fix(softimage): More hardening against corrupted input files (AcademySoftwareFoundation#5155)

* Fix incorrect return check for fread in read_pixels_pure_run_length.
* Detect and report error for unsupported packet type.
* Fix possible infinite loop from malformed RLE packet with a zero
run-count.
* Check for valid y range in read_native_scanline.

Signed-off-by: Larry Gritz <lg@larrygritz.com>

Author: lgritz

src/softimage.imageio/softimageinput.cpp

@@ -175,6 +175,8 @@ SoftimageInput::read_native_scanline(int subimage, int miplevel, int y,
     lock_guard lock(*this);
     if (!seek_subimage(subimage, miplevel))
         return false;
+    if (y < 0 || y >= m_spec.height)
+        return false;
 
     bool result = false;
     if (y == (int)m_scanline_markers.size() - 1) {
@@ -271,9 +273,14 @@ SoftimageInput::read_next_scanline(void* data)
             ok = read_pixels_pure_run_length(cp, data);
         } else if (type == MIXED_RUN_LENGTH) {
             ok = read_pixels_mixed_run_length(cp, data);
+        } else {
+            errorfmt("Unsupported channel packet encoding {:d} in \"{}\"",
+                     int(cp.type), m_filename);
+            close();
+            return false;
         }
         if (!ok) {
-            errorfmt("Failed to read channel packed type {:d} from \"{}\"",
+            errorfmt("Failed to read channel packet type {:d} from \"{}\"",
                      int(cp.type), m_filename);
             close();
             return false;
@@ -345,6 +352,12 @@ SoftimageInput::read_pixels_pure_run_length(
         if (fread(&curCount, 1, 1, m_fd) != 1)
             return false;
 
+        // Zero-length run is malformed
+        if (curCount == 0) {
+            errorfmt("Invalid RLE data");
+            return false;
+        }
+
         // Clamp to avoid writing past the end of the scanline buffer
         if (linePixelCount + curCount > m_pic_header.width)
             curCount = m_pic_header.width - linePixelCount;
@@ -353,7 +366,7 @@ SoftimageInput::read_pixels_pure_run_length(
             // data pointer is set so we're supposed to write data there
             size_t pixelSize   = pixelChannelSize * channels.size();
             uint8_t* pixelData = new uint8_t[pixelSize];
-            if (fread(pixelData, pixelSize, 1, m_fd) != pixelSize)
+            if (fread(pixelData, 1, pixelSize, m_fd) != pixelSize)
                 return false;
 
             // Now we've got the pixel value we need to push it into the data
@@ -472,6 +485,12 @@ SoftimageInput::read_pixels_mixed_run_length(
                 longCount = curCount - 127;
             }
 
+            // Zero-length run is malformed
+            if (longCount == 0) {
+                errorfmt("Invalid RLE data");
+                return false;
+            }
+
             // Clamp to avoid writing past the end of the scanline buffer
             if (linePixelCount + longCount > m_pic_header.width)
                 longCount = m_pic_header.width - linePixelCount;