fix(softimage): Prevent buffer overruns from corrupt RLE (AcademySoftwareFoundation#5142)

* Check for buffer overruns.
* Use check_open.
* Check for invalid bit depths.

Signed-off-by: Larry Gritz <lg@larrygritz.com>

Author: lgritz

src/softimage.imageio/softimageinput.cpp

@@ -120,6 +120,12 @@ SoftimageInput::open(const std::string& name, ImageSpec& spec)
             close();
             return false;
         }
+        // Some validity checking
+        if (curPacket.size != 8 && curPacket.size != 16) {
+            errorfmt("Unsupported bits per channel {}", curPacket.size);
+            close();
+            return false;
+        }
         m_channel_packets.push_back(curPacket);
 
         // Add the number of channels in this packet to nchannels
@@ -135,6 +141,12 @@ SoftimageInput::open(const std::string& name, ImageSpec& spec)
     // Set the details in the ImageSpec
     m_spec = ImageSpec(m_pic_header.width, m_pic_header.height, nchannels,
                        chanType);
+
+    if (!check_open(m_spec, { 0, 65535, 0, 65535, 0, 1, 0, 4 })) {
+        close();
+        return false;
+    }
+
     m_spec.attribute("BitsPerSample", (int)curPacket.size);
 
     m_spec.attribute("softimage:compression", Strutil::join(encodings, ","));
@@ -333,6 +345,10 @@ SoftimageInput::read_pixels_pure_run_length(
         if (fread(&curCount, 1, 1, m_fd) != 1)
             return false;
 
+        // Clamp to avoid writing past the end of the scanline buffer
+        if (linePixelCount + curCount > m_pic_header.width)
+            curCount = m_pic_header.width - linePixelCount;
+
         if (data) {
             // data pointer is set so we're supposed to write data there
             size_t pixelSize   = pixelChannelSize * channels.size();
@@ -456,6 +472,10 @@ SoftimageInput::read_pixels_mixed_run_length(
                 longCount = curCount - 127;
             }
 
+            // Clamp to avoid writing past the end of the scanline buffer
+            if (linePixelCount + longCount > m_pic_header.width)
+                longCount = m_pic_header.width - linePixelCount;
+
             if (data) {
                 // data pointer is set so we're supposed to write data there
                 size_t pixelSize   = pixelChannelSize * channels.size();

testsuite/softimage/ref/out.txt

@@ -30,3 +30,17 @@ Reading ../oiio-images/softimage/astone64.pic
     Stats FiniteCount: 4096 4096 4096 
     Constant: No
     Monochrome: No
+src/broken01.pic :   16 x    4, 1 channel, uint8 softimage
+    SHA-1: 0526ABB83138E5FD9BC07BB83035C89028527301
+    channel list: Y
+    BitsPerSample: 8
+    softimage:compression: "mixed-rle"
+    Stats Min: 0 (of 255)
+    Stats Max: 65 (of 255)
+    Stats Avg: 16.25 (of 255)
+    Stats StdDev: 28.15 (of 255)
+    Stats NanCount: 0 
+    Stats InfCount: 0 
+    Stats FiniteCount: 64 
+    Constant: No
+    Monochrome: Yes

testsuite/softimage/run.py

@@ -4,7 +4,12 @@
 # SPDX-License-Identifier: Apache-2.0
 # https://github.com/AcademySoftwareFoundation/OpenImageIO
 
+redirect = ' >> out.txt 2>&1 '
+ 
 files = [ "A4.pic", "astone64.pic" ]
 for f in files:
     command += info_command (OIIO_TESTSUITE_IMAGEDIR + "/" + f, extraargs="--stats")
 
+# Regression testing of error handling and corrupt files
+command += info_command ("--stats src/broken01.pic",
+                         info_program="iinfo", failureok=True)