fix(rla): lots of additional validity checking and safety (AcademySoftwareFoundation#5094)

* Validity-check resolution of RLA files with check_open. RLA file
headers contain int16_t values for left & right (and top/bottom) window
coordinate, leading to a maximum resolution of 2^16-1.

* Fix potential bug with sign extension in RLE decoding -- if a signed
char is -128, negating it can't make signed char 128 (no such thing), so
must widen the var to an int.

* Fix potential bug by detecting when the number of matte or auxiliary
bits is 0, but the number of matte or aux channels, respectively, is
not.

* Better bounds checking in decode_channel_group. We did the checks
before, but after some accesses that would have been out of bounds! Move
the checks earlier than all the accesses. It actually looks like was the
result of a cut and paste error long ago.

* More care in read_native_scanline for checking valid scanline numbers,
offset into m_sot, and check whether ioseek succeeded (i.e. whether the
offsets loaded from the file are within the range of the size of the
file).

Code and fixes all are from my own brain, but some of the analysis of
which spots have bounds issues were identified in part by conversation
with Claude Code Opus 4.6.

---------

Signed-off-by: Larry Gritz <lg@larrygritz.com>

Author: lgritz

src/rla.imageio/rlainput.cpp

@@ -78,7 +78,7 @@ class RLAInput final : public ImageInput {
 
     /// Helper: read the RLA header and scanline offset table.
     ///
-    inline bool read_header();
+    bool read_header();
 
     /// Helper: read and decode a single channel group consisting of
     /// channels [first_channel .. first_channel+num_channels-1], which
@@ -175,7 +175,7 @@ RLAInput::open(const std::string& name, ImageSpec& newspec)
 
 
 
-inline bool
+bool
 RLAInput::read_header()
 {
     // Read the image header, which should have the same exact layout as
@@ -201,6 +201,17 @@ RLAInput::read_header()
     if (m_rla.NumOfChannelBits == 0)
         m_rla.NumOfChannelBits = 8;  // apparently, this can happen
 
+    if (m_rla.NumOfMatteBits == 0 && m_rla.NumOfMatteChannels > 0) {
+        errorfmt("{} matte channels but 0 matte bits. Possible corrupt input?",
+                 m_rla.NumOfMatteChannels);
+        return false;
+    }
+    if (m_rla.NumOfAuxBits == 0 && m_rla.NumOfAuxChannels > 0) {
+        errorfmt("{} aux channels but 0 aux bits. Possible corrupt input?",
+                 m_rla.NumOfAuxChannels);
+        return false;
+    }
+
     // Immediately following the header is the scanline offset table --
     // one uint32_t for each scanline, giving absolute offsets (from the
     // beginning of the file) where the RLE records start for each
@@ -309,6 +320,15 @@ RLAInput::seek_subimage(int subimage, int miplevel)
     m_spec.full_x      = m_rla.WindowLeft;
     m_spec.full_y      = m_spec.full_height - 1 - m_rla.WindowTop;
 
+    // Check validity of resolutions. The width and height are the difference
+    // between left and right (and top/down) coordinates, which are both
+    // int16_t, giving a maximum of 2^16-1 resolution in each direction. And
+    // the max number of channels is up to 3 color, up to 3 matte, and up to
+    // 256 auxiliary channels.
+    if (!check_open(m_spec, { 0, (1 << 16) - 1, 0, (1 << 16) - 1, 0, 1, 0,
+                              3 + 3 + 256 }))
+        return false;
+
     // set channel formats and stride
     int z_channel = -1;
     m_stride      = 0;
@@ -480,7 +500,7 @@ RLAInput::decode_rle_span(unsigned char* buf, int n, int stride,
 {
     size_t e = 0;
     while (n > 0 && e < elen) {
-        signed char count = (signed char)encoded[e++];
+        int count = (signed char)encoded[e++];
         if (count >= 0) {
             // run count positive: value repeated count+1 times
             for (int i = 0; i <= count && n && e < elen;
@@ -578,6 +598,14 @@ RLAInput::decode_channel_group(int first_channel, short num_channels,
         }
     }
 
+    int bytes_per_chan = ceil2(std::max(int(num_bits), 8)) / 8;
+    if (size_t(offset + (m_spec.width - 1) * pixelsize
+               + num_channels * bytes_per_chan)
+        > m_buf.size()) {
+        errorfmt("Probably corrupt file (buffer overrun avoided)");
+        return false;  // Probably corrupt? Would have overrun
+    }
+
     // If we're little endian, swap endianness in place for 2- and
     // 4-byte pixel data.
     if (littleendian()) {
@@ -604,15 +632,7 @@ RLAInput::decode_channel_group(int first_channel, short num_channels,
     // OIIO conventions.
     if (num_bits == 8 || num_bits == 16 || num_bits == 32) {
         // ok -- no rescaling needed
-    }
-    int bytes_per_chan = ceil2(std::max(int(num_bits), 8)) / 8;
-    if (size_t(offset + (m_spec.width - 1) * pixelsize
-               + num_channels * bytes_per_chan)
-        > m_buf.size()) {
-        errorfmt("Probably corrupt file (buffer overrun avoided)");
-        return false;  // Probably corrupt? Would have overrun
-    }
-    if (num_bits == 10) {
+    } else if (num_bits == 10) {
         // fast, common case -- use templated hard-code
         for (int x = 0; x < m_spec.width; ++x) {
             uint16_t* b = (uint16_t*)(&m_buf[offset + x * pixelsize]);
@@ -658,7 +678,13 @@ RLAInput::read_native_scanline(int subimage, int miplevel, int y, int /*z*/,
     y = m_spec.height - (y - m_spec.y) - 1;
 
     // Seek to scanline start, based on the scanline offset table
-    ioseek(m_sot[y]);
+    if (y < 0 || y >= m_spec.height) {
+        // Invalid scanline
+        return false;
+    }
+    OIIO_DASSERT(m_sot.size() == size_t(m_spec.height));
+    if (!ioseek(m_sot[y]))
+        return false;
 
     // Now decode and interleave the channels.
     // The channels are non-interleaved (i.e. rrrrrgggggbbbbb...).

testsuite/rla/ref/out.txt

@@ -307,19 +307,19 @@ Reading ../oiio-images/rla/imgmake_rgba_nc8.rla
     rla:WhitePoint: 0.31, 0.31, 0.316
 Comparing "../oiio-images/rla/imgmake_rgba_nc8.rla" and "imgmake_rgba_nc8.rla"
 PASS
-oiiotool ERROR: read : "../oiio-images/rla/crash1.rla": Read error: malformed RLE record
+oiiotool ERROR: read : "../oiio-images/rla/crash1.rla": 3 aux channels but 0 aux bits. Possible corrupt input?
 Full command line was:
 > oiiotool ../oiio-images/rla/crash1.rla -o crash1.exr
-oiiotool ERROR: read : "../oiio-images/rla/crash2.rla": Read error: not enough data in scanline 73, channel 0
+oiiotool ERROR: read : "../oiio-images/rla/crash2.rla": 1 aux channels but 0 aux bits. Possible corrupt input?
 Full command line was:
 > oiiotool ../oiio-images/rla/crash2.rla -o crash2.exr
-oiiotool ERROR: read : "src/crash-1629.rla": Read error: malformed RLE record
+oiiotool ERROR: read : "src/crash-1629.rla": rla image full/display resolution must be at least 1x1, but the file specified -16639x256x1. Possible corrupt input?
 Full command line was:
 > oiiotool src/crash-1629.rla -o crash3.exr
-oiiotool ERROR: read : "src/crash-3951.rla": Read error: couldn't read RLE data span
+oiiotool ERROR: read : "src/crash-3951.rla": rla image full/display resolution must be at least 1x1, but the file specified 3834x-31743x1. Possible corrupt input?
 Full command line was:
 > oiiotool src/crash-3951.rla -o crash4.exr
-oiiotool ERROR: read : "src/crash-1.rla": Probably corrupt file (buffer overrun avoided)
+oiiotool ERROR: read : "src/crash-1.rla": rla image full/display resolution must be at least 1x1, but the file specified -281x18999x1. Possible corrupt input?
 Full command line was:
 > oiiotool src/crash-1.rla -o crash5.exr
 Comparing "rlacrop.rla" and "ref/rlacrop.rla"