fix(bmp): Correctly handle the combination of greyscale + RLE compression (#5163)

Fixes #5158

Signed-off-by: Larry Gritz <lg@larrygritz.com>

Author: lgritz

src/bmp.imageio/bmpinput.cpp

@@ -395,13 +395,28 @@ BmpInput::read_native_scanline(int subimage, int miplevel, int y, int /*z*/,
     if ((m_dib_header.compression == RLE4_COMPRESSION
          || m_dib_header.compression == RLE8_COMPRESSION)
         && m_colortable.size()) {
-        for (int x = 0; x < m_spec.width; ++x) {
-            int p = m_uncompressed[int64_t(m_spec.height - 1 - y) * m_spec.width
-                                   + x];
-            auto& c              = colortable(p);
-            mscanline[3 * x]     = c.r;
-            mscanline[3 * x + 1] = c.g;
-            mscanline[3 * x + 2] = c.b;
+        if (m_spec.nchannels == 1) {
+            OIIO_CONTRACT_ASSERT(m_allgray);
+            for (int x = 0; x < m_spec.width; ++x) {
+                int p
+                    = m_uncompressed[int64_t(m_spec.height - 1 - y) * m_spec.width
+                                     + x];
+                auto& c      = colortable(p);
+                mscanline[x] = c.r;
+                // For the "all gray" case, rgb are equal and we're filling in
+                // a 1-channel data buffer.
+            }
+        } else {
+            OIIO_CONTRACT_ASSERT(m_spec.nchannels == 3);
+            for (int x = 0; x < m_spec.width; ++x) {
+                int p
+                    = m_uncompressed[int64_t(m_spec.height - 1 - y) * m_spec.width
+                                     + x];
+                auto& c              = colortable(p);
+                mscanline[3 * x]     = c.r;
+                mscanline[3 * x + 1] = c.g;
+                mscanline[3 * x + 2] = c.b;
+            }
         }
         return true;
     }

testsuite/bmp/ref/out.txt

@@ -291,6 +291,17 @@ Reading ../oiio-images/bmp/gracehopper.bmp
     oiio:ColorSpace: "srgb_rec709_scene"
 Comparing "../oiio-images/bmp/gracehopper.bmp" and "gracehopper.bmp"
 PASS
+Reading src/crash-5158.bmp
+src/crash-5158.bmp   :   13 x   32, 1 channel, uint8 bmp
+    SHA-1: 5BAB75849B9D716B8FEC896E7B0F2D37659B3BAD
+    channel list: Y
+    compression: "rle8"
+    ResolutionUnit: "m"
+    XResolution: 1179648
+    YResolution: 32
+    bmp:bitsperpixel: 8
+    bmp:version: 3
+    oiio:ColorSpace: "srgb_rec709_scene"
 oiiotool ERROR: read : "src/decodecolormap-corrupt.bmp": Possible corrupted header, invalid palette size
 Full command line was:
 > oiiotool --info -v -a --hash src/decodecolormap-corrupt.bmp

testsuite/bmp/run.py

@@ -30,6 +30,11 @@
 # Test BMP of the 56-byte DIB header variety
 command += rw_command ("../oiio-images/bmp", "gracehopper.bmp")
 
+# Regression test for RLE8 heap buffer overflow when palette is all-gray
+# (detected as 1-channel, but RLE path was writing 3 bytes per pixel).
+# See https://github.com/AcademySoftwareFoundation/OpenImageIO/issues/5158
+command += info_command ("src/crash-5158.bmp", hash=1)
+
 # See if we handle these corrupt files with useful error messages
 command += info_command ("src/decodecolormap-corrupt.bmp")
 command += info_command ("src/bad-y.bmp")