Add a per-section ENTROPY column under -E

CMakeLists.txt

@@ -223,6 +223,7 @@ if(Python3_Interpreter_FOUND)
     test_partition
     test_littlefs
     test_partition_overlap
+    test_entropy
     test_esp32_part
     test_esp32_nvs
     test_legacy_fs

src/assess.cpp

@@ -9,6 +9,7 @@ namespace ft {
 
 namespace {
 constexpr double HIGH_ENTROPY = 7.2;  // >= this over a sizable region ⇒ likely encrypted/compressed
+}  // namespace
 
 // Entropy over [off,off+len), sampling if the region is large.
 double region_entropy(const Reader& r, size_t off, size_t len) {
@@ -29,8 +30,6 @@ double region_entropy(const Reader& r, size_t off, size_t len) {
     return shannon_entropy(buf);
 }
 
-}  // namespace
-
 double whole_file_entropy(const Reader& r) { return region_entropy(r, 0, r.size()); }
 
 std::vector<Region> unidentified_regions(const Reader& r, const std::vector<Finding>& findings,

src/assess.hpp

@@ -23,6 +23,10 @@ struct Region {
 // Whole-file entropy (sampled for large files to stay cheap).
 double whole_file_entropy(const Reader& r);
 
+// Shannon entropy (bits/byte, 0-8) over [off, off+len), sampling large ranges so
+// it stays cheap on a multi-GB finding. Used by the -E per-finding entropy column.
+double region_entropy(const Reader& r, size_t off, size_t len);
+
 // Byte ranges not covered by any structural finding, each with its
 // entropy. Only regions >= min_size are returned. Useful to flag encrypted or
 // compressed blobs the identifier didn't recognize.

src/finding.hpp

@@ -62,6 +62,9 @@ struct Member {
     // GPT/MBR partition). SIZE_MAX = no offset (an archive member / UBI volume,
     // which has no single image offset); such members render name-only.
     size_t offset = SIZE_MAX;
+    // Shannon entropy (bits/byte, 0-8) over this member's byte range; computed
+    // only under -E for located members. <0 = not computed.
+    double entropy = -1.0;
 };
 
 struct Finding {
@@ -82,6 +85,10 @@ struct Finding {
     std::string compression;
     std::string arch;
 
+    // Shannon entropy (bits/byte, 0-8) over this finding's byte range; computed
+    // only under -E. <0 = not computed (the default, so it is never emitted).
+    double entropy = -1.0;
+
     // Doc metadata (from the signature definition).
     std::string description;
     std::string vendor;

src/human.cpp

@@ -41,8 +41,24 @@ struct Palette {
         if (s == "warning") return "\033[33m";   // yellow
         return "\033[2m";                        // info: faint
     }
+    // Entropy (bits/byte): high = likely encrypted/packed (red), mid = compressed
+    // (yellow), low = plain. Threshold 7.2 matches the -E "likely encrypted" hint.
+    const char* ent(double e) const {
+        if (!on) return "";
+        if (e >= 7.2) return "\033[31m";   // red
+        if (e >= 6.0) return "\033[33m";   // yellow
+        return "";
+    }
 };
 
+// "7.98", or empty when not computed (entropy < 0). Two decimals, fixed width.
+std::string ent_str(double e) {
+    if (e < 0) return {};
+    char b[16];
+    std::snprintf(b, sizeof(b), "%.2f", e);
+    return b;
+}
+
 std::string human_size(size_t n) {
     static const std::array<const char*, 5> unit{"B", "KB", "MB", "GB", "TB"};
     double v = static_cast<double>(n);
@@ -295,7 +311,15 @@ void emit_findings_tree(std::string& o, const Palette& p, const std::vector<Find
         }
         return w;
     };
+    // Under -E every sized finding is annotated with entropy, so show an ENTROPY
+    // column (between TIER and NOTES) whenever any row carries a computed value.
+    bool show_entropy = false;
+    for (const auto& r : rows) {
+        if (r.f && r.f->entropy >= 0) { show_entropy = true; break; }
+        if (r.mem && r.mem->entropy >= 0) { show_entropy = true; break; }
+    }
     size_t w_off = 6, w_size = 4, w_type = 4, w_tier = 4;
+    const size_t w_ent = 7;  // "ENTROPY"; values are "7.98"
     for (const auto& r : rows) {
         if (r.f) {
             w_off = std::max(w_off, disp_w(r.first));
@@ -317,6 +341,10 @@ void emit_findings_tree(std::string& o, const Palette& p, const std::vector<Find
     col(h, "TYPE", w_type, "", "");
     h += "  ";
     col(h, "TIER", w_tier, "", "");
+    if (show_entropy) {
+        h += "  ";
+        col(h, "ENTROPY", w_ent, "", "");
+    }
     h += "  NOTES";
     o += h + p.reset() + "\n";
 
@@ -342,7 +370,15 @@ void emit_findings_tree(std::string& o, const Palette& p, const std::vector<Find
             line += "  ";
             col(line, human_size(r.mem->size), w_size, "", "");
             line += "  ";
+            // A member has no TIER; under -E fill it blank so the ENTROPY column
+            // stays aligned with the finding rows.
             col(line, r.mem->note, w_type, p.dim(), p.reset());
+            if (show_entropy) {
+                line += "  ";
+                col(line, "", w_tier, "", "");  // blank TIER
+                line += "  ";
+                col(line, ent_str(r.mem->entropy), w_ent, p.ent(r.mem->entropy), p.reset());
+            }
             while (!line.empty() && line.back() == ' ') line.pop_back();
             o += line + "\n";
             continue;
@@ -358,6 +394,10 @@ void emit_findings_tree(std::string& o, const Palette& p, const std::vector<Find
         col(line, r.f->type, w_type, p.sec(*r.f), p.reset());
         line += "  ";
         col(line, r.f->confidence_tier, w_tier, p.tier(r.f->confidence_tier), p.reset());
+        if (show_entropy) {
+            line += "  ";
+            col(line, ent_str(r.f->entropy), w_ent, p.ent(r.f->entropy), p.reset());
+        }
         line += "  ";
         line += p.dim() + notes_for(*r.f, p, verbose) + p.reset();
         while (!line.empty() && line.back() == ' ') line.pop_back();

src/json.cpp

@@ -56,6 +56,15 @@ void kv_num(std::string& o, const char* key, unsigned long long val, bool& first
     o += std::to_string(val);
 }
 
+// A double with 2 decimals (for the -E entropy field).
+void kv_double(std::string& o, const char* key, double val, bool& first) {
+    if (!first) o += ",";
+    first = false;
+    char buf[32];
+    std::snprintf(buf, sizeof(buf), "\"%s\":%.2f", key, val);
+    o += buf;
+}
+
 void emit_finding(std::string& o, const Finding& f, bool with_also_matched);
 
 void emit_finding(std::string& o, const Finding& f, bool with_also_matched) {
@@ -74,6 +83,7 @@ void emit_finding(std::string& o, const Finding& f, bool with_also_matched) {
     if (!f.label.empty()) kv_str(o, "label", f.label, first);
     if (!f.compression.empty()) kv_str(o, "compression", f.compression, first);
     if (!f.arch.empty()) kv_str(o, "arch", f.arch, first);
+    if (f.entropy >= 0) kv_double(o, "entropy", f.entropy, first);
 
     // Archive members (from --list) — emitted even in compact mode (the point of --list).
     if (!f.members.empty()) {
@@ -85,6 +95,11 @@ void emit_finding(std::string& o, const Finding& f, bool with_also_matched) {
             o += "\",\"size\":" + std::to_string(f.members[i].size);
             if (f.members[i].offset != SIZE_MAX)
                 o += ",\"offset\":" + std::to_string(f.members[i].offset);
+            if (f.members[i].entropy >= 0) {
+                char eb[32];
+                std::snprintf(eb, sizeof(eb), ",\"entropy\":%.2f", f.members[i].entropy);
+                o += eb;
+            }
             if (!f.members[i].note.empty()) {
                 o += ",\"note\":\"";
                 escape_to(o, f.members[i].note);

src/main.cpp

@@ -14,6 +14,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <filesystem>
+#include <functional>
 #include <string>
 #include <thread>
 #include <vector>
@@ -582,6 +583,18 @@ int main(int argc, char** argv) {
     if (entropy) {
         regions = ft::unidentified_regions(reader, findings, 4096);
         ent = ft::whole_file_entropy(reader);
+        // Comprehensive per-section entropy: annotate every finding (and its
+        // located members / nested children) so the -E view shows an ENTROPY
+        // column across the whole OFFSET table, not just the unidentified gaps.
+        std::function<void(ft::Finding&)> annotate = [&](ft::Finding& f) {
+            if (f.size > 0) f.entropy = ft::region_entropy(reader, f.offset, f.size);
+            for (auto& m : f.members) {
+                if (m.offset != SIZE_MAX && m.size > 0)
+                    m.entropy = ft::region_entropy(reader, m.offset, m.size);
+                for (auto& c : m.children) annotate(c);
+            }
+        };
+        for (auto& f : findings) annotate(f);
     }
     std::string assessment = ft::assess_file(findings, fm.size(), regions, ent, entropy);
     if (hidden_interior > 0)

tests/run.sh

@@ -22,6 +22,8 @@ python3 tests/test_partition.py
 python3 tests/test_littlefs.py
 # Partition-boundary overrun: a stale finding must not hide a real partition (PR #33).
 python3 tests/test_partition_overlap.py
+# -E per-section entropy column on findings + members (human + JSON).
+python3 tests/test_entropy.py
 # ESP32 partition-table region map + NVS identify/extract (synthetic, self-contained).
 python3 tests/test_esp32_part.py
 python3 tests/test_esp32_nvs.py

tests/test_entropy.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""Regression for the -E per-section entropy column (findings + members).
+
+-E adds a Shannon-entropy value to every finding and every located member, shown
+as an ENTROPY column in the human OFFSET table and an `entropy` field in JSON.
+Without -E there is neither. Fully self-contained (synthetic ext superblocks +
+the gpt fixture). Run: python3 tests/test_entropy.py
+"""
+import json
+import os
+import random
+import struct
+import subprocess
+import sys
+import tempfile
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+MORIA = os.path.join(HERE, "..", "build", "moria")
+sys.path.insert(0, HERE)
+import gen_samples  # noqa: E402
+
+
+def ext_over(buf):
+    """Plant a valid 64 KiB ext superblock at offset 0 of `buf` (finding size =
+    64 KiB, so entropy is dominated by the buffer's contents)."""
+    b = bytearray(buf)
+    sb = 1024
+    struct.pack_into("<I", b, sb + 0, 1000)   # s_inodes_count
+    struct.pack_into("<I", b, sb + 4, 64)     # s_blocks_count_lo -> 64 * 1024 = 64 KiB
+    struct.pack_into("<I", b, sb + 20, 0)     # s_first_data_block
+    struct.pack_into("<I", b, sb + 24, 0)     # s_log_block_size -> 1024
+    struct.pack_into("<H", b, 0x438, 0xEF53)  # magic
+    return bytes(b)
+
+
+def high_entropy_ext():
+    rnd = random.Random(0xE47)
+    return ext_over(bytes(rnd.getrandbits(8) for _ in range(64 * 1024)))
+
+
+def low_entropy_ext():
+    return ext_over(bytes(64 * 1024))
+
+
+def run(data, args):
+    with tempfile.NamedTemporaryFile(suffix=".img") as f:
+        f.write(data)
+        f.flush()
+        r = subprocess.run([MORIA] + args + [f.name], capture_output=True, text=True, timeout=60,
+                           env={**os.environ, "NO_COLOR": "1"})
+    return r.stdout
+
+
+def findings(data, entropy):
+    doc = json.loads(run(data, ["-j"] + (["-E"] if entropy else [])))
+    return doc["findings"]
+
+
+def main():
+    if not os.path.exists(MORIA):
+        print("moria not built", file=sys.stderr)
+        return 1
+    fails = []
+
+    def check(cond, msg):
+        if not cond:
+            fails.append(msg)
+
+    # --- JSON: entropy field is present only under -E, and reflects the data ----
+    hi = findings(high_entropy_ext(), entropy=True)
+    lo = findings(low_entropy_ext(), entropy=True)
+    hie = [x for x in hi if x["type"] == "ext"]
+    loe = [x for x in lo if x["type"] == "ext"]
+    check(hie and hie[0].get("entropy", 0) > 7.5, "high-entropy ext: entropy > 7.5 under -E")
+    check(loe and loe[0].get("entropy", 9) < 1.0, "low-entropy ext: entropy < 1.0 under -E")
+
+    no_e = findings(high_entropy_ext(), entropy=False)
+    check(all("entropy" not in x for x in no_e), "no -E: findings carry no entropy field")
+
+    # --- members get entropy too (the gpt fixture has partition members) --------
+    g = [x for x in findings(gen_samples.gpt_disk(), entropy=True) if x["type"] == "gpt"]
+    check(g and all("entropy" in m for m in g[0].get("members", [])),
+          "gpt members carry entropy under -E")
+    g0 = [x for x in findings(gen_samples.gpt_disk(), entropy=False) if x["type"] == "gpt"]
+    check(g0 and all("entropy" not in m for m in g0[0].get("members", [])),
+          "no -E: gpt members carry no entropy")
+
+    # --- human: ENTROPY column present only under -E ----------------------------
+    human_e = run(high_entropy_ext(), ["-E"])
+    human_0 = run(high_entropy_ext(), [])
+    hdr_e = next((ln for ln in human_e.splitlines() if ln.startswith("OFFSET")), "")
+    hdr_0 = next((ln for ln in human_0.splitlines() if ln.startswith("OFFSET")), "")
+    check("ENTROPY" in hdr_e, "human -E: ENTROPY column in the header")
+    check("ENTROPY" not in hdr_0, "human without -E: no ENTROPY column")
+    # the high-entropy value shows in the table
+    check(any("8.00" in ln for ln in human_e.splitlines()),
+          "human -E: the 8.00 entropy value is rendered")
+
+    print("-" * 60)
+    if fails:
+        for m in fails:
+            print("FAIL:", m)
+        return 1
+    print("PASS: -E entropy column on findings + members (human + JSON), off by default")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())