From 63647cb5a7ac820ac728611c5668095218720198 Mon Sep 17 00:00:00 2001 From: Matt Brown Date: Thu, 17 Sep 2026 17:37:29 -0400 Subject: [PATCH] feat: add a per-section ENTROPY column under -E -E previously reported entropy only for the unidentified gaps between findings. Since -E is an explicit request for entropy analysis, make it comprehensive: it now annotates every finding and every located member (partition entries, UBI volumes) with the Shannon entropy over its byte range and shows it as an ENTROPY column in the human OFFSET table (between TIER and NOTES), plus an `entropy` field in JSON. High values are colored to draw the eye: red at >= 7.2 (likely encrypted/packed), yellow at >= 6.0 (compressed), plain below. This surfaces at a glance a section whose type and randomness disagree, e.g. an "ext" partition reading 8.0 is encrypted, not a real filesystem. - region_entropy() is promoted to the public assess API (it already stride-samples large ranges, so it stays cheap on a multi-GB finding). - Finding and Member gain an `entropy` field (-1 = not computed, the default, so nothing changes without -E). main.cpp annotates findings/members/nested children when -E is set. - The column appears only when a row carries a computed value, so default output is byte-identical. Test: tests/test_entropy.py (self-contained) asserts a random-filled ext reads > 7.5 and a zero-filled ext reads < 1.0 under -E, that the entropy field and the ENTROPY column are absent without -E, and that gpt members carry entropy. Wired into run.sh and CTest. --- CMakeLists.txt | 1 + src/assess.cpp | 3 +- src/assess.hpp | 4 ++ src/finding.hpp | 7 +++ src/human.cpp | 40 ++++++++++++++++ src/json.cpp | 15 ++++++ src/main.cpp | 13 +++++ tests/run.sh | 2 + tests/test_entropy.py | 109 ++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 tests/test_entropy.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 1633abe..b675f65 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -223,6 +223,7 @@ if(Python3_Interpreter_FOUND) test_partition test_littlefs test_partition_overlap + test_entropy test_esp32_part test_esp32_nvs test_legacy_fs diff --git a/src/assess.cpp b/src/assess.cpp index 2e0a576..024d53a 100644 --- a/src/assess.cpp +++ b/src/assess.cpp @@ -9,6 +9,7 @@ namespace ft { namespace { constexpr double HIGH_ENTROPY = 7.2; // >= this over a sizable region ⇒ likely encrypted/compressed +} // namespace // Entropy over [off,off+len), sampling if the region is large. double region_entropy(const Reader& r, size_t off, size_t len) { @@ -29,8 +30,6 @@ double region_entropy(const Reader& r, size_t off, size_t len) { return shannon_entropy(buf); } -} // namespace - double whole_file_entropy(const Reader& r) { return region_entropy(r, 0, r.size()); } std::vector unidentified_regions(const Reader& r, const std::vector& findings, diff --git a/src/assess.hpp b/src/assess.hpp index 599baad..263d32d 100644 --- a/src/assess.hpp +++ b/src/assess.hpp @@ -23,6 +23,10 @@ struct Region { // Whole-file entropy (sampled for large files to stay cheap). double whole_file_entropy(const Reader& r); +// Shannon entropy (bits/byte, 0-8) over [off, off+len), sampling large ranges so +// it stays cheap on a multi-GB finding. Used by the -E per-finding entropy column. +double region_entropy(const Reader& r, size_t off, size_t len); + // Byte ranges not covered by any structural finding, each with its // entropy. Only regions >= min_size are returned. Useful to flag encrypted or // compressed blobs the identifier didn't recognize. diff --git a/src/finding.hpp b/src/finding.hpp index e33a83e..d73529b 100644 --- a/src/finding.hpp +++ b/src/finding.hpp @@ -62,6 +62,9 @@ struct Member { // GPT/MBR partition). SIZE_MAX = no offset (an archive member / UBI volume, // which has no single image offset); such members render name-only. size_t offset = SIZE_MAX; + // Shannon entropy (bits/byte, 0-8) over this member's byte range; computed + // only under -E for located members. <0 = not computed. + double entropy = -1.0; }; struct Finding { @@ -82,6 +85,10 @@ struct Finding { std::string compression; std::string arch; + // Shannon entropy (bits/byte, 0-8) over this finding's byte range; computed + // only under -E. <0 = not computed (the default, so it is never emitted). + double entropy = -1.0; + // Doc metadata (from the signature definition). std::string description; std::string vendor; diff --git a/src/human.cpp b/src/human.cpp index 7a819a9..c54ccad 100644 --- a/src/human.cpp +++ b/src/human.cpp @@ -41,8 +41,24 @@ struct Palette { if (s == "warning") return "\033[33m"; // yellow return "\033[2m"; // info: faint } + // Entropy (bits/byte): high = likely encrypted/packed (red), mid = compressed + // (yellow), low = plain. Threshold 7.2 matches the -E "likely encrypted" hint. + const char* ent(double e) const { + if (!on) return ""; + if (e >= 7.2) return "\033[31m"; // red + if (e >= 6.0) return "\033[33m"; // yellow + return ""; + } }; +// "7.98", or empty when not computed (entropy < 0). Two decimals, fixed width. +std::string ent_str(double e) { + if (e < 0) return {}; + char b[16]; + std::snprintf(b, sizeof(b), "%.2f", e); + return b; +} + std::string human_size(size_t n) { static const std::array unit{"B", "KB", "MB", "GB", "TB"}; double v = static_cast(n); @@ -295,7 +311,15 @@ void emit_findings_tree(std::string& o, const Palette& p, const std::vectorentropy >= 0) { show_entropy = true; break; } + if (r.mem && r.mem->entropy >= 0) { show_entropy = true; break; } + } size_t w_off = 6, w_size = 4, w_type = 4, w_tier = 4; + const size_t w_ent = 7; // "ENTROPY"; values are "7.98" for (const auto& r : rows) { if (r.f) { w_off = std::max(w_off, disp_w(r.first)); @@ -317,6 +341,10 @@ void emit_findings_tree(std::string& o, const Palette& p, const std::vectorsize), w_size, "", ""); line += " "; + // A member has no TIER; under -E fill it blank so the ENTROPY column + // stays aligned with the finding rows. col(line, r.mem->note, w_type, p.dim(), p.reset()); + if (show_entropy) { + line += " "; + col(line, "", w_tier, "", ""); // blank TIER + line += " "; + col(line, ent_str(r.mem->entropy), w_ent, p.ent(r.mem->entropy), p.reset()); + } while (!line.empty() && line.back() == ' ') line.pop_back(); o += line + "\n"; continue; @@ -358,6 +394,10 @@ void emit_findings_tree(std::string& o, const Palette& p, const std::vectortype, w_type, p.sec(*r.f), p.reset()); line += " "; col(line, r.f->confidence_tier, w_tier, p.tier(r.f->confidence_tier), p.reset()); + if (show_entropy) { + line += " "; + col(line, ent_str(r.f->entropy), w_ent, p.ent(r.f->entropy), p.reset()); + } line += " "; line += p.dim() + notes_for(*r.f, p, verbose) + p.reset(); while (!line.empty() && line.back() == ' ') line.pop_back(); diff --git a/src/json.cpp b/src/json.cpp index 28f1f50..949a4ff 100644 --- a/src/json.cpp +++ b/src/json.cpp @@ -56,6 +56,15 @@ void kv_num(std::string& o, const char* key, unsigned long long val, bool& first o += std::to_string(val); } +// A double with 2 decimals (for the -E entropy field). +void kv_double(std::string& o, const char* key, double val, bool& first) { + if (!first) o += ","; + first = false; + char buf[32]; + std::snprintf(buf, sizeof(buf), "\"%s\":%.2f", key, val); + o += buf; +} + void emit_finding(std::string& o, const Finding& f, bool with_also_matched); void emit_finding(std::string& o, const Finding& f, bool with_also_matched) { @@ -74,6 +83,7 @@ void emit_finding(std::string& o, const Finding& f, bool with_also_matched) { if (!f.label.empty()) kv_str(o, "label", f.label, first); if (!f.compression.empty()) kv_str(o, "compression", f.compression, first); if (!f.arch.empty()) kv_str(o, "arch", f.arch, first); + if (f.entropy >= 0) kv_double(o, "entropy", f.entropy, first); // Archive members (from --list) — emitted even in compact mode (the point of --list). if (!f.members.empty()) { @@ -85,6 +95,11 @@ void emit_finding(std::string& o, const Finding& f, bool with_also_matched) { o += "\",\"size\":" + std::to_string(f.members[i].size); if (f.members[i].offset != SIZE_MAX) o += ",\"offset\":" + std::to_string(f.members[i].offset); + if (f.members[i].entropy >= 0) { + char eb[32]; + std::snprintf(eb, sizeof(eb), ",\"entropy\":%.2f", f.members[i].entropy); + o += eb; + } if (!f.members[i].note.empty()) { o += ",\"note\":\""; escape_to(o, f.members[i].note); diff --git a/src/main.cpp b/src/main.cpp index 00c6da3..d363bdb 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -582,6 +583,18 @@ int main(int argc, char** argv) { if (entropy) { regions = ft::unidentified_regions(reader, findings, 4096); ent = ft::whole_file_entropy(reader); + // Comprehensive per-section entropy: annotate every finding (and its + // located members / nested children) so the -E view shows an ENTROPY + // column across the whole OFFSET table, not just the unidentified gaps. + std::function annotate = [&](ft::Finding& f) { + if (f.size > 0) f.entropy = ft::region_entropy(reader, f.offset, f.size); + for (auto& m : f.members) { + if (m.offset != SIZE_MAX && m.size > 0) + m.entropy = ft::region_entropy(reader, m.offset, m.size); + for (auto& c : m.children) annotate(c); + } + }; + for (auto& f : findings) annotate(f); } std::string assessment = ft::assess_file(findings, fm.size(), regions, ent, entropy); if (hidden_interior > 0) diff --git a/tests/run.sh b/tests/run.sh index 795a09c..6f7b960 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -22,6 +22,8 @@ python3 tests/test_partition.py python3 tests/test_littlefs.py # Partition-boundary overrun: a stale finding must not hide a real partition (PR #33). python3 tests/test_partition_overlap.py +# -E per-section entropy column on findings + members (human + JSON). +python3 tests/test_entropy.py # ESP32 partition-table region map + NVS identify/extract (synthetic, self-contained). python3 tests/test_esp32_part.py python3 tests/test_esp32_nvs.py diff --git a/tests/test_entropy.py b/tests/test_entropy.py new file mode 100644 index 0000000..ed01d8c --- /dev/null +++ b/tests/test_entropy.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Regression for the -E per-section entropy column (findings + members). + +-E adds a Shannon-entropy value to every finding and every located member, shown +as an ENTROPY column in the human OFFSET table and an `entropy` field in JSON. +Without -E there is neither. Fully self-contained (synthetic ext superblocks + +the gpt fixture). Run: python3 tests/test_entropy.py +""" +import json +import os +import random +import struct +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +MORIA = os.path.join(HERE, "..", "build", "moria") +sys.path.insert(0, HERE) +import gen_samples # noqa: E402 + + +def ext_over(buf): + """Plant a valid 64 KiB ext superblock at offset 0 of `buf` (finding size = + 64 KiB, so entropy is dominated by the buffer's contents).""" + b = bytearray(buf) + sb = 1024 + struct.pack_into(" 64 * 1024 = 64 KiB + struct.pack_into(" 1024 + struct.pack_into(" 7.5, "high-entropy ext: entropy > 7.5 under -E") + check(loe and loe[0].get("entropy", 9) < 1.0, "low-entropy ext: entropy < 1.0 under -E") + + no_e = findings(high_entropy_ext(), entropy=False) + check(all("entropy" not in x for x in no_e), "no -E: findings carry no entropy field") + + # --- members get entropy too (the gpt fixture has partition members) -------- + g = [x for x in findings(gen_samples.gpt_disk(), entropy=True) if x["type"] == "gpt"] + check(g and all("entropy" in m for m in g[0].get("members", [])), + "gpt members carry entropy under -E") + g0 = [x for x in findings(gen_samples.gpt_disk(), entropy=False) if x["type"] == "gpt"] + check(g0 and all("entropy" not in m for m in g0[0].get("members", [])), + "no -E: gpt members carry no entropy") + + # --- human: ENTROPY column present only under -E ---------------------------- + human_e = run(high_entropy_ext(), ["-E"]) + human_0 = run(high_entropy_ext(), []) + hdr_e = next((ln for ln in human_e.splitlines() if ln.startswith("OFFSET")), "") + hdr_0 = next((ln for ln in human_0.splitlines() if ln.startswith("OFFSET")), "") + check("ENTROPY" in hdr_e, "human -E: ENTROPY column in the header") + check("ENTROPY" not in hdr_0, "human without -E: no ENTROPY column") + # the high-entropy value shows in the table + check(any("8.00" in ln for ln in human_e.splitlines()), + "human -E: the 8.00 entropy value is rendered") + + print("-" * 60) + if fails: + for m in fails: + print("FAIL:", m) + return 1 + print("PASS: -E entropy column on findings + members (human + JSON), off by default") + return 0 + + +if __name__ == "__main__": + sys.exit(main())