diff --git a/CMakeLists.txt b/CMakeLists.txt index 1633abe..b675f65 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -223,6 +223,7 @@ if(Python3_Interpreter_FOUND) test_partition test_littlefs test_partition_overlap + test_entropy test_esp32_part test_esp32_nvs test_legacy_fs diff --git a/src/assess.cpp b/src/assess.cpp index 2e0a576..024d53a 100644 --- a/src/assess.cpp +++ b/src/assess.cpp @@ -9,6 +9,7 @@ namespace ft { namespace { constexpr double HIGH_ENTROPY = 7.2; // >= this over a sizable region ⇒ likely encrypted/compressed +} // namespace // Entropy over [off,off+len), sampling if the region is large. double region_entropy(const Reader& r, size_t off, size_t len) { @@ -29,8 +30,6 @@ double region_entropy(const Reader& r, size_t off, size_t len) { return shannon_entropy(buf); } -} // namespace - double whole_file_entropy(const Reader& r) { return region_entropy(r, 0, r.size()); } std::vector unidentified_regions(const Reader& r, const std::vector& findings, diff --git a/src/assess.hpp b/src/assess.hpp index 599baad..263d32d 100644 --- a/src/assess.hpp +++ b/src/assess.hpp @@ -23,6 +23,10 @@ struct Region { // Whole-file entropy (sampled for large files to stay cheap). double whole_file_entropy(const Reader& r); +// Shannon entropy (bits/byte, 0-8) over [off, off+len), sampling large ranges so +// it stays cheap on a multi-GB finding. Used by the -E per-finding entropy column. +double region_entropy(const Reader& r, size_t off, size_t len); + // Byte ranges not covered by any structural finding, each with its // entropy. Only regions >= min_size are returned. Useful to flag encrypted or // compressed blobs the identifier didn't recognize. diff --git a/src/finding.hpp b/src/finding.hpp index e33a83e..d73529b 100644 --- a/src/finding.hpp +++ b/src/finding.hpp @@ -62,6 +62,9 @@ struct Member { // GPT/MBR partition). SIZE_MAX = no offset (an archive member / UBI volume, // which has no single image offset); such members render name-only. size_t offset = SIZE_MAX; + // Shannon entropy (bits/byte, 0-8) over this member's byte range; computed + // only under -E for located members. <0 = not computed. + double entropy = -1.0; }; struct Finding { @@ -82,6 +85,10 @@ struct Finding { std::string compression; std::string arch; + // Shannon entropy (bits/byte, 0-8) over this finding's byte range; computed + // only under -E. <0 = not computed (the default, so it is never emitted). + double entropy = -1.0; + // Doc metadata (from the signature definition). std::string description; std::string vendor; diff --git a/src/human.cpp b/src/human.cpp index 7a819a9..c54ccad 100644 --- a/src/human.cpp +++ b/src/human.cpp @@ -41,8 +41,24 @@ struct Palette { if (s == "warning") return "\033[33m"; // yellow return "\033[2m"; // info: faint } + // Entropy (bits/byte): high = likely encrypted/packed (red), mid = compressed + // (yellow), low = plain. Threshold 7.2 matches the -E "likely encrypted" hint. + const char* ent(double e) const { + if (!on) return ""; + if (e >= 7.2) return "\033[31m"; // red + if (e >= 6.0) return "\033[33m"; // yellow + return ""; + } }; +// "7.98", or empty when not computed (entropy < 0). Two decimals, fixed width. +std::string ent_str(double e) { + if (e < 0) return {}; + char b[16]; + std::snprintf(b, sizeof(b), "%.2f", e); + return b; +} + std::string human_size(size_t n) { static const std::array unit{"B", "KB", "MB", "GB", "TB"}; double v = static_cast(n); @@ -295,7 +311,15 @@ void emit_findings_tree(std::string& o, const Palette& p, const std::vectorentropy >= 0) { show_entropy = true; break; } + if (r.mem && r.mem->entropy >= 0) { show_entropy = true; break; } + } size_t w_off = 6, w_size = 4, w_type = 4, w_tier = 4; + const size_t w_ent = 7; // "ENTROPY"; values are "7.98" for (const auto& r : rows) { if (r.f) { w_off = std::max(w_off, disp_w(r.first)); @@ -317,6 +341,10 @@ void emit_findings_tree(std::string& o, const Palette& p, const std::vectorsize), w_size, "", ""); line += " "; + // A member has no TIER; under -E fill it blank so the ENTROPY column + // stays aligned with the finding rows. col(line, r.mem->note, w_type, p.dim(), p.reset()); + if (show_entropy) { + line += " "; + col(line, "", w_tier, "", ""); // blank TIER + line += " "; + col(line, ent_str(r.mem->entropy), w_ent, p.ent(r.mem->entropy), p.reset()); + } while (!line.empty() && line.back() == ' ') line.pop_back(); o += line + "\n"; continue; @@ -358,6 +394,10 @@ void emit_findings_tree(std::string& o, const Palette& p, const std::vectortype, w_type, p.sec(*r.f), p.reset()); line += " "; col(line, r.f->confidence_tier, w_tier, p.tier(r.f->confidence_tier), p.reset()); + if (show_entropy) { + line += " "; + col(line, ent_str(r.f->entropy), w_ent, p.ent(r.f->entropy), p.reset()); + } line += " "; line += p.dim() + notes_for(*r.f, p, verbose) + p.reset(); while (!line.empty() && line.back() == ' ') line.pop_back(); diff --git a/src/json.cpp b/src/json.cpp index 28f1f50..949a4ff 100644 --- a/src/json.cpp +++ b/src/json.cpp @@ -56,6 +56,15 @@ void kv_num(std::string& o, const char* key, unsigned long long val, bool& first o += std::to_string(val); } +// A double with 2 decimals (for the -E entropy field). +void kv_double(std::string& o, const char* key, double val, bool& first) { + if (!first) o += ","; + first = false; + char buf[32]; + std::snprintf(buf, sizeof(buf), "\"%s\":%.2f", key, val); + o += buf; +} + void emit_finding(std::string& o, const Finding& f, bool with_also_matched); void emit_finding(std::string& o, const Finding& f, bool with_also_matched) { @@ -74,6 +83,7 @@ void emit_finding(std::string& o, const Finding& f, bool with_also_matched) { if (!f.label.empty()) kv_str(o, "label", f.label, first); if (!f.compression.empty()) kv_str(o, "compression", f.compression, first); if (!f.arch.empty()) kv_str(o, "arch", f.arch, first); + if (f.entropy >= 0) kv_double(o, "entropy", f.entropy, first); // Archive members (from --list) — emitted even in compact mode (the point of --list). if (!f.members.empty()) { @@ -85,6 +95,11 @@ void emit_finding(std::string& o, const Finding& f, bool with_also_matched) { o += "\",\"size\":" + std::to_string(f.members[i].size); if (f.members[i].offset != SIZE_MAX) o += ",\"offset\":" + std::to_string(f.members[i].offset); + if (f.members[i].entropy >= 0) { + char eb[32]; + std::snprintf(eb, sizeof(eb), ",\"entropy\":%.2f", f.members[i].entropy); + o += eb; + } if (!f.members[i].note.empty()) { o += ",\"note\":\""; escape_to(o, f.members[i].note); diff --git a/src/main.cpp b/src/main.cpp index 00c6da3..d363bdb 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -582,6 +583,18 @@ int main(int argc, char** argv) { if (entropy) { regions = ft::unidentified_regions(reader, findings, 4096); ent = ft::whole_file_entropy(reader); + // Comprehensive per-section entropy: annotate every finding (and its + // located members / nested children) so the -E view shows an ENTROPY + // column across the whole OFFSET table, not just the unidentified gaps. + std::function annotate = [&](ft::Finding& f) { + if (f.size > 0) f.entropy = ft::region_entropy(reader, f.offset, f.size); + for (auto& m : f.members) { + if (m.offset != SIZE_MAX && m.size > 0) + m.entropy = ft::region_entropy(reader, m.offset, m.size); + for (auto& c : m.children) annotate(c); + } + }; + for (auto& f : findings) annotate(f); } std::string assessment = ft::assess_file(findings, fm.size(), regions, ent, entropy); if (hidden_interior > 0) diff --git a/tests/run.sh b/tests/run.sh index 795a09c..6f7b960 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -22,6 +22,8 @@ python3 tests/test_partition.py python3 tests/test_littlefs.py # Partition-boundary overrun: a stale finding must not hide a real partition (PR #33). python3 tests/test_partition_overlap.py +# -E per-section entropy column on findings + members (human + JSON). +python3 tests/test_entropy.py # ESP32 partition-table region map + NVS identify/extract (synthetic, self-contained). python3 tests/test_esp32_part.py python3 tests/test_esp32_nvs.py diff --git a/tests/test_entropy.py b/tests/test_entropy.py new file mode 100644 index 0000000..ed01d8c --- /dev/null +++ b/tests/test_entropy.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Regression for the -E per-section entropy column (findings + members). + +-E adds a Shannon-entropy value to every finding and every located member, shown +as an ENTROPY column in the human OFFSET table and an `entropy` field in JSON. +Without -E there is neither. Fully self-contained (synthetic ext superblocks + +the gpt fixture). Run: python3 tests/test_entropy.py +""" +import json +import os +import random +import struct +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +MORIA = os.path.join(HERE, "..", "build", "moria") +sys.path.insert(0, HERE) +import gen_samples # noqa: E402 + + +def ext_over(buf): + """Plant a valid 64 KiB ext superblock at offset 0 of `buf` (finding size = + 64 KiB, so entropy is dominated by the buffer's contents).""" + b = bytearray(buf) + sb = 1024 + struct.pack_into(" 64 * 1024 = 64 KiB + struct.pack_into(" 1024 + struct.pack_into(" 7.5, "high-entropy ext: entropy > 7.5 under -E") + check(loe and loe[0].get("entropy", 9) < 1.0, "low-entropy ext: entropy < 1.0 under -E") + + no_e = findings(high_entropy_ext(), entropy=False) + check(all("entropy" not in x for x in no_e), "no -E: findings carry no entropy field") + + # --- members get entropy too (the gpt fixture has partition members) -------- + g = [x for x in findings(gen_samples.gpt_disk(), entropy=True) if x["type"] == "gpt"] + check(g and all("entropy" in m for m in g[0].get("members", [])), + "gpt members carry entropy under -E") + g0 = [x for x in findings(gen_samples.gpt_disk(), entropy=False) if x["type"] == "gpt"] + check(g0 and all("entropy" not in m for m in g0[0].get("members", [])), + "no -E: gpt members carry no entropy") + + # --- human: ENTROPY column present only under -E ---------------------------- + human_e = run(high_entropy_ext(), ["-E"]) + human_0 = run(high_entropy_ext(), []) + hdr_e = next((ln for ln in human_e.splitlines() if ln.startswith("OFFSET")), "") + hdr_0 = next((ln for ln in human_0.splitlines() if ln.startswith("OFFSET")), "") + check("ENTROPY" in hdr_e, "human -E: ENTROPY column in the header") + check("ENTROPY" not in hdr_0, "human without -E: no ENTROPY column") + # the high-entropy value shows in the table + check(any("8.00" in ln for ln in human_e.splitlines()), + "human -E: the 8.00 entropy value is rendered") + + print("-" * 60) + if fails: + for m in fails: + print("FAIL:", m) + return 1 + print("PASS: -E entropy column on findings + members (human + JSON), off by default") + return 0 + + +if __name__ == "__main__": + sys.exit(main())