From 85e9f7ff88881ef6a344ce2df2ea1c1ac446ce35 Mon Sep 17 00:00:00 2001 From: Matt Brown Date: Sat, 19 Sep 2026 00:52:50 -0400 Subject: [PATCH] feat(ubifs): recover orphaned inodes into lost+found on rootless captures A partial UBIFS capture missing its root LEB has no root inode 1 and no dentry parented at root, so the tree-only walk from root yields nothing even when valid inode/dentry/data nodes for real files are present. Add an orphan pass that re-roots everything unreachable from root inode 1 under a synthetic lost+found/: named subtrees, parent-link cycles, and data-bearing lone inodes. Lone inodes require real content so a metadata-only inode can't extract as a zero-filled shell. Status is marked partial only when the root inode was absent. Surface extraction status/warnings the human view was dropping: render a finding's own manifest-node status/warning on its row, link a finding to its extraction subtree by a unique offset when the type differs (a `ubi` finding's entry is labelled `ubifs`), and route extraction warnings and hard statuses into the diagnostics table so the NOTES column stays terse. Move the errors-only banner to just above the footer. --- src/extract/ubifs.cpp | 82 ++++++++++++++++++++++ src/human.cpp | 108 ++++++++++++++++++++++++---- tests/test_extract.py | 159 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 334 insertions(+), 15 deletions(-) diff --git a/src/extract/ubifs.cpp b/src/extract/ubifs.cpp index a7cc796..f92f437 100644 --- a/src/extract/ubifs.cpp +++ b/src/extract/ubifs.cpp @@ -97,6 +97,7 @@ struct Ctx { std::map inodes{}; std::map> dents{}; // keyed by parent inode std::set stack{}; + std::set visited{}; // inodes reached by the root walk (orphan detection) }; std::optional le32(const Reader& r, uint64_t o) { return r.at(o, Endian::Little); } @@ -284,6 +285,7 @@ void walk_dir(Ctx& c, uint32_t pino, const std::string& rel, size_t depth) { if (depth > MAX_DEPTH) { c.truncated = true; return; } if (c.stack.count(pino)) return; c.stack.insert(pino); + c.visited.insert(pino); auto it = c.dents.find(pino); if (it != c.dents.end()) { std::map best; @@ -310,6 +312,7 @@ void walk_dir(Ctx& c, uint32_t pino, const std::string& rel, size_t depth) { void write_inode(Ctx& c, uint32_t ino, const std::string& rel) { auto it = c.inodes.find(ino); if (it == c.inodes.end()) { c.truncated = true; return; } + c.visited.insert(ino); const InodeInfo& in = it->second; const uint32_t type = in.mode & S_IFMT; const std::string full = c.subdir + "/" + rel; @@ -332,12 +335,91 @@ void write_inode(Ctx& c, uint32_t ino, const std::string& rel) { } } +// Recover inodes the root walk never reached. moria is a best-effort recovery +// tool, not a forensic one: a partial capture whose root LEB is missing (common +// in flash dumps) still holds valid inode/dentry/data nodes for real files, and +// dropping them the way a tree-only walker does is the failure mode this exists +// to avoid. Everything unreachable from root inode 1 is re-rooted under a +// synthetic lost+found/ so it lands on disk anyway. Gated so a healthy image +// with no orphans produces no lost+found/. +void recover_orphans(Ctx& c) { + // Every inode named as a child by some dentry. A parent that is itself a + // child is reachable through that other dentry, so it is not a subtree top. + std::set child_inodes; + for (auto& [pino, dvec] : c.dents) + for (auto& d : dvec) + if (d.inum <= 0xffffffffull) child_inodes.insert(static_cast(d.inum)); + + const std::string lf = "lost+found"; + const size_t f0 = c.out.files, d0 = c.out.dirs, s0 = c.out.symlinks; + bool started = false; + auto ensure_lf = [&]() { + if (!started) { c.root.make_dir(c.subdir + "/" + lf); started = true; } + }; + + // 1. Orphan directory subtrees: a parent the root walk never reached and that + // no dentry names as a child — the top of a dangling tree. + for (auto& [pino, dvec] : c.dents) { + (void)dvec; + if (c.visited.count(pino) || child_inodes.count(pino)) continue; + ensure_lf(); + walk_dir(c, pino, lf + "/inode_" + std::to_string(pino), 0); + } + + // 2. Residual cyclic clusters: parents still unreached after (1) because every + // member is a child of another member, so none qualified as a top. Walk them + // anyway so a parent-link cycle can't make a whole subtree vanish. + for (auto& [pino, dvec] : c.dents) { + (void)dvec; + if (c.visited.count(pino)) continue; + ensure_lf(); + walk_dir(c, pino, lf + "/inode_" + std::to_string(pino), 0); + } + + // 3. Orphan lone inodes: a file/symlink with content but no dentry anywhere + // (its directory entry was lost). Emit it named by inode so its data isn't + // dropped. Require real content: with no name to carry information, a + // metadata-only inode whose data nodes did not survive would extract as a + // zero-filled shell (build_content fills holes with zero) — nothing dressed + // up as a file. A named orphan (1/2) is always worth emitting; a nameless + // one only when it carries data to recover. + for (auto& [ino, in] : c.inodes) { + if (c.visited.count(ino) || child_inodes.count(ino)) continue; + const uint32_t type = in.mode & S_IFMT; + const bool has_content = (type == S_IFREG && !in.data.empty()) || + (type == S_IFLNK && !in.inline_data.empty()); + if (!has_content) continue; + ensure_lf(); + write_inode(c, ino, lf + "/inode_" + std::to_string(ino)); + } + + if (started) { + std::string parts; + auto add = [&](size_t n, std::string what) { + if (!n) return; + if (n == 1 && !what.empty() && what.back() == 's') what.pop_back(); // singular + if (!parts.empty()) parts += ", "; + parts += std::to_string(n) + " " + what; + }; + add(c.out.files - f0, "files"); + add(c.out.dirs - d0, "dirs"); + add(c.out.symlinks - s0, "symlinks"); + c.out.warnings.push_back("recovered " + parts + + " unreachable from root inode into lost+found/"); + } +} + // Parse one UBIFS image (Reader over the volume) into `root/subdir`. void parse_ubifs(const Reader& img, SafeRoot& root, const std::string& subdir, Extracted& out, bool& truncated) { Ctx c{img, 0, img.size(), root, subdir, out}; scan_nodes(c); + // A capture missing its root LEB has neither a root inode node nor any dentry + // parented at root; the extraction is then inherently partial (recovery only). + const bool root_present = c.inodes.count(UBIFS_ROOT_INO) || c.dents.count(UBIFS_ROOT_INO); walk_dir(c, UBIFS_ROOT_INO, "", 0); + recover_orphans(c); + if (!root_present) c.truncated = true; if (c.truncated) truncated = true; } diff --git a/src/human.cpp b/src/human.cpp index c54ccad..36924db 100644 --- a/src/human.cpp +++ b/src/human.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -425,6 +426,50 @@ std::vector gather_diagnostics(const std::vector& fs) { return rows; } +// Every manifest root begins with the top-level "0x-" dir; that hex is +// the container's offset in the input file — the useful offset for a nested entry, +// whose own e.offset is relative to its parent payload. +size_t parse_root_offset(const std::string& root) { + if (root.size() > 2 && root[0] == '0' && (root[1] == 'x' || root[1] == 'X')) + return static_cast(std::strtoull(root.c_str() + 2, nullptr, 16)); + return 0; +} + +// Compact identifier for a nested extraction: the deepest ".extracted" path +// segment minus the suffix (e.g. "vol_3.img"). Empty for a top-level entry. +std::string extraction_label(const std::string& root) { + const std::string ext = ".extracted"; + std::string best; + for (size_t start = 0; start <= root.size();) { + size_t slash = root.find('/', start); + size_t len = (slash == std::string::npos) ? root.size() - start : slash - start; + std::string seg = root.substr(start, len); + if (seg.size() > ext.size() && seg.compare(seg.size() - ext.size(), ext.size(), ext) == 0) + best = seg.substr(0, seg.size() - ext.size()); + if (slash == std::string::npos) break; + start = slash + 1; + } + return best; +} + +// Extraction-side trouble for the diagnostics table: every manifest entry's +// warnings (e.g. a lost+found recovery) and hard statuses (error/unsupported). +// This keeps the detail in one scannable place so the NOTES column stays terse. +void gather_extraction_diags(std::vector& rows, const Manifest& man) { + std::set seen; // dedupe identical severity+message + for (const auto& e : man.entries) { + const size_t off = parse_root_offset(e.root); + const std::string label = extraction_label(e.root); + auto add = [&](const std::string& sev, const std::string& msg) { + std::string full = label.empty() ? msg : label + ": " + msg; + if (!seen.insert(sev + "\x1f" + full).second) return; + rows.push_back({sev, "extract", e.type, full, off}); + }; + for (const auto& w : e.warnings) add("warning", w); + if (e.status != "ok" && e.status != "partial") add("error", e.status); + } +} + // The diagnostics section: a table SEVERITY | OFFSET | TYPE | MESSAGE listing // every finding's diagnostics. The single place to scan for trouble; the NOTES // column flags each in situ. @@ -526,10 +571,19 @@ void emit_extraction_tree(std::string& o, const Palette& p, const std::vector int { - for (size_t i = 0; i < n; ++i) - if (parent[i] < 0 && es[i].offset == f.offset && es[i].type == f.type) - return static_cast(i); - return -1; + // Prefer an exact offset+type match; else fall back to a unique offset + // match, so a container whose top extraction entry is labelled by its inner + // filesystem (a `ubi` finding whose extraction entry is `ubifs`) still links + // to its subtree instead of dropping it from the human view. + int exact = -1, by_off = -1, off_count = 0; + for (size_t i = 0; i < n; ++i) { + if (parent[i] >= 0 || es[i].offset != f.offset) continue; + if (es[i].type == f.type) exact = static_cast(i); + by_off = static_cast(i); + ++off_count; + } + if (exact >= 0) return exact; + return off_count == 1 ? by_off : -1; }; // Extraction-side NOTES: warnings, plus a bare status when it is not clean. A @@ -541,7 +595,9 @@ void emit_extraction_tree(std::string& o, const Palette& p, const std::vector / unsupported: @@ -635,8 +691,16 @@ void emit_extraction_tree(std::string& o, const Palette& p, const std::vector= 0) { + std::string exn = ex_notes(es[node], !kids[node].empty()); + if (!exn.empty()) r.notes_str += (r.notes_str.empty() ? "" : " ") + exn; + } + rows.push_back(r); if (node >= 0) emit_kids(node, ""); } @@ -707,19 +771,21 @@ std::string emit_file_human(const std::vector& findings, Palette p{color}; std::string o; - // Errors-only banner at the very top: a "moria could not do this" result - // must not be buried under a long findings table. Warnings/info live only in - // the diagnostics section and the NOTES column. + // Diagnostics drive an errors-only banner emitted lower down (just above the + // footer). Warnings/info live only in the diagnostics section and the NOTES + // column; extraction warnings/statuses are merged in below. auto diags = gather_diagnostics(findings); + if (extraction) { + gather_extraction_diags(diags, *extraction); + auto rank = [](const std::string& s) { return s == "error" ? 0 : s == "warning" ? 1 : 2; }; + std::sort(diags.begin(), diags.end(), [&](const DiagRow& a, const DiagRow& b) { + if (rank(a.severity) != rank(b.severity)) return rank(a.severity) < rank(b.severity); + return a.offset < b.offset; + }); + } size_t nerr = 0; for (const auto& d : diags) if (d.severity == "error") ++nerr; - if (nerr > 0) { - o += p.sev("error"); - o += (nerr == 1 ? "! 1 error" : "! " + std::to_string(nerr) + " errors"); - o += " — see diagnostics below\n\n"; - o += p.reset(); - } if (findings.empty()) { o += p.dim(); @@ -747,6 +813,18 @@ std::string emit_file_human(const std::vector& findings, } } + // Errors-only banner, just above the footer: a "moria could not do this" + // result sits right before the "-> extracted to ..." line so it is the last + // thing before the run summary, not buried at the top. + if (nerr > 0) { + o += "\n"; + o += p.sev("error"); + o += (nerr == 1 ? "! 1 error" : "! " + std::to_string(nerr) + " errors"); + o += " — see diagnostics above"; + o += p.reset(); + o += "\n"; + } + if (!footer.empty()) { o += "\n"; o += p.dim() + footer + p.reset(); diff --git a/tests/test_extract.py b/tests/test_extract.py index 61f2151..942d4c6 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -282,6 +282,164 @@ def peb(vol_id, lnum, sqnum, marker): return 1 +def test_ubifs_orphan_recovery(work): + """A partial UBIFS capture missing its root LEB has no root inode 1 and no + dentry parented at root, so a tree-only walk from root yields nothing even + though valid inode/dentry/data nodes for real files are present. moria must + recover the unreachable content into a synthetic lost+found/. Self-contained: + a hand-built raw UBIFS (no mkfs tools, UBIFS node CRC = init 0xFFFFFFFF, no + final xor -> zlib.crc32(body) ^ 0xffffffff). Covers a named orphan subtree + (cat 1), a parent-link cycle (cat 2), a data-bearing lone inode (cat 3), and + the lone-inode content gate (a metadata-only inode must NOT emit a zero + shell). Returns failures.""" + import struct + import zlib + + ubicrc = lambda b: zlib.crc32(b) ^ 0xFFFFFFFF + S_IFDIR, S_IFREG = 0o40000, 0o100000 + + def node(ntype, body_after_ch, sqnum=1): + # body_after_ch is everything past the 24-byte common header; the node + # length is 24 + len(body_after_ch). + length = 24 + len(body_after_ch) + n = bytearray(length) + struct.pack_into(" orphan.txt (inode 101). No root. + ino(100, S_IFDIR | 0o755, 4096), + dent(100, 101, "orphan.txt"), + ino(101, S_IFREG | 0o644, len(c1)), + data(101, 0, c1), + # cat 3: data-bearing lone inode 200 (no dentry references it). + ino(200, S_IFREG | 0o644, len(c2)), + data(200, 0, c2), + # cat 3 gate: metadata-only lone inode 201 (no data node) must be skipped. + ino(201, S_IFREG | 0o644, 4096), + # cat 2: dirs 300<->301 each name the other (both are children, so neither + # is a subtree top); 301 also holds cyclefile (inode 302). + ino(300, S_IFDIR | 0o755, 4096), + ino(301, S_IFDIR | 0o755, 4096), + dent(300, 301, "b"), + dent(301, 300, "a"), + dent(301, 302, "cyclefile"), + ino(302, S_IFREG | 0o644, len(c3)), + data(302, 0, c3), + ] + img = bytearray() + for n in nodes: + img += n + if len(img) % 8: + img += b"\x00" * (8 - len(img) % 8) + + src = os.path.join(work, "orphan.ubifs") + with open(src, "wb") as f: + f.write(img) + outdir = os.path.join(work, "orphan.out") + r = subprocess.run([MORIA, "-j", "--extract", "-C", outdir, src], capture_output=True) + try: + entry = json.loads(r.stdout.decode())["extraction"]["extracted"][0] + except Exception: + print(f"FAIL [ubifs-orphan]: no extraction manifest\n{r.stdout[:200]}") + return 1 + root = os.path.join(outdir, entry["root"]) + + def rd(rel): + p = os.path.join(root, rel) + if not os.path.isfile(p): + return None + with open(p, "rb") as fh: + return fh.read() + + failures = 0 + lf = "lost+found" + checks = [ + (os.path.join(lf, "inode_100", "orphan.txt"), c1, "named subtree (cat 1)"), + (os.path.join(lf, "inode_200"), c2, "data-bearing lone inode (cat 3)"), + (os.path.join(lf, "inode_300", "b", "cyclefile"), c3, "cyclic cluster (cat 2)"), + ] + for rel, want, label in checks: + if rd(rel) != want: + print(f"FAIL [ubifs-orphan]: {label} missing/wrong at {rel}") + failures += 1 + + # cat 3 gate: the metadata-only inode 201 must not extract a zero shell. + if os.path.exists(os.path.join(root, lf, "inode_201")): + print("FAIL [ubifs-orphan]: metadata-only inode_201 emitted a zero shell") + failures += 1 + + # No root inode -> everything is under lost+found, and status is partial. + strays = [os.path.join(dp, f) for dp, _, fs in os.walk(root) for f in fs + if lf not in os.path.relpath(os.path.join(dp, f), root).split(os.sep)] + if strays: + print(f"FAIL [ubifs-orphan]: files outside lost+found: {strays[:3]}") + failures += 1 + if entry.get("status") != "partial": + print(f"FAIL [ubifs-orphan]: status={entry.get('status')!r}, expected 'partial'") + failures += 1 + + # The recovery must reach the human view, not just -j: the warning is surfaced + # on the finding row (extraction status/warning on a finding's own manifest + # node, and offset-fallback linkage for containers labelled by their inner FS). + h = subprocess.run([MORIA, "--extract", "-C", os.path.join(work, "orphan.hout"), src], + capture_output=True).stdout + # terse tag on the finding/tree row, full sentence in the diagnostics table. + if b"lost+found" not in h: + print(f"FAIL [ubifs-orphan]: recovery not surfaced in human -e output\n{h[:300]}") + failures += 1 + if b"diagnostics:" not in h or b"recovered" not in h: + print(f"FAIL [ubifs-orphan]: recovery detail not in diagnostics section\n{h[:400]}") + failures += 1 + + if not failures: + print("PASS [ubifs-orphan]: cat1 subtree + cat2 cycle + cat3 lone inode " + "recovered, zero-shell gated, status partial") + return failures + + def test_tar(work, src, expected): """Pack the tree with GNU tar (gnu/pax/ustar), extract with moria, compare. Needs `tar`. Exercises long names, prefix, and pax path records. Returns @@ -1721,6 +1879,7 @@ def main(): failures += test_jffs2(work, src, expected) failures += test_ubifs(work, src, expected) failures += test_ubi_sparse_volume(work) + failures += test_ubifs_orphan_recovery(work) failures += test_tar(work, src, expected) failures += test_romfs(work, src, expected) failures += test_yaffs2(work, src, expected)