From 396b3a4fbf50538ce2462d080fefb03147ffc0bb Mon Sep 17 00:00:00 2001 From: Efi Jeremiah Date: Thu, 3 Sep 2026 00:24:33 +0300 Subject: [PATCH] feat(analyze): one analysis command; serve/investigate/report share the run - internal/analysis: Run() orchestrates normalize (only when missing/stale), endpoint + shell-history correlation before detection, streaming detections, rule packs, MCP audit (+ gateway), provenance; writes detections/{findings, mcp-audit,provenance,corroboration,analysis}.json de-duplicated and severity-sorted; earlier correlation results survive re-analysis unless the overlay is re-parsed; Stale()/Ensure() for consumers. - cli: `analyze` (triage = alias) with --endpoint/--shell-history/--gateway-*/ --rules/--honeytokens/--renormalize/--json; serve, investigate, report call analysis.Ensure and render the persisted run; help text regrouped by workflow step in plain language. - serve UI: empty extras sections hidden; message no longer tells the user to run other commands. - tests: all stages produce their files; states and correlation findings kept across re-runs; renormalize invalidates; Ensure no-op; sorted output. - docs: README four-step quick start, CHANGELOG; version 0.13.0-dev. --- CHANGELOG.md | 21 ++ README.md | 67 ++---- docs/endpoint-corroboration.md | 2 +- internal/analysis/analysis.go | 372 +++++++++++++++++++++++++++++ internal/analysis/analysis_test.go | 125 ++++++++++ internal/cli/analyst_cmds.go | 7 +- internal/cli/analyze.go | 158 ++++-------- internal/cli/cli.go | 93 ++++---- internal/cli/report_cmds.go | 11 +- internal/serve/serve.go | 60 +---- internal/serve/serve_test.go | 11 +- internal/serve/ui.html | 5 +- internal/version/version.go | 2 +- 13 files changed, 654 insertions(+), 280 deletions(-) create mode 100644 internal/analysis/analysis.go create mode 100644 internal/analysis/analysis_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0906d12..b83ae3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,27 @@ and the project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ## [Unreleased] +### Added +- **`agentdfir analyze `** — one command runs every analysis stage in the + right order: normalize (only when the overlay is missing or the package is + newer), endpoint correlation (`--endpoint`, `--shell-history`), detections, + rule packs, MCP audit (+ `--gateway-log`), instruction provenance; writes one + consistent `detections/` set plus `analysis.json`. `triage` is the same command. + `serve`, `investigate` and `report` now run this analysis automatically when + results are missing or stale and render the SAME run — no more "run X and + reload", no recomputation with different states. Single-stage commands + (`correlate`, `mcp audit`, `provenance`, `normalize`) remain for scripting. + +### Changed +- Help text reorganized by workflow step (detect → collect → analyze → look → + export; before-an-incident; trust & keys) in plain language. +- Findings file is de-duplicated and severity-sorted across all stages. + +### Fixed +- Re-running analysis without endpoint logs no longer discards earlier + CORROBORATED/CONTRADICTED states or correlation findings; a re-parse + (`--renormalize`, or a package sealed after the overlay) invalidates them + explicitly instead of silently. ## [0.12.1] — 2026-09-02 Distribution-only release. No runtime code changes. diff --git a/README.md b/README.md index 61e617b..cf6dd86 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ AI-generated text is **never** automatically treated as proof of execution. Ever An agent claiming *"I executed curl example.com"* with no matching tool call stays `REPORTED` — and AgentDFIR shows you exactly that. -## ⚡ Quick start +## ⚡ Quick start — four steps **Install** — one static binary, zero runtime dependencies. Full guide with air-gap, checksum and Sigstore verification steps: [docs/install.md](docs/install.md). @@ -62,62 +62,25 @@ go install github.com/efij/AgentDFIR/cmd/agentdfir@latest go build -trimpath -o agentdfir ./cmd/agentdfir ``` -```sh -# Discover installed AI tooling — never executes suspect binaries -./agentdfir detect - -# Forensic acquisition (lossless, sealed, hash-chained) -./agentdfir collect --product claude --operator "Your Name" - -# From an offline image / copied home directory -./agentdfir collect --product claude --path /mnt/image/Users/suspect \ - --case-id CASE-2026-042 --authorization "IR-TICKET-123" - -# From a KAPE / Velociraptor / CyLR tree: every product, every user, one package -./agentdfir collect --import /cases/host42/kape-output --case-id CASE-2026-042 - -# From a container (read-only docker export) or a CI artifact / support bundle / vendor export -./agentdfir collect --docker devcontainer-3a1f -./agentdfir collect --archive copilot-run-9921.zip - -# Tamper-evident verification — one flipped byte anywhere fails -./agentdfir verify CASE-2026-042.adfir - -# Investigate -./agentdfir timeline CASE-2026-042.adfir # unified, evidence-linked timeline -./agentdfir triage CASE-2026-042.adfir # detections + IR-ready findings -./agentdfir report CASE-2026-042.adfir --format pdf # one-file PDF: findings, timeline, custody, integrity - -# Train / test / demo with synthetic incidents -./agentdfir simulate --scenario orphan-agent --out demo-profile -# Browser case explorer — agent tree, timeline scrubber, raw evidence, findings (127.0.0.1 only) -./agentdfir serve CASE-2026-042.adfir --open - -# Live watch — or a real-time sensor: detections pushed to your SOC within one poll interval -./agentdfir monitor -./agentdfir monitor --detect --alert https://soc.example/hook --honeytokens canaries.txt -./agentdfir investigate CASE-2026-042.adfir -./agentdfir replay --session 9b2d CASE-2026-042.adfir - -# Org rule packs + honeytokens -./agentdfir triage --rules ./rules --honeytokens canaries.txt CASE-2026-042.adfir - -# Second witness: corroborate the transcript against OS telemetry (auditd / Sysmon / EDR exports) -./agentdfir correlate CASE-2026-042.adfir /var/log/audit/audit.log -./agentdfir triage CASE-2026-042.adfir --endpoint sysmon.xml - -# Who wrote each line of CLAUDE.md / .cursorrules — and did it come from a tool result? -./agentdfir provenance CASE-2026-042.adfir CLAUDE.md +```sh +./agentdfir detect # 1. what AI agents are on this machine (never runs them) +./agentdfir collect --product claude # 2. sealed, hash-chained evidence package +./agentdfir analyze CASE-2026-042.adfir # 3. every analysis stage, one command +./agentdfir serve CASE-2026-042.adfir --open # 4. browse: agent tree, timeline, raw evidence, findings +``` -# MCP supply-chain audit: every server, every agent, read-only — plus gateway-log correlation -./agentdfir mcp audit -./agentdfir mcp audit CASE-2026-042.adfir --gateway-log gw.jsonl --gateway-server gateway +Add a second witness and the same commands upgrade every finding from *the agent says* to *the OS confirms*: -# Add a brand-new AI agent product with one signed JSON file — no Go -./agentdfir packs init foo-agent --config-dir .foo && ./agentdfir packs add foo-agent.product.json +```sh +./agentdfir analyze CASE-2026-042.adfir --endpoint /var/log/audit/audit.log # auditd / Sysmon XML / EDR exports +./agentdfir analyze CASE-2026-042.adfir --gateway-log mcp-gateway.jsonl # your MCP gateway's own log ``` +Other ways in: `collect --path `, `--import `, `--docker `, `--archive `. +Other ways out: `report --format pdf|html|ocsf|sarif|timesketch|…`, `rules export --sigma`. Before an incident: `monitor --detect --alert `, `mcp audit`. +Every command is listed by workflow step in `agentdfir help`. + Example finding: ``` diff --git a/docs/endpoint-corroboration.md b/docs/endpoint-corroboration.md index 4350c44..4300cfb 100644 --- a/docs/endpoint-corroboration.md +++ b/docs/endpoint-corroboration.md @@ -16,7 +16,7 @@ An agent's transcript is witness #1: the agent's own diary. It can be wrong (the agentdfir correlate CASE-42.adfir /var/log/audit/audit.log # Linux auditd agentdfir correlate CASE-42.adfir sysmon.xml # Windows Sysmon (XML export) agentdfir correlate CASE-42.adfir procs.jsonl netconns.csv # Velociraptor / osquery / eslogger / EDR exports -agentdfir triage CASE-42.adfir --endpoint audit.log # same, inside triage +agentdfir analyze CASE-42.adfir --endpoint audit.log # same, inside the one-shot analysis ``` Format is sniffed per file (`--format auditd|sysmon-xml|jsonl|csv` to override). `--window 3s` sets the match window. Results are written back to `normalized/events.jsonl` (states + an evidence note naming the corroborating record) and to `detections/corroboration.json`; `triage` merges the findings so every downstream report, OCSF/SARIF export and PDF carries the upgraded states. diff --git a/internal/analysis/analysis.go b/internal/analysis/analysis.go new file mode 100644 index 0000000..b51735c --- /dev/null +++ b/internal/analysis/analysis.go @@ -0,0 +1,372 @@ +// Package analysis is the one place a package gets analyzed. It runs the +// stages in the right order — normalize (only when needed), endpoint +// correlation, detections, rule packs, MCP audit, provenance — and writes +// one consistent set of results under /detections/. Every command +// that shows or exports results (analyze/triage, serve, investigate, +// report) goes through here, so nothing is ever "not run yet" and later +// stages never clobber earlier ones. +package analysis + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/efij/AgentDFIR/internal/correlate" + "github.com/efij/AgentDFIR/internal/detect" + "github.com/efij/AgentDFIR/internal/endpoint" + "github.com/efij/AgentDFIR/internal/mcpaudit" + "github.com/efij/AgentDFIR/internal/normalize" + "github.com/efij/AgentDFIR/internal/provenance" + "github.com/efij/AgentDFIR/internal/rulepack" + "github.com/efij/AgentDFIR/internal/schema" +) + +// Options are the optional inputs an analyst may add. +type Options struct { + EndpointLogs []string // auditd / Sysmon XML / JSONL-CSV exports (second witness) + EndpointFormat endpoint.Format + Window time.Duration + ShellHistory string + GatewayLog string + GatewayMap string + GatewayServers []string + RulesDir string + Honeytokens []string + SpawnThreshold int + KnownDests []string + Renormalize bool // force re-parse even if the overlay is current + Log io.Writer // progress lines; nil = silent +} + +// Result summarizes one run. +type Result struct { + Events int + Entities int + Renormalized bool + Findings []schema.Finding + Correlation *correlate.EndpointResult + MCPServers int + Provenance int // instruction files attributed + StageNotes []string +} + +func (o *Options) logf(format string, a ...any) { + if o.Log != nil { + fmt.Fprintf(o.Log, format+"\n", a...) + } +} + +// Stale reports whether results need (re)computing: no overlay, no +// findings, or the package was sealed after the overlay was written. +func Stale(pkg string) bool { + ev, err := os.Stat(filepath.Join(pkg, "normalized", "events.jsonl")) + if err != nil { + return true + } + if _, err := os.Stat(filepath.Join(pkg, "detections", "findings.json")); err != nil { + return true + } + if man, err := os.Stat(filepath.Join(pkg, "manifest.json")); err == nil && man.ModTime().After(ev.ModTime()) { + return true + } + return false +} + +// Ensure runs a default analysis only when results are missing or stale. +func Ensure(pkg string, log io.Writer) (*Result, error) { + if !Stale(pkg) { + return nil, nil + } + return Run(pkg, Options{Log: log}) +} + +// Run executes every stage and persists results. +func Run(pkg string, o Options) (*Result, error) { + if o.SpawnThreshold <= 0 { + o.SpawnThreshold = 10 + } + res := &Result{} + dir := filepath.Join(pkg, "normalized") + detDir := filepath.Join(pkg, "detections") + for _, d := range []string{dir, detDir} { + if err := os.MkdirAll(d, 0o700); err != nil { + return nil, err + } + } + + // ---- 1. normalize (streaming) — only when the overlay is missing/stale. + evPath := filepath.Join(dir, "events.jsonl") + needNorm := o.Renormalize + if fi, err := os.Stat(evPath); err != nil { + needNorm = true + } else if man, err := os.Stat(filepath.Join(pkg, "manifest.json")); err == nil && man.ModTime().After(fi.ModTime()) { + needNorm = true + } + var entities []schema.Entity + if needNorm { + f, err := os.OpenFile(evPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + return nil, err + } + enc := json.NewEncoder(f) + sr, err := normalize.ParseStream(pkg, func(ev schema.Event) error { return enc.Encode(ev) }) + if err != nil { + f.Close() + return nil, err + } + f.Close() + if err := writeJSONL(filepath.Join(dir, "entities.jsonl"), len(sr.Entities), func(i int) any { return sr.Entities[i] }); err != nil { + return nil, err + } + if err := writeJSONL(filepath.Join(dir, "relationships.jsonl"), len(sr.Relationships), func(i int) any { return sr.Relationships[i] }); err != nil { + return nil, err + } + entities, res.Events, res.Renormalized = sr.Entities, sr.EventCount, true + o.logf("Normalized: %d events, %d entities, %d relationships", sr.EventCount, len(sr.Entities), len(sr.Relationships)) + } else { + entities = readJSONL[schema.Entity](filepath.Join(dir, "entities.jsonl")) + res.Events = countLines(evPath) + o.logf("Normalized: reusing overlay (%d events); corroboration states preserved", res.Events) + } + res.Entities = len(entities) + + // ---- 2. second witness (runs BEFORE detection so findings carry the states). + var findings []schema.Finding + if len(o.EndpointLogs) > 0 || o.ShellHistory != "" { + events := LoadEvents(pkg) + if o.ShellHistory != "" { + if cres, err := correlate.Apply(events, &correlate.ShellHistoryAdapter{Path: o.ShellHistory}); err == nil && cres.Corroborated > 0 { + o.logf("Shell history: %d tool call(s) corroborated", cres.Corroborated) + } + } + if len(o.EndpointLogs) > 0 { + var records []endpoint.Record + for _, p := range o.EndpointLogs { + lr, err := endpoint.Load(p, o.EndpointFormat) + if err != nil { + return nil, fmt.Errorf("endpoint log %s: %w", p, err) + } + o.logf("Endpoint log %s: %s, %d records (%d skipped)", filepath.Base(p), lr.Format, len(lr.Records), lr.Skipped) + records = append(records, lr.Records...) + } + cres, cf := correlate.Endpoint(events, records, correlate.EndpointOptions{Window: o.Window, KnownDests: o.KnownDests}) + res.Correlation = cres + findings = append(findings, cf...) + writeJSON(filepath.Join(detDir, "corroboration.json"), struct { + Summary *correlate.EndpointResult `json:"summary"` + Findings []schema.Finding `json:"findings"` + }{cres, cf}) + o.logf("Endpoint correlation: %d checked — %d CORROBORATED, %d CONTRADICTED, %d outside coverage; %d unlogged agent records", + cres.ToolCalls, cres.Corroborated, cres.Contradicted, cres.OutsideCover, cres.Unlogged) + } + if err := writeJSONL(evPath, len(events), func(i int) any { return events[i] }); err != nil { + return nil, err + } + } + + // Earlier second-witness results stay part of the case as long as the + // overlay they were computed on is still in use; a re-parse invalidates them. + corrPath := filepath.Join(detDir, "corroboration.json") + if len(o.EndpointLogs) == 0 { + if res.Renormalized { + _ = os.Remove(corrPath) + } else if data, err := os.ReadFile(corrPath); err == nil { + var prev struct { + Summary *correlate.EndpointResult `json:"summary"` + Findings []schema.Finding `json:"findings"` + } + if json.Unmarshal(data, &prev) == nil { + res.Correlation = prev.Summary + findings = append(findings, prev.Findings...) + o.logf("Endpoint correlation: reusing earlier results (%d finding(s))", len(prev.Findings)) + } + } + } + + // ---- 3. detections (streaming over the overlay). + det, err := detect.RunStream(pkg, entities, detect.Options{Honeytokens: o.Honeytokens, SpawnThreshold: o.SpawnThreshold, KnownDestinations: o.KnownDests}) + if err != nil { + return nil, err + } + findings = append(findings, det...) + + // ---- 4. declarative rule packs. + if o.RulesDir != "" { + packs, err := rulepack.LoadDir(o.RulesDir) + if err != nil { + return nil, fmt.Errorf("rule packs: %w", err) + } + extra, err := rulepack.Apply(packs, &schema.Normalized{Events: LoadEvents(pkg)}, pkg) + if err != nil { + return nil, fmt.Errorf("rule packs: %w", err) + } + findings = append(findings, extra...) + o.logf("Rule packs: %d pack(s), %d finding(s)", len(packs), len(extra)) + } + + // ---- 5. MCP supply-chain audit (+ gateway corroboration). + inv, mcpExtra, err := mcpaudit.ScanPackage(pkg) + if err == nil { + mf := append(mcpExtra, mcpaudit.Evaluate(inv)...) + var gw *mcpaudit.GatewaySummary + if o.GatewayLog != "" { + m := mcpaudit.DefaultGatewayMap + if o.GatewayMap != "" { + if data, err := os.ReadFile(o.GatewayMap); err == nil { + _ = json.Unmarshal(data, &m) + } + } + recs, unparsed, err := mcpaudit.LoadGatewayLog(o.GatewayLog, m) + if err != nil { + return nil, fmt.Errorf("gateway log: %w", err) + } + sum, gf := mcpaudit.CorrelateGateway(LoadEvents(pkg), recs, o.GatewayServers, 3) + sum.Unparsed = unparsed + gw = &sum + mf = append(mf, gf...) + } + res.MCPServers = len(inv.Servers) + findings = append(findings, mf...) + writeJSON(filepath.Join(detDir, "mcp-audit.json"), struct { + Inventory *mcpaudit.Inventory `json:"inventory"` + Findings []schema.Finding `json:"findings"` + Gateway *mcpaudit.GatewaySummary `json:"gateway,omitempty"` + }{inv, mf, gw}) + o.logf("MCP audit: %d server(s) in %d config(s), %d finding(s)", len(inv.Servers), len(inv.Configs), len(mf)) + } else { + res.StageNotes = append(res.StageNotes, "mcp audit skipped: "+err.Error()) + } + + // ---- 6. instruction & memory provenance. + if prov, err := provenance.Run(pkg, LoadEvents(pkg), ""); err == nil { + res.Provenance = len(prov.Files) + findings = append(findings, prov.Findings...) + writeJSON(filepath.Join(detDir, "provenance.json"), prov) + o.logf("Provenance: %d instruction file(s) attributed, %d write(s) to uncollected instruction paths, %d finding(s)", len(prov.Files), len(prov.OtherWrite), len(prov.Findings)) + } else { + res.StageNotes = append(res.StageNotes, "provenance skipped: "+err.Error()) + } + + // ---- 7. one findings file, severity-sorted, de-duplicated. + findings = dedupe(findings) + sortBySeverity(findings) + res.Findings = findings + writeJSON(filepath.Join(detDir, "findings.json"), findings) + writeJSON(filepath.Join(detDir, "analysis.json"), map[string]any{ + "analyzed_utc": time.Now().UTC().Format(time.RFC3339), "events": res.Events, "renormalized": res.Renormalized, + "findings": len(findings), "endpoint_logs": o.EndpointLogs, "gateway_log": o.GatewayLog, "rules_dir": o.RulesDir, + "honeytokens": len(o.Honeytokens), "notes": res.StageNotes, + }) + return res, nil +} + +// LoadEvents reads the overlay into memory (for stages that need it). +func LoadEvents(pkg string) []schema.Event { + return readJSONL[schema.Event](filepath.Join(pkg, "normalized", "events.jsonl")) +} + +// LoadEntities reads the overlay entities. +func LoadEntities(pkg string) []schema.Entity { + return readJSONL[schema.Entity](filepath.Join(pkg, "normalized", "entities.jsonl")) +} + +// LoadFindings reads the persisted findings. +func LoadFindings(pkg string) []schema.Finding { + var out []schema.Finding + if data, err := os.ReadFile(filepath.Join(pkg, "detections", "findings.json")); err == nil { + _ = json.Unmarshal(data, &out) + } + return out +} + +func dedupe(f []schema.Finding) []schema.Finding { + seen := map[string]bool{} + var out []schema.Finding + for _, x := range f { + key := x.RuleID + "|" + x.SessionID + "|" + x.AgentID + "|" + strings.Join(x.EvidenceRefs, "|") + "|" + x.Title + if seen[key] { + continue + } + seen[key] = true + out = append(out, x) + } + return out +} + +var sevRank = map[string]int{"CRITICAL": 5, "HIGH": 4, "MEDIUM": 3, "LOW": 2, "INFO": 1} + +func sortBySeverity(f []schema.Finding) { + for i := 1; i < len(f); i++ { + for j := i; j > 0 && sevRank[f[j].Severity] > sevRank[f[j-1].Severity]; j-- { + f[j], f[j-1] = f[j-1], f[j] + } + } +} + +func writeJSON(path string, v any) { + data, err := json.MarshalIndent(v, "", " ") + if err != nil { + return + } + _ = os.WriteFile(path, append(data, '\n'), 0o600) +} + +func writeJSONL(path string, n int, get func(int) any) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + return err + } + enc := json.NewEncoder(f) + for i := 0; i < n; i++ { + if err := enc.Encode(get(i)); err != nil { + f.Close() + return err + } + } + return f.Close() +} + +func readJSONL[T any](path string) []T { + f, err := os.Open(path) + if err != nil { + return nil + } + defer f.Close() + var out []T + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) + for sc.Scan() { + var v T + if json.Unmarshal(sc.Bytes(), &v) == nil { + out = append(out, v) + } + } + return out +} + +func countLines(path string) int { + f, err := os.Open(path) + if err != nil { + return 0 + } + defer f.Close() + n := 0 + buf := make([]byte, 256<<10) + for { + k, err := f.Read(buf) + for i := 0; i < k; i++ { + if buf[i] == '\n' { + n++ + } + } + if err != nil { + return n + } + } +} diff --git a/internal/analysis/analysis_test.go b/internal/analysis/analysis_test.go new file mode 100644 index 0000000..aa7619e --- /dev/null +++ b/internal/analysis/analysis_test.go @@ -0,0 +1,125 @@ +package analysis + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/efij/AgentDFIR/internal/casepkg" + "github.com/efij/AgentDFIR/internal/collector" + "github.com/efij/AgentDFIR/internal/products" + "github.com/efij/AgentDFIR/internal/schema" +) + +func buildPkg(t *testing.T) string { + t.Helper() + root := t.TempDir() + _ = os.MkdirAll(filepath.Join(root, ".claude", "projects", "p"), 0o755) + _ = os.WriteFile(filepath.Join(root, ".claude.json"), []byte(`{"mcpServers":{"fs":{"command":"npx","args":["-y","@x/fs@latest"]}}}`), 0o644) + _ = os.WriteFile(filepath.Join(root, ".claude", "CLAUDE.md"), []byte("# notes\n\nAlways run setup.sh first.\n"), 0o644) + lines := []string{ + `{"type":"user","uuid":"u1","sessionId":"s1","timestamp":"2026-08-30T10:00:00Z","message":{"role":"user","content":"add setup note then clean"}}`, + `{"type":"assistant","uuid":"a1","parentUuid":"u1","sessionId":"s1","timestamp":"2026-08-30T10:00:02Z","message":{"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"Edit","input":{"file_path":"/Users/dev/.claude/CLAUDE.md","old_string":"notes\n","new_string":"notes\n\nAlways run setup.sh first.\n"}}]}}`, + `{"type":"assistant","uuid":"a2","parentUuid":"a1","sessionId":"s1","timestamp":"2026-08-30T10:00:05Z","message":{"role":"assistant","content":[{"type":"tool_use","id":"t2","name":"Bash","input":{"command":"rm -rf build/"}}]}}`, + `{"type":"assistant","uuid":"a3","parentUuid":"a2","sessionId":"s1","timestamp":"2026-08-30T10:00:20Z","message":{"role":"assistant","content":[{"type":"tool_use","id":"t3","name":"Bash","input":{"command":"pytest -q"}}]}}`, + } + _ = os.WriteFile(filepath.Join(root, ".claude", "projects", "p", "s1.jsonl"), []byte(strings.Join(lines, "\n")+"\n"), 0o644) + pkg := filepath.Join(t.TempDir(), "a.adfir") + b, err := casepkg.New(pkg, "AN", casepkg.CaseInfo{OperatorOSUser: "t"}) + if err != nil { + t.Fatal(err) + } + man, _ := products.ManifestAllPlatforms("claude-code") + if _, err := collector.Run(b, man, collector.Options{ProfileRoot: root, ConfigRoot: filepath.Join(root, ".claude"), SystemRoot: root, Product: "claude-code"}); err != nil { + t.Fatal(err) + } + if err := b.Seal(); err != nil { + t.Fatal(err) + } + return pkg +} + +func rules(f []schema.Finding) map[string]int { + m := map[string]int{} + for _, x := range f { + m[x.RuleID]++ + } + return m +} + +func TestRunAllStagesAndPreserveStates(t *testing.T) { + pkg := buildPkg(t) + if !Stale(pkg) { + t.Fatal("fresh package must be stale") + } + // auditd log: corroborates rm, nothing for pytest → CONTRADICTED. + audit := filepath.Join(t.TempDir(), "audit.log") + _ = os.WriteFile(audit, []byte( + "type=SYSCALL msg=audit(1788084005.300:101): arch=c000003e syscall=59 success=yes exit=0 ppid=1 pid=4411 uid=1000 comm=\"zsh\" exe=\"/bin/zsh\"\n"+ + "type=EXECVE msg=audit(1788084005.300:101): argc=3 a0=\"/bin/zsh\" a1=\"-c\" a2=\"rm -rf build/\"\n"+ + "type=SYSCALL msg=audit(1788084060.000:102): arch=c000003e syscall=59 success=yes exit=0 ppid=1 pid=9000 uid=0 comm=\"cron\" exe=\"/usr/sbin/cron\"\n"+ + "type=EXECVE msg=audit(1788084060.000:102): argc=1 a0=\"cron\"\n"), 0o600) + res, err := Run(pkg, Options{EndpointLogs: []string{audit}}) + if err != nil { + t.Fatal(err) + } + r := rules(res.Findings) + for _, want := range []string{"ENDPOINT_CONTRADICTED_COMMAND", "DESTRUCTIVE_COMMAND", "UNPINNED_MCP_PACKAGE", "INSTRUCTION_FILE_WRITTEN_BY_AGENT"} { + if r[want] == 0 { + t.Errorf("stage output missing %s (have %v)", want, r) + } + } + for _, f := range []string{"findings.json", "mcp-audit.json", "provenance.json", "corroboration.json", "analysis.json"} { + if _, err := os.Stat(filepath.Join(pkg, "detections", f)); err != nil { + t.Errorf("%s not written", f) + } + } + if res.Correlation == nil || res.Correlation.Contradicted != 1 || res.Correlation.Corroborated != 1 { + t.Fatalf("correlation: %+v", res.Correlation) + } + if Stale(pkg) { + t.Fatal("analyzed package must not be stale") + } + // A second run WITHOUT the endpoint log must reuse the overlay and keep CONTRADICTED/CORROBORATED. + res2, err := Run(pkg, Options{}) + if err != nil { + t.Fatal(err) + } + if res2.Renormalized { + t.Fatal("current overlay must be reused, not re-parsed") + } + states := map[string]int{} + for _, e := range LoadEvents(pkg) { + if e.EventType == schema.EventToolCall && e.Command != "" { + states[e.Corroboration]++ + } + } + if states[schema.StateContradicted] != 1 || states[schema.StateCorroborated] != 1 { + t.Fatalf("states lost on re-analysis: %v", states) + } + if rules(res2.Findings)["ENDPOINT_CONTRADICTED_COMMAND"] != 1 { + t.Fatal("earlier correlation findings must survive a re-analysis without logs") + } + // Detection findings inherit the states. + for _, f := range res2.Findings { + if f.RuleID == "DESTRUCTIVE_COMMAND" && f.Status != schema.StateCorroborated { + t.Fatalf("DESTRUCTIVE_COMMAND should read CORROBORATED, got %s", f.Status) + } + } + // Ensure is a no-op now, and --renormalize resets states. + if r3, _ := Ensure(pkg, nil); r3 != nil { + t.Fatal("Ensure must skip a current analysis") + } + res4, _ := Run(pkg, Options{Renormalize: true}) + if !res4.Renormalized || rules(res4.Findings)["ENDPOINT_CONTRADICTED_COMMAND"] != 0 { + t.Fatalf("renormalize must re-parse and drop stale correlation results: %+v", rules(res4.Findings)) + } + // Findings file is de-duplicated and severity-sorted. + fs := LoadFindings(pkg) + for i := 1; i < len(fs); i++ { + if sevRank[fs[i].Severity] > sevRank[fs[i-1].Severity] { + t.Fatal("findings not sorted by severity") + } + } +} diff --git a/internal/cli/analyst_cmds.go b/internal/cli/analyst_cmds.go index 98f50a6..a092e79 100644 --- a/internal/cli/analyst_cmds.go +++ b/internal/cli/analyst_cmds.go @@ -5,6 +5,7 @@ import ( "encoding/json" "flag" "fmt" + "github.com/efij/AgentDFIR/internal/analysis" "os" "path/filepath" "sort" @@ -28,12 +29,12 @@ func cmdInvestigate(args []string) int { fmt.Fprintln(os.Stderr, "usage: agentdfir investigate ") return 2 } - res, err := normalize.ParsePackage(args[0]) - if err != nil { + if _, err := analysis.Ensure(args[0], os.Stdout); err != nil { fmt.Fprintln(os.Stderr, "error:", err) return 1 } - findings := detect.RunPackage(res, args[0]) + res := &schema.Normalized{Events: analysis.LoadEvents(args[0])} + findings := analysis.LoadFindings(args[0]) fmt.Printf("Loaded %d events, %d entities, %d findings. Type `help`.\n", len(res.Events), len(res.Entities), len(findings)) diff --git a/internal/cli/analyze.go b/internal/cli/analyze.go index 9faa668..ff1943a 100644 --- a/internal/cli/analyze.go +++ b/internal/cli/analyze.go @@ -5,16 +5,13 @@ import ( "encoding/json" "flag" "fmt" + "github.com/efij/AgentDFIR/internal/analysis" "os" "path/filepath" "sort" "strings" - "github.com/efij/AgentDFIR/internal/correlate" - "github.com/efij/AgentDFIR/internal/detect" - "github.com/efij/AgentDFIR/internal/endpoint" "github.com/efij/AgentDFIR/internal/normalize" - "github.com/efij/AgentDFIR/internal/rulepack" "github.com/efij/AgentDFIR/internal/sanitize" "github.com/efij/AgentDFIR/internal/schema" "github.com/efij/AgentDFIR/internal/simulate" @@ -99,133 +96,66 @@ func cmdTimeline(args []string) int { return 0 } -// cmdTriage runs normalize + detections and prints IR-ready findings. -func cmdTriage(args []string) int { - fs := flag.NewFlagSet("triage", flag.ContinueOnError) - triageShellHistory := fs.String("shell-history", "", "correlate against a shell history file (endpoint evidence)") - var endpointLogs multiFlag - fs.Var(&endpointLogs, "endpoint", "OS telemetry log to corroborate against (auditd, Sysmon XML, JSONL/CSV export); repeatable") - rulesDir := fs.String("rules", "", "directory of declarative JSON rule packs") +// cmdAnalyze runs EVERY analysis stage in the right order and prints the +// findings. `triage` is the same command (kept for scripts and habit). +func cmdAnalyze(args []string) int { + fs := flag.NewFlagSet("analyze", flag.ContinueOnError) + var endpointLogs, gwServers multiFlag + fs.Var(&endpointLogs, "endpoint", "OS telemetry log (auditd, Sysmon XML, JSONL/CSV export) to check the transcript against; repeatable") + fs.Var(&gwServers, "gateway-server", "transcript MCP server name routed through the gateway; repeatable") + shellHistory := fs.String("shell-history", "", "shell history file to check commands against") + gwLog := fs.String("gateway-log", "", "MCP gateway log (JSONL) to check MCP calls against") + gwMap := fs.String("gateway-map", "", "field-name map for the gateway log") + rulesDir := fs.String("rules", "", "directory of extra JSON rule packs") honeyFile := fs.String("honeytokens", "", "file of planted canary markers (one per line)") spawnTh := fs.Int("spawn-threshold", 10, "AGENT_SPAWN_EXPLOSION per-session threshold") knownDest := fs.String("known-destinations", "", "comma-separated extra allowlisted network destinations") - // Package may come first or last (`triage pkg --endpoint x` is how people type it). + renorm := fs.Bool("renormalize", false, "re-parse the evidence even if the overlay is current (discards earlier corroboration states)") + asJSON := fs.Bool("json", false, "print findings as JSON") pkg, rest := splitPositional(args) if err := fs.Parse(rest); err != nil || (pkg == "" && fs.NArg() != 1) || (pkg != "" && fs.NArg() != 0) { - fmt.Fprintln(os.Stderr, "usage: agentdfir triage [--endpoint ]... [--shell-history ] [--rules ]") + fmt.Fprintln(os.Stderr, "usage: agentdfir analyze [--endpoint ]... [--gateway-log ] [--rules ] [--honeytokens ]") return 2 } if pkg == "" { pkg = fs.Arg(0) } - // Streaming pipeline (v0.5.1): events are written to the overlay as - // they are parsed and never all held in memory; detection re-reads - // the overlay in bounded passes. Memory scales with sessions/agents, - // not event count. - dir := filepath.Join(pkg, "normalized") - if err := os.MkdirAll(dir, 0o700); err != nil { - fmt.Fprintln(os.Stderr, "error:", err) - return 1 - } - evFile, err := os.OpenFile(filepath.Join(dir, "events.jsonl"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) - if err != nil { - fmt.Fprintln(os.Stderr, "error:", err) - return 1 - } - enc := json.NewEncoder(evFile) - sr, err := normalize.ParseStream(pkg, func(ev schema.Event) error { return enc.Encode(ev) }) - if err != nil { - evFile.Close() - fmt.Fprintln(os.Stderr, "error:", err) - return 1 - } - evFile.Close() - if err := writeJSONL(filepath.Join(dir, "entities.jsonl"), len(sr.Entities), func(i int) any { return sr.Entities[i] }); err != nil { - fmt.Fprintln(os.Stderr, "error:", err) - return 1 - } - if err := writeJSONL(filepath.Join(dir, "relationships.jsonl"), len(sr.Relationships), func(i int) any { return sr.Relationships[i] }); err != nil { - fmt.Fprintln(os.Stderr, "error:", err) - return 1 + opts := analysis.Options{EndpointLogs: endpointLogs, ShellHistory: *shellHistory, GatewayLog: *gwLog, GatewayMap: *gwMap, + GatewayServers: gwServers, RulesDir: *rulesDir, SpawnThreshold: *spawnTh, Renormalize: *renorm, Log: os.Stdout} + if *honeyFile != "" { + data, err := os.ReadFile(*honeyFile) + if err != nil { + fmt.Fprintln(os.Stderr, "honeytokens:", err) + return 1 + } + opts.Honeytokens = strings.Split(string(data), "\n") } - fmt.Printf("Normalized: %d events, %d entities, %d relationships -> %s\n", - sr.EventCount, len(sr.Entities), len(sr.Relationships), dir) - var extraDest []string if *knownDest != "" { - extraDest = strings.Split(*knownDest, ",") + opts.KnownDests = strings.Split(*knownDest, ",") } - // Second-witness correlation runs BEFORE detection so findings carry - // the upgraded/downgraded corroboration states; results are persisted - // to the overlay (shell-history states used to be lost here). - var corrFindings []schema.Finding - if *triageShellHistory != "" || len(endpointLogs) > 0 { - evs := loadEvents(dir) - if *triageShellHistory != "" { - cres, cerr := correlate.Apply(evs, &correlate.ShellHistoryAdapter{Path: *triageShellHistory}) - if cerr == nil && cres.Corroborated > 0 { - fmt.Printf("Endpoint correlation: %d tool call(s) corroborated by shell history.\n", cres.Corroborated) - } - if len(endpointLogs) == 0 { - if err := writeJSONL(filepath.Join(dir, "events.jsonl"), len(evs), func(i int) any { return evs[i] }); err != nil { - fmt.Fprintln(os.Stderr, "error:", err) - return 1 - } - } - } - if len(endpointLogs) > 0 { - f, _, rc := runEndpointCorrelation(pkg, endpointLogs, endpoint.FormatAuto, correlate.EndpointOptions{KnownDests: extraDest}, evs) - if rc != 0 { - return rc - } - corrFindings = f - } + if *asJSON { + opts.Log = nil } - var honey []string - if *honeyFile != "" { - if data, herr := os.ReadFile(*honeyFile); herr == nil { - honey = strings.Split(string(data), "\n") - } else { - fmt.Fprintln(os.Stderr, "honeytokens:", herr) - } - } - findings, err := detect.RunStream(pkg, sr.Entities, detect.Options{ - Honeytokens: honey, SpawnThreshold: *spawnTh, KnownDestinations: extraDest, - }) + res, err := analysis.Run(pkg, opts) if err != nil { fmt.Fprintln(os.Stderr, "error:", err) return 1 } - if *rulesDir != "" { - packs, perr := rulepack.LoadDir(*rulesDir) - if perr != nil { - fmt.Fprintln(os.Stderr, "rule packs:", perr) - return 1 - } - extra, perr := rulepack.Apply(packs, &schema.Normalized{Events: loadEvents(dir)}, pkg) - if perr != nil { - fmt.Fprintln(os.Stderr, "rule packs:", perr) - return 1 - } - if len(extra) > 0 { - fmt.Printf("Rule packs: %d pack(s) contributed %d finding(s).\n", len(packs), len(extra)) - findings = append(findings, extra...) - } - } - - if len(corrFindings) > 0 { - findings = append(findings, corrFindings...) + for _, n := range res.StageNotes { + fmt.Fprintln(os.Stderr, "note:", n) } - detDir := filepath.Join(pkg, "detections") - if err := os.MkdirAll(detDir, 0o700); err != nil { - fmt.Fprintln(os.Stderr, "error:", err) - return 1 - } - data, _ := json.MarshalIndent(findings, "", " ") - if err := os.WriteFile(filepath.Join(detDir, "findings.json"), append(data, '\n'), 0o600); err != nil { - fmt.Fprintln(os.Stderr, "error:", err) - return 1 + if *asJSON { + data, _ := json.MarshalIndent(res.Findings, "", " ") + fmt.Println(string(data)) + return exitFor(res.Findings) } + printTriageFindings(res.Findings) + fmt.Printf("\nResults: %s (open them: agentdfir serve %s)\n", filepath.Join(pkg, "detections"), pkg) + return exitFor(res.Findings) +} +// printTriageFindings renders findings the way analysts read them. +func printTriageFindings(findings []schema.Finding) { fmt.Printf("\n%d finding(s):\n", len(findings)) for _, f := range findings { fmt.Printf("\n%s — %s [%s]\n", f.Severity, sanitize.Terminal(f.Title), f.RuleID) @@ -247,14 +177,10 @@ func cmdTriage(args []string) int { fmt.Printf(" Evidence: %s\n", sanitize.Terminal(e)) } fmt.Printf(" Status: %s Endpoint corroboration: %s\n", f.Status, f.Endpoint) - atlas := f.MitreATLAS - if atlas == "" { - atlas = "not mapped (no valid technique)" + if f.MitreATTACK != "" || f.MitreATLAS != "" { + fmt.Printf(" MITRE: %s %s\n", f.MitreATTACK, f.MitreATLAS) } - fmt.Printf(" MITRE ATLAS: %s\n", atlas) } - fmt.Printf("\nfindings written to %s\n", filepath.Join(detDir, "findings.json")) - return 0 } // cmdSimulate generates a synthetic incident profile. diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 39cfb75..a188859 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -20,53 +20,54 @@ import ( "github.com/efij/AgentDFIR/internal/version" ) -const usage = `agentdfir — open-source DFIR for AI agents +const usage = `agentdfir — open-source DFIR for AI agents. Evidence in, verdicts out. Nothing leaves your machine. -Usage: - agentdfir detect discover installed AI tooling - agentdfir collect [flags] forensic acquisition into an .adfir package - agentdfir verify verify a sealed evidence package - agentdfir normalize parse raw evidence into normalized events - agentdfir timeline print the unified, evidence-linked timeline - agentdfir triage normalize + run detections, print findings - agentdfir correlate ... second witness: auditd / Sysmon XML / JSONL-CSV exports - agentdfir provenance [file] who wrote each line of CLAUDE.md / rules / settings, and why - agentdfir simulate [flags] generate a synthetic incident scenario - agentdfir diff configuration drift between two packages - agentdfir baseline create|check org known-good profiles - agentdfir report HTML/PDF/JSON/CSV/STIX/OTel/OCSF/SARIF/Timesketch/l2tcsv - agentdfir export --support derived, redacted support package - agentdfir keygen generate ed25519 signing keypair - agentdfir sign --key sign a sealed package (SEAL.sig) - agentdfir inspect [--reveal-sensitive] artifact inventory + secret scan - agentdfir encrypt encrypt a package (AGENTDFIR_PASSPHRASE) - agentdfir decrypt decrypt an encrypted package - agentdfir investigate interactive analyst explorer - agentdfir serve [--port N] local case explorer in the browser (127.0.0.1 only, read-only) - agentdfir replay step through a session, states inline - agentdfir monitor [dirs...] [--detect --alert ] live tail; with --detect a real-time sensor - agentdfir explain deterministic case digest (no AI, no transmission) - agentdfir update-packs install signed knowledge-pack overrides - agentdfir rules validate|export validate rule packs · export to Sigma YAML - agentdfir packs list|validate|add|remove|init declarative product packs (new agents, no Go) - agentdfir mcp audit [|--profile ] MCP server inventory + supply-chain findings (read-only) - agentdfir version print version +THE 4-STEP WORKFLOW + 1. See what's installed agentdfir detect + 2. Collect evidence agentdfir collect --product claude (sealed .adfir package) + 3. Analyze everything agentdfir analyze (detections, MCP audit, provenance…) + 4. Look at the results agentdfir serve (browser, 127.0.0.1 only) -Collect flags: - --product product to collect (default: claude) - --out output package directory (default: ./.adfir) - --case-id case identifier (default: generated) - --operator asserted operator name for chain of custody - --path offline profile root (mounted image / copied home) - --import KAPE / Velociraptor / CyLR output tree or image: discover every - user profile, collect ALL products for ALL users into one package - --docker container (docker export, read-only; AGENTDFIR_DOCKER=podman) or saved export - --archive zip / tar / tar.gz: CI artifact, support bundle, vendor data export - --authorization authorization reference (ticket / legal basis) - --max-file-mb per-artifact size bound in MiB (default 512) +COLLECT — where the evidence is + agentdfir collect --product [--live] this machine, this user + agentdfir collect --product claude --path /mnt/image/Users/x a copied home / mounted image + agentdfir collect --import every user, every agent, one package + agentdfir collect --docker a container (read-only export) + agentdfir collect --archive CI artifact, support bundle, vendor export + agentdfir verify prove the package was not modified -Detection never executes discovered binaries. Collection is always -lossless; nothing is redacted at acquisition time. +ANALYZE — one command runs every stage, in order + agentdfir analyze detections + MCP audit + provenance + --endpoint add OS telemetry: which tool calls the OS confirms or contradicts + --gateway-log add your MCP gateway log: which MCP calls it confirms + --rules --honeytokens extra rule packs, planted canaries + (triage = same as analyze. Single stages, for scripts: correlate · mcp audit · provenance · normalize) + +LOOK — same results, different views + agentdfir serve [--open] browser: agent tree, timeline, raw evidence, findings + agentdfir timeline the unified timeline in your terminal + agentdfir investigate interactive terminal explorer + agentdfir replay step through one session + agentdfir explain plain-language case digest (no AI, nothing sent anywhere) + +EXPORT — hand results to other tools + agentdfir report --format pdf|html|json|csv|stix|otel|ocsf|sarif|timesketch|l2tcsv|all + agentdfir export --support redacted package for vendor support + agentdfir rules export --sigma detection rules as Sigma for your SIEM + +BEFORE AN INCIDENT + agentdfir monitor --detect --alert live sensor: findings pushed as they happen + agentdfir mcp audit MCP servers on this machine: unpinned, plaintext, poisoned + agentdfir baseline create|check · agentdfir diff known-good configs and drift + agentdfir simulate --scenario orphan-agent synthetic incident to train and test + +TRUST & KEYS + agentdfir keygen · sign --key · encrypt · decrypt · inspect + agentdfir packs list|add|validate|init add a new AI agent product with one signed JSON file + agentdfir rules validate · update-packs · version + +Collect flags: --out --case-id --operator --authorization --max-file-mb --sign +Nothing is executed on the suspect host, no agent is touched, nothing is transmitted. Help for one command: agentdfir --help ` // Main dispatches and returns the process exit code. @@ -97,8 +98,8 @@ func Main(args []string) int { return cmdNormalize(args[1:]) case "timeline": return cmdTimeline(args[1:]) - case "triage": - return cmdTriage(args[1:]) + case "analyze", "triage": + return cmdAnalyze(args[1:]) case "simulate": return cmdSimulate(args[1:]) case "diff": diff --git a/internal/cli/report_cmds.go b/internal/cli/report_cmds.go index 9dd124d..169740a 100644 --- a/internal/cli/report_cmds.go +++ b/internal/cli/report_cmds.go @@ -3,15 +3,15 @@ package cli import ( "flag" "fmt" + "github.com/efij/AgentDFIR/internal/analysis" + "github.com/efij/AgentDFIR/internal/schema" "os" "path/filepath" "strings" "github.com/efij/AgentDFIR/internal/casepkg" - "github.com/efij/AgentDFIR/internal/detect" "github.com/efij/AgentDFIR/internal/encrypt" "github.com/efij/AgentDFIR/internal/export" - "github.com/efij/AgentDFIR/internal/normalize" "github.com/efij/AgentDFIR/internal/report" "github.com/efij/AgentDFIR/internal/sanitize" "github.com/efij/AgentDFIR/internal/seal" @@ -47,12 +47,13 @@ func cmdReport(args []string) int { } ci, _ := report.ReadCaseInfo(pkg) vres, _ := casepkg.Verify(pkg) - res, err := normalize.ParsePackage(pkg) - if err != nil { + // Reports render the SAME analysis run the analyst saw (states included). + if _, err := analysis.Ensure(pkg, os.Stdout); err != nil { fmt.Fprintln(os.Stderr, "error:", err) return 1 } - findings := detect.RunPackage(res, pkg) + res := &schema.Normalized{Events: analysis.LoadEvents(pkg), Entities: analysis.LoadEntities(pkg)} + findings := analysis.LoadFindings(pkg) c := &report.Case{ Manifest: man, CaseInfo: ci, Verify: vres, diff --git a/internal/serve/serve.go b/internal/serve/serve.go index 07e6a73..3797f98 100644 --- a/internal/serve/serve.go +++ b/internal/serve/serve.go @@ -26,9 +26,8 @@ import ( "sync" "time" + "github.com/efij/AgentDFIR/internal/analysis" "github.com/efij/AgentDFIR/internal/casepkg" - "github.com/efij/AgentDFIR/internal/detect" - "github.com/efij/AgentDFIR/internal/normalize" "github.com/efij/AgentDFIR/internal/report" "github.com/efij/AgentDFIR/internal/sanitize" "github.com/efij/AgentDFIR/internal/schema" @@ -83,21 +82,13 @@ func Load(pkg string, opts Options) (*Server, error) { s.sig = "INVALID — " + sr.Reason } - evPath := filepath.Join(pkg, "normalized", "events.jsonl") - if _, err := os.Stat(evPath); err != nil { - // Normalize into the overlay so later commands share the same data. - res, err := normalize.ParsePackage(pkg) - if err != nil { - return nil, err - } - if err := writeOverlay(pkg, res); err != nil { - return nil, err - } - s.entities, s.rels = res.Entities, res.Relationships - } else { - s.entities = readJSONL[schema.Entity](filepath.Join(pkg, "normalized", "entities.jsonl")) - s.rels = readJSONL[schema.Relationship](filepath.Join(pkg, "normalized", "relationships.jsonl")) + // Analysis is never "not run yet": compute it when missing or stale. + if _, err := analysis.Ensure(pkg, os.Stdout); err != nil { + return nil, err } + evPath := filepath.Join(pkg, "normalized", "events.jsonl") + s.entities = readJSONL[schema.Entity](filepath.Join(pkg, "normalized", "entities.jsonl")) + s.rels = readJSONL[schema.Relationship](filepath.Join(pkg, "normalized", "relationships.jsonl")) f, err := os.Open(evPath) if err != nil { return nil, err @@ -116,14 +107,7 @@ func Load(pkg string, opts Options) (*Server, error) { s.events = append(s.events, ev) } } - // Findings: reuse triage output when present, else compute. - fPath := filepath.Join(pkg, "detections", "findings.json") - if data, err := os.ReadFile(fPath); err == nil { - _ = json.Unmarshal(data, &s.findings) - } - if len(s.findings) == 0 { - s.findings = detect.RunPackage(&schema.Normalized{Events: s.events, Entities: s.entities, Relationships: s.rels}, pkg) - } + s.findings = analysis.LoadFindings(pkg) return s, nil } @@ -613,34 +597,6 @@ func readJSONL[T any](path string) []T { return out } -func writeOverlay(pkg string, res *schema.Normalized) error { - dir := filepath.Join(pkg, "normalized") - if err := os.MkdirAll(dir, 0o700); err != nil { - return err - } - write := func(name string, n int, get func(int) any) error { - f, err := os.OpenFile(filepath.Join(dir, name), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) - if err != nil { - return err - } - enc := json.NewEncoder(f) - for i := 0; i < n; i++ { - if err := enc.Encode(get(i)); err != nil { - f.Close() - return err - } - } - return f.Close() - } - if err := write("events.jsonl", len(res.Events), func(i int) any { return res.Events[i] }); err != nil { - return err - } - if err := write("entities.jsonl", len(res.Entities), func(i int) any { return res.Entities[i] }); err != nil { - return err - } - return write("relationships.jsonl", len(res.Relationships), func(i int) any { return res.Relationships[i] }) -} - // Serve runs the HTTP server on ln until it stops; idle timeouts keep a // forgotten instance from holding sockets. func (s *Server) Serve(ln net.Listener) error { diff --git a/internal/serve/serve_test.go b/internal/serve/serve_test.go index 77f0aa3..ec6dcc6 100644 --- a/internal/serve/serve_test.go +++ b/internal/serve/serve_test.go @@ -182,10 +182,15 @@ func TestServeAPI(t *testing.T) { if len(bk) < 3 { t.Fatalf("buckets: %v", bk) } - // extras empty until analyses run + // analysis ran on load: MCP audit and provenance results exist (empty for this fixture). _, body = get(t, srv, "/api/extras", "") - if strings.TrimSpace(string(body)) != "{}" { - t.Fatalf("extras: %s", body) + var extras map[string]any + _ = json.Unmarshal(body, &extras) + if _, ok := extras["mcp"]; !ok { + t.Fatalf("extras missing mcp audit: %s", body) + } + if _, ok := extras["provenance"]; !ok { + t.Fatalf("extras missing provenance: %s", body) } // Loopback listener only. ln, url, err := s.Listen(0) diff --git a/internal/serve/ui.html b/internal/serve/ui.html index 97e5e47..eb8bde9 100644 --- a/internal/serve/ui.html +++ b/internal/serve/ui.html @@ -309,7 +309,10 @@

Sessions & agents

// ---- extras api('/api/extras').then(x=>{ const box=$('#extrasbox'); box.innerHTML=''; - if(!Object.keys(x).length){box.appendChild(el('div','empty','No MCP audit, endpoint corroboration or provenance results in this package yet. Run `agentdfir mcp audit`, `agentdfir correlate` or `agentdfir provenance` on it and reload.')); return;} + // Drop sections that ran but found nothing, so only real content shows. + if(x.mcp&&!((x.mcp.inventory||{}).servers||[]).length&&!(x.mcp.findings||[]).length) delete x.mcp; + if(x.provenance&&!(x.provenance.files||[]).length&&!(x.provenance.writes_to_instruction_paths||[]).length) delete x.provenance; + if(!Object.keys(x).length){box.appendChild(el('div','empty','Nothing to show here: this package has no MCP server configuration, no instruction/memory files written by an agent, and no OS telemetry was supplied. To add the second witness, re-run: agentdfir analyze --endpoint ')); return;} if(x.mcp){const c=card('MCP supply-chain audit'); const inv=x.mcp.inventory||{}; c.appendChild(el('p','',(inv.servers||[]).length+' server(s) across '+(inv.configs_seen||[]).length+' config file(s)')); const t=table(['host','server','scope','transport','pinned','identity']); (inv.servers||[]).forEach(s=>row(t,[s.host,s.name,s.scope,s.transport,s.command?(s.pinned?'yes':'NO'):'-',s.package?s.package_manager+' '+s.package:(s.url||s.command||'')])); c.appendChild(t); findingsTable(c,x.mcp.findings||[]); if(x.mcp.gateway){const g=x.mcp.gateway; c.appendChild(el('p','','Gateway: '+g.records+' records · '+g.matched_with_transcript+' matched · '+g.gateway_only+' gateway-only · '+g.transcript_only+' transcript-only · '+g.denied+' denied · '+g.errors+' errors · p95 '+g.p95_latency_ms+' ms'));} box.appendChild(c);} diff --git a/internal/version/version.go b/internal/version/version.go index 8ab9476..55779a8 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -3,7 +3,7 @@ package version // Version is the collector version. Overridable at build time via // -ldflags "-X github.com/efij/AgentDFIR/internal/version.Version=vX.Y.Z". -var Version = "0.12.1" +var Version = "0.13.0-dev" // ADFIRVersion is the evidence package format version this binary writes. const ADFIRVersion = "0.1"