From 9b0fbe77e903d752534c69880c4015647b6d1f39 Mon Sep 17 00:00:00 2001 From: Aliya Alias Date: Sat, 3 Oct 2026 11:17:59 +0800 Subject: [PATCH 01/21] fix(web): stop the server on the first Ctrl+C while a page is open The live event stream only ended when the browser disconnected, so uvicorn waited forever for it during shutdown and a second Ctrl+C dumped a traceback. A shutdown flag set from the server's exit handler now ends open streams, with a 3-second graceful-shutdown limit as a backstop. --- glaive/cli.py | 12 +++++++++++- glaive/web/app.py | 7 ++++++- tests/v02/test_cli.py | 3 ++- tests/v02/test_web.py | 12 ++++++++++++ 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/glaive/cli.py b/glaive/cli.py index 173a956..b222d4c 100644 --- a/glaive/cli.py +++ b/glaive/cli.py @@ -232,7 +232,17 @@ def _serve(case: Path, host: str = "127.0.0.1", port: int = 8765, console.print(f"GLAIVE web app for [bold]{session.case_name}[/]: {url}") if not no_browser: webbrowser.open(url) - uvicorn.run(create_app(session, token=token), host=host, port=port, log_level="warning") + from glaive.web.app import shutting_down + + class _Server(uvicorn.Server): + def handle_exit(self, sig: int, frame: object) -> None: # noqa: D102 + shutting_down.set() + super().handle_exit(sig, frame) + + shutting_down.clear() + config = uvicorn.Config(create_app(session, token=token), host=host, port=port, + log_level="warning", timeout_graceful_shutdown=3) + _Server(config).run() @app.command() diff --git a/glaive/web/app.py b/glaive/web/app.py index d1e0688..f6efc82 100644 --- a/glaive/web/app.py +++ b/glaive/web/app.py @@ -55,6 +55,11 @@ def _hostname(netloc: str) -> str: return netloc.rsplit(":", 1)[0] +# Set when the server starts shutting down, so open event streams (browser tabs) +# end at once instead of keeping Ctrl+C waiting. +shutting_down = threading.Event() + + class LocalOnlyMiddleware: """Protection for the token-less local mode (plain ASGI, so streaming works). @@ -306,7 +311,7 @@ async def stream() -> Any: try: for e in backlog[-400:]: yield f"data: {json.dumps(e, default=str)}\n\n" - while not await request.is_disconnected(): + while not shutting_down.is_set() and not await request.is_disconnected(): try: e = q.get_nowait() yield f"data: {json.dumps(e, default=str)}\n\n" diff --git a/tests/v02/test_cli.py b/tests/v02/test_cli.py index 477235d..adc99e8 100644 --- a/tests/v02/test_cli.py +++ b/tests/v02/test_cli.py @@ -51,7 +51,8 @@ def test_demo_serve_starts_local_server(tmp_path: Path, monkeypatch) -> None: # from glaive import cli started: dict = {} - monkeypatch.setattr(uvicorn, "run", lambda app, **kw: started.update(kw)) + monkeypatch.setattr(uvicorn.Server, "run", lambda self, *a, **kw: started.update( + host=self.config.host, port=self.config.port)) monkeypatch.setattr(cli.webbrowser, "open", lambda url: started.update(url=url)) for key in [k for k in os.environ if k.endswith("_API_KEY")]: monkeypatch.delenv(key) diff --git a/tests/v02/test_web.py b/tests/v02/test_web.py index 4f19a25..7eaaffa 100644 --- a/tests/v02/test_web.py +++ b/tests/v02/test_web.py @@ -120,3 +120,15 @@ def test_page_makes_no_third_party_requests() -> None: load fonts, scripts or styles from anywhere else.""" page = (STATIC / "index.html").read_text(encoding="utf-8") assert "http://" not in page and "https://" not in page + + +def test_event_stream_ends_when_server_shuts_down(client: TestClient) -> None: + # An open browser tab must not keep Ctrl+C waiting. + from glaive.web import app as web + + web.shutting_down.set() + try: + r = client.get("/api/events") + assert r.status_code == 200 and r.text.startswith("data: ") + finally: + web.shutting_down.clear() From 146cff1080a8e4827a83343654983b9e9f121979 Mon Sep 17 00:00:00 2001 From: Aliya Alias Date: Sat, 3 Oct 2026 11:18:20 +0800 Subject: [PATCH 02/21] docs: add roadmap for v0.3 to v1.0 --- ROADMAP.md | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 ROADMAP.md diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..9d823ea --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,64 @@ +# Roadmap + +Where GLAIVE is going and why. Each version has one theme and a "done when" +test that can be checked, not a feature wish-list. + +| Version | Theme | Done when... | +|---|---|---| +| v0.2 Core (released) | Make it a real product | You can drop in a Windows triage folder and get a verified report in the browser, with or without an AI model | +| v0.3 AI depth | Make the AI trustworthy, and prove it | Accuracy is published on public data it was not built on, AI is compared with rules alone, "Ask the case" answers with citations, log poisoning is blocked, and personal data never leaves the machine unmasked | +| v0.4 Scale and integrations | Fit into real teams | GB-scale cases; memory, registry, KAPE and Linux evidence; SIEM, threat intel and chat-ops (Feishu, Slack); M365 and cloud audit logs; Azure OpenAI and Bedrock; a shared team server with Docker Compose | +| v0.5 Frontier | Things nobody else has | GLAIVE-mini trained with the gate as its reward, forensics of hijacked AI agents, multimodal evidence, training / CTF mode | +| v1.0 Stable | Ready for outside users | Stable case file and plugin API, docs, signed releases, an independent security review, a public leaderboard | + +## v0.3 AI depth + +Measurement comes first, so every later change can show whether it helped. + +- **Benchmark on public data.** EVTX-ATTACK-SAMPLES (labelled by ATT&CK + tactic), OTRF Security-Datasets (labelled by technique) and a benign + baseline (NextronSystems evtx-baseline) for false positives. +- **Rules alone vs rules plus AI**, per model provider, with cost and time. +- **Confidence calibration**: how often findings at each confidence level match + the answer key. +- **Privacy**: case data is pseudonymised before it is sent to a cloud model, + and a local-only mode refuses cloud models entirely. +- **Audit trail**: every model and tool call is traced (OpenTelemetry + conventions, local file by default). +- **Evidence search (GraphRAG)**: hybrid keyword and vector search over the + evidence graph, with recall@k measured. +- **Past-case memory**: findings from earlier cases, searchable on request. +- **Ask the case**: answers cite the evidence; uncited or ungrounded sentences + are removed. +- **Log poisoning**: a measured attack set, not only unit tests. + +## v0.4 Scale and integrations + +DuckDB for large cases, parallel per-host investigation, Volatility in the +pipeline, registry hives, KAPE / EZ-tools output, Linux logs, M365 / Entra +and CloudTrail, SIEM and threat-intel connectors, Feishu and Slack, +Azure OpenAI and Bedrock, provider prompt caching, Docker Compose. + +## v0.5 Frontier + +- **GLAIVE-mini**: a small open model trained with reinforcement learning + where the verification gate is the reward (grounded claims, real citations, + no inflated confidence). It ships only if it beats its base model on + held-out public data. +- **AI-agent forensics**: investigate hijacked AI agents from MCP tool logs + and agent transcripts. +- **Multimodal evidence**: screenshots, phishing emails and documents. +- **Training mode**: guided cases and CTF-style challenges. + +## Deliberately not planned + +These come up often. They do not make GLAIVE better at its job: + +- Fine-tuning on synthetic data (results would not be honest; see GLAIVE-mini + for the version that would be). +- Agent frameworks such as LangGraph or CrewAI: the hand-written agents are + easier to audit, which matters in forensics. +- Message queues (Kafka, Celery) and Kubernetes: GLAIVE is local-first. +- A semantic response cache: every case is different, and a shared cache could + leak data between cases. +- Server vector databases: the search index lives in the case folder. From f19d1ae50ede1cf65cc7497a14da92d6138a54b2 Mon Sep 17 00:00:00 2001 From: Aliya Alias Date: Sat, 3 Oct 2026 11:19:32 +0800 Subject: [PATCH 03/21] feat(ingest): read NXLog and Winlogbeat JSON exports OTRF Security-Datasets ship NXLog/Logstash records with every field at the top level and the real host in 'Hostname' ('host' is the log collector). Elastic Winlogbeat nests fields under 'winlog'. Both shapes now normalize to GLAIVE events, with Sysmon UtcTime preferred over the shipper's receive time. --- glaive/ingestion/jsonl.py | 73 +++++++++++++++++++++++++++++++++++++++ tests/v02/test_jsonl.py | 38 ++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/glaive/ingestion/jsonl.py b/glaive/ingestion/jsonl.py index 0f3f6d2..c73d52e 100644 --- a/glaive/ingestion/jsonl.py +++ b/glaive/ingestion/jsonl.py @@ -9,6 +9,12 @@ or `evtx_dump -o jsonl` (Event.System / Event.EventData nesting, or flat EventID/TimeCreated/Computer/Channel keys). This makes the synthetic demo case, test fixtures and exports from other tools all ingestible. + +Also log-shipper exports: + - NXLog / Logstash (used by OTRF Security-Datasets): every field at the top + level, the host in "Hostname" (not "host", which is the collector). + - Winlogbeat / Elastic: {"winlog": {"event_id", "computer_name", + "channel", "event_data": {...}}, "@timestamp": ...}. """ from __future__ import annotations @@ -24,6 +30,68 @@ _normalize_time_string, ) +# NXLog / Logstash bookkeeping fields: everything else in a flat NXLog record +# is event data. +_SHIPPER_FIELDS = frozenset({ + "EventID", "EventTime", "EventReceivedTime", "EventType", "SourceModuleName", + "SourceModuleType", "SourceName", "ProviderGuid", "Hostname", "host", "port", "tags", + "Channel", "Keywords", "SeverityValue", "Severity", "Opcode", "OpcodeValue", "RecordNumber", + "ExecutionProcessID", "ThreadID", "Task", "Version", "Category", "@version", "@timestamp", + "ERROR_EVT_UNRESOLVED", "AccountType", "AccountName", "Domain", "UserID", +}) + + +def _from_nxlog(obj: dict[str, Any]) -> dict | None: + """Flat NXLog / Logstash record (OTRF Security-Datasets).""" + computer = obj.get("Hostname") + # Sysmon's UtcTime is the event time; @timestamp is when Logstash received it. + ts = obj.get("UtcTime") or obj.get("@timestamp") or obj.get("EventTime") + eid = obj.get("EventID") + if eid is None or not ts or not computer: + return None + raw = {k: "" if v is None else (v if isinstance(v, str) else json.dumps(v) + if isinstance(v, (dict, list)) else str(v)) + for k, v in obj.items() if k not in _SHIPPER_FIELDS} + raw = _canon_all(raw) + return { + "event_id": int(eid), + "time_created": _normalize_time_string(str(ts)), + "computer": str(computer), + "channel": obj.get("Channel"), + "provider": obj.get("SourceName"), + "threat_name": raw.get("Threat Name"), + "action": raw.get("Action Name"), + "file_path": _clean_defender_path(raw.get("Path")), + "raw_data": raw, + "_record_id": obj.get("RecordNumber"), + "_process_id": obj.get("ExecutionProcessID"), + } + + +def _from_winlogbeat(obj: dict[str, Any]) -> dict | None: + wl = obj["winlog"] + eid = wl.get("event_id") + ts = obj.get("@timestamp") + computer = wl.get("computer_name") + if eid is None or not ts or not computer: + return None + data = wl.get("event_data") or {} + raw = _canon_all({k: "" if v is None else str(v) for k, v in data.items()}) + proc = wl.get("process") or {} + return { + "event_id": int(eid), + "time_created": _normalize_time_string(str(ts)), + "computer": str(computer), + "channel": wl.get("channel"), + "provider": wl.get("provider_name"), + "threat_name": raw.get("Threat Name"), + "action": raw.get("Action Name"), + "file_path": _clean_defender_path(raw.get("Path")), + "raw_data": raw, + "_record_id": wl.get("record_id"), + "_process_id": proc.get("pid") if isinstance(proc, dict) else None, + } + def _from_flat(obj: dict[str, Any]) -> dict | None: eid = obj.get("event_id", obj.get("EventID", obj.get("EventId"))) @@ -60,6 +128,11 @@ def normalize_json_event(obj: Any) -> dict | None: ev = obj["Event"] rec = (ev.get("System") or {}).get("EventRecordID") return _json_event_to_dict(ev, rec) + if isinstance(obj.get("winlog"), dict): + return _from_winlogbeat(obj) + if "Hostname" in obj and "EventID" in obj and not any( + k in obj for k in ("EventData", "raw_data", "Payload")): + return _from_nxlog(obj) return _from_flat(obj) diff --git a/tests/v02/test_jsonl.py b/tests/v02/test_jsonl.py index ac0be46..885f97e 100644 --- a/tests/v02/test_jsonl.py +++ b/tests/v02/test_jsonl.py @@ -44,3 +44,41 @@ def test_bad_record_in_json_array_is_skipped(tmp_path: Path) -> None: stats: dict[str, int] = {} assert [e["event_id"] for e in iter_json_events(p, stats)] == [1] assert stats["records_skipped_malformed"] == 1 + + +def test_nxlog_flat_record_as_in_otrf_datasets() -> None: + # Shape of OTRF Security-Datasets (NXLog -> Logstash): fields at the top + # level, the real host in "Hostname", "host" is the log collector. + ev = normalize_json_event({ + "EventID": 1, "Channel": "Microsoft-Windows-Sysmon/Operational", + "SourceName": "Microsoft-Windows-Sysmon", "host": "wec.internal.cloudapp.net", + "Hostname": "WORKSTATION5.theshire.local", "EventTime": "2020-09-21 18:58:57", + "@timestamp": "2020-09-21T22:58:59.043Z", "UtcTime": "2020-09-21 22:58:57.121", + "RecordNumber": 4242, "ExecutionProcessID": 3172, "port": 64545, + "Image": "C:\\windows\\system32\\whoami.exe", "CommandLine": "whoami /user", + "ParentImage": "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", + "ProcessId": "8840", "tags": ["mordorDataset"]}) + assert ev is not None + assert ev["computer"] == "WORKSTATION5.theshire.local" + assert ev["time_created"] == "2020-09-21T22:58:57.121000+00:00" # UtcTime, not EventTime + assert ev["raw_data"]["CommandLine"] == "whoami /user" + assert "port" not in ev["raw_data"] and "Hostname" not in ev["raw_data"] + assert ev["_record_id"] == 4242 and ev["provider"] == "Microsoft-Windows-Sysmon" + + +def test_nxlog_record_without_utctime_uses_timestamp() -> None: + ev = normalize_json_event({"EventID": 4624, "Channel": "Security", + "Hostname": "DC1", "@timestamp": "2020-09-21T22:59:14.904Z", + "TargetUserName": "pgustavo", "LogonType": 3}) + assert ev is not None and ev["time_created"].startswith("2020-09-21T22:59:14") + assert ev["raw_data"]["LogonType"] == "3" + + +def test_winlogbeat_record() -> None: + ev = normalize_json_event({ + "@timestamp": "2026-03-01T10:00:00.000Z", + "winlog": {"event_id": 4688, "computer_name": "WS01", "channel": "Security", + "provider_name": "Microsoft-Windows-Security-Auditing", "record_id": 77, + "event_data": {"NewProcessName": "C:\\Windows\\System32\\cmd.exe"}}}) + assert ev is not None and ev["computer"] == "WS01" and ev["event_id"] == 4688 + assert ev["raw_data"]["NewProcessName"].endswith("cmd.exe") and ev["_record_id"] == 77 From c0f6ee0c93abf76712d60854ec528d21c77350d5 Mon Sep 17 00:00:00 2001 From: Aliya Alias Date: Sat, 3 Oct 2026 11:21:39 +0800 Subject: [PATCH 04/21] feat(detection): tag alerts with ATT&CK tactics Adds a compact technique-to-tactic table generated from the official Enterprise ATT&CK v19.2 STIX bundle. Alerts now record tactics from Sigma tags and their techniques. v19 renamed Defense Evasion to Stealth and revoked T1562.x in favour of T1685; old names and revoked IDs still resolve, because rules and public datasets use them. --- .github/workflows/ci.yml | 2 +- glaive/detection/attack.json | 1 + glaive/detection/attack.py | 122 +++++++++++++++++++++++++++++++++++ glaive/detection/sigma.py | 5 ++ glaive/graph/nodes.py | 1 + glaive/ingestion/pipeline.py | 7 +- pyproject.toml | 2 +- tests/v03/__init__.py | 0 tests/v03/conftest.py | 2 + tests/v03/test_attack.py | 64 ++++++++++++++++++ 10 files changed, 202 insertions(+), 4 deletions(-) create mode 100644 glaive/detection/attack.json create mode 100644 glaive/detection/attack.py create mode 100644 tests/v03/__init__.py create mode 100644 tests/v03/conftest.py create mode 100644 tests/v03/test_attack.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b128e79..37ccfd3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: python -m pip install --upgrade pip python -m pip install -e ".[dev]" "${{ matrix.mcp }}" - name: Lint - run: ruff check glaive tests/v02 + run: ruff check glaive tests/v02 tests/v03 - name: Tests run: python -m pytest -q - name: Bypass (adversarial) tests diff --git a/glaive/detection/attack.json b/glaive/detection/attack.json new file mode 100644 index 0000000..8af925c --- /dev/null +++ b/glaive/detection/attack.json @@ -0,0 +1 @@ +{"source":"MITRE ATT&CK Enterprise","version":"19.2","notice":"(c) The MITRE Corporation. Reproduced and distributed with the permission of The MITRE Corporation (https://attack.mitre.org/resources/legal-and-branding/terms-of-use/).","tactics":{"TA0001":{"name":"Initial Access","shortname":"initial-access"},"TA0002":{"name":"Execution","shortname":"execution"},"TA0003":{"name":"Persistence","shortname":"persistence"},"TA0004":{"name":"Privilege Escalation","shortname":"privilege-escalation"},"TA0005":{"name":"Stealth","shortname":"stealth"},"TA0006":{"name":"Credential Access","shortname":"credential-access"},"TA0007":{"name":"Discovery","shortname":"discovery"},"TA0008":{"name":"Lateral Movement","shortname":"lateral-movement"},"TA0009":{"name":"Collection","shortname":"collection"},"TA0010":{"name":"Exfiltration","shortname":"exfiltration"},"TA0011":{"name":"Command and Control","shortname":"command-and-control"},"TA0040":{"name":"Impact","shortname":"impact"},"TA0042":{"name":"Resource Development","shortname":"resource-development"},"TA0043":{"name":"Reconnaissance","shortname":"reconnaissance"},"TA0112":{"name":"Defense Impairment","shortname":"defense-impairment"}},"techniques":{"T1001":["TA0011"],"T1001.001":["TA0011"],"T1001.002":["TA0011"],"T1001.003":["TA0011"],"T1002":["TA0009","TA0010"],"T1003":["TA0006"],"T1003.001":["TA0006"],"T1003.002":["TA0006"],"T1003.003":["TA0006"],"T1003.004":["TA0006"],"T1003.005":["TA0006"],"T1003.006":["TA0006"],"T1003.007":["TA0006"],"T1003.008":["TA0006"],"T1004":["TA0003","TA0004"],"T1005":["TA0009"],"T1006":["TA0005"],"T1007":["TA0007"],"T1008":["TA0011"],"T1009":["TA0005"],"T1010":["TA0007"],"T1011":["TA0010"],"T1011.001":["TA0010"],"T1012":["TA0007"],"T1013":["TA0003","TA0004"],"T1014":["TA0005"],"T1015":["TA0003","TA0004"],"T1016":["TA0007"],"T1016.001":["TA0007"],"T1016.002":["TA0007"],"T1017":["TA0002","TA0008"],"T1018":["TA0007"],"T1019":["TA0003","TA0005"],"T1020":["TA0010"],"T1020.001":["TA0010"],"T1021":["TA0008"],"T1021.001":["TA0008"],"T1021.002":["TA0008"],"T1021.003":["TA0008"],"T1021.004":["TA0008"],"T1021.005":["TA0008"],"T1021.006":["TA0008"],"T1021.007":["TA0008"],"T1021.008":["TA0008"],"T1022":["TA0009","TA0010"],"T1023":["TA0003","TA0004"],"T1024":["TA0011"],"T1025":["TA0009"],"T1027":["TA0005"],"T1027.001":["TA0005"],"T1027.002":["TA0005"],"T1027.003":["TA0005"],"T1027.004":["TA0005"],"T1027.005":["TA0005"],"T1027.006":["TA0005"],"T1027.007":["TA0005"],"T1027.008":["TA0005"],"T1027.009":["TA0005"],"T1027.010":["TA0005"],"T1027.011":["TA0005"],"T1027.012":["TA0005"],"T1027.013":["TA0005"],"T1027.014":["TA0005"],"T1027.015":["TA0005"],"T1027.016":["TA0005"],"T1027.017":["TA0005"],"T1027.018":["TA0005"],"T1028":["TA0002","TA0008"],"T1029":["TA0010"],"T1030":["TA0010"],"T1031":["TA0003","TA0004"],"T1032":["TA0011"],"T1033":["TA0007"],"T1035":["TA0002"],"T1036":["TA0005"],"T1036.001":["TA0005"],"T1036.002":["TA0005"],"T1036.003":["TA0005"],"T1036.004":["TA0005"],"T1036.005":["TA0005"],"T1036.006":["TA0005"],"T1036.007":["TA0005"],"T1036.008":["TA0005"],"T1036.009":["TA0005"],"T1036.010":["TA0005"],"T1036.011":["TA0005"],"T1036.012":["TA0005"],"T1037":["TA0003","TA0004"],"T1037.001":["TA0003","TA0004"],"T1037.002":["TA0003","TA0004"],"T1037.003":["TA0003","TA0004"],"T1037.004":["TA0003","TA0004"],"T1037.005":["TA0003","TA0004"],"T1038":["TA0002","TA0003","TA0004","TA0005"],"T1039":["TA0009"],"T1040":["TA0006","TA0007"],"T1041":["TA0010"],"T1042":["TA0003","TA0004"],"T1044":["TA0002","TA0003","TA0004","TA0005"],"T1045":["TA0005"],"T1046":["TA0007"],"T1047":["TA0002"],"T1048":["TA0010"],"T1048.001":["TA0010"],"T1048.002":["TA0010"],"T1048.003":["TA0010"],"T1049":["TA0007"],"T1050":["TA0003","TA0004"],"T1052":["TA0010"],"T1052.001":["TA0010"],"T1053":["TA0002","TA0003","TA0004"],"T1053.001":["TA0002","TA0003","TA0004"],"T1053.002":["TA0002","TA0003","TA0004"],"T1053.003":["TA0002","TA0003","TA0004"],"T1053.005":["TA0002","TA0003","TA0004"],"T1053.006":["TA0002","TA0003","TA0004"],"T1053.007":["TA0002","TA0003","TA0004"],"T1054":["TA0005","TA0112"],"T1055":["TA0004","TA0005"],"T1055.001":["TA0004","TA0005"],"T1055.002":["TA0004","TA0005"],"T1055.003":["TA0004","TA0005"],"T1055.004":["TA0004","TA0005"],"T1055.005":["TA0004","TA0005"],"T1055.008":["TA0004","TA0005"],"T1055.009":["TA0004","TA0005"],"T1055.011":["TA0004","TA0005"],"T1055.012":["TA0004","TA0005"],"T1055.013":["TA0004","TA0005"],"T1055.014":["TA0004","TA0005"],"T1055.015":["TA0004","TA0005"],"T1056":["TA0006","TA0009"],"T1056.001":["TA0006","TA0009"],"T1056.002":["TA0006","TA0009"],"T1056.003":["TA0006","TA0009"],"T1056.004":["TA0006","TA0009"],"T1057":["TA0007"],"T1058":["TA0002","TA0003","TA0004","TA0005"],"T1059":["TA0002"],"T1059.001":["TA0002"],"T1059.002":["TA0002"],"T1059.003":["TA0002"],"T1059.004":["TA0002"],"T1059.005":["TA0002"],"T1059.006":["TA0002"],"T1059.007":["TA0002"],"T1059.008":["TA0002"],"T1059.009":["TA0002"],"T1059.010":["TA0002"],"T1059.011":["TA0002"],"T1059.012":["TA0002"],"T1059.013":["TA0002"],"T1060":["TA0003","TA0004"],"T1063":["TA0007"],"T1065":["TA0011"],"T1066":["TA0005"],"T1067":["TA0003","TA0005"],"T1068":["TA0004"],"T1069":["TA0007"],"T1069.001":["TA0007"],"T1069.002":["TA0007"],"T1069.003":["TA0007"],"T1070":["TA0005"],"T1070.001":["TA0005","TA0112"],"T1070.002":["TA0005","TA0112"],"T1070.003":["TA0005"],"T1070.004":["TA0005"],"T1070.005":["TA0005"],"T1070.006":["TA0005"],"T1070.007":["TA0005"],"T1070.008":["TA0005"],"T1070.009":["TA0005"],"T1070.010":["TA0005"],"T1071":["TA0011"],"T1071.001":["TA0011"],"T1071.002":["TA0011"],"T1071.003":["TA0011"],"T1071.004":["TA0011"],"T1071.005":["TA0011"],"T1072":["TA0002","TA0008"],"T1073":["TA0002","TA0005"],"T1074":["TA0009"],"T1074.001":["TA0009"],"T1074.002":["TA0009"],"T1075":["TA0008"],"T1076":["TA0008"],"T1077":["TA0008"],"T1078":["TA0001","TA0003","TA0004","TA0005"],"T1078.001":["TA0001","TA0003","TA0004","TA0005"],"T1078.002":["TA0001","TA0003","TA0004","TA0005"],"T1078.003":["TA0001","TA0003","TA0004","TA0005"],"T1078.004":["TA0001","TA0003","TA0004","TA0005"],"T1079":["TA0011"],"T1080":["TA0008"],"T1081":["TA0006"],"T1082":["TA0007"],"T1083":["TA0007"],"T1084":["TA0003","TA0004"],"T1085":["TA0002","TA0005"],"T1086":["TA0002"],"T1087":["TA0007"],"T1087.001":["TA0007"],"T1087.002":["TA0007"],"T1087.003":["TA0007"],"T1087.004":["TA0007"],"T1088":["TA0004","TA0005"],"T1089":["TA0005","TA0112"],"T1090":["TA0011"],"T1090.001":["TA0011"],"T1090.002":["TA0011"],"T1090.003":["TA0011"],"T1090.004":["TA0011"],"T1091":["TA0001","TA0008"],"T1092":["TA0011"],"T1093":["TA0004","TA0005"],"T1094":["TA0011"],"T1095":["TA0011"],"T1096":["TA0005"],"T1097":["TA0008"],"T1098":["TA0003","TA0004"],"T1098.001":["TA0003","TA0004"],"T1098.002":["TA0003","TA0004"],"T1098.003":["TA0003","TA0004"],"T1098.004":["TA0003","TA0004"],"T1098.005":["TA0003","TA0004"],"T1098.006":["TA0003","TA0004"],"T1098.007":["TA0003","TA0004"],"T1099":["TA0005"],"T1100":["TA0003","TA0004"],"T1101":["TA0003","TA0004"],"T1102":["TA0011"],"T1102.001":["TA0011"],"T1102.002":["TA0011"],"T1102.003":["TA0011"],"T1103":["TA0003","TA0004"],"T1104":["TA0011"],"T1105":["TA0011"],"T1106":["TA0002"],"T1107":["TA0005"],"T1109":["TA0003","TA0005"],"T1110":["TA0006"],"T1110.001":["TA0006"],"T1110.002":["TA0006"],"T1110.003":["TA0006"],"T1110.004":["TA0006"],"T1111":["TA0006"],"T1112":["TA0003","TA0112"],"T1113":["TA0009"],"T1114":["TA0009"],"T1114.001":["TA0009"],"T1114.002":["TA0009"],"T1114.003":["TA0009"],"T1115":["TA0009"],"T1116":["TA0005","TA0112"],"T1117":["TA0002","TA0005"],"T1118":["TA0002","TA0005"],"T1119":["TA0009"],"T1120":["TA0007"],"T1121":["TA0002","TA0005"],"T1122":["TA0003","TA0004","TA0005"],"T1123":["TA0009"],"T1124":["TA0007"],"T1125":["TA0009"],"T1126":["TA0005"],"T1127":["TA0002","TA0005"],"T1127.001":["TA0002","TA0005"],"T1127.002":["TA0002","TA0005"],"T1127.003":["TA0002","TA0005"],"T1128":["TA0003","TA0004"],"T1129":["TA0002"],"T1130":["TA0005","TA0112"],"T1131":["TA0003","TA0004"],"T1132":["TA0011"],"T1132.001":["TA0011"],"T1132.002":["TA0011"],"T1133":["TA0001","TA0003"],"T1134":["TA0004","TA0005"],"T1134.001":["TA0004","TA0005"],"T1134.002":["TA0004","TA0005"],"T1134.003":["TA0004","TA0005"],"T1134.004":["TA0004","TA0005"],"T1134.005":["TA0004","TA0005"],"T1135":["TA0007"],"T1136":["TA0003"],"T1136.001":["TA0003"],"T1136.002":["TA0003"],"T1136.003":["TA0003"],"T1137":["TA0003"],"T1137.001":["TA0003"],"T1137.002":["TA0003"],"T1137.003":["TA0003"],"T1137.004":["TA0003"],"T1137.005":["TA0003"],"T1137.006":["TA0003"],"T1138":["TA0003","TA0004"],"T1139":["TA0006"],"T1140":["TA0005"],"T1141":["TA0006","TA0009"],"T1142":["TA0006"],"T1143":["TA0005"],"T1144":["TA0005","TA0112"],"T1145":["TA0006"],"T1146":["TA0005"],"T1147":["TA0005"],"T1148":["TA0005","TA0112"],"T1150":["TA0003","TA0004","TA0005"],"T1151":["TA0002","TA0005"],"T1152":["TA0002","TA0003","TA0005"],"T1154":["TA0002","TA0003","TA0004"],"T1155":["TA0002"],"T1156":["TA0003","TA0004"],"T1157":["TA0002","TA0003","TA0004","TA0005"],"T1158":["TA0003","TA0005"],"T1159":["TA0003","TA0004"],"T1160":["TA0003","TA0004"],"T1161":["TA0003","TA0004"],"T1162":["TA0003","TA0004"],"T1163":["TA0003","TA0004"],"T1164":["TA0003","TA0004"],"T1165":["TA0003","TA0004"],"T1166":["TA0003","TA0004"],"T1167":["TA0006"],"T1168":["TA0002","TA0003","TA0004"],"T1169":["TA0004"],"T1170":["TA0002","TA0005"],"T1171":["TA0006","TA0009"],"T1172":["TA0011"],"T1173":["TA0002"],"T1174":["TA0003","TA0006","TA0112"],"T1176":["TA0003"],"T1176.001":["TA0003"],"T1176.002":["TA0003"],"T1177":["TA0002","TA0003","TA0004"],"T1178":["TA0004","TA0005"],"T1179":["TA0003","TA0004","TA0006","TA0009"],"T1180":["TA0003","TA0004"],"T1181":["TA0004","TA0005"],"T1182":["TA0003","TA0004"],"T1183":["TA0003","TA0004","TA0005"],"T1184":["TA0008"],"T1185":["TA0009"],"T1186":["TA0004","TA0005"],"T1187":["TA0006"],"T1188":["TA0011"],"T1189":["TA0001"],"T1190":["TA0001"],"T1191":["TA0002","TA0005"],"T1192":["TA0001"],"T1193":["TA0001"],"T1194":["TA0001"],"T1195":["TA0001"],"T1195.001":["TA0001"],"T1195.002":["TA0001"],"T1195.003":["TA0001"],"T1196":["TA0002","TA0005"],"T1197":["TA0002","TA0003","TA0005"],"T1198":["TA0003","TA0005","TA0112"],"T1199":["TA0001"],"T1200":["TA0001"],"T1201":["TA0007"],"T1202":["TA0005"],"T1203":["TA0002"],"T1204":["TA0002"],"T1204.001":["TA0002"],"T1204.002":["TA0002"],"T1204.003":["TA0002"],"T1204.004":["TA0002"],"T1204.005":["TA0002"],"T1205":["TA0003","TA0005","TA0011"],"T1205.001":["TA0003","TA0005","TA0011"],"T1205.002":["TA0003","TA0005","TA0011"],"T1206":["TA0004"],"T1207":["TA0112"],"T1208":["TA0006"],"T1209":["TA0003","TA0004"],"T1210":["TA0008"],"T1211":["TA0005"],"T1212":["TA0006"],"T1213":["TA0009"],"T1213.001":["TA0009"],"T1213.002":["TA0009"],"T1213.003":["TA0009"],"T1213.004":["TA0009"],"T1213.005":["TA0009"],"T1213.006":["TA0009"],"T1214":["TA0006"],"T1215":["TA0003","TA0004"],"T1216":["TA0005"],"T1216.001":["TA0005"],"T1216.002":["TA0005"],"T1217":["TA0007"],"T1218":["TA0005"],"T1218.001":["TA0005"],"T1218.002":["TA0005"],"T1218.003":["TA0005"],"T1218.004":["TA0005"],"T1218.005":["TA0005"],"T1218.007":["TA0005"],"T1218.008":["TA0005"],"T1218.009":["TA0005"],"T1218.010":["TA0005"],"T1218.011":["TA0005"],"T1218.012":["TA0005"],"T1218.013":["TA0005"],"T1218.014":["TA0005"],"T1218.015":["TA0005"],"T1219":["TA0011"],"T1219.001":["TA0011"],"T1219.002":["TA0011"],"T1219.003":["TA0011"],"T1220":["TA0005"],"T1221":["TA0005"],"T1222":["TA0112"],"T1222.001":["TA0112"],"T1222.002":["TA0112"],"T1223":["TA0002","TA0005"],"T1480":["TA0005"],"T1480.001":["TA0005"],"T1480.002":["TA0005"],"T1482":["TA0007"],"T1483":["TA0011"],"T1484":["TA0004","TA0112"],"T1484.001":["TA0004","TA0112"],"T1484.002":["TA0004","TA0112"],"T1485":["TA0040"],"T1485.001":["TA0040"],"T1486":["TA0040"],"T1487":["TA0040"],"T1488":["TA0040"],"T1489":["TA0040"],"T1490":["TA0040"],"T1491":["TA0040"],"T1491.001":["TA0040"],"T1491.002":["TA0040"],"T1492":["TA0040"],"T1493":["TA0040"],"T1494":["TA0040"],"T1495":["TA0040"],"T1496":["TA0040"],"T1496.001":["TA0040"],"T1496.002":["TA0040"],"T1496.003":["TA0040"],"T1496.004":["TA0040"],"T1497":["TA0005","TA0007"],"T1497.001":["TA0005","TA0007"],"T1497.002":["TA0005","TA0007"],"T1497.003":["TA0005","TA0007"],"T1498":["TA0040"],"T1498.001":["TA0040"],"T1498.002":["TA0040"],"T1499":["TA0040"],"T1499.001":["TA0040"],"T1499.002":["TA0040"],"T1499.003":["TA0040"],"T1499.004":["TA0040"],"T1500":["TA0005"],"T1501":["TA0003","TA0004"],"T1502":["TA0004","TA0005"],"T1503":["TA0006"],"T1504":["TA0003","TA0004"],"T1505":["TA0003"],"T1505.001":["TA0003"],"T1505.002":["TA0003"],"T1505.003":["TA0003"],"T1505.004":["TA0003"],"T1505.005":["TA0003"],"T1505.006":["TA0003"],"T1506":["TA0005","TA0008"],"T1514":["TA0004"],"T1518":["TA0007"],"T1518.001":["TA0007"],"T1518.002":["TA0007"],"T1519":["TA0003","TA0004"],"T1522":["TA0006"],"T1525":["TA0003"],"T1526":["TA0007"],"T1527":["TA0005","TA0008"],"T1528":["TA0006"],"T1529":["TA0040"],"T1530":["TA0009"],"T1531":["TA0040"],"T1534":["TA0008"],"T1535":["TA0005"],"T1536":["TA0005","TA0112"],"T1537":["TA0010"],"T1538":["TA0007"],"T1539":["TA0006"],"T1542":["TA0003","TA0005"],"T1542.001":["TA0003","TA0005"],"T1542.002":["TA0003","TA0005"],"T1542.003":["TA0003","TA0005"],"T1542.004":["TA0003","TA0005"],"T1542.005":["TA0003","TA0005"],"T1543":["TA0003","TA0004"],"T1543.001":["TA0003","TA0004"],"T1543.002":["TA0003","TA0004"],"T1543.003":["TA0003","TA0004"],"T1543.004":["TA0003","TA0004"],"T1543.005":["TA0003","TA0004"],"T1546":["TA0003","TA0004"],"T1546.001":["TA0003","TA0004"],"T1546.002":["TA0003","TA0004"],"T1546.003":["TA0003","TA0004"],"T1546.004":["TA0003","TA0004"],"T1546.005":["TA0003","TA0004"],"T1546.006":["TA0003","TA0004"],"T1546.007":["TA0003","TA0004"],"T1546.008":["TA0003","TA0004"],"T1546.009":["TA0003","TA0004"],"T1546.010":["TA0003","TA0004"],"T1546.011":["TA0003","TA0004"],"T1546.012":["TA0003","TA0004"],"T1546.013":["TA0003","TA0004"],"T1546.014":["TA0003","TA0004"],"T1546.015":["TA0003","TA0004"],"T1546.016":["TA0003","TA0004"],"T1546.017":["TA0003","TA0004"],"T1546.018":["TA0003","TA0004"],"T1547":["TA0003","TA0004"],"T1547.001":["TA0003","TA0004"],"T1547.002":["TA0003","TA0004"],"T1547.003":["TA0003","TA0004"],"T1547.004":["TA0003","TA0004"],"T1547.005":["TA0003","TA0004"],"T1547.006":["TA0003","TA0004"],"T1547.007":["TA0003","TA0004"],"T1547.008":["TA0003","TA0004"],"T1547.009":["TA0003","TA0004"],"T1547.010":["TA0003","TA0004"],"T1547.011":["TA0003","TA0004","TA0112"],"T1547.012":["TA0003","TA0004"],"T1547.013":["TA0003","TA0004"],"T1547.014":["TA0003","TA0004"],"T1547.015":["TA0003","TA0004"],"T1548":["TA0004"],"T1548.001":["TA0004"],"T1548.002":["TA0004"],"T1548.003":["TA0004"],"T1548.004":["TA0004"],"T1548.005":["TA0004"],"T1548.006":["TA0004"],"T1550":["TA0008"],"T1550.001":["TA0008"],"T1550.002":["TA0008"],"T1550.003":["TA0008"],"T1550.004":["TA0008"],"T1552":["TA0006"],"T1552.001":["TA0006"],"T1552.002":["TA0006"],"T1552.003":["TA0006"],"T1552.004":["TA0006"],"T1552.005":["TA0006"],"T1552.006":["TA0006"],"T1552.007":["TA0006"],"T1552.008":["TA0006"],"T1553":["TA0112"],"T1553.001":["TA0112"],"T1553.002":["TA0112"],"T1553.003":["TA0112"],"T1553.004":["TA0112"],"T1553.005":["TA0112"],"T1553.006":["TA0112"],"T1554":["TA0003"],"T1555":["TA0006"],"T1555.001":["TA0006"],"T1555.002":["TA0006"],"T1555.003":["TA0006"],"T1555.004":["TA0006"],"T1555.005":["TA0006"],"T1555.006":["TA0006"],"T1556":["TA0003","TA0006","TA0112"],"T1556.001":["TA0003","TA0006","TA0112"],"T1556.002":["TA0003","TA0006","TA0112"],"T1556.003":["TA0003","TA0006","TA0112"],"T1556.004":["TA0003","TA0006","TA0112"],"T1556.005":["TA0003","TA0006","TA0112"],"T1556.006":["TA0003","TA0006","TA0112"],"T1556.007":["TA0003","TA0006","TA0112"],"T1556.008":["TA0003","TA0006","TA0112"],"T1556.009":["TA0003","TA0006","TA0112"],"T1557":["TA0006","TA0009"],"T1557.001":["TA0006","TA0009"],"T1557.002":["TA0006","TA0009"],"T1557.003":["TA0006","TA0009"],"T1557.004":["TA0006","TA0009"],"T1558":["TA0006"],"T1558.001":["TA0006"],"T1558.002":["TA0006"],"T1558.003":["TA0006"],"T1558.004":["TA0006"],"T1558.005":["TA0006"],"T1559":["TA0002"],"T1559.001":["TA0002"],"T1559.002":["TA0002"],"T1559.003":["TA0002"],"T1560":["TA0009"],"T1560.001":["TA0009"],"T1560.002":["TA0009"],"T1560.003":["TA0009"],"T1561":["TA0040"],"T1561.001":["TA0040"],"T1561.002":["TA0040"],"T1562":["TA0005","TA0112"],"T1562.001":["TA0005","TA0112"],"T1562.002":["TA0005","TA0112"],"T1562.003":["TA0005","TA0112"],"T1562.004":["TA0005","TA0112"],"T1562.006":["TA0005","TA0112"],"T1562.007":["TA0005","TA0112"],"T1562.008":["TA0005","TA0112"],"T1562.009":["TA0005","TA0112"],"T1562.010":["TA0005","TA0112"],"T1562.011":["TA0005","TA0112"],"T1562.012":["TA0005","TA0112"],"T1562.013":["TA0005","TA0112"],"T1563":["TA0008"],"T1563.001":["TA0008"],"T1563.002":["TA0008"],"T1564":["TA0005"],"T1564.001":["TA0005"],"T1564.002":["TA0005"],"T1564.003":["TA0005"],"T1564.004":["TA0005"],"T1564.005":["TA0005"],"T1564.006":["TA0005"],"T1564.007":["TA0005"],"T1564.008":["TA0005"],"T1564.009":["TA0005"],"T1564.010":["TA0005"],"T1564.011":["TA0005"],"T1564.012":["TA0005"],"T1564.013":["TA0005"],"T1564.014":["TA0005"],"T1565":["TA0040"],"T1565.001":["TA0040"],"T1565.002":["TA0040"],"T1565.003":["TA0040"],"T1566":["TA0001"],"T1566.001":["TA0001"],"T1566.002":["TA0001"],"T1566.003":["TA0001"],"T1566.004":["TA0001"],"T1567":["TA0010"],"T1567.001":["TA0010"],"T1567.002":["TA0010"],"T1567.003":["TA0010"],"T1567.004":["TA0010"],"T1568":["TA0011"],"T1568.001":["TA0011"],"T1568.002":["TA0011"],"T1568.003":["TA0011"],"T1569":["TA0002"],"T1569.001":["TA0002"],"T1569.002":["TA0002"],"T1569.003":["TA0002"],"T1570":["TA0008"],"T1571":["TA0011"],"T1572":["TA0011"],"T1573":["TA0011"],"T1573.001":["TA0011"],"T1573.002":["TA0011"],"T1574":["TA0002","TA0005"],"T1574.001":["TA0002","TA0005"],"T1574.002":["TA0002","TA0005"],"T1574.004":["TA0002","TA0005"],"T1574.005":["TA0002","TA0005"],"T1574.006":["TA0002","TA0005"],"T1574.007":["TA0002","TA0005"],"T1574.008":["TA0002","TA0005"],"T1574.009":["TA0002","TA0005"],"T1574.010":["TA0002","TA0005"],"T1574.011":["TA0002","TA0005"],"T1574.012":["TA0002","TA0005"],"T1574.013":["TA0002","TA0005"],"T1574.014":["TA0002","TA0005"],"T1578":["TA0112"],"T1578.001":["TA0112"],"T1578.002":["TA0112"],"T1578.003":["TA0112"],"T1578.004":["TA0112"],"T1578.005":["TA0112"],"T1580":["TA0007"],"T1583":["TA0042"],"T1583.001":["TA0042"],"T1583.002":["TA0042"],"T1583.003":["TA0042"],"T1583.004":["TA0042"],"T1583.005":["TA0042"],"T1583.006":["TA0042"],"T1583.007":["TA0042"],"T1583.008":["TA0042"],"T1584":["TA0042"],"T1584.001":["TA0042"],"T1584.002":["TA0042"],"T1584.003":["TA0042"],"T1584.004":["TA0042"],"T1584.005":["TA0042"],"T1584.006":["TA0042"],"T1584.007":["TA0042"],"T1584.008":["TA0042"],"T1585":["TA0042"],"T1585.001":["TA0042"],"T1585.002":["TA0042"],"T1585.003":["TA0042"],"T1586":["TA0042"],"T1586.001":["TA0042"],"T1586.002":["TA0042"],"T1586.003":["TA0042"],"T1587":["TA0042"],"T1587.001":["TA0042"],"T1587.002":["TA0042"],"T1587.003":["TA0042"],"T1587.004":["TA0042"],"T1588":["TA0042"],"T1588.001":["TA0042"],"T1588.002":["TA0042"],"T1588.003":["TA0042"],"T1588.004":["TA0042"],"T1588.005":["TA0042"],"T1588.006":["TA0042"],"T1588.007":["TA0042"],"T1589":["TA0043"],"T1589.001":["TA0043"],"T1589.002":["TA0043"],"T1589.003":["TA0043"],"T1590":["TA0043"],"T1590.001":["TA0043"],"T1590.002":["TA0043"],"T1590.003":["TA0043"],"T1590.004":["TA0043"],"T1590.005":["TA0043"],"T1590.006":["TA0043"],"T1591":["TA0043"],"T1591.001":["TA0043"],"T1591.002":["TA0043"],"T1591.003":["TA0043"],"T1591.004":["TA0043"],"T1592":["TA0043"],"T1592.001":["TA0043"],"T1592.002":["TA0043"],"T1592.003":["TA0043"],"T1592.004":["TA0043"],"T1593":["TA0043"],"T1593.001":["TA0043"],"T1593.002":["TA0043"],"T1593.003":["TA0043"],"T1594":["TA0043"],"T1595":["TA0043"],"T1595.001":["TA0043"],"T1595.002":["TA0043"],"T1595.003":["TA0043"],"T1596":["TA0043"],"T1596.001":["TA0043"],"T1596.002":["TA0043"],"T1596.003":["TA0043"],"T1596.004":["TA0043"],"T1596.005":["TA0043"],"T1597":["TA0043"],"T1597.001":["TA0043"],"T1597.002":["TA0043"],"T1598":["TA0043"],"T1598.001":["TA0043"],"T1598.002":["TA0043"],"T1598.003":["TA0043"],"T1598.004":["TA0043"],"T1599":["TA0112"],"T1599.001":["TA0112"],"T1600":["TA0112"],"T1600.001":["TA0112"],"T1600.002":["TA0112"],"T1601":["TA0112"],"T1601.001":["TA0112"],"T1601.002":["TA0112"],"T1602":["TA0009"],"T1602.001":["TA0009"],"T1602.002":["TA0009"],"T1606":["TA0006"],"T1606.001":["TA0006"],"T1606.002":["TA0006"],"T1608":["TA0042"],"T1608.001":["TA0042"],"T1608.002":["TA0042"],"T1608.003":["TA0042"],"T1608.004":["TA0042"],"T1608.005":["TA0042"],"T1608.006":["TA0042"],"T1609":["TA0002"],"T1610":["TA0002"],"T1611":["TA0004"],"T1612":["TA0005"],"T1613":["TA0007"],"T1614":["TA0007"],"T1614.001":["TA0007"],"T1615":["TA0007"],"T1619":["TA0007"],"T1620":["TA0005"],"T1621":["TA0006"],"T1622":["TA0005","TA0007"],"T1647":["TA0112"],"T1648":["TA0002"],"T1649":["TA0006"],"T1650":["TA0042"],"T1651":["TA0002"],"T1652":["TA0007"],"T1653":["TA0003"],"T1654":["TA0007"],"T1656":["TA0005"],"T1657":["TA0040"],"T1659":["TA0001","TA0011"],"T1665":["TA0011"],"T1666":["TA0112"],"T1667":["TA0040"],"T1668":["TA0003"],"T1669":["TA0001"],"T1671":["TA0003"],"T1672":["TA0005"],"T1673":["TA0007"],"T1674":["TA0002"],"T1675":["TA0002"],"T1677":["TA0002"],"T1678":["TA0005"],"T1679":["TA0005"],"T1680":["TA0007"],"T1681":["TA0043"],"T1682":["TA0043"],"T1683":["TA0042"],"T1683.001":["TA0042"],"T1683.002":["TA0042"],"T1684":["TA0005"],"T1684.001":["TA0005"],"T1684.002":["TA0005"],"T1685":["TA0112"],"T1685.001":["TA0112"],"T1685.002":["TA0112"],"T1685.003":["TA0112"],"T1685.004":["TA0112"],"T1685.005":["TA0112"],"T1685.006":["TA0112"],"T1686":["TA0112"],"T1686.001":["TA0112"],"T1686.002":["TA0112"],"T1686.003":["TA0112"],"T1687":["TA0112"],"T1688":["TA0112"],"T1689":["TA0112"],"T1690":["TA0112"]},"revoked_by":{"T1002":"T1560","T1004":"T1547.004","T1009":"T1027.001","T1013":"T1547.010","T1015":"T1546.008","T1017":"T1072","T1019":"T1542.001","T1022":"T1560","T1023":"T1547.009","T1024":"T1573","T1028":"T1021.006","T1031":"T1543.003","T1032":"T1573","T1035":"T1569.002","T1038":"T1574.001","T1042":"T1546.001","T1044":"T1574.010","T1045":"T1027.002","T1050":"T1543.003","T1053.001":"T1053.002","T1054":"T1685","T1058":"T1574.011","T1060":"T1547.001","T1063":"T1518.001","T1065":"T1571","T1066":"T1027.005","T1067":"T1542.003","T1070.001":"T1685.005","T1070.002":"T1685.006","T1073":"T1574.002","T1075":"T1550.002","T1076":"T1021.001","T1077":"T1021.002","T1079":"T1573","T1081":"T1552.001","T1084":"T1546.003","T1085":"T1218.011","T1086":"T1059.001","T1088":"T1548.002","T1089":"T1685","T1093":"T1055.012","T1094":"T1095","T1096":"T1564.004","T1097":"T1550.003","T1099":"T1070.006","T1100":"T1505.003","T1101":"T1547.005","T1103":"T1546.010","T1107":"T1070.004","T1109":"T1542.002","T1116":"T1553.002","T1117":"T1218.010","T1118":"T1218.004","T1121":"T1218.009","T1122":"T1546.015","T1126":"T1070.005","T1128":"T1546.007","T1130":"T1553.004","T1131":"T1547.002","T1138":"T1546.011","T1139":"T1552.003","T1141":"T1056.002","T1142":"T1555.001","T1143":"T1564.003","T1144":"T1553.001","T1145":"T1552.004","T1146":"T1070.003","T1147":"T1564.002","T1148":"T1690","T1150":"T1547.011","T1151":"T1036.006","T1152":"T1569.001","T1154":"T1546.005","T1155":"T1059.002","T1156":"T1546.004","T1157":"T1574.004","T1158":"T1564.001","T1159":"T1543.001","T1160":"T1543.004","T1161":"T1546.006","T1162":"T1547.011","T1163":"T1037.004","T1164":"T1547.007","T1165":"T1037.005","T1166":"T1548.001","T1167":"T1555.002","T1168":"T1053","T1169":"T1548.003","T1170":"T1218.005","T1171":"T1557.001","T1172":"T1090.004","T1173":"T1559.002","T1174":"T1556.002","T1177":"T1547.008","T1178":"T1134.005","T1179":"T1056.004","T1180":"T1546.002","T1181":"T1055.011","T1182":"T1546.009","T1183":"T1546.012","T1184":"T1563.001","T1186":"T1055.013","T1188":"T1090.003","T1191":"T1218.003","T1192":"T1566.002","T1193":"T1566.001","T1194":"T1566.003","T1196":"T1218.002","T1198":"T1553.003","T1206":"T1548.003","T1208":"T1558.003","T1209":"T1547.003","T1214":"T1552.002","T1215":"T1547.006","T1223":"T1218.001","T1483":"T1568.002","T1487":"T1561.002","T1488":"T1561.001","T1492":"T1565.001","T1493":"T1565.002","T1494":"T1565.003","T1500":"T1027.004","T1501":"T1543.002","T1502":"T1134.004","T1503":"T1555.003","T1504":"T1546.013","T1506":"T1550.004","T1514":"T1548.004","T1519":"T1546.014","T1522":"T1552.005","T1527":"T1550.001","T1536":"T1578.004","T1547.011":"T1647","T1562":"T1685","T1562.001":"T1685","T1562.002":"T1685.001","T1562.003":"T1690","T1562.004":"T1686","T1562.006":"T1685","T1562.007":"T1686.001","T1562.008":"T1685.002","T1562.009":"T1688","T1562.010":"T1689","T1562.011":"T1685.003","T1562.012":"T1685.004","T1562.013":"T1686.002","T1574.002":"T1574.001","T1656":"T1684.001","T1672":"T1684.002"}} \ No newline at end of file diff --git a/glaive/detection/attack.py b/glaive/detection/attack.py new file mode 100644 index 0000000..2cc9b20 --- /dev/null +++ b/glaive/detection/attack.py @@ -0,0 +1,122 @@ +"""MITRE ATT&CK lookups: which tactics a technique belongs to. + + tactics_for("T1003.001") -> ["TA0006"] (Credential Access) + tactic_id("defense_evasion") -> "TA0005" + tactic_name("TA0005") -> "Stealth" + +The table in attack.json was generated from the official Enterprise ATT&CK +STIX bundle (version in table()["version"]). ATT&CK v19 renamed Defense Evasion +(TA0005) to Stealth and split part of it into Defense Impairment (TA0112); +older names, Sigma tags ("attack.defense_evasion", "attack.defense-evasion") +and tactic IDs are all accepted. v19 also revoked techniques such as T1562.001 +(now T1685); revoked IDs are kept, with the tactics of both the old and the new +technique, because rules and datasets still use them. + +(c) The MITRE Corporation. Reproduced and distributed with the permission of +The MITRE Corporation. +""" +from __future__ import annotations + +import json +import re +from functools import lru_cache +from pathlib import Path +from typing import Any + +_TABLE_PATH = Path(__file__).parent / "attack.json" +_TID = re.compile(r"^T\d{4}(?:\.\d{3})?$") + +# Names used before ATT&CK v19 (and by datasets labelled before then). +_ALIASES = {"defense-evasion": "TA0005", "defenseevasion": "TA0005"} +# A label of "Defense Evasion" also covers what v19 moved to Defense Impairment. +EQUIVALENT_TACTICS = {"TA0005": frozenset({"TA0005", "TA0112"}), + "TA0112": frozenset({"TA0005", "TA0112"})} + + +@lru_cache(maxsize=1) +def table() -> dict[str, Any]: + return json.loads(_TABLE_PATH.read_text(encoding="utf-8")) + + +def _key(text: str) -> str: + return re.sub(r"[\s_]+", "-", text.strip().lower()) + + +@lru_cache(maxsize=256) +def tactic_id(name_or_tag: str) -> str | None: + """'Credential Access', 'credential_access', 'attack.credential-access', + 'TA0006' -> 'TA0006'. None if it is not a tactic.""" + s = name_or_tag.strip() + if s.lower().startswith("attack."): + s = s[7:] + if re.fullmatch(r"(?i)ta\d{4}", s): + return s.upper() if s.upper() in table()["tactics"] else None + k = _key(s) + if k in _ALIASES: + return _ALIASES[k] + for tid, t in table()["tactics"].items(): + if k in (t["shortname"], _key(t["name"])): + return tid + return None + + +def tactic_name(tid: str) -> str: + t = table()["tactics"].get(tid) + return t["name"] if t else tid + + +def tactics_for(technique: str) -> list[str]: + """Tactic IDs for a technique ID. Sub-techniques fall back to their parent + when the table does not list them.""" + t = technique.strip().upper() + if not _TID.match(t): + return [] + techs = table()["techniques"] + return list(techs.get(t) or techs.get(t.split(".")[0]) or []) + + +def tactics_from_tags(tags: list[str]) -> list[str]: + """Tactic IDs named directly in Sigma tags (attack.execution ...).""" + out: list[str] = [] + for tag in tags: + low = tag.lower() + if not low.startswith("attack.") or re.fullmatch(r"attack\.[tsg]\d{4}(\.\d{3})?", low): + continue + tid = tactic_id(tag) + if tid and tid not in out: + out.append(tid) + return out + + +def tactics_of(techniques: list[str], tags: list[str] | None = None) -> list[str]: + """All tactics implied by technique IDs plus those named in tags.""" + out = tactics_from_tags(tags or []) + for t in techniques: + for tid in tactics_for(t): + if tid not in out: + out.append(tid) + return out + + +def current_id(technique: str) -> str: + """The ID ATT&CK uses today: T1562.001 -> T1685 (revoked in v19).""" + t = technique.strip().upper() + seen = set() + while t in table()["revoked_by"] and t not in seen: + seen.add(t) + t = table()["revoked_by"][t] + return t + + +def same_tactic(a: str, b: str) -> bool: + return a == b or b in EQUIVALENT_TACTICS.get(a, frozenset()) + + +def same_technique(found: str, expected: str) -> bool: + """T1003.006 matches an expected T1003 (and vice versa): the parent + technique is what most labels and rules agree on. A revoked ID matches the + technique that replaced it.""" + f, e = found.strip().upper(), expected.strip().upper() + fs = {f, current_id(f)} + es = {e, current_id(e)} + return bool(fs & es) or bool({x.split(".")[0] for x in fs} & {x.split(".")[0] for x in es}) diff --git a/glaive/detection/sigma.py b/glaive/detection/sigma.py index 127862f..17fe857 100644 --- a/glaive/detection/sigma.py +++ b/glaive/detection/sigma.py @@ -33,6 +33,7 @@ import yaml +from glaive.detection.attack import tactics_of from glaive.ingestion.windows import classify_channel logger = logging.getLogger(__name__) @@ -359,6 +360,10 @@ def mitre_techniques(self) -> list[str]: out.append(m.group(1).upper()) return out + @property + def mitre_tactics(self) -> list[str]: + return tactics_of(self.mitre_techniques, self.tags) + def matches(self, family: str, event_id: int, view: dict[str, str]) -> bool: return self._accepts(family, event_id) and self._match(view) diff --git a/glaive/graph/nodes.py b/glaive/graph/nodes.py index 75a745a..9ab6f59 100644 --- a/glaive/graph/nodes.py +++ b/glaive/graph/nodes.py @@ -718,6 +718,7 @@ class Alert(Node): detection_time: datetime = Field(..., description="Timestamp of the triggering event.") description: str | None = None mitre_techniques: list[str] = Field(default_factory=list, description="e.g. ['T1059.001'].") + mitre_tactics: list[str] = Field(default_factory=list, description="e.g. ['TA0002'].") event_id: int | None = None event_record_id: int | None = Field(None, description="EVTX EventRecordID, for traceability.") channel: str | None = None diff --git a/glaive/ingestion/pipeline.py b/glaive/ingestion/pipeline.py index 6822e69..7ef35ba 100644 --- a/glaive/ingestion/pipeline.py +++ b/glaive/ingestion/pipeline.py @@ -25,6 +25,7 @@ from pathlib import Path, PurePosixPath from typing import Any +from glaive.detection.attack import tactics_of from glaive.detection.correlations import ( CorrelationHit, brute_force_then_success, @@ -199,7 +200,8 @@ def _matched_fields(ev: dict, limit: int = 6) -> dict[str, str]: def _alert_node(ev: dict, rule_id: str, title: str, level: str, description: str, - mitre: list[str], source: str, matched: dict[str, str] | None = None) -> Alert | None: + mitre: list[str], source: str, matched: dict[str, str] | None = None, + tactics: list[str] | None = None) -> Alert | None: t = parse_time(ev.get("time_created")) if t is None or not ev.get("_evidence_hash"): return None @@ -208,6 +210,7 @@ def _alert_node(ev: dict, rule_id: str, title: str, level: str, description: str derivation=f"{source} rule {rule_id} on {ev.get('_derivation', 'event')}", host_hostname=ev["computer"], rule_id=rule_id, title=title, level=level, detection_time=t, description=description or None, mitre_techniques=mitre, + mitre_tactics=tactics if tactics is not None else tactics_of(mitre), event_id=ev.get("event_id"), event_record_id=ev.get("_record_id"), channel=ev.get("channel"), matched_fields=matched or _matched_fields(ev), source=source) @@ -327,7 +330,7 @@ def add_alert(node: Alert | None, uid: str | None, related: list[str]) -> None: for ev in all_events: for rule in engine.match(ev): node = _alert_node(ev, rule.id, rule.title, rule.level, rule.description, - rule.mitre_techniques, "sigma") + rule.mitre_techniques, "sigma", tactics=rule.mitre_tactics) add_alert(node, ev.get("_uid"), []) if node is not None: alert_records.append({"host": ev["computer"], "time": node.detection_time, diff --git a/pyproject.toml b/pyproject.toml index e9e6bbf..6544a62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ build-backend = "setuptools.build_meta" include = ["glaive*"] [tool.setuptools.package-data] -"glaive.detection" = ["rules/*.yml"] +"glaive.detection" = ["rules/*.yml", "attack.json"] "glaive.web" = ["static/*"] [tool.ruff] diff --git a/tests/v03/__init__.py b/tests/v03/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/v03/conftest.py b/tests/v03/conftest.py new file mode 100644 index 0000000..fbf35da --- /dev/null +++ b/tests/v03/conftest.py @@ -0,0 +1,2 @@ +"""Fixtures for the v0.3 tests: the demo case is shared with the v0.2 suite.""" +from tests.v02.conftest import demo_dir, demo_session # noqa: F401 diff --git a/tests/v03/test_attack.py b/tests/v03/test_attack.py new file mode 100644 index 0000000..f511e31 --- /dev/null +++ b/tests/v03/test_attack.py @@ -0,0 +1,64 @@ +"""MITRE ATT&CK technique -> tactic lookups.""" +from __future__ import annotations + +from pathlib import Path + +from glaive.detection.attack import ( + same_tactic, + same_technique, + table, + tactic_id, + tactic_name, + tactics_for, + tactics_from_tags, +) +from glaive.detection.sigma import load_rules +from glaive.mcp_server.session import GlaiveSession + + +def test_table_is_the_official_enterprise_matrix() -> None: + t = table() + assert t["source"] == "MITRE ATT&CK Enterprise" and float(t["version"]) >= 19 + assert len(t["techniques"]) > 600 and len(t["tactics"]) >= 14 + + +def test_technique_to_tactic() -> None: + assert tactics_for("T1003.001") == ["TA0006"] + assert "TA0002" in tactics_for("t1059.001") + assert tactics_for("T1059.999") == tactics_for("T1059") # unknown sub-technique: parent + assert tactics_for("not-a-technique") == [] + + +def test_tactic_names_old_and_new() -> None: + assert tactic_id("Credential Access") == tactic_id("credential_access") == "TA0006" + assert tactic_id("attack.command-and-control") == "TA0011" + # ATT&CK v19 renamed Defense Evasion to Stealth; old labels still resolve. + assert tactic_id("Defense Evasion") == tactic_id("attack.defense_evasion") == "TA0005" + assert tactic_name("TA0005") == "Stealth" + assert same_tactic("TA0005", "TA0112") and not same_tactic("TA0005", "TA0006") + assert tactic_id("Not A Tactic") is None and tactic_id("TA9999") is None + + +def test_tags_and_technique_matching() -> None: + assert tactics_from_tags(["attack.execution", "attack.t1059.001", "attack.s0002"]) == ["TA0002"] + assert same_technique("T1003.006", "T1003") and same_technique("T1003", "T1003.001") + assert not same_technique("T1059.001", "T1003") + # v19 revoked T1562.001 in favour of T1685; both spellings match. + assert same_technique("T1562.001", "T1685") and "TA0112" in tactics_for("T1562.001") + + +def test_every_builtin_rule_has_a_tactic() -> None: + rules, _ = load_rules() + assert rules and all(r.mitre_tactics for r in rules), \ + [r.title for r in rules if not r.mitre_tactics] + + +def test_alerts_carry_tactics(demo_dir: Path, tmp_path: Path) -> None: + from glaive.ingestion.pipeline import ingest_path + + s = GlaiveSession(analysis_dir=tmp_path / "c") + ingest_path(s, demo_dir) + alerts = list(s.graph.find_nodes("Alert")) + assert alerts and all(a.mitre_tactics for a in alerts) + dump = next(a for a in alerts if "Credential Dumping" in a.title) + assert "TA0006" in dump.mitre_tactics From 64ba3026ae6067e1cbcfdc5ee4f687547cdff321 Mon Sep 17 00:00:00 2001 From: Aliya Alias Date: Sat, 3 Oct 2026 11:36:58 +0800 Subject: [PATCH 05/21] perf(detection): check each event only against rules for its log source Which rules accept a (log family, event id) pair depends only on their logsource, so it is now computed once per pair and cached. With the 2,200 SigmaHQ Windows rules, EVTX-ATTACK-SAMPLES (35,807 events) goes from 33s to 13s with identical alerts; a test checks the result against evaluating every rule. --- glaive/detection/sigma.py | 18 +++++++++++++++++- tests/v02/test_sigma.py | 16 ++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/glaive/detection/sigma.py b/glaive/detection/sigma.py index 17fe857..53de14b 100644 --- a/glaive/detection/sigma.py +++ b/glaive/detection/sigma.py @@ -440,9 +440,25 @@ class SigmaEngine: def __init__(self, rules: list[SigmaRule]) -> None: self.rules = rules + # Which rules accept a (log family, event id) depends only on the rule's + # logsource, so it is worked out once per pair instead of per event. + # With the 2,000+ SigmaHQ rules most events are only checked against a + # handful of candidates. + self._candidates: dict[tuple[str, int], list[SigmaRule]] = {} + + def candidates(self, family: str, event_id: int) -> list[SigmaRule]: + key = (family, event_id) + found = self._candidates.get(key) + if found is None: + found = [r for r in self.rules if r._accepts(family, event_id)] + self._candidates[key] = found + return found def match(self, ev: dict[str, Any]) -> list[SigmaRule]: family = classify_channel(ev) event_id = ev.get("event_id") or 0 + rules = self.candidates(family, event_id) + if not rules: + return [] view = event_view(ev) - return [r for r in self.rules if r.matches(family, event_id, view)] + return [r for r in rules if r._match(view)] diff --git a/tests/v02/test_sigma.py b/tests/v02/test_sigma.py index b0bebbc..00d4728 100644 --- a/tests/v02/test_sigma.py +++ b/tests/v02/test_sigma.py @@ -126,3 +126,19 @@ def test_rule_files_are_valid_yaml() -> None: for f in BUILTIN_RULES_DIR.glob("*.yml"): doc = yaml.safe_load(f.read_text(encoding="utf-8")) assert {"title", "id", "logsource", "detection", "level"} <= set(doc), f.name + + +def test_engine_prefilter_gives_the_same_matches_as_checking_every_rule(demo_dir: Path) -> None: + from glaive.detection.sigma import event_view + from glaive.ingestion.jsonl import iter_json_events + from glaive.ingestion.windows import classify_channel + + rules, _ = load_rules() + engine = SigmaEngine(rules) + events = [e for f in sorted(demo_dir.glob("*.jsonl")) for e in iter_json_events(f)] + assert len(events) > 100 + for e in events: + view, fam, eid = event_view(e), classify_channel(e), e.get("event_id") or 0 + brute = [r.id for r in rules if r.matches(fam, eid, view)] + assert [r.id for r in engine.match(e)] == brute + assert len(engine._candidates) < len(events) # worked out once per (family, event id) From 1be92571b9df94c876c758e68d3a57867ceaf3e6 Mon Sep 17 00:00:00 2001 From: Aliya Alias Date: Sat, 3 Oct 2026 11:44:40 +0800 Subject: [PATCH 06/21] feat(ingest): extract .tar, .tar.gz and .tgz evidence archives Several public datasets (some OTRF Security-Datasets, NextronSystems evtx-baseline) and many triage tools ship tar archives. They get the same protection as zips: path traversal refused, size and ratio limits, and only regular files are written (links, devices and fifos are skipped). --- glaive/ingestion/pipeline.py | 101 +++++++++++++++++++++++++++-------- glaive/web/static/index.html | 2 +- tests/v02/test_pipeline.py | 58 ++++++++++++++++++++ 3 files changed, 139 insertions(+), 22 deletions(-) diff --git a/glaive/ingestion/pipeline.py b/glaive/ingestion/pipeline.py index 7ef35ba..18f40aa 100644 --- a/glaive/ingestion/pipeline.py +++ b/glaive/ingestion/pipeline.py @@ -3,7 +3,8 @@ summary = ingest_path(session, Path("./triage.zip")) Steps: - 1. Collect files (folders walked; .zip archives safely extracted). + 1. Collect files (folders walked; .zip and .tar/.tar.gz/.tgz archives + safely extracted). 2. Hash every file into the evidence store (chain of custody first). 3. Detect each file's format from its bytes and read its events (EVTX binary, JSON / JSON-Lines exports). @@ -17,6 +18,7 @@ import logging import os +import tarfile import zipfile from collections import Counter from collections.abc import Callable @@ -108,29 +110,86 @@ def to_dict(self) -> dict[str, Any]: # ---- file collection ----------------------------------------------------------- -def safe_extract(archive: Path, dest: Path) -> list[Path]: - """Extract a zip with zip-slip, zip-bomb and symlink protection.""" +_TAR_SUFFIXES = (".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz") + + +def is_archive(path: Path) -> bool: + name = path.name.lower() + if name.endswith(".zip"): + return zipfile.is_zipfile(path) + if name.endswith(_TAR_SUFFIXES): + try: + return tarfile.is_tarfile(path) + except OSError: + return False + return False + + +def _archive_stem(path: Path) -> str: + name = path.name + for suffix in (".zip", *_TAR_SUFFIXES): + if name.lower().endswith(suffix): + return name[: -len(suffix)] + return path.stem + + +def _safe_target(archive: Path, dest: Path, member: str) -> Path: + name = PurePosixPath(member.replace("\\", "/")) + if name.is_absolute() or ".." in name.parts or ":" in member: + raise ArchiveError(f"{archive.name}: unsafe path {member!r}") + target = (dest / Path(*name.parts)).resolve() + if dest not in target.parents: + raise ArchiveError(f"{archive.name}: unsafe path {member!r}") + return target + + +def _check_limits(archive: Path, count: int, total: int, compressed: int) -> None: + if count > MAX_ARCHIVE_FILES: + raise ArchiveError(f"{archive.name}: too many files ({count})") + if total > MAX_ARCHIVE_BYTES: + raise ArchiveError(f"{archive.name}: uncompressed size {total} exceeds limit") + if total / max(compressed, 1) > MAX_COMPRESSION_RATIO: + raise ArchiveError(f"{archive.name}: compression ratio looks like a zip bomb") + + +def _extract_tar(archive: Path, dest: Path) -> list[Path]: + """Regular files only: links, devices and fifos are skipped, never created.""" out: list[Path] = [] + try: + with tarfile.open(archive) as tf: + members = [m for m in tf.getmembers() if m.isfile()] + _check_limits(archive, len(members), sum(m.size for m in members), + archive.stat().st_size) + for m in members: + target = _safe_target(archive, dest, m.name) + src = tf.extractfile(m) + if src is None: + continue + target.parent.mkdir(parents=True, exist_ok=True) + with src, open(target, "wb") as dst: + while chunk := src.read(1 << 20): + dst.write(chunk) + out.append(target) + except (tarfile.TarError, EOFError) as e: + raise ArchiveError(f"{archive.name}: unreadable archive: {e}") from e + return out + + +def safe_extract(archive: Path, dest: Path) -> list[Path]: + """Extract a zip or tar archive with path-traversal, bomb and symlink + protection.""" dest = dest.resolve() + if not archive.name.lower().endswith(".zip"): + return _extract_tar(archive, dest) + out: list[Path] = [] with zipfile.ZipFile(archive) as zf: infos = [i for i in zf.infolist() if not i.is_dir()] - if len(infos) > MAX_ARCHIVE_FILES: - raise ArchiveError(f"{archive.name}: too many files ({len(infos)})") - total = sum(i.file_size for i in infos) - if total > MAX_ARCHIVE_BYTES: - raise ArchiveError(f"{archive.name}: uncompressed size {total} exceeds limit") - compressed = sum(i.compress_size for i in infos) or 1 - if total / compressed > MAX_COMPRESSION_RATIO: - raise ArchiveError(f"{archive.name}: compression ratio looks like a zip bomb") + _check_limits(archive, len(infos), sum(i.file_size for i in infos), + sum(i.compress_size for i in infos)) for info in infos: - name = PurePosixPath(info.filename.replace("\\", "/")) - if name.is_absolute() or ".." in name.parts or ":" in info.filename: - raise ArchiveError(f"{archive.name}: unsafe path {info.filename!r}") + target = _safe_target(archive, dest, info.filename) if (info.external_attr >> 16) & 0o170000 == 0o120000: continue # skip symlinks stored in the archive - target = (dest / Path(*name.parts)).resolve() - if dest not in target.parents: - raise ArchiveError(f"{archive.name}: unsafe path {info.filename!r}") target.parent.mkdir(parents=True, exist_ok=True) with zf.open(info) as src, open(target, "wb") as dst: while chunk := src.read(1 << 20): @@ -140,12 +199,12 @@ def safe_extract(archive: Path, dest: Path) -> list[Path]: def collect_files(path: Path, work_dir: Path, session: Any = None) -> list[Path]: - """Expand a path into evidence files (walk folders, extract zips).""" + """Expand a path into evidence files (walk folders, extract archives).""" path = Path(path) if path.is_file(): - if zipfile.is_zipfile(path) and path.suffix.lower() == ".zip": - sha = session.store.ingest(path) if session is not None else path.stem - dest = work_dir / f"{path.stem}-{sha[:12]}" + if is_archive(path): + sha = session.store.ingest(path) if session is not None else _archive_stem(path) + dest = work_dir / f"{_archive_stem(path)}-{sha[:12]}" if session is not None: session.log("pipeline", "archive_extracted", archive=path.name, sha256=sha) files: list[Path] = [] diff --git a/glaive/web/static/index.html b/glaive/web/static/index.html index 5b6063f..94de600 100644 --- a/glaive/web/static/index.html +++ b/glaive/web/static/index.html @@ -120,7 +120,7 @@