diff --git a/.github/workflows/sensor-gemini.yml b/.github/workflows/sensor-gemini.yml new file mode 100644 index 0000000..168df5c --- /dev/null +++ b/.github/workflows/sensor-gemini.yml @@ -0,0 +1,23 @@ +name: Gemini CLI parser platforms +on: + pull_request: + paths: ['Sensor/**', '.github/workflows/sensor-gemini.yml'] + push: + branches: [main] + paths: ['Sensor/**', '.github/workflows/sensor-gemini.yml'] +permissions: + contents: read +jobs: + parser: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v7 + - run: uv sync --extra dev --python 3.12 + working-directory: Sensor + - run: uv run pytest tests/test_gemini_parser.py -q + working-directory: Sensor diff --git a/README.md b/README.md index a1741af..68f62e8 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,9 @@ See **[docs/REPRODUCIBILITY.md](docs/REPRODUCIBILITY.md)** for the full evaluati Component documentation: +ADR Sensor also captures [Gemini CLI session journals](Sensor/README.md#gemini-cli) +on macOS, Linux, and Windows, including tool results and nested subagent sessions. + - [Discovery/README.md](Discovery/README.md): endpoint inventory, probes, and the fingerprint catalog - [Sensor/README.md](Sensor/README.md): telemetry collection and unified schema - [Detection/README.md](Detection/README.md): ADR-Bench, detector baselines, MCP infrastructure diff --git a/Sensor/README.md b/Sensor/README.md index a7fc469..cc6e5e1 100644 --- a/Sensor/README.md +++ b/Sensor/README.md @@ -21,6 +21,7 @@ ADR Sensor is a Python library that collects telemetry from AI coding agents to | **DeepSeek Harness** | `dsh` | JSONL/Zstandard (`~/.dsh/sessions/`) | macOS, Linux, Windows | | **Warp Terminal** | `warp` | SQLite (`warp.sqlite`) | macOS, Windows | | **opencode** | `opencode` | SQLite (`opencode.db`) or JSON tree | macOS, Linux | +| **Gemini CLI** | `gemini` | JSONL journals + legacy JSON chats | macOS, Linux, Windows | ### Claude Desktop Agent Mode @@ -119,13 +120,62 @@ adr-sensor --source dsh ``` +### Gemini CLI + +The `gemini` source reads current `chats/**/*.jsonl` journals and legacy +`chats/*.json` conversation snapshots. It captures user/assistant text, tool +arguments and results, recorded status and approval requests, model and token +usage, and nested subagent sessions. Start timestamps remain stable when sessions +resume; `--save-sessions` refreshes changed snapshots in place. + +| Host | Default scan root (per-user home) | +| ---- | -------------------------------- | +| macOS / Linux | `~/.gemini/tmp/` | +| Windows | `%USERPROFILE%\.gemini\tmp\` | +| macOS Seatbelt sandbox | `~/.cache/.gemini/tmp/` (also scanned on macOS) | + +`GEMINI_CLI_HOME` overrides the **parent home directory**, so the path becomes +`$GEMINI_CLI_HOME/.gemini/tmp`, not `$GEMINI_CLI_HOME/tmp`. Project directories +can be hashes or readable identifiers. Project paths come from `.project_root` +or `projects.json`; a project hash alone is not a filesystem path. A custom +acquired root can be supplied through `GeminiParser(base_path=Path("/capture/tmp"))`. +WSL/container sessions belong to their own filesystem and home. + +ADR consolidates repeated journal messages by ID, so tool progress updates do +not duplicate messages or token totals. Earlier activity survives rewind and +checkpoint records. `session_context.history_scope` is `all_recorded_branches`; +this is recorded activity, not a reconstruction of only the model's current +context. Source message metadata, typed content, tool IDs, and recorded thought +summaries remain in `session_context`. The parser adds no redaction or truncation; +upstream output limits and deleted files cannot be recovered. Unknown or malformed +records do not abort other sessions, and malformed-record counts are reported. + +Only CLI chat records are covered. Prompt-only `logs.json`, editor chat storage, +shell history, unsaved sessions, and files outside these roots are not collected. +The default lookback is 14 days by file modification time; `--all-history` +includes older files. Explicit `mcp_`/qualified tool names identify MCP calls; +server attribution is left empty when the recorded name is ambiguous. + +```bash +uv run adr-sensor --source gemini --no-save +uv run adr-sensor --source gemini --save-sessions --all-history +``` + +Contracts verified against upstream +[record types](https://github.com/google-gemini/gemini-cli/blob/9c1b0a610534d6f8120964cf2672c07807d8fc90/packages/core/src/services/chatRecordingTypes.ts), +[journal writer](https://github.com/google-gemini/gemini-cli/blob/9c1b0a610534d6f8120964cf2672c07807d8fc90/packages/core/src/services/chatRecordingService.ts), +[storage paths](https://github.com/google-gemini/gemini-cli/blob/9c1b0a610534d6f8120964cf2672c07807d8fc90/packages/core/src/config/storage.ts), +and [supported platforms](https://geminicli.com/docs/get-started/installation/). +Tests use synthetic records matching these contracts and run on all three hosts; +they do not require a Gemini account. + ## Architecture ``` ┌─────────────────────────────────────────────────────────────────┐ │ AI Agent Logs │ │ Claude, Cursor, Cline, Codex, Copilot CLI, Warp │ -│ Claude Desktop, opencode, DeepSeek Harness │ +│ Claude Desktop, opencode, Gemini CLI, DeepSeek Harness │ └───────────────────────────────┬─────────────────────────────────┘ ▼ ┌─────────────────────────────────────────────────────────────────┐ @@ -191,6 +241,7 @@ adr-sensor --source copilot adr-sensor --source dsh adr-sensor --source claude_desktop adr-sensor --source opencode +adr-sensor --source gemini # Save individual session files (incremental) adr-sensor --save-sessions @@ -456,6 +507,7 @@ cannot run on the current platform are skipped rather than failing. | ----------------- | -------------------------- | ----------------------------------------------------------------- | | `CODEX_HOME` | Codex parser | Codex data root containing `sessions/` and optional `state_*.sqlite` catalogs (default `~/.codex`) | | `COPILOT_HOME` | Copilot parser | Copilot CLI data root containing `session-state/` (default `~/.copilot`) | +| `GEMINI_CLI_HOME` | Gemini parser | Parent home containing `.gemini/tmp/`; on macOS also `.cache/.gemini/tmp/` | | `DSH_HOME` | DeepSeek Harness parser | Harness data root containing `sessions/` (default `~/.dsh`) | | `XDG_CACHE_HOME` | `AgentObserver` | Base for `--save-sessions` output (`$XDG_CACHE_HOME/adr_sensor`, default `~/.cache/adr_sensor`) | | `XDG_DATA_HOME` | opencode parser | Overrides the opencode data directory (default `~/.local/share/opencode`) | @@ -513,6 +565,7 @@ adr-sensor/ │ │ ├── codex_parser.py │ │ ├── copilot_parser.py │ │ ├── dsh_parser.py +│ │ ├── gemini_parser.py │ │ ├── opencode_parser.py │ │ └── warp_parser.py │ ├── schemas/ diff --git a/Sensor/adr_sensor/__init__.py b/Sensor/adr_sensor/__init__.py index 1a3be17..eb97c1e 100644 --- a/Sensor/adr_sensor/__init__.py +++ b/Sensor/adr_sensor/__init__.py @@ -3,8 +3,8 @@ Security observability library for AI coding agents. Collects telemetry from Claude Code, Cursor, Cline, OpenAI Codex CLI, GitHub Copilot CLI, DeepSeek -Harness, Warp Terminal, opencode, and Claude Desktop Agent Mode (including -Dispatch sessions) to enable threat detection and security monitoring. +Harness, Warp Terminal, opencode, Gemini CLI, and Claude Desktop Agent Mode +(including Dispatch sessions) to enable threat detection and security monitoring. Usage: from adr_sensor import AgentObserver diff --git a/Sensor/adr_sensor/cli.py b/Sensor/adr_sensor/cli.py index 492f3f5..511b763 100644 --- a/Sensor/adr_sensor/cli.py +++ b/Sensor/adr_sensor/cli.py @@ -56,6 +56,7 @@ def main(): adr-sensor --source copilot Ingest GitHub Copilot CLI logs only adr-sensor --source dsh Ingest DeepSeek Harness logs only adr-sensor --source opencode Ingest opencode logs only + adr-sensor --source gemini Ingest Gemini CLI chat sessions adr-sensor --save-sessions Save individual session files adr-sensor --output-format jsonl Export as JSONL adr-sensor --otel-config ./otel.json Export logs to an OTLP/HTTP endpoint diff --git a/Sensor/adr_sensor/observer.py b/Sensor/adr_sensor/observer.py index 19bdcc1..b8546b0 100644 --- a/Sensor/adr_sensor/observer.py +++ b/Sensor/adr_sensor/observer.py @@ -29,6 +29,7 @@ from .parsers.copilot_parser import CopilotParser from .parsers.cursor_parser import CursorParser from .parsers.dsh_parser import DshParser +from .parsers.gemini_parser import GeminiParser from .parsers.opencode_parser import OpencodeParser from .parsers.warp_parser import WarpParser from .schemas.agent_event_schema import AgentEvent @@ -63,6 +64,7 @@ class AgentObserver: ("copilot", "GitHub Copilot CLI"), ("dsh", "DeepSeek Harness"), ("opencode", "opencode"), + ("gemini", "Gemini CLI"), ) #: Sources that only produce logs on some operating systems. A source absent @@ -71,7 +73,7 @@ class AgentObserver: "claude_desktop": ("Darwin", "Windows"), } - CONTENT_AWARE_INCREMENTAL_SOURCES = frozenset({"codex", "copilot", "dsh"}) + CONTENT_AWARE_INCREMENTAL_SOURCES = frozenset({"codex", "copilot", "dsh", "gemini"}) def __init__(self, output_dir: Optional[Path] = None, max_age_days: Optional[int] = None): """Initialize the AgentObserver. @@ -97,6 +99,7 @@ def __init__(self, output_dir: Optional[Path] = None, max_age_days: Optional[int ) self.output_dir = output_dir if output_dir else Path("output") + self.gemini_parser = GeminiParser(max_age_days=max_age_days) if max_age_days is not None else GeminiParser() self.output_dir.mkdir(exist_ok=True) def _emit_error(self, error_payload: Dict[str, Any]) -> None: @@ -132,7 +135,7 @@ def ingest_all( Args: source_filter: Which source to ingest. One of 'all', 'claude', 'cursor', - 'claude_desktop', 'cline', 'warp', 'codex', 'copilot', 'dsh', 'opencode'. + 'claude_desktop', 'cline', 'warp', 'codex', 'copilot', 'dsh', 'opencode', 'gemini'. Returns: Tuple of (agent_events, system_configs). diff --git a/Sensor/adr_sensor/parsers/__init__.py b/Sensor/adr_sensor/parsers/__init__.py index 6d3f11f..f2a5091 100644 --- a/Sensor/adr_sensor/parsers/__init__.py +++ b/Sensor/adr_sensor/parsers/__init__.py @@ -12,6 +12,7 @@ from .copilot_parser import CopilotParser from .cursor_parser import CursorParser from .dsh_parser import DshParser +from .gemini_parser import GeminiParser from .opencode_parser import OpencodeParser from .warp_parser import WarpParser @@ -21,6 +22,7 @@ "ClaudeParser", "ClineParser", "CodexParser", + "GeminiParser", "CopilotParser", "CursorParser", "DshParser", diff --git a/Sensor/adr_sensor/parsers/gemini_parser.py b/Sensor/adr_sensor/parsers/gemini_parser.py new file mode 100644 index 0000000..5f336f3 --- /dev/null +++ b/Sensor/adr_sensor/parsers/gemini_parser.py @@ -0,0 +1,291 @@ +"""Read Gemini CLI chat journals and legacy conversation snapshots. + +The source contract is Google's chatRecordingTypes.ts / chatRecordingService.ts. +Journals upsert complete messages by ID; ADR retains activity across rewinds +and checkpoints because removing model context does not undo executed actions. +""" + +import json +import os +import platform +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +from ..schemas.agent_event_schema import AgentEvent, ChatMessage, ToolUsage +from ..utils.timestamp_utils import normalize_timestamp +from .base_parser import BaseParser + + +class GeminiParser(BaseParser): + """Capture persisted Gemini CLI messages, tools, metadata and subagents.""" + + def __init__(self, max_age_days: int = 14, base_path: Optional[Path] = None): + home_override = os.environ.get("GEMINI_CLI_HOME") + home = Path(home_override).expanduser() if home_override else Path.home() + self.base_paths = [home / ".gemini" / "tmp"] + if platform.system() == "Darwin": + self.base_paths.append(home / ".cache" / ".gemini" / "tmp") + if base_path is not None: + self.base_paths = [Path(base_path).expanduser()] + self.max_age_days = max_age_days + + def parse_all(self) -> List[AgentEvent]: + entries: Dict[str, AgentEvent] = {} + cutoff = (datetime.now(timezone.utc) - timedelta(days=self.max_age_days)).timestamp() + seen_paths = set() + for base in self.base_paths: + if not base.is_dir(): + continue + for chats in sorted(base.glob("*/chats")): + for path in sorted(chats.rglob("*")): + if path.suffix not in {".json", ".jsonl"}: + continue + try: + if not path.is_file() or path.resolve() in seen_paths: + continue + seen_paths.add(path.resolve()) + if self.max_age_days > 0 and path.stat().st_mtime < cutoff: + continue + entry = self.parse_file(path) + if entry is None or not entry.has_meaningful_content(): + continue + old = entries.get(entry.session_id) + # Legacy .json can remain after migration to .jsonl. + if old is None or self._revision(entry) > self._revision(old): + entries[entry.session_id] = entry + except (OSError, ValueError) as exc: + print(f"[GEMINI] Unable to read {path}: {exc}") + return list(entries.values()) + + @staticmethod + def _revision(entry: AgentEvent) -> tuple: + context = entry.session_context or {} + return ( + normalize_timestamp(context["last_event_at"]), + str(entry.raw_log_path).endswith(".jsonl"), + context["event_count"], + ) + + @staticmethod + def _timestamp(value: Any) -> Optional[datetime]: + if value is None or isinstance(value, bool): + return None + try: + return normalize_timestamp(value) + except (TypeError, ValueError, OverflowError, OSError): + return None + + def parse_file(self, path: Path) -> Optional[AgentEvent]: + """Normalize a file without changing it or applying content redaction.""" + metadata: Dict[str, Any] = {} + messages: Dict[str, dict] = {} + controls = [] + permissions = [] + timestamps = [] + event_count = 0 + malformed = 0 + + def add_message(message: Any) -> None: + if not isinstance(message, dict) or not isinstance(message.get("id"), str): + return + messages[message["id"]] = message + timestamp = self._timestamp(message.get("timestamp")) + if timestamp: + timestamps.append(timestamp) + calls = message.get("toolCalls") + for call in calls if isinstance(calls, list) else []: + if not isinstance(call, dict): + continue + timestamp = self._timestamp(call.get("timestamp")) + if timestamp: + timestamps.append(timestamp) + if call.get("status") == "awaiting_approval": + permission = {"message_id": message["id"], "tool_call": call} + if permission not in permissions: + permissions.append(permission) + + try: + # Capture before reading: appended records can advance the revision, + # but later writes must not give a partial read a newer file timestamp. + modified_at = datetime.fromtimestamp(path.stat().st_mtime, timezone.utc) + with path.open(encoding="utf-8") as handle: + if path.suffix == ".json": + records = [json.load(handle)] + else: + records = [] + for line in handle: + if not line.strip(): + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError: + malformed += 1 + for record in records: + if not isinstance(record, dict): + malformed += 1 + continue + event_count += 1 + if "$rewindTo" in record: + controls.append(record) + continue + if "id" in record: + add_message(record) + continue + update = record.get("$set", record) + if not isinstance(update, dict): + malformed += 1 + continue + if "$set" in record: + controls.append({"$set": {k: v for k, v in update.items() if k != "messages"}}) + metadata.update({k: v for k, v in update.items() if k != "messages"}) + checkpoint = update.get("messages") + for message in checkpoint if isinstance(checkpoint, list) else []: + add_message(message) + except (OSError, UnicodeError, ValueError) as exc: + print(f"[GEMINI] Unable to parse {path}: {exc}") + return None + + session_id = metadata.get("sessionId") + if not isinstance(session_id, str) or not session_id: + return None + history = [] + message_metadata = {} + model = None + usage = {} + for message_id, message in messages.items(): + role = message.get("type") + details = {k: v for k, v in message.items() if k not in {"content", "toolCalls"}} + if not isinstance(message.get("content", ""), str): + details["content_parts"] = message.get("content") + message_metadata[message_id] = details + if role not in {"user", "gemini"}: + details["content"] = message.get("content") + continue + calls = message.get("toolCalls") + tools = [] + details["tool_metadata"] = [] + for call in calls if isinstance(calls, list) else []: + if not isinstance(call, dict) or not isinstance(call.get("name"), str): + continue + tools.append(self._tool(call)) + details["tool_metadata"].append({k: v for k, v in call.items() if k not in {"args", "result"}}) + content = self._text(message.get("content")) + if content or tools or isinstance(message.get("content"), (dict, list)) and message["content"]: + history.append( + ChatMessage( + role="assistant" if role == "gemini" else "user", + content=content, + tools=tools, + sequence_id=message_id, + ) + ) + if role == "gemini": + if isinstance(message.get("model"), str): + model = message["model"] + tokens = message.get("tokens") + if isinstance(tokens, dict): + for key, value in tokens.items(): + if isinstance(value, (int, float)) and not isinstance(value, bool) and value >= 0: + usage[key] = usage.get(key, 0) + value + + if not history: + return None + updated = self._timestamp(metadata.get("lastUpdated")) + if updated: + timestamps.append(updated) + timestamp = self._timestamp(metadata.get("startTime")) or (min(timestamps) if timestamps else modified_at) + context = { + "last_event_at": (max(timestamps) if timestamps else modified_at).isoformat(), + "event_count": event_count, + "session_metadata": metadata, + "message_metadata": message_metadata, + "history_scope": "all_recorded_branches", + "journal_controls": controls, + "permission_requests": permissions, + "malformed_records": malformed, + } + if metadata.get("kind") == "subagent" and path.parent.name != "chats": + context["parent_session_id"] = path.parent.name + if malformed: + print(f"[GEMINI] Skipped {malformed} malformed records in {path}") + return AgentEvent( + timestamp=timestamp, + source="gemini", + session_id=f"gemini_{session_id}", + project_path=self._project_path(path), + model=model, + chat_history=history, + raw_log_path=str(path), + session_context=context, + token_usage={ + "cumulative": { + target: usage[source] + for source, target in ( + ("input", "input_tokens"), + ("output", "output_tokens"), + ("cached", "cached_input_tokens"), + ("thoughts", "reasoning_output_tokens"), + ("tool", "tool_tokens"), + ("total", "total_tokens"), + ) + if source in usage + } + } + if usage + else None, + ) + + @staticmethod + def _project_path(path: Path) -> Optional[str]: + chats = next((parent for parent in path.parents if parent.name == "chats"), None) + if chats is None: + return None + project = chats.parent + try: + marker = (project / ".project_root").read_text(encoding="utf-8").strip() + if marker: + return marker + except (OSError, UnicodeError): + pass + try: + registry = json.loads((project.parent.parent / "projects.json").read_text(encoding="utf-8")) + projects = registry.get("projects", {}) if isinstance(registry, dict) else {} + if isinstance(projects, dict): + return next((key for key, value in projects.items() if value == project.name), None) + except (OSError, UnicodeError, ValueError): + pass + return None + + @staticmethod + def _text(content: Any) -> str: + if isinstance(content, str): + return content + parts = content if isinstance(content, list) else [content] + return "\n".join( + part if isinstance(part, str) else part["text"] + for part in parts + if isinstance(part, str) or isinstance(part, dict) and isinstance(part.get("text"), str) + ) + + @staticmethod + def _tool(call: dict) -> ToolUsage: + name = call["name"] + server = None + # Only the explicit qualified separator is unambiguous; older bare + # names and truncated names do not establish server identity. + if "__" in name and "..." not in name: + server = name.removeprefix("mcp_").split("__", 1)[0] or None + status = call.get("status") if isinstance(call.get("status"), str) else None + result = call.get("result") + serialized = result if isinstance(result, str) else json.dumps(result, ensure_ascii=False) + args = call.get("args", {}) + return ToolUsage( + tool_name=name, + tool_type="mcp_tool" if server or name.startswith("mcp_") else "function_call", + server_name=server, + arguments=args if isinstance(args, dict) else {"raw": args}, + result=serialized if result is not None else None, + status=status, + error=serialized if status == "error" and result is not None else None, + ) diff --git a/Sensor/adr_sensor/schemas/agent_event_schema.py b/Sensor/adr_sensor/schemas/agent_event_schema.py index 65cb74a..acf64ec 100644 --- a/Sensor/adr_sensor/schemas/agent_event_schema.py +++ b/Sensor/adr_sensor/schemas/agent_event_schema.py @@ -1,7 +1,7 @@ """ Agent Event Schema for AI agent telemetry ingestion. Normalizes logs from Claude Code, Cursor, Cline, Codex, Copilot CLI, DeepSeek -Harness, Warp, opencode, and Claude Desktop into a common format for security analysis. +Harness, Warp, opencode, Gemini CLI, and Claude Desktop into a common format for security analysis. """ import hashlib @@ -46,7 +46,7 @@ class AgentEvent: # Core fields timestamp: datetime - source: str # claude, cursor, cline, warp, codex, copilot, dsh, claude_desktop, opencode + source: str # claude, cursor, cline, warp, codex, copilot, dsh, claude_desktop, opencode, gemini session_id: str # Chat history diff --git a/Sensor/tests/test_gemini_parser.py b/Sensor/tests/test_gemini_parser.py new file mode 100644 index 0000000..838d9d8 --- /dev/null +++ b/Sensor/tests/test_gemini_parser.py @@ -0,0 +1,198 @@ +"""Synthetic fixtures matching Google's persisted ConversationRecord contract.""" + +import json +import os +from copy import deepcopy +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from adr_sensor.observer import AgentObserver +from adr_sensor.parsers.gemini_parser import GeminiParser + + +def records(session_id="main-session"): + return [ + {"sessionId": session_id, "projectHash": "hash", "startTime": "2026-01-01T00:00:00Z", "kind": "main"}, + {"id": "user1", "type": "user", "timestamp": "2026-01-01T00:00:01Z", "content": "Inspect this project"}, + { + "id": "assistant1", + "type": "gemini", + "timestamp": "2026-01-01T00:00:02Z", + "content": [{"text": "Reading"}], + "model": "gemini-test", + "thoughts": [{"subject": "Plan", "description": "Inspect files"}], + "tokens": {"input": 10, "output": 2, "cached": 3, "total": 12}, + "toolCalls": [ + { + "id": "call1", + "name": "mcp_test_server__read", + "args": {"path": "config.json"}, + "timestamp": "2026-01-01T00:00:02Z", + "status": "awaiting_approval", + } + ], + }, + ] + + +def write(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(json.dumps(row) for row in data) + "\n", encoding="utf-8") + return path + + +@pytest.mark.parametrize("host", ["Darwin", "Linux", "Windows"]) +@pytest.mark.parametrize("override", [False, True]) +def test_home_resolution(tmp_path, monkeypatch, host, override): + monkeypatch.setattr("adr_sensor.parsers.gemini_parser.platform.system", lambda: host) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.setenv("GEMINI_CLI_HOME", str(tmp_path / "custom") if override else "") + home = tmp_path / "custom" if override else tmp_path + parser = GeminiParser() + assert parser.base_paths[0] == home / ".gemini/tmp" + assert len(parser.base_paths) == (2 if host == "Darwin" else 1) + if host == "Darwin": + assert parser.base_paths[1] == home / ".cache/.gemini/tmp" + + +def test_journal_updates_keep_results_permissions_and_metadata(tmp_path): + rows = records() + finished = deepcopy(rows[-1]) + finished["toolCalls"][0].update( + status="success", + result=[ + {"functionResponse": {"id": "call1", "name": "read", "response": {"output": "secret-value-" + "x" * 4000}}} + ], + ) + rows.extend([finished, {"$set": {"lastUpdated": "2026-01-01T00:00:03Z"}}]) + path = write(tmp_path / "project/chats/session-test.jsonl", rows) + (tmp_path / "project/.project_root").write_text("C:\\work\\project", encoding="utf-8") + before = path.read_bytes(), path.stat().st_mtime_ns + event = GeminiParser(base_path=tmp_path).parse_all()[0] + assert len(event.chat_history) == 2 + assert event.project_path == "C:\\work\\project" + tool = event.chat_history[-1].tools[0] + assert tool.server_name == "test_server" + assert tool.status == "success" + assert "secret-value-" + "x" * 4000 in tool.result + assert tool.arguments == {"path": "config.json"} + assert event.model == "gemini-test" + assert event.token_usage["cumulative"]["input_tokens"] == 10 # Upserts must not double count. + assert event.session_context["permission_requests"][0]["tool_call"]["id"] == "call1" + assert event.session_context["message_metadata"]["assistant1"]["thoughts"][0]["subject"] == "Plan" + assert (path.read_bytes(), path.stat().st_mtime_ns) == before + + +def test_legacy_snapshot_and_jsonl_migration_deduplicate(tmp_path): + rows = records() + legacy = dict(rows[0], messages=rows[1:]) + path = tmp_path / "project/chats/session-old.json" + path.parent.mkdir(parents=True) + path.write_text(json.dumps(legacy, indent=2), encoding="utf-8") + parser = GeminiParser(base_path=tmp_path) + assert len(parser.parse_all()[0].chat_history) == 2 + journal = write(path.with_suffix(".jsonl"), rows) + events = parser.parse_all() + assert len(events) == 1 + assert events[0].raw_log_path == str(journal) + + +def test_rewind_checkpoint_subagent_and_non_chat_files(tmp_path): + rows = records() + rows += [ + {"$rewindTo": "assistant1"}, + {"$set": {"messages": [rows[1]]}}, + {"id": "user2", "type": "user", "content": "Try another approach"}, + ] + write(tmp_path / "project/chats/session-main.jsonl", rows) + child = records("child-session") + child[0]["kind"] = "subagent" + write(tmp_path / "project/chats/main-session/child-session.jsonl", child) + write(tmp_path / "project/logs/unrelated.jsonl", records("ignore")) + events = {event.session_id: event for event in GeminiParser(base_path=tmp_path).parse_all()} + assert set(events) == {"gemini_main-session", "gemini_child-session"} + assert len(events["gemini_main-session"].chat_history) == 3 + assert events["gemini_main-session"].chat_history[1].tools # Abandoned actions survive. + assert events["gemini_child-session"].session_context["parent_session_id"] == "main-session" + + +def test_missing_malformed_and_age_filtering(tmp_path): + parser = GeminiParser(base_path=tmp_path) + assert parser.parse_all() == [] + path = write(tmp_path / "project/chats/session-old.jsonl", records()) + with path.open("a", encoding="utf-8") as handle: + handle.write('[]\n{"$set": null}\n{"unfinished":') + event = parser.parse_all()[0] + assert event.session_context["malformed_records"] == 3 + os.utime(path, (1, 1)) + assert parser.parse_all() == [] + assert len(GeminiParser(max_age_days=0, base_path=tmp_path).parse_all()) == 1 + path.write_text("{}", encoding="utf-8") + assert parser.parse_all() == [] + + +def test_resumed_export_updates_same_file_and_skips_unchanged(tmp_path): + path = write(tmp_path / "input/project/chats/session.jsonl", records()) + parser = GeminiParser(base_path=tmp_path / "input") + observer = AgentObserver(output_dir=tmp_path / "output", max_age_days=0) + observer.gemini_parser = parser + event = observer.ingest_all("gemini")[0][0] + assert observer.gemini_parser.max_age_days == 14 + output = observer.save_sessions_to_individual_files([event], tmp_path / "output")[0] + assert observer.filter_entries_by_existing_files([event], tmp_path / "output") == [] + resumed = {"id": "user2", "type": "user", "timestamp": "2026-01-02T00:00:00Z", "content": "Continue work"} + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(resumed) + "\n") + updated = parser.parse_all()[0] + assert updated.timestamp == datetime(2026, 1, 1, tzinfo=timezone.utc) + selected = observer.filter_entries_by_existing_files([updated], tmp_path / "output") + assert len(selected) == 1 + assert observer.save_sessions_to_individual_files(selected, tmp_path / "output") == [output] + assert len(json.loads(output.read_text(encoding="utf-8"))["chat_history"]) == 3 + + +def test_status_and_structured_content_are_preserved(tmp_path): + rows = records() + rows[1]["content"] = [{"text": "Inspect image"}, {"inlineData": {"mimeType": "image/png", "data": "abc"}}] + rows[-1]["toolCalls"] = [ + { + "id": "fail", + "name": "run_shell_command", + "args": {"command": "false"}, + "status": "error", + "result": [{"functionResponse": {"response": {"error": "denied"}}}], + }, + {"id": "cancel", "name": "read_file", "args": {}, "status": "cancelled"}, + {"id": "pending", "name": "custom", "args": {}, "status": "executing"}, + ] + event = GeminiParser().parse_file(write(tmp_path / "session.jsonl", rows)) + tools = event.chat_history[-1].tools + assert tools[0].status == "error" and "denied" in tools[0].error + assert tools[1].status == "cancelled" and tools[1].result is None + assert tools[2].status == "executing" and tools[2].result is None + assert event.session_context["message_metadata"]["user1"]["content_parts"][1]["inlineData"]["data"] == "abc" + + +def test_project_registry_and_corrupt_optional_metadata(tmp_path): + path = write(tmp_path / "tmp/project/chats/session.jsonl", records()) + registry = tmp_path / "projects.json" + registry.write_text(json.dumps({"projects": {"/work/repo": "project"}}), encoding="utf-8") + parser = GeminiParser(base_path=tmp_path / "tmp") + assert parser.parse_all()[0].project_path == "/work/repo" + registry.write_text("{", encoding="utf-8") + assert parser.parse_all()[0].project_path is None + assert path.exists() + + +def test_mixed_string_and_object_content_parts_and_image_only_messages(tmp_path): + rows = records() + rows[1]["content"] = ["Inspect this image", {"text": "and report findings"}] + image_part = {"inlineData": {"mimeType": "image/png", "data": "abc"}} + rows.append({"id": "image", "type": "user", "content": image_part}) + event = GeminiParser().parse_file(write(tmp_path / "session.jsonl", rows)) + assert event.chat_history[0].content == "Inspect this image\nand report findings" + assert event.chat_history[-1].sequence_id == "image" + assert event.chat_history[-1].role == "user" and event.chat_history[-1].content == "" + assert event.session_context["message_metadata"]["image"]["content_parts"] == image_part