diff --git a/.github/scripts/next-release-version.sh b/.github/scripts/next-release-version.sh new file mode 100755 index 000000000..f8530e473 --- /dev/null +++ b/.github/scripts/next-release-version.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Computes the next release version for the Release workflow. +# +# usage: next-release-version.sh [tag ...] +# +# The base is the highest of the workspace version and every plain `vX.Y.Z` +# tag given. The workspace version alone is not a safe base: releases are +# published from a tag, and the version in Cargo.toml can lag the latest tag +# (it sat at 2.1.4 while v2.2.0 through v2.8.0 existed, so the workflow kept +# proposing a tag that was already taken). Prints ` `. +set -euo pipefail + +bump="${1:?bump (patch|minor|major) required}" +workspace_version="${2:?workspace version required}" +shift 2 + +semver='^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$' +if ! [[ "$workspace_version" =~ $semver ]]; then + echo "Workspace version '${workspace_version}' is not X.Y.Z" >&2 + exit 1 +fi + +base="$workspace_version" +for tag in "$@"; do + version="${tag#v}" + # Pre-release and other non-release tags never set the base. + if [[ "$tag" == v* && "$version" =~ $semver ]]; then + base="$(printf '%s\n%s\n' "$base" "$version" | sort -V | tail -n1)" + fi +done + +IFS=. read -r major minor patch <<< "$base" +case "$bump" in + major) + major=$((major + 1)) + minor=0 + patch=0 + ;; + minor) + minor=$((minor + 1)) + patch=0 + ;; + patch) + patch=$((patch + 1)) + ;; + *) + echo "Unsupported bump: ${bump}" >&2 + exit 1 + ;; +esac + +echo "${base} ${major}.${minor}.${patch}" diff --git a/.github/scripts/next-release-version.test.sh b/.github/scripts/next-release-version.test.sh new file mode 100755 index 000000000..f3d75b917 --- /dev/null +++ b/.github/scripts/next-release-version.test.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Tests for next-release-version.sh. Run: .github/scripts/next-release-version.test.sh +set -euo pipefail + +script="$(cd "$(dirname "$0")" && pwd)/next-release-version.sh" +failures=0 + +expect() { + local name="$1" want="$2" + shift 2 + local got + if got="$("$script" "$@" 2>/dev/null)"; then :; else got="error"; fi + if [[ "$got" != "$want" ]]; then + echo "FAIL ${name}: want '${want}', got '${got}'" >&2 + failures=$((failures + 1)) + fi +} + +# The regression: a stale workspace version must not hide newer tags. +expect "stale workspace minor" "2.8.0 2.9.0" minor 2.1.4 v2.1.4 v2.2.0 v2.7.0 v2.8.0 +expect "stale workspace patch" "2.8.0 2.8.1" patch 2.1.4 v2.8.0 v2.2.0 +expect "stale workspace major" "2.8.0 3.0.0" major 2.1.4 v2.8.0 +# Numeric, not lexical, ordering. +expect "numeric ordering" "2.10.0 2.11.0" minor 2.1.4 v2.9.0 v2.10.0 +# A workspace version ahead of every tag still wins. +expect "workspace ahead" "3.0.0 3.1.0" minor 3.0.0 v2.8.0 +expect "no tags" "2.1.4 2.1.5" patch 2.1.4 +# Pre-release and foreign tags never set the base. +expect "prerelease ignored" "2.8.0 2.9.0" minor 2.1.4 v2.8.0 v3.0.0-rc1 nightly 9.9.9 +# A leading-zero tag is not a release version (08 would be octal). +expect "leading zero ignored" "2.8.0 2.9.0" minor 2.1.4 v2.8.0 v2.08.0 +# Bad input fails. +expect "bad bump" "error" hotfix 2.1.4 +expect "bad workspace version" "error" minor 2.1 v2.8.0 + +if ((failures > 0)); then + echo "${failures} failure(s)" >&2 + exit 1 +fi +echo "next-release-version: ok" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23c397c4f..06f3aea97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,9 @@ jobs: - name: Check formatting run: cargo fmt --all -- --check + - name: Release version script + run: .github/scripts/next-release-version.test.sh + - name: Clippy run: cargo clippy --workspace --all-targets -- -D warnings diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bec23a7d1..ac41d5192 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -85,30 +85,15 @@ jobs: exit 1 fi - IFS=. read -r major minor patch <<< "$current_version" - case "${{ inputs.bump }}" in - major) - major=$((major + 1)) - minor=0 - patch=0 - ;; - minor) - minor=$((minor + 1)) - patch=0 - ;; - patch) - patch=$((patch + 1)) - ;; - *) - echo "Unsupported bump: ${{ inputs.bump }}" >&2 - exit 1 - ;; - esac - - next_version="${major}.${minor}.${patch}" + # The workspace version can lag the latest tag (releases are cut + # from tags), so the next version is derived from the highest of + # the two; see the script. + git fetch --tags origin + mapfile -t existing_tags < <(git tag --list 'v*') + versions="$(.github/scripts/next-release-version.sh "${{ inputs.bump }}" "$current_version" "${existing_tags[@]}")" + read -r base_version next_version <<< "$versions" tag="v${next_version}" - git fetch --tags origin if git rev-parse --verify --quiet "refs/tags/${tag}"; then echo "Tag ${tag} already exists" >&2 exit 1 @@ -116,6 +101,7 @@ jobs: { echo "current_version=${current_version}" + echo "base_version=${base_version}" echo "next_version=${next_version}" echo "tag=${tag}" } >> "$GITHUB_OUTPUT" diff --git a/CHANGELOG.md b/CHANGELOG.md index f75606e13..0f2263cfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,55 @@ # Changelog +## Unreleased: the path-based transcript API is removed (tinyagents#453) + +**Breaking.** `tinyagents-session` no longer reads or writes a workspace +directory for transcripts. Everything goes through the store port +(`TranscriptLocator`/`TranscriptHistory`/`TranscriptRead`, served by +`DriverSessionStores`, `WorkspaceStores` or `InMemorySessionStores`); the only +reader of classic `session_raw/*.jsonl` files is the one-time, read-only +`legacy::import_transcripts_once` / `WorkspaceStores`, and +`legacy::rewrite_legacy_transcripts` for pre-import fixups. + +Removed from `tinyagents_session::transcript`: `FileTranscriptLocator`, +`FileTranscriptHistory`, `AgentTranscriptFiles`, `agent_transcript_root`, +`write_transcript*`, `append_transcript_turn*`, `append_interrupted_partial`, +`append_tools_record`, `read_transcript`, `read_transcript_display`, +`read_transcript_legacy_md`, `adopt_legacy_session_transcripts`, +`migrate_layout_if_needed`, `find_latest_transcript`, +`find_root_transcript_for_thread*`, `find_root_transcripts_for_thread`, +`read_thread_usage_summary` (and `ThreadUsageSummary`/`SubagentArchetypeUsage`), +`resolve_keyed_transcript_path*`, `append_background_message` and its +lock-notification variant, `TranscriptLocator::file_backing`, `view::get_page`, +`view::get_page_scoped`, `view::project_thread*`, `view::resolve_files_scoped`, +the view's projection cache, and `thread_spend`. Replacements: +`append_background_message_in`, `view::get_page_in`, +`view::project_thread_in(locator, thread_id, agent_id, lookup)`, +`spend::thread_spend_in`, `TranscriptLocator::thread_transcripts` / +`root_for_thread_scoped` / `read_session_transcript`, and +`TranscriptLocator::adopt_legacy` (driver stores). +`SessionAdoption`, `SessionTurnGuard`, `lock_session_turn` and the rest of the +locator-based surface are unchanged. The `fs2` dependency is gone. + +Fixes (the two deferred tinyagents#463 findings): + +- A background append's idempotency check considered every row of the head + generation's log, so a key dropped by a compaction *inside the same + generation* still reported `Duplicate` and the delivery was lost. It now + looks at the generation's live context only (rows after the last compaction + plus that compaction's replacement), which is what the contract always said: + a key a compaction dropped is appended again. +- The driver-backed thread listing read the index alone, so a root whose + index refresh failed stayed invisible until something read or wrote that + transcript again. A listing now compares written index documents with + written transcript logs and re-indexes any missing stem before it answers. +- `legacy::import_transcripts` now carries a classic line's top-level + `background` provenance onto the imported row, so an out-of-band delivery + keeps its origin and idempotency key after import. + +The Release workflow derives the next version from the highest of the +workspace version and the latest `v*` tag instead of the (stale) workspace +version alone, so it no longer proposes a tag that already exists. + ## Unreleased: locator-based transcript reads and writes (tinyagents#453) Hosts can now do through a `TranscriptLocator` everything they used the @@ -27,8 +77,7 @@ driver store: - `legacy::rewrite_legacy_transcripts(workspace_dir, rewrite) -> LegacyRewriteReport` for pre-import fixups of classic `session_raw/` files. -The path API is unchanged; `project_from_files` and `thread_spend` now share -the locator code path. +(The path API was still present at this point and was removed in the entry above.) Review follow-ups: `rewrite_legacy_transcripts` keeps a transcript's recorded tool declarations, counts unreadable directories and entries in `errors` diff --git a/Cargo.lock b/Cargo.lock index 01a22064f..03914ac22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2003,7 +2003,6 @@ version = "2.1.4" dependencies = [ "anyhow", "async-trait", - "chrono", "dotenvy", "futures", "reqwest", @@ -2099,6 +2098,7 @@ dependencies = [ "tinyagents-registry", "tinyagents-session", "tinyinference-llm", + "tinystoragedrivers-core", "tinytools", "tokio", "tracing", @@ -2111,7 +2111,6 @@ dependencies = [ "anyhow", "async-trait", "chrono", - "fs2", "hex", "parking_lot", "serde", @@ -2120,6 +2119,7 @@ dependencies = [ "tempfile", "thiserror", "tinyagents-harness", + "tinyagents-session", "tinyagents-tasks", "tinystoragedrivers-core", "tinystoragedrivers-sqlite", diff --git a/crates/tinyagents-integration-tests/Cargo.toml b/crates/tinyagents-integration-tests/Cargo.toml index a9b17f6a4..e363fa398 100644 --- a/crates/tinyagents-integration-tests/Cargo.toml +++ b/crates/tinyagents-integration-tests/Cargo.toml @@ -10,7 +10,6 @@ publish = false [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } -chrono = { workspace = true } dotenvy = "0.15" futures = { workspace = true } reqwest = { workspace = true } diff --git a/crates/tinyagents-integration-tests/tests/feature_session_transcript.rs b/crates/tinyagents-integration-tests/tests/feature_session_transcript.rs index 8839f29e6..416d238d5 100644 --- a/crates/tinyagents-integration-tests/tests/feature_session_transcript.rs +++ b/crates/tinyagents-integration-tests/tests/feature_session_transcript.rs @@ -1,13 +1,17 @@ -//! End-to-end contract coverage for append-only session transcripts. +//! End-to-end contract coverage for append-only session transcripts, through +//! the storage-driver store (SQLite) a host installs. -use chrono::Local; +mod common; + +use common::storage; use serde_json::json; +use tinyagents_session::port::{DriverSessionStores, SessionStoreProvider}; +use tinyagents_session::thread_spend_in; use tinyagents_session::transcript::{ - DisplayRecord, FileTranscriptHistory, MessageUsage, TranscriptHistory, TranscriptMessage, - TranscriptMeta, TranscriptRead, TranscriptToolCall, TurnUsage, append_interrupted_partial, - append_transcript_turn, find_root_transcript_for_thread, read_thread_usage_summary, - read_transcript, read_transcript_display, + DisplayRecord, MessageUsage, TranscriptMessage, TranscriptMeta, TranscriptPartial, + TranscriptToolCall, TranscriptTurn, TurnUsage, }; +use tinystoragedrivers_sqlite::SqliteStorage; fn meta(turn_count: usize, input_tokens: u64, output_tokens: u64) -> TranscriptMeta { TranscriptMeta { @@ -58,13 +62,18 @@ fn usage(iteration: u32, input: u64, output: u64) -> TurnUsage { #[test] fn append_only_transcript_reopens_with_compacted_context_and_full_display_history() { - let workspace = tempfile::tempdir().unwrap(); + let database = tempfile::tempdir().unwrap(); + let open = || { + DriverSessionStores::new(std::sync::Arc::new( + SqliteStorage::open(database.path().join("sessions.sqlite")).unwrap(), + )) + .unwrap() + .for_agent("researcher") + .transcripts + }; + let locator = open(); let stem = "1700000000_researcher"; - let path = workspace - .path() - .join("session_raw") - .join(format!("{stem}.jsonl")); - let companion_date = Local::now().format("%Y_%m_%d").to_string(); + let history = locator.open_stem(stem, meta(0, 0, 0)).unwrap(); let initial = vec![ TranscriptMessage { @@ -86,15 +95,29 @@ fn append_only_transcript_reopens_with_compacted_context_and_full_display_histor TranscriptMessage::new("user", "Find the durable transcript contract."), TranscriptMessage::assistant("I will search the contract."), ]; - append_transcript_turn( - &path, + let turn = |prev: &[TranscriptMessage], + next: &[TranscriptMessage], + meta: &TranscriptMeta, + usage: &TurnUsage, + request_id: &str| { + history + .append_turn(TranscriptTurn { + prev, + next, + meta, + turn_usage: Some(usage), + request_id: Some(request_id), + tools: None, + }) + .unwrap(); + }; + turn( &[], &initial, &meta(1, 8, 3), - Some(&usage(1, 8, 3)), - Some("request-initial"), - ) - .unwrap(); + &usage(1, 8, 3), + "request-initial", + ); let extended = [ initial.clone(), @@ -104,40 +127,49 @@ fn append_only_transcript_reopens_with_compacted_context_and_full_display_histor ], ] .concat(); - append_transcript_turn( - &path, + turn( &initial, &extended, &meta(2, 15, 7), - Some(&usage(2, 7, 4)), - Some("request-extension"), - ) - .unwrap(); + &usage(2, 7, 4), + "request-extension", + ); let compacted = vec![ TranscriptMessage::new("system", "Preserve source provenance."), TranscriptMessage::assistant("Summary: transcript persistence is append-only."), TranscriptMessage::new("user", "Continue from the compacted context."), ]; - append_transcript_turn( - &path, + turn( &extended, &compacted, &meta(3, 21, 9), - Some(&usage(3, 6, 2)), - Some("request-compaction"), - ) - .unwrap(); - append_interrupted_partial( - &path, - "This partial must not reach a resumed model.", - Some("request-interrupted"), - Some(4), - Some("stream was cancelled"), - ) - .unwrap(); + &usage(3, 6, 2), + "request-compaction", + ); + assert!( + locator + .append_interrupted_partial( + "thread-transcript-e2e", + None, + &TranscriptPartial { + content: "This partial must not reach a resumed model.".into(), + reasoning_content: Some("stream was cancelled".into()), + iteration: Some(4), + }, + Some("request-interrupted"), + ) + .unwrap() + ); - let replayed = read_transcript(&path).unwrap(); + // A fresh handle over the same database, as after a restart. + drop(history); + drop(locator); + let locator = open(); + let reopened = locator + .root_for_thread("thread-transcript-e2e") + .expect("the thread's root is found by lookup"); + let replayed = reopened.read_session().unwrap().unwrap(); assert_eq!(replayed.meta.turn_count, 3); assert_eq!(replayed.meta.input_tokens, 21); assert_eq!(replayed.meta.output_tokens, 9); @@ -159,7 +191,7 @@ fn append_only_transcript_reopens_with_compacted_context_and_full_display_histor .all(|message| !message.content.contains("partial must not")) ); - let display = read_transcript_display(&path).unwrap(); + let display = reopened.read_display().unwrap().unwrap(); assert_eq!(display.records.len(), 7); assert!(matches!( &display.records[0], @@ -191,39 +223,13 @@ fn append_only_transcript_reopens_with_compacted_context_and_full_display_histor && message.reasoning_content.as_deref() == Some("stream was cancelled") )); - let markdown = workspace.path().join("sessions").join(companion_date); - let markdown = markdown.join(format!("{stem}.md")); - assert!( - markdown.is_file(), - "missing companion: {}", - markdown.display() - ); - assert!( - std::fs::read_to_string(&markdown) - .unwrap() - .contains("Summary: transcript persistence is append-only.") - ); - - assert_eq!( - find_root_transcript_for_thread(workspace.path(), "thread-transcript-e2e"), - Some(path.clone()) - ); - let summary = read_thread_usage_summary(workspace.path(), "thread-transcript-e2e").unwrap(); - assert_eq!(summary.input_tokens, 21); - assert_eq!(summary.output_tokens, 9); - assert_eq!(summary.cached_input_tokens, 3); - assert_eq!(summary.turn_count, 3); - assert_eq!(summary.last_turn_input_tokens, 6); - assert_eq!(summary.last_turn_output_tokens, 2); - assert_eq!(summary.model.as_deref(), Some("fixture-model")); - assert!(summary.subagents.is_empty()); - - let reopened = FileTranscriptHistory::new(workspace.path(), stem, meta(0, 0, 0)).unwrap(); - assert_eq!(reopened.path(), path); - let reopened_session = TranscriptRead::read_session(&reopened).unwrap().unwrap(); - assert_eq!(reopened_session.messages, replayed.messages); - assert_eq!( - TranscriptHistory::messages(&reopened).unwrap(), - replayed.messages - ); + let spend = thread_spend_in(locator.as_ref(), "thread-transcript-e2e"); + assert!(spend.found_transcript); + assert_eq!(spend.root.input_tokens, 8 + 7 + 6); + assert_eq!(spend.root.output_tokens, 3 + 4 + 2); + assert_eq!(spend.root.turns, 3); + assert_eq!(spend.root.last_input_tokens, 6); + assert_eq!(spend.root.last_output_tokens, 2); + assert_eq!(spend.root.model.as_deref(), Some("fixture-model")); + assert!(spend.subagents.is_empty()); } diff --git a/crates/tinyagents-integration-tests/tests/session_conformance.rs b/crates/tinyagents-integration-tests/tests/session_conformance.rs index 20535c3ee..bf3e81860 100644 --- a/crates/tinyagents-integration-tests/tests/session_conformance.rs +++ b/crates/tinyagents-integration-tests/tests/session_conformance.rs @@ -4,7 +4,7 @@ //! the same contract suite is run against every backend the crate owns, so a //! defect in one cannot hide behind another. The run ledger is certified on the //! memory and sqlite drivers. `TranscriptHistory` genuinely has two -//! implementations — the production file-backed one and the in-memory test +//! implementations — the production driver-backed one and the in-memory test //! double — and both are certified here. mod common; @@ -17,7 +17,7 @@ use tinyagents_session::testkit::InMemoryTranscriptHistory; use tinyagents_session::testkit::conformance::{ run_ledger_conformance, transcript_history_conformance, }; -use tinyagents_session::transcript::{FileTranscriptHistory, TranscriptMeta}; +use tinyagents_session::transcript::TranscriptMeta; #[tokio::test] async fn run_ledger_satisfies_the_conformance_suite_on_memory() { @@ -55,10 +55,16 @@ fn contract_meta() -> TranscriptMeta { } #[test] -fn file_transcript_history_satisfies_the_conformance_suite() { - let dir = tempfile::tempdir().unwrap(); - let history = FileTranscriptHistory::new(dir.path(), "session", contract_meta()).unwrap(); - transcript_history_conformance(&history); +fn driver_transcript_history_satisfies_the_conformance_suite() { + use tinyagents_session::port::{DriverSessionStores, SessionStoreProvider}; + let stores = + DriverSessionStores::new(Arc::new(tinystoragedrivers_core::MemoryStorage::new())).unwrap(); + let history = stores + .for_agent("contract") + .transcripts + .open_stem("session", contract_meta()) + .unwrap(); + transcript_history_conformance(history.as_ref()); } #[test] diff --git a/crates/tinyagents-runtime/Cargo.toml b/crates/tinyagents-runtime/Cargo.toml index 51f8db814..e03236044 100644 --- a/crates/tinyagents-runtime/Cargo.toml +++ b/crates/tinyagents-runtime/Cargo.toml @@ -26,6 +26,7 @@ tracing = { workspace = true } [dev-dependencies] anyhow = { workspace = true } +tinystoragedrivers-core = { workspace = true } serde_json = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, default-features = true, features = ["macros", "rt-multi-thread", "sync", "time"] } diff --git a/crates/tinyagents-runtime/src/lib.rs b/crates/tinyagents-runtime/src/lib.rs index 1a2b3bb90..cb0077fcb 100644 --- a/crates/tinyagents-runtime/src/lib.rs +++ b/crates/tinyagents-runtime/src/lib.rs @@ -90,3 +90,5 @@ pub trait TranscriptCodec: Send + Sync { #[cfg(test)] #[path = "lib_tests.rs"] mod test; +#[cfg(test)] +mod test_store; diff --git a/crates/tinyagents-runtime/src/lib_prefix_refresh_tests.rs b/crates/tinyagents-runtime/src/lib_prefix_refresh_tests.rs index 08db4dce2..5fa4eddba 100644 --- a/crates/tinyagents-runtime/src/lib_prefix_refresh_tests.rs +++ b/crates/tinyagents-runtime/src/lib_prefix_refresh_tests.rs @@ -348,8 +348,7 @@ async fn legacy_mixed_role_prefix_resumes_repeatedly_then_records_boundary_on_co #[tokio::test] async fn mixed_role_refresh_preserves_overlapping_conversation_on_request_and_cold_resume() { - let directory = tempfile::tempdir().unwrap(); - let locator = Arc::new(FileTranscriptLocator::new(directory.path().to_path_buf())); + let locator = crate::test_store::memory(); let identity = SessionRef::scoped("refresh", "agent-id"); let hooks = Arc::new(RefreshHook::default()); let prefix = vec![Message::system("new"), Message::user("first question")]; @@ -400,8 +399,7 @@ async fn mixed_role_refresh_preserves_overlapping_conversation_on_request_and_co #[tokio::test] async fn prefix_extension_containing_prior_rows_forces_successor_for_normal_and_partial_turns() { for partial in [false, true] { - let directory = tempfile::tempdir().unwrap(); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); + let locator = crate::test_store::memory(); let identity = SessionRef::scoped("extension", "agent-id"); let mut initial_meta = meta(); initial_meta.turn_count = 1; @@ -413,8 +411,10 @@ async fn prefix_extension_containing_prior_rows_forces_successor_for_normal_and_ let mut example = TranscriptMessage::new("user", "example"); example.extra_metadata = Some(serde_json::json!({"opaque":"conversation provenance"})); example.request_id = Some("original-turn".into()); + // As a store returns a row it already holds: its request id is recorded. + example.preserve_request_id = true; sealed.append(example).unwrap(); - let sealed_before = std::fs::read(sealed.path()).unwrap(); + let sealed_before = crate::test_store::fingerprint(sealed.as_ref()); let original_row = sealed.read_session().unwrap().unwrap().messages[1].clone(); let prefix = vec![Message::system("base"), Message::user("example")]; let expected = vec![ @@ -455,7 +455,10 @@ async fn prefix_extension_containing_prior_rows_forces_successor_for_normal_and_ head.generation, 1, "changed committed prefix must seal generation zero" ); - assert_eq!(std::fs::read(sealed.path()).unwrap(), sealed_before); + assert_eq!( + crate::test_store::fingerprint(sealed.as_ref()), + sealed_before + ); let persisted = locator .read_session_transcript(&head) .unwrap() @@ -471,11 +474,7 @@ async fn prefix_extension_containing_prior_rows_forces_successor_for_normal_and_ drop(session); let mut reopened = SessionBuilder::new(Arc::new(Driver::new(vec![]))) .codec(Arc::new(RetainedCodec)) - .session( - Arc::new(FileTranscriptLocator::new(directory.path())), - identity, - meta(), - ) + .session(locator.clone(), identity, meta()) .build() .unwrap(); assert_eq!(reopened.resume(&options).await.unwrap().history, expected); @@ -490,8 +489,7 @@ async fn compacted_legacy_mixed_role_prefix_resumes_then_records_exact_boundary( } async fn legacy_prefix_contract(compacted: bool, recorded_root: bool) { - let directory = tempfile::tempdir().unwrap(); - let locator = Arc::new(FileTranscriptLocator::new(directory.path().to_path_buf())); + let locator = crate::test_store::memory(); let identity = SessionRef::scoped("legacy-prefix", "agent-id"); let mut legacy_meta = meta(); legacy_meta.turn_count = 1; @@ -587,11 +585,7 @@ async fn legacy_prefix_contract(compacted: bool, recorded_root: bool) { drop(session); let mut reopened = SessionBuilder::new(Arc::new(Driver::new(vec![]))) .codec(codec) - .session( - Arc::new(FileTranscriptLocator::new(directory.path().to_path_buf())), - identity, - meta(), - ) + .session(locator.clone(), identity, meta()) .build() .unwrap(); assert_eq!(reopened.resume(&options).await.unwrap().history, committed); diff --git a/crates/tinyagents-runtime/src/lib_tests.rs b/crates/tinyagents-runtime/src/lib_tests.rs index 69af45fce..766e88ef1 100644 --- a/crates/tinyagents-runtime/src/lib_tests.rs +++ b/crates/tinyagents-runtime/src/lib_tests.rs @@ -6,16 +6,15 @@ use std::{ sync::{Arc, Mutex}, }; +use crate::test_store::{Destination, fingerprint, memory, read, read_display, seed, try_read}; use async_trait::async_trait; use tinyagents_harness::{ context::{RunConfig, RunContext}, runtime::AgentHarness, }; use tinyagents_session::transcript::{ - DisplayRecord, FileTranscriptLocator, SessionRef, SessionTranscript, TranscriptHistory, - TranscriptLocator, TranscriptMessage, TranscriptMeta, TranscriptRead, TranscriptTurn, - TurnUsage, read_transcript, read_transcript_display, resolve_keyed_transcript_path, - session_stem, write_transcript, + DisplayRecord, SessionRef, SessionTranscript, TranscriptHistory, TranscriptLocator, + TranscriptMessage, TranscriptMeta, TranscriptRead, TranscriptTurn, TurnUsage, session_stem, }; use tinyinference_llm::message::Message; use tinyinference_llm::providers::MockModel; @@ -791,9 +790,8 @@ async fn harness_driver_rejects_a_snapshot_not_registered_by_the_harness() { } #[tokio::test] -async fn file_history_commits_partial_model_history_and_display_only_partial_together() { - let directory = tempfile::tempdir().unwrap(); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); +async fn history_commits_partial_model_history_and_display_only_partial_together() { + let locator = memory(); let codec = Arc::new(Codec { turn_usage: Mutex::new(Some(turn_usage())), ..Default::default() @@ -811,7 +809,7 @@ async fn file_history_commits_partial_model_history_and_display_only_partial_tog partial: Some(partial), })]))) .codec(codec) - .transcript(locator, "agent", meta()) + .transcript(locator.clone(), "agent", meta()) .build() .unwrap(); assert!( @@ -823,19 +821,17 @@ async fn file_history_commits_partial_model_history_and_display_only_partial_tog .await .is_err() ); - let path = directory.path().join("session_raw/agent.jsonl"); - let persisted = read_transcript(&path).unwrap(); + let persisted = read(locator.as_ref(), "agent"); assert_eq!(persisted.messages[0].content, "recoverable"); assert_eq!(persisted.messages[0].turn_usage, Some(turn_usage())); - assert!(read_transcript_display(&path).unwrap().records.iter().any(|record| matches!(record, + assert!(read_display(locator.as_ref(), "agent").records.iter().any(|record| matches!(record, DisplayRecord::Message(message) if message.interrupted && message.message.content == "display partial" ))); } #[tokio::test] async fn codec_turn_usage_is_written_with_the_successful_turn_append() { - let directory = tempfile::tempdir().unwrap(); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); + let locator = memory(); let codec = Arc::new(Codec { turn_usage: Mutex::new(Some(turn_usage())), ..Default::default() @@ -844,7 +840,7 @@ async fn codec_turn_usage_is_written_with_the_successful_turn_append() { Message::assistant("final"), ]))]))) .codec(codec) - .transcript(locator, "agent", meta()) + .transcript(locator.clone(), "agent", meta()) .build() .unwrap(); @@ -856,7 +852,7 @@ async fn codec_turn_usage_is_written_with_the_successful_turn_append() { .await .unwrap(); - let transcript = read_transcript(&directory.path().join("session_raw/agent.jsonl")).unwrap(); + let transcript = read(locator.as_ref(), "agent"); assert_eq!(transcript.messages.len(), 1); assert_eq!(transcript.messages[0].content, "final"); assert_eq!(transcript.messages[0].turn_usage, Some(turn_usage())); @@ -975,8 +971,7 @@ async fn cancellation_persists_the_rounds_a_cancel_aware_driver_returns() { (true, &["stop me", "completed round"][..]), (false, &[][..]), ] { - let directory = tempfile::tempdir().unwrap(); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); + let locator = memory(); let options = TurnOptions::default(); let cancellation = options.cancellation.clone(); let started = Arc::new(tokio::sync::Notify::new()); @@ -985,7 +980,7 @@ async fn cancellation_persists_the_rounds_a_cancel_aware_driver_returns() { let mut session = SessionBuilder::new(Arc::new(driver)) .codec(Arc::new(Codec::default())) .hooks(hook) - .transcript(locator, "agent", meta()) + .transcript(locator.clone(), "agent", meta()) .build() .unwrap(); let turn = tokio::spawn(async move { @@ -1004,8 +999,7 @@ async fn cancellation_persists_the_rounds_a_cancel_aware_driver_returns() { [SessionTerminal::Cancelled] )); assert!(events.receipts.lock().unwrap().is_empty()); - let path = directory.path().join("session_raw/agent.jsonl"); - let persisted: Vec = read_transcript(&path) + let persisted: Vec = try_read(locator.as_ref(), "agent") .map(|transcript| transcript.messages.into_iter().map(|m| m.content).collect()) .unwrap_or_default(); assert_eq!(persisted, expected, "returns_partial={returns_partial}"); @@ -2534,7 +2528,7 @@ fn session_outcome(history: Vec, output: &str) -> DriverOutcome { /// reads and appends to the file the first one wrote. #[tokio::test] async fn a_restarted_session_continues_the_same_transcript() { - let directory = tempfile::tempdir().unwrap(); + let locator = memory(); let session_ref = SessionRef::scoped("thread-9fa08", "agent-id"); let mut first = SessionBuilder::new(Arc::new(Driver::new(vec![Ok(session_outcome( @@ -2545,11 +2539,7 @@ async fn a_restarted_session_continues_the_same_transcript() { "when?", ))]))) .codec(Arc::new(Codec::default())) - .session( - Arc::new(FileTranscriptLocator::new(directory.path())), - session_ref.clone(), - meta(), - ) + .session(locator.clone(), session_ref.clone(), meta()) .build() .unwrap(); first @@ -2571,11 +2561,7 @@ async fn a_restarted_session_continues_the_same_transcript() { "about kashmir", ))]))) .codec(Arc::new(Codec::default())) - .session( - Arc::new(FileTranscriptLocator::new(directory.path())), - session_ref.clone(), - meta(), - ) + .session(locator.clone(), session_ref.clone(), meta()) .build() .unwrap(); let resumed = second @@ -2598,18 +2584,18 @@ async fn a_restarted_session_continues_the_same_transcript() { .await .unwrap(); - let roots: Vec<_> = std::fs::read_dir(directory.path().join("session_raw")) - .unwrap() - .flatten() - .map(|entry| entry.file_name().to_string_lossy().to_string()) - .collect(); + let roots = locator + .thread_transcripts("thread-9fa08", None) + .expect("the thread has transcripts") + .roots; assert_eq!( roots.len(), 1, - "one conversation, one transcript: {roots:?}" + "one conversation, one transcript: {:?}", + roots.iter().map(|root| &root.stem).collect::>() ); - let persisted = read_transcript(&directory.path().join("session_raw").join(&roots[0])).unwrap(); + let persisted = roots[0].transcript.read_session().unwrap().unwrap(); let contents: Vec<&str> = persisted .messages .iter() @@ -2634,9 +2620,8 @@ async fn a_restarted_session_continues_the_same_transcript() { /// conversation's own history. #[tokio::test] async fn a_compaction_opens_the_next_generation_and_leaves_the_sealed_one_intact() { - let directory = tempfile::tempdir().unwrap(); let session_ref = SessionRef::scoped("thread-1", "agent-id"); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); + let locator = memory(); let mut session = SessionBuilder::new(Arc::new(Driver::new(vec![ Ok(session_outcome( @@ -2666,11 +2651,10 @@ async fn a_compaction_opens_the_next_generation_and_leaves_the_sealed_one_intact ) .await .unwrap(); - let sealed = directory - .path() - .join("session_raw") - .join(format!("{}.jsonl", session_stem(&session_ref))); - let sealed_bytes = std::fs::read(&sealed).unwrap(); + let sealed = locator + .read_session_transcript(&session_ref) + .expect("the first generation exists"); + let sealed_bytes = fingerprint(sealed.as_ref()); session .turn( @@ -2681,15 +2665,14 @@ async fn a_compaction_opens_the_next_generation_and_leaves_the_sealed_one_intact .unwrap(); assert_eq!( - std::fs::read(&sealed).unwrap(), + fingerprint(sealed.as_ref()), sealed_bytes, - "the sealed generation must be byte-identical after a compaction" + "the sealed generation must be unchanged after a compaction" + ); + let carried = read( + locator.as_ref(), + &session_stem(&session_ref.next_generation()), ); - let successor = directory.path().join("session_raw").join(format!( - "{}.jsonl", - session_stem(&session_ref.next_generation()) - )); - let carried = read_transcript(&successor).unwrap(); assert_eq!( carried .messages @@ -2709,9 +2692,8 @@ async fn a_compaction_opens_the_next_generation_and_leaves_the_sealed_one_intact /// replaying the sealed one. #[tokio::test] async fn a_restart_after_a_compaction_resumes_the_head_generation() { - let directory = tempfile::tempdir().unwrap(); let session_ref = SessionRef::scoped("thread-1", "agent-id"); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); + let locator = memory(); locator .open_session(&session_ref, meta()) .unwrap() @@ -2781,9 +2763,8 @@ async fn resumed_compaction_keeps_the_original_system_prefix_frozen() { } } - let directory = tempfile::tempdir().unwrap(); let session_ref = SessionRef::scoped("thread-prefix", "agent-id"); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); + let locator = memory(); let mut thread_meta = meta(); thread_meta.thread_id = Some("thread-prefix".into()); let root = locator @@ -2885,9 +2866,8 @@ async fn resumed_compaction_keeps_the_original_system_prefix_frozen() { ["replacement", "changing history summary", "later"] ); - let altered_dir = tempfile::tempdir().unwrap(); let altered_ref = SessionRef::scoped("altered-prefix", "agent-id"); - let altered_locator = Arc::new(FileTranscriptLocator::new(altered_dir.path())); + let altered_locator = memory(); let sealed = altered_locator.open_session(&altered_ref, meta()).unwrap(); for (role, content) in [ ("system", "old stable"), @@ -2937,9 +2917,8 @@ async fn resumed_compaction_keeps_the_original_system_prefix_frozen() { // A successor that committed a replacement prefix records its own exact // boundary. A later cold resume must use that generation's count rather // than compare its rows with generation zero's different prompt. - let recorded_dir = tempfile::tempdir().unwrap(); let recorded_ref = SessionRef::scoped("recorded-prefix", "agent-id"); - let recorded_locator = Arc::new(FileTranscriptLocator::new(recorded_dir.path())); + let recorded_locator = memory(); let sealed = recorded_locator .open_session(&recorded_ref, meta()) .unwrap(); @@ -2982,9 +2961,8 @@ async fn resumed_compaction_keeps_the_original_system_prefix_frozen() { assert_eq!(recorded.prefix_snapshot().messages().len(), 1); assert_eq!(resumed.history[1].text(), "changing history summary"); - let few_shot_dir = tempfile::tempdir().unwrap(); let few_shot_ref = SessionRef::scoped("few-shot-prefix", "agent-id"); - let few_shot_locator = Arc::new(FileTranscriptLocator::new(few_shot_dir.path())); + let few_shot_locator = memory(); let mut few_shot_meta = meta(); few_shot_meta.prefix_message_count = Some(2); let few_shot_history = few_shot_locator @@ -3021,9 +2999,8 @@ async fn resumed_compaction_keeps_the_original_system_prefix_frozen() { // A Thread scan can read a different file from the session-bound write // destination. Rebinding to that destination must replace the cached // stored-boundary count as well as its raw rows. - let rebound_dir = tempfile::tempdir().unwrap(); let destination_ref = SessionRef::scoped("destination-prefix", "agent-id"); - let rebound_locator = Arc::new(FileTranscriptLocator::new(rebound_dir.path())); + let rebound_locator = memory(); let mut destination_meta = meta(); destination_meta.thread_id = Some("shared-prefix-thread".into()); destination_meta.created = "2026-01-01T00:00:00Z".into(); @@ -3040,21 +3017,19 @@ async fn resumed_compaction_keeps_the_original_system_prefix_frozen() { .append(TranscriptMessage::new(role, content)) .unwrap(); } - let scanned_path = resolve_keyed_transcript_path(rebound_dir.path(), "2000_other").unwrap(); let mut scanned_meta = meta(); scanned_meta.thread_id = Some("shared-prefix-thread".into()); scanned_meta.created = "2026-02-01T00:00:00Z".into(); scanned_meta.prefix_message_count = Some(1); - write_transcript( - &scanned_path, + seed( + rebound_locator.as_ref(), + "2000_other", &[ TranscriptMessage::new("system", "scanned stable"), TranscriptMessage::new("user", "scanned turn"), ], &scanned_meta, - None, - ) - .unwrap(); + ); let mut rebound = SessionBuilder::new(Arc::new(Driver::new(Vec::new()))) .codec(Arc::new(RoleCodec)) .session(rebound_locator, destination_ref, meta()) @@ -3247,9 +3222,8 @@ async fn session_driver_receives_the_frozen_system_boundary() { /// corrupt the destination. #[tokio::test] async fn thread_resume_on_a_session_bound_target_does_not_corrupt_its_own_destination() { - let directory = tempfile::tempdir().unwrap(); let session_ref = SessionRef::scoped("thread-1", "agent-id"); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); + let locator = memory(); // The session's own destination already carries one real message from an // earlier session-mode turn. @@ -3268,16 +3242,15 @@ async fn thread_resume_on_a_session_bound_target_does_not_corrupt_its_own_destin let mut legacy = meta(); legacy.thread_id = Some("thread-1".into()); legacy.created = "zzz-later".into(); - tinyagents_session::transcript::write_transcript( - &directory.path().join("session_raw/legacy_other.jsonl"), + seed( + locator.as_ref(), + "legacy_other", &[ TranscriptMessage::new("user", "legacy one"), TranscriptMessage::new("user", "legacy two"), ], &legacy, - None, - ) - .unwrap(); + ); // The driver extends whatever it was handed by exactly one message. let mut session = SessionBuilder::new(Arc::new(Driver::new(vec![Ok(session_outcome( @@ -3311,11 +3284,7 @@ async fn thread_resume_on_a_session_bound_target_does_not_corrupt_its_own_destin // message: this assertion catches exactly that class of bug rather than // merely checking "some content survived", which an append-only writer // satisfies by construction even when it drops the wrong number of rows. - let destination_path = directory - .path() - .join("session_raw") - .join(format!("{}.jsonl", session_stem(&session_ref))); - let on_disk = read_transcript(&destination_path).unwrap(); + let on_disk = read(locator.as_ref(), &session_stem(&session_ref)); assert_eq!( on_disk.messages.len(), 3, @@ -3345,9 +3314,8 @@ async fn thread_resume_on_a_session_bound_target_does_not_corrupt_its_own_destin /// has its own, different metadata. #[tokio::test] async fn thread_resume_on_a_session_bound_target_reloads_the_destinations_own_metadata() { - let directory = tempfile::tempdir().unwrap(); let session_ref = SessionRef::scoped("thread-1", "agent-id"); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); + let locator = memory(); // The destination's own metadata: a distinct `created` from whatever the // scan below will find. `agent_id` has to match the builder's seed @@ -3370,16 +3338,15 @@ async fn thread_resume_on_a_session_bound_target_reloads_the_destinations_own_me let mut legacy = meta(); legacy.thread_id = Some("thread-1".into()); legacy.created = "zzz-scanned-created".into(); - tinyagents_session::transcript::write_transcript( - &directory.path().join("session_raw/legacy_other.jsonl"), + seed( + locator.as_ref(), + "legacy_other", &[TranscriptMessage::new( "user", "from a different file entirely", )], &legacy, - None, - ) - .unwrap(); + ); let mut session = SessionBuilder::new(Arc::new(Driver::new(vec![Ok(session_outcome( vec![ @@ -3401,11 +3368,7 @@ async fn thread_resume_on_a_session_bound_target_reloads_the_destinations_own_me .await .unwrap(); - let destination_path = directory - .path() - .join("session_raw") - .join(format!("{}.jsonl", session_stem(&session_ref))); - let on_disk = read_transcript(&destination_path).unwrap(); + let on_disk = read(locator.as_ref(), &session_stem(&session_ref)); assert_eq!( on_disk.meta.created, "destination-created", "the destination's own created timestamp must survive, not the scanned file's" @@ -3416,9 +3379,8 @@ async fn thread_resume_on_a_session_bound_target_reloads_the_destinations_own_me /// byte-for-byte unchanged — instead of the actual head. #[tokio::test] async fn thread_resume_on_a_session_bound_target_writes_the_head_not_a_sealed_generation() { - let directory = tempfile::tempdir().unwrap(); let session_ref = SessionRef::scoped("thread-1", "agent-id"); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); + let locator = memory(); // `ResumeMode::Thread` matches on `_meta.thread_id`, so the seed for both // generations needs it set — otherwise `root_for_thread_scoped` finds // neither file and `resume` returns early before ever reaching the bind @@ -3439,11 +3401,10 @@ async fn thread_resume_on_a_session_bound_target_writes_the_head_not_a_sealed_ge head_handle .append(TranscriptMessage::new("user", "head generation 1")) .unwrap(); - let sealed_path = directory - .path() - .join("session_raw") - .join(format!("{}.jsonl", session_stem(&session_ref))); - let sealed_bytes_before = std::fs::read(&sealed_path).unwrap(); + let sealed = locator + .read_session_transcript(&session_ref) + .expect("generation 0 exists"); + let sealed_bytes_before = fingerprint(sealed.as_ref()); let mut session = SessionBuilder::new(Arc::new(Driver::new(vec![Ok(session_outcome( vec![ @@ -3466,20 +3427,18 @@ async fn thread_resume_on_a_session_bound_target_writes_the_head_not_a_sealed_ge .unwrap(); assert_eq!( - std::fs::read(&sealed_path).unwrap(), + fingerprint(sealed.as_ref()), sealed_bytes_before, - "generation 0 must stay sealed and byte-identical" + "generation 0 must stay sealed and unchanged" ); - let head_path = directory.path().join("session_raw").join(format!( - "{}.jsonl", - session_stem(&session_ref.next_generation()) - )); - let head_contents: Vec = read_transcript(&head_path) - .unwrap() - .messages - .into_iter() - .map(|message| message.content) - .collect(); + let head_contents: Vec = read( + locator.as_ref(), + &session_stem(&session_ref.next_generation()), + ) + .messages + .into_iter() + .map(|message| message.content) + .collect(); assert!( head_contents.contains(&"new turn".to_string()), "the new turn must land in the head generation, not the sealed one: {head_contents:?}" @@ -3491,26 +3450,21 @@ async fn thread_resume_on_a_session_bound_target_writes_the_head_not_a_sealed_ge /// regains the turns newest-wins lookup had stranded. #[tokio::test] async fn a_first_session_resume_adopts_a_pre_identity_conversation() { - let directory = tempfile::tempdir().unwrap(); + let store = crate::test_store::driver(); let mut legacy = meta(); legacy.thread_id = Some("thread-9fa08".into()); legacy.created = "2026-09-22T07:30:47Z".into(); - tinyagents_session::transcript::write_transcript( - &tinyagents_session::transcript::resolve_keyed_transcript_path( - directory.path(), - "1790062247_orchestrator", - ) - .unwrap(), + seed( + store.as_ref(), + "1790062247_orchestrator", &[TranscriptMessage::new("user", "plan a trip to kashmir")], &legacy, - None, - ) - .unwrap(); + ); let mut session = SessionBuilder::new(Arc::new(Driver::new(Vec::new()))) .codec(Arc::new(Codec::default())) .session( - Arc::new(FileTranscriptLocator::new(directory.path())), + store, SessionRef::scoped("thread-9fa08", "agent-id"), meta(), ) @@ -3534,11 +3488,10 @@ async fn a_first_session_resume_adopts_a_pre_identity_conversation() { #[tokio::test] async fn a_session_resume_with_no_history_anywhere_loads_nothing() { - let directory = tempfile::tempdir().unwrap(); let mut session = SessionBuilder::new(Arc::new(Driver::new(Vec::new()))) .codec(Arc::new(Codec::default())) .session( - Arc::new(FileTranscriptLocator::new(directory.path())), + memory(), SessionRef::scoped("thread-fresh", "agent-id"), meta(), ) @@ -3574,7 +3527,7 @@ fn two_tools() -> ToolSnapshot { /// requests can be inspected. `turn` numbers the call so each outcome /// extends the stored history instead of reading as a compaction. async fn one_file_turn( - locator: Arc, + locator: Arc, sent: ToolSnapshot, retain: bool, turn: usize, @@ -3605,60 +3558,54 @@ fn recorded_tools_session() -> SessionRef { SessionRef::scoped("thread-tools", "agent-id") } -fn recorded_tools_path(directory: &std::path::Path) -> std::path::PathBuf { - let dir = directory.join("session_raw"); - let mut files: Vec<_> = std::fs::read_dir(&dir) - .unwrap() - .map(|entry| entry.unwrap().path()) - .filter(|path| path.extension().is_some_and(|ext| ext == "jsonl")) - .collect(); - assert_eq!(files.len(), 1, "one transcript for one session: {files:?}"); - files.remove(0) -} - -fn tools_records(path: &std::path::Path) -> usize { - std::fs::read_to_string(path) - .unwrap() - .lines() - .filter(|line| line.contains(r#""kind":"tools""#)) - .count() +/// The one transcript a one-session test wrote: its stem. +fn recorded_tools_stem(locator: &dyn TranscriptLocator) -> String { + let roots = locator + .thread_transcripts("thread-tools", None) + .expect("the thread has a transcript") + .roots; + assert_eq!(roots.len(), 1, "one transcript for one session"); + roots[0].stem.clone() } #[tokio::test] async fn each_turn_records_the_tools_it_was_sent_with() { - let directory = tempfile::tempdir().unwrap(); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); - let (_, recorded) = one_file_turn(locator, two_tools(), false, 0).await; + let locator = memory(); + let (_, recorded) = one_file_turn(locator.clone(), two_tools(), false, 0).await; - let path = recorded_tools_path(directory.path()); - let transcript = read_transcript(&path).unwrap(); + let stem = recorded_tools_stem(locator.as_ref()); + let transcript = read(locator.as_ref(), &stem); assert_eq!(transcript.meta.prefix_message_count, Some(0)); let stored = ToolSnapshot::from_json(transcript.tools.as_ref().unwrap()).unwrap(); assert_eq!(tool_names(&stored), vec!["alpha", "beta"]); assert_eq!(tool_names(&recorded.unwrap()), vec!["alpha", "beta"]); // A request-state record, never a displayed message. assert_eq!( - read_transcript_display(&path).unwrap().records.len(), + read_display(locator.as_ref(), &stem).records.len(), transcript.messages.len() ); } #[tokio::test] async fn a_resumed_session_keeps_tools_the_new_process_did_not_rebuild() { - let directory = tempfile::tempdir().unwrap(); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); + let locator = memory(); one_file_turn(locator.clone(), two_tools(), true, 0).await; // A "restarted" process that has only rebuilt `alpha` so far. - let (driver, recorded) = one_file_turn(locator, tools("alpha"), true, 1).await; + let (driver, recorded) = one_file_turn(locator.clone(), tools("alpha"), true, 1).await; let sent = &driver.requests.lock().unwrap()[0].tools; assert_eq!(tool_names(sent), vec!["alpha", "beta"]); assert_eq!(tool_names(&recorded.unwrap()), vec!["alpha", "beta"]); - // Every ordinary turn records its sent snapshot so concurrent session - // handles cannot leave an earlier turn's declarations in force. - let path = recorded_tools_path(directory.path()); - assert_eq!(tools_records(&path), 2); + // The turn records its merged snapshot, so the stored declarations are + // the retained pair rather than the `alpha` this process rebuilt. + let stored = read(locator.as_ref(), &recorded_tools_stem(locator.as_ref())) + .tools + .unwrap(); + assert_eq!( + tool_names(&ToolSnapshot::from_json(&stored).unwrap()), + vec!["alpha", "beta"] + ); } #[tokio::test] @@ -3707,32 +3654,32 @@ async fn in_memory_sessions_retain_tools_sent_on_earlier_turns() { #[tokio::test] async fn thread_resume_restores_tools_from_its_write_destination() { - let directory = tempfile::tempdir().unwrap(); let session_ref = SessionRef::scoped("thread-tools", "agent-id"); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); - let destination = directory - .path() - .join("session_raw") - .join(format!("{}.jsonl", session_stem(&session_ref))); + let locator = memory(); + let destination_rows = [TranscriptMessage::new("user", "destination")]; locator .open_session(&session_ref, meta()) .unwrap() - .append(TranscriptMessage::new("user", "destination")) - .unwrap(); - tinyagents_session::transcript::append_tools_record(&destination, &two_tools().to_json()) + .append_turn(TranscriptTurn { + prev: &[], + next: &destination_rows, + meta: &meta(), + turn_usage: None, + request_id: None, + tools: Some(&two_tools().to_json()), + }) .unwrap(); let mut scanned_meta = meta(); scanned_meta.thread_id = Some("thread-tools".into()); scanned_meta.created = "zzz-scanned-created".into(); - tinyagents_session::transcript::write_transcript( - &directory.path().join("session_raw/scanned.jsonl"), + seed( + locator.as_ref(), + "scanned", &[TranscriptMessage::new("user", "scanned")], &scanned_meta, - None, - ) - .unwrap(); + ); let driver = Arc::new(Driver::new(vec![Ok(outcome(vec![ Message::user("scanned"), @@ -3743,7 +3690,7 @@ async fn thread_resume_restores_tools_from_its_write_destination() { .codec(Arc::new(Codec::default())) .hooks(hook) .retain_recorded_tools(true) - .session(locator, session_ref, meta()) + .session(locator.clone(), session_ref, meta()) .build() .unwrap(); @@ -3764,18 +3711,17 @@ async fn thread_resume_restores_tools_from_its_write_destination() { #[tokio::test] async fn without_retention_a_changed_tool_set_is_sent_and_recorded_as_is() { - let directory = tempfile::tempdir().unwrap(); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); + let locator = memory(); one_file_turn(locator.clone(), two_tools(), false, 0).await; - let (driver, _) = one_file_turn(locator, tools("alpha"), false, 1).await; + let (driver, _) = one_file_turn(locator.clone(), tools("alpha"), false, 1).await; assert_eq!( tool_names(&driver.requests.lock().unwrap()[0].tools), vec!["alpha"] ); - let path = recorded_tools_path(directory.path()); - assert_eq!(tools_records(&path), 2); - let stored = read_transcript(&path).unwrap().tools.unwrap(); + let stored = read(locator.as_ref(), &recorded_tools_stem(locator.as_ref())) + .tools + .unwrap(); assert_eq!( tool_names(&ToolSnapshot::from_json(&stored).unwrap()), vec!["alpha"] @@ -3784,8 +3730,7 @@ async fn without_retention_a_changed_tool_set_is_sent_and_recorded_as_is() { #[tokio::test] async fn an_exact_tool_turn_neither_merges_nor_records() { - let directory = tempfile::tempdir().unwrap(); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); + let locator = memory(); one_file_turn(locator.clone(), two_tools(), true, 0).await; let mut history: Vec = (0..2).map(|_| Message::user("x")).collect(); @@ -3799,7 +3744,7 @@ async fn an_exact_tool_turn_neither_merges_nor_records() { .codec(Arc::new(Codec::default())) .hooks(hook) .retain_recorded_tools(true) - .session(locator, recorded_tools_session(), meta()) + .session(locator.clone(), recorded_tools_session(), meta()) .build() .unwrap(); session @@ -3811,8 +3756,7 @@ async fn an_exact_tool_turn_neither_merges_nor_records() { .unwrap(); assert!(driver.requests.lock().unwrap()[0].tools.specs().is_empty()); - let stored = read_transcript(&recorded_tools_path(directory.path())) - .unwrap() + let stored = read(locator.as_ref(), &recorded_tools_stem(locator.as_ref())) .tools .unwrap(); assert_eq!( @@ -3823,8 +3767,7 @@ async fn an_exact_tool_turn_neither_merges_nor_records() { #[tokio::test] async fn an_exact_tool_turn_carries_recorded_tools_into_a_compaction_generation() { - let directory = tempfile::tempdir().unwrap(); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); + let locator = memory(); one_file_turn(locator.clone(), two_tools(), true, 0).await; // Return a reduced history so persistence opens a successor generation. @@ -3839,7 +3782,7 @@ async fn an_exact_tool_turn_carries_recorded_tools_into_a_compaction_generation( .codec(Arc::new(Codec::default())) .hooks(hook) .retain_recorded_tools(true) - .session(locator, recorded_tools_session(), meta()) + .session(locator.clone(), recorded_tools_session(), meta()) .build() .unwrap(); session @@ -3850,12 +3793,9 @@ async fn an_exact_tool_turn_carries_recorded_tools_into_a_compaction_generation( .await .unwrap(); - let head = std::fs::read_dir(directory.path().join("session_raw")) - .unwrap() - .map(|entry| entry.unwrap().path()) - .find(|path| path.to_string_lossy().contains(".g1.jsonl")) - .expect("compaction successor must exist"); - let stored = read_transcript(&head).unwrap().tools.unwrap(); + let head = locator.head_generation(&recorded_tools_session()); + assert_eq!(head.generation, 1, "compaction successor must exist"); + let stored = read(locator.as_ref(), &session_stem(&head)).tools.unwrap(); assert_eq!( tool_names(&ToolSnapshot::from_json(&stored).unwrap()), vec!["alpha", "beta"] @@ -3906,7 +3846,7 @@ async fn resume_adopts_the_stored_prefix_and_its_committed_turns() { /// says to: lazily from the *current* workspace dir, "never frozen at build /// time". That necessarily yields a fresh `Arc` per `before_resume`. struct FreshLocatorEachTurn { - workspace_dir: PathBuf, + store: Arc, session: SessionRef, built: Mutex, } @@ -3922,7 +3862,7 @@ impl SessionHooks for FreshLocatorEachTurn { *self.built.lock().unwrap() += 1; Ok(ResumePreparation { transcript: Some(TranscriptTarget::for_session( - Arc::new(FileTranscriptLocator::new(self.workspace_dir.clone())), + Arc::new(Destination::new("store", &self.store)), self.session.clone(), meta(), )), @@ -3962,11 +3902,10 @@ impl SessionHooks for FreshLocatorEachTurn { /// The turn count matters: turn one binds, so the guard is only consulted from /// turn two onwards. A single-turn test passes either way. #[tokio::test] -async fn a_rebuilt_locator_over_the_same_directory_is_the_same_binding() { - let directory = tempfile::tempdir().unwrap(); +async fn a_rebuilt_locator_over_the_same_destination_is_the_same_binding() { let session_ref = SessionRef::scoped("thread-1", "agent-id"); let hook = Arc::new(FreshLocatorEachTurn { - workspace_dir: directory.path().to_path_buf(), + store: memory(), session: session_ref.clone(), built: Mutex::new(0), }); @@ -4021,34 +3960,33 @@ async fn a_rebuilt_locator_over_the_same_directory_is_the_same_binding() { /// The converse: two genuinely different destinations must still be rejected, /// so the loosened comparison did not turn the redirection guard off. #[tokio::test] -async fn a_locator_over_a_different_directory_is_not_the_same_binding() { - let first = tempfile::tempdir().unwrap(); - let second = tempfile::tempdir().unwrap(); +async fn a_locator_over_a_different_destination_is_not_the_same_binding() { + let store = memory(); let session_ref = SessionRef::scoped("thread-1", "agent-id"); let bound = TranscriptTarget::for_session( - Arc::new(FileTranscriptLocator::new(first.path())), + Arc::new(Destination::new("store-a", &store)), session_ref.clone(), meta(), ); let same = TranscriptTarget::for_session( - Arc::new(FileTranscriptLocator::new(first.path())), + Arc::new(Destination::new("store-a", &store)), session_ref.clone(), meta(), ); let elsewhere = TranscriptTarget::for_session( - Arc::new(FileTranscriptLocator::new(second.path())), + Arc::new(Destination::new("store-b", &store)), session_ref, meta(), ); assert!( bound.same_binding(&same), - "same directory, different allocation: the same durable destination" + "same destination, different allocation: the same durable destination" ); assert!( !bound.same_binding(&elsewhere), - "a different directory is a real redirection and must stay rejected" + "a different destination is a real redirection and must stay rejected" ); } @@ -4117,8 +4055,7 @@ async fn explicit_prefix_refresh_preserves_history_and_survives_cold_resume() { .collect()) } } - let directory = tempfile::tempdir().unwrap(); - let locator = Arc::new(FileTranscriptLocator::new(directory.path().to_path_buf())); + let locator = memory(); let identity = SessionRef::scoped("managed", "agent-id"); let prefix = vec![Message::system("base"), Message::system("tools")]; let first = vec![ diff --git a/crates/tinyagents-runtime/src/session_tests.rs b/crates/tinyagents-runtime/src/session_tests.rs index d3c0f842e..e4afaebe4 100644 --- a/crates/tinyagents-runtime/src/session_tests.rs +++ b/crates/tinyagents-runtime/src/session_tests.rs @@ -1,11 +1,11 @@ use super::*; +use crate::test_store::{memory, read, seed}; use crate::{DriverFailure, DriverOutcome, SessionBuilder, TranscriptCodec}; use async_trait::async_trait; use std::sync::Arc; use tinyagents_session::transcript::{ - BackgroundAppend, BackgroundAppendOutcome, FileTranscriptLocator, SessionRef, - SessionTranscript, TranscriptLocator, TranscriptMessage, TranscriptMeta, read_transcript, - resolve_keyed_transcript_path, session_stem, + BackgroundAppend, BackgroundAppendOutcome, SessionRef, SessionTranscript, TranscriptMessage, + TranscriptMeta, session_stem, }; use tinyinference_llm::message::Message; use tokio::sync::Notify; @@ -97,22 +97,18 @@ impl SessionDriver for GatedDriver { /// session and the write falls back to the target's existing session. #[tokio::test] async fn a_running_turn_holds_off_a_background_append_into_its_session() { - let directory = tempfile::tempdir().unwrap(); - let locator = Arc::new(FileTranscriptLocator::new(directory.path())); + let locator = memory(); let session_ref = SessionRef::scoped("thread-1", "agent"); let selected_session = SessionRef::scoped("thread-2", "agent"); - let path = - resolve_keyed_transcript_path(directory.path(), &session_stem(&session_ref)).unwrap(); - tinyagents_session::transcript::write_transcript( - &path, + seed( + locator.as_ref(), + &session_stem(&session_ref), &[ TranscriptMessage::user("remind me at 5"), TranscriptMessage::assistant("Scheduled."), ], &meta(), - None, - ) - .unwrap(); + ); let started = Arc::new(Notify::new()); let release = Arc::new(Notify::new()); let mut session = SessionBuilder::new(Arc::new(GatedDriver { @@ -153,19 +149,20 @@ async fn a_running_turn_holds_off_a_background_append_into_its_session() { let delivery_locator = locator.clone(); let delivery_session = session_ref.clone(); - let delivery_started = Arc::new(Notify::new()); - let delivery_started_signal = delivery_started.clone(); let mut delivery = tokio::spawn(async move { - tinyagents_session::transcript::append_background_message_with_lock_notification( - &delivery_locator, + tinyagents_session::transcript::append_background_message_in( + delivery_locator.as_ref(), &delivery_session, TranscriptMessage::assistant("Time to stretch!"), BackgroundAppend::new("run-1", serde_json::json!({"kind": "cron"})), - &delivery_started_signal, ) .await }); - delivery_started.notified().await; + // Single-threaded runtime: yielding lets the delivery run up to the turn + // lock it must wait on. + for _ in 0..50 { + tokio::task::yield_now().await; + } assert!( timeout(Duration::from_secs(1), &mut delivery) .await @@ -179,17 +176,14 @@ async fn a_running_turn_holds_off_a_background_append_into_its_session() { delivery.await.unwrap().unwrap(), BackgroundAppendOutcome::Appended { generation: 1 } ); - let head_path = resolve_keyed_transcript_path( - directory.path(), + let contents: Vec = read( + locator.as_ref(), &session_stem(&locator.head_generation(&session_ref)), ) - .unwrap(); - let contents: Vec = read_transcript(&head_path) - .unwrap() - .messages - .into_iter() - .map(|message| message.content) - .collect(); + .messages + .into_iter() + .map(|message| message.content) + .collect(); assert_eq!( contents, [ diff --git a/crates/tinyagents-runtime/src/test_store.rs b/crates/tinyagents-runtime/src/test_store.rs new file mode 100644 index 000000000..e31e22c6d --- /dev/null +++ b/crates/tinyagents-runtime/src/test_store.rs @@ -0,0 +1,183 @@ +//! Test fixtures over a store-port transcript locator. +//! +//! Runtime tests bind sessions to an in-memory locator and read back what the +//! session persisted by stem, the way a host reads a store, rather than +//! naming any file. + +use std::sync::Arc; + +use tinyagents_session::port::{DriverSessionStores, SessionStoreProvider}; +use tinyagents_session::transcript::{ + DisplaySessionTranscript, SessionRef, SessionTranscript, ThreadTranscripts, TranscriptHistory, + TranscriptLocator, TranscriptMessage, TranscriptMeta, TranscriptRead, TranscriptTurn, +}; +use tinystoragedrivers_core::MemoryStorage; + +/// An empty transcript store: the production driver-backed locator, over memory. +pub(crate) fn memory() -> Arc { + driver() +} + +/// Writes `messages` as the first turn of the transcript `stem`. +pub(crate) fn seed( + locator: &dyn TranscriptLocator, + stem: &str, + messages: &[TranscriptMessage], + meta: &TranscriptMeta, +) { + locator + .open_stem(stem, meta.clone()) + .expect("open stem") + .append_turn(TranscriptTurn { + prev: &[], + next: messages, + meta, + turn_usage: None, + request_id: None, + tools: None, + }) + .expect("seed transcript"); +} + +/// The model-context replay of the transcript `stem`; panics when it is absent. +pub(crate) fn read(locator: &dyn TranscriptLocator, stem: &str) -> SessionTranscript { + try_read(locator, stem).unwrap_or_else(|| panic!("transcript {stem} is absent")) +} + +/// The model-context replay of the transcript `stem`, when it exists. +pub(crate) fn try_read(locator: &dyn TranscriptLocator, stem: &str) -> Option { + locator + .open_stem(stem, placeholder_meta()) + .expect("open stem") + .read_session() + .expect("read transcript") +} + +/// The display projection of the transcript `stem`; panics when it is absent. +pub(crate) fn read_display( + locator: &dyn TranscriptLocator, + stem: &str, +) -> DisplaySessionTranscript { + locator + .open_stem(stem, placeholder_meta()) + .expect("open stem") + .read_display() + .expect("read display") + .unwrap_or_else(|| panic!("transcript {stem} is absent")) +} + +/// A `_meta` for opening a stem only to read it (never written: the stem exists +/// or the read is `None`). +fn placeholder_meta() -> TranscriptMeta { + TranscriptMeta { + session_id: None, + parent_session_id: None, + agent_name: String::new(), + agent_id: None, + agent_type: None, + dispatcher: String::new(), + provider: None, + model: None, + created: String::new(), + updated: String::new(), + turn_count: 0, + prefix_message_count: None, + input_tokens: 0, + output_tokens: 0, + cached_input_tokens: 0, + charged_amount_usd: 0.0, + thread_id: None, + task_id: None, + } +} + +/// Everything `handle` would show a reader (model context and display), as one +/// comparable value, so a test can assert a transcript is byte-for-byte what +/// it was. +pub(crate) fn fingerprint(handle: &dyn tinyagents_session::transcript::TranscriptRead) -> String { + format!( + "{:?}\n{:?}", + handle.read_session().expect("read transcript"), + handle.read_display().expect("read display") + ) +} + +/// An in-memory store addressed under a fixed destination key, so tests can +/// hand out separate `Arc`s that stand for one durable store (or, with +/// another key, for a different one). +pub(crate) struct Destination { + key: String, + store: Arc, +} + +impl Destination { + pub(crate) fn new(key: &str, store: &Arc) -> Self { + Self { + key: key.to_string(), + store: Arc::clone(store), + } + } +} + +impl TranscriptLocator for Destination { + fn destination_key(&self) -> Option { + Some(self.key.clone()) + } + fn latest_for_agent(&self, agent_name: &str) -> Option> { + self.store.latest_for_agent(agent_name) + } + fn root_for_thread(&self, thread_id: &str) -> Option> { + self.store.root_for_thread(thread_id) + } + fn root_for_thread_scoped( + &self, + thread_id: &str, + agent_id: Option<&str>, + ) -> Option> { + self.store.root_for_thread_scoped(thread_id, agent_id) + } + fn thread_transcripts( + &self, + thread_id: &str, + agent_id: Option<&str>, + ) -> Option { + self.store.thread_transcripts(thread_id, agent_id) + } + fn open_stem( + &self, + stem: &str, + seed: TranscriptMeta, + ) -> anyhow::Result> { + self.store.open_stem(stem, seed) + } + fn session_exists(&self, session: &SessionRef) -> bool { + self.store.session_exists(session) + } + fn read_session_transcript(&self, session: &SessionRef) -> Option> { + self.store.read_session_transcript(session) + } + fn begin_generation( + &self, + session: &SessionRef, + seed: TranscriptMeta, + ) -> anyhow::Result<(SessionRef, Arc)> { + self.store.begin_generation(session, seed) + } + fn begin_generation_from_baseline( + &self, + session: &SessionRef, + seed: TranscriptMeta, + baseline: &[TranscriptMessage], + ) -> anyhow::Result<(SessionRef, Arc)> { + self.store + .begin_generation_from_baseline(session, seed, baseline) + } +} + +/// An empty driver-backed transcript store (the production store, over memory). +pub(crate) fn driver() -> Arc { + DriverSessionStores::new(Arc::new(MemoryStorage::new())) + .expect("driver session stores over memory") + .for_agent("runtime-tests") + .transcripts +} diff --git a/crates/tinyagents-session/Cargo.toml b/crates/tinyagents-session/Cargo.toml index d8590f57c..7d32a8638 100644 --- a/crates/tinyagents-session/Cargo.toml +++ b/crates/tinyagents-session/Cargo.toml @@ -9,7 +9,6 @@ rust-version.workspace = true description = "Durable TinyAgents session history and run ledger." [dependencies] -fs2 = "0.4" async-trait = { workspace = true } chrono = { workspace = true } anyhow = { workspace = true } @@ -53,6 +52,9 @@ tracing = ["tinyagents-harness/tracing"] storage-drivers = [] [dev-dependencies] +# Tests build transcript fixtures as classic JSONL and import them through +# `legacy`, so they need the importer even under `--no-default-features`. +tinyagents-session = { path = ".", features = ["legacy-import"] } tinystoragedrivers-sqlite = { workspace = true } tokio = { workspace = true, default-features = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/tinyagents-session/src/README.md b/crates/tinyagents-session/src/README.md index 9f61a95f8..25a4a883a 100644 --- a/crates/tinyagents-session/src/README.md +++ b/crates/tinyagents-session/src/README.md @@ -2,7 +2,7 @@ Driver-backed history for agent sessions (`DriverRunLedger` over any `tinystoragedrivers` backend), a restart-survivable ledger for background -agent/workflow execution, a JSONL-backed transcript store for KV-cache-stable resume, and a driver-backed chat +agent/workflow execution, a store-port transcript seam for KV-cache-stable resume, and a driver-backed chat thread/message store for the product-facing chat log. ## Why this is a top-level module @@ -20,7 +20,7 @@ would imply a dependency that exists in neither direction. | `harness::store` | "what is this run working with right now?" | during a run | | `graph::checkpoint` | "how do I resume this interrupted run?" | until resumed | | **`session` (driver ledger)** | "what happened, what did it cost, how did runs nest?" | indefinitely | -| **`session::transcript` (JSONL)** | "what exact messages did the model see?" | indefinitely, resumed verbatim | +| **`session::transcript` (store port)** | "what exact messages did the model see?" | indefinitely, resumed verbatim | | **`session::threads` (driver)** | "which chats does the user have, and what is in them?" | indefinitely | The driver-backed history (`run_ledger`) is queryable history that @@ -32,14 +32,11 @@ stream a provider (and its prompt cache) already saw; see its own ## Layout -The ledger lives wherever the host's storage backend puts it; the transcript -and thread files are still derived from the workspace root: +Everything lives wherever the host's storage backend puts it: ```text -{host storage backend} ← sessions, run ledger (collections) -{workspace_dir}/session_raw/{stem}.jsonl ← JSONL: transcripts (source of truth) -{workspace_dir}/sessions/YYYY_MM_DD/{stem}.md ← human-readable transcript view -(chat threads + messages live in the storage driver, not on disk) +{host storage backend} ← sessions, run ledger, transcripts, chat threads (collections) +{workspace_dir}/session_raw/{stem}.jsonl ← classic transcripts: read once by legacy::import_transcripts_once; rewritten in place only by legacy::rewrite_legacy_transcripts ``` ## Public surface @@ -58,9 +55,9 @@ reachable under `session::`, `session::run_ledger::`, and - **Run ledger** — agent runs, workflow runs, teams, members, tasks, run events, and telemetry, with the claim/completion coordination primitives — see its own [README](./run_ledger/README.md) -- **Transcripts** — full-rewrite and append-only writers, model-context and - display readers, thread lookups and usage summaries — see its own - [README](./transcript/README.md) +- **Transcripts** — the `TranscriptLocator`/`TranscriptHistory` store seam, + background appends, the per-session turn lock, thread spend and the display + view — see its own [README](./transcript/README.md) - **Chat threads** — `DriverConversationStore` and its wire types (re-exported at the root) under `session::threads` — see its own [README](./threads/README.md) - **Store port** — `SessionStoreProvider`, `AgentStores`, `TurnStates`, @@ -71,7 +68,7 @@ reachable under `session::`, `session::run_ledger::`, and - **Testkit** — `testkit::conformance::run_ledger_conformance` and `transcript_history_conformance`: contract suites certifying that a run ledger workspace or a `TranscriptHistory` implementation behaves like the - bundled ones. Run against the memory and SQLite run ledgers and both `FileTranscriptHistory` and the in-memory + bundled ones. Run against the memory and SQLite run ledgers and both the driver-backed and the in-memory `testkit::InMemoryTranscriptHistory` double in `crates/tinyagents-integration-tests/tests/session_conformance.rs`. diff --git a/crates/tinyagents-session/src/legacy/rewrite_tests.rs b/crates/tinyagents-session/src/legacy/rewrite_tests.rs index 4ab113512..408cfaf1d 100644 --- a/crates/tinyagents-session/src/legacy/rewrite_tests.rs +++ b/crates/tinyagents-session/src/legacy/rewrite_tests.rs @@ -177,35 +177,34 @@ fn symlinks_are_neither_followed_nor_rewritten() { #[tokio::test] async fn a_rewrite_keeps_background_delivery_provenance() { use crate::transcript::{ - BackgroundAppend, BackgroundAppendOutcome, DisplayRecord, FileTranscriptLocator, - SessionRef, TranscriptLocator, append_background_message, read_transcript_display, - resolve_keyed_transcript_path, session_stem, + BackgroundAppend, BackgroundAppendOutcome, BackgroundOrigin, DisplayRecord, SessionRef, + append_background_message_in, read_transcript_display, resolve_keyed_transcript_path, + session_stem, }; let dir = tempfile::tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); let session = SessionRef::scoped("thread-1", "orchestrator"); - locator - .open_session(&session, meta()) - .unwrap() - .replace(&[ + let path = resolve_keyed_transcript_path(dir.path(), &session_stem(&session)).unwrap(); + // A classic transcript holding one out-of-band delivery, as the classic + // background writer recorded it (a top-level `background` line field). + let origin = BackgroundOrigin { + idempotency_key: "run-1".into(), + provenance: serde_json::json!({"kind": "cron"}), + }; + write_transcript( + &path, + &[ TranscriptMessage::system("keep ### SECRET"), TranscriptMessage::user("hi"), - ]) - .unwrap(); - let options = || BackgroundAppend::new("run-1", serde_json::json!({"kind": "cron"})); - let deliver = || { - append_background_message( - &locator, - &session, - TranscriptMessage::assistant("Time to stretch!"), - options(), - ) - }; - assert_eq!( - deliver().await.unwrap(), - BackgroundAppendOutcome::Appended { generation: 0 } - ); + crate::transcript::display_with_background_row( + TranscriptMessage::assistant("Time to stretch!"), + &origin, + ), + ], + &meta(), + None, + ) + .unwrap(); let report = rewrite_legacy_transcripts(dir.path(), |session| { session.messages[0].content = "keep".into(); @@ -213,9 +212,8 @@ async fn a_rewrite_keeps_background_delivery_provenance() { }); assert_eq!(report.rewritten, 1); - let path = resolve_keyed_transcript_path(dir.path(), &session_stem(&session)).unwrap(); let display = read_transcript_display(&path).unwrap(); - let origin = display + let kept = display .records .iter() .find_map(|record| match record { @@ -223,9 +221,20 @@ async fn a_rewrite_keeps_background_delivery_provenance() { DisplayRecord::Compaction(_) => None, }) .expect("the rewritten row keeps its provenance"); - assert_eq!(origin.idempotency_key, "run-1"); + assert_eq!(kept.idempotency_key, "run-1"); + + // Imported into a store, the delivery is still a duplicate. + let locator = crate::transcript::test_locator::imported_locator(dir.path()); + let outcome = append_background_message_in( + locator.as_ref(), + &session, + TranscriptMessage::assistant("Time to stretch!"), + BackgroundAppend::new("run-1", serde_json::json!({"kind": "cron"})), + ) + .await + .unwrap(); assert_eq!( - deliver().await.unwrap(), + outcome, BackgroundAppendOutcome::Duplicate { generation: 0 }, "the idempotency key survives the rewrite" ); diff --git a/crates/tinyagents-session/src/legacy/transcripts.rs b/crates/tinyagents-session/src/legacy/transcripts.rs index cd4126cc9..72e80fe66 100644 --- a/crates/tinyagents-session/src/legacy/transcripts.rs +++ b/crates/tinyagents-session/src/legacy/transcripts.rs @@ -116,6 +116,18 @@ fn count_markdown_only(workspace_dir: &Path, stems: &std::collections::HashSet TranscriptMessage { + match &message.background { + Some(origin) => { + crate::transcript::display_with_background_row(message.message.clone(), origin) + } + None => message.message.clone(), + } +} + /// Replay one transcript's display records against `history`; returns the /// number of messages replayed. fn replay( @@ -137,7 +149,7 @@ fn replay( return Ok(()); } let mut next = logical.clone(); - next.extend(group.iter().map(|m| m.message.clone())); + next.extend(group.iter().map(stored_row)); let usage = group.iter().rev().find_map(|m| m.turn_usage.clone()); let request_id = group.iter().rev().find_map(|m| m.request_id.clone()); history.append_turn(TranscriptTurn { @@ -187,11 +199,8 @@ fn replay( } DisplayRecord::Compaction(marker) => { flush(&mut logical, &mut group, &mut replayed)?; - let next: Vec = marker - .replacement - .iter() - .map(|m| m.message.clone()) - .collect(); + let next: Vec = + marker.replacement.iter().map(stored_row).collect(); let usage = marker .replacement .iter() diff --git a/crates/tinyagents-session/src/lib.rs b/crates/tinyagents-session/src/lib.rs index 034b48941..7e68f42aa 100644 --- a/crates/tinyagents-session/src/lib.rs +++ b/crates/tinyagents-session/src/lib.rs @@ -147,7 +147,7 @@ pub use threads::{ ConversationPurgeStats, ConversationThread, CreateConversationThread, CrossThreadHit, ThreadMessage, ThreadMessagePatch, }; -pub use transcript::spend::{ThreadSpend, TranscriptSpend, thread_spend, transcript_spend}; +pub use transcript::spend::{ThreadSpend, TranscriptSpend, thread_spend_in, transcript_spend}; pub use turn_state::TurnStateMirror; pub use types::{ SessionMessage, SessionRecord, SessionSearchParams, SessionSearchResult, SessionStatus, diff --git a/crates/tinyagents-session/src/port/drivers/transcripts_display.rs b/crates/tinyagents-session/src/port/drivers/transcripts_display.rs index afbcb0a43..8abf731f4 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts_display.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts_display.rs @@ -8,7 +8,9 @@ //! was written. Entries keep no wall clock, so a marker's `ts` is `None` and //! intermediate tool-call rows carry no per-step timestamp. //! - **Thread listing.** [`TranscriptLocator::thread_transcripts`] reads the -//! index: the thread's written roots, ordered like the file view (by +//! index (first re-indexing any written transcript whose index document is +//! missing, see [`DriverTranscriptLocator::repair_unindexed`]): the thread's +//! written roots, ordered like the file view (by //! `_meta.created`, session chains oldest first, pre-identity roots dropped //! once a session root exists), and the sub-agent transcripts that descend //! from a root stem or (unscoped) name the thread. @@ -154,6 +156,72 @@ impl DriverTranscriptLocator { } } + /// Re-indexes every written transcript that has no index document. + /// + /// A write whose index refresh failed (see `HistoryInner::index`) leaves a + /// durable entry log with no lookup document, and nothing repairs that + /// until the transcript is next read or written; until then the thread + /// listing, which reads the index alone, would not see it. Detecting this + /// is two counts (written index documents against stems whose first entry + /// is written); only a mismatch pays for finding the missing stems, and + /// reading each one through its own handle runs the ordinary read-time + /// index repair. + fn repair_unindexed(&self) { + let missing = self.run(|docs, declared| async move { + declared.ensure(&docs).await?; + let first_entries = Filter::eq("seq", 0).and(Filter::eq("written", true)); + let indexed = docs.count(INDEX, &Filter::eq("written", true)).await?; + let logs = docs.count(ENTRIES, &first_entries).await?; + if logs <= indexed { + return Ok(Vec::new()); + } + let indexed_stems: HashSet = docs + .query_all(INDEX, &Query::filter(Filter::eq("written", true))) + .await? + .iter() + .filter_map(|item| Indexed::of(&item.doc)) + .map(|indexed| indexed.stem) + .collect(); + Ok(docs + .query_all(ENTRIES, &Query::filter(first_entries)) + .await? + .iter() + .filter_map(|item| item.doc.get("stem").and_then(Value::as_str)) + .filter(|stem| !indexed_stems.contains(*stem)) + .map(str::to_string) + .collect::>()) + }); + let missing = match missing { + Ok(missing) => missing, + Err(error) => { + tracing::warn!( + target: "tinyagents_session::port::drivers", + locator = %self.label, + %error, + "[session-store] unindexed transcript scan failed" + ); + return; + } + }; + for stem in missing { + tracing::debug!( + target: "tinyagents_session::port::drivers", + locator = %self.label, + %stem, + "[session-store] re-indexing a transcript whose index document is missing" + ); + if let Err(error) = self.handle(&stem, discovered_seed(&stem)).read_session() { + tracing::warn!( + target: "tinyagents_session::port::drivers", + locator = %self.label, + %stem, + %error, + "[session-store] re-indexing an unindexed transcript failed" + ); + } + } + } + fn listed(&self, stem: String) -> ThreadTranscript { let transcript: Arc = Arc::new(self.handle(&stem, discovered_seed(&stem))); @@ -258,6 +326,7 @@ impl DriverTranscriptLocator { if thread_id.is_empty() { return None; } + self.repair_unindexed(); let mut filter = Filter::eq("thread_id", thread_id).and(Filter::eq("subagent", false)); if let Some(agent_id) = agent_id { filter = filter.and(Filter::eq("agent_id", agent_id)); diff --git a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs index 97dbf8129..213890957 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs @@ -627,7 +627,7 @@ fn a_display_read_repairs_an_index_a_write_could_not_refresh() { raw.delete(INDEX, &doc_key(&["s"]), Precondition::None) .await }); - assert!(locator.thread_transcripts("t", None).is_none()); + assert!(locator.root_for_thread("t").is_none()); // A handle whose replay has not indexed the write (as after a refresh // that failed every attempt). locator @@ -635,10 +635,77 @@ fn a_display_read_repairs_an_index_a_write_could_not_refresh() { .read_display() .unwrap() .unwrap(); + assert!( + locator.root_for_thread("t").is_some(), + "the display read re-indexed it" + ); +} + +/// The thread listing reads the index alone, so a root whose index document +/// was never written (every refresh attempt after its write failed) used to +/// stay invisible until something read or wrote that transcript again. +#[test] +fn a_listing_sees_and_repairs_a_root_whose_index_refresh_failed() { + let docs = docs(); + let locator = locator(&docs); + locator + .handle("s", meta("t")) + .append(message("user", "a")) + .unwrap(); + let raw = Arc::clone(&docs); + on_bridge(async move { + raw.delete(INDEX, &doc_key(&["s"]), Precondition::None) + .await + }); + assert!( + locator.root_for_thread("t").is_none(), + "the lookup index is missing the root" + ); + let listed = locator .thread_transcripts("t", None) - .expect("the display read re-indexed it"); + .expect("the listing finds the unindexed root"); + assert_eq!(listed.roots.len(), 1); assert_eq!(listed.roots[0].stem, "s"); + assert!( + locator.root_for_thread("t").is_some(), + "the listing re-indexed the root for later lookups" + ); +} + +/// An unindexed sub-agent transcript is listed under its root too. +#[test] +fn a_listing_repairs_an_unindexed_subagent_of_an_indexed_root() { + let docs = docs(); + let locator = locator(&docs); + locator + .handle("100_root", meta("t")) + .append(message("user", "a")) + .unwrap(); + locator + .handle("100_root__200_child", meta("t")) + .append(message("user", "b")) + .unwrap(); + let raw = Arc::clone(&docs); + on_bridge(async move { + raw.delete( + INDEX, + &doc_key(&["100_root__200_child"]), + Precondition::None, + ) + .await + }); + + let listed = locator.thread_transcripts("t", None).expect("listed"); + assert_eq!(listed.roots.len(), 1); + assert_eq!( + listed + .subagents + .iter() + .map(|sub| sub.stem.as_str()) + .collect::>(), + ["100_root__200_child"] + ); } #[test] diff --git a/crates/tinyagents-session/src/port/mod_tests.rs b/crates/tinyagents-session/src/port/mod_tests.rs index 4b1a9c091..2b8fd0b25 100644 --- a/crates/tinyagents-session/src/port/mod_tests.rs +++ b/crates/tinyagents-session/src/port/mod_tests.rs @@ -1,19 +1,26 @@ use super::*; use crate::testkit::conformance::session_store_conformance; -use crate::transcript::FileTranscriptLocator; use tinyagents_harness::store::{InMemoryAppendStore, InMemoryStore}; -/// The file-backed transcript and turn-state building blocks (with in-memory -/// key-value and journal stores), assembled the way a desktop host's provider -/// assembles them: everything under one workspace, shared by every agent. -struct FileStores { - workspace: std::path::PathBuf, +/// A provider that shares one in-memory transcript locator between every +/// agent (with in-memory turn-state, key-value and journal stores), the way a +/// single-user host shares one store. +struct SharedStores { + transcripts: Arc, } -impl SessionStoreProvider for FileStores { +impl Default for SharedStores { + fn default() -> Self { + Self { + transcripts: Arc::new(InMemoryTranscriptLocator::new("shared-stores")), + } + } +} + +impl SessionStoreProvider for SharedStores { fn for_agent(&self, _agent_id: &str) -> AgentStores { AgentStores { - transcripts: Arc::new(FileTranscriptLocator::new(&self.workspace)), + transcripts: self.transcripts.clone(), turn_states: Arc::new(InMemoryTurnStates::default()), kv: Arc::new(InMemoryStore::new()), journal: Arc::new(InMemoryAppendStore::new()), @@ -22,11 +29,8 @@ impl SessionStoreProvider for FileStores { } #[tokio::test] -async fn the_file_building_blocks_meet_the_session_store_contract() { - let dir = tempfile::tempdir().unwrap(); - let provider = FileStores { - workspace: dir.path().to_path_buf(), - }; +async fn the_shared_in_memory_building_blocks_meet_the_session_store_contract() { + let provider = SharedStores::default(); session_store_conformance(&provider).await; assert!(provider.recover().is_ok(), "recovery defaults to a no-op"); assert!(provider.destination_key().is_none()); @@ -53,11 +57,8 @@ fn the_in_memory_turn_states_serve_the_port() { } #[test] -fn file_providers_explicitly_leave_actor_rebinding_unsupported() { - let dir = tempfile::tempdir().unwrap(); - let provider = FileStores { - workspace: dir.path().to_path_buf(), - }; +fn shared_providers_explicitly_leave_actor_rebinding_unsupported() { + let provider = SharedStores::default(); assert!( provider .bind_actor(Arc::new(tinystoragedrivers_core::MemoryStorage::new())) @@ -95,11 +96,8 @@ fn actor_factory_arc_forwarding_preserves_native_refusal_and_source() { #[test] /// Pins the default checked accessor for an unchanged file provider. -fn checked_accessor_keeps_legacy_file_provider_compatible() { - let dir = tempfile::tempdir().unwrap(); - let provider: &dyn SessionStoreProvider = &FileStores { - workspace: dir.path().to_path_buf(), - }; +fn checked_accessor_keeps_legacy_provider_compatible() { + let provider: &dyn SessionStoreProvider = &SharedStores::default(); let stores = provider.try_for_agent("a").unwrap(); assert!(stores.turn_states.list().unwrap().is_empty()); } @@ -187,10 +185,7 @@ fn optional_run_ledger_provider_preserves_handle_and_defaults_to_none() { .unwrap() .is_none() ); - let dir = tempfile::tempdir().unwrap(); - let default = FileStores { - workspace: dir.path().to_path_buf(), - }; + let default = SharedStores::default(); assert!(default.run_ledger_for_agent("allowed").unwrap().is_none()); } diff --git a/crates/tinyagents-session/src/testkit/conformance.rs b/crates/tinyagents-session/src/testkit/conformance.rs index b9835a3b2..f99ca6b65 100644 --- a/crates/tinyagents-session/src/testkit/conformance.rs +++ b/crates/tinyagents-session/src/testkit/conformance.rs @@ -12,7 +12,7 @@ //! [`crate::transcript::TranscriptHistory`]/[`crate::transcript::TranscriptRead`], //! which — unlike the ledger — genuinely has two independent //! implementations: the production -//! [`FileTranscriptHistory`](crate::transcript::FileTranscriptHistory) and +//! [`DriverTranscriptHistory`](crate::port::DriverTranscriptHistory) and //! the in-memory [`super::InMemoryTranscriptHistory`] double. Both are //! certified against the same suite in this crate's own tests. //! diff --git a/crates/tinyagents-session/src/testkit/in_memory_transcript.rs b/crates/tinyagents-session/src/testkit/in_memory_transcript.rs index 4ab124ab9..a6548a040 100644 --- a/crates/tinyagents-session/src/testkit/in_memory_transcript.rs +++ b/crates/tinyagents-session/src/testkit/in_memory_transcript.rs @@ -1,7 +1,7 @@ //! An in-memory [`TranscriptHistory`] double. //! -//! [`FileTranscriptHistory`](crate::transcript::FileTranscriptHistory) is the -//! only production implementation of the trait; this is a second, independent +//! The driver-backed history ([`crate::port::DriverTranscriptHistory`]) is the +//! production implementation of the trait; this is a second, independent //! one so [`TranscriptHistory`]'s conformance suite //! ([`super::conformance::transcript_history_conformance`]) actually certifies //! the *contract*, not one backend's accidental behavior, and so a test that @@ -19,8 +19,7 @@ use crate::transcript::{ /// A [`TranscriptHistory`] backed by a `Vec` behind a /// [`Mutex`], with no filesystem I/O at all. /// -/// Mirrors [`FileTranscriptHistory`](crate::transcript::FileTranscriptHistory)'s -/// observable contract: +/// Mirrors the driver-backed history's observable contract: /// /// * [`TranscriptRead::read_session`] returns `Ok(None)` until the first /// write (matching "the file does not exist yet"), and `Some` — including an diff --git a/crates/tinyagents-session/src/transcript.rs b/crates/tinyagents-session/src/transcript.rs index 16f67bcc4..427d76558 100644 --- a/crates/tinyagents-session/src/transcript.rs +++ b/crates/tinyagents-session/src/transcript.rs @@ -1,101 +1,45 @@ -//! Session transcript persistence for KV cache stability. -//! -//! **Source of truth**: `session_raw/{stem}.jsonl` — a *flat* directory. -//! -//! Each JSONL file starts with a single metadata line (identified by an -//! `_meta` key) followed by one JSON object per record. On every -//! write the companion `.md` file is re-rendered for human readability -//! under `sessions/{YYYY_MM_DD}/{stem}.md`; it is **never** read back — -//! all round-trip / resume logic uses the JSONL. -//! -//! ## Append-only log (Phase A — transcript-derived view) -//! -//! The JSONL is an **append-only event log**. [`write_transcript`] still -//! full-rewrites (used by one-shot writers: migrations, sub-agent runners, -//! tests), but the incremental session persistence path -//! (`persist_session_transcript`) uses [`append_transcript_turn`], which -//! never rewrites existing lines. It classifies the incoming logical -//! message set against what was last persisted: -//! -//! - **Pure extension** (previously-persisted messages are a prefix of the -//! new set): only the new tail message lines are appended. -//! - **Reduction / rewrite** (context reduction dropped or replaced earlier -//! turns — the previously-persisted set is *not* a prefix): a single -//! `{"kind":"compaction","replacement":[…]}` record is appended carrying -//! the full reduced message set. Earlier turns stay on disk untouched. -//! -//! Cumulative `_meta` totals are kept fresh without rewriting the file by -//! appending a fresh `{"_meta":{…}}` line each turn; readers take the **last** -//! `_meta` line as authoritative (line 1 remains a valid fallback for old -//! cores that only read the header). -//! -//! ### Two read paths -//! -//! - **Model context** ([`read_transcript`] / `read_transcript_jsonl`) replays -//! the log: message lines accumulate, a `compaction` record **replaces** the -//! accumulator with its `replacement`, and `interrupted:true` partial lines -//! are **skipped** (they never entered the model's context). The result is -//! byte-identical to what the old full-rewrite approach produced. -//! - **Display** ([`read_transcript_display`]) returns **every** record in file -//! order — pre-compaction history, compaction markers, and interrupted -//! partials — so the UI projection (Phase B) can render the full timeline. -//! -//! ### Compatibility -//! -//! Existing files (zero compaction records, no `version`) read identically. -//! New record kinds and fields are additive: `MessageLine` carries a -//! `#[serde(flatten)] _extra` catch-all and `MetaPayload` does not set -//! `deny_unknown_fields`, so an **old** core reading a **new** file skips -//! unknown-kind lines (a compaction record fails the `role`/`content` -//! requirement and is logged+skipped) rather than crashing. The `_meta` -//! carries a `version` field (`TRANSCRIPT_SCHEMA_VERSION`) for future readers. -//! -//! ### Typed rows (`"v":2`) +//! Session transcripts: the store-port seam hosts persist conversations +//! through, and the types they exchange. +//! +//! A conversation's transcript lives in whatever store the host installed +//! (see [`crate::port`]): [`TranscriptLocator`] resolves transcripts by thread, +//! agent or [`SessionRef`] and binds a write handle, and +//! [`TranscriptHistory`]/[`TranscriptRead`] are those handles. This crate no +//! longer reads or writes a workspace directory on that path; the only code +//! touching classic `session_raw/{stem}.jsonl` files is in [`crate::legacy`]: +//! the one-time, read-only import and `rewrite_legacy_transcripts`, which +//! rewrites them in place. +//! +//! ## Append-only log +//! +//! A transcript is an **append-only event log** whichever store holds it. An +//! ordinary turn appends only its new rows. A reduction (context reduction +//! dropped or replaced earlier turns, so the previously-persisted set is not a +//! prefix of the new one) is recorded as a compaction that carries the full +//! reduced message set; the earlier rows stay in the log. +//! +//! - **Model context** ([`TranscriptRead::read_session`]) replays the log: +//! rows accumulate, a compaction **replaces** the accumulator, and +//! interrupted partials are **skipped** (they never entered the model's +//! context). +//! - **Display** ([`TranscriptRead::read_display`]) returns **every** record in +//! order: pre-compaction history, compaction markers and interrupted +//! partials, so the UI projection ([`view`]) can render the full timeline. +//! +//! ## Typed rows //! //! The in-memory [`TranscriptMessage`] is typed: `content` is plain text, and a -//! native tool-call round or an image is carried in fields -//! (`tool_calls`, `tool_call_id`, `parts`) instead of a string envelope or an -//! `[OH_IMAGE:..]` marker inside `content`. The stored line mirrors it: -//! `{"v":2,"shape":"assistant_calls","content":"","tool_calls":[..]}`, -//! `{"v":2,"shape":"tool_result","tool_call_id":"..","content":""}` or -//! `{"v":2,"shape":"user_parts","parts":[{"type":"text",..},{"type":"image",..}]}`. -//! -//! **Read-both.** Every row written before typed rows keeps its envelope or -//! marker in `content`; both readers lift it into the typed fields -//! ([`TranscriptMessage::normalized`], lenient: any well-formed native envelope, -//! a tool-result envelope, `[OH_IMAGE:..]` markers). A line the reader cannot -//! type stays a plain text row. Existing bytes are never rewritten; a file may -//! hold both generations. A host that still builds string rows gets the same -//! typed lines for any canonical envelope/marker string. -//! -//! [`TranscriptMessage::legacy_content`] rebuilds the old string for the places -//! that must keep it (a journal, a host file shared with older binaries). -//! **Downgrade is unsupported:** a reader that predates `"v"` would see a typed -//! row's plain `content` without its tool calls or images. -//! -//! ## Storage layout -//! -//! ```text -//! {workspace}/session_raw/{stem}.jsonl ← source of truth (flat) -//! {workspace}/sessions/YYYY_MM_DD/{stem}.md ← human-readable view -//! ``` -//! -//! `stem` is `{unix_ts}_{agent_id}` for a root session, or -//! `{parent_chain}__{unix_ts}_{agent_id}` for a sub-agent. Because the -//! stem starts with the unix timestamp at agent-build time, a directory -//! listing of `session_raw/` is naturally sorted by creation time and -//! `find_latest_transcript` becomes O(scan one dir, filter by suffix) -//! — it does not depend on the calendar date, so a session that's been -//! idle for weeks resumes the same way as one from yesterday. -//! -//! ## Backward compatibility -//! -//! Older releases wrote into `session_raw/DDMMYYYY/{stem}.jsonl` (and -//! the legacy `sessions/DDMMYYYY/{stem}.md`). [`find_latest_transcript`] -//! falls back to scanning those date-grouped dirs when the flat -//! directory yields nothing, so users upgrading don't lose resume. -//! -//! ## JSONL schema +//! native tool-call round or an image is carried in fields (`tool_calls`, +//! `tool_call_id`, `parts`) instead of a string envelope or an `[OH_IMAGE:..]` +//! marker inside `content`. Rows a classic file stored with the envelope or +//! marker in `content` are lifted into the typed fields on import +//! ([`TranscriptMessage::normalized`], lenient: any well-formed native +//! envelope, a tool-result envelope, `[OH_IMAGE:..]` markers); a line the +//! reader cannot type stays a plain text row. +//! [`TranscriptMessage::legacy_content`] rebuilds the old string for the +//! places that must keep it. +//! +//! ## Classic JSONL format (read by the legacy import only) //! //! **Line 1 (meta):** //! ```json @@ -110,69 +54,62 @@ //! {"role":"tool","content":"..."} //! ``` //! -//! Only `role` and `content` are required. All other fields are optional. -//! UI-visible rows may also carry a stable `id` and `extra_metadata` so -//! the session transcript can eventually replace the separate thread -//! message log without losing message-level addressing. +//! Only `role` and `content` are required. Compaction records +//! (`{"kind":"compaction","replacement":[…]}`) and later `_meta` lines (the +//! last wins) are additive; unknown record kinds are skipped. //! //! ## Module layout //! //! | File | Role | //! |-----------------|-------------------------------------------------------------| //! | `types` | Public domain types (`TranscriptMeta`, projections, usage). | -//! | `jsonl` | Line shapes and line ⇄ message conversions. | -//! | `writer` | Full rewrite, append-only turn delta, interrupted partials. | -//! | `reader` | Model-context replay, display projection, meta-only scans. | -//! | `thread_lookup` | Thread → root transcript lookup and usage summaries. | -//! | `paths` | `session_raw` / `sessions` path resolution and resume scan. | -//! | `markdown` | Human-readable `.md` companion rendering. | -//! | `legacy_md` | Legacy HTML-comment `.md` reader. | -//! | `migration` | One-shot legacy date-grouped layout conversion. | -//! | `session` | [`SessionRef`] identity and its deterministic stem. | -//! | `adoption` | Folding pre-identity transcripts into a session. | +//! | `history` | The [`TranscriptLocator`]/[`TranscriptHistory`] store seam. | +//! | `background` | Out-of-band appends into a session's head generation. | +//! | `turn_lock` | The per-session turn lock. | +//! | `spend` | Token/cost totals read back from transcripts. | +//! | `view` | Display projection and pagination of a thread. | +//! | `session` | [`SessionRef`] identity and its deterministic stem. | +//! | `jsonl`, `reader`, `writer`, `legacy_md`, `markdown`, `paths` | The classic JSONL codec the legacy import and rewrite use. | mod adoption; -mod agent_files; mod background; mod display; mod history; +#[cfg_attr(not(feature = "legacy-import"), allow(dead_code, unused_imports))] mod jsonl; +#[cfg_attr(not(feature = "legacy-import"), allow(dead_code, unused_imports))] mod legacy_md; +#[cfg_attr(not(feature = "legacy-import"), allow(dead_code, unused_imports))] mod markdown; -mod migration; +#[cfg_attr(not(feature = "legacy-import"), allow(dead_code, unused_imports))] mod paths; +#[cfg_attr(not(feature = "legacy-import"), allow(dead_code, unused_imports))] mod reader; mod session; pub mod spend; -mod thread_lookup; +#[cfg(test)] +pub(crate) mod test_locator; mod turn_lock; mod types; pub mod view; +#[cfg_attr(not(feature = "legacy-import"), allow(dead_code, unused_imports))] mod writer; -pub use adoption::{SessionAdoption, adopt_legacy_session_transcripts}; -pub use agent_files::{AgentTranscriptFiles, agent_transcript_root}; -pub use background::{ - append_background_message, append_background_message_in, - append_background_message_with_lock_notification, -}; +pub use adoption::SessionAdoption; +pub use background::append_background_message_in; pub use display::BACKGROUND_METADATA_KEY; +#[cfg_attr(not(feature = "legacy-import"), allow(unused_imports))] +pub(crate) use display::with_background as display_with_background_row; pub(crate) use display::{display_from_session, display_message_of, partial_display_message}; pub(crate) use history::same_transcript_messages; pub use history::{ - FileTranscriptHistory, FileTranscriptLocator, TranscriptHistory, TranscriptLocator, - TranscriptPartial, TranscriptRead, TranscriptTurn, TruncateCut, + TranscriptHistory, TranscriptLocator, TranscriptPartial, TranscriptRead, TranscriptTurn, + TruncateCut, }; pub(crate) use jsonl::stamped_rows; -pub use legacy_md::read_transcript_legacy_md; -pub use migration::{TranscriptLayoutMigration, migrate_layout_if_needed}; -pub use paths::{find_latest_transcript, resolve_keyed_transcript_path}; -pub use reader::{read_transcript, read_transcript_display}; +#[cfg_attr(not(feature = "legacy-import"), allow(unused_imports))] +pub(crate) use reader::{read_transcript, read_transcript_display}; pub use session::{SessionRef, session_stem}; -pub use thread_lookup::{ - find_root_transcript_for_thread, find_root_transcript_for_thread_scoped, - find_root_transcripts_for_thread, read_thread_usage_summary, -}; pub use turn_lock::{SessionTurnGuard, lock_session_turn}; pub use types::{ BackgroundAppend, BackgroundAppendOutcome, BackgroundOrigin, CompactionMarker, DisplayMessage, @@ -180,18 +117,19 @@ pub use types::{ ThreadTranscript, ThreadTranscripts, ToolFailure, TranscriptMediaRef, TranscriptMessage, TranscriptMeta, TranscriptPart, TranscriptToolCall, TurnUsage, UsageCostSource, }; +#[cfg_attr(not(feature = "legacy-import"), allow(unused_imports))] pub(crate) use writer::write_transcript_with_tools; -pub use writer::{ - append_interrupted_partial, append_tools_record, append_transcript_turn, - append_transcript_turn_with_partial, write_transcript, write_transcript_if_absent, - write_transcript_if_absent_with_tools, +#[cfg(test)] +pub(crate) use writer::{ + append_interrupted_partial, append_tools_record, append_transcript_turn, write_transcript, + write_transcript_if_absent, }; // ── Tests ───────────────────────────────────────────────────────────── #[cfg(test)] -#[path = "transcript/locator_parity_tests.rs"] -mod locator_parity_tests; +pub(crate) use paths::resolve_keyed_transcript_path; + #[cfg(test)] #[path = "transcript/transcript_tests.rs"] mod test; diff --git a/crates/tinyagents-session/src/transcript/README.md b/crates/tinyagents-session/src/transcript/README.md index 77e419f78..72fbe869e 100644 --- a/crates/tinyagents-session/src/transcript/README.md +++ b/crates/tinyagents-session/src/transcript/README.md @@ -1,39 +1,39 @@ # `session::transcript` — durable, provider-neutral session transcripts -An append-only, JSONL-backed transcript store used for KV-cache-stable -resume: `{workspace}/session_raw/{stem}.jsonl` is the source of truth, with a -human-readable `.md` companion re-rendered on every write. See the extensive -module doc in `../transcript.rs` for the on-disk schema, the append-only log -format (pure extension vs. compaction records), and backward compatibility -with the legacy date-grouped layout — this README covers the module's shape -and how its files relate rather than repeating that. +The store-port seam hosts persist conversations through. A transcript lives in +whatever store the host installed (`crate::port`: the `tinystoragedrivers` +backed `DriverSessionStores`, or the in-memory double); this module defines +the traits and types, and nothing in it reads or writes a workspace directory. +The only code touching classic `session_raw/{stem}.jsonl` files is in +`crate::legacy`: the one-time, read-only import (`import_transcripts_once`, +`WorkspaceStores`) and `rewrite_legacy_transcripts`, which rewrites them in +place. Both parse them with the internal JSONL codec kept here +(`jsonl.rs`, `reader.rs`, `legacy_md.rs`). See the module doc in +`../transcript.rs` for the append-only log model and the classic line schema. ## Why this exists, and how it differs from `crate::ops` / `crate::run_ledger` -This module persists the literal message stream a model saw, byte-for-byte, -so a resumed run gets exactly the same context (and, for providers with -prompt caching, hits the same cache) it would have if the process never -restarted. `crate::ops` records a *searchable summary* of a session -(messages, tool calls, cost) for history/search; `crate::run_ledger` records -*execution state* (run status, checkpoints, coordination). All three can -describe the same conversation from different angles and are written -independently — a host that only wants search doesn't need transcripts, and -vice versa. +This module persists the literal message stream a model saw, so a resumed run +gets exactly the same context (and, for providers with prompt caching, hits +the same cache) it would have if the process never restarted. `crate::ops` +records a *searchable summary* of a session (messages, tool calls, cost) for +history/search; `crate::run_ledger` records *execution state* (run status, +checkpoints, coordination). All three can describe the same conversation from +different angles and are written independently. ## Public surface -Re-exported from `transcript::` (see `../transcript.rs`); reachable as -`session::transcript::`. +Re-exported from `transcript::` (see `../transcript.rs`). ### Types (`types.rs`) - `TranscriptMessage` / `TranscriptToolCall` — the durable, provider-neutral message and tool-call shapes. Deliberately not an inference message: this - is an on-disk compatibility boundary, and hosts convert at their own edge. -- `TranscriptMeta` / `SessionTranscript` — the `_meta` header and the parsed + is a compatibility boundary, and hosts convert at their own edge. +- `TranscriptMeta` / `SessionTranscript` — the `_meta` header and the model-context transcript (`meta` + exact message array). The optional `_meta.prefix_message_count` records how many leading messages were frozen - prompt tiers in this generation; older files omit it. + prompt tiers in this generation. - `MessageUsage` / `TurnUsage` — per-turn usage and provenance attached to the last assistant message of a turn. `input` / `output` / `cached_input` sum every provider call of the turn (its spend); `last_call_input` / @@ -44,39 +44,43 @@ Re-exported from `transcript::` (see `../transcript.rs`); reachable as falling back to the mean request size for records written before them. - `DisplayMessage` / `CompactionMarker` / `DisplayRecord` / `DisplaySessionTranscript` — the display projection, which (unlike the - model-context view) preserves every record in file order. + model-context view) preserves every record in log order. - `ToolFailure` — provider-neutral failure status for a tool-result row. -### Writing (`writer.rs`) - -- `write_transcript` — one-shot **full rewrite**; used by migrations, - sub-agent runners, and tests. -- `append_transcript_turn` — the incremental, append-only path: appends new - tail lines on pure extension, or a `compaction` record on context - reduction. Never rewrites existing lines. -- `append_interrupted_partial` — appends a display-only partial assistant - answer when a stream was cancelled mid-turn; skipped by the model-context - reader. +### Store seam (`history.rs`) + +- `TranscriptLocator` — resolves transcripts by thread, agent or + `SessionRef`, lists a thread's transcripts (`thread_transcripts`: the roots + with session chains oldest first and pre-identity roots dropped once a + session root exists, plus sub-agent transcripts), binds a session's write + handle (`open_session` / `open_stem`), and opens compaction generations + (`begin_generation`, `begin_generation_from_baseline`, + `truncate_into_next_generation`). `destination_key()` names the durable + destination so rebuilt locators over one store compare equal. +- `TranscriptHistory` / `TranscriptRead` — the write and read handles a turn + path holds. `TranscriptRead` has no write methods, so a discovered + transcript cannot be appended to by construction. `read_session()` is the + model-context replay (compactions replace the accumulator, interrupted + partials are skipped); `read_display()` is the display projection (every + record, including pre-compaction history, compaction markers and interrupted + partials). +- `TranscriptTurn` — the borrowed argument bundle for + `TranscriptHistory::append_turn`. ### Background appends (`background.rs`, `turn_lock.rs`) -- `append_background_message(&FileTranscriptLocator, &SessionRef, +- `append_background_message_in(&dyn TranscriptLocator, &SessionRef, TranscriptMessage, BackgroundAppend) -> Result` (async) — appends one plain assistant message to a session's **head** generation from outside any turn (e.g. a scheduled job reporting back into the chat that created it). Outcomes: `Appended { generation }`, - `Duplicate { generation }` (the idempotency key is already in the head), - `StaleGeneration { expected, head }` (a pinned `expected_generation` no - longer matches), `NoSession` (no transcript; none is created). The line is - an ordinary message line plus an additive top-level - `"background": {"idempotency_key", "provenance"}` field, so the - model-context reader resumes it as an assistant row and the display reader - exposes the origin as `DisplayMessage::background`. -- `append_background_message_in(&dyn TranscriptLocator, …)` — the same - contract through any locator. A file-backed one (`file_backing()`) writes - the identical JSONL line; any other store appends through its history - handle and keeps the provenance in the row's `extra_metadata` under - `BACKGROUND_METADATA_KEY`, which the display read lifts back out. + `Duplicate { generation }` (the idempotency key is already held by the head + generation's *live* context: a key dropped by a compaction, in the same + generation or a successor, is free again), `StaleGeneration { expected, + head }` (a pinned `expected_generation` no longer matches), `NoSession` (no + transcript; none is created). The provenance is kept in the row's + `extra_metadata` under `BACKGROUND_METADATA_KEY`, which the display read + lifts back out as `DisplayMessage::background`. - `lock_session_turn(&dyn TranscriptLocator, &SessionRef) -> Option` (async) — the per-session, in-process turn lock, keyed by `destination_key()` + the generation-0 stem. A live turn holds it @@ -85,30 +89,12 @@ Re-exported from `transcript::` (see `../transcript.rs`); reachable as awaits it, so it always lands between turns. Not reentrant: never await a background append into a session from inside that session's own turn. -### Reading (`reader.rs`) - -- `read_transcript` — model-context replay: compaction records replace the - accumulator, interrupted partials are skipped. Falls back to the legacy - `.md` reader when no `.jsonl` exists. -- `read_transcript_display` — display projection: every record in file - order, including pre-compaction history and interrupted partials. -- `TranscriptRead::read_display()` — the same projection through any handle - (a driver store replays its entry log: a later `set` is a compaction - marker, a `clear` an empty one, a partial an interrupted record; entries - keep no clock, so markers carry no `ts`). -- `TranscriptLocator::thread_transcripts(thread_id, agent_id)` — the roots - (session chains oldest first, pre-identity roots dropped once a session - root exists) and sub-agent transcripts backing a thread, as - `ThreadTranscripts`. `view::get_page_in` and `spend::thread_spend_in` take a - locator and use it, so a host never names a transcript file. - ### Spend and cost provenance (`spend.rs`) - `transcript_spend(&SessionTranscript)` sums usage in the logical context. Compaction removes older rows from that view; use - `thread_spend(workspace_dir, thread_id)` (or `thread_spend_in(locator, - thread_id)` for any store) for the full append history. -- `thread_spend` returns `ThreadSpend`: `root` contains the orchestrator's + `thread_spend_in(locator, thread_id)` for the full append history. +- `thread_spend_in` returns `ThreadSpend`: `root` contains the orchestrator's own spend, and `subagents` groups descendant runs by archetype, including children whose headers name a different worker thread. Retained root rows across generations are counted once. @@ -139,108 +125,47 @@ Present the known priced amount with its source and separately re-price the unpriced tokens at host-selected rates, or report their cost as unavailable. Do not describe an estimated amount as a provider charge. -### Paths and resume (`paths.rs`) - -- `resolve_keyed_transcript_path` — resolves/creates - `session_raw/{stem}.jsonl` for a given stem. -- `find_latest_transcript` — newest-transcript scan for an agent, with a - fallback to the legacy `session_raw/DDMMYYYY/` layout. - -### Thread lookups (`thread_lookup.rs`) - -- `find_root_transcript_for_thread` / `find_root_transcript_for_thread_scoped` - / `find_root_transcripts_for_thread` — locate a thread's root transcript(s) - by `_meta.thread_id`, optionally scoped by `_meta.agent_id` to avoid - splicing one agent's history into another's when a `thread_id` is reused. -- `read_thread_usage_summary` — aggregated token/cost usage for a thread - across all its root transcripts, plus a per-archetype sub-agent breakdown. - -### Legacy compatibility (`legacy_md.rs`) - -- `read_transcript_legacy_md` — parses the pre-JSONL HTML-comment `.md` - format; one-release migration compat. +### View (`view/`) -### History/locator seam (`history.rs`) - -- `TranscriptHistory` / `TranscriptRead` / `TranscriptLocator` — the traits a - host turn path holds (`Arc`) instead of calling the free functions - directly, so a discovered transcript that turns out to be a legacy `.md` - file cannot be appended to by construction (`TranscriptRead` has no write - methods). -- `FileTranscriptHistory` / `FileTranscriptLocator` — the default - implementations, each a thin wrapper over exactly one free function per - method. -- `TranscriptTurn` — the borrowed argument bundle for - `TranscriptHistory::append_turn`, shaped to mirror - `append_transcript_turn`'s signature one-for-one. +`view::get_page_in(locator, thread_id, agent_id, cursor, limit)` projects a +thread's listed transcripts and returns one newest-first page; +`view::project_thread_in` returns the whole projection (optionally placing +sub-agents by a `SpawnCallLookup`). See `view/README.md`. ## File-by-file map | File | Role | | --- | --- | | `types.rs` | Public domain types: messages, meta, usage, display projection. | -| `jsonl.rs` | Internal JSONL line shapes and line ⇄ message conversions. | -| `writer.rs` | Full rewrite, append-only turn delta, interrupted partials. | -| `reader.rs` | Model-context replay, display projection, meta-only/usage scans. | -| `paths.rs` | Path resolution, sanitization, and the newest-transcript resume scan. | -| `thread_lookup.rs` | Thread → root transcript lookup and usage summaries. | -| `spend.rs` | Full-history thread spend and cost provenance splits. | -| `markdown.rs` | Human-readable `.md` companion rendering (never read back). | -| `legacy_md.rs` | Legacy HTML-comment `.md` reader (migration compat). | | `history.rs` | `TranscriptHistory` / `TranscriptRead` / `TranscriptLocator` seam. | | `background.rs` | Out-of-band, idempotent assistant appends into a session's head. | | `turn_lock.rs` | Per-session async turn lock shared by live turns and background appends. | -| `*_tests.rs` | Module-local unit tests (`transcript_tests.rs`, `session_tests.rs`, ...). | - -`../transcript.rs` (the module root, one level up) wires these together, -re-exports the public surface, and carries the full format specification in -its module doc. +| `spend.rs` | Full-history thread spend and cost provenance splits. | +| `session.rs` | `SessionRef` identity and its deterministic stem. | +| `display.rs` | Row ⇄ display projection helpers. | +| `jsonl.rs` | Internal classic JSONL line shapes and line ⇄ message conversions. | +| `reader.rs` | Classic-file model-context replay and display projection (legacy import). | +| `writer.rs` | Classic-file full rewrite (`legacy::rewrite_legacy_transcripts`); turn-append helpers are test fixtures. | +| `paths.rs`, `markdown.rs` | Stem sanitisation and the `.md` companion the classic rewrite re-renders. | +| `legacy_md.rs` | Legacy HTML-comment `.md` reader (import compatibility). | +| `test_locator.rs` | Test fixtures: classic files imported into a driver-backed locator. | +| `*_tests.rs` | Module-local unit tests. | ## Operational constraints -**The JSONL is the only source of truth; the `.md` companion is never read -back.** All round-trip and resume logic reads `.jsonl`. A companion write -failure is logged and swallowed (`writer.rs::render_md_companion`) so it can -never take down state persistence. - -**`append_transcript_turn` never rewrites existing lines.** It diffs the -incoming logical message set against what was previously persisted -(`common_prefix_len`, compared on `role`/`content`/`id`): a prefix match -appends only the new tail; anything else appends a full `compaction` record -carrying the reduced set, leaving earlier lines untouched. - -**Readers must take the *last* `_meta` line, not the first.** Cumulative -totals are kept fresh by appending a fresh `_meta` line each turn rather than -rewriting line 1; line 1 remains a valid fallback only for old cores that -never learned to look further. - -**Interrupted partials are display-only.** `interrupted: true` lines are -skipped by `read_transcript`'s model-context replay so a resumed context -never carries a truncated assistant answer, but they are included by -`read_transcript_display` for the UI timeline. - -**Root vs. sub-agent transcripts are distinguished by stem shape**, not a -separate directory: a stem containing `__` is a sub-agent transcript -(`{parent_chain}__{unix_ts}_{agent_id}`); a stem without it is a root -session (`{unix_ts}_{agent_id}`). `thread_lookup.rs` and `paths.rs` both rely -on this to avoid folding delegated sub-agent history into a thread's main -timeline. - -**`FileTranscriptHistory`'s generic trait path always re-reads the file -before writing** (`write_logical_set`), unlike the turn path proper, which -tracks its previously-persisted set in memory. This is deliberate: messages -reconstructed from disk have had turn-usage fields hoisted and -`failure`/`failure_detail` lifted out of `extra_metadata`, so feeding that -back in as `prev` for an in-memory-tracked caller would mismatch at the -first such message and force a full compaction record every turn. - -**File history mutations are coordinated across processes.** A persistent -advisory lock under the workspace's `.transcript-locks/` directory serializes -each file mutation. `begin_generation` keeps the successor's lock from its -existence check through its first successful write, so a crash releases the -reservation and leaves no phantom generation. Turn appends also compare the -caller baseline with the current logical transcript while holding that lock; -a stale session receives a persistence error instead of writing a compaction -that hides another session's newer turns. Lock files are deliberately -retained: removing a lock inode while another process waits could split future -writers across different locks. +**Compaction never erases.** A reduction is recorded as a compaction (or a +successor generation), and the earlier rows stay in the log: the model reads +the reduced context, the display read shows everything. + +**Interrupted partials are display-only.** They are skipped by the +model-context replay so a resumed context never carries a truncated assistant +answer, but they are included by the display read for the UI timeline. + +**Root vs. sub-agent transcripts are distinguished by stem shape**: a stem +containing `__` is a sub-agent transcript (`{parent_chain}__{unix_ts}_{agent_id}`); +a stem without it is a root session. Listing and spend rely on this to avoid +folding delegated sub-agent history into a thread's main timeline. + +**Classic-file readers must take the *last* `_meta` line, not the first.** +Cumulative totals were kept fresh by appending a fresh `_meta` line each +turn; line 1 remains a valid fallback only for the oldest files. diff --git a/crates/tinyagents-session/src/transcript/adoption.rs b/crates/tinyagents-session/src/transcript/adoption.rs index 041d041c3..07e8bca88 100644 --- a/crates/tinyagents-session/src/transcript/adoption.rs +++ b/crates/tinyagents-session/src/transcript/adoption.rs @@ -1,28 +1,7 @@ -//! Adoption of conversations written before session identity existed. -//! -//! A host that minted a transcript stem per process left one conversation -//! spread across several root transcripts, and resume only ever loaded the -//! newest of them — so the opening turns of a thread became unreachable to the -//! model while the UI, which concatenates every matching root, still displayed -//! them. -//! -//! [`adopt_legacy_session_transcripts`] closes that gap once per conversation. -//! The first time a session is resumed and has no transcript of its own, the -//! legacy roots for its thread are read in `_meta.created` order and written -//! into the session's generation 0. The legacy files are never modified, -//! moved, or deleted: adoption only ever *adds* the file the session layer -//! will use from then on. +//! The result of folding a conversation's pre-identity transcripts into its +//! session ([`TranscriptLocator::adopt_legacy`](super::TranscriptLocator::adopt_legacy)). -use anyhow::{Context, Result}; -use std::path::{Path, PathBuf}; -use std::time::{Duration, SystemTime}; - -use super::paths::resolve_keyed_transcript_path; -use super::reader::read_transcript; -use super::session::{SessionRef, session_stem}; -use super::thread_lookup::find_root_transcripts_for_thread_reporting_unreadable; -use super::types::{TranscriptMessage, TranscriptMeta}; -use super::writer::write_transcript_if_absent_with_tools; +use std::path::PathBuf; /// What adoption did for one session. #[derive(Debug, Clone, PartialEq, Eq)] @@ -34,347 +13,3 @@ pub struct SessionAdoption { /// Messages carried over. pub messages: usize, } - -/// An adoption lock older than this is assumed to belong to a process that -/// crashed mid-adoption rather than one still working, and is reclaimed -/// rather than left blocking every future resume forever. -const STALE_LOCK_AGE: Duration = Duration::from_secs(60); - -/// Fold the legacy root transcripts of `session`'s thread into its own -/// generation 0, if it has none yet and any exist. -/// -/// Returns `Ok(None)` when there is nothing to do — the session already has a -/// transcript, the thread has no legacy roots, or a concurrent adopter is -/// already handling this session — which makes repeat calls harmless. -/// -/// `thread_id` is passed separately from `session` because the legacy files are -/// keyed by `_meta.thread_id`, which is what the host had before it had a -/// session key, and the two are not required to be the same string. -// keep until v3.0 (review after 2027-01-11): hosts must publish an explicit -// pre-SessionRef upgrade/import boundary before removing this one-shot adopter. -// Offline workspaces may still contain timestamped transcripts; absence of new -// writes does not prove that every durable conversation has been adopted. -pub fn adopt_legacy_session_transcripts( - workspace_dir: &Path, - session: &SessionRef, - thread_id: &str, - seed_meta: &TranscriptMeta, -) -> Result> { - let stem = session_stem(session); - let destination = resolve_keyed_transcript_path(workspace_dir, &stem)?; - if destination.exists() { - return Ok(None); - } - - // Two processes can otherwise both pass the check above, both scan the - // same legacy roots, and race to write `destination`: whichever full - // rewrite lands last wins and the other's turns (if it observed a - // different, possibly more complete, set of roots) are gone with no way - // to notice, because both callers now see `destination.exists()` and - // return `Ok(None)`. Holding this lock for the whole check-scan-write - // makes one adoption per session the only one that can ever run at a - // time; every other concurrent caller backs off and treats the winner's - // result as its own. - let lock_path = adoption_lock_path(&destination); - let Some(_lock) = AdoptionLock::acquire(&lock_path)? else { - tracing::debug!( - "[transcript-adoption] session={stem} adoption already in progress elsewhere; \ - skipping" - ); - return Ok(None); - }; - // Re-check now that the lock is held: another process may have finished - // adoption (or written the session's own first turn) between the - // unlocked check above and acquiring the lock. - if destination.exists() { - return Ok(None); - } - - // Oldest first, by `_meta.created`. Anything already pointing at this - // session's own file is excluded so a partially-adopted workspace cannot - // fold a file into itself. - // - // The scan itself (not just the fold loop below) can hit a candidate it - // cannot read at all — before it even knows whether that file's - // `_meta.thread_id` would have matched this thread. A silently-dropped - // candidate here would let the fold below complete and finalize on only - // the *other*, readable roots: the destination would then exist as the - // idempotency marker, and the unreadable file's turns would never be - // retried even once it became readable again. `unreadable` makes that - // case visible so it can defer the whole call instead. - let (legacy, unreadable): (Vec, bool) = - find_root_transcripts_for_thread_reporting_unreadable(workspace_dir, thread_id); - let legacy: Vec = legacy - .into_iter() - .filter(|path| path != &destination) - .collect(); - anyhow::ensure!( - !unreadable, - "deferring adoption: at least one root transcript in this workspace could not be read, \ - and it may belong to thread {thread_id}" - ); - if legacy.is_empty() { - return Ok(None); - } - - let mut messages: Vec = Vec::new(); - let mut meta = seed_meta.clone(); - meta.session_id = Some(session.session_id()); - meta.parent_session_id = session.parent_session_id(); - meta.thread_id = Some(thread_id.to_string()); - meta.turn_count = 0; - meta.prefix_message_count = None; - meta.input_tokens = 0; - meta.output_tokens = 0; - meta.cached_input_tokens = 0; - meta.charged_amount_usd = 0.0; - let mut earliest_created: Option = None; - let mut latest_updated: Option = None; - let mut adopted = Vec::new(); - let mut tools = None; - - for path in legacy { - // A legacy candidate must be read to be filtered, so an unreadable - // one fails the whole call rather than being silently skipped. The - // destination is this call's own idempotency marker: skipping it and - // writing anyway would create that marker over an *incomplete* fold, - // and because `destination.exists()` short-circuits every later - // call, the skipped file's turns would never be retried — even after - // the file became readable again. Failing instead leaves nothing on - // disk, so a later resume simply tries the whole fold again. - let transcript = read_transcript(&path).with_context(|| { - format!( - "legacy root {} is unreadable; deferring adoption rather than finalizing a \ - fold that would permanently drop its turns", - path.display() - ) - })?; - - // Session-identified files are not pre-identity legacy transcripts: - // they are either one of this thread's *other* agents (when - // `session.agent_id` is set, matching the rule - // `find_root_transcript_for_thread_scoped` already applies), or a - // session file — including one already adopted — that happens to - // share this thread id. Folding either in would mix another agent's - // history into this one, or duplicate content that adoption already - // recovered once. - if transcript.meta.session_id.is_some() { - continue; - } - if let Some(expected_agent) = session.agent_id.as_deref() - && transcript.meta.agent_id.as_deref() != Some(expected_agent) - { - continue; - } - - if messages.is_empty() && !transcript.messages.is_empty() { - // The folded transcript starts with this source's messages. A - // later source may have its own prompt, but its boundary is not - // the prefix of the combined conversation. - meta.prefix_message_count = transcript.meta.prefix_message_count; - } - messages.extend(transcript.messages); - if transcript.tools.is_some() { - tools = transcript.tools; - } - meta.turn_count += transcript.meta.turn_count; - meta.input_tokens += transcript.meta.input_tokens; - meta.output_tokens += transcript.meta.output_tokens; - meta.cached_input_tokens += transcript.meta.cached_input_tokens; - meta.charged_amount_usd += transcript.meta.charged_amount_usd; - if !transcript.meta.created.is_empty() - && earliest_created - .as_ref() - .is_none_or(|earliest| transcript.meta.created < *earliest) - { - earliest_created = Some(transcript.meta.created.clone()); - } - if !transcript.meta.updated.is_empty() - && latest_updated - .as_ref() - .is_none_or(|latest| transcript.meta.updated > *latest) - { - latest_updated = Some(transcript.meta.updated.clone()); - } - adopted.push(path); - } - - if messages.is_empty() { - return Ok(None); - } - if let Some(created) = earliest_created { - meta.created = created; - } - if let Some(updated) = latest_updated { - meta.updated = updated; - } - - // `write_transcript_if_absent` never overwrites an existing destination: - // the adoption lock only serializes competing *adopters*, not a normal - // session turn independently creating this same first transcript while - // adoption is still scanning. If that happened, `destination` now holds - // real conversation data that must not be clobbered with an adoption - // fold that started before it existed — so a lost race here discards - // this call's fold and reports `Ok(None)`, the same as "nothing to do". - if !write_transcript_if_absent_with_tools(&destination, &messages, &meta, tools.as_ref())? { - tracing::debug!( - "[transcript-adoption] session={stem} lost the race to a concurrent write; \ - discarding this fold" - ); - return Ok(None); - } - tracing::info!( - "[transcript-adoption] session={stem} adopted {} legacy root(s) totalling {} message(s)", - adopted.len(), - messages.len() - ); - Ok(Some(SessionAdoption { - path: destination, - messages: messages.len(), - adopted, - })) -} - -/// The exclusive-create lock path guarding one destination's adoption. -fn adoption_lock_path(destination: &Path) -> PathBuf { - let mut file_name = destination - .file_name() - .map(|name| name.to_os_string()) - .unwrap_or_default(); - file_name.push(".adopting"); - destination.with_file_name(file_name) -} - -/// An exclusive-create file lock held for the duration of one adoption. -/// -/// Backed by [`std::fs::OpenOptions::create_new`] rather than an in-process -/// mutex because concurrent adopters are typically separate processes (two -/// hosts, or a process restarted mid-turn) with no shared memory to -/// synchronize on. -/// -/// The correctness boundary that actually prevents data loss is -/// [`write_transcript_if_absent`]'s atomic publish, not this lock — the lock -/// is an efficiency optimization that lets concurrent *adopters* avoid -/// redundant scanning, not the thing standing between two writers and a -/// corrupted file. That is deliberate: reclaiming a lock purely by file age -/// (below) can never be made fully race-free without OS-level leases this -/// module does not have, so the design accepts an occasional double-scan -/// under reclamation rather than a lock a crashed owner can block forever — -/// knowing that even a full double-scan-and-write race resolves safely -/// through the atomic publish underneath it. -struct AdoptionLock { - path: PathBuf, - /// Written into the lock file's content at creation. [`Drop`] reads the - /// file back and only removes it when the content still matches this - /// token, so a lock this instance *lost* ownership of (reclaimed by - /// another process as stale, see [`Self::acquire`]) is never unlinked - /// out from under its new, legitimate owner. - token: String, -} - -impl AdoptionLock { - /// Acquires the lock at `path`, or returns `Ok(None)` when another - /// process already holds a fresh one. - /// - /// A lock older than [`STALE_LOCK_AGE`] is reclaimed on the assumption - /// that its owner crashed before releasing it — otherwise a single crash - /// mid-adoption would block that session's adoption forever. The - /// remaining reclaim-under-a-live-owner race this cannot fully close - /// (two processes both observe the same stale lock; see [`Self::token`] - /// for how `Drop` avoids compounding it) resolves safely because the - /// eventual writes still go through [`write_transcript_if_absent`]. - fn acquire(path: &Path) -> Result> { - let token = lock_token(); - match Self::create_exclusive(path, &token)? { - true => Ok(Some(Self { - path: path.to_path_buf(), - token, - })), - false if lock_is_stale(path) => { - tracing::warn!( - "[transcript-adoption] reclaiming stale adoption lock {}", - path.display() - ); - let _ = std::fs::remove_file(path); - match Self::create_exclusive(path, &token)? { - true => Ok(Some(Self { - path: path.to_path_buf(), - token, - })), - // Lost the race to reclaim it — the winner will finish - // the adoption (or lose its own race to a normal write, - // safely, via `write_transcript_if_absent`). - false => Ok(None), - } - } - false => Ok(None), - } - } - - /// Attempts to create `path` exclusively with `token` as its content. - /// Returns `Ok(true)` on success, `Ok(false)` when `path` already - /// exists. - fn create_exclusive(path: &Path, token: &str) -> Result { - use std::io::Write; - match std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(path) - { - Ok(mut file) => { - // Best-effort: the exclusive create above is what actually - // establishes ownership. A failed or partial token write - // only widens `Drop`'s safety margin (it would then decline - // to remove a lock it cannot positively confirm as its own), - // it never narrows it. - let _ = file.write_all(token.as_bytes()); - Ok(true) - } - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Ok(false), - Err(error) => { - Err(error).with_context(|| format!("create adoption lock {}", path.display())) - } - } - } - - /// Whether this instance's token is still what is on disk at `path`. - fn still_owns(&self) -> bool { - std::fs::read_to_string(&self.path).is_ok_and(|contents| contents == self.token) - } -} - -impl Drop for AdoptionLock { - fn drop(&mut self) { - if self.still_owns() { - let _ = std::fs::remove_file(&self.path); - } - } -} - -fn lock_is_stale(path: &Path) -> bool { - std::fs::metadata(path) - .and_then(|metadata| metadata.modified()) - .and_then(|modified| { - SystemTime::now() - .duration_since(modified) - .map_err(|_| std::io::Error::other("clock went backwards")) - }) - .is_ok_and(|age| age > STALE_LOCK_AGE) -} - -/// A per-process-unique token for one lock acquisition: pid plus a -/// monotonically increasing in-process counter. Not a cryptographic nonce — -/// it only has to distinguish this acquisition from acquisitions by other -/// processes and from earlier acquisitions in this one, which pid+counter -/// already guarantees deterministically and without any external -/// dependency. -fn lock_token() -> String { - use std::sync::atomic::{AtomicU64, Ordering}; - static COUNTER: AtomicU64 = AtomicU64::new(0); - let nonce = COUNTER.fetch_add(1, Ordering::Relaxed); - format!("{}-{nonce}", std::process::id()) -} - -#[cfg(test)] -#[path = "adoption_tests.rs"] -mod test; diff --git a/crates/tinyagents-session/src/transcript/adoption_tests.rs b/crates/tinyagents-session/src/transcript/adoption_tests.rs deleted file mode 100644 index 077fd2339..000000000 --- a/crates/tinyagents-session/src/transcript/adoption_tests.rs +++ /dev/null @@ -1,860 +0,0 @@ -use super::*; -use crate::transcript::{ - FileTranscriptLocator, MessageUsage, TranscriptLocator, TranscriptToolCall, TurnUsage, - append_tools_record, append_transcript_turn, read_transcript, write_transcript, - write_transcript_if_absent, -}; -use tempfile::tempdir; - -fn legacy_meta(created: &str, updated: &str, thread_id: &str) -> TranscriptMeta { - TranscriptMeta { - session_id: None, - parent_session_id: None, - agent_name: "orchestrator".into(), - agent_id: Some("orchestrator".into()), - agent_type: Some("root".into()), - dispatcher: "native".into(), - provider: None, - model: Some("model".into()), - created: created.into(), - updated: updated.into(), - turn_count: 1, - prefix_message_count: None, - input_tokens: 10, - output_tokens: 5, - cached_input_tokens: 2, - charged_amount_usd: 0.5, - thread_id: Some(thread_id.into()), - task_id: None, - } -} - -fn write_legacy(dir: &Path, stem: &str, created: &str, body: &str, thread_id: &str) { - let path = resolve_keyed_transcript_path(dir, stem).unwrap(); - write_transcript( - &path, - &[TranscriptMessage::new("user", body)], - &legacy_meta(created, created, thread_id), - None, - ) - .unwrap(); -} - -/// The shape that cost a real user their thread: one conversation spread -/// across three timestamped stems, of which resume only ever saw the newest. -#[test] -fn legacy_roots_fold_into_one_session_in_created_order() { - let dir = tempdir().unwrap(); - let thread = "thread-9fa08c44"; - write_legacy( - dir.path(), - "1790062247_orchestrator", - "2026-09-22T07:30:47Z", - "i want to plan a trip to kashmir", - thread, - ); - write_legacy( - dir.path(), - "1790100297_orchestrator", - "2026-09-22T18:05:22Z", - "hello?", - thread, - ); - write_legacy( - dir.path(), - "1790100352_orchestrator", - "2026-09-22T18:06:27Z", - "what did i ask here?", - thread, - ); - - let session = SessionRef::scoped(thread, "orchestrator"); - let adoption = adopt_legacy_session_transcripts( - dir.path(), - &session, - thread, - &legacy_meta("", "", thread), - ) - .unwrap() - .expect("three legacy roots should be adopted"); - - assert_eq!(adoption.adopted.len(), 3); - assert_eq!(adoption.messages, 3); - - let adopted = read_transcript(&adoption.path).unwrap(); - let contents: Vec<&str> = adopted - .messages - .iter() - .map(|message| message.content.as_str()) - .collect(); - assert_eq!( - contents, - [ - "i want to plan a trip to kashmir", - "hello?", - "what did i ask here?" - ] - ); - assert_eq!(adopted.meta.session_id, Some(session.session_id())); - assert_eq!(adopted.meta.thread_id.as_deref(), Some(thread)); -} - -#[test] -fn adoption_preserves_the_first_sources_frozen_prefix_count() { - let dir = tempdir().unwrap(); - let thread = "thread-prefix-adoption"; - let first = resolve_keyed_transcript_path(dir.path(), "1000_orchestrator").unwrap(); - let mut first_meta = legacy_meta("2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z", thread); - first_meta.prefix_message_count = Some(1); - write_transcript( - &first, - &[ - TranscriptMessage::new("system", "stable"), - TranscriptMessage::new("system", "history summary"), - TranscriptMessage::new("user", "first"), - ], - &first_meta, - None, - ) - .unwrap(); - let second = resolve_keyed_transcript_path(dir.path(), "2000_orchestrator").unwrap(); - let mut second_meta = legacy_meta("2026-02-01T00:00:00Z", "2026-02-01T00:00:00Z", thread); - second_meta.prefix_message_count = Some(2); - write_transcript( - &second, - &[TranscriptMessage::new("user", "later")], - &second_meta, - None, - ) - .unwrap(); - - let session = SessionRef::scoped(thread, "orchestrator"); - let adopted = adopt_legacy_session_transcripts( - dir.path(), - &session, - thread, - &legacy_meta("", "", thread), - ) - .unwrap() - .unwrap(); - - assert_eq!( - read_transcript(&adopted.path) - .unwrap() - .meta - .prefix_message_count, - Some(1) - ); -} - -#[test] -fn adoption_sums_usage_and_spans_the_whole_conversation() { - let dir = tempdir().unwrap(); - let thread = "thread-1"; - write_legacy(dir.path(), "1000_a", "2026-01-01T00:00:00Z", "one", thread); - write_legacy(dir.path(), "2000_a", "2026-02-02T00:00:00Z", "two", thread); - - let session = SessionRef::scoped(thread, "orchestrator"); - let adoption = adopt_legacy_session_transcripts( - dir.path(), - &session, - thread, - &legacy_meta("", "", thread), - ) - .unwrap() - .unwrap(); - - let meta = read_transcript(&adoption.path).unwrap().meta; - assert_eq!(meta.turn_count, 2); - assert_eq!(meta.input_tokens, 20); - assert_eq!(meta.output_tokens, 10); - assert_eq!(meta.cached_input_tokens, 4); - assert_eq!(meta.charged_amount_usd, 1.0); - assert_eq!(meta.created, "2026-01-01T00:00:00Z"); - assert_eq!(meta.updated, "2026-02-02T00:00:00Z"); -} - -#[test] -fn adoption_preserves_the_newest_legacy_tool_snapshot() { - let dir = tempdir().unwrap(); - let thread = "thread-tools"; - write_legacy(dir.path(), "1000_a", "2026-01-01T00:00:00Z", "one", thread); - write_legacy(dir.path(), "2000_a", "2026-01-02T00:00:00Z", "two", thread); - let newer = resolve_keyed_transcript_path(dir.path(), "2000_a").unwrap(); - append_tools_record(&newer, &serde_json::json!([{"name": "newer"}])).unwrap(); - - let session = SessionRef::scoped(thread, "orchestrator"); - let adoption = adopt_legacy_session_transcripts( - dir.path(), - &session, - thread, - &legacy_meta("", "", thread), - ) - .unwrap() - .expect("legacy roots should be adopted"); - - assert_eq!( - read_transcript(&adoption.path).unwrap().tools, - Some(serde_json::json!([{"name": "newer"}])) - ); -} - -#[test] -fn adoption_never_touches_the_files_it_reads() { - let dir = tempdir().unwrap(); - let thread = "thread-1"; - write_legacy(dir.path(), "1000_a", "2026-01-01T00:00:00Z", "one", thread); - let legacy_path = resolve_keyed_transcript_path(dir.path(), "1000_a").unwrap(); - let before = std::fs::read(&legacy_path).unwrap(); - - let session = SessionRef::scoped(thread, "orchestrator"); - adopt_legacy_session_transcripts(dir.path(), &session, thread, &legacy_meta("", "", thread)) - .unwrap() - .unwrap(); - - assert_eq!(std::fs::read(&legacy_path).unwrap(), before); -} - -#[test] -fn adoption_is_idempotent_and_never_re_folds() { - let dir = tempdir().unwrap(); - let thread = "thread-1"; - write_legacy(dir.path(), "1000_a", "2026-01-01T00:00:00Z", "one", thread); - let session = SessionRef::scoped(thread, "orchestrator"); - let seed = legacy_meta("", "", thread); - - let first = adopt_legacy_session_transcripts(dir.path(), &session, thread, &seed) - .unwrap() - .unwrap(); - // The adopted transcript is itself a root matching this thread, so a - // second pass must recognise the session as already backed rather than - // folding the conversation into itself. - let second = adopt_legacy_session_transcripts(dir.path(), &session, thread, &seed).unwrap(); - - assert!(second.is_none()); - assert_eq!(read_transcript(&first.path).unwrap().messages.len(), 1); -} - -#[test] -fn a_thread_with_no_legacy_roots_adopts_nothing() { - let dir = tempdir().unwrap(); - let session = SessionRef::scoped("thread-fresh", "orchestrator"); - let result = adopt_legacy_session_transcripts( - dir.path(), - &session, - "thread-fresh", - &legacy_meta("", "", "thread-fresh"), - ) - .unwrap(); - - assert!(result.is_none()); -} - -#[test] -fn an_adopted_session_is_what_the_locator_then_resolves() { - let dir = tempdir().unwrap(); - let thread = "thread-1"; - write_legacy(dir.path(), "1000_a", "2026-01-01T00:00:00Z", "one", thread); - let session = SessionRef::scoped(thread, "orchestrator"); - adopt_legacy_session_transcripts(dir.path(), &session, thread, &legacy_meta("", "", thread)) - .unwrap() - .unwrap(); - - let locator = FileTranscriptLocator::new(dir.path()); - let read = locator - .read_session_transcript(&session) - .expect("the adopted transcript backs the session"); - assert_eq!( - read.read_session().unwrap().unwrap().messages[0].content, - "one" - ); -} - -// ── Legacy OpenHuman layouts ────────────────────────────────────────── -// -// OpenHuman wrote transcripts several ways before session identity existed. -// Adoption has to recover a conversation from each of them, because these are -// exactly the files sitting in users' workspaces today. - -/// A sub-agent transcript shares its parent's `thread_id`. Folding one into the -/// root conversation would splice a delegated worker's private reasoning into -/// the user's chat, so the `parent__child` stems must stay out. -#[test] -fn subagent_siblings_are_never_folded_into_the_root_conversation() { - let dir = tempdir().unwrap(); - let thread = "thread-1"; - write_legacy( - dir.path(), - "1000_orchestrator", - "2026-01-01T00:00:00Z", - "user ask", - thread, - ); - write_legacy( - dir.path(), - "1000_orchestrator__1001_researcher", - "2026-01-01T00:00:01Z", - "worker chatter", - thread, - ); - - let session = SessionRef::scoped(thread, "orchestrator"); - let adoption = adopt_legacy_session_transcripts( - dir.path(), - &session, - thread, - &legacy_meta("", "", thread), - ) - .unwrap() - .unwrap(); - - let contents: Vec = read_transcript(&adoption.path) - .unwrap() - .messages - .into_iter() - .map(|message| message.content) - .collect(); - assert_eq!(contents, ["user ask"]); - assert_eq!(adoption.adopted.len(), 1); -} - -/// OpenHuman's pre-session-key naming was `{agent}_{index}` with no timestamp -/// at all. Those stems carry a `thread_id` in `_meta` just the same, so they -/// must adopt like any other root. -#[test] -fn legacy_indexed_openhuman_stems_are_adopted() { - let dir = tempdir().unwrap(); - let thread = "thread-1"; - write_legacy( - dir.path(), - "orchestrator_1", - "2026-01-01T00:00:00Z", - "first", - thread, - ); - write_legacy( - dir.path(), - "orchestrator_2", - "2026-01-02T00:00:00Z", - "second", - thread, - ); - - let session = SessionRef::scoped(thread, "orchestrator"); - let adoption = adopt_legacy_session_transcripts( - dir.path(), - &session, - thread, - &legacy_meta("", "", thread), - ) - .unwrap() - .unwrap(); - - let contents: Vec = read_transcript(&adoption.path) - .unwrap() - .messages - .into_iter() - .map(|message| message.content) - .collect(); - assert_eq!(contents, ["first", "second"]); -} - -/// The date-grouped `session_raw/DDMMYYYY/` layout is only reachable to the -/// thread scan after the layout migration has flattened it, so the two steps -/// have to compose: migrate, then adopt. -#[test] -fn date_grouped_openhuman_transcripts_adopt_after_the_layout_migration() { - let dir = tempdir().unwrap(); - let thread = "thread-1"; - let legacy_dir = dir.path().join("session_raw").join("01012026"); - std::fs::create_dir_all(&legacy_dir).unwrap(); - write_transcript( - &legacy_dir.join("1000_orchestrator.jsonl"), - &[TranscriptMessage::new("user", "from the dated layout")], - &legacy_meta("2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z", thread), - None, - ) - .unwrap(); - - let session = SessionRef::scoped(thread, "orchestrator"); - let seed = legacy_meta("", "", thread); - // Before flattening, the dated file is invisible to the root scan. - assert!( - adopt_legacy_session_transcripts(dir.path(), &session, thread, &seed) - .unwrap() - .is_none() - ); - - crate::transcript::migrate_layout_if_needed(dir.path()).unwrap(); - let adoption = adopt_legacy_session_transcripts(dir.path(), &session, thread, &seed) - .unwrap() - .expect("the flattened transcript is adoptable"); - - assert_eq!( - read_transcript(&adoption.path).unwrap().messages[0].content, - "from the dated layout" - ); -} - -/// Transcripts written before `_meta.session_id` existed deserialize with it -/// absent. Adoption must not require it — it is the whole population being -/// migrated. -#[test] -fn transcripts_without_session_identity_still_adopt() { - let dir = tempdir().unwrap(); - let thread = "thread-1"; - write_legacy( - dir.path(), - "1000_a", - "2026-01-01T00:00:00Z", - "pre-identity", - thread, - ); - let legacy = - read_transcript(&resolve_keyed_transcript_path(dir.path(), "1000_a").unwrap()).unwrap(); - assert_eq!(legacy.meta.session_id, None); - assert_eq!(legacy.meta.parent_session_id, None); - - let session = SessionRef::scoped(thread, "orchestrator"); - let adoption = adopt_legacy_session_transcripts( - dir.path(), - &session, - thread, - &legacy_meta("", "", thread), - ) - .unwrap() - .unwrap(); - - let adopted = read_transcript(&adoption.path).unwrap(); - assert_eq!(adopted.messages[0].content, "pre-identity"); - assert_eq!(adopted.meta.session_id, Some(session.session_id())); -} - -/// Tool calls, tool results and usage are the reason the transcript is the -/// resume source rather than the prose conversation log. Adoption must carry -/// them across intact. -#[test] -fn adoption_preserves_tool_rounds_and_usage_of_legacy_transcripts() { - let dir = tempdir().unwrap(); - let thread = "thread-1"; - let mut assistant = TranscriptMessage::assistant("calling a tool"); - assistant.id = Some("call-1".into()); - assistant.turn_usage = Some(TurnUsage { - provider: "openrouter".into(), - model: "model".into(), - usage: MessageUsage { - input: 11, - output: 7, - cached_input: 3, - context_window: 1000, - cost_usd: 0.25, - ..Default::default() - }, - ts: "2026-01-01T00:00:00Z".into(), - reasoning_content: Some("thinking".into()), - tool_calls: vec![TranscriptToolCall { - id: "call-1".into(), - name: "web_search".into(), - arguments: "{\"q\":\"kashmir\"".into(), - extra_content: None, - }], - iteration: 2, - }); - let mut tool_result = TranscriptMessage::new("tool", "search results"); - tool_result.id = Some("call-1".into()); - tool_result.cache_breakpoints = vec![1]; - - write_transcript( - &resolve_keyed_transcript_path(dir.path(), "1000_a").unwrap(), - &[ - TranscriptMessage::new("user", "find me something"), - assistant, - tool_result, - ], - &legacy_meta("2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z", thread), - None, - ) - .unwrap(); - - let session = SessionRef::scoped(thread, "orchestrator"); - let adoption = adopt_legacy_session_transcripts( - dir.path(), - &session, - thread, - &legacy_meta("", "", thread), - ) - .unwrap() - .unwrap(); - - let adopted = read_transcript(&adoption.path).unwrap(); - assert_eq!(adopted.messages.len(), 3); - let usage = adopted.messages[1] - .turn_usage - .as_ref() - .expect("assistant usage survives adoption"); - assert_eq!(usage.usage.input, 11); - assert_eq!(usage.reasoning_content.as_deref(), Some("thinking")); - assert_eq!(usage.tool_calls[0].name, "web_search"); - // Raw, unrepaired provider JSON is preserved verbatim. - assert_eq!(usage.tool_calls[0].arguments, "{\"q\":\"kashmir\""); - assert_eq!(adopted.messages[2].role, "tool"); - assert_eq!(adopted.messages[2].cache_breakpoints, vec![1]); -} - -/// A different agent scoped to the same thread id writes its own root -/// transcript. Folding it into this agent's adoption would splice one -/// agent's private history into another's — the same mixing -/// `find_root_transcript_for_thread_scoped` exists to prevent for ordinary -/// resume. -#[test] -fn a_different_agents_root_on_the_same_thread_is_never_folded_in() { - let dir = tempdir().unwrap(); - let thread = "thread-1"; - write_legacy(dir.path(), "1000_a", "2026-01-01T00:00:00Z", "mine", thread); - let mut other_agent = legacy_meta("2026-01-01T00:00:01Z", "2026-01-01T00:00:01Z", thread); - other_agent.agent_id = Some("researcher".into()); - write_transcript( - &resolve_keyed_transcript_path(dir.path(), "1000_researcher").unwrap(), - &[TranscriptMessage::new("user", "not mine")], - &other_agent, - None, - ) - .unwrap(); - - let session = SessionRef::scoped(thread, "orchestrator"); - let adoption = adopt_legacy_session_transcripts( - dir.path(), - &session, - thread, - &legacy_meta("", "", thread), - ) - .unwrap() - .unwrap(); - - let contents: Vec = read_transcript(&adoption.path) - .unwrap() - .messages - .into_iter() - .map(|message| message.content) - .collect(); - assert_eq!(contents, ["mine"]); - assert_eq!(adoption.adopted.len(), 1); -} - -/// A file that already carries session identity (already adopted, or one of -/// this session's own later generations sharing the thread id) is not -/// pre-identity legacy content. Folding it in would duplicate history that -/// adoption already recovered, or content this call has no business reading. -#[test] -fn a_session_identified_root_on_the_same_thread_is_never_folded_in() { - let dir = tempdir().unwrap(); - let thread = "thread-1"; - write_legacy( - dir.path(), - "1000_a", - "2026-01-01T00:00:00Z", - "legacy", - thread, - ); - let mut already_adopted = legacy_meta("2026-01-01T00:00:01Z", "2026-01-01T00:00:01Z", thread); - already_adopted.session_id = Some("thread-1~deadbeef.orchestrator~deadbeef".into()); - write_transcript( - &resolve_keyed_transcript_path(dir.path(), "2000_a").unwrap(), - &[TranscriptMessage::new("user", "already adopted elsewhere")], - &already_adopted, - None, - ) - .unwrap(); - - let session = SessionRef::scoped(thread, "orchestrator"); - let adoption = adopt_legacy_session_transcripts( - dir.path(), - &session, - thread, - &legacy_meta("", "", thread), - ) - .unwrap() - .unwrap(); - - let contents: Vec = read_transcript(&adoption.path) - .unwrap() - .messages - .into_iter() - .map(|message| message.content) - .collect(); - assert_eq!(contents, ["legacy"]); - assert_eq!(adoption.adopted.len(), 1); -} - -/// The thread-lookup scan itself silently drops a candidate it cannot read -/// (logs a warning, excludes it) *before* adoption's own fold loop ever sees -/// it — so a fail-fast check inside that loop alone cannot catch this case. -/// The scan must surface "something in this workspace was unreadable" so -/// adoption can defer instead of finalizing on only the readable roots. -#[test] -fn an_unreadable_root_the_scan_itself_drops_defers_adoption_rather_than_finalizing() { - let dir = tempdir().unwrap(); - let thread = "thread-1"; - write_legacy( - dir.path(), - "1000_a", - "2026-01-01T00:00:00Z", - "readable", - thread, - ); - // A `.jsonl` file the scan cannot parse at all — `find_root_transcripts_for_thread` - // would silently exclude this from its result and only log a warning. - let corrupt_path = resolve_keyed_transcript_path(dir.path(), "2000_a").unwrap(); - std::fs::write(&corrupt_path, b"not a valid meta line at all\n").unwrap(); - - let session = SessionRef::scoped(thread, "orchestrator"); - let result = adopt_legacy_session_transcripts( - dir.path(), - &session, - thread, - &legacy_meta("", "", thread), - ); - - assert!( - result.is_err(), - "a workspace-wide unreadable candidate must defer adoption, not finalize on a partial fold" - ); - let destination = resolve_keyed_transcript_path(dir.path(), &session_stem(&session)).unwrap(); - assert!( - !destination.exists(), - "a deferred adoption must not create the idempotency marker" - ); -} - -/// Two independent adopters (simulating two racing processes) for the same -/// session must not both fold the same legacy roots: the lock in -/// [`adopt_legacy_session_transcripts`] serializes them, so the second call -/// backs off and sees the first call's result rather than re-folding (which -/// would duplicate messages) or overwriting it with a stale view. -#[test] -fn concurrent_adoption_attempts_do_not_duplicate_or_race() { - use std::sync::Arc; - use std::sync::Barrier; - - let dir = tempdir().unwrap(); - let dir_path: Arc = Arc::new(dir.path().to_path_buf()); - let thread = "thread-1"; - write_legacy( - dir_path.as_path(), - "1000_a", - "2026-01-01T00:00:00Z", - "one", - thread, - ); - write_legacy( - dir_path.as_path(), - "2000_a", - "2026-01-02T00:00:00Z", - "two", - thread, - ); - - let barrier = Arc::new(Barrier::new(4)); - let handles: Vec<_> = (0..4) - .map(|_| { - let dir_path = Arc::clone(&dir_path); - let barrier = Arc::clone(&barrier); - std::thread::spawn(move || { - let session = SessionRef::scoped(thread, "orchestrator"); - barrier.wait(); - adopt_legacy_session_transcripts( - dir_path.as_path(), - &session, - thread, - &legacy_meta("", "", thread), - ) - }) - }) - .collect(); - - let results: Vec<_> = handles - .into_iter() - .map(|handle| handle.join().unwrap().unwrap()) - .collect(); - assert_eq!( - results.iter().filter(|result| result.is_some()).count(), - 1, - "exactly one racing adopter should perform the fold; results: {results:?}" - ); - - let session = SessionRef::scoped(thread, "orchestrator"); - let destination = - resolve_keyed_transcript_path(dir_path.as_path(), &session_stem(&session)).unwrap(); - let adopted = read_transcript(&destination).unwrap(); - assert_eq!(adopted.messages.len(), 2, "no duplication across racers"); -} - -/// The lock above only serializes competing *adopters* — it says nothing -/// about a normal session turn independently creating this same session's -/// first transcript while adoption is mid-scan (the CodeRabbit-reported -/// gap). `write_transcript_if_absent`, not the lock, is what has to make -/// that safe: whichever of the two legitimately wins the race to publish -/// first must never be silently destroyed by the other. -#[test] -fn a_concurrent_normal_write_and_an_adoption_never_destroy_each_other() { - use std::sync::{Arc, Barrier}; - - for _ in 0..20 { - let dir = tempdir().unwrap(); - let dir_path: Arc = Arc::new(dir.path().to_path_buf()); - let thread = "thread-1"; - write_legacy( - dir_path.as_path(), - "1000_a", - "2026-01-01T00:00:00Z", - "legacy", - thread, - ); - - let barrier = Arc::new(Barrier::new(2)); - - let adopt_dir = Arc::clone(&dir_path); - let adopt_barrier = Arc::clone(&barrier); - let adopt_thread = std::thread::spawn(move || { - let session = SessionRef::scoped(thread, "orchestrator"); - adopt_barrier.wait(); - adopt_legacy_session_transcripts( - adopt_dir.as_path(), - &session, - thread, - &legacy_meta("", "", thread), - ) - .unwrap() - }); - - let write_dir = Arc::clone(&dir_path); - let write_barrier = Arc::clone(&barrier); - let write_thread = std::thread::spawn(move || { - let session = SessionRef::scoped(thread, "orchestrator"); - let destination = - resolve_keyed_transcript_path(write_dir.as_path(), &session_stem(&session)) - .unwrap(); - let mut turn_meta = legacy_meta("2026-01-01T00:00:05Z", "2026-01-01T00:00:05Z", thread); - turn_meta.session_id = Some(session.session_id()); - write_barrier.wait(); - write_transcript_if_absent( - &destination, - &[TranscriptMessage::new("user", "real first turn")], - &turn_meta, - ) - .unwrap() - }); - - let adopt_result = adopt_thread.join().unwrap(); - let write_result = write_thread.join().unwrap(); - - let session = SessionRef::scoped(thread, "orchestrator"); - let destination = - resolve_keyed_transcript_path(dir_path.as_path(), &session_stem(&session)).unwrap(); - assert!(destination.exists(), "one of the two must have published"); - let contents: Vec = read_transcript(&destination) - .unwrap() - .messages - .into_iter() - .map(|message| message.content) - .collect(); - - // Exactly one side wins, and the file reflects that winner alone — - // never a mix, and the loser's data never touched disk. - if adopt_result.is_some() { - assert_eq!(contents, ["legacy"]); - assert!( - !write_result, - "adoption published first; the concurrent write must have lost" - ); - } else { - assert_eq!(contents, ["real first turn"]); - assert!( - write_result, - "the concurrent write published first; adoption must have lost" - ); - } - } -} - -/// A legacy transcript whose turns were compacted replays as its reduced set. -/// Adoption folds what the model would actually have seen, not the raw lines. -#[test] -fn adoption_folds_the_replayed_context_of_a_compacted_legacy_transcript() { - let dir = tempdir().unwrap(); - let thread = "thread-1"; - let path = resolve_keyed_transcript_path(dir.path(), "1000_a").unwrap(); - let meta = legacy_meta("2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z", thread); - write_transcript( - &path, - &[ - TranscriptMessage::new("user", "one"), - TranscriptMessage::new("user", "two"), - ], - &meta, - None, - ) - .unwrap(); - // A compaction record reduces the logical set, as OpenHuman's trim did. - let retained = [TranscriptMessage::new("user", "two")]; - append_transcript_turn( - &path, - &[ - TranscriptMessage::new("user", "one"), - TranscriptMessage::new("user", "two"), - ], - &retained, - &meta, - None, - None, - ) - .unwrap(); - - let session = SessionRef::scoped(thread, "orchestrator"); - let adoption = adopt_legacy_session_transcripts( - dir.path(), - &session, - thread, - &legacy_meta("", "", thread), - ) - .unwrap() - .unwrap(); - - let contents: Vec = read_transcript(&adoption.path) - .unwrap() - .messages - .into_iter() - .map(|message| message.content) - .collect(); - assert_eq!(contents, ["two"]); -} - -/// A stale-lock reclaim (see `AdoptionLock::acquire`) means two -/// `AdoptionLock` values can briefly both believe they hold the same path. -/// The one that lost that race must never unlink the file out from under -/// the one that actually holds it now — that is exactly what its -/// ownership token (compared in `Drop`) exists to prevent. -#[test] -fn drop_never_removes_a_lock_another_owner_now_holds() { - let dir = tempdir().unwrap(); - let lock_path = dir.path().join("thread-1.jsonl.adopting"); - - let first = AdoptionLock::acquire(&lock_path).unwrap().unwrap(); - // Simulate a concurrent stale-lock reclaim by another process: it - // removes the file and creates its own with a fresh token, exactly - // what `AdoptionLock::acquire`'s reclaim branch does. - std::fs::remove_file(&lock_path).unwrap(); - let second = AdoptionLock::acquire(&lock_path).unwrap().unwrap(); - - drop(first); - assert!( - lock_path.exists(), - "the first (now-stale) lock's Drop must not remove the second owner's live lock" - ); - - drop(second); - assert!( - !lock_path.exists(), - "the still-legitimate second owner's Drop must remove its own lock" - ); -} diff --git a/crates/tinyagents-session/src/transcript/agent_files.rs b/crates/tinyagents-session/src/transcript/agent_files.rs deleted file mode 100644 index 0e04e7f52..000000000 --- a/crates/tinyagents-session/src/transcript/agent_files.rs +++ /dev/null @@ -1,227 +0,0 @@ -//! Transcript files of one embedded agent. -//! -//! An agent derived on a shared workspace keeps its transcripts under -//! `/agents//session_raw/`. Conversations it wrote before that -//! layout, in `/session_raw/`, stay readable: a lookup that misses -//! the agent's own directory falls back to the shared one, and the first time -//! such a session is opened for writing its file is copied into the agent's -//! directory. The shared file is never modified. - -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use super::{ - FileTranscriptLocator, SessionAdoption, SessionRef, ThreadTranscripts, TranscriptHistory, - TranscriptLocator, TranscriptMessage, TranscriptMeta, TranscriptPartial, TranscriptRead, - find_root_transcript_for_thread_scoped, resolve_keyed_transcript_path, session_stem, -}; - -/// The workspace root an agent's transcripts are written under. -#[must_use] -pub fn agent_transcript_root(workspace_dir: &Path, agent_id: &str) -> PathBuf { - workspace_dir.join("agents").join(agent_id) -} - -/// The agent's own transcript files, with a read-only fallback to the shared -/// workspace's. -pub struct AgentTranscriptFiles { - own: FileTranscriptLocator, - own_root: PathBuf, - legacy: FileTranscriptLocator, - legacy_root: PathBuf, -} - -impl AgentTranscriptFiles { - /// `agent_id`'s transcripts on `workspace_dir`. - #[must_use] - pub fn new(workspace_dir: &Path, agent_id: &str) -> Self { - let own_root = agent_transcript_root(workspace_dir, agent_id); - Self { - own: FileTranscriptLocator::new(own_root.clone()), - own_root, - legacy: FileTranscriptLocator::new(workspace_dir.to_path_buf()), - legacy_root: workspace_dir.to_path_buf(), - } - } - - /// Copies `stem` from the shared directory into the agent's when only - /// the shared one has it. - fn adopt_stem(&self, stem: &str) { - let Ok(own) = resolve_keyed_transcript_path(&self.own_root, stem) else { - return; - }; - if own.exists() { - return; - } - let legacy = self - .legacy_root - .join("session_raw") - .join(own.file_name().unwrap_or_default()); - if legacy.is_file() { - self.copy_in(&legacy, &own); - } - } - - /// Copies the shared root transcript of `thread_id` into the agent's - /// directory when the agent has none of its own. - fn adopt_thread_root(&self, thread_id: &str, agent_id: Option<&str>) { - if find_root_transcript_for_thread_scoped(&self.own_root, thread_id, agent_id).is_some() { - return; - } - let Some(legacy) = - find_root_transcript_for_thread_scoped(&self.legacy_root, thread_id, agent_id) - else { - return; - }; - let Some(name) = legacy.file_name() else { - return; - }; - let own_dir = self.own_root.join("session_raw"); - if std::fs::create_dir_all(&own_dir).is_ok() { - self.copy_in(&legacy, &own_dir.join(name)); - } - } - - fn copy_in(&self, legacy: &Path, own: &Path) { - match std::fs::copy(legacy, own) { - Ok(_) => tracing::debug!( - "[session_store] adopted shared transcript into the agent's directory file={}", - own.file_name().unwrap_or_default().to_string_lossy() - ), - Err(error) => tracing::warn!( - "[session_store] could not adopt shared transcript file={} error={error}", - own.file_name().unwrap_or_default().to_string_lossy() - ), - } - } -} - -impl TranscriptLocator for AgentTranscriptFiles { - fn file_backing(&self) -> Option<&FileTranscriptLocator> { - Some(&self.own) - } - - fn thread_transcripts( - &self, - thread_id: &str, - agent_id: Option<&str>, - ) -> Option { - let own = self.own.thread_transcripts(thread_id, agent_id); - let legacy = self.legacy.thread_transcripts(thread_id, agent_id); - match (own, legacy) { - (Some(mut own), Some(legacy)) => { - // The agent's roots supersede the shared ones (an adopted - // root is a copy), but sub-agents spawned before the move - // exist only in the shared directory: list both, the agent's - // copy winning a stem both hold. - let held: std::collections::HashSet = - own.subagents.iter().map(|sub| sub.stem.clone()).collect(); - let shared: Vec<_> = legacy - .subagents - .into_iter() - .filter(|sub| !held.contains(&sub.stem)) - .collect(); - if !shared.is_empty() { - tracing::debug!( - "[session_store] listing {} shared sub-agent transcript(s) beside the agent's own", - shared.len() - ); - own.subagents.extend(shared); - own.subagents.sort_by(|a, b| a.stem.cmp(&b.stem)); - } - Some(own) - } - (own, legacy) => own.or(legacy), - } - } - - fn destination_key(&self) -> Option { - self.own.destination_key() - } - - fn latest_for_agent(&self, agent_name: &str) -> Option> { - self.own - .latest_for_agent(agent_name) - .or_else(|| self.legacy.latest_for_agent(agent_name)) - } - - fn root_for_thread(&self, thread_id: &str) -> Option> { - self.own - .root_for_thread(thread_id) - .or_else(|| self.legacy.root_for_thread(thread_id)) - } - - fn root_for_thread_scoped( - &self, - thread_id: &str, - agent_id: Option<&str>, - ) -> Option> { - self.own - .root_for_thread_scoped(thread_id, agent_id) - .or_else(|| self.legacy.root_for_thread_scoped(thread_id, agent_id)) - } - - fn open_stem( - &self, - stem: &str, - seed: TranscriptMeta, - ) -> anyhow::Result> { - self.adopt_stem(stem); - self.own.open_stem(stem, seed) - } - - fn session_exists(&self, session: &SessionRef) -> bool { - self.adopt_stem(&session_stem(session)); - self.own.session_exists(session) - } - - fn read_session_transcript(&self, session: &SessionRef) -> Option> { - self.adopt_stem(&session_stem(session)); - self.own.read_session_transcript(session) - } - - fn adopt_legacy( - &self, - session: &SessionRef, - thread_id: &str, - seed: &TranscriptMeta, - ) -> anyhow::Result> { - self.own.adopt_legacy(session, thread_id, seed) - } - - fn append_interrupted_partial( - &self, - thread_id: &str, - agent_id: Option<&str>, - partial: &TranscriptPartial, - request_id: Option<&str>, - ) -> anyhow::Result { - self.adopt_thread_root(thread_id, agent_id); - self.own - .append_interrupted_partial(thread_id, agent_id, partial, request_id) - } - - fn begin_generation( - &self, - session: &SessionRef, - seed: TranscriptMeta, - ) -> anyhow::Result<(SessionRef, Arc)> { - self.adopt_stem(&session_stem(session)); - self.own.begin_generation(session, seed) - } - - fn begin_generation_from_baseline( - &self, - session: &SessionRef, - seed: TranscriptMeta, - baseline: &[TranscriptMessage], - ) -> anyhow::Result<(SessionRef, Arc)> { - self.adopt_stem(&session_stem(session)); - self.own - .begin_generation_from_baseline(session, seed, baseline) - } -} - -#[cfg(test)] -#[path = "agent_files_tests.rs"] -mod tests; diff --git a/crates/tinyagents-session/src/transcript/agent_files_tests.rs b/crates/tinyagents-session/src/transcript/agent_files_tests.rs deleted file mode 100644 index 5fa5f07dd..000000000 --- a/crates/tinyagents-session/src/transcript/agent_files_tests.rs +++ /dev/null @@ -1,218 +0,0 @@ -use super::*; -use crate::transcript::TranscriptMessage; - -fn meta() -> TranscriptMeta { - TranscriptMeta { - session_id: None, - parent_session_id: None, - agent_name: "alpha".into(), - agent_id: Some("alpha".into()), - agent_type: None, - dispatcher: "native".into(), - provider: None, - model: None, - created: "2026-10-01T00:00:00Z".into(), - updated: "2026-10-01T00:00:00Z".into(), - turn_count: 1, - prefix_message_count: None, - input_tokens: 0, - output_tokens: 0, - cached_input_tokens: 0, - charged_amount_usd: 0.0, - thread_id: Some("thread-1".into()), - task_id: None, - } -} - -fn messages(session: &dyn TranscriptRead) -> Vec { - session - .read_session() - .expect("readable") - .expect("written") - .messages - .iter() - .map(|message| message.content.clone()) - .collect() -} - -#[test] -fn new_sessions_are_written_under_the_agents_directory() { - let workspace = tempfile::tempdir().expect("tempdir"); - let files = AgentTranscriptFiles::new(workspace.path(), "alpha"); - let session = SessionRef::scoped("thread-1", "alpha"); - - files - .open_session(&session, meta()) - .expect("bind") - .replace(&[TranscriptMessage::user("hello")]) - .expect("write"); - - let own = workspace.path().join("agents/alpha/session_raw"); - assert!(std::fs::read_dir(&own).expect("own dir").next().is_some()); - assert!( - !workspace - .path() - .join("session_raw") - .join(format!("{}.jsonl", session_stem(&session))) - .exists() - ); -} - -#[test] -fn a_shared_session_is_read_then_continued_without_touching_the_shared_file() { - let workspace = tempfile::tempdir().expect("tempdir"); - let session = SessionRef::scoped("thread-1", "alpha"); - FileTranscriptLocator::new(workspace.path()) - .open_session(&session, meta()) - .expect("bind shared") - .replace(&[TranscriptMessage::user("before")]) - .expect("write shared"); - let shared_path = workspace - .path() - .join("session_raw") - .join(format!("{}.jsonl", session_stem(&session))); - let shared_bytes = std::fs::read(&shared_path).expect("shared file"); - - let files = AgentTranscriptFiles::new(workspace.path(), "alpha"); - let found = files - .read_session_transcript(&session) - .expect("the shared session is found"); - assert_eq!(messages(found.as_ref()), ["before"]); - - files - .open_session(&session, meta()) - .expect("bind own") - .replace(&[ - TranscriptMessage::user("before"), - TranscriptMessage::user("after"), - ]) - .expect("continue"); - - assert_eq!( - std::fs::read(&shared_path).expect("shared file"), - shared_bytes - ); - let continued = files - .read_session_transcript(&session) - .expect("own session"); - assert_eq!(messages(continued.as_ref()), ["before", "after"]); -} - -#[test] -fn another_agent_does_not_adopt_a_session_scoped_to_someone_else() { - let workspace = tempfile::tempdir().expect("tempdir"); - let alpha_session = SessionRef::scoped("thread-1", "alpha"); - AgentTranscriptFiles::new(workspace.path(), "alpha") - .open_session(&alpha_session, meta()) - .expect("bind") - .replace(&[TranscriptMessage::user("alpha only")]) - .expect("write"); - - let beta = AgentTranscriptFiles::new(workspace.path(), "beta"); - assert!( - beta.read_session_transcript(&SessionRef::scoped("thread-1", "beta")) - .is_none() - ); - assert!(beta.read_session_transcript(&alpha_session).is_none()); -} - -#[test] -fn fixture_workspace_resumes_by_stem_without_readoption_or_renaming() { - // A workspace laid out before per-agent directories: the transcript sits in - // the shared `session_raw/` under its deterministic, timestamp-free stem. - let workspace = tempfile::tempdir().expect("tempdir"); - let session = SessionRef::scoped("thread-9", "alpha"); - let stem = session_stem(&session); - FileTranscriptLocator::new(workspace.path()) - .open_session(&session, meta()) - .expect("bind shared") - .replace(&[TranscriptMessage::user("old turn")]) - .expect("write shared"); - let shared = workspace - .path() - .join("session_raw") - .join(format!("{stem}.jsonl")); - assert!(shared.is_file(), "stem names the shared file unchanged"); - - let files = AgentTranscriptFiles::new(workspace.path(), "alpha"); - assert!(files.session_exists(&session)); - let resumed = files.read_session_transcript(&session).expect("resumes"); - assert_eq!(messages(resumed.as_ref()), ["old turn"]); - - // The copy keeps the very same file name, so a second process resolves it - // by the same stem. - let own = workspace - .path() - .join("agents/alpha/session_raw") - .join(format!("{stem}.jsonl")); - assert!(own.is_file()); - let again = AgentTranscriptFiles::new(workspace.path(), "alpha"); - assert!(again.session_exists(&session)); -} - -#[test] -fn shared_sub_agents_stay_listed_after_the_root_is_adopted() { - let workspace = tempfile::tempdir().expect("tempdir"); - let session = SessionRef::scoped("thread-1", "alpha"); - let shared = FileTranscriptLocator::new(workspace.path()); - shared - .open_session(&session, meta()) - .expect("bind shared") - .replace(&[TranscriptMessage::user("before")]) - .expect("write shared"); - // Sub-agents spawned before the agent kept its own directory, one of - // which the agent's directory also holds by the same stem. - for stem in ["1700000000_alpha__1_old", "1700000000_alpha__2_both"] { - shared - .open_stem(stem, meta()) - .expect("bind shared sub-agent") - .replace(&[TranscriptMessage::user(stem)]) - .expect("write shared sub-agent"); - } - - let files = AgentTranscriptFiles::new(workspace.path(), "alpha"); - files - .open_session(&session, meta()) - .expect("bind own") - .replace(&[ - TranscriptMessage::user("before"), - TranscriptMessage::user("after"), - ]) - .expect("continue"); - for stem in ["1700000000_alpha__2_both", "1700000000_alpha__3_new"] { - files - .open_stem(stem, meta()) - .expect("bind own sub-agent") - .replace(&[TranscriptMessage::user(stem)]) - .expect("write own sub-agent"); - } - - let listed = files - .thread_transcripts("thread-1", None) - .expect("the thread is listed"); - let roots: Vec<_> = listed.roots.iter().map(|root| root.stem.clone()).collect(); - assert_eq!(roots, [session_stem(&session)]); - assert_eq!( - messages(listed.roots[0].transcript.as_ref()), - ["before", "after"], - "the agent's own root is listed" - ); - let subs: Vec<_> = listed - .subagents - .iter() - .map(|sub| sub.stem.clone()) - .collect(); - assert_eq!( - subs, - [ - "1700000000_alpha__1_old", - "1700000000_alpha__2_both", - "1700000000_alpha__3_new" - ] - ); - let own_root = agent_transcript_root(workspace.path(), "alpha"); - assert!( - listed.subagents[1].transcript.path().starts_with(&own_root), - "the agent's copy wins a stem both directories hold" - ); -} diff --git a/crates/tinyagents-session/src/transcript/background.rs b/crates/tinyagents-session/src/transcript/background.rs index 30d3f994a..b9aacb3f8 100644 --- a/crates/tinyagents-session/src/transcript/background.rs +++ b/crates/tinyagents-session/src/transcript/background.rs @@ -3,14 +3,14 @@ //! A conversation's transcript is normally written only by its own live turns. //! Some work finishes *outside* any turn but belongs in the conversation — a //! scheduled job created from a chat reports back into that chat, so the next -//! turn can see it ("did you remind me?"). [`append_background_message`] is +//! turn can see it ("did you remind me?"). [`append_background_message_in`] is //! that writer: one plain assistant message, appended to the session's head //! generation, idempotent per caller key, and marked with where it came from. //! //! # Concurrency contract //! //! The live turn path appends a *delta* against the baseline it read when the -//! turn started, and validates that baseline under the file lock. A foreign +//! turn started, and validates that baseline when it writes. A foreign //! line written between that read and the persist makes the baseline stale and //! the user's turn fails to persist. So: //! @@ -30,24 +30,17 @@ //! baseline and its next persist fails rather than hiding the line. //! - The turn lock is **in-process**. Another process appending mid-turn is //! still caught by the baseline check (that turn fails to persist; nothing is -//! corrupted), and the append itself takes the same cross-process file locks -//! as every other writer, so bytes never interleave. +//! corrupted), and the append itself goes through the store's own write +//! path like every other writer. use super::display::with_background; +use super::history::TranscriptLocator; use super::history::seed_meta_for_discovered; -use super::history::{FileTranscriptHistory, FileTranscriptLocator, TranscriptLocator}; -use super::jsonl::{LineKind, build_message_line, classify_line}; -use super::paths::resolve_keyed_transcript_path; -use super::reader::read_jsonl_lines; use super::session::{SessionRef, session_stem}; -use super::turn_lock::{lock_session_turn_or_identity, lock_session_turn_with_notification}; +use super::turn_lock::lock_session_turn_or_identity; use super::types::{ BackgroundAppend, BackgroundAppendOutcome, BackgroundOrigin, DisplayRecord, TranscriptMessage, }; -use super::writer::append_bytes; -use anyhow::Context; -use std::path::Path; -use tokio::sync::Notify; /// How many times the head may move under an unpinned append before it gives /// up. Each move is a compaction or a fork landing between head resolution @@ -61,143 +54,50 @@ const MAX_HEAD_MOVES: usize = 3; /// whatever `session.generation` says. With /// [`BackgroundAppend::expected_generation`] set, a head that has moved on /// returns [`BackgroundAppendOutcome::StaleGeneration`] and writes nothing. -/// - **Idempotent.** A head generation that already holds a line with the same -/// [`BackgroundAppend::idempotency_key`] returns -/// [`BackgroundAppendOutcome::Duplicate`] and writes nothing. The key is -/// checked within the head generation only. A compaction that carries the -/// message into a successor carries its provenance too (the model-context -/// row keeps it in `extra_metadata`, and the writer puts it back on the -/// line), so a retry against that successor is still a duplicate; one the -/// compaction dropped is appended again. +/// - **Idempotent.** A head generation whose *live* context already holds a +/// message delivered with the same [`BackgroundAppend::idempotency_key`] +/// returns [`BackgroundAppendOutcome::Duplicate`] and writes nothing. The +/// live context is what a compaction left: rows written before a +/// compaction of the same generation count only if that compaction's +/// replacement kept them (a kept row keeps its provenance in +/// `extra_metadata`). A key a compaction dropped is therefore appended +/// again, in the same generation or in a successor. /// - **Never creates a session.** No transcript yet is /// [`BackgroundAppendOutcome::NoSession`]. /// -/// The line is an ordinary message line, so the model-context reader -/// ([`read_transcript`](super::read_transcript)) replays it as an assistant row -/// and the display reader shows it. It additionally carries a top-level -/// `background` field (a [`BackgroundOrigin`]: the key and the caller's -/// `provenance`), surfaced as [`DisplayMessage::background`](super::DisplayMessage::background); -/// readers that predate the field ignore it. No `_meta` line is appended, so -/// turn counts and usage totals are untouched, and the `.md` companion is next -/// re-rendered by the following turn. +/// The message is appended through the head's +/// [`TranscriptHistory`](super::TranscriptHistory) handle, with the provenance +/// recorded in the row's `extra_metadata` under +/// [`BACKGROUND_METADATA_KEY`](super::BACKGROUND_METADATA_KEY), which the +/// display read lifts back into [`DisplayMessage::background`](super::DisplayMessage::background). /// /// Awaits the session's turn lock first; see the module documentation for the -/// contract this places on live turns. The file I/O itself is synchronous and -/// small (one scan of the head file, one append). +/// contract this places on live turns. The turn lock is the only +/// serialisation: the duplicate check and the append are two calls, so two +/// processes appending the same key to one store at once could both write it. +/// A locator with no [`destination_key`](TranscriptLocator::destination_key) +/// is locked by its own identity instead, so appends through that one locator +/// still serialise. /// /// # Errors /// /// When `message` is not a plain assistant message (another role, tool calls, -/// or an interrupted partial), when the key is empty, on I/O failure, or when -/// another process holds the head's successor reserved mid-compaction. -pub async fn append_background_message( - locator: &FileTranscriptLocator, - session: &SessionRef, - message: TranscriptMessage, - options: BackgroundAppend, -) -> anyhow::Result { - append_background_message_impl(locator, locator, session, message, options, None).await -} - -/// Test-oriented variant that signals immediately before waiting for the -/// shared turn lock. The append and production path are otherwise identical. -#[doc(hidden)] -pub async fn append_background_message_with_lock_notification( - locator: &FileTranscriptLocator, +/// or an interrupted partial), when the key is empty, or on any store error. +pub async fn append_background_message_in( + locator: &dyn TranscriptLocator, session: &SessionRef, message: TranscriptMessage, options: BackgroundAppend, - lock_attempted: &Notify, ) -> anyhow::Result { - append_background_message_impl( - locator, - locator, - session, - message, - options, - Some(lock_attempted), - ) - .await + append_background_message_impl(locator, session, message, options).await } -/// `files` writes the JSONL line; `resolver` resolves the head and decides -/// whether the session exists. They differ for a wrapper over a file locator -/// (such as [`AgentTranscriptFiles`](super::AgentTranscriptFiles)) whose -/// existence checks adopt a transcript into `files` before it is written. async fn append_background_message_impl( - files: &FileTranscriptLocator, - resolver: &dyn TranscriptLocator, - session: &SessionRef, - message: TranscriptMessage, - options: BackgroundAppend, - lock_attempted: Option<&Notify>, -) -> anyhow::Result { - let message = validate(message, &options)?; - let root = session.first_generation(); - let stem = session_stem(&root); - let key = options.idempotency_key.as_str(); - tracing::debug!(session = %stem, idempotency_key = %key, expected_generation = ?options.expected_generation, "[transcript-background] append requested"); - - let _turn = lock_session_turn_with_notification(resolver, &root, lock_attempted).await; - for _ in 0..MAX_HEAD_MOVES { - let head = resolver.head_generation(&root); - if !resolver.session_exists(&head) { - tracing::debug!(session = %stem, idempotency_key = %key, "[transcript-background] no transcript; nothing written"); - return Ok(BackgroundAppendOutcome::NoSession); - } - if let Some(stale) = stale(&options, head.generation) { - tracing::debug!(session = %stem, idempotency_key = %key, ?stale, "[transcript-background] head moved on; nothing written"); - return Ok(stale); - } - match append_to_head(files, &head, &message, &options)? { - Attempt::Done(outcome) => { - tracing::debug!(session = %stem, idempotency_key = %key, ?outcome, "[transcript-background] append finished"); - return Ok(outcome); - } - Attempt::HeadMoved => { - tracing::debug!(session = %stem, idempotency_key = %key, generation = head.generation, "[transcript-background] head sealed during append; re-resolving"); - } - } - } - anyhow::bail!( - "session {stem} head kept moving during a background append ({MAX_HEAD_MOVES} attempts)" - ) -} - -/// [`append_background_message`] through any [`TranscriptLocator`]. -/// -/// A file-backed locator ([`TranscriptLocator::file_backing`]) goes through -/// the file writer, so the JSONL line is exactly what -/// [`append_background_message`] writes. Any other store gets the same -/// contract — the turn lock, head-only, idempotent within the head -/// generation, never creating a session — with the message appended through -/// its [`TranscriptHistory`](super::TranscriptHistory) handle and the -/// provenance recorded in the row's `extra_metadata` under -/// [`BACKGROUND_METADATA_KEY`](super::BACKGROUND_METADATA_KEY), which the -/// display read lifts back into [`DisplayMessage::background`](super::DisplayMessage::background). -/// -/// For such a store the turn lock is the only serialisation: the duplicate -/// check and the append are two calls, so two processes appending the same key -/// to one store at once could both write it. A locator with no -/// [`destination_key`](TranscriptLocator::destination_key) is locked by its -/// own identity instead, so appends through that one locator still serialise. -/// -/// # Errors -/// -/// As [`append_background_message`], plus any store error. -pub async fn append_background_message_in( locator: &dyn TranscriptLocator, session: &SessionRef, message: TranscriptMessage, options: BackgroundAppend, ) -> anyhow::Result { - if let Some(files) = locator.file_backing() { - // Existence and head resolution go through `locator` itself, so a - // wrapper that adopts transcripts on lookup (an agent's own files over - // a shared workspace) still finds a session only its fallback holds. - return append_background_message_impl(files, locator, session, message, options, None) - .await; - } let message = validate(message, &options)?; let root = session.first_generation(); let stem = session_stem(&root); @@ -245,15 +145,9 @@ fn append_to_stored_head( let generation = head.generation; let handle = locator.open_session(head, seed_meta_for_discovered(""))?; let key = options.idempotency_key.as_str(); - let held = handle.read_display()?.is_some_and(|display| { - display.records.iter().any(|record| match record { - DisplayRecord::Message(message) => carries_key(message.background.as_ref(), key), - DisplayRecord::Compaction(compaction) => compaction - .replacement - .iter() - .any(|message| carries_key(message.background.as_ref(), key)), - }) - }); + let held = handle + .read_display()? + .is_some_and(|display| live_context_holds_key(&display.records, key)); if held { return Ok(Attempt::Done(BackgroundAppendOutcome::Duplicate { generation, @@ -275,6 +169,28 @@ fn append_to_stored_head( })) } +/// Whether the *live* context of `records` holds a message delivered with +/// `key`. A compaction replaces the context accumulated so far, so only rows +/// after the last compaction and that compaction's own replacement count; a +/// key the compaction dropped is not held. +fn live_context_holds_key(records: &[DisplayRecord], key: &str) -> bool { + let mut held = false; + for record in records { + match record { + DisplayRecord::Message(message) => { + held |= carries_key(message.background.as_ref(), key); + } + DisplayRecord::Compaction(compaction) => { + held = compaction + .replacement + .iter() + .any(|message| carries_key(message.background.as_ref(), key)); + } + } + } + held +} + fn carries_key(origin: Option<&BackgroundOrigin>, key: &str) -> bool { origin.is_some_and(|origin| origin.idempotency_key == key) } @@ -286,43 +202,6 @@ enum Attempt { HeadMoved, } -/// Scans and appends to `head` under its file write locks. -fn append_to_head( - locator: &FileTranscriptLocator, - head: &SessionRef, - message: &TranscriptMessage, - options: &BackgroundAppend, -) -> anyhow::Result { - let workspace = locator.workspace_dir(); - let path = resolve_keyed_transcript_path(workspace, &session_stem(head))?; - let successor = - resolve_keyed_transcript_path(workspace, &session_stem(&head.next_generation()))?; - let history = FileTranscriptHistory::opened_at(path.clone(), seed_meta_for_discovered("")); - history.with_write_locks(|| { - if successor.is_file() { - return Ok(Attempt::HeadMoved); - } - let generation = head.generation; - if holds_idempotency_key(&path, &options.idempotency_key)? { - return Ok(Attempt::Done(BackgroundAppendOutcome::Duplicate { - generation, - })); - } - let mut line = build_message_line(message, message.turn_usage.as_ref(), None, false); - line.ts = line.ts.or_else(|| Some(chrono::Utc::now().to_rfc3339())); - line.background = Some(BackgroundOrigin { - idempotency_key: options.idempotency_key.clone(), - provenance: options.provenance.clone(), - }); - let mut buf = serde_json::to_string(&line).context("serialise background message line")?; - buf.push('\n'); - append_bytes(&path, buf.as_bytes())?; - Ok(Attempt::Done(BackgroundAppendOutcome::Appended { - generation, - })) - }) -} - /// `StaleGeneration` when the caller pinned a generation other than `head`. fn stale(options: &BackgroundAppend, head: u32) -> Option { options @@ -363,34 +242,6 @@ fn validate( Ok(normalized) } -/// Whether any message line of the transcript at `path` was delivered with -/// `key`. Lines that do not parse are skipped, exactly as the readers skip them. -fn holds_idempotency_key(path: &Path, key: &str) -> anyhow::Result { - let matches = |origin: &Option| { - origin - .as_ref() - .is_some_and(|origin| origin.idempotency_key == key) - }; - for line in read_jsonl_lines(path)?.into_iter().flatten() { - let line = line.trim(); - if line.is_empty() { - continue; - } - let found = match classify_line(line) { - Ok(LineKind::Message(message)) => matches(&message.background), - Ok(LineKind::Compaction(compaction)) => compaction - .replacement - .iter() - .any(|message| matches(&message.background)), - _ => false, - }; - if found { - return Ok(true); - } - } - Ok(false) -} - #[cfg(test)] #[path = "background_tests.rs"] mod tests; diff --git a/crates/tinyagents-session/src/transcript/background_tests.rs b/crates/tinyagents-session/src/transcript/background_tests.rs index 1a02737da..45c4528cb 100644 --- a/crates/tinyagents-session/src/transcript/background_tests.rs +++ b/crates/tinyagents-session/src/transcript/background_tests.rs @@ -1,11 +1,10 @@ use super::*; +use crate::transcript::test_locator::driver_locator; use crate::transcript::{ - DisplayRecord, ToolFailure, TranscriptLocator, TranscriptMeta, TranscriptPart, TranscriptTurn, - lock_session_turn, read_transcript, read_transcript_display, resolve_keyed_transcript_path, - session_stem, + DisplayRecord, ToolFailure, TranscriptHistory, TranscriptLocator, TranscriptMeta, + TranscriptPart, TranscriptRead, TranscriptTurn, lock_session_turn, }; -use std::path::{Path, PathBuf}; -use tempfile::tempdir; +use std::sync::Arc; fn meta() -> TranscriptMeta { TranscriptMeta { @@ -41,10 +40,14 @@ fn options(key: &str) -> BackgroundAppend { ) } +fn locator() -> Arc { + driver_locator() +} + /// Writes one turn the way the live turn path does: through the session's own /// history handle, diffing `next` against `prev`. fn live_turn( - locator: &FileTranscriptLocator, + locator: &dyn TranscriptLocator, session: &SessionRef, prev: &[TranscriptMessage], next: &[TranscriptMessage], @@ -69,69 +72,72 @@ fn first_turn() -> Vec { ] } -fn head_path(dir: &Path, session: &SessionRef) -> PathBuf { - let locator = FileTranscriptLocator::new(dir); +/// The head generation's transcript of `session`. +fn head(locator: &dyn TranscriptLocator, session: &SessionRef) -> Arc { let head = locator.head_generation(session); - resolve_keyed_transcript_path(dir, &session_stem(&head)).unwrap() + locator + .read_session_transcript(&head) + .expect("the head generation holds a transcript") } -#[tokio::test] -async fn an_appended_message_is_resumed_as_an_assistant_turn() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - live_turn(&locator, &session(), &[], &first_turn()); +/// The head generation's model-context messages. +fn head_messages(locator: &dyn TranscriptLocator, session: &SessionRef) -> Vec { + head(locator, session) + .read_session() + .unwrap() + .expect("transcript present") + .messages +} - let outcome = append_background_message( - &locator, +async fn deliver( + locator: &dyn TranscriptLocator, + key: &str, +) -> anyhow::Result { + append_background_message_in( + locator, &session(), TranscriptMessage::assistant("Time to stretch!"), - options("run-1"), + options(key), ) .await - .unwrap(); +} + +#[tokio::test] +async fn an_appended_message_is_resumed_as_an_assistant_turn() { + let locator = locator(); + live_turn(locator.as_ref(), &session(), &[], &first_turn()); + + let outcome = deliver(locator.as_ref(), "run-1").await.unwrap(); assert_eq!(outcome, BackgroundAppendOutcome::Appended { generation: 0 }); - let resumed = read_transcript(&head_path(dir.path(), &session())).unwrap(); - let last = resumed.messages.last().unwrap(); - assert_eq!(resumed.messages.len(), 3); + let resumed = head_messages(locator.as_ref(), &session()); + let last = resumed.last().unwrap(); + assert_eq!(resumed.len(), 3); assert_eq!(last.role, "assistant"); assert_eq!(last.content, "Time to stretch!"); } #[tokio::test] async fn a_session_without_a_transcript_is_reported_and_not_created() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); + let locator = locator(); - let outcome = append_background_message( - &locator, - &session(), - TranscriptMessage::assistant("Time to stretch!"), - options("run-1"), - ) - .await - .unwrap(); + let outcome = deliver(locator.as_ref(), "run-1").await.unwrap(); assert_eq!(outcome, BackgroundAppendOutcome::NoSession); - assert!(!head_path(dir.path(), &session()).exists()); + assert!(!locator.session_exists(&locator.head_generation(&session()))); } #[tokio::test] async fn the_display_reader_shows_the_message_with_its_provenance() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - live_turn(&locator, &session(), &[], &first_turn()); + let locator = locator(); + live_turn(locator.as_ref(), &session(), &[], &first_turn()); - append_background_message( - &locator, - &session(), - TranscriptMessage::assistant("Time to stretch!"), - options("run-1"), - ) - .await - .unwrap(); + deliver(locator.as_ref(), "run-1").await.unwrap(); - let display = read_transcript_display(&head_path(dir.path(), &session())).unwrap(); + let display = head(locator.as_ref(), &session()) + .read_display() + .unwrap() + .unwrap(); let Some(DisplayRecord::Message(last)) = display.records.last() else { panic!("expected a message record last"); }; @@ -154,35 +160,24 @@ async fn the_display_reader_shows_the_message_with_its_provenance() { #[tokio::test] async fn a_repeated_idempotency_key_is_a_duplicate_and_writes_nothing() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - live_turn(&locator, &session(), &[], &first_turn()); - let message = TranscriptMessage::assistant("Time to stretch!"); - let first = append_background_message(&locator, &session(), message.clone(), options("run-1")) - .await - .unwrap(); - let before = std::fs::read(head_path(dir.path(), &session())).unwrap(); + let locator = locator(); + live_turn(locator.as_ref(), &session(), &[], &first_turn()); + let first = deliver(locator.as_ref(), "run-1").await.unwrap(); + let before = head_messages(locator.as_ref(), &session()); - let again = append_background_message(&locator, &session(), message.clone(), options("run-1")) - .await - .unwrap(); + let again = deliver(locator.as_ref(), "run-1").await.unwrap(); assert_eq!(first, BackgroundAppendOutcome::Appended { generation: 0 }); assert_eq!(again, BackgroundAppendOutcome::Duplicate { generation: 0 }); - assert_eq!( - std::fs::read(head_path(dir.path(), &session())).unwrap(), - before - ); + assert_eq!(head_messages(locator.as_ref(), &session()), before); // A different key is a different delivery. - let other = append_background_message(&locator, &session(), message, options("run-2")) - .await - .unwrap(); + let other = deliver(locator.as_ref(), "run-2").await.unwrap(); assert_eq!(other, BackgroundAppendOutcome::Appended { generation: 0 }); } /// Seals generation 0 and opens generation 1 with `retained`, the way a /// compaction in the live turn path does. -fn compact(locator: &FileTranscriptLocator, retained: &[TranscriptMessage]) { +fn compact(locator: &dyn TranscriptLocator, retained: &[TranscriptMessage]) { let (_, handle) = locator.begin_generation(&session(), meta()).unwrap(); handle .append_turn(TranscriptTurn { @@ -198,19 +193,20 @@ fn compact(locator: &FileTranscriptLocator, retained: &[TranscriptMessage]) { #[tokio::test] async fn a_stale_expected_generation_writes_nothing() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - live_turn(&locator, &session(), &[], &first_turn()); - compact(&locator, &[TranscriptMessage::system("summary")]); - let sealed = resolve_keyed_transcript_path(dir.path(), &session_stem(&session())).unwrap(); - let head = head_path(dir.path(), &session()); - let (sealed_before, head_before) = ( - std::fs::read(&sealed).unwrap(), - std::fs::read(&head).unwrap(), - ); + let locator = locator(); + live_turn(locator.as_ref(), &session(), &[], &first_turn()); + compact(locator.as_ref(), &[TranscriptMessage::system("summary")]); + let sealed_before = locator + .read_session_transcript(&session()) + .unwrap() + .read_session() + .unwrap() + .unwrap() + .messages; + let head_before = head_messages(locator.as_ref(), &session()); - let outcome = append_background_message( - &locator, + let outcome = append_background_message_in( + locator.as_ref(), &session(), TranscriptMessage::assistant("Time to stretch!"), options("run-1").expecting_generation(0), @@ -225,19 +221,25 @@ async fn a_stale_expected_generation_writes_nothing() { head: 1 } ); - assert_eq!(std::fs::read(&sealed).unwrap(), sealed_before); - assert_eq!(std::fs::read(&head).unwrap(), head_before); + let sealed_after = locator + .read_session_transcript(&session()) + .unwrap() + .read_session() + .unwrap() + .unwrap() + .messages; + assert_eq!(sealed_after, sealed_before); + assert_eq!(head_messages(locator.as_ref(), &session()), head_before); } #[tokio::test] async fn the_append_lands_in_the_head_generation() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - live_turn(&locator, &session(), &[], &first_turn()); - compact(&locator, &[TranscriptMessage::system("summary")]); + let locator = locator(); + live_turn(locator.as_ref(), &session(), &[], &first_turn()); + compact(locator.as_ref(), &[TranscriptMessage::system("summary")]); - let outcome = append_background_message( - &locator, + let outcome = append_background_message_in( + locator.as_ref(), &session(), TranscriptMessage::assistant("Time to stretch!"), options("run-1").expecting_generation(1), @@ -246,18 +248,19 @@ async fn the_append_lands_in_the_head_generation() { .unwrap(); assert_eq!(outcome, BackgroundAppendOutcome::Appended { generation: 1 }); - let head = read_transcript(&head_path(dir.path(), &session())).unwrap(); assert_eq!( - head.messages + head_messages(locator.as_ref(), &session()) .iter() .map(|message| message.content.as_str()) .collect::>(), ["summary", "Time to stretch!"] ); - let sealed = read_transcript( - &resolve_keyed_transcript_path(dir.path(), &session_stem(&session())).unwrap(), - ) - .unwrap(); + let sealed = locator + .read_session_transcript(&session()) + .unwrap() + .read_session() + .unwrap() + .unwrap(); assert_eq!(sealed.messages.len(), 2); } @@ -269,25 +272,19 @@ async fn the_append_lands_in_the_head_generation() { /// yields, so "it is still waiting" is decided by the lock, not by timing. #[tokio::test] async fn a_background_append_waits_for_the_live_turn_and_keeps_both_in_order() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let path = head_path(dir.path(), &session()); - live_turn(&locator, &session(), &[], &first_turn()); + let locator = locator(); + live_turn(locator.as_ref(), &session(), &[], &first_turn()); // Turn two starts: lock, then read its baseline. - let turn = lock_session_turn(&locator, &session()).await.unwrap(); - let baseline = read_transcript(&path).unwrap().messages; - - let workspace = dir.path().to_path_buf(); - let delivery = tokio::spawn(async move { - append_background_message( - &FileTranscriptLocator::new(workspace), - &session(), - TranscriptMessage::assistant("Time to stretch!"), - options("run-1"), - ) + let turn = lock_session_turn(locator.as_ref(), &session()) .await - }); + .unwrap(); + let baseline = head_messages(locator.as_ref(), &session()); + + let delivery = { + let locator = Arc::clone(&locator); + tokio::spawn(async move { deliver(locator.as_ref(), "run-1").await }) + }; for _ in 0..50 { tokio::task::yield_now().await; } @@ -295,13 +292,13 @@ async fn a_background_append_waits_for_the_live_turn_and_keeps_both_in_order() { !delivery.is_finished(), "delivery must wait for the live turn" ); - assert_eq!(read_transcript(&path).unwrap().messages, baseline); + assert_eq!(head_messages(locator.as_ref(), &session()), baseline); // Turn two persists against the baseline it read, then releases the lock. let mut second = baseline.clone(); second.push(TranscriptMessage::user("what's the weather?")); second.push(TranscriptMessage::assistant("Sunny.")); - live_turn(&locator, &session(), &baseline, &second); + live_turn(locator.as_ref(), &session(), &baseline, &second); drop(turn); assert_eq!( @@ -310,16 +307,16 @@ async fn a_background_append_waits_for_the_live_turn_and_keeps_both_in_order() { ); // Turn three resumes over everything and appends on top of it. - let _turn = lock_session_turn(&locator, &session()).await.unwrap(); - let resumed = read_transcript(&path).unwrap().messages; + let _turn = lock_session_turn(locator.as_ref(), &session()) + .await + .unwrap(); + let resumed = head_messages(locator.as_ref(), &session()); let mut third = resumed.clone(); third.push(TranscriptMessage::user("did you remind me?")); third.push(TranscriptMessage::assistant("Yes, at 5.")); - live_turn(&locator, &session(), &resumed, &third); + live_turn(locator.as_ref(), &session(), &resumed, &third); - let contents: Vec<(String, String)> = read_transcript(&path) - .unwrap() - .messages + let contents: Vec<(String, String)> = head_messages(locator.as_ref(), &session()) .into_iter() .map(|message| (message.role, message.content)) .collect(); @@ -340,10 +337,9 @@ async fn a_background_append_waits_for_the_live_turn_and_keeps_both_in_order() { #[tokio::test] async fn only_plain_assistant_messages_with_a_key_are_accepted() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - live_turn(&locator, &session(), &[], &first_turn()); - let before = std::fs::read(head_path(dir.path(), &session())).unwrap(); + let locator = locator(); + live_turn(locator.as_ref(), &session(), &[], &first_turn()); + let before = head_messages(locator.as_ref(), &session()); let mut interrupted = TranscriptMessage::assistant("half an ans"); interrupted.interrupted = true; let calls = TranscriptMessage::assistant_with_calls( @@ -377,49 +373,23 @@ async fn only_plain_assistant_messages_with_a_key_are_accepted() { (interrupted, "run-1"), (TranscriptMessage::assistant("fine"), " "), ] { - let result = append_background_message(&locator, &session(), message, options(key)).await; + let result = + append_background_message_in(locator.as_ref(), &session(), message, options(key)).await; assert!(result.is_err(), "{result:?}"); } - assert_eq!( - std::fs::read(head_path(dir.path(), &session())).unwrap(), - before - ); + assert_eq!(head_messages(locator.as_ref(), &session()), before); } -/// A line whose `background` value is not a [`BackgroundOrigin`] (a newer or -/// foreign writer) still resumes as a message; only the origin is dropped. #[tokio::test] -async fn an_unreadable_origin_does_not_cost_the_message() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - live_turn(&locator, &session(), &[], &first_turn()); - let path = head_path(dir.path(), &session()); - append_bytes( - &path, - b"{\"role\":\"assistant\",\"content\":\"from elsewhere\",\"background\":42}\n", - ) - .unwrap(); - - let resumed = read_transcript(&path).unwrap(); - assert_eq!(resumed.messages.last().unwrap().content, "from elsewhere"); - let display = read_transcript_display(&path).unwrap(); - let Some(DisplayRecord::Message(last)) = display.records.last() else { - panic!("expected a message record last"); - }; - assert_eq!(last.background, None); -} - -#[test] -fn an_attempt_against_a_sealed_head_writes_nothing_and_asks_to_re_resolve() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - live_turn(&locator, &session(), &[], &first_turn()); - compact(&locator, &[TranscriptMessage::system("summary")]); - let sealed = resolve_keyed_transcript_path(dir.path(), &session_stem(&session())).unwrap(); - let before = std::fs::read(&sealed).unwrap(); - - let attempt = append_to_head( - &locator, +async fn an_attempt_against_a_sealed_head_writes_nothing_and_asks_to_re_resolve() { + let locator = locator(); + live_turn(locator.as_ref(), &session(), &[], &first_turn()); + compact(locator.as_ref(), &[TranscriptMessage::system("summary")]); + let sealed = locator.read_session_transcript(&session()).unwrap(); + let before = sealed.read_session().unwrap().unwrap().messages; + + let attempt = append_to_stored_head( + locator.as_ref(), &session(), &TranscriptMessage::assistant("Time to stretch!"), &options("run-1"), @@ -427,76 +397,33 @@ fn an_attempt_against_a_sealed_head_writes_nothing_and_asks_to_re_resolve() { .unwrap(); assert!(matches!(attempt, Attempt::HeadMoved)); - assert_eq!(std::fs::read(&sealed).unwrap(), before); -} - -#[tokio::test] -async fn an_agents_files_adopt_a_shared_session_before_the_append() { - let dir = tempdir().unwrap(); - // The conversation exists only in the shared workspace, as written - // before the agent kept its own directory. - let shared = FileTranscriptLocator::new(dir.path()); - live_turn(&shared, &session(), &[], &first_turn()); - let shared_path = head_path(dir.path(), &session()); - let before = std::fs::read(&shared_path).unwrap(); - - let agent = super::super::AgentTranscriptFiles::new(dir.path(), "orchestrator"); - let outcome = append_background_message_in( - &agent, - &session(), - TranscriptMessage::assistant("Time to stretch!"), - options("run-1"), - ) - .await - .unwrap(); - - assert_eq!(outcome, BackgroundAppendOutcome::Appended { generation: 0 }); - let own_root = super::super::agent_transcript_root(dir.path(), "orchestrator"); - let own = read_transcript(&head_path(&own_root, &session())).unwrap(); - assert_eq!( - own.messages.last().unwrap().content, - "Time to stretch!", - "the adopted copy receives the message" - ); - assert_eq!( - std::fs::read(&shared_path).unwrap(), - before, - "the shared file is never modified" - ); + assert_eq!(sealed.read_session().unwrap().unwrap().messages, before); } -/// A store-like locator that names no destination (and is not file backed), -/// over real transcript files. -struct Unnamed(FileTranscriptLocator); +/// A store-like locator that names no destination, over real transcripts. +struct Unnamed(Arc); impl TranscriptLocator for Unnamed { - fn latest_for_agent( - &self, - agent_name: &str, - ) -> Option> { + fn latest_for_agent(&self, agent_name: &str) -> Option> { self.0.latest_for_agent(agent_name) } - fn root_for_thread( - &self, - thread_id: &str, - ) -> Option> { + fn root_for_thread(&self, thread_id: &str) -> Option> { self.0.root_for_thread(thread_id) } fn open_stem( &self, stem: &str, seed: TranscriptMeta, - ) -> anyhow::Result> { + ) -> anyhow::Result> { self.0.open_stem(stem, seed) } } #[tokio::test] async fn appends_through_one_unnamed_locator_are_serialised() { - let dir = tempdir().unwrap(); - let files = FileTranscriptLocator::new(dir.path()); - live_turn(&files, &session(), &[], &first_turn()); - let locator = std::sync::Arc::new(Unnamed(FileTranscriptLocator::new(dir.path()))); + let inner = locator(); + live_turn(inner.as_ref(), &session(), &[], &first_turn()); + let locator = Arc::new(Unnamed(inner)); assert!(locator.destination_key().is_none()); let held = super::super::turn_lock::lock_session_turn_or_identity( @@ -505,16 +432,8 @@ async fn appends_through_one_unnamed_locator_are_serialised() { ) .await; let delivery = { - let locator = std::sync::Arc::clone(&locator); - tokio::spawn(async move { - append_background_message_in( - locator.as_ref(), - &session(), - TranscriptMessage::assistant("Time to stretch!"), - options("run-1"), - ) - .await - }) + let locator = Arc::clone(&locator); + tokio::spawn(async move { deliver(locator.as_ref(), "run-1").await }) }; for _ in 0..50 { tokio::task::yield_now().await; @@ -531,39 +450,29 @@ async fn appends_through_one_unnamed_locator_are_serialised() { } /// A compaction that keeps the delivered message carries its provenance into -/// the successor, so a retry is a duplicate there too — as it is for a store -/// that keeps rows. +/// the successor, so a retry is a duplicate there too. #[tokio::test] async fn a_delivery_carried_through_a_compaction_is_still_a_duplicate() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - live_turn(&locator, &session(), &[], &first_turn()); - let bound = session(); - let deliver = || { - append_background_message( - &locator, - &bound, - TranscriptMessage::assistant("Time to stretch!"), - options("run-1"), - ) - }; + let locator = locator(); + live_turn(locator.as_ref(), &session(), &[], &first_turn()); assert_eq!( - deliver().await.unwrap(), + deliver(locator.as_ref(), "run-1").await.unwrap(), BackgroundAppendOutcome::Appended { generation: 0 } ); // The runtime compacts from the context it read back. - let retained = read_transcript(&head_path(dir.path(), &session())) - .unwrap() - .messages; - compact(&locator, &retained); + let retained = head_messages(locator.as_ref(), &session()); + compact(locator.as_ref(), &retained); assert_eq!( - deliver().await.unwrap(), + deliver(locator.as_ref(), "run-1").await.unwrap(), BackgroundAppendOutcome::Duplicate { generation: 1 } ); - let head = read_transcript_display(&head_path(dir.path(), &session())).unwrap(); - let carried = head + let display = head(locator.as_ref(), &session()) + .read_display() + .unwrap() + .unwrap(); + let carried = display .records .iter() .filter_map(|record| match record { @@ -573,3 +482,54 @@ async fn a_delivery_carried_through_a_compaction_is_still_a_duplicate() { .count(); assert_eq!(carried, 1, "the successor's row keeps its provenance"); } + +/// A compaction inside the *same* generation (the handle's `replace`) that +/// drops the delivered message removes it from the live context, so the key is +/// free again: retrying the delivery appends it instead of reporting a +/// duplicate of a row the model no longer sees. +#[tokio::test] +async fn a_delivery_dropped_by_a_same_generation_compaction_is_appended_again() { + let locator = locator(); + live_turn(locator.as_ref(), &session(), &[], &first_turn()); + assert_eq!( + deliver(locator.as_ref(), "run-1").await.unwrap(), + BackgroundAppendOutcome::Appended { generation: 0 } + ); + + head_handle(locator.as_ref()) + .replace(&[TranscriptMessage::system("summary")]) + .unwrap(); + + assert_eq!( + deliver(locator.as_ref(), "run-1").await.unwrap(), + BackgroundAppendOutcome::Appended { generation: 0 }, + "the compaction dropped the delivery, so the key is not held" + ); + assert_eq!( + deliver(locator.as_ref(), "run-1").await.unwrap(), + BackgroundAppendOutcome::Duplicate { generation: 0 }, + "the fresh delivery is held again" + ); +} + +/// The same-generation counterpart: a compaction that keeps the delivery keeps +/// its provenance, so the retry is still a duplicate. +#[tokio::test] +async fn a_delivery_kept_by_a_same_generation_compaction_is_still_a_duplicate() { + let locator = locator(); + live_turn(locator.as_ref(), &session(), &[], &first_turn()); + deliver(locator.as_ref(), "run-1").await.unwrap(); + + let retained = head_messages(locator.as_ref(), &session()); + head_handle(locator.as_ref()).replace(&retained).unwrap(); + + assert_eq!( + deliver(locator.as_ref(), "run-1").await.unwrap(), + BackgroundAppendOutcome::Duplicate { generation: 0 } + ); +} + +fn head_handle(locator: &dyn TranscriptLocator) -> Arc { + let head = locator.head_generation(&session()); + locator.open_session(&head, meta()).unwrap() +} diff --git a/crates/tinyagents-session/src/transcript/history.rs b/crates/tinyagents-session/src/transcript/history.rs index 6e3e17b16..d384a965c 100644 --- a/crates/tinyagents-session/src/transcript/history.rs +++ b/crates/tinyagents-session/src/transcript/history.rs @@ -17,20 +17,14 @@ //! rewrite. [`TranscriptHistory::clear`] is therefore an empty compaction. //! -use std::collections::{HashMap, VecDeque}; -use std::fs::{self, File, OpenOptions}; -use std::path::{Component, Path, PathBuf}; -use std::sync::{Arc, Mutex, OnceLock, Weak}; +use std::path::Path; +use std::sync::Arc; use crate::transcript::types::TranscriptMessage; -use fs2::FileExt; use crate::transcript::{ DisplaySessionTranscript, SessionAdoption, SessionRef, SessionTranscript, ThreadTranscript, - ThreadTranscripts, TranscriptMeta, TurnUsage, adopt_legacy_session_transcripts, - find_latest_transcript, find_root_transcript_for_thread, - find_root_transcript_for_thread_scoped, read_transcript, resolve_keyed_transcript_path, - session_stem, + ThreadTranscripts, TranscriptMeta, TurnUsage, session_stem, }; /// The stem a read handle is bound to: its path's last component, without the @@ -65,8 +59,7 @@ const MAX_GENERATIONS: u32 = 4096; /// `prev` is a field rather than handle state on purpose: the turn path tracks /// the previously-persisted logical set in memory on `Agent` /// (`persisted_transcript_messages`) precisely so it never has to re-read a -/// growing file, and a disk re-read is not a faithful substitute — see -/// `FileTranscriptHistory::write_logical_set_locked`. +/// growing log, and a store re-read is not a faithful substitute. pub struct TranscriptTurn<'a> { /// Logical message set already persisted, for the extension-vs-compaction diff. pub prev: &'a [TranscriptMessage], @@ -152,9 +145,7 @@ pub trait TranscriptHistory: TranscriptRead { /// reads hold. /// /// Split out of [`TranscriptHistory`] rather than added as one more method on it, -/// for a reason that is not stylistic: a *discovered* transcript can still be a -/// legacy `.md` file (see [`FileTranscriptHistory::opened_at`]), and -/// `append_transcript_turn` writes JSONL. Handing discovery results out as +/// for a reason that is not stylistic: handing discovery results out as /// `Arc` makes it impossible to `append_turn` into /// one by construction, instead of by convention. /// @@ -171,7 +162,7 @@ pub trait TranscriptRead: Send + Sync { /// The model-context replay of this transcript, `_meta` included, or /// `Ok(None)` when the file does not exist. /// - /// Exactly [`read_transcript`], so compaction records have already replaced + /// The log replayed, so compaction records have already replaced /// the accumulator and `interrupted: true` partials are already skipped — /// §3.1's "single most important constraint". Returning the whole /// [`SessionTranscript`] rather than messages alone is what lets the shadow @@ -194,11 +185,10 @@ pub trait TranscriptRead: Send + Sync { /// this session's own write handle. /// /// One injected object covers the whole turn path: both resume reads and the -/// first-write bind. A host holds it as `Option>` -/// and falls back to [`FileTranscriptLocator`] built from the *current* -/// `workspace_dir` — lazily, never frozen at build time, -/// because tests reassign `agent.workspace_dir` after `build()` and a -/// build-time locator would silently keep pointing at the old directory. +/// first-write bind. A host holds it as `Option>`, +/// built from the store it currently targets rather than frozen at build time, +/// so a host that rebuilds its locator per turn still resolves the same +/// destination (see [`Self::destination_key`]). pub trait TranscriptLocator: Send + Sync { /// The durable destination this locator addresses, when it can name one. /// @@ -228,9 +218,9 @@ pub trait TranscriptLocator: Send + Sync { fn root_for_thread(&self, thread_id: &str) -> Option>; /// [`Self::root_for_thread`], additionally scoped to `agent_id` when - /// given — see - /// [`transcript::find_root_transcript_for_thread_scoped`](crate::transcript::find_root_transcript_for_thread_scoped) - /// for why. Defaults to the unscoped lookup so an implementor that never + /// given: one caller-supplied thread id can be handed to several + /// independently configured agents, and each needs its own transcript. + /// Defaults to the unscoped lookup so an implementor that never /// serves several distinct agents over the same `thread_id` (this test /// double, notably) does not have to know about agent scoping at all. fn root_for_thread_scoped( @@ -264,15 +254,6 @@ pub trait TranscriptLocator: Send + Sync { }) } - /// The file locator this one writes through, when it is file-backed. - /// - /// Lets a generic writer keep the JSONL line format exact (for example - /// [`append_background_message_in`](super::append_background_message_in) - /// records provenance as a line field there). `None` for any other store. - fn file_backing(&self) -> Option<&FileTranscriptLocator> { - None - } - /// Binds (creating on first write) this session's own write handle for /// `stem`, with `seed` used only when no file exists yet. fn open_stem( @@ -587,457 +568,6 @@ impl TruncateCut { } } -/// The default [`TranscriptLocator`]: real files under -/// `{workspace_dir}/session_raw`. -/// -/// Thin by design — each method wraps exactly one `transcript::` free function -/// and changes nothing about it, so swapping the turn path onto the locator is -/// behaviour-preserving. -pub struct FileTranscriptLocator { - workspace_dir: PathBuf, -} - -impl FileTranscriptLocator { - /// Builds a locator rooted at `workspace_dir` (i.e. it resolves - /// `{workspace_dir}/session_raw/...`). - pub fn new(workspace_dir: impl Into) -> Self { - Self { - workspace_dir: workspace_dir.into(), - } - } - - /// The workspace root this locator resolves `session_raw/` under. - pub fn workspace_dir(&self) -> &Path { - &self.workspace_dir - } -} - -impl TranscriptLocator for FileTranscriptLocator { - fn file_backing(&self) -> Option<&FileTranscriptLocator> { - Some(self) - } - - /// The same file set the path-based view projects - /// ([`resolve_files_scoped`](crate::transcript::view::resolve_files_scoped)). - fn thread_transcripts( - &self, - thread_id: &str, - agent_id: Option<&str>, - ) -> Option { - let (roots, subagents) = crate::transcript::view::resolve_files_scoped( - &self.workspace_dir, - thread_id, - agent_id, - )?; - let listed = |paths: Vec| -> Vec { - paths - .into_iter() - .map(|path| { - let stem = stem_of_handle(&path); - let transcript: Arc = Arc::new( - FileTranscriptHistory::opened_at(path, seed_meta_for_discovered(&stem)), - ); - ThreadTranscript { stem, transcript } - }) - .collect() - }; - Some(ThreadTranscripts { - roots: listed(roots), - subagents: listed(subagents), - }) - } - - /// The workspace root every lookup and bind resolves under, which is - /// this locator's only field and therefore its whole identity. - fn destination_key(&self) -> Option { - // The resolved `session_raw/` directory, not the workspace root: two - // workspaces whose `session_raw` is a symlink to one directory share - // their transcript files, so they must share the turn lock. - Some( - symlink_normalized_path(&self.workspace_dir.join("session_raw")) - .to_string_lossy() - .into_owned(), - ) - } - - fn latest_for_agent(&self, agent_name: &str) -> Option> { - let path = find_latest_transcript(&self.workspace_dir, agent_name).or_else(|| { - let dir = self.workspace_dir.join("session_raw"); - let mut best: Option<(String, PathBuf)> = None; - for entry in fs::read_dir(dir).ok()?.flatten() { - let candidate = entry.path(); - if candidate.extension().and_then(|ext| ext.to_str()) != Some("jsonl") { - continue; - } - let Ok(transcript) = read_transcript(&candidate) else { - continue; - }; - if transcript.meta.parent_session_id.is_none() - && (transcript.meta.agent_name == agent_name - || transcript.meta.agent_id.as_deref() == Some(agent_name)) - && best - .as_ref() - .is_none_or(|(updated, _)| transcript.meta.updated > *updated) - { - best = Some((transcript.meta.updated, candidate)); - } - } - best.map(|(_, path)| path).or_else(|| { - crate::transcript::paths::find_latest_legacy_transcript( - &self.workspace_dir, - agent_name, - ) - }) - })?; - tracing::debug!( - "[transcript-history] locator latest_for_agent agent={agent_name} path={}", - path.display() - ); - Some(Arc::new(FileTranscriptHistory::opened_at( - path, - seed_meta_for_discovered(agent_name), - ))) - } - - fn append_interrupted_partial( - &self, - thread_id: &str, - agent_id: Option<&str>, - partial: &TranscriptPartial, - request_id: Option<&str>, - ) -> anyhow::Result { - let Some(path) = - find_root_transcript_for_thread_scoped(&self.workspace_dir, thread_id, agent_id) - else { - return Ok(false); - }; - if partial.content.is_empty() { - return Ok(false); - } - // Bind the lock handle to the current head so partial writes share - // the generation and successor-reservation locks with head writers. - let root_path = path; - let path = head_generation_path(&root_path); - let history = - FileTranscriptHistory::opened_at(path.clone(), seed_meta_for_discovered(thread_id)); - history.with_write_locks(|| { - anyhow::ensure!( - head_generation_path(&root_path) == path, - "transcript head advanced during partial append; retry" - ); - crate::transcript::append_interrupted_partial( - &path, - &partial.content, - request_id, - partial.iteration, - partial.reasoning_content.as_deref(), - )?; - tracing::debug!( - "[transcript-history] locator appended interrupted partial thread={thread_id} chars={} path={}", - partial.content.len(), - path.display() - ); - Ok(true) - }) - } - - fn root_for_thread(&self, thread_id: &str) -> Option> { - let path = find_root_transcript_for_thread(&self.workspace_dir, thread_id)?; - tracing::debug!( - "[transcript-history] locator root_for_thread thread={thread_id} path={}", - path.display() - ); - Some(Arc::new(FileTranscriptHistory::opened_at( - path, - seed_meta_for_discovered(thread_id), - ))) - } - - fn root_for_thread_scoped( - &self, - thread_id: &str, - agent_id: Option<&str>, - ) -> Option> { - // Same cross-dir, newest-wins scan as `root_for_thread`, additionally - // filtered on `_meta.agent_id` so one runtime agent's resume cannot - // pick up a different agent's transcript for a caller-reused - // `thread_id` — see `find_root_transcript_for_thread_scoped`. - let path = - find_root_transcript_for_thread_scoped(&self.workspace_dir, thread_id, agent_id)?; - tracing::debug!( - "[transcript-history] locator root_for_thread_scoped thread={thread_id} \ - agent_id={agent_id:?} path={}", - path.display() - ); - Some(Arc::new(FileTranscriptHistory::opened_at( - path, - seed_meta_for_discovered(thread_id), - ))) - } - - fn open_stem( - &self, - stem: &str, - seed: TranscriptMeta, - ) -> anyhow::Result> { - Ok(Arc::new(FileTranscriptHistory::new( - &self.workspace_dir, - stem, - seed, - )?)) - } - - fn session_exists(&self, session: &SessionRef) -> bool { - // A direct path probe, not a read: `head_generation` calls this once - // per generation and only needs to know whether the file is there. - // `is_file()` rather than `exists()`: a directory, FIFO or other - // non-regular entry occupying the canonical path must not be - // reported as an existing generation — reads/appends against it - // would fail (or, for a directory, silently target the wrong thing) - // downstream, and `head_generation`'s chain walk would stop at a - // phantom "generation" that was never actually written. - resolve_keyed_transcript_path(&self.workspace_dir, &session_stem(session)) - .is_ok_and(|path| path.is_file()) - } - - fn read_session_transcript(&self, session: &SessionRef) -> Option> { - let stem = session_stem(session); - let path = resolve_keyed_transcript_path(&self.workspace_dir, &stem).ok()?; - if !path.is_file() { - return None; - } - tracing::debug!( - "[transcript-history] locator read_session session={stem} path={}", - path.display() - ); - Some(Arc::new(FileTranscriptHistory::opened_at( - path, - seed_meta_for_discovered(&stem), - ))) - } - - fn adopt_legacy( - &self, - session: &SessionRef, - thread_id: &str, - seed: &TranscriptMeta, - ) -> anyhow::Result> { - adopt_legacy_session_transcripts(&self.workspace_dir, session, thread_id, seed) - } - - fn begin_generation( - &self, - session: &SessionRef, - seed: TranscriptMeta, - ) -> anyhow::Result<(SessionRef, Arc)> { - begin_file_generation_with_baseline(&self.workspace_dir, session, seed, None) - } - - fn begin_generation_from_baseline( - &self, - session: &SessionRef, - seed: TranscriptMeta, - baseline: &[TranscriptMessage], - ) -> anyhow::Result<(SessionRef, Arc)> { - begin_file_generation_with_baseline(&self.workspace_dir, session, seed, Some(baseline)) - } -} - -/// Return a stable absolute key even before the workspace has been created. -/// `canonicalize` cannot resolve a path with a missing component, but the -/// in-process turn lock still needs equivalent relative and absolute paths to -/// identify the same destination. -fn absolute_normalized_path(path: &Path) -> PathBuf { - let absolute = if path.is_absolute() { - path.to_path_buf() - } else { - std::env::current_dir() - .unwrap_or_else(|_| PathBuf::from(".")) - .join(path) - }; - let mut normalized = PathBuf::new(); - for component in absolute.components() { - match component { - Component::CurDir => {} - Component::ParentDir => { - if normalized.components().count() > 1 - && !matches!( - normalized.components().next_back(), - Some(Component::RootDir) - ) - { - normalized.pop(); - } - } - Component::Prefix(_) | Component::RootDir | Component::Normal(_) => { - normalized.push(component.as_os_str()); - } - } - } - normalized -} - -/// Returns an absolute key with symlinks resolved in every existing prefix. -/// The final workspace directory may not exist yet, so canonicalizing the -/// whole path is not sufficient for locator identity. -fn symlink_normalized_path(path: &Path) -> PathBuf { - enum OwnedComponent { - CurDir, - ParentDir, - Prefix(std::ffi::OsString), - RootDir, - Normal(std::ffi::OsString), - } - - fn owned_components(path: &Path) -> VecDeque { - path.components() - .map(|component| match component { - Component::CurDir => OwnedComponent::CurDir, - Component::ParentDir => OwnedComponent::ParentDir, - Component::Prefix(prefix) => { - OwnedComponent::Prefix(prefix.as_os_str().to_os_string()) - } - Component::RootDir => OwnedComponent::RootDir, - Component::Normal(name) => OwnedComponent::Normal(name.to_os_string()), - }) - .collect() - } - - let absolute = if path.is_absolute() { - path.to_path_buf() - } else { - std::env::current_dir() - .unwrap_or_else(|_| PathBuf::from(".")) - .join(path) - }; - let fallback = absolute_normalized_path(&absolute); - let mut resolved = PathBuf::new(); - let mut pending = owned_components(&absolute); - let mut symlink_hops = 0; - - while let Some(component) = pending.pop_front() { - match component { - OwnedComponent::CurDir => {} - OwnedComponent::ParentDir => { - if resolved.components().count() > 1 - && !matches!(resolved.components().next_back(), Some(Component::RootDir)) - { - resolved.pop(); - } - } - OwnedComponent::Prefix(prefix) => resolved.push(prefix), - OwnedComponent::RootDir => resolved.push(std::path::MAIN_SEPARATOR.to_string()), - OwnedComponent::Normal(name) => { - let candidate = resolved.join(&name); - match fs::symlink_metadata(&candidate) { - Ok(metadata) if metadata.file_type().is_symlink() => { - symlink_hops += 1; - if symlink_hops > 40 { - return fallback; - } - let target = match fs::read_link(&candidate) { - Ok(target) => target, - Err(_) => return fallback, - }; - let mut replacement = if target.is_absolute() { - owned_components(&target) - } else { - owned_components(&resolved) - }; - if !target.is_absolute() { - replacement.extend(owned_components(&target)); - } - replacement.extend(pending); - pending = replacement; - resolved.clear(); - } - _ => resolved.push(name), - } - } - } - } - resolved -} - -fn begin_file_generation_with_baseline( - workspace_dir: &Path, - session: &SessionRef, - seed: TranscriptMeta, - baseline: Option<&[TranscriptMessage]>, -) -> anyhow::Result<(SessionRef, Arc)> { - let successor = session.next_generation(); - anyhow::ensure!( - successor.generation <= MAX_GENERATIONS, - "session {} has reached the {MAX_GENERATIONS}-generation compaction limit; \ - refusing to create generation {}", - session.session_id(), - successor.generation - ); - let stem = session_stem(&successor); - let path = resolve_keyed_transcript_path(workspace_dir, &stem)?; - let parent_path = resolve_keyed_transcript_path(workspace_dir, &session_stem(session))?; - let mut lock_paths = vec![parent_path.clone(), path.clone()]; - lock_paths.dedup(); - let mut locks = Vec::with_capacity(lock_paths.len()); - for lock_path in &lock_paths { - let mut lock = generation_lock(lock_path)?; - lock.lock_exclusive()?; - locks.push((lock_path.clone(), lock)); - } - let parent_lock = locks - .iter() - .position(|(locked_path, _)| locked_path == &parent_path) - .map(|index| locks.remove(index).1); - let successor_lock = locks - .iter() - .position(|(locked_path, _)| locked_path == &path) - .map(|index| locks.remove(index).1) - .expect("successor path lock acquired"); - if let Some(baseline) = baseline { - let current = if parent_path.is_file() { - read_transcript(&parent_path)?.messages - } else { - Vec::new() - }; - anyhow::ensure!( - same_transcript_messages(¤t, baseline), - "transcript baseline is stale for {}; reload the session before persisting", - parent_path.display() - ); - } - anyhow::ensure!( - !path_entry_exists(&path)?, - "session generation {stem} already exists; refusing to overwrite a sealed transcript" - ); - - let mut meta = seed; - meta.session_id = Some(successor.session_id()); - meta.parent_session_id = successor.parent_session_id(); - tracing::info!( - "[transcript-history] sealed session={} and opened generation {} at {}", - session.session_id(), - successor.generation, - path.display() - ); - let mut history = FileTranscriptHistory::new(workspace_dir, &stem, meta)?; - // The parent is protected while the successor slot is selected and - // reserved. Later parent writers take the parent lock first, then fail - // promptly on the still-absent, reserved successor slot. - drop(parent_lock); - *history.generation_reservation.get_mut().unwrap() = Some(GenerationReservation { - _successor: successor_lock, - }); - Ok((successor, Arc::new(history))) -} - -fn path_entry_exists(path: &Path) -> anyhow::Result { - match std::fs::symlink_metadata(path) { - Ok(_) => Ok(true), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), - Err(error) => Err(error.into()), - } -} - pub(crate) fn same_transcript_messages( left: &[TranscriptMessage], right: &[TranscriptMessage], @@ -1047,11 +577,11 @@ pub(crate) fn same_transcript_messages( /// A placeholder `_meta` for a handle bound to an already-existing transcript. /// -/// `seed_meta` is consulted only when the file is **absent**, and a discovered -/// path exists by definition, so this value is never written. It exists because -/// [`FileTranscriptHistory`] is one type serving both roles; giving read-only -/// handles a `None` meta would mean an `Option` field every write path then has -/// to unwrap for no benefit. +/// `seed_meta` is consulted only when the transcript is **absent**, and a +/// discovered one exists by definition, so this value is never written. One +/// history type serves both roles; giving read-only handles a `None` meta +/// would mean an `Option` field every write path then has to unwrap for no +/// benefit. pub(super) fn seed_meta_for_discovered(agent_name: &str) -> TranscriptMeta { TranscriptMeta { session_id: None, @@ -1074,588 +604,3 @@ pub(super) fn seed_meta_for_discovered(agent_name: &str) -> TranscriptMeta { task_id: None, } } - -/// A lossless history backed by one `session_raw/{stem}.jsonl` transcript. -/// -/// Construct with [`FileTranscriptHistory::new`] (workspace-rooted, i.e. -/// `{workspace}/session_raw/`). The `seed_meta` is used only when the -/// transcript file does not exist yet; for an existing file the authoritative -/// cumulative `_meta` is read back from disk so turn counts and token rollups -/// keep accumulating rather than resetting. -pub struct FileTranscriptHistory { - /// Fully-resolved transcript file, fixed at construction. - /// - /// Resolved eagerly rather than derived per call from a `(workspace, stem)` - /// pair: the old shape hardcoded `{workspace}/session_raw/`, which is the - /// **wrong directory** for a canonical session and would have silently - /// cross-written into the canonical session's transcripts the moment this - /// handle was wired into the turn path. - path: PathBuf, - /// `_meta` used for the very first write, before a file exists. - seed_meta: TranscriptMeta, - /// Cross-process reservation held from generation selection through its - /// first successful write. A crash releases the advisory lock. - generation_reservation: Mutex>, - /// Serializes callers sharing this handle while its first-write reservation - /// is being consumed. The process-wide path lock alone is acquired too late - /// to prevent two callers from both observing the reservation. - write_serial: Mutex<()>, -} - -struct GenerationReservation { - _successor: GenerationLock, -} - -/// Releases a successfully acquired advisory lock before closing its descriptor. -/// -/// Closing alone leaves flock held when a concurrent fork inherited the open -/// file description. Explicit unlock makes the writer/reservation lifetime -/// authoritative even while that inherited descriptor survives until exec. -struct GenerationLock { - file: File, - held: bool, -} - -impl GenerationLock { - fn lock_exclusive(&mut self) -> std::io::Result<()> { - FileExt::lock_exclusive(&self.file)?; - self.held = true; - Ok(()) - } - - fn try_lock_exclusive(&mut self) -> std::io::Result<()> { - FileExt::try_lock_exclusive(&self.file)?; - self.held = true; - Ok(()) - } -} - -impl Drop for GenerationLock { - fn drop(&mut self) { - if self.held { - // Closing remains the final release if the OS refuses explicit unlock. - let _ = FileExt::unlock(&self.file); - } - } -} - -impl FileTranscriptHistory { - fn acquire_write_lock(&self) -> anyhow::Result> { - if self.generation_reservation.lock().unwrap().is_some() { - return Ok(None); - } - let stem = self - .path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or_default(); - let (parent_stem, generation) = match stem.rsplit_once(".g") { - Some((parent, suffix)) if suffix.parse::().is_ok() => { - (parent, suffix.parse::().unwrap()) - } - _ => (stem, 0), - }; - let successor = self - .path - .with_file_name(format!("{parent_stem}.g{}.jsonl", generation + 1)); - // All writers and generation creators acquire parent before successor. - // This lets ordinary writes serialize on the parent while a generation - // reservation on an absent successor fails promptly instead of - // blocking the generation's first write. - let paths = vec![self.path.clone(), successor.clone()]; - let mut locks = Vec::with_capacity(paths.len()); - for lock_path in paths { - let mut lock = generation_lock(&lock_path)?; - if lock_path == successor && !path_entry_exists(&successor)? { - // An absent successor with a held lock is reserved by a - // generation creator, which needs its first write to release - // that reservation. Do not wait here: this parent write would - // otherwise deadlock the creator permanently. - lock.try_lock_exclusive().map_err(|error| { - anyhow::anyhow!( - "successor generation is reserved for {}; retry after it commits: {error}", - self.path.display() - ) - })?; - } else { - // Use the same sorted acquisition order as generation creation - // for locks that are not an unwritten successor reservation. - lock.lock_exclusive()?; - } - locks.push((lock_path, lock)); - } - let parent_lock = locks - .iter() - .position(|(path, _)| path == &self.path) - .map(|index| locks.remove(index).1) - .expect("parent path lock acquired"); - let successor_lock = locks - .iter() - .position(|(path, _)| path == &successor) - .map(|index| locks.remove(index).1) - .expect("successor path lock acquired"); - Ok(Some((successor_lock, parent_lock))) - } - - fn finish_generation_reservation(&self, success: bool) -> anyhow::Result<()> { - if success && let Some(reservation) = self.generation_reservation.lock().unwrap().take() { - drop(reservation); - } - Ok(()) - } - - /// Binds a history handle to `{workspace_dir}/session_raw/{stem}.jsonl`. - /// - pub fn new( - workspace_dir: impl AsRef, - stem: &str, - seed_meta: TranscriptMeta, - ) -> anyhow::Result { - let path = resolve_keyed_transcript_path(workspace_dir.as_ref(), stem)?; - tracing::debug!( - "[transcript-history] bound stem={stem} path={}", - path.display() - ); - Ok(Self { - path, - seed_meta, - generation_reservation: Mutex::new(None), - write_serial: Mutex::new(()), - }) - } - - /// Binds a handle to an **already-discovered** transcript file, verbatim. - /// - /// Deliberately does **not** go through `resolve_keyed_transcript_path*`, - /// which the two stem constructors above use. That helper `create_dir_all`s - /// its parent and forces a `.jsonl` extension — both wrong for a discovered - /// path: `find_latest_transcript` can still return a legacy `.md` - /// file (`read_transcript` routes by extension), and re-resolving would - /// mangle it into a sibling `.jsonl` that does not exist while creating - /// stray directories on a pure read. - /// - /// Hand the result out as `Arc`, not - /// `Arc` — see [`TranscriptRead`]'s doc. - pub fn opened_at(path: PathBuf, seed_meta: TranscriptMeta) -> Self { - tracing::debug!( - "[transcript-history] opened discovered path={}", - path.display() - ); - Self { - path, - seed_meta, - generation_reservation: Mutex::new(None), - write_serial: Mutex::new(()), - } - } - - /// This handle's transcript file. - pub fn path(&self) -> &Path { - &self.path - } - - /// Reads the current transcript, or `None` when no file exists yet. - /// - /// A missing transcript is the normal first-turn state, not an error. - fn read(&self) -> anyhow::Result> { - if !self.path.exists() { - return Ok(None); - } - read_transcript(&self.path).map(Some) - } - - /// The logical (model-context) message set currently on disk. - /// - /// Routes through [`read_transcript`], so compaction records have already - /// replaced the accumulator and `interrupted: true` partials are skipped. - fn persisted(&self) -> anyhow::Result> { - Ok(self.read()?.map(|t| t.messages).unwrap_or_default()) - } - - /// The `_meta` to write: the file's own cumulative meta when it exists, - /// otherwise this handle's seed. - /// - /// Uses the existing durable metadata as a default when a caller does not - /// caller-computed meta. The **turn path must never route through here** — - /// it computes `turn_count` and the four token/cost rollups fresh each turn, - /// and re-reading the file's `_meta` would freeze them at the previous - /// turn's values, silently breaking `read_thread_usage_summary`. - fn meta_for_write(&self) -> anyhow::Result { - Ok(self - .read()? - .map(|t| t.meta) - .unwrap_or_else(|| self.seed_meta.clone())) - } -} - -/// Opens the stable advisory-lock file associated with a transcript path. -/// The file is intentionally retained after unlock; deleting lock files can -/// split waiters across different inodes and invalidate mutual exclusion. -fn generation_lock(path: &Path) -> anyhow::Result { - let parent = path.parent().unwrap_or_else(|| Path::new(".")); - let lock_root = parent.parent().unwrap_or(parent).join(".transcript-locks"); - let lock_dir = lock_root.join(parent.file_name().unwrap_or_default()); - fs::create_dir_all(&lock_dir)?; - let lock_name = path.file_name().unwrap_or_default(); - let lock_path = lock_dir.join(lock_name); - OpenOptions::new() - .create(true) - .truncate(false) - .read(true) - .write(true) - .open(lock_path) - .map(|file| GenerationLock { file, held: false }) - .map_err(Into::into) -} - -/// A process-wide, per-path mutex serializing the read-modify-write sequence -/// [`FileTranscriptHistory::append`]/`replace`/`clear` run against one file. -/// -/// [`SessionRef`]'s own doc names this as a supported shape: two cores in one -/// process sharing a workspace should both see and extend one conversation. -/// Without this, two `FileTranscriptHistory` instances bound to the same -/// path (a legitimate, common way to get there — `open_session` is called -/// fresh per `Session::resume`) can each read the file's current content, -/// compute a diff against that now-stale view, and write. Whichever finishes -/// its own read first computes a `next` that does not extend what the file -/// looks like by the time it *writes* — `append_transcript_turn_with_partial` -/// then reads that mismatch as "the context was reduced" and appends a -/// **compaction record** instead of a plain tail, and a compaction's -/// replacement value is what canonical reads return going forward. The -/// other write's whole contribution becomes unreachable, even though its -/// bytes are still physically on disk as a now-superseded line — a silent -/// lost update, not a crash. -/// -/// Keyed by path rather than by `Arc>` identity because the two -/// racing instances are typically *separate* `FileTranscriptHistory` values, -/// not a shared handle. Entries are [`Weak`] and swept opportunistically so -/// the registry does not grow for the lifetime of a long-running host: once -/// every in-flight critical section for a path finishes, nothing keeps that -/// path's entry alive, and the next unrelated call reclaims the slot. -fn path_lock(path: &Path) -> Arc> { - static REGISTRY: OnceLock>>>> = OnceLock::new(); - let registry = REGISTRY.get_or_init(|| Mutex::new(HashMap::new())); - let mut locks = registry - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - locks.retain(|_, weak| weak.strong_count() > 0); - if let Some(existing) = locks.get(path).and_then(Weak::upgrade) { - return existing; - } - let fresh = Arc::new(Mutex::new(())); - locks.insert(path.to_path_buf(), Arc::downgrade(&fresh)); - fresh -} - -impl TranscriptRead for FileTranscriptHistory { - fn path(&self) -> &Path { - &self.path - } - - /// Same call the free-function readers make, on the same path, with the - /// same return type — so there is nothing left for the round trip to lose. - fn read_session(&self) -> anyhow::Result> { - if !self.path.exists() { - tracing::debug!( - "[transcript-history] read_session absent path={}", - self.path.display() - ); - return Ok(None); - } - let session = read_transcript(&self.path)?; - tracing::debug!( - "[transcript-history] read_session messages={} path={}", - session.messages.len(), - self.path.display() - ); - Ok(Some(session)) - } - - /// [`read_transcript_display`](crate::transcript::read_transcript_display) - /// on this handle's file. A legacy `.md` transcript has no display log, so - /// it projects its model context instead. - fn read_display(&self) -> anyhow::Result> { - if !self.path.exists() { - return Ok(None); - } - if self.path.extension().and_then(|ext| ext.to_str()) != Some("jsonl") { - return Ok(self.read_session()?.map(super::display_from_session)); - } - crate::transcript::read_transcript_display(&self.path).map(Some) - } -} - -impl FileTranscriptHistory { - /// The actual `append_turn` write. Assumes the caller already holds - /// [`path_lock`] for [`Self::path`] — never call this directly; every - /// public entry point below acquires the lock once and then routes - /// through here (and [`Self::append_turn_with_partial_locked`]) so the - /// lock is taken exactly once per call, never nested (this crate's - /// `Mutex` is not reentrant). - fn append_turn_locked(&self, turn: TranscriptTurn<'_>) -> anyhow::Result<()> { - self.validate_turn_baseline(turn.prev)?; - tracing::debug!( - "[transcript-history] append_turn prev={} next={} usage={} request_id={:?} path={}", - turn.prev.len(), - turn.next.len(), - turn.turn_usage.is_some(), - turn.request_id, - self.path.display() - ); - crate::transcript::writer::append_transcript_turn_with_extras( - &self.path, - turn.prev, - turn.next, - turn.meta, - turn.turn_usage, - turn.request_id, - crate::transcript::writer::AppendTranscriptExtras { - partial: None, - tools: turn.tools, - }, - )?; - Ok(()) - } - - /// [`Self::append_turn_locked`]'s counterpart for the display-partial - /// variant. Same locking contract. - fn append_turn_with_partial_locked( - &self, - turn: TranscriptTurn<'_>, - partial: Option<&TranscriptPartial>, - ) -> anyhow::Result<()> { - self.validate_turn_baseline(turn.prev)?; - tracing::debug!( - "[transcript-history] append_turn_with_partial prev={} next={} partial={} path={}", - turn.prev.len(), - turn.next.len(), - partial.is_some(), - self.path.display() - ); - crate::transcript::writer::append_transcript_turn_with_extras( - &self.path, - turn.prev, - turn.next, - turn.meta, - turn.turn_usage, - turn.request_id, - crate::transcript::writer::AppendTranscriptExtras { - partial, - tools: turn.tools, - }, - )?; - Ok(()) - } - - fn validate_turn_baseline(&self, prev: &[TranscriptMessage]) -> anyhow::Result<()> { - if self.generation_reservation.lock().unwrap().is_some() || !self.path.is_file() { - return Ok(()); - } - let stem = self - .path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or_default(); - let (parent_stem, generation) = match stem.rsplit_once(".g") { - Some((parent, suffix)) if suffix.parse::().is_ok() => { - (parent, suffix.parse::().unwrap()) - } - _ => (stem, 0), - }; - let successor_stem = format!("{parent_stem}.g{}", generation + 1); - let successor_path = self.path.with_file_name(format!("{successor_stem}.jsonl")); - anyhow::ensure!( - !successor_path.is_file(), - "session generation {successor_stem} already exists; reload the session before persisting" - ); - let disk = self.persisted()?; - let same = disk.len() == prev.len() - && disk - .iter() - .zip(prev) - .all(|(left, right)| left.same_row_as(right)); - anyhow::ensure!( - same, - "transcript baseline is stale for {}; reload the session before persisting", - self.path.display() - ); - Ok(()) - } - - /// Writes `next` as the new logical set, diffing against what is - /// persisted. Assumes the caller already holds [`path_lock`] for - /// [`Self::path`] — see [`Self::append_turn_locked`]'s doc for why. - /// - /// Routes through [`Self::append_turn_locked`] so every write in this - /// module — trait-driven and turn-path alike — funnels through one call - /// to [`append_transcript_turn`], and the extension-vs-compaction - /// decision stays with the format owner rather than drifting here. - /// - /// The `self.persisted()` disk re-read is what the generic trait path has - /// to do, and is deliberately **not** what the turn path does. - /// [`read_transcript`] reconstructs `TranscriptMessage`s from line records: the - /// `failure` / `failure_detail` fields have been lifted out of - /// `extra_metadata` and turn-usage fields hoisted to top-level line fields. - /// Feeding that back in as `prev` would make `common_prefix_len` mismatch - /// at the first such message, so the writer would emit a full compaction - /// record — re-appending the entire message set — on every single turn. - fn write_logical_set_locked(&self, next: &[TranscriptMessage]) -> anyhow::Result<()> { - let prev = self.persisted()?; - let meta = self.meta_for_write()?; - self.append_turn_locked(TranscriptTurn { - prev: &prev, - next, - meta: &meta, - turn_usage: None, - request_id: None, - tools: None, - }) - } -} - -impl FileTranscriptHistory { - /// Runs `write` holding exactly the locks every other mutation of this - /// file takes (handle serial, cross-process advisory lock, process-wide - /// path lock), so an out-of-band writer can never interleave its bytes - /// with a turn append or a generation being sealed. - pub(super) fn with_write_locks( - &self, - write: impl FnOnce() -> anyhow::Result, - ) -> anyhow::Result { - let _serial = self.write_serial.lock().unwrap_or_else(|p| p.into_inner()); - let os_lock = self.acquire_write_lock()?; - let lock = path_lock(&self.path); - let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); - let result = write(); - drop(os_lock); - result - } -} - -impl TranscriptHistory for FileTranscriptHistory { - /// Pure forwarder: every argument reaches the transcript writer's turn append - /// untouched, so the bytes this writes are identical to what the free - /// function would have written at the call site. - /// - /// This — not [`TranscriptHistory::append`] — is the turn path's own - /// write call (`Session::persist` in `tinyagents-runtime` calls - /// [`TranscriptHistory::append_turn_with_partial`] directly), so the - /// same [`path_lock`] serialization `append`/`replace`/`clear` need - /// applies here too: two `FileTranscriptHistory` handles bound to the - /// same successor generation (two compactions racing on - /// `TranscriptLocator::begin_generation` for one session) would - /// otherwise both see the file absent and both take the writer's - /// create-fresh path, and whichever `fs::write` lands last would - /// silently discard the other's retained set. - fn append_turn(&self, turn: TranscriptTurn<'_>) -> anyhow::Result<()> { - let _serial = self.write_serial.lock().unwrap_or_else(|p| p.into_inner()); - let os_lock = self.acquire_write_lock()?; - let lock = path_lock(&self.path); - let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); - let result = self.append_turn_locked(turn); - self.finish_generation_reservation(result.is_ok())?; - drop(os_lock); - result - } - - fn append_turn_with_partial( - &self, - turn: TranscriptTurn<'_>, - partial: Option<&TranscriptPartial>, - ) -> anyhow::Result<()> { - let _serial = self.write_serial.lock().unwrap_or_else(|p| p.into_inner()); - let os_lock = self.acquire_write_lock()?; - let lock = path_lock(&self.path); - let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); - let result = self.append_turn_with_partial_locked(turn, partial); - self.finish_generation_reservation(result.is_ok())?; - drop(os_lock); - result - } - - fn messages(&self) -> anyhow::Result> { - self.persisted() - } - - fn append(&self, message: TranscriptMessage) -> anyhow::Result<()> { - let _serial = self.write_serial.lock().unwrap_or_else(|p| p.into_inner()); - let os_lock = self.acquire_write_lock()?; - let lock = path_lock(&self.path); - let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); - let mut next = self.persisted()?; - next.push(message); - let result = self.write_logical_set_locked(&next); - self.finish_generation_reservation(result.is_ok())?; - drop(os_lock); - result - } - - fn replace(&self, messages: &[TranscriptMessage]) -> anyhow::Result<()> { - let _serial = self.write_serial.lock().unwrap_or_else(|p| p.into_inner()); - let os_lock = self.acquire_write_lock()?; - let lock = path_lock(&self.path); - let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); - let result = self.write_logical_set_locked(messages); - self.finish_generation_reservation(result.is_ok())?; - drop(os_lock); - result - } - - fn clear(&self) -> anyhow::Result<()> { - let _serial = self.write_serial.lock().unwrap_or_else(|p| p.into_inner()); - let os_lock = self.acquire_write_lock()?; - let lock = path_lock(&self.path); - let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); - if !self.path.exists() { - self.finish_generation_reservation(true)?; - drop(os_lock); - return Ok(()); - } - let result = self.write_logical_set_locked(&[]); - self.finish_generation_reservation(result.is_ok())?; - drop(os_lock); - result - } -} - -/// The newest generation of the transcript at `path`: the sibling -/// `{base}.g{n}.jsonl` with the largest `n`, or `path` itself when no successor -/// exists. `path` may name any generation of the chain. -fn head_generation_path(path: &Path) -> PathBuf { - let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) else { - return path.to_path_buf(); - }; - let base = match stem.rsplit_once(".g") { - Some((base, generation)) if generation.parse::().is_ok() => base, - _ => stem, - }; - let Some(parent) = path.parent() else { - return path.to_path_buf(); - }; - std::fs::read_dir(parent) - .into_iter() - .flatten() - .filter_map(Result::ok) - .filter_map(|entry| { - let candidate = entry.path(); - if candidate.extension().and_then(|ext| ext.to_str()) != Some("jsonl") { - return None; - } - let name = candidate.file_stem()?.to_str()?; - let generation = if name == base { - 0 - } else { - name.strip_prefix(base)? - .strip_prefix(".g")? - .parse::() - .ok()? - }; - Some((generation, candidate)) - }) - .max_by_key(|(generation, _)| *generation) - .map_or_else(|| path.to_path_buf(), |(_, head)| head) -} - -#[cfg(test)] -#[path = "history_tests.rs"] -mod tests; diff --git a/crates/tinyagents-session/src/transcript/history_tests.rs b/crates/tinyagents-session/src/transcript/history_tests.rs deleted file mode 100644 index 6698a46ec..000000000 --- a/crates/tinyagents-session/src/transcript/history_tests.rs +++ /dev/null @@ -1,88 +0,0 @@ -use super::*; - -#[test] -fn parent_components_never_pop_the_filesystem_root() { - assert_eq!( - absolute_normalized_path(Path::new("/a/../..")), - PathBuf::from("/") - ); -} - -#[test] -fn parent_components_collapse_normal_segments() { - assert_eq!( - absolute_normalized_path(Path::new("/a/b/../c/./d")), - PathBuf::from("/a/c/d") - ); -} - -#[test] -fn completed_writer_releases_successor_lock_even_when_descriptor_is_inherited() { - let workspace = tempfile::tempdir().unwrap(); - let history = FileTranscriptHistory::new( - workspace.path(), - "conversation", - seed_meta_for_discovered("agent"), - ) - .unwrap(); - let locks = history.acquire_write_lock().unwrap().unwrap(); - // File::try_clone shares the open file description, as a forked child does. - // The child must not extend a completed parent's advisory lock lifetime. - let inherited_successor = locks.0.file.try_clone().unwrap(); - drop(locks); - history - .append(TranscriptMessage::new("user", "after completed writer")) - .unwrap(); - assert_eq!(history.messages().unwrap().len(), 1); - drop(inherited_successor); -} - -#[test] -fn failed_lock_attempt_does_not_release_the_existing_owner() { - let workspace = tempfile::tempdir().unwrap(); - let path = workspace.path().join("session_raw/conversation.jsonl"); - let mut owner = generation_lock(&path).unwrap(); - owner.lock_exclusive().unwrap(); - let mut failed = generation_lock(&path).unwrap(); - assert!(failed.try_lock_exclusive().is_err()); - drop(failed); - let mut next = generation_lock(&path).unwrap(); - assert!(next.try_lock_exclusive().is_err()); - drop(owner); - next.try_lock_exclusive().unwrap(); -} - -#[test] -fn committed_generation_releases_reservation_even_when_descriptor_is_inherited() { - let workspace = tempfile::tempdir().unwrap(); - let history = FileTranscriptHistory::new( - workspace.path(), - "conversation.g1", - seed_meta_for_discovered("agent"), - ) - .unwrap(); - let mut held = generation_lock(&history.path).unwrap(); - held.lock_exclusive().unwrap(); - let inherited = held.file.try_clone().unwrap(); - *history.generation_reservation.lock().unwrap() = - Some(GenerationReservation { _successor: held }); - history - .append(TranscriptMessage::new("user", "first generation write")) - .unwrap(); - let mut observer = generation_lock(&history.path).unwrap(); - observer.try_lock_exclusive().unwrap(); - drop(observer); - let peer = FileTranscriptHistory::new( - workspace.path(), - "conversation.g1", - seed_meta_for_discovered("agent"), - ) - .unwrap(); - peer.append(TranscriptMessage::new( - "assistant", - "peer after committed generation", - )) - .unwrap(); - assert_eq!(peer.messages().unwrap().len(), 2); - drop(inherited); -} diff --git a/crates/tinyagents-session/src/transcript/locator_parity_tests.rs b/crates/tinyagents-session/src/transcript/locator_parity_tests.rs deleted file mode 100644 index 3358d5c67..000000000 --- a/crates/tinyagents-session/src/transcript/locator_parity_tests.rs +++ /dev/null @@ -1,403 +0,0 @@ -//! The locator-generic reads and writes (`read_display`, -//! `thread_transcripts`, `view::get_page_in`, `spend::thread_spend_in`, -//! `append_background_message_in`) give the same answer over the JSONL files -//! and over a driver store, for one scenario written through the port alone. - -use std::sync::Arc; - -use serde_json::Value; -use tinystoragedrivers_core::MemoryStorage; - -use crate::port::{DriverSessionStores, SessionStoreProvider}; -use crate::transcript::spend::{ThreadSpend, thread_spend, thread_spend_in}; -use crate::transcript::view::{get_page, get_page_in}; -use crate::transcript::{ - BackgroundAppend, BackgroundAppendOutcome, DisplayRecord, FileTranscriptLocator, MessageUsage, - SessionRef, TranscriptLocator, TranscriptMessage, TranscriptMeta, TranscriptPartial, - TranscriptTurn, TurnUsage, append_background_message_in, session_stem, -}; - -const THREAD: &str = "thread-1"; -const AGENT: &str = "orchestrator"; -const CHILD_SUFFIX: &str = "1767225600_researcher"; - -fn meta(agent: &str, session: Option<&SessionRef>) -> TranscriptMeta { - TranscriptMeta { - session_id: session.map(SessionRef::session_id), - parent_session_id: session.and_then(SessionRef::parent_session_id), - agent_name: agent.into(), - agent_id: Some(agent.into()), - agent_type: Some( - if session.is_some() { - "root" - } else { - "subagent" - } - .into(), - ), - dispatcher: "native".into(), - provider: None, - model: Some("model-a".into()), - created: "2026-01-01T00:00:00Z".into(), - updated: "2026-01-01T00:00:00Z".into(), - turn_count: 1, - prefix_message_count: None, - input_tokens: 0, - output_tokens: 0, - cached_input_tokens: 0, - charged_amount_usd: 0.0, - thread_id: Some(THREAD.into()), - task_id: None, - } -} - -fn usage(input: u64, output: u64, cost: f64) -> TurnUsage { - TurnUsage { - provider: "provider".into(), - model: "model-a".into(), - usage: MessageUsage { - input, - output, - cost_usd: cost, - ..MessageUsage::default() - }, - ts: "2026-01-01T00:00:01Z".into(), - reasoning_content: None, - tool_calls: Vec::new(), - iteration: 1, - } -} - -fn root() -> SessionRef { - SessionRef::scoped(THREAD, AGENT) -} - -fn child_stem() -> String { - format!("{}__{CHILD_SUFFIX}", session_stem(&root())) -} - -/// Two turns (the second interrupted after a partial), a compaction into a -/// second generation with one more turn, and one sub-agent run. -fn write_scenario(locator: &dyn TranscriptLocator) { - let session = root(); - let seed = meta(AGENT, Some(&session)); - let handle = locator.open_session(&session, seed.clone()).unwrap(); - let first = vec![ - TranscriptMessage::system("system prompt"), - TranscriptMessage::user("question one"), - TranscriptMessage::assistant("answer one"), - ]; - handle - .append_turn(TranscriptTurn { - prev: &[], - next: &first, - meta: &seed, - turn_usage: Some(&usage(10, 5, 0.01)), - request_id: Some("r1"), - tools: None, - }) - .unwrap(); - let mut second = first.clone(); - second.push(TranscriptMessage::user("question two")); - second.push(TranscriptMessage::assistant("answer two")); - handle - .append_turn_with_partial( - TranscriptTurn { - prev: &handle.messages().unwrap(), - next: &second, - meta: &seed, - turn_usage: Some(&usage(20, 7, 0.02)), - request_id: Some("r2"), - tools: None, - }, - Some(&TranscriptPartial::new("half an answer")), - ) - .unwrap(); - - let (successor, next) = locator.begin_generation(&session, seed.clone()).unwrap(); - let successor_meta = meta(AGENT, Some(&successor)); - let compacted = vec![ - TranscriptMessage::system("system prompt"), - TranscriptMessage::assistant("summary of turns one and two"), - TranscriptMessage::user("question three"), - TranscriptMessage::assistant("answer three"), - ]; - next.append_turn(TranscriptTurn { - prev: &[], - next: &compacted, - meta: &successor_meta, - turn_usage: Some(&usage(30, 9, 0.03)), - request_id: Some("r3"), - tools: None, - }) - .unwrap(); - - let child_meta = meta("researcher", None); - let child = locator - .open_stem(&child_stem(), child_meta.clone()) - .unwrap(); - child - .append_turn(TranscriptTurn { - prev: &[], - next: &[ - TranscriptMessage::user("research this"), - TranscriptMessage::assistant("researched"), - ], - meta: &child_meta, - turn_usage: Some(&usage(3, 2, 0.005)), - request_id: None, - tools: None, - }) - .unwrap(); -} - -fn driver() -> Arc { - DriverSessionStores::new(Arc::new(MemoryStorage::new())) - .unwrap() - .for_agent(AGENT) - .transcripts -} - -/// `value` without wall-clock fields: the file writer stamps compaction -/// markers and partials with the time of the write, a driver store keeps none. -fn without_clock(mut value: Value) -> Value { - fn strip(value: &mut Value) { - match value { - Value::Object(map) => { - map.remove("ts"); - map.values_mut().for_each(strip); - } - Value::Array(items) => items.iter_mut().for_each(strip), - _ => {} - } - } - strip(&mut value); - value -} - -fn spend_summary(spend: &ThreadSpend) -> Value { - serde_json::json!({ - "found": spend.found_transcript, - "root": [spend.root.input_tokens, spend.root.output_tokens, spend.root.turns], - "root_cost": format!("{:.4}", spend.root.cost_usd), - "subagents": spend.subagents.iter().map(|(agent, (spend, runs))| { - serde_json::json!([agent, spend.input_tokens, spend.output_tokens, spend.turns, runs]) - }).collect::>(), - }) -} - -#[test] -fn the_thread_listing_matches_across_stores() { - let dir = tempfile::tempdir().unwrap(); - let files = FileTranscriptLocator::new(dir.path()); - let store = driver(); - write_scenario(&files); - write_scenario(store.as_ref()); - - let stems = |locator: &dyn TranscriptLocator| { - let listed = locator - .thread_transcripts(THREAD, None) - .expect("thread has roots"); - ( - listed - .roots - .iter() - .map(|t| t.stem.clone()) - .collect::>(), - listed - .subagents - .iter() - .map(|t| t.stem.clone()) - .collect::>(), - ) - }; - let (roots, subagents) = stems(&files); - assert_eq!( - roots, - vec![ - session_stem(&root()), - session_stem(&root().next_generation()) - ] - ); - assert_eq!(subagents, vec![child_stem()]); - assert_eq!(stems(store.as_ref()), (roots, subagents)); - assert!(files.thread_transcripts("no-such-thread", None).is_none()); - assert!(store.thread_transcripts("no-such-thread", None).is_none()); -} - -#[test] -fn the_display_read_matches_across_stores() { - let dir = tempfile::tempdir().unwrap(); - let files = FileTranscriptLocator::new(dir.path()); - let store = driver(); - write_scenario(&files); - write_scenario(store.as_ref()); - - let shape = |locator: &dyn TranscriptLocator| -> Vec { - let head = locator.read_session_transcript(&root()).unwrap(); - head.read_display() - .unwrap() - .unwrap() - .records - .iter() - .map(|record| match record { - DisplayRecord::Message(message) => format!( - "{}:{}:{}:{:?}", - message.message.role, - message.message.content, - message.interrupted, - message.request_id - ), - DisplayRecord::Compaction(marker) => { - format!("compaction:{}", marker.replacement.len()) - } - }) - .collect() - }; - let file_shape = shape(&files); - assert!( - file_shape - .iter() - .any(|row| row == "assistant:half an answer:true:Some(\"r2\")"), - "{file_shape:?}" - ); - assert_eq!(shape(store.as_ref()), file_shape); -} - -#[test] -fn the_projected_page_matches_across_stores_and_the_path_view() { - let dir = tempfile::tempdir().unwrap(); - let files = FileTranscriptLocator::new(dir.path()); - let store = driver(); - write_scenario(&files); - write_scenario(store.as_ref()); - - let page = |page: crate::transcript::view::TranscriptPage| { - without_clock(serde_json::to_value(page).unwrap()) - }; - let by_path = page(get_page(dir.path(), THREAD, None, Some(200))); - let file_page = page(get_page_in(&files, THREAD, None, None, Some(200))); - assert_eq!(file_page, by_path, "the locator view is the path view"); - assert_eq!(file_page["hasTranscript"], Value::Bool(true)); - let text = file_page.to_string(); - assert!(text.contains("answer three") && text.contains("researched")); - assert_eq!( - page(get_page_in(store.as_ref(), THREAD, None, None, Some(200))), - file_page - ); - - let empty = get_page_in(store.as_ref(), "no-such-thread", None, None, None); - assert!(!empty.has_transcript && empty.items.is_empty()); -} - -#[test] -fn spend_matches_across_stores_and_the_path_walk() { - let dir = tempfile::tempdir().unwrap(); - let files = FileTranscriptLocator::new(dir.path()); - let store = driver(); - write_scenario(&files); - write_scenario(store.as_ref()); - - let by_path = spend_summary(&thread_spend(dir.path(), THREAD)); - let file_spend = spend_summary(&thread_spend_in(&files, THREAD)); - assert_eq!(file_spend, by_path); - assert_eq!( - file_spend["subagents"], - serde_json::json!([["researcher", 3, 2, 1, 1]]), - "the sub-agent run is counted once, under its archetype" - ); - assert_eq!( - spend_summary(&thread_spend_in(store.as_ref(), THREAD)), - file_spend - ); - assert!(!thread_spend_in(store.as_ref(), "no-such-thread").found_transcript); -} - -#[test] -fn a_subagent_written_through_a_driver_keeps_its_usage() { - let store = driver(); - write_scenario(store.as_ref()); - let child = store - .thread_transcripts(THREAD, None) - .unwrap() - .subagents - .remove(0); - let session = child.transcript.read_session().unwrap().unwrap(); - let last = session.messages.last().unwrap(); - assert_eq!(last.role, "assistant"); - let recorded = last - .turn_usage - .as_ref() - .expect("usage on the last assistant row"); - assert_eq!((recorded.usage.input, recorded.usage.output), (3, 2)); -} - -async fn background_outcomes(locator: &dyn TranscriptLocator) -> Vec { - let message = || TranscriptMessage::assistant("reminder delivered"); - let options = - |key: &str| BackgroundAppend::new(key, serde_json::json!({"kind": "cron", "run_id": key})); - let mut outcomes = Vec::new(); - outcomes.push( - append_background_message_in(locator, &root(), message(), options("run-1")) - .await - .unwrap(), - ); - write_scenario(locator); - for key in ["run-1", "run-1"] { - outcomes.push( - append_background_message_in(locator, &root(), message(), options(key)) - .await - .unwrap(), - ); - } - let mut pinned = options("run-2"); - pinned.expected_generation = Some(0); - outcomes.push( - append_background_message_in(locator, &root(), message(), pinned) - .await - .unwrap(), - ); - outcomes -} - -#[tokio::test(flavor = "multi_thread")] -async fn background_append_matches_across_stores() { - let dir = tempfile::tempdir().unwrap(); - let files = FileTranscriptLocator::new(dir.path()); - let store = driver(); - let expected = vec![ - BackgroundAppendOutcome::NoSession, - BackgroundAppendOutcome::Appended { generation: 1 }, - BackgroundAppendOutcome::Duplicate { generation: 1 }, - BackgroundAppendOutcome::StaleGeneration { - expected: 0, - head: 1, - }, - ]; - assert_eq!(background_outcomes(&files).await, expected); - assert_eq!(background_outcomes(store.as_ref()).await, expected); - - for locator in [&files as &dyn TranscriptLocator, store.as_ref()] { - let head = locator - .read_session_transcript(&root().next_generation()) - .unwrap(); - let display = head.read_display().unwrap().unwrap(); - let Some(DisplayRecord::Message(last)) = display.records.last() else { - panic!("the delivery is the last record"); - }; - assert_eq!(last.message.content, "reminder delivered"); - let origin = last.background.as_ref().expect("provenance recorded"); - assert_eq!(origin.idempotency_key, "run-1"); - assert_eq!(origin.provenance["kind"], "cron"); - assert!( - last.message.extra_metadata.is_none(), - "the provenance is lifted out of the row's metadata" - ); - // The model context replays the delivery as an ordinary row. - let session = head.read_session().unwrap().unwrap(); - assert_eq!( - session.messages.last().unwrap().content, - "reminder delivered" - ); - } -} diff --git a/crates/tinyagents-session/src/transcript/migration.rs b/crates/tinyagents-session/src/transcript/migration.rs deleted file mode 100644 index 91f9ec75e..000000000 --- a/crates/tinyagents-session/src/transcript/migration.rs +++ /dev/null @@ -1,339 +0,0 @@ -//! One-shot migration of legacy date-grouped transcript layouts. -//! -//! Older hosts wrote transcripts beneath `session_raw/DDMMYYYY/` and markdown -//! companions beneath `sessions/DDMMYYYY/`. [`migrate_layout_if_needed`] -//! moves those artifacts into the flat transcript layout without overwriting a -//! file already present at the destination. Its workspace-local marker makes -//! completed migrations idempotent. - -use anyhow::{Context, Result}; -use std::fs; -use std::path::Path; - -/// Marker file that signals the v1 session-layout migration has run for a -/// workspace. It lives under `state/migrations/` to keep the workspace root -/// tidy. -const MIGRATION_MARKER: &str = "state/migrations/session_layout_v1.done"; - -/// Counts and non-fatal diagnostics from a session-layout migration. -#[derive(Debug, Default, Clone)] -pub struct TranscriptLayoutMigration { - /// Legacy JSONL transcripts moved into flat `session_raw/`. - pub jsonl_moved: usize, - /// JSONL files retained in the legacy directory because the destination existed. - pub jsonl_skipped: usize, - /// Markdown files moved into ISO-date directories. - pub md_moved: usize, - /// Markdown files retained in the legacy directory because the destination existed. - pub md_skipped: usize, - /// Empty legacy date directories removed after moving their contents. - pub legacy_dirs_pruned: usize, - /// Whether the migration marker was present before this invocation. - pub already_done: bool, - /// Non-fatal filesystem diagnostics encountered while migrating. - pub warnings: Vec, -} - -/// Migrate a workspace's legacy transcript layout if it has not run before. -/// -/// Detects `session_raw/{DDMMYYYY}/...jsonl` and `sessions/{DDMMYYYY}/...md`, -/// moves JSONL files into flat `session_raw/`, and moves markdown files into -/// `sessions/{YYYY_MM_DD}/`. Existing destination files are never -/// overwritten: they are retained in the legacy location and reported in -/// [`TranscriptLayoutMigration::warnings`]. A successful run writes a marker, including -/// when there were no legacy artifacts, so later starts perform no scan. -/// -/// Individual filesystem failures are collected as warnings rather than -/// returned, allowing a host to continue startup and use legacy read fallback. -pub fn migrate_layout_if_needed(workspace_dir: &Path) -> Result { - let marker_path = workspace_dir.join(MIGRATION_MARKER); - if marker_path.exists() { - tracing::debug!( - "[session-migration] marker present at {} — skipping", - marker_path.display() - ); - return Ok(TranscriptLayoutMigration { - already_done: true, - ..Default::default() - }); - } - - let mut outcome = TranscriptLayoutMigration::default(); - - let raw_root = workspace_dir.join("session_raw"); - if raw_root.is_dir() { - migrate_raw_jsonl(&raw_root, &mut outcome)?; - } - - let sessions_root = workspace_dir.join("sessions"); - if sessions_root.is_dir() { - migrate_md_directories(&sessions_root, &mut outcome)?; - } - - write_marker(&marker_path, &outcome).context("write session-migration marker")?; - - tracing::info!( - "[session-migration] complete: jsonl moved={} skipped={}, md moved={} skipped={}, legacy dirs pruned={}, warnings={}", - outcome.jsonl_moved, - outcome.jsonl_skipped, - outcome.md_moved, - outcome.md_skipped, - outcome.legacy_dirs_pruned, - outcome.warnings.len(), - ); - - Ok(outcome) -} - -fn migrate_raw_jsonl(raw_root: &Path, outcome: &mut TranscriptLayoutMigration) -> Result<()> { - let entries = match fs::read_dir(raw_root) { - Ok(it) => it, - Err(err) => { - outcome - .warnings - .push(format!("read_dir({}) failed: {err}", raw_root.display())); - return Ok(()); - } - }; - - for entry in entries.flatten() { - let path = entry.path(); - if !path.is_dir() { - continue; - } - let Some(name) = path.file_name().and_then(|s| s.to_str()) else { - continue; - }; - if !is_ddmmyyyy(name) { - continue; - } - move_jsonl_files_up(&path, raw_root, outcome); - prune_if_empty(&path, outcome); - } - - Ok(()) -} - -fn move_jsonl_files_up( - legacy_dir: &Path, - flat_dir: &Path, - outcome: &mut TranscriptLayoutMigration, -) { - let entries = match fs::read_dir(legacy_dir) { - Ok(it) => it, - Err(err) => { - outcome - .warnings - .push(format!("read_dir({}) failed: {err}", legacy_dir.display())); - return; - } - }; - - for entry in entries.flatten() { - let path = entry.path(); - if !path.is_file() || path.extension().and_then(|s| s.to_str()) != Some("jsonl") { - continue; - } - let Some(file_name) = path.file_name() else { - continue; - }; - let dest = flat_dir.join(file_name); - match hard_link_then_remove(&path, &dest) { - Ok(()) => { - outcome.jsonl_moved += 1; - tracing::debug!( - "[session-migration] moved {} → {}", - path.display(), - dest.display() - ); - } - Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { - outcome.jsonl_skipped += 1; - outcome.warnings.push(format!( - "skip move: destination already exists at {} (legacy file kept at {})", - dest.display(), - path.display() - )); - } - Err(err) => outcome.warnings.push(format!( - "atomic move({} → {}) failed: {err}", - path.display(), - dest.display() - )), - } - } -} - -fn migrate_md_directories( - sessions_root: &Path, - outcome: &mut TranscriptLayoutMigration, -) -> Result<()> { - let entries = match fs::read_dir(sessions_root) { - Ok(it) => it, - Err(err) => { - outcome.warnings.push(format!( - "read_dir({}) failed: {err}", - sessions_root.display() - )); - return Ok(()); - } - }; - - for entry in entries.flatten() { - let path = entry.path(); - if !path.is_dir() { - continue; - } - let Some(name) = path.file_name().and_then(|s| s.to_str()) else { - continue; - }; - let Some(iso) = ddmmyyyy_to_yyyy_mm_dd(name) else { - continue; - }; - let dest = sessions_root.join(&iso); - match reserve_md_destination(&dest) { - Ok(created) => { - let moved_before = outcome.md_moved; - if created { - tracing::debug!( - "[session-migration] reserved markdown destination {}", - dest.display() - ); - } - merge_md_dirs(&path, &dest, outcome); - prune_if_empty(&path, outcome); - if created && !path.exists() { - // The legacy implementation moved this entire directory in - // one rename and recorded one move regardless of its file - // count. Keep that marker/report contract stable for a - // complete fresh-directory migration; merging into an - // existing destination remains per-file. - outcome.md_moved = moved_before + 1; - } - } - Err(err) => outcome.warnings.push(format!( - "reserve markdown destination {} failed: {err}", - dest.display() - )), - } - } - - Ok(()) -} - -/// Reserve a markdown destination directory without replacing a concurrent -/// creator's directory. `create_dir` is the atomic no-replace operation: an -/// `AlreadyExists` result means another process (or an earlier migration) owns -/// the directory and its files must be merged rather than replaced. -fn reserve_md_destination(destination: &Path) -> std::io::Result { - if let Some(parent) = destination.parent() { - fs::create_dir_all(parent)?; - } - match fs::create_dir(destination) { - Ok(()) => Ok(true), - Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => Ok(false), - Err(err) => Err(err), - } -} - -fn merge_md_dirs(legacy: &Path, dest: &Path, outcome: &mut TranscriptLayoutMigration) { - let entries = match fs::read_dir(legacy) { - Ok(it) => it, - Err(err) => { - outcome - .warnings - .push(format!("read_dir({}) failed: {err}", legacy.display())); - return; - } - }; - for entry in entries.flatten() { - let src = entry.path(); - if !src.is_file() { - continue; - } - let Some(file_name) = src.file_name() else { - continue; - }; - let target = dest.join(file_name); - match hard_link_then_remove(&src, &target) { - Ok(()) => outcome.md_moved += 1, - Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { - outcome.md_skipped += 1; - outcome.warnings.push(format!( - "skip md merge: {} already exists (legacy at {} kept)", - target.display(), - src.display() - )); - } - Err(err) => outcome.warnings.push(format!( - "atomic md move({} → {}) failed: {err}", - src.display(), - target.display() - )), - } - } -} - -/// Atomically create `destination` as a second link to `source`, then remove -/// `source`. `hard_link` refuses an existing destination, so a competing -/// writer can never be replaced between discovery and transfer. -fn hard_link_then_remove(source: &Path, destination: &Path) -> std::io::Result<()> { - fs::hard_link(source, destination)?; - fs::remove_file(source) -} - -fn prune_if_empty(dir: &Path, outcome: &mut TranscriptLayoutMigration) { - match fs::read_dir(dir) { - Ok(mut it) => { - if it.next().is_some() { - return; - } - } - Err(_) => return, - } - if fs::remove_dir(dir).is_ok() { - outcome.legacy_dirs_pruned += 1; - } -} - -fn write_marker(marker_path: &Path, outcome: &TranscriptLayoutMigration) -> Result<()> { - if let Some(parent) = marker_path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("create marker dir {}", parent.display()))?; - } - let body = format!( - "openhuman session_layout migration v1\nrun_at: {}\njsonl_moved: {}\nmd_moved: {}\nlegacy_dirs_pruned: {}\nwarnings: {}\n", - chrono::Utc::now().to_rfc3339(), - outcome.jsonl_moved, - outcome.md_moved, - outcome.legacy_dirs_pruned, - outcome.warnings.len(), - ); - fs::write(marker_path, body) - .with_context(|| format!("write marker {}", marker_path.display()))?; - Ok(()) -} - -fn is_ddmmyyyy(name: &str) -> bool { - name.len() == 8 && name.chars().all(|c| c.is_ascii_digit()) -} - -fn ddmmyyyy_to_yyyy_mm_dd(name: &str) -> Option { - if !is_ddmmyyyy(name) { - return None; - } - let dd = &name[0..2]; - let mm = &name[2..4]; - let yyyy = &name[4..8]; - Some(format!("{yyyy}_{mm}_{dd}")) -} - -/// Return the migration marker path for `workspace_dir`. -#[cfg(test)] -fn marker_path_for(workspace_dir: &Path) -> std::path::PathBuf { - workspace_dir.join(MIGRATION_MARKER) -} - -#[cfg(test)] -#[path = "migration_tests.rs"] -mod tests; diff --git a/crates/tinyagents-session/src/transcript/migration_tests.rs b/crates/tinyagents-session/src/transcript/migration_tests.rs deleted file mode 100644 index 9dd03f55d..000000000 --- a/crates/tinyagents-session/src/transcript/migration_tests.rs +++ /dev/null @@ -1,213 +0,0 @@ -use super::*; -use std::fs; -use tempfile::TempDir; - -fn write_file(path: &std::path::Path, body: &str) { - fs::create_dir_all(path.parent().unwrap()).unwrap(); - fs::write(path, body).unwrap(); -} - -#[test] -fn fresh_workspace_writes_marker_with_no_moves() { - let dir = TempDir::new().unwrap(); - let outcome = migrate_layout_if_needed(dir.path()).unwrap(); - assert!(!outcome.already_done); - assert_eq!(outcome.jsonl_moved, 0); - assert_eq!(outcome.md_moved, 0); - assert!(marker_path_for(dir.path()).exists()); -} - -#[test] -fn second_run_is_a_noop() { - let dir = TempDir::new().unwrap(); - let _first = migrate_layout_if_needed(dir.path()).unwrap(); - let second = migrate_layout_if_needed(dir.path()).unwrap(); - assert!(second.already_done); - assert_eq!(second.jsonl_moved, 0); - assert_eq!(second.warnings.len(), 0); -} - -#[test] -fn moves_legacy_jsonl_files_up_to_flat_session_raw() { - let dir = TempDir::new().unwrap(); - let ws = dir.path(); - let legacy_a = ws.join("session_raw").join("01052026"); - let legacy_b = ws.join("session_raw").join("02052026"); - write_file(&legacy_a.join("1714000000_main.jsonl"), "a"); - write_file(&legacy_a.join("1714000001_welcome.jsonl"), "b"); - write_file(&legacy_b.join("1714999999_orchestrator.jsonl"), "c"); - - let outcome = migrate_layout_if_needed(ws).unwrap(); - assert_eq!(outcome.jsonl_moved, 3); - assert_eq!(outcome.legacy_dirs_pruned, 2); - - let raw_root = ws.join("session_raw"); - assert!(raw_root.join("1714000000_main.jsonl").exists()); - assert!(raw_root.join("1714000001_welcome.jsonl").exists()); - assert!(raw_root.join("1714999999_orchestrator.jsonl").exists()); - assert!(!legacy_a.exists(), "legacy date dir should be removed"); - assert!(!legacy_b.exists(), "legacy date dir should be removed"); -} - -#[test] -fn jsonl_destination_collision_is_skipped_with_warning() { - let dir = TempDir::new().unwrap(); - let ws = dir.path(); - let raw_root = ws.join("session_raw"); - write_file(&raw_root.join("1714000000_main.jsonl"), "new"); - write_file( - &raw_root.join("01052026").join("1714000000_main.jsonl"), - "old", - ); - - let outcome = migrate_layout_if_needed(ws).unwrap(); - assert_eq!(outcome.jsonl_moved, 0); - assert_eq!(outcome.jsonl_skipped, 1); - assert!( - outcome - .warnings - .iter() - .any(|w| w.contains("already exists")) - ); - assert_eq!( - fs::read_to_string(raw_root.join("1714000000_main.jsonl")).unwrap(), - "new" - ); - assert_eq!( - fs::read_to_string(raw_root.join("01052026").join("1714000000_main.jsonl")).unwrap(), - "old" - ); -} - -#[test] -fn atomic_transfer_preserves_destination_created_before_transfer() { - let dir = TempDir::new().unwrap(); - let source = dir.path().join("legacy.jsonl"); - let destination = dir.path().join("canonical.jsonl"); - write_file(&source, "legacy contents"); - - // This models another process creating the canonical file after migration - // has discovered `source`, but before its atomic transfer operation. - write_file(&destination, "concurrent contents"); - let error = hard_link_then_remove(&source, &destination).unwrap_err(); - - assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); - assert_eq!( - fs::read_to_string(&destination).unwrap(), - "concurrent contents" - ); - assert_eq!(fs::read_to_string(&source).unwrap(), "legacy contents"); -} - -#[test] -fn renames_md_ddmmyyyy_dirs_to_iso() { - let dir = TempDir::new().unwrap(); - let ws = dir.path(); - let legacy_md = ws.join("sessions").join("01052026"); - write_file(&legacy_md.join("main_0.md"), "x"); - write_file(&legacy_md.join("main_1.md"), "y"); - - let outcome = migrate_layout_if_needed(ws).unwrap(); - assert_eq!( - outcome.md_moved, 1, - "a freshly reserved date directory preserves the legacy one-directory count" - ); - let iso = ws.join("sessions").join("2026_05_01"); - assert!(iso.is_dir()); - assert!(iso.join("main_0.md").exists()); - assert!(iso.join("main_1.md").exists()); - assert!( - !legacy_md.exists(), - "DDMMYYYY md dir should be gone after rename" - ); -} - -#[test] -fn merges_md_when_iso_dir_already_exists() { - let dir = TempDir::new().unwrap(); - let ws = dir.path(); - let legacy = ws.join("sessions").join("01052026"); - let iso = ws.join("sessions").join("2026_05_01"); - write_file(&legacy.join("main_0.md"), "legacy"); - write_file(&legacy.join("main_1.md"), "legacy"); - write_file(&iso.join("main_1.md"), "newer"); - - let outcome = migrate_layout_if_needed(ws).unwrap(); - assert_eq!(outcome.md_moved, 1); - assert_eq!(outcome.md_skipped, 1); - assert_eq!(fs::read_to_string(iso.join("main_0.md")).unwrap(), "legacy"); - assert_eq!(fs::read_to_string(iso.join("main_1.md")).unwrap(), "newer"); -} - -#[test] -fn markdown_destination_reserved_by_a_concurrent_creator_is_merged_safely() { - let dir = TempDir::new().unwrap(); - let ws = dir.path(); - let legacy = ws.join("sessions").join("01052026"); - let destination = ws.join("sessions").join("2026_05_01"); - write_file(&legacy.join("new.md"), "legacy file"); - write_file(&legacy.join("shared.md"), "legacy copy"); - - // Model a second process reserving and populating the ISO directory after - // this migration has discovered the legacy directory but before it can - // reserve the destination leaf. - fs::create_dir_all(&destination).unwrap(); - write_file(&destination.join("shared.md"), "concurrent copy"); - - let outcome = migrate_layout_if_needed(ws).unwrap(); - assert_eq!(outcome.md_moved, 1); - assert_eq!(outcome.md_skipped, 1); - assert_eq!( - fs::read_to_string(destination.join("new.md")).unwrap(), - "legacy file" - ); - assert_eq!( - fs::read_to_string(destination.join("shared.md")).unwrap(), - "concurrent copy" - ); - assert_eq!( - fs::read_to_string(legacy.join("shared.md")).unwrap(), - "legacy copy" - ); -} - -#[test] -fn ignores_non_date_subdirectories_in_session_raw() { - let dir = TempDir::new().unwrap(); - let ws = dir.path(); - let weird = ws.join("session_raw").join("my_notes"); - write_file(&weird.join("random.jsonl"), "keep me"); - - let outcome = migrate_layout_if_needed(ws).unwrap(); - assert_eq!(outcome.jsonl_moved, 0); - assert!(weird.is_dir(), "non-date subdir must be left alone"); - assert!(weird.join("random.jsonl").exists()); -} - -#[test] -fn ddmmyyyy_to_iso_handles_boundary_dates() { - assert_eq!( - ddmmyyyy_to_yyyy_mm_dd("01012026").as_deref(), - Some("2026_01_01") - ); - assert_eq!( - ddmmyyyy_to_yyyy_mm_dd("31122099").as_deref(), - Some("2099_12_31") - ); - assert!(ddmmyyyy_to_yyyy_mm_dd("abc12345").is_none()); - assert!(ddmmyyyy_to_yyyy_mm_dd("1234567").is_none(), "7 digits"); - assert!(ddmmyyyy_to_yyyy_mm_dd("123456789").is_none(), "9 digits"); -} - -#[test] -fn marker_persists_run_metadata() { - let dir = TempDir::new().unwrap(); - let ws = dir.path(); - let legacy = ws.join("session_raw").join("01052026"); - write_file(&legacy.join("1714000000_main.jsonl"), "a"); - - migrate_layout_if_needed(ws).unwrap(); - let marker = fs::read_to_string(marker_path_for(ws)).unwrap(); - assert!(marker.contains("jsonl_moved: 1")); - assert!(marker.contains("openhuman session_layout migration v1")); -} diff --git a/crates/tinyagents-session/src/transcript/paths.rs b/crates/tinyagents-session/src/transcript/paths.rs index c7d8628fc..7c3824169 100644 --- a/crates/tinyagents-session/src/transcript/paths.rs +++ b/crates/tinyagents-session/src/transcript/paths.rs @@ -1,38 +1,8 @@ -//! Transcript path resolution: the flat `session_raw/{stem}.jsonl` layout, -//! the dated `.md` companion, legacy date-grouped fallbacks, and the -//! newest-transcript scan used for resume. +//! Transcript file naming shared by the stem codec and the legacy JSONL +//! reader/writer: stem sanitisation and the `.md` companion path. -use anyhow::{Context, Result}; -use std::fs; use std::path::{Path, PathBuf}; -/// Resolve a transcript path under `session_raw/{stem}.jsonl` — a -/// *flat* directory keyed only by stem. Used by the session-key flow: -/// the stem is `"{unix_ts}_{agent_id}"` for a root session, or -/// `"{parent_chain}__{session_key}"` for a sub-agent, so nested -/// delegations still produce a single flat filename that encodes the -/// parent → child path. -/// -/// Creates the directory if needed. Overwrites are intentional: the -/// `Agent` persists the same transcript file across every turn of a -/// session, and every sub-agent spawn gets a unique timestamp in its -/// own key so collisions are effectively impossible. -pub fn resolve_keyed_transcript_path(workspace_dir: &Path, stem: &str) -> Result { - let raw_dir = raw_session_dir(workspace_dir); - resolve_keyed_transcript_path_in_dir(&raw_dir, stem) -} - -/// Like [`resolve_keyed_transcript_path`], but takes the raw session -/// directory directly rather than deriving it from a workspace root — the -/// seam tests use to point at a tempdir without going through the full -/// `{workspace}/session_raw` layout. -pub fn resolve_keyed_transcript_path_in_dir(raw_dir: &Path, stem: &str) -> Result { - fs::create_dir_all(raw_dir) - .with_context(|| format!("create session_raw dir {}", raw_dir.display()))?; - let sanitized = sanitize_stem(stem); - Ok(raw_dir.join(format!("{sanitized}.jsonl"))) -} - /// Sanitize a user-supplied transcript stem so it never escapes the /// `session_raw/` directory. Allows ASCII alphanumerics plus a small /// punctuation set (`_`, `-`, `.`); every other byte is replaced with @@ -55,84 +25,6 @@ pub(super) fn sanitize_stem(stem: &str) -> String { } } -/// Find the most recent transcript for `agent_name`. -/// -/// **Primary**: scan the flat `session_raw/` directory and pick the -/// newest matching stem (root sessions only — sub-agents are skipped). -/// **Fallback**: scan the legacy `session_raw/DDMMYYYY/` dirs (today -/// and yesterday) and the legacy `sessions/DDMMYYYY/` markdown dirs so -/// users upgrading from the date-grouped layout don't lose resume. -/// The fallback is one-release transitional and can be removed once -/// existing transcripts have rolled forward. -pub fn find_latest_transcript(workspace_dir: &Path, agent_name: &str) -> Option { - let sanitized = sanitize_agent_name(agent_name); - let raw_root = raw_session_dir(workspace_dir); - let sessions_root = workspace_dir.join("sessions"); - - // Primary path: flat session_raw/ directory. The stem-suffix scan - // is naturally date-independent, so an idle thread resumes the same - // way today as it did weeks ago. - if raw_root.is_dir() - && let Some(path) = latest_in_dir(&raw_root, &sanitized) - { - return Some(path); - } - - // Fallback: legacy date-grouped layout (one-release migration - // window). Today first, then yesterday — matches the previous - // behaviour so we don't regress while users still have files in - // the old structure. - let today = chrono::Local::now().format("%d%m%Y").to_string(); - let yesterday = (chrono::Local::now() - chrono::Duration::days(1)) - .format("%d%m%Y") - .to_string(); - - for date_str in [&today, &yesterday] { - let raw_dir = raw_root.join(date_str); - if raw_dir.is_dir() - && let Some(path) = latest_in_dir(&raw_dir, &sanitized) - { - return Some(path); - } - let legacy_dir = sessions_root.join(date_str); - if legacy_dir.is_dir() - && let Some(path) = latest_in_dir(&legacy_dir, &sanitized) - { - return Some(path); - } - } - - None -} - -/// Find the newest transcript in the date-grouped layouts without consulting -/// the flat layout, so a canonical transcript cannot hide a legacy candidate. -pub fn find_latest_legacy_transcript(workspace_dir: &Path, agent_name: &str) -> Option { - let sanitized = sanitize_agent_name(agent_name); - let raw_root = raw_session_dir(workspace_dir); - let sessions_root = workspace_dir.join("sessions"); - let today = chrono::Local::now().format("%d%m%Y").to_string(); - let yesterday = (chrono::Local::now() - chrono::Duration::days(1)) - .format("%d%m%Y") - .to_string(); - - [&today, &yesterday] - .into_iter() - .flat_map(|date| { - [raw_root.join(date), sessions_root.join(date)] - .into_iter() - .filter_map(|dir| latest_in_dir(&dir, &sanitized)) - }) - .next() -} - -/// Flat directory for the JSONL source of truth, e.g. -/// `{workspace}/session_raw`. Stems start with `{unix_ts}` so the -/// listing is naturally time-ordered without a date subdirectory. -pub(super) fn raw_session_dir(workspace_dir: &Path) -> PathBuf { - workspace_dir.join("session_raw") -} - /// Given a `session_raw/{stem}.jsonl` path, derive the companion /// `sessions/YYYY_MM_DD/{stem}.md` path. The date is taken from the /// local clock at write time — fine for browsing because the source @@ -177,101 +69,14 @@ pub(super) fn md_companion_path(jsonl_path: &Path) -> PathBuf { out.with_extension("md") } -/// Sanitizes an agent name for use as a filename prefix: keeps -/// alphanumerics, `-`, and `_`; replaces everything else with `_`. -pub(super) fn sanitize_agent_name(name: &str) -> String { - name.chars() - .map(|c| { - if c.is_alphanumeric() || c == '-' || c == '_' { - c - } else { - '_' - } - }) - .collect() -} - -/// Find the latest transcript file for `agent_prefix` in `dir`. -/// -/// Prefers `.jsonl` files; falls back to `.md` if no `.jsonl` exists -/// (legacy sessions). When both exist for the same index the `.jsonl` -/// wins. -pub(super) fn latest_in_dir(dir: &Path, agent_prefix: &str) -> Option { - // Two transcript-naming schemes coexist on disk: - // * Legacy: `{agent}_{index}.jsonl|.md` — strictly increasing - // index, used by the now-removed `resolve_new_transcript_path`. - // * Keyed: `{unix_ts}_{agent}.jsonl` (root session) or - // `{parent_chain}__{unix_ts}_{agent}.jsonl` (sub-agent). The - // root stem starts with `{unix_ts}_{agent}` and has no `__` - // prefix segment. - // - // For resume we only care about root sessions (sub-agents rebuild - // from scratch), so we scan for filenames matching either scheme - // and pick the newest. "Newest" is the largest sort key — indices - // and unix timestamps both order naturally as integers. - let legacy_prefix = format!("{}_", agent_prefix); - let keyed_suffix = format!("_{}", agent_prefix); - let mut best_jsonl: Option<(u64, PathBuf)> = None; - let mut best_md: Option<(u64, PathBuf)> = None; - - let entries = fs::read_dir(dir).ok()?; - for entry in entries.flatten() { - let name = entry.file_name(); - let name_str = name.to_string_lossy(); - // Extract the stem minus extension. - let (stem, is_jsonl) = if let Some(s) = name_str.strip_suffix(".jsonl") { - (s, true) - } else if let Some(s) = name_str.strip_suffix(".md") { - (s, false) - } else { - continue; - }; - // Skip sub-agent transcripts — they carry at least one `__` - // separator in their stem (e.g. - // `{orch_key}__{planner_key}`). Root resume never targets a - // sub-agent's transcript directly. - if stem.contains("__") { - continue; - } - // Determine sort key. Keyed filenames end with - // `_{agent_prefix}`: everything before that is the unix - // timestamp. Legacy filenames start with `{agent_prefix}_`: - // everything after is the numeric index. - let sort_key: u64 = if let Some(ts_part) = stem.strip_suffix(&keyed_suffix) { - match ts_part.parse::() { - Ok(ts) => ts, - Err(_) => continue, - } - } else if let Some(idx_part) = stem.strip_prefix(&legacy_prefix) { - match idx_part.parse::() { - Ok(idx) => idx, - Err(_) => continue, - } - } else { - continue; - }; - let slot = if is_jsonl { - &mut best_jsonl - } else { - &mut best_md - }; - if slot.as_ref().is_none_or(|(best, _)| sort_key > *best) { - *slot = Some((sort_key, entry.path())); - } - } - - // Prefer the best .jsonl; fall back to .md if no .jsonl exists. - match (best_jsonl, best_md) { - (Some(jsonl), Some(md)) => { - // Take the one with the higher index; on a tie prefer .jsonl. - if md.0 > jsonl.0 { - Some(md.1) - } else { - Some(jsonl.1) - } - } - (Some(jsonl), None) => Some(jsonl.1), - (None, Some(md)) => Some(md.1), - (None, None) => None, - } +/// Test fixture: `{workspace}/session_raw/{stem}.jsonl`, creating the +/// directory. +#[cfg(test)] +pub(crate) fn resolve_keyed_transcript_path( + workspace_dir: &Path, + stem: &str, +) -> anyhow::Result { + let raw_dir = workspace_dir.join("session_raw"); + std::fs::create_dir_all(&raw_dir)?; + Ok(raw_dir.join(format!("{}.jsonl", sanitize_stem(stem)))) } diff --git a/crates/tinyagents-session/src/transcript/reader.rs b/crates/tinyagents-session/src/transcript/reader.rs index a53b12fa2..027fcf2bc 100644 --- a/crates/tinyagents-session/src/transcript/reader.rs +++ b/crates/tinyagents-session/src/transcript/reader.rs @@ -9,8 +9,7 @@ use super::jsonl::{ use super::legacy_md::read_transcript_legacy_md; use super::types::TranscriptMessage; use super::types::{ - CompactionMarker, DisplayRecord, DisplaySessionTranscript, MessageUsage, SessionTranscript, - TranscriptMeta, + CompactionMarker, DisplayRecord, DisplaySessionTranscript, SessionTranscript, TranscriptMeta, }; use anyhow::{Context, Result}; use std::fs; @@ -275,77 +274,3 @@ pub fn read_transcript_display(path: &Path) -> Result Ok(DisplaySessionTranscript { meta, records }) } - -/// Parse the authoritative `_meta` of a root transcript JSONL. -/// -/// Append-only files carry the immutable header on line 1 plus a refreshed -/// `_meta` line per turn (cumulative totals). The **last** `_meta` line wins, -/// so a multi-turn session reports its running totals — not just the first -/// turn's. Falls back to line 1 for legacy single-header files. -pub(super) fn read_transcript_meta_only(path: &Path) -> Option { - let lines = read_jsonl_lines(path).ok()?; - let mut latest: Option = None; - for line in &lines { - let Some(line) = line else { - latest.as_ref()?; - continue; - }; - let line = line.trim(); - if line.is_empty() { - continue; - } - if let Ok(ml) = serde_json::from_str::(line) { - latest = Some(meta_from_payload(ml.meta)); - } else if latest.is_none() { - // The first non-empty line must be a valid meta header. - return None; - } - } - latest -} - -/// Extract the last assistant message's usage + model from a transcript JSONL. -/// Only the final assistant message of a turn carries these (see the JSONL -/// format docs at the top of this module). Compaction records and refreshed -/// `_meta` lines are skipped; a `compaction` record's `replacement` assistant -/// rows are considered so a compacted transcript still surfaces its latest -/// usage. -pub(super) fn read_last_assistant_usage(path: &Path) -> Option<(MessageUsage, Option)> { - let lines = read_jsonl_lines(path).ok()?; - let mut result = None; - let mut seen_first = false; - for line in &lines { - let Some(line) = line else { - if !seen_first { - return None; - } - continue; - }; - let line = line.trim(); - if line.is_empty() { - continue; - } - if !seen_first { - seen_first = true; // first non-empty line is the `_meta` header - continue; - } - match classify_line(line) { - Ok(LineKind::Message(ml)) if ml.role == "assistant" && !ml.interrupted => { - if let Some(usage) = ml.usage { - result = Some((usage, ml.model)); - } - } - Ok(LineKind::Compaction(cl)) => { - for ml in &cl.replacement { - if ml.role == "assistant" - && let Some(usage) = ml.usage.clone() - { - result = Some((usage, ml.model.clone())); - } - } - } - _ => {} - } - } - result -} diff --git a/crates/tinyagents-session/src/transcript/session.rs b/crates/tinyagents-session/src/transcript/session.rs index 1d862ab06..8a5795f97 100644 --- a/crates/tinyagents-session/src/transcript/session.rs +++ b/crates/tinyagents-session/src/transcript/session.rs @@ -6,7 +6,7 @@ //! //! 1. Every cold boot produced a new stem, so one conversation accumulated //! several transcript files. Resume picked the newest by `_meta.created` -//! ([`super::find_root_transcript_for_thread`]), so any file that persisted +//! (`root_for_thread`, [`super::TranscriptLocator`]), so any file that persisted //! less than it was seeded with silently shortened the conversation for //! every session after it. //! 2. Two processes over one workspace each minted their own stem and neither @@ -42,8 +42,8 @@ pub(crate) const SUBAGENT_SEPARATOR: &str = "__"; /// passes its `thread_id`. `agent_id` scopes it, because one caller-supplied /// key can legitimately be handed to several independently configured agents /// and each needs its own transcript (see -/// [`super::find_root_transcript_for_thread_scoped`] for the bug that taught us -/// this). `generation` distinguishes the segments a compaction creates. +/// [`TranscriptLocator::root_for_thread_scoped`](super::TranscriptLocator::root_for_thread_scoped) +/// for the bug that taught us this). `generation` distinguishes the segments a compaction creates. #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct SessionRef { /// The host's stable name for the conversation. diff --git a/crates/tinyagents-session/src/transcript/spend.rs b/crates/tinyagents-session/src/transcript/spend.rs index 4204e6d97..002f5378c 100644 --- a/crates/tinyagents-session/src/transcript/spend.rs +++ b/crates/tinyagents-session/src/transcript/spend.rs @@ -6,12 +6,10 @@ //! host's concern and stays out of here. use std::collections::{BTreeMap, HashSet}; -use std::path::{Path, PathBuf}; use super::{ DisplayRecord, DisplaySessionTranscript, SessionTranscript, ThreadTranscript, TranscriptLocator, TranscriptMessage, TranscriptMeta, TurnUsage, UsageCostSource, - find_root_transcripts_for_thread, read_transcript_display, }; /// One transcript's own spend, summed from the per-turn `turn_usage` records @@ -159,7 +157,7 @@ pub fn context_output_tokens_of(usage: &TurnUsage) -> u64 { /// /// This sums only what `transcript.messages` still holds: after a compaction /// that is the reduced context, so it undercounts a compacted session. The -/// thread aggregate ([`thread_spend`]) reads the full append history instead. +/// thread aggregate ([`thread_spend_in`]) reads the full append history instead. pub fn transcript_spend(transcript: &SessionTranscript) -> TranscriptSpend { spend_from_usages( transcript @@ -238,24 +236,6 @@ fn spend_from_usages<'a>( /// writing the rows it retained from its parent, and those rows keep their /// original `request_id`, so a row whose `request_id` is in that set is /// carried history, not new spend. The ids this file counts are added to it. -fn file_spend( - path: &Path, - earlier_requests: &mut HashSet, -) -> Option<(TranscriptSpend, TranscriptMeta)> { - let display = match read_transcript_display(path) { - Ok(display) => display, - Err(err) => { - tracing::warn!( - "[transcript:spend] skipping unreadable transcript {}: {err}", - path.display() - ); - return None; - } - }; - Some(display_spend(display, earlier_requests)) -} - -/// [`file_spend`] of an already-read display projection. fn display_spend( display: DisplaySessionTranscript, earlier_requests: &mut HashSet, @@ -339,41 +319,6 @@ fn carried_from(prior: &TranscriptMessage, row: &TranscriptMessage) -> bool { && prior.turn_usage == row.turn_usage } -/// Every descendant transcript of `root`, at any delegation depth. -/// -/// Sub-agent transcripts are named `{root_stem}__{child}` (and a grandchild -/// chains another `__`), which is the *only* durable record of the parent→child -/// relation. Selecting them by `_meta.thread_id` instead — as the session -/// crate's own summary helper does — drops the majority of them: a delegation -/// gets its own worker thread, and `inherited_thread_id` lets that worker id win -/// over the parent's, so the child's header names a thread the caller never -/// asked about (#6460). -fn descendant_transcripts(root: &Path) -> Vec { - let Some(dir) = root.parent() else { - return Vec::new(); - }; - let Some(root_stem) = root.file_stem().and_then(|s| s.to_str()) else { - return Vec::new(); - }; - let prefix = format!("{root_stem}__"); - let Ok(entries) = std::fs::read_dir(dir) else { - return Vec::new(); - }; - let mut matches: Vec = entries - .flatten() - .map(|entry| entry.path()) - .filter(|path| { - path.extension().and_then(|s| s.to_str()) == Some("jsonl") - && path - .file_stem() - .and_then(|s| s.to_str()) - .is_some_and(|stem| stem.starts_with(&prefix)) - }) - .collect(); - matches.sort(); - matches -} - /// One thread's spend, split into the orchestrator's own and each sub-agent /// archetype's, each counted exactly once. #[derive(Debug, Clone, Default)] @@ -385,50 +330,12 @@ pub struct ThreadSpend { pub found_transcript: bool, } -/// Walk a thread's root transcripts and their descendants, totalling each file's -/// own recorded spend. -/// -/// Correct-by-construction because every transcript records only what the agent -/// that owns it spent (see the host's per-turn usage codec): the walk visits -/// each file once, so no token is counted twice however deep the delegation went, -/// and a child whose usage never reached its parent's in-turn ledger (#6459) is -/// still counted from its own file. -pub fn thread_spend(workspace_dir: &Path, thread_id: &str) -> ThreadSpend { - let mut out = ThreadSpend::default(); - let roots = find_root_transcripts_for_thread(workspace_dir, thread_id); - // Shared across the thread's root files (oldest first), so rows a later - // generation carried over from an earlier one are not counted twice. - let mut root_requests = HashSet::new(); - for root in &roots { - out.found_transcript = true; - if let Some((spend, meta)) = file_spend(root, &mut root_requests) { - out.add_root(spend, meta); - } - let children = descendant_transcripts(root) - .into_iter() - .map(|path| (stem_of(&path), path)); - for chain in subagent_chains(children) { - // One run per chain: a compacted sub-agent's `.gN` successors open - // with rows carried from the earlier generation, which keep their - // `request_id` and must not be counted again. - let mut requests = HashSet::new(); - let mut new_run = true; - for child in chain { - if let Some((spend, child_meta)) = file_spend(&child, &mut requests) { - out.add_subagent(spend, &child_meta, std::mem::take(&mut new_run)); - } - } - } - } - out -} - -/// [`thread_spend`] over any [`TranscriptLocator`]: the thread's roots and +/// A thread's spend over a [`TranscriptLocator`]: the thread's roots and /// sub-agent transcripts come from /// [`TranscriptLocator::thread_transcripts`], each counted once — the roots /// in projection order with one shared set of counted requests, every listed -/// sub-agent into its archetype. Unlike the file walk, a sub-agent the -/// listing finds by its `_meta.thread_id` (not only by stem) is counted too. +/// sub-agent into its archetype. A sub-agent the listing finds by +/// its `_meta.thread_id` (not only by stem) is counted too. pub fn thread_spend_in(locator: &dyn TranscriptLocator, thread_id: &str) -> ThreadSpend { let mut out = ThreadSpend::default(); let Some(listed) = locator.thread_transcripts(thread_id, None) else { @@ -447,7 +354,7 @@ pub fn thread_spend_in(locator: &dyn TranscriptLocator, thread_id: &str) -> Thre .iter() .map(|child| (child.stem.clone(), child)); for chain in subagent_chains(children) { - // As in the file walk: one run and one set of counted requests per + // One run and one set of counted requests per // compaction chain. let mut requests = HashSet::new(); let mut new_run = true; @@ -489,14 +396,8 @@ fn subagent_chains(items: impl IntoIterator) -> Vec String { - path.file_stem() - .map(|stem| stem.to_string_lossy().into_owned()) - .unwrap_or_default() -} - /// A listed transcript's display projection; unreadable or missing ones are -/// skipped with a warning, as the file walk skips an unreadable file. +/// skipped with a warning. fn read_listed(listed: &ThreadTranscript) -> Option { match listed.transcript.read_display() { Ok(found) => found, diff --git a/crates/tinyagents-session/src/transcript/spend_cost_tests.rs b/crates/tinyagents-session/src/transcript/spend_cost_tests.rs index b2a9452a9..92f17baa1 100644 --- a/crates/tinyagents-session/src/transcript/spend_cost_tests.rs +++ b/crates/tinyagents-session/src/transcript/spend_cost_tests.rs @@ -1,4 +1,6 @@ use super::*; +use crate::transcript::test_locator::thread_spend; +use std::path::{Path, PathBuf}; use super::tests::{meta, turn_usage}; use crate::transcript::{MessageUsage, append_transcript_turn, read_transcript}; diff --git a/crates/tinyagents-session/src/transcript/spend_tests.rs b/crates/tinyagents-session/src/transcript/spend_tests.rs index db69a5cc3..308f114f7 100644 --- a/crates/tinyagents-session/src/transcript/spend_tests.rs +++ b/crates/tinyagents-session/src/transcript/spend_tests.rs @@ -6,9 +6,11 @@ //! the core actually persists. use super::*; +use crate::transcript::test_locator::{driver_locator, thread_spend}; use crate::transcript::{ MessageUsage, TranscriptMessage, TranscriptMeta, TurnUsage, append_transcript_turn, }; +use std::path::Path; const MODEL: &str = "openrouter/deepseek/deepseek-v4-flash"; @@ -531,10 +533,9 @@ fn request_less_compaction_with_usage_identical_to_history_still_counts() { /// counted in the sealed generation. #[test] fn retained_rows_in_a_successor_generation_are_not_counted_twice() { - use crate::transcript::{FileTranscriptLocator, SessionRef, TranscriptLocator, TruncateCut}; - let tmp = tempfile::tempdir().expect("tempdir"); + use crate::transcript::{SessionRef, TruncateCut}; let thread = "thread-generations"; - let locator = FileTranscriptLocator::new(tmp.path()); + let locator = driver_locator(); let session = SessionRef::scoped(thread, "orchestrator"); let first = locator .open_session(&session, meta("orchestrator", "root", Some(thread))) @@ -548,7 +549,7 @@ fn retained_rows_in_a_successor_generation_are_not_counted_twice() { answer.preserve_request_id = true; first.append(answer).expect("append answer"); - let before = thread_spend(tmp.path(), thread); + let before = thread_spend_in(locator.as_ref(), thread); assert_eq!(before.root.input_tokens, 4_000); locator @@ -559,11 +560,15 @@ fn retained_rows_in_a_successor_generation_are_not_counted_twice() { ) .expect("next generation"); assert_eq!( - find_root_transcripts_for_thread(tmp.path(), thread).len(), + locator + .thread_transcripts(thread, None) + .expect("thread listed") + .roots + .len(), 2, "both generations are root transcripts of the thread" ); - let after = thread_spend(tmp.path(), thread); + let after = thread_spend_in(locator.as_ref(), thread); assert_eq!(after.root.input_tokens, 4_000, "retained row counted twice"); assert_eq!(after.root.turns, 1); } @@ -701,11 +706,9 @@ fn last_call_fields_round_trip_and_default_on_old_lines() { /// A compacted sub-agent leaves `{stem}` and `{stem}.g1` behind; the successor /// opens with the rows it carried over (same `request_id`). The pair is one -/// run, and the carried turn is counted once, by the file walk and the -/// locator walk alike. +/// run, and the carried turn is counted once, by the locator walk. #[test] fn a_compacted_subagent_is_one_run_counted_once() { - use crate::transcript::FileTranscriptLocator; let tmp = tempfile::tempdir().expect("tempdir"); let thread = "thread-subagent-generations"; let root_stem = "1790000002_orchestrator_generations"; @@ -730,22 +733,12 @@ fn a_compacted_subagent_is_one_run_counted_once() { &[(5_000, 50, 0), (7_000, 70, 0)], ); - for (walk, spend) in [ - ("files", thread_spend(tmp.path(), thread)), - ( - "locator", - thread_spend_in(&FileTranscriptLocator::new(tmp.path()), thread), - ), - ] { - let (researcher, runs) = spend.subagents.get("researcher").expect("child counted"); - assert_eq!(*runs, 1, "{walk}: one delegation across its generations"); - assert_eq!( - researcher.input_tokens, 12_000, - "{walk}: carried turn counted once" - ); - assert_eq!(researcher.output_tokens, 120, "{walk}"); - assert_eq!(researcher.turns, 2, "{walk}"); - } + let spend = thread_spend(tmp.path(), thread); + let (researcher, runs) = spend.subagents.get("researcher").expect("child counted"); + assert_eq!(*runs, 1, "one delegation across its generations"); + assert_eq!(researcher.input_tokens, 12_000, "carried turn counted once"); + assert_eq!(researcher.output_tokens, 120); + assert_eq!(researcher.turns, 2); } #[test] diff --git a/crates/tinyagents-session/src/transcript/test_locator.rs b/crates/tinyagents-session/src/transcript/test_locator.rs new file mode 100644 index 000000000..d5e444216 --- /dev/null +++ b/crates/tinyagents-session/src/transcript/test_locator.rs @@ -0,0 +1,37 @@ +//! Test fixtures: classic `session_raw/*.jsonl` files on disk, served through a +//! store-port locator. +//! +//! The crate no longer reads or writes a workspace directory at run time; the +//! one-time legacy import is the only reader of those files. Tests that build a +//! transcript fixture as raw JSONL therefore import it into an in-memory +//! locator and exercise the store-port APIs against that. + +use std::path::Path; +use std::sync::Arc; + +use tinystoragedrivers_core::MemoryStorage; + +use super::TranscriptLocator; +use crate::port::{DriverSessionStores, SessionStoreProvider}; + +/// An empty driver-backed locator (the production store, over memory). +pub(crate) fn driver_locator() -> Arc { + let provider = DriverSessionStores::new(Arc::new(MemoryStorage::new())) + .expect("driver session stores over memory"); + provider.for_agent("test").transcripts +} + +/// A driver-backed locator holding every transcript under +/// `/session_raw` (flat and date-grouped), imported id-preserving. +pub(crate) fn imported_locator(workspace: &Path) -> Arc { + let locator = driver_locator(); + crate::legacy::import_transcripts(workspace, locator.as_ref()) + .expect("import fixture transcripts"); + locator +} + +/// [`thread_spend_in`](super::spend::thread_spend_in) over the fixtures +/// imported from `workspace`. +pub(crate) fn thread_spend(workspace: &Path, thread_id: &str) -> super::spend::ThreadSpend { + super::spend::thread_spend_in(imported_locator(workspace).as_ref(), thread_id) +} diff --git a/crates/tinyagents-session/src/transcript/thread_lookup.rs b/crates/tinyagents-session/src/transcript/thread_lookup.rs deleted file mode 100644 index 49c582ec2..000000000 --- a/crates/tinyagents-session/src/transcript/thread_lookup.rs +++ /dev/null @@ -1,298 +0,0 @@ -//! Thread-keyed lookups over the canonical `session_raw` store: locating a thread's -//! root transcripts and summing its token / cost usage. - -use super::paths::raw_session_dir; -use super::reader::{read_last_assistant_usage, read_transcript, read_transcript_meta_only}; -use super::types::{SubagentArchetypeUsage, ThreadUsageSummary}; -use std::collections::BTreeMap; -use std::fs; -use std::path::{Path, PathBuf}; - -/// Find the newest root transcript whose metadata declares `thread_id`. -/// -/// Root transcripts live directly under `session_raw/` and do not carry -/// the `__` separator used for sub-agent siblings. This helper is the -/// bridge PR-2 can use to route UI thread reads to the canonical root -/// transcript without accidentally folding delegated worker transcripts -/// into the main chat timeline. -pub fn find_root_transcript_for_thread(workspace_dir: &Path, thread_id: &str) -> Option { - find_root_transcripts_for_thread(workspace_dir, thread_id).pop() -} - -/// [`find_root_transcript_for_thread`], additionally scoped to `agent_id` -/// when given. -/// -/// Plain thread-id matching is ambiguous when several distinct agents can be -/// given the same caller-supplied `thread_id` — agents can share one -/// `session_raw/` store, keyed only by thread id in `_meta.thread_id`. Two -/// agents handed the same id would otherwise resolve to whichever one wrote -/// the newest matching transcript, splicing one agent's history (and its -/// caller's data) into another agent's turn. `agent_id` filters on -/// `_meta.agent_id`, the agent definition id written at turn time, so the -/// resume this seeds only ever pulls from the calling agent's own history. -/// -/// `agent_id: None` retains the unscoped behaviour for callers that guarantee -/// one logical agent drives each `thread_id` (#5351). -pub fn find_root_transcript_for_thread_scoped( - workspace_dir: &Path, - thread_id: &str, - agent_id: Option<&str>, -) -> Option { - let mut matches = find_root_transcripts_for_thread(workspace_dir, thread_id); - if let Some(agent_id) = agent_id { - matches.retain(|path| { - read_transcript(path) - .ok() - .and_then(|transcript| transcript.meta.agent_id) - .as_deref() - == Some(agent_id) - }); - } - matches.pop() -} - -/// Finds every root transcript whose metadata declares `thread_id`, oldest -/// first by `meta.created`. -/// -/// Underlies [`find_root_transcript_for_thread`] (which takes the newest) and -/// [`super::history::TranscriptLocator::root_for_thread_scoped`]'s -/// agent-id filtering; exposed directly for callers that need the full -/// ordered history rather than just the latest match. -pub fn find_root_transcripts_for_thread(workspace_dir: &Path, thread_id: &str) -> Vec { - root_transcripts_for_thread_in_dir(&raw_session_dir(workspace_dir), thread_id).0 -} - -/// [`find_root_transcripts_for_thread`], additionally reporting whether the -/// scan hit any root `.jsonl` file it could not read at all — matching or -/// not, since a read failure happens *before* the thread-id comparison, so -/// which thread an unreadable file belonged to can never be determined. -/// -/// Exists for [`super::adoption::adopt_legacy_session_transcripts`]: that -/// caller's idempotency marker is the destination file it writes, so folding -/// only the *readable* matches and reporting success would permanently -/// strand an unreadable file's turns — the marker's existence stops every -/// later retry. [`find_root_transcripts_for_thread`] itself is used by -/// callers (thread resume, usage summaries) that already treat "unreadable" -/// as "absent" and are safe to keep doing so; only adoption's -/// once-and-only-once contract needs to know the difference. -pub fn find_root_transcripts_for_thread_reporting_unreadable( - workspace_dir: &Path, - thread_id: &str, -) -> (Vec, bool) { - root_transcripts_for_thread_in_dir(&raw_session_dir(workspace_dir), thread_id) -} - -fn root_transcripts_for_thread_in_dir(raw_dir: &Path, thread_id: &str) -> (Vec, bool) { - let thread_id = thread_id.trim(); - if thread_id.is_empty() { - return (Vec::new(), false); - } - - let entries = match fs::read_dir(raw_dir) { - Ok(entries) => entries, - // A workspace with no session_raw/ yet has genuinely adopted - // nothing — not an error. Any other failure (permissions, a - // transient I/O error) means the scan could not actually see - // whether a matching root exists, which callers relying on - // `unreadable` (adoption's idempotency contract) must not treat the - // same as "confirmed nothing here". - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return (Vec::new(), false); - } - Err(error) => { - tracing::warn!( - "[transcript] could not scan {} for thread {thread_id}: {error}", - raw_dir.display() - ); - return (Vec::new(), true); - } - }; - let mut any_unreadable = false; - // Keyed by `meta.created` so the order is chronological rather than - // lexicographic. Modern stems are `{unix_ts}_{agent_id}` and sort the same - // either way, but a legacy `{agent}_{index}` root encodes no time at all — - // and because digits sort before letters, every legacy root sorted *after* - // every modern one regardless of when it was written. `project_from_files` - // concatenates these in order, so that reordered the rendered view and - // could attach a sub-agent trail to the wrong turn. - // - // An explicit loop rather than a filter/filter_map chain: both the - // directory-entry read and the transcript read below can independently - // fail and need to set the same `any_unreadable` flag, and two closures - // cannot each hold a mutable borrow of it at once. - let mut matches: Vec<(String, PathBuf)> = Vec::new(); - for entry in entries { - let path = match entry { - Ok(entry) => entry.path(), - Err(error) => { - // An entry the directory iterator itself could not read - // (e.g. a race with concurrent deletion, a transient I/O - // error) is exactly as invisible to this scan as a file that - // failed `read_transcript` below — the `.flatten()` this - // loop replaced would have hidden it from every caller, - // including adoption's fail-closed contract. - tracing::warn!( - "[transcript] could not read a directory entry in {}: {error}", - raw_dir.display() - ); - any_unreadable = true; - continue; - } - }; - let is_candidate = path.extension().and_then(|s| s.to_str()) == Some("jsonl") - && path - .file_stem() - .and_then(|s| s.to_str()) - .is_some_and(|stem| !stem.contains("__")); - if !is_candidate { - continue; - } - match read_transcript(&path) { - Ok(transcript) if transcript.meta.thread_id.as_deref() == Some(thread_id) => { - matches.push((transcript.meta.created.clone(), path)); - } - Ok(_) => {} - Err(err) => { - tracing::warn!( - "[transcript] skipping unreadable root transcript candidate {}: {err}", - path.display() - ); - any_unreadable = true; - } - } - } - - // A compaction generation inherits its predecessor's `meta.created` - // (`begin_generation` seeds from the live meta), so `created` alone ties a - // whole chain. The generation number breaks that tie before the path does: - // by path, `X.g1.jsonl` sorts *before* `X.jsonl` (`g` < `j`) and `.g10` - // before `.g2`, which made the newest-wins lookups resolve a sealed - // generation instead of the head. Path stays the final tiebreak so the - // order is total and deterministic. - matches.sort_by(|left, right| { - left.0 - .cmp(&right.0) - .then_with(|| path_generation(&left.1).cmp(&path_generation(&right.1))) - .then_with(|| left.1.cmp(&right.1)) - }); - ( - matches.into_iter().map(|(_, path)| path).collect(), - any_unreadable, - ) -} - -/// The compaction generation a transcript path encodes: `n` for a -/// `{stem}.g{n}.jsonl` successor, `0` otherwise. -/// -/// Unambiguous because [`super::session::session_stem`] never lets a `.` -/// through inside a component and appends a digest to every component, so a -/// trailing `.g{digits}` can only be the generation suffix it writes. -pub(crate) fn path_generation(path: &Path) -> u32 { - path.file_stem() - .and_then(|stem| stem.to_str()) - .and_then(|stem| stem.rsplit_once(".g")) - .filter(|(_, digits)| !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit())) - .and_then(|(_, digits)| digits.parse().ok()) - .unwrap_or(0) -} - -/// Summed token/cost usage for `thread_id` across its root transcripts, or -/// `None` when the thread has no persisted turns yet. -pub fn read_thread_usage_summary( - workspace_dir: &Path, - thread_id: &str, -) -> Option { - let thread_id = thread_id.trim(); - if thread_id.is_empty() { - return None; - } - - // Single scan: split the thread's transcripts into root (orchestrator) and - // `__` sub-agent files. Root totals stay the parent's; sub-agent files are - // grouped by archetype for the per-agent breakdown. - let mut root_matches: Vec = Vec::new(); - let mut sub_matches: Vec = Vec::new(); - let raw_dir = raw_session_dir(workspace_dir); - let Ok(entries) = fs::read_dir(&raw_dir) else { - return None; - }; - for path in entries.flatten().map(|entry| entry.path()) { - if path.extension().and_then(|s| s.to_str()) != Some("jsonl") { - continue; - } - let Some(stem) = path.file_stem().and_then(|s| s.to_str()) else { - continue; - }; - let is_subagent = stem.contains("__"); - let matches_thread = read_transcript_meta_only(&path) - .map(|m| m.thread_id.as_deref() == Some(thread_id)) - .unwrap_or(false); - if !matches_thread { - continue; - } - if is_subagent { - sub_matches.push(path); - } else { - root_matches.push(path); - } - } - - if root_matches.is_empty() && sub_matches.is_empty() { - return None; - } - root_matches.sort_by(|left, right| left.file_name().cmp(&right.file_name())); - - let mut summary = ThreadUsageSummary::default(); - for path in &root_matches { - if let Some(meta) = read_transcript_meta_only(path) { - summary.input_tokens = summary.input_tokens.saturating_add(meta.input_tokens); - summary.output_tokens = summary.output_tokens.saturating_add(meta.output_tokens); - summary.cached_input_tokens = summary - .cached_input_tokens - .saturating_add(meta.cached_input_tokens); - summary.cost_usd += meta.charged_amount_usd; - summary.turn_count = summary.turn_count.saturating_add(meta.turn_count); - } - } - - // Newest root transcript drives the last-turn gauge + model + updated stamp. - if let Some(newest) = root_matches.last() { - if let Some(meta) = read_transcript_meta_only(newest) { - summary.updated = meta.updated; - } - if let Some((usage, model)) = read_last_assistant_usage(newest) { - summary.last_turn_input_tokens = usage.input; - summary.last_turn_output_tokens = usage.output; - summary.model = model; - } - } - - // Group sub-agent transcripts by archetype (`agent_name`). - let mut groups: BTreeMap = BTreeMap::new(); - for path in &sub_matches { - let Some(meta) = read_transcript_meta_only(path) else { - continue; - }; - let group = - groups - .entry(meta.agent_name.clone()) - .or_insert_with(|| SubagentArchetypeUsage { - agent_id: meta.agent_name.clone(), - ..Default::default() - }); - group.input_tokens = group.input_tokens.saturating_add(meta.input_tokens); - group.output_tokens = group.output_tokens.saturating_add(meta.output_tokens); - group.cached_input_tokens = group - .cached_input_tokens - .saturating_add(meta.cached_input_tokens); - group.runs = group.runs.saturating_add(1); - if group.model.is_none() - && let Some((_, model)) = read_last_assistant_usage(path) - { - group.model = model; - } - } - summary.subagents = groups.into_values().collect(); - - Some(summary) -} diff --git a/crates/tinyagents-session/src/transcript/transcript_tests.rs b/crates/tinyagents-session/src/transcript/transcript_tests.rs index 87654e5b1..7636b41ef 100644 --- a/crates/tinyagents-session/src/transcript/transcript_tests.rs +++ b/crates/tinyagents-session/src/transcript/transcript_tests.rs @@ -6,7 +6,6 @@ //! Consolidated here per AGENTS.md: one `test.rs` per module directory. use super::*; -use std::sync::{Arc, Barrier}; use tempfile::tempdir; fn meta() -> TranscriptMeta { @@ -182,7 +181,6 @@ fn an_invalid_utf8_tail_does_not_block_cold_resume() { assert_eq!(model.messages.len(), 2); assert_eq!(model.messages[1].content, "after crash"); assert_eq!(read_transcript_display(&path).unwrap().records.len(), 2); - assert!(super::reader::read_transcript_meta_only(&path).is_some()); } #[test] @@ -255,156 +253,8 @@ fn append_replays_delta_compaction_request_ids_and_interrupted_partials() { assert!(raw.contains("request-c")); } -#[test] -fn file_history_never_converts_or_drops_durable_fields() { - let dir = tempdir().unwrap(); - let history = FileTranscriptHistory::new(dir.path(), "session", meta()).unwrap(); - let assistant = TranscriptMessage { - id: Some("assistant-id".into()), - role: "assistant".into(), - content: r#"{"tool_calls":[{"id":"call-1"}]}"#.into(), - tool_calls: Vec::new(), - tool_call_id: None, - parts: None, - legacy: Default::default(), - extra_metadata: Some(serde_json::json!({ - "trusted_verbatim": true, - "artifacts": ["artifact-1"], - "provider_extension": {"opaque": [1, 2]} - })), - cache_breakpoints: vec![4], - turn_usage: None, - request_id: None, - preserve_request_id: false, - interrupted: false, - tool_failure: None, - }; - TranscriptHistory::append(&history, assistant.clone()).unwrap(); - let replayed = TranscriptHistory::messages(&history).unwrap(); - assert_eq!(replayed.len(), 1); - assert_eq!(replayed[0].id, assistant.id); - assert_eq!(replayed[0].role, assistant.role); - assert_eq!(replayed[0].content, assistant.content); - assert_eq!(replayed[0].extra_metadata, assistant.extra_metadata); - assert_eq!(replayed[0].cache_breakpoints, vec![4]); - TranscriptHistory::replace(&history, std::slice::from_ref(&assistant)).unwrap(); - TranscriptHistory::clear(&history).unwrap(); - assert!(TranscriptHistory::messages(&history).unwrap().is_empty()); - let display = read_transcript_display(history.path()).unwrap(); - assert!( - display - .records - .iter() - .any(|record| matches!(record, DisplayRecord::Compaction(_))) - ); - let raw = std::fs::read_to_string(history.path()).unwrap(); - assert!(raw.contains("trusted_verbatim")); - assert!(raw.contains("artifact-1")); - assert!(raw.contains("\"cache_breakpoints\":[4]")); -} - -#[test] -fn discovery_and_legacy_read_replay_the_canonical_format() { - let dir = tempdir().unwrap(); - let root = resolve_keyed_transcript_path(dir.path(), "1714000000_agent").unwrap(); - let mut root_meta = meta(); - root_meta.thread_id = Some("thread-1".into()); - write_transcript( - &root, - &[TranscriptMessage::new("user", "hello")], - &root_meta, - None, - ) - .unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let found = locator.root_for_thread("thread-1").unwrap(); - assert_eq!( - found.read_session().unwrap().unwrap().messages[0].content, - "hello" - ); - - let legacy = dir.path().join("legacy.md"); - std::fs::write(&legacy, "\n\nlegacy body\n\n").unwrap(); - assert_eq!( - read_transcript_legacy_md(&legacy).unwrap().messages[0].content, - "legacy body" - ); -} - // ── Session identity ────────────────────────────────────────────────── -/// One conversation, two cold sessions: the second must find the first's file -/// rather than mint a second stem for the same thread. This is the regression -/// that cost a real user the opening turns of a thread. -#[test] -fn one_session_resolves_to_one_transcript_across_separate_bindings() { - let dir = tempdir().unwrap(); - let session = SessionRef::scoped("thread-9fa08", "orchestrator"); - - let first = FileTranscriptLocator::new(dir.path()) - .open_session(&session, meta()) - .unwrap(); - first - .append(TranscriptMessage::new( - "user", - "i want to plan a trip to kashmir", - )) - .unwrap(); - - // A brand-new locator and handle, as a restarted process would build. - let second = FileTranscriptLocator::new(dir.path()) - .open_session(&session, meta()) - .unwrap(); - second - .append(TranscriptMessage::new("user", "hello?")) - .unwrap(); - - assert_eq!(first.path(), second.path()); - let messages = second.messages().unwrap(); - assert_eq!(messages.len(), 2); - assert_eq!(messages[0].content, "i want to plan a trip to kashmir"); - assert_eq!(messages[1].content, "hello?"); - - let roots: Vec<_> = std::fs::read_dir(dir.path().join("session_raw")) - .unwrap() - .flatten() - .map(|entry| entry.file_name()) - .collect(); - assert_eq!( - roots.len(), - 1, - "one conversation must not sprawl: {roots:?}" - ); -} - -#[test] -fn an_unwritten_session_reads_as_absent_rather_than_erroring() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-new", "orchestrator"); - - assert!(locator.read_session_transcript(&session).is_none()); - assert!(!locator.session_exists(&session)); - assert_eq!(locator.head_generation(&session), session); -} - -/// A directory occupying a session's canonical `.jsonl` path must never be -/// treated as an existing generation: reads/appends against it would fail -/// (or silently target the wrong thing), and `head_generation`'s chain walk -/// would otherwise stop at a phantom "generation" that was never written. -#[test] -fn a_directory_at_the_canonical_path_is_never_treated_as_an_existing_session() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-1", "orchestrator"); - - let path = resolve_keyed_transcript_path(dir.path(), &session_stem(&session)).unwrap(); - std::fs::create_dir_all(&path).unwrap(); - - assert!(!locator.session_exists(&session)); - assert!(locator.read_session_transcript(&session).is_none()); -} - #[test] fn session_identity_round_trips_through_the_jsonl_meta() { let dir = tempdir().unwrap(); @@ -431,551 +281,6 @@ fn session_identity_round_trips_through_the_jsonl_meta() { ); } -/// A compaction must never destroy what it replaces. It seals the current -/// generation and opens the next, so the replaced turns stay on disk. -#[test] -fn a_compaction_seals_a_generation_and_leaves_it_untouched() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-1", "orchestrator"); - - let first = locator.open_session(&session, meta()).unwrap(); - for turn in ["one", "two", "three"] { - first.append(TranscriptMessage::new("user", turn)).unwrap(); - } - let sealed_path = first.path().to_path_buf(); - let sealed_bytes = std::fs::read(&sealed_path).unwrap(); - - let (successor, handle) = locator.begin_generation(&session, meta()).unwrap(); - // The successor is bound but empty; the retained set is written through the - // ordinary turn path so usage and request ids are recorded as usual. - handle - .replace(&[TranscriptMessage::new("user", "three")]) - .unwrap(); - - assert_eq!(successor.generation, 1); - assert_eq!( - std::fs::read(&sealed_path).unwrap(), - sealed_bytes, - "the sealed generation must be byte-identical afterwards" - ); - assert_ne!(handle.path(), sealed_path); - - let carried = handle.messages().unwrap(); - assert_eq!(carried.len(), 1); - assert_eq!(carried[0].content, "three"); - - let successor_meta = handle.read_session().unwrap().unwrap().meta; - assert_eq!( - successor_meta.session_id.as_deref(), - Some(session_stem(&successor).as_str()) - ); - assert_eq!( - successor_meta.parent_session_id.as_deref(), - Some(session_stem(&session).as_str()) - ); -} - -/// A fork for edit/regenerate must give the same untouched-sealed-file -/// guarantee a compaction gives, and the chain must walk both generations. -#[test] -fn truncate_into_next_generation_seals_the_head_byte_identical_and_chain_walks_both() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-1", "orchestrator"); - - let first = locator.open_session(&session, meta()).unwrap(); - for turn in ["one", "two", "three", "four"] { - first.append(TranscriptMessage::new("user", turn)).unwrap(); - } - let sealed_path = first.path().to_path_buf(); - let sealed_bytes = std::fs::read(&sealed_path).unwrap(); - - let (successor, handle, truncated) = locator - .truncate_into_next_generation(&session, TruncateCut::BeforeIndex(2), meta()) - .unwrap(); - - assert_eq!(successor.generation, 1); - assert_eq!( - std::fs::read(&sealed_path).unwrap(), - sealed_bytes, - "the sealed head must be byte-identical afterwards" - ); - assert_ne!(handle.path(), sealed_path); - - // The new head carries only the retained prefix. - let contents: Vec<_> = truncated.iter().map(|m| m.content.as_str()).collect(); - assert_eq!(contents, vec!["one", "two"]); - assert_eq!(handle.messages().unwrap().len(), 2); - - // Parent recorded exactly as `begin_generation` records it. - let successor_meta = handle.read_session().unwrap().unwrap().meta; - assert_eq!( - successor_meta.parent_session_id.as_deref(), - Some(session_stem(&session).as_str()) - ); - - // Both generations are reachable by walking the chain — nothing is lost. - let chain = locator.session_chain(&session); - assert_eq!(chain.len(), 2); - assert_eq!(chain[0], session); - assert_eq!(chain[1], successor); - assert_eq!(locator.head_generation(&session), successor); -} - -#[test] -fn truncate_into_next_generation_by_message_id_cuts_at_that_message() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-1", "orchestrator"); - - let first = locator.open_session(&session, meta()).unwrap(); - let mut keep_a = TranscriptMessage::new("user", "keep-a"); - keep_a.id = Some("m1".into()); - let mut cut_here = TranscriptMessage::new("assistant", "cut-here"); - cut_here.id = Some("m2".into()); - let mut dropped = TranscriptMessage::new("user", "dropped"); - dropped.id = Some("m3".into()); - for message in [keep_a, cut_here, dropped] { - first.append(message).unwrap(); - } - - let (_, handle, truncated) = locator - .truncate_into_next_generation(&session, TruncateCut::BeforeMessageId("m2".into()), meta()) - .unwrap(); - - assert_eq!(truncated.len(), 1); - assert_eq!(truncated[0].id.as_deref(), Some("m1")); - assert_eq!(handle.messages().unwrap().len(), 1); -} - -#[test] -fn truncate_into_next_generation_by_unknown_message_id_fails() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-1", "orchestrator"); - locator - .open_session(&session, meta()) - .unwrap() - .append(TranscriptMessage::new("user", "one")) - .unwrap(); - - let err = locator.truncate_into_next_generation( - &session, - TruncateCut::BeforeMessageId("nonexistent".into()), - meta(), - ); - assert!( - err.is_err(), - "an unresolvable id must fail rather than silently keep everything" - ); -} - -#[test] -fn truncate_into_next_generation_last_assistant_turn_drops_only_the_trailing_answer() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-1", "orchestrator"); - - let first = locator.open_session(&session, meta()).unwrap(); - for (role, content) in [ - ("user", "first question"), - ("assistant", "first answer"), - ("user", "second question"), - ("assistant", "second answer, to regenerate"), - ] { - first.append(TranscriptMessage::new(role, content)).unwrap(); - } - - let (_, handle, truncated) = locator - .truncate_into_next_generation(&session, TruncateCut::LastAssistantTurn, meta()) - .unwrap(); - - let contents: Vec<_> = truncated.iter().map(|m| m.content.as_str()).collect(); - assert_eq!( - contents, - vec!["first question", "first answer", "second question"] - ); - assert_eq!(handle.messages().unwrap().len(), 3); -} - -#[test] -fn truncate_into_next_generation_operates_on_the_current_head_not_the_root() { - // A fork after an earlier compaction must truncate the head generation's - // messages, not the sealed root's. - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-1", "orchestrator"); - - locator - .open_session(&session, meta()) - .unwrap() - .append(TranscriptMessage::new("user", "root-only")) - .unwrap(); - let (compacted, compacted_handle) = locator.begin_generation(&session, meta()).unwrap(); - for turn in ["alpha", "beta", "gamma"] { - compacted_handle - .append(TranscriptMessage::new("user", turn)) - .unwrap(); - } - - let (successor, handle, truncated) = locator - .truncate_into_next_generation(&session, TruncateCut::BeforeIndex(1), meta()) - .unwrap(); - - assert_eq!(successor.generation, 2); - assert_eq!(successor.parent_session_id(), Some(compacted.session_id())); - let contents: Vec<_> = truncated.iter().map(|m| m.content.as_str()).collect(); - assert_eq!(contents, vec!["alpha"]); - assert_eq!(handle.messages().unwrap().len(), 1); -} - -/// After a compaction, a resume must land on the newest generation — the one -/// the model is actually continuing — not on the sealed original. -#[test] -fn head_generation_follows_the_compaction_chain() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-1", "orchestrator"); - - locator - .open_session(&session, meta()) - .unwrap() - .append(TranscriptMessage::new("user", "one")) - .unwrap(); - assert_eq!(locator.head_generation(&session), session); - - let (first_successor, first_handle) = locator.begin_generation(&session, meta()).unwrap(); - first_handle - .append(TranscriptMessage::new("user", "one")) - .unwrap(); - assert_eq!(locator.head_generation(&session), first_successor); - - let (second_successor, second_handle) = - locator.begin_generation(&first_successor, meta()).unwrap(); - second_handle - .append(TranscriptMessage::new("user", "one")) - .unwrap(); - assert_eq!(locator.head_generation(&session).generation, 2); - assert_eq!(locator.head_generation(&session), second_successor); -} - -#[test] -fn opening_a_generation_that_already_exists_is_refused() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-1", "orchestrator"); - - let (_, handle) = locator.begin_generation(&session, meta()).unwrap(); - handle - .append(TranscriptMessage::new("user", "one")) - .unwrap(); - let second = locator.begin_generation(&session, meta()); - - assert!( - second.is_err(), - "sealing the same generation twice would overwrite durable history" - ); -} - -#[test] -fn clearing_an_absent_successor_releases_its_reservation() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-clear-reservation", "orchestrator"); - let (_, handle) = locator.begin_generation(&session, meta()).unwrap(); - - handle.clear().unwrap(); - // Keep the original handle alive: clear must release its reservation itself. - assert!(locator.begin_generation(&session, meta()).is_ok()); -} - -#[test] -fn non_file_entry_occupying_successor_is_rejected() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-occupied-successor", "orchestrator"); - let successor = session.next_generation(); - let path = resolve_keyed_transcript_path(dir.path(), &session_stem(&successor)).unwrap(); - std::fs::create_dir(&path).unwrap(); - - assert!(locator.begin_generation(&session, meta()).is_err()); -} - -/// Concurrent generation selection must reserve the successor slot until -/// its first write. Exactly one process may commit that generation. -#[test] -fn concurrent_begin_generation_allows_only_one_successor_writer() { - let dir = tempdir().unwrap(); - let session = FileTranscriptLocator::new(dir.path()); - let root = SessionRef::scoped("thread-1", "orchestrator"); - // Seal generation 0 so both racers are compacting into the same, - // already-known successor generation 1. - session - .open_session(&root, meta()) - .unwrap() - .append(TranscriptMessage::new("user", "sealed")) - .unwrap(); - - let locator = Arc::new(FileTranscriptLocator::new(dir.path())); - let left_locator = Arc::clone(&locator); - let left_root = root.clone(); - let left = std::thread::spawn(move || { - let (_, handle) = left_locator.begin_generation(&left_root, meta())?; - handle.append(TranscriptMessage::new("user", "from left"))?; - anyhow::Ok(()) - }); - - let right_locator = Arc::clone(&locator); - let right_root = root.clone(); - let right = std::thread::spawn(move || { - let (_, handle) = right_locator.begin_generation(&right_root, meta())?; - handle.append(TranscriptMessage::new("user", "from right"))?; - anyhow::Ok(()) - }); - - let outcomes = [left.join().unwrap(), right.join().unwrap()]; - assert_eq!(outcomes.iter().filter(|result| result.is_ok()).count(), 1); - - let successor = root.next_generation(); - let handle = locator.open_session(&successor, meta()).unwrap(); - let contents: Vec = handle - .messages() - .unwrap() - .into_iter() - .map(|message| message.content) - .collect(); - assert_eq!( - contents.len(), - 1, - "only the winning generation is visible: {contents:?}" - ); -} - -#[test] -fn cross_process_generation_worker_refuses_a_committed_slot() { - let Ok(workspace) = std::env::var("TINYAGENTS_GENERATION_WORKER_DIR") else { - return; - }; - let marker = std::env::var("TINYAGENTS_GENERATION_WORKER_MARKER").unwrap(); - std::fs::write(marker, "ready").unwrap(); - let locator = FileTranscriptLocator::new(workspace); - let result = locator.begin_generation( - &SessionRef::scoped("thread-cross-process", "orchestrator"), - meta(), - ); - assert!( - result.is_err(), - "the committed generation slot must be occupied" - ); -} - -#[test] -fn generation_reservation_coordinates_separate_processes() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let root = SessionRef::scoped("thread-cross-process", "orchestrator"); - locator - .open_session(&root, meta()) - .unwrap() - .append(TranscriptMessage::new("user", "sealed")) - .unwrap(); - let (_, winner) = locator.begin_generation(&root, meta()).unwrap(); - - let marker = dir.path().join("worker-ready"); - let child = std::process::Command::new(std::env::current_exe().unwrap()) - .args([ - "--exact", - "transcript::test::cross_process_generation_worker_refuses_a_committed_slot", - ]) - .env("TINYAGENTS_GENERATION_WORKER_DIR", dir.path()) - .env("TINYAGENTS_GENERATION_WORKER_MARKER", &marker) - .spawn() - .unwrap(); - for _ in 0..500 { - if marker.exists() { - break; - } - std::thread::sleep(std::time::Duration::from_millis(10)); - } - assert!(marker.exists(), "worker did not start in time"); - - winner - .append(TranscriptMessage::new("user", "winner")) - .unwrap(); - let output = child.wait_with_output().unwrap(); - assert!( - output.status.success(), - "worker failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - assert_eq!( - locator - .open_session(&root.next_generation(), meta()) - .unwrap() - .messages() - .unwrap() - .len(), - 1, - "the committed generation contains only the winning writer's payload" - ); -} - -/// Two handles on one session — the shape two cores over one workspace -/// produce — must both land in the same file, with neither losing the other's -/// turns. -#[test] -fn concurrent_handles_on_one_session_both_extend_it() { - let dir = tempdir().unwrap(); - let session = SessionRef::scoped("thread-1", "orchestrator"); - let left = FileTranscriptLocator::new(dir.path()) - .open_session(&session, meta()) - .unwrap(); - let right = FileTranscriptLocator::new(dir.path()) - .open_session(&session, meta()) - .unwrap(); - - // Seed the file first, so both appends below exercise the append-only - // path — `write_logical_set` re-reading `persisted` fresh immediately - // before every write, which is the actual claim under test — rather than - // racing on first-write file creation, a distinct, pre-existing concern - // this test is not about. - left.append(TranscriptMessage::new("user", "seed")).unwrap(); - - // Genuinely overlapping, not merely interleaved: both handles race to - // append from separate OS threads, released together by a barrier so - // neither can start before the other is ready. A handle that cached its - // own view of `persisted` instead of re-reading it fresh before every - // write could lose whichever append the barrier let land second. - let barrier = Arc::new(Barrier::new(2)); - let left_barrier = Arc::clone(&barrier); - let left_thread = std::thread::spawn(move || { - left_barrier.wait(); - left.append(TranscriptMessage::new("user", "from left")) - .unwrap(); - }); - let right_barrier = Arc::clone(&barrier); - let right_thread = std::thread::spawn(move || { - right_barrier.wait(); - right - .append(TranscriptMessage::new("user", "from right")) - .unwrap(); - }); - left_thread.join().unwrap(); - right_thread.join().unwrap(); - - let reread = FileTranscriptLocator::new(dir.path()) - .open_session(&session, meta()) - .unwrap(); - let mut contents: Vec = reread - .messages() - .unwrap() - .into_iter() - .map(|message| message.content) - .collect(); - assert_eq!(contents.remove(0), "seed"); - contents.sort(); - assert_eq!( - contents, - ["from left", "from right"], - "an overlapping append from either handle must not be lost" - ); -} - -#[test] -fn turn_write_rejects_a_stale_baseline_instead_of_compacting_over_newer_data() { - let dir = tempdir().unwrap(); - let session = SessionRef::scoped("thread-stale", "orchestrator"); - let locator = FileTranscriptLocator::new(dir.path()); - let first = locator.open_session(&session, meta()).unwrap(); - let old = vec![TranscriptMessage::new("user", "original")]; - first - .append_turn(TranscriptTurn { - prev: &[], - next: &old, - meta: &meta(), - turn_usage: None, - request_id: None, - tools: None, - }) - .unwrap(); - - let stale_handle = locator.open_session(&session, meta()).unwrap(); - let newer = vec![ - old[0].clone(), - TranscriptMessage::new("assistant", "new turn"), - ]; - first - .append_turn(TranscriptTurn { - prev: &old, - next: &newer, - meta: &meta(), - turn_usage: None, - request_id: None, - tools: None, - }) - .unwrap(); - - let stale_next = vec![TranscriptMessage::new("user", "replacement")]; - let result = stale_handle.append_turn(TranscriptTurn { - prev: &old, - next: &stale_next, - meta: &meta(), - turn_usage: None, - request_id: None, - tools: None, - }); - assert!(result.is_err(), "stale writers must report a conflict"); - let actual = stale_handle.messages().unwrap(); - assert_eq!(actual.len(), 2); - assert_eq!(actual[1].content, "new turn"); -} - -#[test] -fn bound_parent_rejects_appends_after_a_successor_is_committed() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let parent = SessionRef::scoped("thread-sealed-parent", "orchestrator"); - let old_handle = locator.open_session(&parent, meta()).unwrap(); - old_handle - .append(TranscriptMessage::new("user", "parent")) - .unwrap(); - let (_, successor) = locator.begin_generation(&parent, meta()).unwrap(); - successor - .append(TranscriptMessage::new("user", "successor")) - .unwrap(); - - let result = old_handle.append(TranscriptMessage::new("assistant", "late parent write")); - assert!(result.is_err(), "a committed successor seals its parent"); - assert_eq!(old_handle.messages().unwrap().len(), 1); -} - -/// The model reads only the head generation, but a host rendering or -/// exporting the conversation needs every segment, in order. -#[test] -fn a_session_chain_lists_every_generation_oldest_first() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-1", "orchestrator"); - - assert!(locator.session_chain(&session).is_empty()); - - locator - .open_session(&session, meta()) - .unwrap() - .append(TranscriptMessage::new("user", "one")) - .unwrap(); - let (successor, handle) = locator.begin_generation(&session, meta()).unwrap(); - handle - .append(TranscriptMessage::new("user", "two")) - .unwrap(); - - let chain = locator.session_chain(&session); - assert_eq!(chain, vec![session.clone(), successor]); - // Asking from any generation returns the same whole chain. - assert_eq!(locator.session_chain(&chain[1]), chain); -} - /// [`write_transcript_if_absent`] is what keeps adoption from clobbering a /// destination a concurrent normal turn created while adoption was still /// scanning legacy roots (see `adoption::adopt_legacy_session_transcripts`). @@ -1159,45 +464,6 @@ fn a_turn_stamps_iteration_and_ts_on_every_step_it_appends() { assert_eq!(with_usage, 1); } -/// A compaction successor inherits its predecessor's `created`, so the thread -/// scan has to order a generation chain by generation — by path alone -/// `X.g1` sorts before `X` and `X.g10` before `X.g2`, and the newest-wins -/// lookup resolved a sealed generation. -#[test] -fn thread_scan_orders_a_generation_chain_by_generation() { - let dir = tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-gen", "orchestrator"); - let mut seed = meta(); - seed.thread_id = Some("thread-gen".into()); - locator - .open_session(&session, seed.clone()) - .unwrap() - .append(TranscriptMessage::new("user", "g0")) - .unwrap(); - let mut head = session.clone(); - for n in 1..=10 { - let (successor, handle) = locator.begin_generation(&head, seed.clone()).unwrap(); - handle - .append(TranscriptMessage::new("user", format!("g{n}"))) - .unwrap(); - head = successor; - } - - let roots = find_root_transcripts_for_thread(dir.path(), "thread-gen"); - let generations: Vec = roots - .iter() - .map(|path| super::thread_lookup::path_generation(path)) - .collect(); - assert_eq!(generations, (0..=10).collect::>()); - let newest = find_root_transcript_for_thread(dir.path(), "thread-gen").unwrap(); - assert_eq!( - read_transcript(&newest).unwrap().messages[0].content, - "g10", - "newest-wins resolves the head generation" - ); -} - #[test] fn role_constructors_build_bare_rows() { for (row, role) in [ diff --git a/crates/tinyagents-session/src/transcript/turn_lock.rs b/crates/tinyagents-session/src/transcript/turn_lock.rs index 8f4c836ec..786cc52ac 100644 --- a/crates/tinyagents-session/src/transcript/turn_lock.rs +++ b/crates/tinyagents-session/src/transcript/turn_lock.rs @@ -19,10 +19,7 @@ use super::history::TranscriptLocator; use super::session::{SessionRef, session_stem}; use std::collections::HashMap; -use std::future::Future; -use std::pin::Pin; use std::sync::{Arc, Mutex, OnceLock, Weak}; -use std::task::{Context, Poll}; use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard}; /// Proof that the holder owns `session`'s turn lock. Released on drop. @@ -74,55 +71,6 @@ pub(crate) async fn lock_session_turn_or_identity( SessionTurnGuard { _guard: guard } } -#[doc(hidden)] -pub(crate) async fn lock_session_turn_with_notification( - locator: &dyn TranscriptLocator, - session: &SessionRef, - attempted: Option<&tokio::sync::Notify>, -) -> Option { - let destination = locator.destination_key()?; - let stem = session_stem(&session.first_generation()); - let lock = registry_lock(destination, stem.clone()); - tracing::debug!(session = %stem, "[transcript-turn-lock] acquiring"); - let guard = NotifyOnPoll::new(lock.lock_owned(), attempted).await; - tracing::debug!(session = %stem, "[transcript-turn-lock] acquired"); - Some(SessionTurnGuard { _guard: guard }) -} - -struct NotifyOnPoll<'a, F> { - future: Pin>, - notify: Option<&'a tokio::sync::Notify>, - notified: bool, -} - -impl<'a, F> NotifyOnPoll<'a, F> { - fn new(future: F, notify: Option<&'a tokio::sync::Notify>) -> Self { - Self { - future: Box::pin(future), - notify, - notified: false, - } - } -} - -impl Future for NotifyOnPoll<'_, F> { - type Output = F::Output; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - // Poll the wrapped future first: a waiter woken by the notification - // then knows the lock attempt has already been made (and parked, if it - // is held), instead of racing a not-yet-polled acquisition. - let result = self.future.as_mut().poll(cx); - if !self.notified { - if let Some(notify) = self.notify { - notify.notify_one(); - } - self.notified = true; - } - result - } -} - /// The shared mutex for `(destination, stem)`, created on first use. fn registry_lock(destination: String, stem: String) -> Arc> { type Registry = Mutex>>>; diff --git a/crates/tinyagents-session/src/transcript/turn_lock_tests.rs b/crates/tinyagents-session/src/transcript/turn_lock_tests.rs index 8e30f4d3a..cfbe61969 100644 --- a/crates/tinyagents-session/src/transcript/turn_lock_tests.rs +++ b/crates/tinyagents-session/src/transcript/turn_lock_tests.rs @@ -1,5 +1,5 @@ use super::*; -use crate::transcript::{FileTranscriptLocator, TranscriptHistory, TranscriptMeta, TranscriptRead}; +use crate::transcript::{TranscriptHistory, TranscriptMeta, TranscriptRead}; /// Whether `lock_session_turn` for `session` would have to wait right now. /// @@ -13,10 +13,28 @@ async fn is_held(locator: &dyn TranscriptLocator, session: &SessionRef) -> bool } } +/// A locator naming a fixed destination, so separate instances stand for +/// separate handles over one store. +struct Keyed(&'static str); + +impl TranscriptLocator for Keyed { + fn destination_key(&self) -> Option { + Some(self.0.to_string()) + } + fn latest_for_agent(&self, _: &str) -> Option> { + None + } + fn root_for_thread(&self, _: &str) -> Option> { + None + } + fn open_stem(&self, _: &str, _: TranscriptMeta) -> anyhow::Result> { + anyhow::bail!("not a store") + } +} + #[tokio::test] async fn a_held_turn_lock_blocks_the_same_session_until_released() { - let dir = tempfile::tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); + let locator = Keyed("store-a"); let session = SessionRef::scoped("thread-1", "agent"); let guard = lock_session_turn(&locator, &session).await.unwrap(); @@ -27,150 +45,39 @@ async fn a_held_turn_lock_blocks_the_same_session_until_released() { } #[tokio::test] -async fn every_generation_and_every_locator_over_one_workspace_share_the_lock() { - let dir = tempfile::tempdir().unwrap(); +async fn every_generation_and_every_locator_over_one_destination_share_the_lock() { let session = SessionRef::scoped("thread-1", "agent"); - let _guard = lock_session_turn(&FileTranscriptLocator::new(dir.path()), &session) + let _guard = lock_session_turn(&Keyed("store-a"), &session) .await .unwrap(); // A compaction moves the head on; it is still the same conversation. A // host also rebuilds its locator per turn rather than sharing one. - let rebuilt = FileTranscriptLocator::new(dir.path()); + let rebuilt = Keyed("store-a"); assert!(is_held(&rebuilt, &session.next_generation().next_generation()).await); } #[tokio::test] -async fn missing_workspace_paths_are_normalized_for_locking() { - let dir = tempfile::tempdir().unwrap(); - let missing = dir.path().join("not-created"); - let equivalent = missing.join("child").join(".."); +async fn other_sessions_and_other_destinations_are_independent() { + let locator = Keyed("store-a"); let session = SessionRef::scoped("thread-1", "agent"); + let _guard = lock_session_turn(&locator, &session).await.unwrap(); - let first = FileTranscriptLocator::new(&missing); - let second = FileTranscriptLocator::new(&equivalent); - let _guard = lock_session_turn(&first, &session).await.unwrap(); - assert!( - tokio::time::timeout( - std::time::Duration::from_millis(20), - lock_session_turn(&second, &session), - ) - .await - .is_err() - ); + assert!(!is_held(&locator, &SessionRef::scoped("thread-2", "agent")).await); + assert!(!is_held(&locator, &SessionRef::scoped("thread-1", "other-agent")).await); + assert!(!is_held(&Keyed("store-b"), &session).await); } -#[cfg(unix)] #[tokio::test] -async fn missing_workspace_below_a_symlink_is_normalized_for_locking() { - use std::os::unix::fs::symlink; - - let dir = tempfile::tempdir().unwrap(); - let real = dir.path().join("real"); - std::fs::create_dir(&real).unwrap(); - let link = dir.path().join("link"); - symlink(&real, &link).unwrap(); +async fn distinct_in_memory_locators_do_not_share_a_lock() { + use crate::port::InMemoryTranscriptLocator; + let first = InMemoryTranscriptLocator::new("first"); + let second = InMemoryTranscriptLocator::new("second"); let session = SessionRef::scoped("thread-1", "agent"); - let first = FileTranscriptLocator::new(real.join("not-created")); - let second = FileTranscriptLocator::new(link.join("not-created")); - let _guard = lock_session_turn(&first, &session).await.unwrap(); - assert!( - tokio::time::timeout( - std::time::Duration::from_millis(20), - lock_session_turn(&second, &session), - ) - .await - .is_err() - ); -} -#[cfg(unix)] -#[tokio::test] -async fn dangling_workspace_symlink_is_normalized_for_locking() { - use std::os::unix::fs::symlink; - - let dir = tempfile::tempdir().unwrap(); - let real = dir.path().join("future"); - let link = dir.path().join("link"); - symlink(&real, &link).unwrap(); - let session = SessionRef::scoped("thread-1", "agent"); - - let _guard = lock_session_turn(&FileTranscriptLocator::new(&real), &session) - .await - .unwrap(); - assert!( - tokio::time::timeout( - std::time::Duration::from_millis(20), - lock_session_turn(&FileTranscriptLocator::new(&link), &session), - ) - .await - .is_err() - ); -} - -#[cfg(unix)] -#[tokio::test] -async fn parent_components_are_resolved_after_symlinks() { - use std::os::unix::fs::symlink; - - let dir = tempfile::tempdir().unwrap(); - let real = dir.path().join("real"); - std::fs::create_dir_all(real.join("nested")).unwrap(); - let link = dir.path().join("link"); - symlink(real.join("nested"), &link).unwrap(); - let session = SessionRef::scoped("thread-1", "agent"); - - let _guard = lock_session_turn( - &FileTranscriptLocator::new(real.join("workspace")), - &session, - ) - .await - .unwrap(); - assert!( - tokio::time::timeout( - std::time::Duration::from_millis(20), - lock_session_turn( - &FileTranscriptLocator::new(link.join("..").join("workspace")), - &session, - ), - ) - .await - .is_err() - ); -} - -#[cfg(unix)] -#[tokio::test] -async fn absolute_parent_components_clamp_at_root() { - let session = SessionRef::scoped("thread-1", "agent"); - let _guard = lock_session_turn( - &FileTranscriptLocator::new(std::path::Path::new("/../tmp")), - &session, - ) - .await - .unwrap(); - assert!( - tokio::time::timeout( - std::time::Duration::from_millis(20), - lock_session_turn(&FileTranscriptLocator::new("/tmp"), &session), - ) - .await - .is_err() - ); -} - -#[tokio::test] -async fn other_sessions_and_other_workspaces_are_independent() { - let dir = tempfile::tempdir().unwrap(); - let elsewhere = tempfile::tempdir().unwrap(); - let locator = FileTranscriptLocator::new(dir.path()); - let session = SessionRef::scoped("thread-1", "agent"); - let _guard = lock_session_turn(&locator, &session).await.unwrap(); - - assert!(!is_held(&locator, &SessionRef::scoped("thread-2", "agent")).await); - assert!(!is_held(&locator, &SessionRef::scoped("thread-1", "other-agent")).await); - assert!(!is_held(&FileTranscriptLocator::new(elsewhere.path()), &session).await); + assert!(is_held(&first, &session).await); + assert!(!is_held(&second, &session).await); } /// A locator that cannot name its destination cannot be told apart from any @@ -185,7 +92,7 @@ impl TranscriptLocator for Anonymous { None } fn open_stem(&self, _: &str, _: TranscriptMeta) -> anyhow::Result> { - anyhow::bail!("not file backed") + anyhow::bail!("anonymous") } } @@ -197,25 +104,3 @@ async fn an_anonymous_locator_has_no_turn_lock() { .is_none() ); } - -#[cfg(unix)] -#[tokio::test] -async fn workspaces_sharing_a_symlinked_session_raw_share_the_lock() { - use std::os::unix::fs::symlink; - - let dir = tempfile::tempdir().unwrap(); - let shared = dir.path().join("shared-raw"); - std::fs::create_dir(&shared).unwrap(); - let first_ws = dir.path().join("ws-a"); - let second_ws = dir.path().join("ws-b"); - std::fs::create_dir(&first_ws).unwrap(); - std::fs::create_dir(&second_ws).unwrap(); - symlink(&shared, first_ws.join("session_raw")).unwrap(); - symlink(&shared, second_ws.join("session_raw")).unwrap(); - let session = SessionRef::scoped("thread-1", "agent"); - let first = FileTranscriptLocator::new(&first_ws); - let second = FileTranscriptLocator::new(&second_ws); - - let _guard = lock_session_turn(&first, &session).await.unwrap(); - assert!(is_held(&second, &session).await); -} diff --git a/crates/tinyagents-session/src/transcript/types.rs b/crates/tinyagents-session/src/transcript/types.rs index 1f8d16304..620247d97 100644 --- a/crates/tinyagents-session/src/transcript/types.rs +++ b/crates/tinyagents-session/src/transcript/types.rs @@ -761,7 +761,7 @@ pub struct DisplayMessage { /// `failure: true`). pub failure_detail: Option, /// Who delivered this line out of band, when it was written by - /// [`append_background_message`](super::append_background_message) + /// [`append_background_message`](super::append_background_message_in) /// rather than by a live turn. `None` for every turn-written line. pub background: Option, } @@ -790,53 +790,12 @@ pub struct DisplaySessionTranscript { pub records: Vec, } -/// Aggregated token/cost usage for a chat thread, summed across **all** of the -/// thread's root session transcripts (a thread reopened across days/restarts -/// produces several files). `last_turn_*`, `model`, and `updated` come from the -/// newest transcript so the UI can render a context-window gauge for the most -/// recent turn. Returns `None` when no transcript exists yet (a brand-new -/// thread with no completed turns). -#[derive(Debug, Clone, Default, PartialEq)] -pub struct ThreadUsageSummary { - /// Orchestrator (parent) token totals — the root transcript(s) only. Root - /// transcripts never include sub-agent calls (those go to a separate - /// observer + their own `__` transcript files); see [`Self::subagents`]. - pub input_tokens: u64, - pub output_tokens: u64, - pub cached_input_tokens: u64, - pub cost_usd: f64, - pub turn_count: usize, - /// Input/output tokens of the most recent assistant turn (context gauge). - pub last_turn_input_tokens: u64, - pub last_turn_output_tokens: u64, - /// Model that served the most recent turn, if recorded. - pub model: Option, - /// RFC-3339 `updated` of the newest transcript. - pub updated: String, - /// Per-archetype sub-agent spend, reconstructed from the thread's `__` - /// sub-agent transcripts (grouped by `agent_name`). - pub subagents: Vec, -} - -/// One sub-agent archetype's summed spend within a thread (e.g. all `coder` -/// runs). `model` is the model that served one of its runs, used to price it. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct SubagentArchetypeUsage { - pub agent_id: String, - pub input_tokens: u64, - pub output_tokens: u64, - pub cached_input_tokens: u64, - /// How many sub-agent runs of this archetype contributed. - pub runs: usize, - pub model: Option, -} - // ── Background appends ─────────────────────────────────────────────── /// Where an out-of-band transcript line came from, recorded on the line as an /// additive top-level `background` field. /// -/// Written by [`append_background_message`](super::append_background_message). +/// Written by [`append_background_message`](super::append_background_message_in). /// A reader that predates the field keeps the line as an ordinary message (the /// field lands in the line's unknown-field catch-all), so old binaries resume /// the delivered message exactly like a new one does. @@ -851,7 +810,7 @@ pub struct BackgroundOrigin { pub provenance: serde_json::Value, } -/// Options for [`append_background_message`](super::append_background_message). +/// Options for [`append_background_message`](super::append_background_message_in). #[derive(Debug, Clone, PartialEq)] pub struct BackgroundAppend { /// Idempotency key; must be non-empty. See [`BackgroundOrigin::idempotency_key`]. @@ -881,7 +840,7 @@ impl BackgroundAppend { } } -/// What [`append_background_message`](super::append_background_message) did. +/// What [`append_background_message`](super::append_background_message_in) did. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BackgroundAppendOutcome { /// The message was appended to head generation `generation`. diff --git a/crates/tinyagents-session/src/transcript/view/README.md b/crates/tinyagents-session/src/transcript/view/README.md index 5ad1598c3..d3f85e4ca 100644 --- a/crates/tinyagents-session/src/transcript/view/README.md +++ b/crates/tinyagents-session/src/transcript/view/README.md @@ -1,21 +1,22 @@ # Transcript view -This module projects persisted transcript JSONL files into display records and -serves newest-first pages. The transcript files remain the source of truth; -the bounded projection cache in `cache.rs` is an optimization for unscoped -reads. Agent-scoped reads resolve roots separately so one agent's request does -not reuse another scope's cached projection. +This module projects a thread's persisted transcripts (listed by a +`TranscriptLocator`) into display records and serves newest-first pages. The +store stays the source of truth; nothing is cached, so every page reflects the +store as it is now. Agent-scoped reads list only the roots that agent owns. ## Public surface -`get_page` and `get_page_scoped` return `TranscriptPage` values. The `project` -module exposes record and thread projection helpers, while `types.rs` defines +`get_page_in` returns `TranscriptPage` values. The `project` module exposes +record and thread projection helpers (`project_thread_in`, `project_transcripts`), while `types.rs` defines the serialized display item and status shapes. Cursors are opaque exclusive chronological upper bounds; callers should pass back `next_cursor` unchanged. ## Resolution and ordering -`resolve.rs` selects root generations and sub-agent transcript files. +The locator's `thread_transcripts` selects root generations and sub-agent +transcripts; `resolve.rs` drops the rows a successor generation re-wrote from +its predecessor. `subagents.rs` correlates child sessions with spawning calls and places their records in the root conversation. A scoped child must have a durable ownership relationship to an accepted root; a matching thread identifier alone is not @@ -24,8 +25,7 @@ for display. ## Operational constraints -Transcript reads are best-effort over append-only JSONL sources. Malformed or -unreadable candidates are skipped with diagnostics. Keep pagination cursors -stable and do not treat a cache hit as an authorization decision. +Transcript reads are best-effort over append-only sources. Unreadable +candidates are skipped with diagnostics. Keep pagination cursors stable. `status_map.rs` expresses `TranscriptSubagentStatus` in terms of the canonical `OrchestrationTaskStatus` (`From` in, `TryFrom` out; `Cancelled` has no projection). Its serialized strings are unchanged. Mapping table: `tinyagents-tasks` README. diff --git a/crates/tinyagents-session/src/transcript/view/cache.rs b/crates/tinyagents-session/src/transcript/view/cache.rs deleted file mode 100644 index 867542952..000000000 --- a/crates/tinyagents-session/src/transcript/view/cache.rs +++ /dev/null @@ -1,157 +0,0 @@ -//! In-memory, mtime-keyed projection cache for the transcript view. -//! -//! The settled transcript is derived from `session_raw/*.jsonl` on every -//! request. Re-projecting a long thread on each page fetch is wasteful, so we -//! memoize per-thread projections keyed on the backing files' `(path, mtime, -//! len)` signature. An append (new turn, interrupted partial, sub-agent file) -//! changes the signature and transparently invalidates the entry — there are -//! **no disk writes** and no explicit invalidation call. The cache is bounded -//! (LRU over a few dozen threads) so a long-lived core can't grow it without -//! limit. - -use std::collections::HashMap; -use std::collections::VecDeque; -use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex, OnceLock}; -use std::time::SystemTime; - -use super::project; -use super::types::ProjectedTranscript; - -const LOG_PREFIX: &str = "[threads][transcript][cache]"; - -/// Number of distinct threads whose projections are retained. Beyond this the -/// least-recently-used entry is evicted. -const CACHE_CAPACITY: usize = 32; - -/// Signature of one backing file — changes on any append/rewrite. -#[derive(Debug, Clone, PartialEq, Eq)] -struct FileSig { - path: PathBuf, - mtime: Option, - len: u64, -} - -fn file_sig(path: &Path) -> FileSig { - let (mtime, len) = match std::fs::metadata(path) { - Ok(meta) => (meta.modified().ok(), meta.len()), - Err(_) => (None, 0), - }; - FileSig { - path: path.to_path_buf(), - mtime, - len, - } -} - -struct CacheEntry { - signature: Vec, - projected: Arc, -} - -#[derive(Default)] -struct CacheInner { - entries: HashMap, - /// LRU order — front is least-recently-used. - order: VecDeque, -} - -/// Bounded per-thread projection cache. -#[derive(Default)] -pub struct TranscriptViewCache { - inner: Mutex, -} - -impl TranscriptViewCache { - /// Project `thread_id`'s transcript, serving a cached result when the - /// backing files are unchanged. `None` when the thread has no transcript. - pub fn get_or_project( - &self, - workspace_dir: &Path, - thread_id: &str, - ) -> Option> { - let (root_paths, sub_paths) = project::resolve_files(workspace_dir, thread_id)?; - let signature: Vec = root_paths - .iter() - .map(|path| file_sig(path)) - .chain(sub_paths.iter().map(|p| file_sig(p))) - .collect(); - - { - let mut inner = self.inner.lock().ok()?; - if let Some(entry) = inner.entries.get(thread_id) { - if entry.signature == signature { - let projected = entry.projected.clone(); - touch(&mut inner, thread_id); - tracing::debug!( - "{LOG_PREFIX} hit thread={thread_id} items={}", - projected.items.len() - ); - return Some(projected); - } - tracing::debug!("{LOG_PREFIX} miss (signature changed) thread={thread_id}"); - } else { - tracing::debug!("{LOG_PREFIX} miss (cold) thread={thread_id}"); - } - } - - let projected = Arc::new(project::project_from_files( - thread_id, - &root_paths, - &sub_paths, - None, - )); - - let mut inner = self.inner.lock().ok()?; - inner.entries.insert( - thread_id.to_string(), - CacheEntry { - signature, - projected: projected.clone(), - }, - ); - touch(&mut inner, thread_id); - evict_if_needed(&mut inner); - tracing::debug!( - "{LOG_PREFIX} stored thread={thread_id} items={} cache_size={}", - projected.items.len(), - inner.entries.len() - ); - Some(projected) - } - - #[cfg(test)] - fn len(&self) -> usize { - self.inner.lock().unwrap().entries.len() - } -} - -/// Move `thread_id` to the most-recently-used end of the LRU order. -fn touch(inner: &mut CacheInner, thread_id: &str) { - if let Some(pos) = inner.order.iter().position(|t| t == thread_id) { - inner.order.remove(pos); - } - inner.order.push_back(thread_id.to_string()); -} - -/// Evict least-recently-used entries until within [`CACHE_CAPACITY`]. -fn evict_if_needed(inner: &mut CacheInner) { - while inner.entries.len() > CACHE_CAPACITY { - let Some(victim) = inner.order.pop_front() else { - break; - }; - inner.entries.remove(&victim); - tracing::debug!("{LOG_PREFIX} evicted thread={victim}"); - } -} - -/// Process-wide cache singleton. The transcript view is read-only and derived, -/// so one shared cache across RPC calls is correct. -pub fn global() -> &'static TranscriptViewCache { - static CACHE: OnceLock = OnceLock::new(); - CACHE.get_or_init(TranscriptViewCache::default) -} - -#[cfg(test)] -#[path = "cache_tests.rs"] -mod tests; diff --git a/crates/tinyagents-session/src/transcript/view/cache_tests.rs b/crates/tinyagents-session/src/transcript/view/cache_tests.rs deleted file mode 100644 index 014483c8b..000000000 --- a/crates/tinyagents-session/src/transcript/view/cache_tests.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! Cache hit/miss + invalidation tests for the transcript view cache. - -use super::TranscriptViewCache; -use crate::transcript; -use std::path::Path; -use tempfile::TempDir; - -fn meta_line(thread_id: &str) -> String { - format!( - r#"{{"_meta":{{"version":1,"agent":"orchestrator","dispatcher":"native","created":"2026-07-21T00:00:00Z","updated":"2026-07-21T00:00:00Z","turn_count":1,"input_tokens":0,"output_tokens":0,"cached_input_tokens":0,"charged_amount_usd":0.0,"thread_id":"{thread_id}"}}}}"# - ) -} - -fn write_raw(workspace: &Path, stem: &str, thread_id: &str, body: &[&str]) -> std::path::PathBuf { - let path = transcript::resolve_keyed_transcript_path(workspace, stem).unwrap(); - let mut buf = meta_line(thread_id); - buf.push('\n'); - for line in body { - buf.push_str(line); - buf.push('\n'); - } - std::fs::write(&path, buf).unwrap(); - path -} - -#[test] -fn recomputes_when_file_grows() { - let dir = TempDir::new().unwrap(); - let path = write_raw( - dir.path(), - "100_orchestrator", - "thr_cache", - &[r#"{"role":"user","content":"first"}"#], - ); - let cache = TranscriptViewCache::default(); - - let a = cache - .get_or_project(dir.path(), "thr_cache") - .expect("first"); - assert_eq!(a.items.len(), 1); - - // Second call, unchanged file → same cached Arc (hit). - let b = cache - .get_or_project(dir.path(), "thr_cache") - .expect("second"); - assert!( - std::sync::Arc::ptr_eq(&a, &b), - "unchanged file must serve cached Arc" - ); - - // Append a line → file length changes → signature invalidates → recompute. - let mut appended = std::fs::read_to_string(&path).unwrap(); - appended.push_str(r#"{"role":"assistant","content":"second"}"#); - appended.push('\n'); - std::fs::write(&path, appended).unwrap(); - - let c = cache - .get_or_project(dir.path(), "thr_cache") - .expect("third"); - assert!(!std::sync::Arc::ptr_eq(&a, &c), "grown file must recompute"); - assert_eq!( - c.items.len(), - 2, - "recomputed projection reflects the append" - ); -} - -#[test] -fn missing_thread_returns_none() { - let dir = TempDir::new().unwrap(); - let cache = TranscriptViewCache::default(); - assert!(cache.get_or_project(dir.path(), "ghost").is_none()); - assert_eq!(cache.len(), 0); -} diff --git a/crates/tinyagents-session/src/transcript/view/mod.rs b/crates/tinyagents-session/src/transcript/view/mod.rs index b75a385fb..0c488b0ef 100644 --- a/crates/tinyagents-session/src/transcript/view/mod.rs +++ b/crates/tinyagents-session/src/transcript/view/mod.rs @@ -1,25 +1,22 @@ -//! Transcript-derived view: project the append-only `session_raw/*.jsonl` -//! source of truth into typed display items for the chat renderer, with -//! newest-first pagination over a bounded in-memory cache. +//! Transcript-derived view: project a thread's append-only transcripts into typed display items for the chat renderer, with +//! uncached newest-first pagination. //! -//! Entry point: [`get_page`] (used by the `threads.transcript_get` RPC). +//! Entry point: [`get_page_in`] (used by the `threads.transcript_get` RPC). -mod cache; mod project; mod prompt_tools; mod resolve; mod status_map; mod subagents; +#[cfg(test)] +mod test_shims; pub mod types; -use std::path::Path; - use serde::Serialize; -pub use project::{ - project_records, project_thread, project_thread_scoped, project_thread_scoped_with, - project_transcripts, resolve_files_scoped, -}; +pub use project::{project_records, project_thread_in, project_transcripts}; +#[cfg(test)] +pub(crate) use test_shims::{get_page, project_thread_scoped_with}; use crate::transcript::TranscriptLocator; #[allow(deprecated)] @@ -80,42 +77,14 @@ pub struct TranscriptPage { pub has_transcript: bool, } -/// Project `thread_id`'s transcript and return one newest-first page. +/// Projects `thread_id`'s transcripts (from +/// [`TranscriptLocator::thread_transcripts`], scoped to `agent_id` when given) +/// and returns one newest-first page. /// /// `cursor` is the opaque token returned as `next_cursor` by a previous call /// (an offset from the newest item); `None`/empty starts at the newest item. -/// `limit` defaults to [`DEFAULT_LIMIT`] and is clamped to [`MAX_LIMIT`]. -pub fn get_page( - workspace_dir: &Path, - thread_id: &str, - cursor: Option<&str>, - limit: Option, -) -> TranscriptPage { - get_page_scoped(workspace_dir, thread_id, None, cursor, limit) -} - -/// Fetch a page scoped to an owning agent. Use this when thread IDs can be -/// supplied by callers and shared by multiple agents. -pub fn get_page_scoped( - workspace_dir: &Path, - thread_id: &str, - agent_id: Option<&str>, - cursor: Option<&str>, - limit: Option, -) -> TranscriptPage { - let projected = if let Some(agent_id) = agent_id { - project::project_thread_scoped(workspace_dir, thread_id, Some(agent_id)) - .map(std::sync::Arc::new) - } else { - cache::global().get_or_project(workspace_dir, thread_id) - }; - page_of(thread_id, projected.as_deref(), cursor, limit) -} - -/// [`get_page_scoped`] over any [`TranscriptLocator`]: the thread's -/// transcripts come from -/// [`TranscriptLocator::thread_transcripts`] and are projected exactly as the -/// file view projects them. Not cached. +/// `limit` defaults to [`DEFAULT_LIMIT`] and is clamped to [`MAX_LIMIT`]. Not +/// cached. pub fn get_page_in( locator: &dyn TranscriptLocator, thread_id: &str, diff --git a/crates/tinyagents-session/src/transcript/view/project.rs b/crates/tinyagents-session/src/transcript/view/project.rs index 5b3fe3f18..efaa0707f 100644 --- a/crates/tinyagents-session/src/transcript/view/project.rs +++ b/crates/tinyagents-session/src/transcript/view/project.rs @@ -2,20 +2,18 @@ //! //! Turns the append-only log's [`DisplayRecord`]s (message lines, compaction //! markers, interrupted partials) into the frontend's chat vocabulary -//! ([`DisplayItem`]), sanitizing injected scaffolding as it goes. File -//! resolution lives in [`super::resolve`]; sub-agent trails are placed by +//! ([`DisplayItem`]), sanitizing injected scaffolding as it goes. Cross-generation row identity lives in [`super::resolve`]; sub-agent trails are placed by //! [`super::subagents`]. use std::collections::{HashMap, HashSet, VecDeque}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use crate::transcript::history::seed_meta_for_discovered; use crate::transcript::{ - CompactionMarker, DisplayMessage, DisplayRecord, DisplaySessionTranscript, - FileTranscriptHistory, ThreadTranscript, TranscriptRead, + CompactionMarker, DisplayMessage, DisplayRecord, DisplaySessionTranscript, ThreadTranscript, + TranscriptLocator, }; +#[cfg(test)] +pub(crate) use super::test_shims::{project_from_files, project_thread, resolve_files}; use super::{SpawnCallLookup, TOOL_RESULT_FAILURES_METADATA_KEY}; use super::prompt_tools::{self, CallRegistry, PromptToolResult}; @@ -37,95 +35,29 @@ const CHANNEL_CONTEXT_PREFIX: &str = "[Channel context]"; type NativeToolCall = (String, String, String); type NativeToolEnvelope = (String, Vec); -/// Resolve a thread's root transcript, discover its sub-agent siblings, and -/// project everything into display items. Returns `None` when the thread has -/// no root transcript yet (brand-new thread / first turn not persisted). -pub fn project_thread(workspace_dir: &Path, thread_id: &str) -> Option { - project_thread_scoped(workspace_dir, thread_id, None) -} - -/// Project only the transcript roots owned by `agent_id` when supplied. -pub fn project_thread_scoped( - workspace_dir: &Path, - thread_id: &str, - agent_id: Option<&str>, -) -> Option { - let (root_paths, sub_paths) = - resolve::resolve_files_scoped(workspace_dir, thread_id, agent_id)?; - Some(project_from_files(thread_id, &root_paths, &sub_paths, None)) -} - -/// [`project_thread_scoped`] with a [`SpawnCallLookup`], so a sub-agent is -/// placed under the exact tool call that spawned it when the run ledger knows -/// which one that was, instead of by the timestamp/target heuristic. -pub fn project_thread_scoped_with( - workspace_dir: &Path, +/// Lists `thread_id`'s transcripts through `locator` (scoped to `agent_id` +/// when given) and projects them. `None` when the thread has no root +/// transcript yet (brand-new thread, first turn not persisted). +/// +/// With a [`SpawnCallLookup`], a sub-agent is placed under the exact tool call +/// that spawned it when the run ledger knows which one that was, instead of by +/// the timestamp/target heuristic. +pub fn project_thread_in( + locator: &dyn TranscriptLocator, thread_id: &str, agent_id: Option<&str>, - lookup: &dyn SpawnCallLookup, + lookup: Option<&dyn SpawnCallLookup>, ) -> Option { - let (root_paths, sub_paths) = - resolve::resolve_files_scoped(workspace_dir, thread_id, agent_id)?; - Some(project_from_files( + let listed = locator.thread_transcripts(thread_id, agent_id)?; + Some(project_transcripts( thread_id, - &root_paths, - &sub_paths, - Some(lookup), + &listed.roots, + &listed.subagents, + lookup, )) } -/// Resolve the on-disk file set backing a thread's transcript view: the root -/// generations in chain order plus every sub-agent sibling file. `None` when -/// the thread has no root transcript yet. Exposed so the cache can key on -/// these paths (and their mtimes/lengths) without re-projecting. -pub fn resolve_files( - workspace_dir: &Path, - thread_id: &str, -) -> Option<(Vec, Vec)> { - resolve::resolve_files(workspace_dir, thread_id) -} - -/// Resolve the files for one agent's transcript when `agent_id` is supplied. -pub fn resolve_files_scoped( - workspace_dir: &Path, - thread_id: &str, - agent_id: Option<&str>, -) -> Option<(Vec, Vec)> { - resolve::resolve_files_scoped(workspace_dir, thread_id, agent_id) -} - -/// Project a thread from an already-resolved file set (root generations + -/// sub-agent siblings). Missing/unreadable files degrade to empty rather than -/// failing. -/// -/// A root whose `_meta.parent_session_id` names the previous root is that -/// session's next compaction generation: it opens with the retained set -/// rewritten, so those rows are dropped (see [`resolve::drop_retained_rows`]) -/// and a [`DisplayItem::Compaction`] marks the seam instead. -pub fn project_from_files( - thread_id: &str, - root_paths: &[PathBuf], - sub_paths: &[PathBuf], - lookup: Option<&dyn SpawnCallLookup>, -) -> ProjectedTranscript { - let handles = |paths: &[PathBuf]| -> Vec { - paths - .iter() - .map(|path| { - // The same stem rule as `FileTranscriptLocator::thread_transcripts` - // (strips only `.jsonl`), so both paths name a file alike. - let stem = crate::transcript::history::stem_of_handle(path); - let transcript: Arc = Arc::new( - FileTranscriptHistory::opened_at(path.clone(), seed_meta_for_discovered(&stem)), - ); - ThreadTranscript { stem, transcript } - }) - .collect() - }; - project_transcripts(thread_id, &handles(root_paths), &handles(sub_paths), lookup) -} - -/// `project_from_files` over a listed transcript set (see +/// Projects a thread from a listed transcript set (see /// [`TranscriptLocator::thread_transcripts`](crate::transcript::TranscriptLocator::thread_transcripts)), /// so any store projects the same way. Unreadable transcripts degrade to /// empty rather than failing. diff --git a/crates/tinyagents-session/src/transcript/view/resolve.rs b/crates/tinyagents-session/src/transcript/view/resolve.rs index 32e53f0a3..70f0f4c15 100644 --- a/crates/tinyagents-session/src/transcript/view/resolve.rs +++ b/crates/tinyagents-session/src/transcript/view/resolve.rs @@ -1,225 +1,15 @@ -//! Which files back a thread's transcript view, and in what order. +//! Cross-generation row identity for a thread's transcript view. //! -//! Two things make this more than "every root whose `_meta.thread_id` -//! matches": -//! -//! - **Session generations.** A compaction seals generation `n` and opens -//! `{stem}.g{n+1}`, which inherits `_meta.created` and opens with the -//! retained message set rewritten. Ordering by `created` then path put -//! `X.g1` before `X` (and `.g10` before `.g2`), and concatenating every -//! generation rendered the retained rows twice. Generations are ordered by -//! the tinyagents `session_chain` instead, and [`drop_retained_rows`] removes -//! the rewritten prefix when a successor is projected. -//! - **Adopted legacy roots.** A pre-identity conversation's timestamped roots -//! are folded into the session file on first resume and left untouched on -//! disk, so once a session file exists they are duplicates of its head. -//! -//! Sub-agent files are discovered by `_meta.thread_id` as well as by the -//! legacy `{root_stem}__` prefix: a sub-agent's stem chains the parent's -//! *session key* (`{unix}_{agent}`), which for a session-identity thread is not -//! the root file's stem (`{thread}.{agent}` with digests), so the prefix alone -//! found none of them. - -use std::collections::{HashMap, HashSet}; -use std::fs; -use std::io::{BufRead, BufReader}; -use std::path::{Path, PathBuf}; - -use crate::transcript::{ - self, DisplayMessage, DisplayRecord, FileTranscriptLocator, SessionRef, TranscriptLocator, -}; - -const LOG_PREFIX: &str = "[threads][transcript][resolve]"; - -/// The `_meta` header fields resolution needs, read from a file's first line -/// only (the full reader parses the whole file). -#[derive(Debug, Default, Clone)] -pub(super) struct HeadMeta { - pub(super) thread_id: Option, - pub(super) agent_id: Option, - pub(super) session_id: Option, -} - -/// Read the first-line `_meta` header of a transcript. `None` when the file -/// cannot be read or does not start with a meta line. -pub(super) fn read_head_meta(path: &Path) -> Option { - let file = fs::File::open(path).ok()?; - let mut first = String::new(); - BufReader::new(file).read_line(&mut first).ok()?; - let value: serde_json::Value = serde_json::from_str(first.trim()).ok()?; - let meta = value.get("_meta")?; - let field = |key: &str| { - meta.get(key) - .and_then(serde_json::Value::as_str) - .map(str::to_string) - }; - Some(HeadMeta { - thread_id: field("thread_id"), - agent_id: field("agent_id"), - session_id: field("session_id"), - }) -} - -/// Resolve the root generations and sub-agent siblings backing `thread_id`. -/// `None` when the thread has no root transcript yet. -pub fn resolve_files( - workspace_dir: &Path, - thread_id: &str, -) -> Option<(Vec, Vec)> { - resolve_files_scoped(workspace_dir, thread_id, None) -} - -pub(super) fn resolve_files_scoped( - workspace_dir: &Path, - thread_id: &str, - agent_id: Option<&str>, -) -> Option<(Vec, Vec)> { - let mut found = transcript::find_root_transcripts_for_thread(workspace_dir, thread_id); - if let Some(agent_id) = agent_id { - found.retain(|path| { - read_head_meta(path) - .and_then(|meta| meta.agent_id) - .as_deref() - == Some(agent_id) - }); - } - if found.is_empty() { - return None; - } - let raw_dir = found[0].parent()?.to_path_buf(); - let roots = order_root_files(workspace_dir, thread_id, found.clone()); - // Discover sub-agents from every matched root, including legacy roots the - // session-chain ordering drops, so their children still render. - let subs = discover_subagent_files(&raw_dir, thread_id, &found, agent_id.is_some()); - tracing::debug!( - "{LOG_PREFIX} thread={thread_id} roots={} subagent_files={}", - roots.len(), - subs.len() - ); - Some((roots, subs)) -} +//! A compaction seals generation `n` and opens `{stem}.g{n+1}`, which inherits +//! `_meta.created` and opens with the retained message set rewritten. +//! Concatenating every generation would render the retained rows twice, so +//! [`drop_retained_rows`] removes the rewritten prefix when a successor is +//! projected (generations themselves arrive in chain order from the locator's +//! [`thread_transcripts`](crate::transcript::TranscriptLocator::thread_transcripts)). -fn file_name(path: &Path) -> Option<&std::ffi::OsStr> { - path.file_name() -} - -/// Order the thread's roots: each session's generations oldest-first (from -/// `session_chain`), legacy (pre-identity) roots dropped once a session file -/// exists. A thread with no session file keeps the scan's order unchanged. -fn order_root_files(workspace_dir: &Path, thread_id: &str, roots: Vec) -> Vec { - let metas: Vec<(PathBuf, HeadMeta)> = roots - .into_iter() - .map(|path| { - let meta = read_head_meta(&path).unwrap_or_default(); - (path, meta) - }) - .collect(); - if !metas.iter().any(|(_, meta)| meta.session_id.is_some()) { - return metas.into_iter().map(|(path, _)| path).collect(); - } +use std::collections::HashMap; - let locator = FileTranscriptLocator::new(workspace_dir); - let mut seen: HashSet = HashSet::new(); - let mut ordered = Vec::new(); - let mut dropped_legacy = 0usize; - for (path, meta) in &metas { - if meta.session_id.is_none() { - dropped_legacy += 1; - continue; - } - let Some(name) = file_name(path) else { - continue; - }; - if seen.contains(name) { - continue; - } - let chain: Vec = meta - .agent_id - .as_deref() - .map(|agent_id| { - let session = SessionRef::scoped(thread_id, agent_id); - locator - .session_chain(&session) - .iter() - .filter_map(|generation| { - transcript::resolve_keyed_transcript_path( - workspace_dir, - &transcript::session_stem(generation), - ) - .ok() - }) - .collect() - }) - .unwrap_or_default(); - if chain.iter().any(|link| file_name(link) == Some(name)) { - for link in chain { - if let Some(link_name) = file_name(&link) - && seen.insert(link_name.to_os_string()) - { - ordered.push(link); - } - } - } else { - // A session file this thread's own identity does not derive (a - // different key scheme): keep it where the scan put it. - seen.insert(name.to_os_string()); - ordered.push(path.clone()); - } - } - if dropped_legacy > 0 { - tracing::debug!( - "{LOG_PREFIX} thread={thread_id} dropped {dropped_legacy} adopted legacy root(s) \ - in favour of the session chain" - ); - } - ordered -} - -/// Every sub-agent (`__`) transcript of this thread in `raw_dir`: those whose -/// stem extends a root stem (legacy layout), plus those whose `_meta.thread_id` -/// names the thread (session-identity layout). Sorted by path. -fn discover_subagent_files( - raw_dir: &Path, - thread_id: &str, - roots: &[PathBuf], - scoped: bool, -) -> Vec { - let prefixes: Vec = roots - .iter() - .filter_map(|root| root.file_stem().and_then(|stem| stem.to_str())) - .map(|stem| format!("{stem}__")) - .collect(); - let entries = match fs::read_dir(raw_dir) { - Ok(entries) => entries, - Err(error) => { - tracing::debug!( - "{LOG_PREFIX} subagent discovery read_dir failed dir={} error={error}", - raw_dir.display() - ); - return Vec::new(); - } - }; - let mut paths: Vec = entries - .flatten() - .map(|entry| entry.path()) - .filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some("jsonl")) - .filter(|path| { - let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) else { - return false; - }; - if !stem.contains("__") { - return false; - } - prefixes.iter().any(|prefix| stem.starts_with(prefix)) - || (!scoped - && read_head_meta(path) - .and_then(|meta| meta.thread_id) - .is_some_and(|id| id == thread_id)) - }) - .collect(); - paths.sort(); - paths -} +use crate::transcript::{DisplayMessage, DisplayRecord}; /// Identity of one message row for cross-generation de-duplication. type RowKey = (String, String, Option, Option, String); diff --git a/crates/tinyagents-session/src/transcript/view/test_shims.rs b/crates/tinyagents-session/src/transcript/view/test_shims.rs new file mode 100644 index 000000000..efceba10e --- /dev/null +++ b/crates/tinyagents-session/src/transcript/view/test_shims.rs @@ -0,0 +1,104 @@ +//! Test-only workspace-path shims over the locator-based view API. +//! +//! View fixtures are written as raw classic JSONL under a temp workspace; each +//! shim imports that workspace into an in-memory locator and calls the +//! store-port function, so the projection under test is the production one. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use super::types::ProjectedTranscript; +use super::{SpawnCallLookup, TranscriptPage, get_page_in, project}; +use crate::transcript::history::seed_meta_for_discovered; +use crate::transcript::test_locator::imported_locator; +use crate::transcript::{ThreadTranscript, TranscriptRead}; + +pub(crate) fn get_page( + workspace_dir: &Path, + thread_id: &str, + cursor: Option<&str>, + limit: Option, +) -> TranscriptPage { + get_page_in( + imported_locator(workspace_dir).as_ref(), + thread_id, + None, + cursor, + limit, + ) +} + +pub(crate) fn project_thread(workspace_dir: &Path, thread_id: &str) -> Option { + project::project_thread_in( + imported_locator(workspace_dir).as_ref(), + thread_id, + None, + None, + ) +} + +pub(crate) fn project_thread_scoped_with( + workspace_dir: &Path, + thread_id: &str, + agent_id: Option<&str>, + lookup: &dyn SpawnCallLookup, +) -> Option { + project::project_thread_in( + imported_locator(workspace_dir).as_ref(), + thread_id, + agent_id, + Some(lookup), + ) +} + +/// The transcript files (as `session_raw/.jsonl` paths) the locator +/// lists for `thread_id`: root generations in chain order, then sub-agents. +pub(crate) fn resolve_files( + workspace_dir: &Path, + thread_id: &str, +) -> Option<(Vec, Vec)> { + let listed = imported_locator(workspace_dir).thread_transcripts(thread_id, None)?; + let path = |t: &ThreadTranscript| { + workspace_dir + .join("session_raw") + .join(format!("{}.jsonl", t.stem)) + }; + Some(( + listed.roots.iter().map(path).collect(), + listed.subagents.iter().map(path).collect(), + )) +} + +/// Projects the given fixture files (imported into one locator) as a thread's +/// roots and sub-agents. +pub(crate) fn project_from_files( + thread_id: &str, + root_paths: &[PathBuf], + sub_paths: &[PathBuf], + lookup: Option<&dyn SpawnCallLookup>, +) -> ProjectedTranscript { + let workspace = root_paths + .first() + .or_else(|| sub_paths.first()) + .and_then(|path| path.parent()) + .and_then(Path::parent) + .expect("fixture files live under /session_raw"); + let locator = imported_locator(workspace); + let handles = |paths: &[PathBuf]| -> Vec { + paths + .iter() + .map(|path| { + let stem = path + .file_stem() + .and_then(|stem| stem.to_str()) + .expect("utf-8 stem") + .to_owned(); + let transcript: Arc = locator + .open_stem(&stem, seed_meta_for_discovered(&stem)) + .expect("fixture stem imported"); + ThreadTranscript { stem, transcript } + }) + .collect() + }; + project::project_transcripts(thread_id, &handles(root_paths), &handles(sub_paths), lookup) +} diff --git a/crates/tinyagents-session/src/transcript/writer.rs b/crates/tinyagents-session/src/transcript/writer.rs index 4d9c9eb60..547fc6655 100644 --- a/crates/tinyagents-session/src/transcript/writer.rs +++ b/crates/tinyagents-session/src/transcript/writer.rs @@ -2,11 +2,13 @@ //! delta (message tail or compaction record), interrupted partials, and the //! derived `.md` companion. +#[cfg(test)] use super::history::TranscriptPartial; +#[cfg(test)] use super::jsonl::{ - COMPACTION_KIND, CompactionLine, MessageLine, build_message_line, meta_line_json, partial_line, - serialise_message_lines, tools_line_json, + COMPACTION_KIND, CompactionLine, MessageLine, build_message_line, partial_line, }; +use super::jsonl::{meta_line_json, serialise_message_lines, tools_line_json}; use super::markdown::render_markdown; use super::paths::md_companion_path; use super::types::TranscriptMessage; @@ -24,7 +26,8 @@ use std::sync::atomic::{AtomicU64, Ordering}; /// one-shot writer used by migrations, the sub-agent runners, and tests. /// The incremental session-persistence path uses [`append_transcript_turn`] /// instead, which never rewrites existing lines. -pub fn write_transcript( +#[cfg(test)] +pub(crate) fn write_transcript( jsonl_path: &Path, messages: &[TranscriptMessage], meta: &TranscriptMeta, @@ -77,7 +80,8 @@ pub(crate) fn write_transcript_with_tools( /// wants. Returns `Ok(true)` when this call created `jsonl_path`, `Ok(false)` /// when it already existed (some other write already won the race and this /// call's `messages`/`meta` were discarded). -pub fn write_transcript_if_absent( +#[cfg(test)] +pub(crate) fn write_transcript_if_absent( jsonl_path: &Path, messages: &[TranscriptMessage], meta: &TranscriptMeta, @@ -87,7 +91,8 @@ pub fn write_transcript_if_absent( /// Like [`write_transcript_if_absent`], additionally preserving the latest /// recorded tool declarations when a transcript is migrated into a session. -pub fn write_transcript_if_absent_with_tools( +#[cfg(test)] +pub(crate) fn write_transcript_if_absent_with_tools( jsonl_path: &Path, messages: &[TranscriptMessage], meta: &TranscriptMeta, @@ -146,7 +151,8 @@ pub fn write_transcript_if_absent_with_tools( /// /// `request_id`, when supplied by the caller, is stamped on every appended /// line as a turn boundary marker. -pub fn append_transcript_turn( +#[cfg(test)] +pub(crate) fn append_transcript_turn( jsonl_path: &Path, prev_persisted: &[TranscriptMessage], messages: &[TranscriptMessage], @@ -168,38 +174,8 @@ pub fn append_transcript_turn( ) } -/// Appends one logical turn and, when present, its display-only interruption -/// row from one serialized buffer and one file-write operation. -/// -/// The partial is written after the logical delta and refreshed metadata. It -/// has `interrupted: true`, so the model-context reader skips it while the -/// display reader preserves it. Serialization happens before opening the file -/// for append, preventing a serialization failure from leaving a logical turn -/// without its associated display partial. -pub fn append_transcript_turn_with_partial( - jsonl_path: &Path, - prev_persisted: &[TranscriptMessage], - messages: &[TranscriptMessage], - meta: &TranscriptMeta, - turn_usage: Option<&TurnUsage>, - request_id: Option<&str>, - partial: Option<&TranscriptPartial>, -) -> Result<()> { - append_transcript_turn_with_extras( - jsonl_path, - prev_persisted, - messages, - meta, - turn_usage, - request_id, - AppendTranscriptExtras { - partial, - tools: None, - }, - ) -} - /// Optional turn records serialized alongside a turn's logical delta. +#[cfg(test)] pub(crate) struct AppendTranscriptExtras<'a> { pub partial: Option<&'a TranscriptPartial>, pub tools: Option<&'a serde_json::Value>, @@ -207,6 +183,7 @@ pub(crate) struct AppendTranscriptExtras<'a> { /// Appends a turn, optional display partial, and optional tool declarations /// from one serialized buffer and one file-write operation. +#[cfg(test)] pub(crate) fn append_transcript_turn_with_extras( jsonl_path: &Path, prev_persisted: &[TranscriptMessage], @@ -322,7 +299,8 @@ pub(crate) fn append_transcript_turn_with_extras( /// Appends a `{"kind":"tools"}` record naming the tool declarations the /// session's latest turn was sent with. The file must already exist (its /// first line is always `_meta`), so callers append this after the turn. -pub fn append_tools_record(jsonl_path: &Path, tools: &serde_json::Value) -> Result<()> { +#[cfg(test)] +pub(crate) fn append_tools_record(jsonl_path: &Path, tools: &serde_json::Value) -> Result<()> { anyhow::ensure!( jsonl_path.is_file(), "transcript must exist before appending tool declarations: {}", @@ -340,6 +318,7 @@ pub fn append_tools_record(jsonl_path: &Path, tools: &serde_json::Value) -> Resu } /// Appends the optional display-only row to an already assembled turn buffer. +#[cfg(test)] fn serialise_interrupted_partial( partial: Option<&TranscriptPartial>, request_id: Option<&str>, @@ -362,7 +341,8 @@ fn serialise_interrupted_partial( /// resumed context never carries a truncated answer. Does not affect the /// caller's tracked `prev_persisted` (nothing about the logical model context /// changed). No-op when `partial_content` is empty. -pub fn append_interrupted_partial( +#[cfg(test)] +pub(crate) fn append_interrupted_partial( jsonl_path: &Path, partial_content: &str, request_id: Option<&str>, @@ -402,6 +382,7 @@ pub fn append_interrupted_partial( /// intentionally excluded because /// it is enriched (turn usage) between the in-memory history and the persisted /// line, which must not count as a divergence. +#[cfg(test)] fn common_prefix_len(a: &[TranscriptMessage], b: &[TranscriptMessage]) -> usize { a.iter() .zip(b.iter()) @@ -509,6 +490,7 @@ fn atomic_write(path: &Path, contents: &[u8]) -> Result<()> { /// holds the honor of "first write" when it finishes — never clobber a /// conversation's genuine first turn with an adoption fold that started /// scanning before that turn existed. +#[cfg(test)] fn publish_transcript_if_absent(path: &Path, contents: &[u8]) -> Result { let tmp_path = unique_tmp_path(path); @@ -534,6 +516,7 @@ fn publish_transcript_if_absent(path: &Path, contents: &[u8]) -> Result { } /// Append raw bytes to a file, opening in append mode (O(1), no read-back). +#[cfg(test)] pub(super) fn append_bytes(path: &Path, bytes: &[u8]) -> Result<()> { use std::io::{Read, Seek, SeekFrom, Write}; let mut file = fs::OpenOptions::new() diff --git a/crates/tinyagents-session/src/turn_state/mirror/mod_tests.rs b/crates/tinyagents-session/src/turn_state/mirror/mod_tests.rs index 7d62cdaa3..7a9b746ee 100644 --- a/crates/tinyagents-session/src/turn_state/mirror/mod_tests.rs +++ b/crates/tinyagents-session/src/turn_state/mirror/mod_tests.rs @@ -2,15 +2,13 @@ //! ordering keys and interrupted-turn finalization. use super::*; +use crate::port::InMemoryTranscriptLocator; use crate::port::InMemoryTurnStates; -use crate::transcript::FileTranscriptLocator; use crate::transcript::{ - self, DisplayRecord, TranscriptMessage, TranscriptMeta, read_transcript, - read_transcript_display, + DisplayRecord, TranscriptHistory, TranscriptLocator, TranscriptMessage, TranscriptMeta, }; use crate::turn_state::types::{SubagentActivity, ToolTimelineEntry, ToolTimelineStatus}; use crate::turn_state::types::{TranscriptItem, TurnLifecycle}; -use tempfile::tempdir; fn memory_store() -> Arc { Arc::new(InMemoryTurnStates::default()) @@ -22,15 +20,18 @@ fn fresh(thread_id: &str) -> (Arc, TurnStateMirror) { (store, mirror) } -/// A mirror whose interrupted partials land in the transcripts under `dir`. +/// A mirror whose interrupted partials land in `transcripts`. fn mirror_over( - dir: &std::path::Path, + transcripts: &Arc, store: Arc, thread_id: &str, request_id: &str, ) -> TurnStateMirror { - TurnStateMirror::new(store, thread_id, request_id) - .with_transcripts(Arc::new(FileTranscriptLocator::new(dir)), None) + TurnStateMirror::new(store, thread_id, request_id).with_transcripts(transcripts.clone(), None) +} + +fn transcripts() -> Arc { + Arc::new(InMemoryTranscriptLocator::new("mirror-tests")) } /// A store whose conditional write cannot read the stored outcome. @@ -85,9 +86,10 @@ impl TurnStates for UnreadableOutcome { } } -fn seed_root_transcript(workspace: &std::path::Path, thread_id: &str) -> std::path::PathBuf { - let path = transcript::resolve_keyed_transcript_path(workspace, "100_orchestrator") - .expect("resolve path"); +fn seed_root_transcript( + transcripts: &InMemoryTranscriptLocator, + thread_id: &str, +) -> Arc { let meta = TranscriptMeta { session_id: None, parent_session_id: None, @@ -108,14 +110,13 @@ fn seed_root_transcript(workspace: &std::path::Path, thread_id: &str) -> std::pa thread_id: Some(thread_id.to_string()), task_id: None, }; - transcript::write_transcript( - &path, - &[TranscriptMessage::new("user", "hello there")], - &meta, - None, - ) - .expect("seed transcript"); - path + let history = transcripts + .open_stem("100_orchestrator", meta) + .expect("open transcript"); + history + .append(TranscriptMessage::new("user", "hello there")) + .expect("seed transcript"); + history } #[test] @@ -221,9 +222,9 @@ fn subagent_prose_coalesces_per_kind_and_iteration() { #[test] fn finish_marks_an_unfinished_turn_interrupted() { - let dir = tempdir().expect("tempdir"); + let transcripts = transcripts(); let store = memory_store(); - let mut m = mirror_over(dir.path(), store.clone(), "thr_new", "req-1"); + let mut m = mirror_over(&transcripts, store.clone(), "thr_new", "req-1"); m.state.streaming_text = "orphan partial".into(); // No transcript exists yet: must not panic, snapshot keeps the partial. m.finish(); @@ -237,15 +238,18 @@ fn finish_marks_an_unfinished_turn_interrupted() { #[test] fn finish_appends_interrupted_partial_to_existing_transcript() { - let dir = tempdir().expect("tempdir"); - let path = seed_root_transcript(dir.path(), "thr_abc"); - let mut m = mirror_over(dir.path(), memory_store(), "thr_abc", "req-9"); + let transcripts = transcripts(); + let path = seed_root_transcript(&transcripts, "thr_abc"); + let mut m = mirror_over(&transcripts, memory_store(), "thr_abc", "req-9"); m.state.iteration = 2; m.state.thinking = "hmm".into(); m.state.streaming_text = "half an answer".into(); m.finish(); - let model = read_transcript(&path).expect("read model context"); + let model = path + .read_session() + .expect("read model context") + .expect("transcript present"); assert!( !model .messages @@ -253,7 +257,10 @@ fn finish_appends_interrupted_partial_to_existing_transcript() { .any(|msg| msg.content.contains("half an answer")), "interrupted partial must be excluded from the model context" ); - let display = read_transcript_display(&path).expect("read display"); + let display = path + .read_display() + .expect("read display") + .expect("transcript present"); let partial = display .records .iter() @@ -270,13 +277,16 @@ fn finish_appends_interrupted_partial_to_existing_transcript() { #[test] fn finish_after_completion_writes_no_partial() { - let dir = tempdir().expect("tempdir"); - let path = seed_root_transcript(dir.path(), "thr_done"); - let mut m = mirror_over(dir.path(), memory_store(), "thr_done", "req-done"); + let transcripts = transcripts(); + let path = seed_root_transcript(&transcripts, "thr_done"); + let mut m = mirror_over(&transcripts, memory_store(), "thr_done", "req-done"); m.state.streaming_text = "final answer".into(); m.turn_completed = true; m.finish(); - let display = read_transcript_display(&path).expect("read display"); + let display = path + .read_display() + .expect("read display") + .expect("transcript present"); assert!( !display .records @@ -288,10 +298,10 @@ fn finish_after_completion_writes_no_partial() { #[test] fn finish_keeps_a_terminal_outcome_the_driver_already_recorded() { - let dir = tempdir().expect("tempdir"); - let path = seed_root_transcript(dir.path(), "thr_settled"); + let transcripts = transcripts(); + let path = seed_root_transcript(&transcripts, "thr_settled"); let store = memory_store(); - let mut m = mirror_over(dir.path(), store.clone(), "thr_settled", "req-s"); + let mut m = mirror_over(&transcripts, store.clone(), "thr_settled", "req-s"); m.state.streaming_text = "delivered reply".into(); // The turn driver settles the snapshot as completed behind the mirror's back. let mut settled = m.state.clone(); @@ -300,7 +310,10 @@ fn finish_keeps_a_terminal_outcome_the_driver_already_recorded() { m.finish(); let stored = store.get_turn("thr_settled", "req-s").unwrap().unwrap(); assert_eq!(stored.lifecycle, TurnLifecycle::Completed); - let display = read_transcript_display(&path).expect("read display"); + let display = path + .read_display() + .expect("read display") + .expect("transcript present"); assert!( !display .records @@ -312,13 +325,13 @@ fn finish_keeps_a_terminal_outcome_the_driver_already_recorded() { #[test] fn finish_leaves_an_unreadable_snapshot_alone_and_appends_no_partial() { - let dir = tempdir().expect("tempdir"); - let path = seed_root_transcript(dir.path(), "thr_corrupt"); + let transcripts = transcripts(); + let path = seed_root_transcript(&transcripts, "thr_corrupt"); let store = Arc::new(UnreadableOutcome { inner: InMemoryTurnStates::default(), unreadable: std::sync::atomic::AtomicBool::new(false), }); - let mut m = mirror_over(dir.path(), store.clone(), "thr_corrupt", "req-c"); + let mut m = mirror_over(&transcripts, store.clone(), "thr_corrupt", "req-c"); m.state.streaming_text = "maybe delivered".into(); store .unreadable @@ -330,7 +343,10 @@ fn finish_leaves_an_unreadable_snapshot_alone_and_appends_no_partial() { TurnLifecycle::Started, "an outcome that cannot be read must not be overwritten" ); - let display = read_transcript_display(&path).expect("read display"); + let display = path + .read_display() + .expect("read display") + .expect("transcript present"); assert!( !display .records diff --git a/docs/feature-e2e-test-matrix.md b/docs/feature-e2e-test-matrix.md index 2386927a1..d2d461589 100644 --- a/docs/feature-e2e-test-matrix.md +++ b/docs/feature-e2e-test-matrix.md @@ -25,7 +25,7 @@ tests remain valuable but do not close an E2E gap. | Feature | Evidence of gap | Test to add | | --- | --- | --- | | Hosted harness invocation | `feature_harness_hosted_invocation.rs` | Covers composed/screened input, host observer attribution, and public streamed failure sanitization. | -| Session transcript persistence | `feature_session_transcript.rs` | Covers append, compaction, interrupted partials, replay/display projections, Markdown, thread summaries, and `FileTranscriptHistory` reopen. | +| Session transcript persistence | `feature_session_transcript.rs` | Covers append, compaction, interrupted partials, replay/display projections, thread spend, and driver-backed history reopen. | | Registry runtime router | `ModelRouter`/`WorkloadRoute` only have crate-local tests | `feature_registry_router.rs`: route default and capability-gated workloads into harness model selection and verify primary failure falls back to the configured alternate. | | Definition-host boundary | `tinyagents-definition` has one inline unit test | `feature_definition_runtime.rs`: validate/serialize definitions and use them in a hosted delegated harness run, proving declared children succeed and undeclared children fail. | diff --git a/docs/modules/runtime/README.md b/docs/modules/runtime/README.md index 2ae290262..a6cb0c9e1 100644 --- a/docs/modules/runtime/README.md +++ b/docs/modules/runtime/README.md @@ -41,8 +41,8 @@ rows before any commit, replacing host-side shadow history. With a `TranscriptHistory`, a successful turn first passes `before_commit`, the pre-commit validation hook. The runtime then submits its logical transcript delta, metadata, and any supplied `TranscriptPartial` to one -`append_turn_with_partial` operation. `FileTranscriptHistory` serializes those -records into one buffer before one file write. The interrupted partial remains +`append_turn_with_partial` operation. The driver-backed history writes those +records as one entry in one conditional write. The interrupted partial remains in the display projection and is excluded from model-context replay. Custom history implementations that cannot make this combined operation reject diff --git a/docs/modules/session/store-port.md b/docs/modules/session/store-port.md index 4ab92e195..ee2c13330 100644 --- a/docs/modules/session/store-port.md +++ b/docs/modules/session/store-port.md @@ -54,11 +54,11 @@ implement them, so code generic over a store accepts injected handles, and - `InMemorySessionStores`: per-agent, process-lifetime stores (`InMemoryTranscriptLocator`, `InMemoryTurnStates`, the harness's in-memory stores). For tests and hosts that keep nothing. -- `FileTranscriptLocator` implements `TranscriptLocator` (the file turn-state - store is removed; use `InMemoryTurnStates` or `DriverTurnStates`); together - with `open_session_stores` it is the building block of a file-backed provider. This crate does not - assemble one: the host that owns a layout does (OpenHuman's - `openhuman_rpc::session_store`). +- There is no file-backed transcript locator: the path-based transcript API + (`FileTranscriptLocator`, `FileTranscriptHistory`, the workspace-path + readers and writers) is removed. A host uses `DriverSessionStores` (or + `WorkspaceStores`, which also performs the one-time read-only import of a + classic `session_raw/` directory) and `InMemorySessionStores` for tests. - `TranscriptLocator::append_interrupted_partial` carries the display-only partial of an interrupted turn without a file path. - `DriverSessionStores` (feature `storage-drivers`): every agent's stores in diff --git a/docs/spec/README.md b/docs/spec/README.md index 6020cca9c..41e97b3f7 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -206,7 +206,7 @@ Remaining open question: Transcript accounting preserves cost provenance in durable usage records. Hosts should use `TranscriptSpend::cost_split` to separate charged or estimated -amounts from unpriced usage, and `thread_spend` for full-history accounting. +amounts from unpriced usage, and `thread_spend_in` for full-history accounting. See the [transcript contract](../../crates/tinyagents-session/src/transcript/README.md#spend-and-cost-provenance-spendrs). - [Transcript execution projections](../../crates/tinyagents-runtime/src/transcript/README.md): mechanical typed-row conversion; durable codecs remain host-owned.