From 8b2faa383fa7cdd0a8d7c4478f8390726f81179e Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Fri, 4 Sep 2026 15:13:16 -0400 Subject: [PATCH 1/8] start etr1 frontier contracts --- crates/codestory-bench/Cargo.toml | 4 + .../codestory-bench/src/bin/codestory_etr1.rs | 193 ++++++++++++++++++ .../codestory-retrieval/src/lexical_index.rs | 4 +- crates/codestory-retrieval/src/lib.rs | 74 +++++++ 4 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 crates/codestory-bench/src/bin/codestory_etr1.rs diff --git a/crates/codestory-bench/Cargo.toml b/crates/codestory-bench/Cargo.toml index be2cb39b2..713656c2d 100644 --- a/crates/codestory-bench/Cargo.toml +++ b/crates/codestory-bench/Cargo.toml @@ -30,6 +30,10 @@ path = "src/bin/codestory_proof_availability.rs" name = "codestory-witness-seam" path = "src/bin/codestory_witness_seam.rs" +[[bin]] +name = "codestory-etr1" +path = "src/bin/codestory_etr1.rs" + [dev-dependencies] codestory-retrieval = { workspace = true, features = ["benchmark-support"] } criterion = { workspace = true } diff --git a/crates/codestory-bench/src/bin/codestory_etr1.rs b/crates/codestory-bench/src/bin/codestory_etr1.rs new file mode 100644 index 000000000..b4ecb750a --- /dev/null +++ b/crates/codestory-bench/src/bin/codestory_etr1.rs @@ -0,0 +1,193 @@ +//! Benchmark-only implementation of the frozen ETR-1 frontier experiment. + +use anyhow::{Result, ensure}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct ByteRangeV1 { + start: u64, + end: u64, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct LineRangeV1 { + start: u32, + end: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct FrozenFragmentV1 { + fragment_id: String, + project_id: String, + path: String, + content_digest: String, + byte_range: ByteRangeV1, + line_range: LineRangeV1, + source: String, + serialized_row_bytes: u32, +} + +fn sha256(bytes: impl AsRef<[u8]>) -> String { + format!("{:x}", Sha256::digest(bytes.as_ref())) +} + +fn fragment_id(project_id: &str, path: &str, content_digest: &str, range: ByteRangeV1) -> String { + let mut digest = Sha256::new(); + digest.update(b"codestory.frozen-fragment/v1\0"); + for value in [ + project_id.as_bytes(), + path.as_bytes(), + content_digest.as_bytes(), + ] { + digest.update((value.len() as u64).to_le_bytes()); + digest.update(value); + } + digest.update(range.start.to_le_bytes()); + digest.update(range.end.to_le_bytes()); + format!("{:x}", digest.finalize()) +} + +fn select_successors( + score_order: &[(String, f32)], + seeds: &BTreeSet, + prior: &BTreeSet, + limit: usize, +) -> Vec { + let mut selected = Vec::with_capacity(limit.min(score_order.len())); + let mut seen = BTreeSet::new(); + for (fragment_id, _) in score_order { + if !seeds.contains(fragment_id) && !prior.contains(fragment_id) && seen.insert(fragment_id) + { + selected.push(fragment_id.clone()); + if selected.len() == limit { + break; + } + } + } + selected +} + +fn candidate_query_with_shortening( + question: &str, + source: &str, + fits: F, +) -> Result<(String, u32)> +where + F: Fn(&str) -> bool, +{ + ensure!(!question.trim().is_empty(), "question_is_empty"); + ensure!(!source.is_empty(), "seed_source_is_empty"); + let lines = source.split_inclusive('\n').collect::>(); + ensure!(!lines.is_empty(), "seed_source_has_no_lines"); + for retained in (1..=lines.len()).rev() { + let retained_source = lines[..retained].concat(); + if retained_source.trim().is_empty() { + continue; + } + let query = format!("{question}\n\n{retained_source}"); + if fits(&query) { + return Ok((query, u32::try_from(lines.len() - retained)?)); + } + } + anyhow::bail!("no_complete_seed_source_line_fits") +} + +fn natural_seed_prefix(matches: &[T]) -> Vec { + matches.iter().take(16).cloned().collect() +} + +fn main() -> Result<()> { + anyhow::bail!("ETR-1 commands are not implemented yet") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fragment_identity_binds_project_path_digest_and_range() { + let range = ByteRangeV1 { start: 7, end: 19 }; + let baseline = fragment_id("project-a", "src/lib.rs", &"a".repeat(64), range); + assert_eq!(baseline.len(), 64); + assert_ne!( + baseline, + fragment_id("project-b", "src/lib.rs", &"a".repeat(64), range) + ); + assert_ne!( + baseline, + fragment_id("project-a", "src/main.rs", &"a".repeat(64), range) + ); + assert_ne!( + baseline, + fragment_id("project-a", "src/lib.rs", &"b".repeat(64), range) + ); + assert_ne!( + baseline, + fragment_id( + "project-a", + "src/lib.rs", + &"a".repeat(64), + ByteRangeV1 { start: 8, end: 19 } + ) + ); + } + + #[test] + fn natural_seed_prefix_preserves_underfill_and_order() { + assert_eq!(natural_seed_prefix(&[3, 1, 2]), vec![3, 1, 2]); + assert_eq!( + natural_seed_prefix(&(0..20).collect::>()), + (0..16).collect::>() + ); + } + + #[test] + fn cumulative_exclusions_produce_unique_successors() { + let scores = vec![ + ("seed".into(), 1.0), + ("prior".into(), 0.9), + ("new-a".into(), 0.8), + ("new-b".into(), 0.8), + ("new-c".into(), 0.7), + ]; + let selected = select_successors( + &scores, + &BTreeSet::from(["seed".into()]), + &BTreeSet::from(["prior".into()]), + 2, + ); + assert_eq!(selected, ["new-a", "new-b"]); + } + + #[test] + fn query_shortening_keeps_utf8_and_removes_complete_trailing_lines() { + let source = "first α line\nsecond β line\nthird γ line\n"; + let maximum = "question\n\nfirst α line\nsecond β line\n".len(); + let (query, removed) = + candidate_query_with_shortening("question", source, |value| value.len() <= maximum) + .unwrap(); + assert_eq!(query, "question\n\nfirst α line\nsecond β line\n"); + assert_eq!(removed, 1); + assert!(std::str::from_utf8(query.as_bytes()).is_ok()); + assert!(candidate_query_with_shortening("question", source, |_| false).is_err()); + } + + #[test] + fn lexical_contract_uses_the_product_normalizer_and_stop_words() { + assert_eq!( + codestory_retrieval::benchmark_support::etr1_lexical_document("HTTPServer run_pending"), + "http server run pending" + ); + assert_eq!( + codestory_retrieval::benchmark_support::etr1_lexical_query_terms( + "How does HTTPServer run_pending work?" + ), + vec!["http", "server", "run", "pending", "work"] + ); + } +} diff --git a/crates/codestory-retrieval/src/lexical_index.rs b/crates/codestory-retrieval/src/lexical_index.rs index f3fa40be2..36972b910 100644 --- a/crates/codestory-retrieval/src/lexical_index.rs +++ b/crates/codestory-retrieval/src/lexical_index.rs @@ -3917,7 +3917,7 @@ fn lexical_documents_hash(documents: &[LexicalDocument], coverage: &LexicalCover .hash } -fn normalize_lexical_text(value: &str) -> String { +pub(crate) fn normalize_lexical_text(value: &str) -> String { let mut normalized = String::with_capacity(value.len() + value.len() / 8); let mut characters = value.chars().peekable(); let mut previous: Option = None; @@ -4090,7 +4090,7 @@ fn fts_document_frequency(connection: &Connection, token: &str) -> Result .map_err(Into::into) } -fn lexical_query_tokens(query: &str) -> Vec { +pub(crate) fn lexical_query_tokens(query: &str) -> Vec { let mut tokens = Vec::new(); let normalized = normalize_lexical_text(query); for token in normalized diff --git a/crates/codestory-retrieval/src/lib.rs b/crates/codestory-retrieval/src/lib.rs index 7111ebc56..130ae3c01 100644 --- a/crates/codestory-retrieval/src/lib.rs +++ b/crates/codestory-retrieval/src/lib.rs @@ -67,6 +67,80 @@ pub mod benchmark_support { use anyhow::{Context, Result}; use std::path::{Path, PathBuf}; + #[derive(Debug, Clone, Copy, PartialEq)] + pub struct Etr1LexicalMatch { + pub rowid: usize, + pub score: f64, + } + + /// Frozen-fragment FTS5 surface for benchmark-only causal experiments. + /// It deliberately reuses the product's lexical normalization and stopword + /// policy while keeping the fragment representation outside production. + pub struct Etr1LexicalIndex { + connection: rusqlite::Connection, + } + + impl Etr1LexicalIndex { + pub fn new<'a>(documents: impl IntoIterator) -> Result { + use rusqlite::params; + + let connection = rusqlite::Connection::open_in_memory()?; + connection.execute_batch( + "CREATE VIRTUAL TABLE passages USING fts5(content); BEGIN IMMEDIATE;", + )?; + { + let mut insert = + connection.prepare("INSERT INTO passages(rowid, content) VALUES (?1, ?2)")?; + for (ordinal, document) in documents.into_iter().enumerate() { + let rowid = i64::try_from(ordinal.saturating_add(1))?; + insert.execute(params![rowid, etr1_lexical_document(document)])?; + } + } + connection.execute_batch("COMMIT;")?; + Ok(Self { connection }) + } + + pub fn search(&self, query: &str) -> Result<(Vec, Vec)> { + let terms = etr1_lexical_query_terms(query); + if terms.is_empty() { + return Ok((terms, Vec::new())); + } + let expression = terms + .iter() + .map(|term| format!("\"{}\"*", term.replace('"', "\"\""))) + .collect::>() + .join(" OR "); + let mut statement = self.connection.prepare( + "SELECT rowid, bm25(passages) AS score \ + FROM passages WHERE passages MATCH ?1 ORDER BY score, rowid", + )?; + let rows = statement.query_map([expression], |row| { + Ok(Etr1LexicalMatch { + rowid: usize::try_from(row.get::<_, i64>(0)?).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 0, + rusqlite::types::Type::Integer, + Box::new(error), + ) + })?, + score: row.get(1)?, + }) + })?; + Ok((terms, rows.collect::>>()?)) + } + } + + pub fn etr1_lexical_document(value: &str) -> String { + crate::lexical_index::normalize_lexical_text(value) + .split_whitespace() + .collect::>() + .join(" ") + } + + pub fn etr1_lexical_query_terms(query: &str) -> Vec { + crate::lexical_index::lexical_query_tokens(query) + } + pub struct WitnessLexicalPin { generation: String, input_hash: String, From 62fdb766d4be472db228cf741da7e7a69dcc00b6 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Fri, 4 Sep 2026 15:48:46 -0400 Subject: [PATCH 2/8] implement etr1 frontier experiment --- .../codestory-bench/src/bin/codestory_etr1.rs | 253 ++-- .../src/bin/codestory_etr1/contract.rs | 575 +++++++++ .../src/bin/codestory_etr1/prepare.rs | 665 ++++++++++ .../src/bin/codestory_etr1/run.rs | 1142 +++++++++++++++++ .../codestory-retrieval/src/lexical_index.rs | 4 +- crates/codestory-retrieval/src/lib.rs | 45 +- scripts/codestory-etr1-canary.mjs | 83 ++ scripts/codestory-etr1-evaluate.mjs | 212 +++ scripts/codestory-etr1-validate.mjs | 343 +++++ scripts/lib/etr1-evidence.mjs | 196 +++ scripts/tests/etr1-evidence.test.mjs | 207 +++ 11 files changed, 3540 insertions(+), 185 deletions(-) create mode 100644 crates/codestory-bench/src/bin/codestory_etr1/contract.rs create mode 100644 crates/codestory-bench/src/bin/codestory_etr1/prepare.rs create mode 100644 crates/codestory-bench/src/bin/codestory_etr1/run.rs create mode 100644 scripts/codestory-etr1-canary.mjs create mode 100644 scripts/codestory-etr1-evaluate.mjs create mode 100644 scripts/codestory-etr1-validate.mjs create mode 100644 scripts/lib/etr1-evidence.mjs create mode 100644 scripts/tests/etr1-evidence.test.mjs diff --git a/crates/codestory-bench/src/bin/codestory_etr1.rs b/crates/codestory-bench/src/bin/codestory_etr1.rs index b4ecb750a..1fde157e5 100644 --- a/crates/codestory-bench/src/bin/codestory_etr1.rs +++ b/crates/codestory-bench/src/bin/codestory_etr1.rs @@ -1,193 +1,84 @@ //! Benchmark-only implementation of the frozen ETR-1 frontier experiment. use anyhow::{Result, ensure}; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; -use std::collections::BTreeSet; - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -struct ByteRangeV1 { - start: u64, - end: u64, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -struct LineRangeV1 { - start: u32, - end: u32, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -struct FrozenFragmentV1 { - fragment_id: String, - project_id: String, - path: String, - content_digest: String, - byte_range: ByteRangeV1, - line_range: LineRangeV1, - source: String, - serialized_row_bytes: u32, +use clap::{Parser, Subcommand}; +use std::path::PathBuf; + +#[path = "codestory_proof_availability/build_provenance.rs"] +mod build_provenance; +#[path = "codestory_etr1/contract.rs"] +mod contract; +#[path = "codestory_etr1/prepare.rs"] +mod prepare; +#[path = "codestory_etr1/run.rs"] +mod run; + +#[derive(Parser)] +#[command(about = "Run the frozen benchmark-only ETR-1 frontier experiment")] +struct Args { + #[command(subcommand)] + command: Command, } -fn sha256(bytes: impl AsRef<[u8]>) -> String { - format!("{:x}", Sha256::digest(bytes.as_ref())) -} - -fn fragment_id(project_id: &str, path: &str, content_digest: &str, range: ByteRangeV1) -> String { - let mut digest = Sha256::new(); - digest.update(b"codestory.frozen-fragment/v1\0"); - for value in [ - project_id.as_bytes(), - path.as_bytes(), - content_digest.as_bytes(), - ] { - digest.update((value.len() as u64).to_le_bytes()); - digest.update(value); - } - digest.update(range.start.to_le_bytes()); - digest.update(range.end.to_le_bytes()); - format!("{:x}", digest.finalize()) -} - -fn select_successors( - score_order: &[(String, f32)], - seeds: &BTreeSet, - prior: &BTreeSet, - limit: usize, -) -> Vec { - let mut selected = Vec::with_capacity(limit.min(score_order.len())); - let mut seen = BTreeSet::new(); - for (fragment_id, _) in score_order { - if !seeds.contains(fragment_id) && !prior.contains(fragment_id) && seen.insert(fragment_id) - { - selected.push(fragment_id.clone()); - if selected.len() == limit { - break; - } - } - } - selected -} - -fn candidate_query_with_shortening( - question: &str, - source: &str, - fits: F, -) -> Result<(String, u32)> -where - F: Fn(&str) -> bool, -{ - ensure!(!question.trim().is_empty(), "question_is_empty"); - ensure!(!source.is_empty(), "seed_source_is_empty"); - let lines = source.split_inclusive('\n').collect::>(); - ensure!(!lines.is_empty(), "seed_source_has_no_lines"); - for retained in (1..=lines.len()).rev() { - let retained_source = lines[..retained].concat(); - if retained_source.trim().is_empty() { - continue; - } - let query = format!("{question}\n\n{retained_source}"); - if fits(&query) { - return Ok((query, u32::try_from(lines.len() - retained)?)); - } - } - anyhow::bail!("no_complete_seed_source_line_fits") -} - -fn natural_seed_prefix(matches: &[T]) -> Vec { - matches.iter().take(16).cloned().collect() +#[derive(Subcommand)] +enum Command { + /// Authenticate the frozen corpus, rebuild BM25 memberships, and emit exact + /// fragment documents for the existing embedding diagnostic. + Prepare { + #[arg(long)] + evidence_root: PathBuf, + #[arg(long)] + corpus_root: PathBuf, + #[arg(long)] + output_dir: PathBuf, + }, + /// Build paired unconditioned and source-conditioned frontiers from a + /// previously authenticated preparation and exact fragment vectors. + Run { + #[arg(long)] + prepared: PathBuf, + #[arg(long)] + prepared_sha256: String, + #[arg(long)] + fragment_vectors: PathBuf, + #[arg(long)] + fragment_vectors_sha256: String, + #[arg(long)] + state_root: PathBuf, + #[arg(long)] + output_dir: PathBuf, + }, } fn main() -> Result<()> { - anyhow::bail!("ETR-1 commands are not implemented yet") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn fragment_identity_binds_project_path_digest_and_range() { - let range = ByteRangeV1 { start: 7, end: 19 }; - let baseline = fragment_id("project-a", "src/lib.rs", &"a".repeat(64), range); - assert_eq!(baseline.len(), 64); - assert_ne!( - baseline, - fragment_id("project-b", "src/lib.rs", &"a".repeat(64), range) - ); - assert_ne!( - baseline, - fragment_id("project-a", "src/main.rs", &"a".repeat(64), range) - ); - assert_ne!( - baseline, - fragment_id("project-a", "src/lib.rs", &"b".repeat(64), range) - ); - assert_ne!( - baseline, - fragment_id( - "project-a", - "src/lib.rs", - &"a".repeat(64), - ByteRangeV1 { start: 8, end: 19 } - ) - ); - } - - #[test] - fn natural_seed_prefix_preserves_underfill_and_order() { - assert_eq!(natural_seed_prefix(&[3, 1, 2]), vec![3, 1, 2]); - assert_eq!( - natural_seed_prefix(&(0..20).collect::>()), - (0..16).collect::>() + if std::env::args().nth(1).as_deref() == Some("internal-embedding-server") { + ensure!( + build_provenance::SOURCE_DIRTY.trim() == "false", + "dirty_etr1_binary" ); + return codestory_cli::run_native_embedding_server(); } - - #[test] - fn cumulative_exclusions_produce_unique_successors() { - let scores = vec![ - ("seed".into(), 1.0), - ("prior".into(), 0.9), - ("new-a".into(), 0.8), - ("new-b".into(), 0.8), - ("new-c".into(), 0.7), - ]; - let selected = select_successors( - &scores, - &BTreeSet::from(["seed".into()]), - &BTreeSet::from(["prior".into()]), - 2, - ); - assert_eq!(selected, ["new-a", "new-b"]); - } - - #[test] - fn query_shortening_keeps_utf8_and_removes_complete_trailing_lines() { - let source = "first α line\nsecond β line\nthird γ line\n"; - let maximum = "question\n\nfirst α line\nsecond β line\n".len(); - let (query, removed) = - candidate_query_with_shortening("question", source, |value| value.len() <= maximum) - .unwrap(); - assert_eq!(query, "question\n\nfirst α line\nsecond β line\n"); - assert_eq!(removed, 1); - assert!(std::str::from_utf8(query.as_bytes()).is_ok()); - assert!(candidate_query_with_shortening("question", source, |_| false).is_err()); - } - - #[test] - fn lexical_contract_uses_the_product_normalizer_and_stop_words() { - assert_eq!( - codestory_retrieval::benchmark_support::etr1_lexical_document("HTTPServer run_pending"), - "http server run pending" - ); - assert_eq!( - codestory_retrieval::benchmark_support::etr1_lexical_query_terms( - "How does HTTPServer run_pending work?" - ), - vec!["http", "server", "run", "pending", "work"] - ); + let args = Args::parse(); + match args.command { + Command::Prepare { + evidence_root, + corpus_root, + output_dir, + } => prepare::execute(&evidence_root, &corpus_root, &output_dir), + Command::Run { + prepared, + prepared_sha256, + fragment_vectors, + fragment_vectors_sha256, + state_root, + output_dir, + } => run::execute( + &prepared, + &prepared_sha256, + &fragment_vectors, + &fragment_vectors_sha256, + &state_root, + &output_dir, + ), } } diff --git a/crates/codestory-bench/src/bin/codestory_etr1/contract.rs b/crates/codestory-bench/src/bin/codestory_etr1/contract.rs new file mode 100644 index 000000000..751e71887 --- /dev/null +++ b/crates/codestory-bench/src/bin/codestory_etr1/contract.rs @@ -0,0 +1,575 @@ +use crate::build_provenance; +use anyhow::{Context, Result, ensure}; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Component, Path, PathBuf}; + +pub const PARENT_HEAD: &str = "c9c935d87129a79f326b650bbf23d73191df8b4f"; +pub const FRAGMENT_DIAGNOSTIC_SHA256: &str = + "ca185ed13c635bbb4b64cc6760c5025799700359ebcc4dd3bcc53e34f8cf9194"; +pub const FRAGMENT_BUILD_SHA256: &str = + "2201780e1a752db4bfcceb047bf5cd0b5a854733c4050330ef87575b960f3baf"; +pub const MEMBERSHIP_FREEZE_SHA256: &str = + "e6867b5c79706160021ec5edf60792273345ca97cae8377e69934d5e2c9992ee"; +pub const QUESTIONS_SHA256: &str = + "8e7219a59c973c02f8ea93120bb680da46a75b8272153986c76e55bfb73ca3b6"; +pub const ANNOTATIONS_SHA256: &str = + "52b0cc223292bc70f1e4fa3f52b67bf42a91e4d4b9ed997aa12c648c068e9ade"; +pub const MODEL_CONTRACT_SHA256: &str = + "cb0e3c00290f1eb21ecdcd873521d03331069b1efa766fcd1e493e6d4299b4b7"; +pub const MODEL_SHA256: &str = "666db8df27c88570cdc07adca28646260038b8ca65354911d57b936ebf56efaa"; +pub const TOKENIZER_SHA256: &str = + "7465b93c945b7a266481e6785aa13e505c625562c1c046c4b762bb4da4d46082"; +pub const LEXICAL_POLICY_SHA256: &str = + "43b2478d75abd3d5689d05e08c072e4148fd21ab29bcc55533d30a494edf986b"; +pub const FRAGMENT_COUNT: usize = 10_369; +pub const WORDING_COUNT: usize = 72; +pub const VECTOR_DIMENSION: usize = 768; +pub const SEED_LIMIT: usize = 16; +pub const SUCCESSORS_PER_QUERY: usize = 8; +pub const MAX_SUCCESSORS: usize = 128; +pub const MAX_POOL: usize = 144; +pub const PUBLIC_ROWS: usize = 16; +pub const PUBLIC_BYTES: usize = 16 * 1024; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ByteRangeV1 { + pub start: u64, + pub end: u64, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct LineRangeV1 { + pub start: u32, + pub end: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct FrozenFragmentV1 { + pub fragment_id: String, + pub project_id: String, + pub path: String, + pub content_digest: String, + pub byte_range: ByteRangeV1, + pub line_range: LineRangeV1, + pub source: String, + pub serialized_row_bytes: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct FileBinding { + pub path: PathBuf, + pub sha256: String, + pub bytes: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct DeclaredBinding { + pub path: PathBuf, + pub sha256: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct BuildIdentity { + pub source_commit: String, + pub source_tree: String, + pub source_dirty: bool, + pub profile: String, + pub rustc: String, + pub binary_path: PathBuf, + pub binary_sha256: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PreparedRepositoryV1 { + pub repository_id: String, + pub project_id: String, + pub commit: String, + pub local_root: PathBuf, + pub publication: Value, + pub fragment_ids: Vec, + pub score_order_sha256: String, + pub base_serialized_bytes: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PreparedWordingV1 { + pub case_id: String, + pub phrasing_id: String, + pub repository_id: String, + pub group: String, + pub question: String, + pub question_sha256: String, + pub membership: FileBinding, + pub terms: Vec, + pub bm25_match_count: u32, + pub bm25_matches_sha256: String, + pub seed_fragment_ids: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Etr1PreparationV1 { + pub contract: String, + pub authority: String, + pub packet_decision: String, + pub parent_head: String, + pub build: BuildIdentity, + pub method: FileBinding, + pub fixed_inputs: BTreeMap, + pub annotations: DeclaredBinding, + pub model_sha256: String, + pub tokenizer_sha256: String, + pub embedding_input: FileBinding, + pub annotation_access: String, + pub repositories: Vec, + pub fragments: Vec, + pub wordings: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EmbeddingDiagnosticRecord { + pub id: String, + pub purpose: String, + pub text: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EmbeddingDiagnosticInput { + pub contract: String, + pub records: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct QueryReceiptV1 { + pub query_ordinal: u32, + pub seed_fragment_id: String, + pub original_input_sha256: String, + pub encoded_input_sha256: String, + pub encoded_input: String, + pub removed_trailing_source_lines: u32, + pub model_limit_rejections: u32, + pub global_batch_ordinal: u32, + pub score_order_sha256: String, + pub query_vector: Vec, + pub scores: Vec, + pub excluded_before: Vec, + pub retained_successors: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BatchReceiptV1 { + pub global_batch_ordinal: u32, + pub arm: String, + pub query_ordinals: Vec, + pub input_sha256: Vec, + pub wall_ns: u64, + pub completed_tokens: u64, + pub qualification_event_sequence: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct SourceAuthenticationReceiptV1 { + pub fragment_source_bytes: u64, + pub filesystem_bytes_read: u64, + pub authenticated_fragment_ids: Vec, + pub file_digests: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ArmTimingV1 { + pub round_zero_bm25_ns: u64, + pub seed_source_authentication_ns: u64, + pub query_encoding_ns: u64, + pub vector_search_ns: u64, + pub descriptor_mapping_ns: u64, + pub remaining_source_authentication_ns: u64, + pub prepared_state_ns: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ArmFrontierV1 { + pub name: String, + pub search_count: u32, + pub query_receipts: Vec, + pub batch_receipts: Vec, + pub successors: Vec, + pub descriptor_pool: Vec, + pub hydrated_pool: Vec, + pub legally_selectable_pool: Vec, + pub source_authentication: SourceAuthenticationReceiptV1, + pub token_total: u64, + pub timing: ArmTimingV1, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Etr1WordingResultV1 { + pub contract: String, + pub case_id: String, + pub phrasing_id: String, + pub repository_id: String, + pub group: String, + pub question_sha256: String, + pub seed_fragment_ids: Vec, + pub control: ArmFrontierV1, + pub candidate: ArmFrontierV1, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Etr1RunManifestV1 { + pub contract: String, + pub authority: String, + pub experiment_status: String, + pub decision: String, + pub parent_head: String, + pub build: BuildIdentity, + pub preparation: FileBinding, + pub fragment_vectors: FileBinding, + pub method_sha256: String, + pub annotation_access: String, + pub vector_artifact_loaded_before_timing: bool, + pub initial_engine: Value, + pub final_engine: Value, + pub graph_invocations: u32, + pub bge_invocations: u32, + pub symbol_document_invocations: u32, + pub host_query_invocations: u32, + pub production_packet_invocations: u32, + pub qualification_events: FileBinding, + pub qualification_completed_token_total: u64, + pub rows: Vec, +} + +pub fn sha256(bytes: impl AsRef<[u8]>) -> String { + format!("{:x}", Sha256::digest(bytes.as_ref())) +} + +pub fn digest_file(path: &Path) -> Result { + let mut file = File::open(path).with_context(|| format!("open {}", path.display()))?; + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let count = file.read(&mut buffer)?; + if count == 0 { + break; + } + digest.update(&buffer[..count]); + } + Ok(format!("{:x}", digest.finalize())) +} + +pub fn bind_file(path: &Path, expected: Option<&str>) -> Result { + ensure!( + path.is_absolute(), + "binding_path_not_absolute: {}", + path.display() + ); + let metadata = std::fs::symlink_metadata(path)?; + ensure!( + metadata.is_file() && !metadata.file_type().is_symlink(), + "binding_not_regular_file" + ); + let digest = digest_file(path)?; + if let Some(expected) = expected { + ensure!( + digest == expected, + "binding_digest_mismatch: {}", + path.display() + ); + } + Ok(FileBinding { + path: path.to_path_buf(), + sha256: digest, + bytes: metadata.len(), + }) +} + +pub fn read_bound_json(binding: &FileBinding) -> Result { + let bytes = std::fs::read(&binding.path)?; + ensure!( + bytes.len() as u64 == binding.bytes, + "binding_length_changed" + ); + ensure!(sha256(&bytes) == binding.sha256, "binding_digest_changed"); + serde_json::from_slice(&bytes).context("parse bound JSON") +} + +pub fn build_identity() -> Result { + let binary_path = std::fs::canonicalize(std::env::current_exe()?)?; + Ok(BuildIdentity { + source_commit: build_provenance::SOURCE_COMMIT.trim().to_string(), + source_tree: build_provenance::SOURCE_TREE.trim().to_string(), + source_dirty: build_provenance::SOURCE_DIRTY.trim() != "false", + profile: build_provenance::BUILD_PROFILE.trim().to_string(), + rustc: build_provenance::RUSTC_VV.trim().to_string(), + binary_sha256: digest_file(&binary_path)?, + binary_path, + }) +} + +pub fn fragment_id( + project_id: &str, + path: &str, + content_digest: &str, + range: ByteRangeV1, +) -> String { + let mut digest = Sha256::new(); + digest.update(b"codestory.frozen-fragment/v1\0"); + for value in [ + project_id.as_bytes(), + path.as_bytes(), + content_digest.as_bytes(), + ] { + digest.update((value.len() as u64).to_le_bytes()); + digest.update(value); + } + digest.update(range.start.to_le_bytes()); + digest.update(range.end.to_le_bytes()); + format!("{:x}", digest.finalize()) +} + +pub fn select_successors( + score_order: &[(String, f32)], + seeds: &BTreeSet, + prior: &BTreeSet, + limit: usize, +) -> Vec { + let mut selected = Vec::with_capacity(limit.min(score_order.len())); + let mut seen = BTreeSet::new(); + for (fragment_id, _) in score_order { + if !seeds.contains(fragment_id) && !prior.contains(fragment_id) && seen.insert(fragment_id) + { + selected.push(fragment_id.clone()); + if selected.len() == limit { + break; + } + } + } + selected +} + +#[cfg(test)] +pub fn candidate_query_with_shortening( + question: &str, + source: &str, + fits: F, +) -> Result<(String, u32)> +where + F: Fn(&str) -> bool, +{ + ensure!(!question.trim().is_empty(), "question_is_empty"); + ensure!(!source.is_empty(), "seed_source_is_empty"); + let lines = source.split_inclusive('\n').collect::>(); + ensure!(!lines.is_empty(), "seed_source_has_no_lines"); + for retained in (1..=lines.len()).rev() { + let retained_source = lines[..retained].concat(); + if retained_source.trim().is_empty() { + continue; + } + let query = format!("{question}\n\n{retained_source}"); + if fits(&query) { + return Ok((query, u32::try_from(lines.len() - retained)?)); + } + } + anyhow::bail!("no_complete_seed_source_line_fits") +} + +pub fn natural_seed_prefix(matches: &[T]) -> Vec { + matches.iter().take(SEED_LIMIT).cloned().collect() +} + +pub fn validate_relative_path(path: &str) -> Result<()> { + let parsed = Path::new(path); + ensure!( + !path.is_empty() && !parsed.is_absolute() && !path.contains('\\'), + "invalid_relative_path" + ); + ensure!( + parsed + .components() + .all(|component| matches!(component, Component::Normal(_))), + "invalid_relative_path" + ); + Ok(()) +} + +pub fn confined_source_path(root: &Path, relative: &str) -> Result { + validate_relative_path(relative)?; + let root = std::fs::canonicalize(root)?; + let candidate = std::fs::canonicalize(root.join(relative))?; + ensure!(candidate.starts_with(&root), "source_path_escaped_root"); + Ok(candidate) +} + +pub fn serialize_pretty(value: &T) -> Result> { + let mut bytes = serde_json::to_vec_pretty(value)?; + bytes.push(b'\n'); + Ok(bytes) +} + +pub fn write_exclusive(path: &Path, bytes: &[u8]) -> Result<()> { + let mut options = OpenOptions::new(); + options.create_new(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(path)?; + file.write_all(bytes)?; + file.sync_all()?; + Ok(()) +} + +pub fn stage_output_directory(output: &Path) -> Result { + ensure!(output.is_absolute(), "output_path_not_absolute"); + ensure!(!output.exists(), "output_already_exists"); + let parent = output.parent().context("output_parent_missing")?; + ensure!(parent.is_dir(), "output_parent_missing"); + let stage = tempfile::Builder::new() + .prefix(".etr1-stage-") + .tempdir_in(parent)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(stage.path(), std::fs::Permissions::from_mode(0o700))?; + } + Ok(stage) +} + +pub fn publish_output_directory(stage: tempfile::TempDir, output: &Path) -> Result<()> { + let stage_path = stage.keep(); + if let Err(error) = std::fs::rename(&stage_path, output) { + let _ = std::fs::remove_dir_all(&stage_path); + return Err(error.into()); + } + #[cfg(unix)] + File::open(output.parent().context("output_parent_missing")?)?.sync_all()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fragment_identity_binds_every_authority() { + let range = ByteRangeV1 { start: 7, end: 19 }; + let baseline = fragment_id("project-a", "src/lib.rs", &"a".repeat(64), range); + assert_eq!(baseline.len(), 64); + assert_ne!( + baseline, + fragment_id("project-b", "src/lib.rs", &"a".repeat(64), range) + ); + assert_ne!( + baseline, + fragment_id("project-a", "src/main.rs", &"a".repeat(64), range) + ); + assert_ne!( + baseline, + fragment_id("project-a", "src/lib.rs", &"b".repeat(64), range) + ); + assert_ne!( + baseline, + fragment_id( + "project-a", + "src/lib.rs", + &"a".repeat(64), + ByteRangeV1 { start: 8, end: 19 } + ) + ); + } + + #[test] + fn natural_seed_prefix_preserves_underfill_and_order() { + assert_eq!(natural_seed_prefix(&[3, 1, 2]), vec![3, 1, 2]); + assert_eq!( + natural_seed_prefix(&(0..20).collect::>()), + (0..16).collect::>() + ); + } + + #[test] + fn cumulative_exclusions_produce_unique_successors() { + let scores = vec![ + ("seed".into(), 1.0), + ("prior".into(), 0.9), + ("new-a".into(), 0.8), + ("new-b".into(), 0.8), + ]; + assert_eq!( + select_successors( + &scores, + &BTreeSet::from(["seed".into()]), + &BTreeSet::from(["prior".into()]), + 2 + ), + ["new-a", "new-b"] + ); + } + + #[test] + fn query_shortening_keeps_utf8_and_complete_lines() { + let source = "first α line\nsecond β line\nthird γ line\n"; + let maximum = "question\n\nfirst α line\nsecond β line\n".len(); + let (query, removed) = + candidate_query_with_shortening("question", source, |value| value.len() <= maximum) + .unwrap(); + assert_eq!(query, "question\n\nfirst α line\nsecond β line\n"); + assert_eq!(removed, 1); + assert!(candidate_query_with_shortening("question", source, |_| false).is_err()); + } + + #[test] + fn relative_source_paths_reject_aliases_and_escapes() { + assert!(validate_relative_path("src/lib.rs").is_ok()); + for path in [ + "", + "/src/lib.rs", + "../src/lib.rs", + "src/../lib.rs", + "src\\lib.rs", + ] { + assert!(validate_relative_path(path).is_err(), "{path}"); + } + } + + #[test] + fn aborted_or_cancelled_stage_never_publishes_a_partial_experiment() { + let parent = tempfile::tempdir().unwrap(); + let output = parent.path().join("run"); + let stage = stage_output_directory(&output).unwrap(); + write_exclusive(&stage.path().join("partial.json"), b"{}\n").unwrap(); + drop(stage); + assert!(!output.exists()); + } + + #[test] + fn publication_is_no_clobber() { + let parent = tempfile::tempdir().unwrap(); + let output = parent.path().join("run"); + std::fs::create_dir(&output).unwrap(); + assert!(stage_output_directory(&output).is_err()); + } +} diff --git a/crates/codestory-bench/src/bin/codestory_etr1/prepare.rs b/crates/codestory-bench/src/bin/codestory_etr1/prepare.rs new file mode 100644 index 000000000..182cae505 --- /dev/null +++ b/crates/codestory-bench/src/bin/codestory_etr1/prepare.rs @@ -0,0 +1,665 @@ +use super::contract::*; +use anyhow::{Context, Result, bail, ensure}; +use codestory_retrieval::benchmark_support::{Etr1LexicalIndex, Etr1LexicalMatch}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +#[derive(Debug, Deserialize)] +struct SyntaxDiagnostic { + #[serde(rename = "selectedFreeze")] + selected_freeze: ExternalBinding, +} + +#[derive(Debug, Clone, Deserialize)] +struct ExternalBinding { + path: PathBuf, + sha256: String, + bytes: u64, +} + +#[derive(Debug, Deserialize)] +struct SelectedFreeze { + #[serde(rename = "inventoryRows")] + inventory_rows: Vec, +} + +#[derive(Debug, Deserialize)] +struct SelectedRepository { + repository_id: String, + project_root: PathBuf, + prepared: ExternalBinding, + units: UnitBindings, +} + +#[derive(Debug, Deserialize)] +struct UnitBindings { + line: ExternalBinding, +} + +#[derive(Debug, Deserialize)] +struct PriorPreparation { + publication: Value, +} + +#[derive(Debug, Clone, Deserialize)] +struct LineUnit { + start_line: u32, + end_line: u32, + byte_range: ByteRangeV1, + content: String, + available: bool, + snippet: String, + path: String, + content_digest: String, +} + +#[derive(Debug, Deserialize)] +struct Questions { + contract: String, + authority: String, + repositories: Vec, + cases: Vec, +} + +#[derive(Debug, Deserialize)] +struct QuestionRepository { + id: String, + commit: String, + local_root: PathBuf, +} + +#[derive(Debug, Deserialize)] +struct QuestionCase { + case_id: String, + repository_id: String, + group: String, + question: String, + paraphrases: Vec, +} + +#[derive(Debug, Deserialize)] +struct MembershipFreeze { + method: ExternalBinding, + records: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +struct MembershipRecord { + case_id: String, + phrasing_id: String, + repository_id: String, + group: String, + path: PathBuf, + sha256: String, + bytes: u64, +} + +#[derive(Debug, Deserialize)] +struct MembershipFile { + query: String, + method_sha256: String, + arms: BTreeMap, +} + +#[derive(Debug, Deserialize)] +struct MembershipArm { + terms: Vec, + matches: Vec, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +struct MembershipMatch { + rowid: usize, + score: f64, +} + +#[derive(Debug, Deserialize)] +struct ModelContract { + schema_version: u32, + model: ModelIdentity, + embedding: EmbeddingIdentity, + tokenizer_config: TokenizerIdentity, +} + +#[derive(Debug, Deserialize)] +struct ModelIdentity { + sha256: String, +} + +#[derive(Debug, Deserialize)] +struct EmbeddingIdentity { + dimension: usize, + query_prefix: String, + document_prefix: String, + pooling: String, + normalization: String, +} + +#[derive(Debug, Deserialize)] +struct TokenizerIdentity { + tokenizer_sha256: String, +} + +#[derive(Debug, Serialize)] +struct PublicSourceRow<'a> { + kind: &'static str, + path: &'a str, + start_line: u32, + end_line: u32, + snippet: &'a str, + content_digest: &'a str, + byte_range: ByteRangeV1, +} + +fn external_binding(value: &ExternalBinding) -> Result { + let binding = bind_file(&value.path, Some(&value.sha256))?; + ensure!( + binding.bytes == value.bytes, + "external_binding_length_mismatch" + ); + Ok(binding) +} + +pub(super) fn git_head(root: &Path) -> Result { + let output = Command::new("git") + .arg("-C") + .arg(root) + .args(["rev-parse", "HEAD^{commit}"]) + .env("GIT_OPTIONAL_LOCKS", "0") + .env("GIT_TERMINAL_PROMPT", "0") + .output()?; + ensure!(output.status.success(), "repository_commit_unavailable"); + Ok(String::from_utf8(output.stdout)?.trim().to_string()) +} + +fn line_number(bytes: &[u8], offset: usize) -> u32 { + u32::try_from( + bytes[..offset] + .iter() + .filter(|byte| **byte == b'\n') + .count() + + 1, + ) + .unwrap_or(u32::MAX) +} + +fn end_line_number(bytes: &[u8], end: usize) -> u32 { + line_number(bytes, end.saturating_sub(1)) +} + +fn render_snippet(source: &str, start_line: u32) -> String { + let mut snippet = String::from("```text\n"); + for (offset, line) in source.split_inclusive('\n').enumerate() { + let line = line.trim_end_matches(['\r', '\n']); + snippet.push_str(&format!(" {:>5} | {line}\n", start_line as usize + offset)); + } + snippet.push_str("```"); + snippet +} + +fn authenticate_units( + repository: &SelectedRepository, + project_id: &str, + units: &[LineUnit], +) -> Result> { + let mut sources = HashMap::>::new(); + let mut result = Vec::with_capacity(units.len()); + for unit in units { + ensure!(unit.available, "frozen_fragment_unavailable"); + if !sources.contains_key(&unit.path) { + let path = confined_source_path(&repository.project_root, &unit.path)?; + let bytes = std::fs::read(path)?; + ensure!( + sha256(&bytes) == unit.content_digest, + "fragment_file_digest_mismatch" + ); + sources.insert(unit.path.clone(), bytes); + } + let source = &sources[&unit.path]; + let start = usize::try_from(unit.byte_range.start)?; + let end = usize::try_from(unit.byte_range.end)?; + ensure!( + start < end && end <= source.len(), + "fragment_byte_range_invalid" + ); + ensure!( + std::str::from_utf8(&source[..start]).is_ok(), + "fragment_start_splits_utf8" + ); + ensure!( + std::str::from_utf8(&source[..end]).is_ok(), + "fragment_end_splits_utf8" + ); + let observed = std::str::from_utf8(&source[start..end])?; + ensure!(observed == unit.content, "fragment_source_mismatch"); + ensure!( + line_number(source, start) == unit.start_line, + "fragment_start_line_mismatch" + ); + ensure!( + end_line_number(source, end) == unit.end_line, + "fragment_end_line_mismatch" + ); + ensure!( + render_snippet(observed, unit.start_line) == unit.snippet, + "fragment_snippet_mismatch" + ); + let row = PublicSourceRow { + kind: "source_range", + path: &unit.path, + start_line: unit.start_line, + end_line: unit.end_line, + snippet: &unit.snippet, + content_digest: &unit.content_digest, + byte_range: unit.byte_range, + }; + let serialized_row_bytes = u32::try_from(serde_json::to_vec(&row)?.len())?; + result.push(FrozenFragmentV1 { + fragment_id: fragment_id( + project_id, + &unit.path, + &unit.content_digest, + unit.byte_range, + ), + project_id: project_id.to_string(), + path: unit.path.clone(), + content_digest: unit.content_digest.clone(), + byte_range: unit.byte_range, + line_range: LineRangeV1 { + start: unit.start_line, + end: unit.end_line, + }, + source: unit.content.clone(), + serialized_row_bytes, + }); + } + ensure!( + result + .iter() + .all(|fragment| !fragment.source.trim().is_empty()), + "empty_fragment_document" + ); + ensure!( + result + .iter() + .map(|fragment| &fragment.fragment_id) + .collect::>() + .len() + == result.len(), + "duplicate_fragment_identity" + ); + Ok(result) +} + +fn compare_membership( + observed_terms: &[String], + observed: &[Etr1LexicalMatch], + expected: &MembershipArm, +) -> Result<()> { + ensure!(observed_terms == expected.terms, "bm25_terms_mismatch"); + ensure!( + observed.len() == expected.matches.len(), + "bm25_match_count_mismatch" + ); + for (observed, expected) in observed.iter().zip(&expected.matches) { + ensure!(observed.rowid == expected.rowid, "bm25_rowid_mismatch"); + ensure!( + (observed.score - expected.score).abs() <= 1e-12, + "bm25_score_mismatch" + ); + } + Ok(()) +} + +fn method_freeze() -> Value { + json!({ + "contract": "codestory.etr1-method/v1", + "authority": "visible_development_frontier_only", + "question": "same authenticated BM25 seeds; equal second-round search and source ceilings; raw question versus raw question plus verbatim seed source", + "fragment_identity": "sha256(domain || length-framed project_id,path,content_digest || little-endian half-open byte bounds)", + "representation": "non-overlapping complete-line 512-byte frozen fragments; document and public source are the exact fragment text", + "round_zero": {"lane":"content-only FTS5 BM25", "seeds":16, "underfill":"preserved"}, + "second_round": {"searches":"one per actual seed per arm", "successors_per_search":8, "cumulative_exclusion":true, "score":"normalized dot product", "tie_break":"fragment_id"}, + "limits": {"successors":MAX_SUCCESSORS, "descriptor_pool":MAX_POOL, "public_rows":PUBLIC_ROWS, "public_bytes":PUBLIC_BYTES}, + "query": {"control":"raw question", "candidate":"raw question + newline delimiter + seed source", "truncation":"none; complete trailing seed lines may be removed only after a typed model-limit rejection"}, + "graph": false, "bge": false, "symbol_documents": false, "host_queries": false, + "packet_decision": "not_evaluated" + }) +} + +pub fn execute(evidence_root: &Path, corpus_root: &Path, output: &Path) -> Result<()> { + ensure!( + evidence_root.is_absolute() && corpus_root.is_absolute(), + "input_roots_must_be_absolute" + ); + let source_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .context("source_root_missing")?; + ensure!( + output.is_absolute() && !output.starts_with(source_root), + "output_must_be_external" + ); + let build = build_identity()?; + + let diagnostic_binding = bind_file( + &evidence_root.join("syntax-representation-v2/diagnostic.json"), + Some(FRAGMENT_DIAGNOSTIC_SHA256), + )?; + let build_binding = bind_file( + &evidence_root.join("syntax-representation-v2/build.json"), + Some(FRAGMENT_BUILD_SHA256), + )?; + let membership_binding = bind_file( + &evidence_root.join("fragment-eligibility-v1/membership-freeze.json"), + Some(MEMBERSHIP_FREEZE_SHA256), + )?; + let questions_binding = bind_file(&corpus_root.join("questions.json"), Some(QUESTIONS_SHA256))?; + let lexical_binding = bind_file( + &source_root.join("crates/codestory-retrieval/src/lexical_index.rs"), + Some(LEXICAL_POLICY_SHA256), + )?; + let model_binding = bind_file( + &source_root.join("crates/codestory-llama-sys/model-contract.json"), + Some(MODEL_CONTRACT_SHA256), + )?; + + let model: ModelContract = read_bound_json(&model_binding)?; + ensure!( + model.schema_version == 1 && model.model.sha256 == MODEL_SHA256, + "model_identity_mismatch" + ); + ensure!( + model.tokenizer_config.tokenizer_sha256 == TOKENIZER_SHA256, + "tokenizer_identity_mismatch" + ); + ensure!( + model.embedding.dimension == VECTOR_DIMENSION + && model.embedding.query_prefix == "Represent this query for searching relevant code: " + && model.embedding.document_prefix.is_empty() + && model.embedding.pooling == "cls" + && model.embedding.normalization == "l2", + "embedding_contract_mismatch" + ); + + let diagnostic: SyntaxDiagnostic = read_bound_json(&diagnostic_binding)?; + let selected_binding = external_binding(&diagnostic.selected_freeze)?; + let selected: SelectedFreeze = read_bound_json(&selected_binding)?; + let questions: Questions = read_bound_json(&questions_binding)?; + ensure!( + questions.contract == "codestory.visible-questions" + && questions.authority == "visible_development_only" + && questions.cases.len() == 24, + "question_contract_mismatch" + ); + let membership: MembershipFreeze = read_bound_json(&membership_binding)?; + ensure!( + membership.records.len() == WORDING_COUNT, + "membership_record_count_mismatch" + ); + let membership_method = external_binding(&membership.method)?; + + let mut fragments = Vec::new(); + let mut repositories = Vec::new(); + let mut fragment_by_repository = BTreeMap::>::new(); + let mut indexes = BTreeMap::::new(); + for selected_repository in &selected.inventory_rows { + let question_repository = questions + .repositories + .iter() + .find(|repository| repository.id == selected_repository.repository_id) + .context("repository_question_pin_missing")?; + ensure!( + std::fs::canonicalize(&question_repository.local_root)? + == std::fs::canonicalize(&selected_repository.project_root)?, + "repository_root_mismatch" + ); + ensure!( + git_head(&selected_repository.project_root)? == question_repository.commit, + "repository_commit_mismatch" + ); + let prior_binding = external_binding(&selected_repository.prepared)?; + let prior: PriorPreparation = read_bound_json(&prior_binding)?; + let project_id = prior + .publication + .get("project_id") + .and_then(Value::as_str) + .context("project_id_missing")? + .to_string(); + let units_binding = external_binding(&selected_repository.units.line)?; + let units: Vec = read_bound_json(&units_binding)?; + let repo_fragments = authenticate_units(selected_repository, &project_id, &units)?; + let score_order_sha256 = sha256(serde_json::to_vec( + &repo_fragments + .iter() + .map(|fragment| &fragment.fragment_id) + .collect::>(), + )?); + let empty_packet = json!({"publication": prior.publication, "answer_sufficiency":"not_asserted", "support":[], "continuation":[]}); + let base_serialized_bytes = u32::try_from(serde_json::to_vec(&empty_packet)?.len())?; + indexes.insert( + selected_repository.repository_id.clone(), + Etr1LexicalIndex::new( + repo_fragments + .iter() + .map(|fragment| fragment.source.as_str()), + )?, + ); + repositories.push(PreparedRepositoryV1 { + repository_id: selected_repository.repository_id.clone(), + project_id, + commit: question_repository.commit.clone(), + local_root: selected_repository.project_root.clone(), + publication: empty_packet["publication"].clone(), + fragment_ids: repo_fragments + .iter() + .map(|fragment| fragment.fragment_id.clone()) + .collect(), + score_order_sha256, + base_serialized_bytes, + }); + fragments.extend(repo_fragments.iter().cloned()); + fragment_by_repository.insert(selected_repository.repository_id.clone(), repo_fragments); + } + ensure!(fragments.len() == FRAGMENT_COUNT, "fragment_count_mismatch"); + + let mut wordings = Vec::with_capacity(WORDING_COUNT); + for record in &membership.records { + let record_binding = bind_file(&record.path, Some(&record.sha256))?; + ensure!( + record_binding.bytes == record.bytes && !record.group.is_empty(), + "membership_record_binding_mismatch" + ); + let expected: MembershipFile = read_bound_json(&record_binding)?; + ensure!( + expected.method_sha256 == membership_method.sha256, + "membership_method_mismatch" + ); + let case = questions + .cases + .iter() + .find(|case| case.case_id == record.case_id) + .context("question_case_missing")?; + ensure!( + case.repository_id == record.repository_id + && case.group == record.group + && case.paraphrases.len() == 2, + "question_case_binding_mismatch" + ); + let phrasing_index = match record.phrasing_id.as_str() { + "original" => 0, + "paraphrase_1" => 1, + "paraphrase_2" => 2, + _ => bail!("unknown_phrasing_id"), + }; + let question = [&case.question, &case.paraphrases[0], &case.paraphrases[1]][phrasing_index]; + ensure!(expected.query == *question, "membership_question_mismatch"); + let index = indexes + .get(&record.repository_id) + .context("lexical_index_missing")?; + let (terms, observed) = index.search(question)?; + let raw = expected.arms.get("raw").context("raw_membership_missing")?; + compare_membership(&terms, &observed, raw)?; + let repo_fragments = fragment_by_repository + .get(&record.repository_id) + .context("repository_fragments_missing")?; + let seeds = natural_seed_prefix(&observed) + .into_iter() + .map(|item| repo_fragments[item.rowid - 1].fragment_id.clone()) + .collect::>(); + let observed_matches = observed + .iter() + .map(|item| MembershipMatch { + rowid: item.rowid, + score: item.score, + }) + .collect::>(); + wordings.push(PreparedWordingV1 { + case_id: record.case_id.clone(), + phrasing_id: record.phrasing_id.clone(), + repository_id: record.repository_id.clone(), + group: record.group.clone(), + question: question.clone(), + question_sha256: sha256(question.as_bytes()), + membership: record_binding, + terms, + bm25_match_count: u32::try_from(observed.len())?, + bm25_matches_sha256: sha256(serde_json::to_vec(&observed_matches)?), + seed_fragment_ids: seeds, + }); + } + ensure!(wordings.len() == WORDING_COUNT, "wording_count_mismatch"); + + let mut fixed_inputs = BTreeMap::new(); + for (name, binding) in [ + ("fragment_diagnostic", diagnostic_binding), + ("fragment_build", build_binding), + ("selected_fragment_freeze", selected_binding), + ("lexical_membership_freeze", membership_binding), + ("lexical_membership_method", membership_method), + ("questions", questions_binding), + ("model_contract", model_binding), + ("lexical_policy_source", lexical_binding), + ] { + fixed_inputs.insert(name.to_string(), binding); + } + + let method_bytes = serialize_pretty(&method_freeze())?; + let embedding_input = EmbeddingDiagnosticInput { + contract: "codestory.embedding-diagnostic-input/v1".into(), + records: fragments + .iter() + .map(|fragment| EmbeddingDiagnosticRecord { + id: fragment.fragment_id.clone(), + purpose: "document".into(), + text: fragment.source.clone(), + }) + .collect(), + }; + let embedding_bytes = serialize_pretty(&embedding_input)?; + let method_file = output.join("method-freeze.json"); + let embedding_file = output.join("fragment-embedding-input.json"); + let method = FileBinding { + path: method_file, + sha256: sha256(&method_bytes), + bytes: method_bytes.len() as u64, + }; + let embedding_binding = FileBinding { + path: embedding_file, + sha256: sha256(&embedding_bytes), + bytes: embedding_bytes.len() as u64, + }; + let preparation = Etr1PreparationV1 { + contract: "codestory.etr1-preparation/v1".into(), + authority: "visible_development_frontier_only".into(), + packet_decision: "not_evaluated".into(), + parent_head: PARENT_HEAD.into(), + build, + method, + fixed_inputs, + annotations: DeclaredBinding { + path: corpus_root.join("reconciled.json"), + sha256: ANNOTATIONS_SHA256.into(), + }, + model_sha256: MODEL_SHA256.into(), + tokenizer_sha256: TOKENIZER_SHA256.into(), + embedding_input: embedding_binding, + annotation_access: "not_accessed".into(), + repositories, + fragments, + wordings, + }; + let preparation_bytes = serialize_pretty(&preparation)?; + let stage = stage_output_directory(output)?; + write_exclusive(&stage.path().join("method-freeze.json"), &method_bytes)?; + write_exclusive( + &stage.path().join("fragment-embedding-input.json"), + &embedding_bytes, + )?; + write_exclusive(&stage.path().join("preparation.json"), &preparation_bytes)?; + publish_output_directory(stage, output)?; + println!( + "{} {}", + sha256(&preparation_bytes), + output.join("preparation.json").display() + ); + println!( + "{} {}", + preparation.embedding_input.sha256, + preparation.embedding_input.path.display() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn snippet_and_line_contract_are_complete_line_based() { + let source = "alpha\nβeta\n"; + assert_eq!( + render_snippet(source, 4), + "```text\n 4 | alpha\n 5 | βeta\n```" + ); + assert_eq!(line_number(source.as_bytes(), 0), 1); + assert_eq!(line_number(source.as_bytes(), 6), 2); + assert_eq!(end_line_number(source.as_bytes(), source.len()), 2); + } + + #[test] + fn membership_comparison_refuses_score_or_order_drift() { + let expected = MembershipArm { + terms: vec!["alpha".into()], + matches: vec![MembershipMatch { + rowid: 1, + score: -2.0, + }], + }; + assert!( + compare_membership( + &["alpha".into()], + &[Etr1LexicalMatch { + rowid: 1, + score: -2.0, + }], + &expected, + ) + .is_ok() + ); + assert!( + compare_membership( + &["alpha".into()], + &[Etr1LexicalMatch { + rowid: 2, + score: -2.0, + }], + &expected, + ) + .is_err() + ); + } +} diff --git a/crates/codestory-bench/src/bin/codestory_etr1/run.rs b/crates/codestory-bench/src/bin/codestory_etr1/run.rs new file mode 100644 index 000000000..6e1baa856 --- /dev/null +++ b/crates/codestory-bench/src/bin/codestory_etr1/run.rs @@ -0,0 +1,1142 @@ +use super::contract::*; +use anyhow::{Context, Result, ensure}; +use codestory_retrieval::benchmark_support::Etr1LexicalIndex; +use codestory_retrieval::{ + EmbeddingEngineIdentity, PerUserEmbeddingClient, ProductEmbeddingClient, SidecarRuntimeConfig, +}; +use serde::Deserialize; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +const RUN_CONTRACT: &str = "codestory.etr1-run/v1"; +const ROW_CONTRACT: &str = "codestory.etr1-wording/v1"; +const VECTOR_CONTRACT: &str = "codestory.embedding-diagnostic-output/v1"; +const QUERY_BATCH_MAX: usize = 8; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct FragmentVectorArtifact { + contract: String, + authority: String, + packet_decision: String, + input_sha256: String, + source_commit: String, + source_tree: String, + build_profile: String, + rustc: String, + binary_sha256: String, + initial_engine: Value, + final_engine: Value, + whole_encoding_wall_ms: u64, + records: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct FragmentVectorRecord { + id: String, + purpose: String, + text_sha256: String, + vector: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct QualificationEvent { + schema_version: u32, + sequence: u64, + action: String, + status: String, + server_event_sequence: u64, + clock: Value, + #[serde(default)] + snapshot: Option, + #[serde(default)] + details: Option>, +} + +#[derive(Debug, Clone)] +struct QuerySpec { + ordinal: usize, + seed_fragment_id: String, + original_input: String, + encoded_input: String, + removed_trailing_source_lines: u32, + model_limit_rejections: u32, +} + +#[derive(Debug)] +struct EncodedQuery { + spec: QuerySpec, + vector: Vec, + global_batch_ordinal: u32, +} + +#[derive(Debug, Default)] +struct EventCursor { + completed_events: usize, +} + +struct SourceAuthenticator<'a> { + repository: &'a PreparedRepositoryV1, + fragments: &'a HashMap, + file_cache: HashMap>, + receipt: SourceAuthenticationReceiptV1, +} + +impl<'a> SourceAuthenticator<'a> { + fn new( + repository: &'a PreparedRepositoryV1, + fragments: &'a HashMap, + ) -> Self { + Self { + repository, + fragments, + file_cache: HashMap::new(), + receipt: SourceAuthenticationReceiptV1::default(), + } + } + + fn authenticate(&mut self, fragment_id: &str) -> Result<()> { + if self + .receipt + .authenticated_fragment_ids + .iter() + .any(|value| value == fragment_id) + { + return Ok(()); + } + let fragment = self + .fragments + .get(fragment_id) + .copied() + .context("source_fragment_missing")?; + ensure!( + fragment.project_id == self.repository.project_id + && self + .repository + .fragment_ids + .iter() + .any(|value| value == fragment_id), + "source_fragment_repository_mismatch" + ); + ensure!( + fragment_id + == super::contract::fragment_id( + &fragment.project_id, + &fragment.path, + &fragment.content_digest, + fragment.byte_range, + ), + "source_fragment_identity_mismatch" + ); + if !self.file_cache.contains_key(&fragment.path) { + let path = confined_source_path(&self.repository.local_root, &fragment.path)?; + let bytes = fs::read(path)?; + let digest = sha256(&bytes); + ensure!( + digest == fragment.content_digest, + "source_file_digest_mismatch" + ); + self.receipt.filesystem_bytes_read = self + .receipt + .filesystem_bytes_read + .saturating_add(bytes.len() as u64); + self.receipt + .file_digests + .insert(fragment.path.clone(), digest); + self.file_cache.insert(fragment.path.clone(), bytes); + } + let source = &self.file_cache[&fragment.path]; + let start = usize::try_from(fragment.byte_range.start)?; + let end = usize::try_from(fragment.byte_range.end)?; + ensure!( + start < end && end <= source.len(), + "source_fragment_range_invalid" + ); + ensure!( + std::str::from_utf8(&source[..start]).is_ok() + && std::str::from_utf8(&source[..end]).is_ok(), + "source_fragment_range_splits_utf8" + ); + ensure!( + std::str::from_utf8(&source[start..end])? == fragment.source, + "source_fragment_bytes_changed" + ); + ensure!( + observed_line(source, start) == fragment.line_range.start + && observed_end_line(source, end) == fragment.line_range.end, + "source_fragment_lines_changed" + ); + self.receipt.fragment_source_bytes = self + .receipt + .fragment_source_bytes + .saturating_add(fragment.source.len() as u64); + self.receipt + .authenticated_fragment_ids + .push(fragment_id.to_string()); + Ok(()) + } +} + +fn observed_line(bytes: &[u8], offset: usize) -> u32 { + u32::try_from( + bytes[..offset] + .iter() + .filter(|byte| **byte == b'\n') + .count() + + 1, + ) + .unwrap_or(u32::MAX) +} + +fn observed_end_line(bytes: &[u8], end: usize) -> u32 { + observed_line(bytes, end.saturating_sub(1)) +} + +fn private_directory(path: &Path) -> Result<()> { + ensure!(path.is_absolute(), "state_path_not_absolute"); + let metadata = fs::symlink_metadata(path)?; + ensure!( + metadata.is_dir() && !metadata.file_type().is_symlink(), + "state_not_directory" + ); + ensure!(fs::canonicalize(path)? == path, "state_path_not_canonical"); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + ensure!( + metadata.uid() == unsafe { libc::geteuid() } && metadata.mode() & 0o077 == 0, + "state_directory_not_private" + ); + } + #[cfg(not(unix))] + return Err(anyhow::anyhow!( + "etr1_requires_unix_private_directory_validation" + )); + Ok(()) +} + +fn validate_isolated_state(state_root: &Path, runtime: &SidecarRuntimeConfig) -> Result { + private_directory(state_root)?; + let cache = state_root.join("cache"); + let ipc = state_root.join("ipc"); + private_directory(&cache)?; + private_directory(&ipc)?; + ensure!(runtime.cache_root == cache, "etr1_cache_not_isolated"); + ensure!(!runtime.embedding.allow_cpu, "etr1_cpu_fallback_forbidden"); + let gate = codestory_retrieval::qualification_gate_environment(); + ensure!( + gate.directory.as_deref() == Some(ipc.as_os_str()), + "etr1_qualification_directory_not_isolated" + ); + let nonce = gate + .nonce_string() + .context("etr1_qualification_nonce_missing")?; + ensure!( + (16..=64).contains(&nonce.len()) + && nonce + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')), + "etr1_qualification_nonce_invalid" + ); + let events = ipc.join(format!("{nonce}.events.jsonl")); + ensure!(!events.exists(), "etr1_qualification_events_preexisting"); + Ok(events) +} + +fn validate_vector(vector: &[f32]) -> Result<()> { + ensure!( + vector.len() == VECTOR_DIMENSION && vector.iter().all(|value| value.is_finite()), + "vector_shape_invalid" + ); + let norm = vector + .iter() + .map(|value| f64::from(*value).powi(2)) + .sum::(); + ensure!((norm - 1.0).abs() < 0.001, "vector_not_normalized"); + Ok(()) +} + +fn engine_receipt(identity: &EmbeddingEngineIdentity) -> Result { + ensure!( + identity.accelerator_execution_verified + && identity.worker_alive + && identity.load_error.is_none() + && identity.embedded_model + && identity.policy == "accelerated" + && identity.model_digest == MODEL_SHA256, + "engine_execution_unverified" + ); + Ok(serde_json::to_value(identity)?) +} + +fn load_vector_artifact( + path: &Path, + expected_sha256: &str, + preparation: &Etr1PreparationV1, +) -> Result<( + FileBinding, + FragmentVectorArtifact, + HashMap>, +)> { + let binding = bind_file(path, Some(expected_sha256))?; + let artifact: FragmentVectorArtifact = read_bound_json(&binding)?; + ensure!( + artifact.contract == VECTOR_CONTRACT + && artifact.authority == "post_failure_diagnostic_only" + && artifact.packet_decision == "not_evaluated" + && artifact.input_sha256 == preparation.embedding_input.sha256 + && artifact.source_commit == preparation.build.source_commit + && artifact.source_tree == preparation.build.source_tree + && artifact.build_profile == preparation.build.profile + && artifact.rustc == preparation.build.rustc + && artifact.whole_encoding_wall_ms > 0, + "fragment_vector_artifact_identity_mismatch" + ); + ensure!( + artifact.initial_engine["model_digest"] == MODEL_SHA256 + && artifact.final_engine["model_digest"] == MODEL_SHA256 + && artifact.initial_engine["server_instance_id"] + == artifact.final_engine["server_instance_id"] + && artifact.initial_engine["load_generation"] + == artifact.final_engine["load_generation"] + && artifact.initial_engine["accelerator_execution_verified"] == true + && artifact.final_engine["accelerator_execution_verified"] == true, + "fragment_vector_engine_mismatch" + ); + ensure!( + artifact.records.len() == preparation.fragments.len(), + "fragment_vector_count_mismatch" + ); + let mut vectors = HashMap::with_capacity(artifact.records.len()); + for (record, fragment) in artifact.records.iter().zip(&preparation.fragments) { + ensure!( + record.id == fragment.fragment_id + && record.purpose == "document" + && record.text_sha256 == sha256(fragment.source.as_bytes()), + "fragment_vector_record_binding_mismatch" + ); + validate_vector(&record.vector)?; + ensure!( + vectors + .insert(record.id.clone(), record.vector.clone()) + .is_none(), + "duplicate_fragment_vector" + ); + } + ensure!( + artifact.binary_sha256.len() == 64 + && artifact + .binary_sha256 + .bytes() + .all(|byte| byte.is_ascii_hexdigit()), + "fragment_vector_binary_identity_invalid" + ); + Ok((binding, artifact, vectors)) +} + +fn read_completed_events(path: &Path) -> Result> { + let bytes = fs::read(path).context("read qualification event log")?; + ensure!( + bytes.ends_with(b"\n"), + "qualification_event_log_unterminated" + ); + let mut events = Vec::new(); + let mut previous = 0_u64; + for line in bytes + .split(|byte| *byte == b'\n') + .filter(|line| !line.is_empty()) + { + let event: QualificationEvent = serde_json::from_slice(line)?; + ensure!( + event.schema_version == 1 && event.sequence > previous, + "qualification_event_sequence_invalid" + ); + previous = event.sequence; + if event.action == "completed_tokens" { + ensure!( + event.status == "completed" + && event.server_event_sequence > 0 + && event.clock.is_object() + && event.snapshot.is_none(), + "qualification_token_event_invalid" + ); + events.push(event); + } + } + Ok(events) +} + +fn consume_completed_event(path: &Path, cursor: &mut EventCursor) -> Result<(u64, u64)> { + let events = read_completed_events(path)?; + ensure!( + events.len() == cursor.completed_events + 1, + "qualification_completed_event_count_invalid" + ); + let event = &events[cursor.completed_events]; + cursor.completed_events += 1; + let details = event + .details + .as_ref() + .context("qualification_token_details_missing")?; + ensure!( + details + .get("request_id") + .is_some_and(|value| !value.is_empty()) + && details + .get("native_completion_sequence") + .and_then(|value| value.parse::().ok()) + .is_some_and(|value| value > 0), + "qualification_token_identity_missing" + ); + let tokens = details + .get("completed_tokens") + .context("qualification_completed_tokens_missing")? + .parse::()?; + ensure!(tokens > 0, "qualification_completed_tokens_zero"); + Ok((tokens, event.sequence)) +} + +fn input_too_long(error: &anyhow::Error) -> bool { + let text = format!("{error:#}"); + text.contains("native_embedding_input_too_long") || text.contains("embedding input is too long") +} + +fn shorten_single_query(spec: &mut QuerySpec, question: &str, source: &str) -> Result<()> { + let lines = source.split_inclusive('\n').collect::>(); + let retained = lines + .len() + .checked_sub(usize::try_from(spec.removed_trailing_source_lines)?) + .context("all_seed_source_lines_removed")?; + ensure!(retained > 1, "no_complete_seed_source_line_fits"); + let next_source = lines[..retained - 1].concat(); + ensure!( + !next_source.trim().is_empty(), + "no_complete_seed_source_line_fits" + ); + spec.removed_trailing_source_lines += 1; + spec.encoded_input = format!("{question}\n\n{next_source}"); + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn encode_partition( + client: &ProductEmbeddingClient, + specs: &mut [QuerySpec], + questions: &[String], + seed_sources: &[String], + qualification_events: &Path, + cursor: &mut EventCursor, + next_batch_ordinal: &mut u32, + batches: &mut Vec, + results: &mut Vec, + arm: &str, +) -> Result<()> { + ensure!( + !specs.is_empty() && specs.len() <= QUERY_BATCH_MAX, + "query_batch_size_invalid" + ); + let inputs = specs + .iter() + .map(|spec| spec.encoded_input.clone()) + .collect::>(); + let started = Instant::now(); + match client.embed_queries_with_control(&inputs, Some(Duration::from_secs(60)), &|| false) { + Ok(vectors) => { + let wall_ns = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX); + ensure!(vectors.len() == specs.len(), "query_vector_count_mismatch"); + for vector in &vectors { + validate_vector(vector)?; + } + let (completed_tokens, sequence) = + consume_completed_event(qualification_events, cursor)?; + let ordinal = *next_batch_ordinal; + *next_batch_ordinal = (*next_batch_ordinal).saturating_add(1); + batches.push(BatchReceiptV1 { + global_batch_ordinal: ordinal, + arm: arm.to_string(), + query_ordinals: specs.iter().map(|spec| spec.ordinal as u32).collect(), + input_sha256: specs + .iter() + .map(|spec| sha256(spec.encoded_input.as_bytes())) + .collect(), + wall_ns, + completed_tokens, + qualification_event_sequence: sequence, + }); + for (spec, vector) in specs.iter().cloned().zip(vectors) { + results.push(EncodedQuery { + spec, + vector, + global_batch_ordinal: ordinal, + }); + } + Ok(()) + } + Err(error) if input_too_long(&error) => { + ensure!(arm == "candidate", "raw_question_exceeds_model_limit"); + for spec in specs.iter_mut() { + spec.model_limit_rejections = spec.model_limit_rejections.saturating_add(1); + } + if specs.len() > 1 { + let middle = specs.len() / 2; + let (left, right) = specs.split_at_mut(middle); + encode_partition( + client, + left, + questions, + seed_sources, + qualification_events, + cursor, + next_batch_ordinal, + batches, + results, + arm, + )?; + encode_partition( + client, + right, + questions, + seed_sources, + qualification_events, + cursor, + next_batch_ordinal, + batches, + results, + arm, + ) + } else { + let index = specs[0].ordinal; + shorten_single_query(&mut specs[0], &questions[index], &seed_sources[index])?; + encode_partition( + client, + specs, + questions, + seed_sources, + qualification_events, + cursor, + next_batch_ordinal, + batches, + results, + arm, + ) + } + } + Err(error) => Err(error).context("encode ETR-1 query batch"), + } +} + +#[allow(clippy::too_many_arguments)] +fn encode_queries( + client: &ProductEmbeddingClient, + arm: &str, + specs: &mut [QuerySpec], + questions: &[String], + seed_sources: &[String], + qualification_events: &Path, + cursor: &mut EventCursor, + next_batch_ordinal: &mut u32, +) -> Result<(Vec, Vec)> { + ensure!( + specs.len() == questions.len() && specs.len() == seed_sources.len(), + "query_spec_input_count_mismatch" + ); + let mut encoded = Vec::with_capacity(specs.len()); + let mut batches = Vec::new(); + for start in (0..specs.len()).step_by(QUERY_BATCH_MAX) { + let end = (start + QUERY_BATCH_MAX).min(specs.len()); + encode_partition( + client, + &mut specs[start..end], + questions, + seed_sources, + qualification_events, + cursor, + next_batch_ordinal, + &mut batches, + &mut encoded, + arm, + )?; + } + encoded.sort_by_key(|query| query.spec.ordinal); + ensure!( + encoded + .iter() + .enumerate() + .all(|(ordinal, query)| query.spec.ordinal == ordinal), + "encoded_query_order_invalid" + ); + Ok((encoded, batches)) +} + +fn score_fragments( + query: &[f32], + fragment_ids: &[String], + vectors: &HashMap>, +) -> Result<(Vec, Vec<(String, f32)>)> { + validate_vector(query)?; + let mut scores = Vec::with_capacity(fragment_ids.len()); + for fragment_id in fragment_ids { + let vector = vectors + .get(fragment_id) + .context("repository_fragment_vector_missing")?; + let score = query + .iter() + .zip(vector) + .fold(0.0_f32, |sum, (left, right)| sum + left * right); + ensure!(score.is_finite(), "semantic_score_nonfinite"); + scores.push(score); + } + let mut ranked = fragment_ids + .iter() + .cloned() + .zip(scores.iter().copied()) + .collect::>(); + ranked.sort_by(|left, right| { + right + .1 + .total_cmp(&left.1) + .then_with(|| left.0.cmp(&right.0)) + }); + Ok((scores, ranked)) +} + +fn exact_legally_selectable_pool( + descriptor_pool: &[String], + repository: &PreparedRepositoryV1, + fragments: &HashMap, +) -> Result> { + let mut seen = HashSet::new(); + let mut result = Vec::new(); + for fragment_id in descriptor_pool { + if !seen.insert(fragment_id) { + continue; + } + let fragment = fragments + .get(fragment_id) + .copied() + .context("legally_selectable_fragment_missing")?; + let public_bytes = usize::try_from(repository.base_serialized_bytes)? + .saturating_add(usize::try_from(fragment.serialized_row_bytes)?); + if public_bytes <= PUBLIC_BYTES { + result.push(fragment_id.clone()); + } + } + Ok(result) +} + +#[allow(clippy::too_many_arguments)] +fn run_arm( + name: &str, + wording: &PreparedWordingV1, + repository: &PreparedRepositoryV1, + fragments: &HashMap, + vectors: &HashMap>, + lexical: &Etr1LexicalIndex, + client: &ProductEmbeddingClient, + qualification_events: &Path, + cursor: &mut EventCursor, + next_batch_ordinal: &mut u32, +) -> Result { + let bm25_started = Instant::now(); + let (_, matches) = lexical.search(&wording.question)?; + let observed_seeds = natural_seed_prefix(&matches) + .into_iter() + .map(|item| repository.fragment_ids[item.rowid - 1].clone()) + .collect::>(); + let round_zero_bm25_ns = u64::try_from(bm25_started.elapsed().as_nanos()).unwrap_or(u64::MAX); + ensure!( + observed_seeds == wording.seed_fragment_ids, + "round_zero_seed_drift" + ); + let seeds = observed_seeds.into_iter().collect::>(); + let seed_sources = wording + .seed_fragment_ids + .iter() + .map(|id| { + fragments + .get(id) + .map(|fragment| fragment.source.clone()) + .context("seed_fragment_missing") + }) + .collect::>>()?; + let mut authenticator = SourceAuthenticator::new(repository, fragments); + let seed_auth_started = Instant::now(); + for fragment_id in &wording.seed_fragment_ids { + authenticator.authenticate(fragment_id)?; + } + let seed_source_authentication_ns = + u64::try_from(seed_auth_started.elapsed().as_nanos()).unwrap_or(u64::MAX); + + let query_inputs = if name == "control" { + wording + .seed_fragment_ids + .iter() + .map(|_| wording.question.clone()) + .collect::>() + } else { + ensure!(name == "candidate", "unknown_etr1_arm"); + seed_sources + .iter() + .map(|source| format!("{}\n\n{source}", wording.question)) + .collect() + }; + let query_questions = wording + .seed_fragment_ids + .iter() + .map(|_| wording.question.clone()) + .collect::>(); + let mut specs = query_inputs + .into_iter() + .enumerate() + .map(|(ordinal, input)| QuerySpec { + ordinal, + seed_fragment_id: wording.seed_fragment_ids[ordinal].clone(), + original_input: input.clone(), + encoded_input: input, + removed_trailing_source_lines: 0, + model_limit_rejections: 0, + }) + .collect::>(); + let encoding_started = Instant::now(); + let (encoded, batch_receipts) = encode_queries( + client, + name, + &mut specs, + &query_questions, + &seed_sources, + qualification_events, + cursor, + next_batch_ordinal, + )?; + let query_encoding_ns = + u64::try_from(encoding_started.elapsed().as_nanos()).unwrap_or(u64::MAX); + + let vector_started = Instant::now(); + let mut scored = Vec::with_capacity(encoded.len()); + for query in &encoded { + scored.push(score_fragments( + &query.vector, + &repository.fragment_ids, + vectors, + )?); + } + let vector_search_ns = u64::try_from(vector_started.elapsed().as_nanos()).unwrap_or(u64::MAX); + + let mapping_started = Instant::now(); + let mut prior = BTreeSet::new(); + let mut successors = Vec::new(); + let mut query_receipts = Vec::with_capacity(encoded.len()); + for (query, (scores, ranked)) in encoded.into_iter().zip(scored) { + let excluded_before = seeds.iter().chain(prior.iter()).cloned().collect(); + let selected = select_successors(&ranked, &seeds, &prior, SUCCESSORS_PER_QUERY); + ensure!( + selected.len() <= SUCCESSORS_PER_QUERY, + "successor_query_limit_exceeded" + ); + prior.extend(selected.iter().cloned()); + successors.extend(selected.iter().cloned()); + query_receipts.push(QueryReceiptV1 { + query_ordinal: u32::try_from(query.spec.ordinal)?, + seed_fragment_id: query.spec.seed_fragment_id, + original_input_sha256: sha256(query.spec.original_input.as_bytes()), + encoded_input_sha256: sha256(query.spec.encoded_input.as_bytes()), + encoded_input: query.spec.encoded_input, + removed_trailing_source_lines: query.spec.removed_trailing_source_lines, + model_limit_rejections: query.spec.model_limit_rejections, + global_batch_ordinal: query.global_batch_ordinal, + score_order_sha256: repository.score_order_sha256.clone(), + query_vector: query.vector, + scores, + excluded_before, + retained_successors: selected, + }); + } + ensure!( + successors.len() <= MAX_SUCCESSORS + && successors.iter().collect::>().len() == successors.len(), + "successor_pool_invalid" + ); + let mut descriptor_pool = wording.seed_fragment_ids.clone(); + descriptor_pool.extend(successors.iter().cloned()); + ensure!( + descriptor_pool.len() <= MAX_POOL, + "descriptor_pool_limit_exceeded" + ); + let descriptor_mapping_ns = + u64::try_from(mapping_started.elapsed().as_nanos()).unwrap_or(u64::MAX); + + let remaining_auth_started = Instant::now(); + for fragment_id in &successors { + authenticator.authenticate(fragment_id)?; + } + let remaining_source_authentication_ns = + u64::try_from(remaining_auth_started.elapsed().as_nanos()).unwrap_or(u64::MAX); + let hydrated_pool = descriptor_pool.clone(); + ensure!( + authenticator.receipt.authenticated_fragment_ids.len() == hydrated_pool.len(), + "hydrated_pool_authentication_incomplete" + ); + let legally_selectable_pool = + exact_legally_selectable_pool(&hydrated_pool, repository, fragments)?; + let timing = ArmTimingV1 { + round_zero_bm25_ns, + seed_source_authentication_ns, + query_encoding_ns, + vector_search_ns, + descriptor_mapping_ns, + remaining_source_authentication_ns, + prepared_state_ns: round_zero_bm25_ns + .saturating_add(seed_source_authentication_ns) + .saturating_add(query_encoding_ns) + .saturating_add(vector_search_ns) + .saturating_add(descriptor_mapping_ns) + .saturating_add(remaining_source_authentication_ns), + }; + Ok(ArmFrontierV1 { + name: name.to_string(), + search_count: u32::try_from(query_receipts.len())?, + query_receipts, + batch_receipts, + successors, + descriptor_pool, + hydrated_pool, + legally_selectable_pool, + source_authentication: authenticator.receipt, + token_total: 0, + timing, + }) +} + +fn finalize_arm(mut arm: ArmFrontierV1) -> ArmFrontierV1 { + arm.token_total = arm + .batch_receipts + .iter() + .map(|batch| batch.completed_tokens) + .sum(); + arm +} + +pub fn execute( + prepared: &Path, + prepared_sha256: &str, + fragment_vectors: &Path, + fragment_vectors_sha256: &str, + state_root: &Path, + output: &Path, +) -> Result<()> { + let source_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .context("source_root_missing")?; + ensure!( + output.is_absolute() && !output.starts_with(source_root) && !output.starts_with(state_root), + "output_must_be_private_external_evidence" + ); + let preparation_binding = bind_file(prepared, Some(prepared_sha256))?; + let preparation: Etr1PreparationV1 = read_bound_json(&preparation_binding)?; + let build = build_identity()?; + ensure!( + !build.source_dirty + && preparation.contract == "codestory.etr1-preparation/v1" + && preparation.authority == "visible_development_frontier_only" + && preparation.packet_decision == "not_evaluated" + && preparation.parent_head == PARENT_HEAD + && preparation.annotation_access == "not_accessed" + && preparation.annotations.sha256 == ANNOTATIONS_SHA256 + && preparation.model_sha256 == MODEL_SHA256 + && preparation.tokenizer_sha256 == TOKENIZER_SHA256 + && preparation.fragments.len() == FRAGMENT_COUNT + && preparation.wordings.len() == WORDING_COUNT + && preparation.build.source_commit == build.source_commit + && preparation.build.source_tree == build.source_tree, + "etr1_preparation_identity_mismatch" + ); + ensure!( + bind_file(&preparation.method.path, Some(&preparation.method.sha256))?.bytes + == preparation.method.bytes + && bind_file( + &preparation.embedding_input.path, + Some(&preparation.embedding_input.sha256) + )? + .bytes + == preparation.embedding_input.bytes, + "etr1_preparation_child_binding_mismatch" + ); + + // The full document-vector artifact is read and authenticated before any + // request timing begins. It is never rebuilt or paged during ETR-1. + let (vector_binding, vector_artifact, vectors) = + load_vector_artifact(fragment_vectors, fragment_vectors_sha256, &preparation)?; + let runtime = SidecarRuntimeConfig::local(); + let qualification_events = validate_isolated_state(state_root, &runtime)?; + codestory_cli::install_native_embedding_client_transport()?; + let mut residency = PerUserEmbeddingClient::for_runtime(&runtime)?.acquire_residency_lease()?; + let initial_engine = engine_receipt(residency.identity())?; + let client = ProductEmbeddingClient::new(&runtime); + + let fragment_map = preparation + .fragments + .iter() + .map(|fragment| (fragment.fragment_id.clone(), fragment)) + .collect::>(); + ensure!( + fragment_map.len() == FRAGMENT_COUNT, + "fragment_map_identity_collision" + ); + let repository_map = preparation + .repositories + .iter() + .map(|repository| (repository.repository_id.clone(), repository)) + .collect::>(); + let mut lexical = BTreeMap::new(); + for repository in &preparation.repositories { + ensure!( + super::prepare::git_head(&repository.local_root)? == repository.commit, + "repository_commit_changed" + ); + lexical.insert( + repository.repository_id.clone(), + Etr1LexicalIndex::new(repository.fragment_ids.iter().map(|id| { + fragment_map + .get(id) + .expect("prepared repository fragment identity exists") + .source + .as_str() + }))?, + ); + } + + let mut cursor = EventCursor::default(); + let mut next_batch_ordinal = 0_u32; + let mut rows = Vec::with_capacity(WORDING_COUNT); + for wording in &preparation.wordings { + let repository = repository_map + .get(&wording.repository_id) + .copied() + .context("wording_repository_missing")?; + let index = lexical + .get(&wording.repository_id) + .context("wording_lexical_index_missing")?; + let control = finalize_arm(run_arm( + "control", + wording, + repository, + &fragment_map, + &vectors, + index, + &client, + &qualification_events, + &mut cursor, + &mut next_batch_ordinal, + )?); + let candidate = finalize_arm(run_arm( + "candidate", + wording, + repository, + &fragment_map, + &vectors, + index, + &client, + &qualification_events, + &mut cursor, + &mut next_batch_ordinal, + )?); + ensure!( + control.search_count == wording.seed_fragment_ids.len() as u32 + && candidate.search_count == control.search_count + && control.descriptor_pool.len() <= MAX_POOL + && candidate.descriptor_pool.len() <= MAX_POOL, + "etr1_arm_budget_mismatch" + ); + rows.push(Etr1WordingResultV1 { + contract: ROW_CONTRACT.into(), + case_id: wording.case_id.clone(), + phrasing_id: wording.phrasing_id.clone(), + repository_id: wording.repository_id.clone(), + group: wording.group.clone(), + question_sha256: wording.question_sha256.clone(), + seed_fragment_ids: wording.seed_fragment_ids.clone(), + control, + candidate, + }); + } + ensure!(rows.len() == WORDING_COUNT, "etr1_row_count_mismatch"); + let final_engine = engine_receipt(&residency.revalidate()?)?; + for key in [ + "server_instance_id", + "load_generation", + "model_digest", + "ggml_build_identity", + ] { + ensure!( + initial_engine[key] == final_engine[key], + "engine_identity_changed:{key}" + ); + } + let event_bytes = fs::read(&qualification_events)?; + let completed = read_completed_events(&qualification_events)?; + ensure!( + completed.len() == cursor.completed_events + && rows + .iter() + .flat_map(|row| [&row.control, &row.candidate]) + .map(|arm| arm.batch_receipts.len()) + .sum::() + == completed.len(), + "qualification_event_reconciliation_failed" + ); + let total_tokens = rows + .iter() + .flat_map(|row| [&row.control, &row.candidate]) + .map(|arm| arm.token_total) + .sum::(); + + let stage = stage_output_directory(output)?; + let rows_directory = stage.path().join("rows"); + fs::create_dir(&rows_directory)?; + let mut row_bindings = Vec::with_capacity(rows.len()); + for row in &rows { + let name = format!("{}--{}.json", row.case_id, row.phrasing_id); + let bytes = serialize_pretty(row)?; + write_exclusive(&rows_directory.join(&name), &bytes)?; + row_bindings.push(FileBinding { + path: output.join("rows").join(name), + sha256: sha256(&bytes), + bytes: bytes.len() as u64, + }); + } + write_exclusive( + &stage.path().join("qualification-events.jsonl"), + &event_bytes, + )?; + let qualification_binding = FileBinding { + path: output.join("qualification-events.jsonl"), + sha256: sha256(&event_bytes), + bytes: event_bytes.len() as u64, + }; + let manifest = Etr1RunManifestV1 { + contract: RUN_CONTRACT.into(), + authority: "visible_development_frontier_only".into(), + experiment_status: "awaiting_validation".into(), + decision: "not_evaluated".into(), + parent_head: PARENT_HEAD.into(), + build, + preparation: preparation_binding, + fragment_vectors: vector_binding, + method_sha256: preparation.method.sha256, + annotation_access: "not_accessed".into(), + vector_artifact_loaded_before_timing: true, + initial_engine, + final_engine, + graph_invocations: 0, + bge_invocations: 0, + symbol_document_invocations: 0, + host_query_invocations: 0, + production_packet_invocations: 0, + qualification_events: qualification_binding, + qualification_completed_token_total: total_tokens, + rows: row_bindings, + }; + let manifest_bytes = serialize_pretty(&manifest)?; + write_exclusive(&stage.path().join("run.json"), &manifest_bytes)?; + publish_output_directory(stage, output)?; + println!( + "{} {}", + sha256(&manifest_bytes), + output.join("run.json").display() + ); + // Keep the residency lease alive through the complete no-clobber publish. + drop(residency); + drop(vector_artifact); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn unit_vector(index: usize) -> Vec { + let mut vector = vec![0.0; VECTOR_DIMENSION]; + vector[index] = 1.0; + vector + } + + #[test] + fn normalized_dot_scores_use_fragment_id_as_only_tie_break() { + let ids = vec!["z".to_string(), "a".to_string(), "m".to_string()]; + let vectors = HashMap::from([ + ("z".to_string(), unit_vector(0)), + ("a".to_string(), unit_vector(0)), + ("m".to_string(), unit_vector(1)), + ]); + let (scores, ranked) = score_fragments(&unit_vector(0), &ids, &vectors).unwrap(); + assert_eq!(scores, [1.0, 1.0, 0.0]); + assert_eq!( + ranked + .iter() + .map(|value| value.0.as_str()) + .collect::>(), + ["a", "z", "m"] + ); + } + + #[test] + fn complete_line_retry_removes_only_a_trailing_seed_line() { + let mut spec = QuerySpec { + ordinal: 0, + seed_fragment_id: "seed".into(), + original_input: "question\n\na\nb\n".into(), + encoded_input: "question\n\na\nb\n".into(), + removed_trailing_source_lines: 0, + model_limit_rejections: 1, + }; + shorten_single_query(&mut spec, "question", "a\nb\n").unwrap(); + assert_eq!(spec.encoded_input, "question\n\na\n"); + assert_eq!(spec.removed_trailing_source_lines, 1); + assert!(shorten_single_query(&mut spec, "question", "a\nb\n").is_err()); + } + + #[test] + fn legal_pool_deduplicates_exact_ids_without_making_seeds_compulsory() { + let repository = PreparedRepositoryV1 { + repository_id: "repo".into(), + project_id: "project".into(), + commit: "c".into(), + local_root: PathBuf::from("/tmp/repo"), + publication: Value::Null, + fragment_ids: vec!["seed".into(), "successor".into()], + score_order_sha256: "x".into(), + base_serialized_bytes: 100, + }; + let seed = FrozenFragmentV1 { + fragment_id: "seed".into(), + project_id: "project".into(), + path: "seed.rs".into(), + content_digest: "a".repeat(64), + byte_range: ByteRangeV1 { start: 0, end: 1 }, + line_range: LineRangeV1 { start: 1, end: 1 }, + source: "s".into(), + serialized_row_bytes: u32::try_from(PUBLIC_BYTES).unwrap(), + }; + let successor = FrozenFragmentV1 { + fragment_id: "successor".into(), + serialized_row_bytes: 10, + ..seed.clone() + }; + let map = HashMap::from([("seed".into(), &seed), ("successor".into(), &successor)]); + assert_eq!( + exact_legally_selectable_pool( + &["seed".into(), "successor".into(), "successor".into()], + &repository, + &map, + ) + .unwrap(), + ["successor"] + ); + } +} diff --git a/crates/codestory-retrieval/src/lexical_index.rs b/crates/codestory-retrieval/src/lexical_index.rs index 36972b910..f3fa40be2 100644 --- a/crates/codestory-retrieval/src/lexical_index.rs +++ b/crates/codestory-retrieval/src/lexical_index.rs @@ -3917,7 +3917,7 @@ fn lexical_documents_hash(documents: &[LexicalDocument], coverage: &LexicalCover .hash } -pub(crate) fn normalize_lexical_text(value: &str) -> String { +fn normalize_lexical_text(value: &str) -> String { let mut normalized = String::with_capacity(value.len() + value.len() / 8); let mut characters = value.chars().peekable(); let mut previous: Option = None; @@ -4090,7 +4090,7 @@ fn fts_document_frequency(connection: &Connection, token: &str) -> Result .map_err(Into::into) } -pub(crate) fn lexical_query_tokens(query: &str) -> Vec { +fn lexical_query_tokens(query: &str) -> Vec { let mut tokens = Vec::new(); let normalized = normalize_lexical_text(query); for token in normalized diff --git a/crates/codestory-retrieval/src/lib.rs b/crates/codestory-retrieval/src/lib.rs index 130ae3c01..840beee2e 100644 --- a/crates/codestory-retrieval/src/lib.rs +++ b/crates/codestory-retrieval/src/lib.rs @@ -131,14 +131,55 @@ pub mod benchmark_support { } pub fn etr1_lexical_document(value: &str) -> String { - crate::lexical_index::normalize_lexical_text(value) + etr1_normalize_lexical_text(value) .split_whitespace() .collect::>() .join(" ") } pub fn etr1_lexical_query_terms(query: &str) -> Vec { - crate::lexical_index::lexical_query_tokens(query) + const LEXICAL_STOP_WORDS: &[&str] = &[ + "about", "after", "and", "are", "cite", "does", "explain", "file", "files", "flow", + "flows", "for", "from", "how", "into", "level", "path", "source", "sources", "support", + "that", "the", "through", "top", "what", "where", "which", "with", + ]; + + let mut tokens = Vec::new(); + let normalized = etr1_normalize_lexical_text(query); + for token in normalized + .split_whitespace() + .filter(|token| token.len() >= 2) + .filter(|token| !LEXICAL_STOP_WORDS.contains(token)) + { + if !tokens.iter().any(|existing| existing == token) { + tokens.push(token.to_string()); + } + } + tokens + } + + fn etr1_normalize_lexical_text(value: &str) -> String { + let mut normalized = String::with_capacity(value.len() + value.len() / 8); + let mut characters = value.chars().peekable(); + let mut previous: Option = None; + while let Some(character) = characters.next() { + let next = characters.peek().copied(); + if character.is_uppercase() + && previous.is_some_and(|value: char| value.is_lowercase() || value.is_numeric()) + || character.is_uppercase() + && previous.is_some_and(|value: char| value.is_uppercase()) + && next.is_some_and(|value| value.is_lowercase()) + { + normalized.push(' '); + } + if character.is_alphanumeric() { + normalized.extend(character.to_lowercase()); + } else { + normalized.push(' '); + } + previous = Some(character); + } + normalized } pub struct WitnessLexicalPin { diff --git a/scripts/codestory-etr1-canary.mjs b/scripts/codestory-etr1-canary.mjs new file mode 100644 index 000000000..8c984894f --- /dev/null +++ b/scripts/codestory-etr1-canary.mjs @@ -0,0 +1,83 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { parseArgs } from "node:util"; +import { fileURLToPath } from "node:url"; +import { maximizeCoveredAtoms, scoreOrder, selectSuccessors, sha256, + validateVector } from "./lib/etr1-evidence.mjs"; + +async function main() { + const { values } = parseArgs({ options: { diagnostic: { type: "string" }, + "state-root": { type: "string" }, "output-dir": { type: "string" } } }); + for (const name of ["diagnostic", "state-root", "output-dir"]) + assert.ok(values[name] && path.isAbsolute(values[name]), `missing absolute --${name}`); + await mkdir(values["output-dir"], { mode: 0o700 }); + const documents = [ + { id: "seed", text: "pub fn process() { dispatch(); }\n" }, + { id: "target", text: "fn dispatch() { persist(); }\n" }, + { id: "noise-a", text: "fn render_banner() { paint(); }\n" }, + { id: "noise-b", text: "fn parse_flags() { validate(); }\n" }, + ]; + const question = "How does process reach persistence?"; + const input = { contract: "codestory.embedding-diagnostic-input/v1", records: [ + ...documents.map(({ id, text }) => ({ id, purpose: "document", text })), + { id: "control", purpose: "query", text: question }, + { id: "candidate", purpose: "query", text: `${question}\n\n${documents[0].text}` }, + ] }; + const inputBytes = Buffer.from(JSON.stringify(input)), inputPath = path.join(values["output-dir"], "input.json"); + await writeFile(inputPath, inputBytes, { flag: "wx", mode: 0o600 }); + const vectorPath = path.join(values["state-root"], "canary-vectors.json"); + const execution = spawnSync(values.diagnostic, ["--input", inputPath, "--input-sha256", sha256(inputBytes), + "--state-root", values["state-root"], "--output", vectorPath], + { encoding: "utf8", timeout: 180_000, maxBuffer: 4 * 1024 * 1024, env: process.env }); + assert.equal(execution.error, undefined, `embedding diagnostic failed to launch: ${execution.error}`); + assert.equal(execution.status, 0, `embedding diagnostic failed: ${execution.stderr}`); + const vectorBytes = await readFile(vectorPath), artifact = JSON.parse(vectorBytes); + assert.equal(artifact.contract, "codestory.embedding-diagnostic-output/v1"); + assert.equal(artifact.input_sha256, sha256(inputBytes)); + assert.equal(artifact.records.length, input.records.length); + const vectors = new Map(artifact.records.map((record, index) => { + assert.equal(record.id, input.records[index].id); + assert.equal(record.text_sha256, sha256(input.records[index].text)); + validateVector(record.vector, `canary vector ${record.id}`); + return [record.id, record.vector]; + })); + const dot = (left, right) => left.reduce((sum, value, index) => + Math.fround(sum + Math.fround(Math.fround(value) * Math.fround(right[index]))), 0); + const frontiers = {}; + for (const arm of ["control", "candidate"]) { + const scores = documents.map(({ id }) => dot(vectors.get(arm), vectors.get(id))); + const successors = selectSuccessors(scoreOrder(documents.map(({ id }) => id), scores), + new Set(["seed"]), new Set()); + frontiers[arm] = { scores, successors, legal_pool: ["seed", ...successors] }; + } + const frontierBytes = Buffer.from(JSON.stringify(frontiers)); + await writeFile(path.join(values["output-dir"], "frontiers.json"), frontierBytes, + { flag: "wx", mode: 0o600 }); + // Synthetic truth is constructed only after both frontier outputs are frozen. + const costs = new Map(documents.map(({ id, text }) => [id, Buffer.byteLength(text) + 128])); + const evaluated = Object.fromEntries(Object.entries(frontiers).map(([arm, frontier]) => { + const requirement = frontier.legal_pool.includes("target") ? [["target"]] : [null]; + return [arm, maximizeCoveredAtoms(requirement, costs, 256)]; + })); + const events = await readFile(path.join(values["state-root"], "ipc", + `${process.env.CODESTORY_EMBED_QUALIFICATION_NONCE}.events.jsonl`)); + const completed = events.toString("utf8").trimEnd().split("\n").map(JSON.parse) + .filter((event) => event.action === "completed_tokens"); + assert.ok(completed.length >= 2 && completed.every((event) => Number(event.details.completed_tokens) > 0), + "canary token completion evidence missing"); + const receipt = { contract: "codestory.etr1-synthetic-canary/v1", experiment_status: "valid", + packet_decision: "not_evaluated", input_sha256: sha256(inputBytes), + vectors_sha256: sha256(vectorBytes), frontiers_sha256: sha256(frontierBytes), + diagnostic_binary_sha256: sha256(await readFile(values.diagnostic)), + qualification_events_sha256: sha256(events), completed_token_events: completed.length, + vector_artifact_bytes: (await stat(vectorPath)).size, evaluated }; + const receiptBytes = Buffer.from(`${JSON.stringify(receipt, null, 2)}\n`); + const output = path.join(values["output-dir"], "receipt.json"); + await writeFile(output, receiptBytes, { flag: "wx", mode: 0o600 }); + console.log(`${sha256(receiptBytes)} ${output}`); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) + main().catch((error) => { console.error(error.message); process.exitCode = 1; }); diff --git a/scripts/codestory-etr1-evaluate.mjs b/scripts/codestory-etr1-evaluate.mjs new file mode 100644 index 000000000..71a91457c --- /dev/null +++ b/scripts/codestory-etr1-evaluate.mjs @@ -0,0 +1,212 @@ +import assert from "node:assert/strict"; +import { readFile, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { parseArgs } from "node:util"; +import { fileURLToPath } from "node:url"; +import { evaluateArm, maximizeCoveredAtoms, mean, percentile, requiredFragments, sha256 } from "./lib/etr1-evidence.mjs"; +import { validateEtr1 } from "./codestory-etr1-validate.mjs"; + +async function readBound(file, digest) { + const bytes = await readFile(file); + assert.equal(sha256(bytes), digest, `artifact digest changed: ${file}`); + return { bytes, value: JSON.parse(bytes) }; +} + +function comparableOptimum(actual, expected) { + assert.equal(actual.covered, expected.covered, "oracle covered-atom optimum changed"); + assert.deepEqual(actual.selected, expected.selected, "oracle selected-row optimum changed"); + assert.equal(actual.rows, expected.rows, "oracle row optimum changed"); + assert.equal(actual.public_bytes, expected.public_bytes, "oracle byte optimum changed"); +} + +export async function reproduceOracleFixtures({ oracle, preparation, annotations }) { + assert.equal(oracle.contract, "codestory.fragment-budget-oracle/v1"); + assert.equal(oracle.authority, "answer_aware_post_failure_diagnostic_only"); + assert.equal(oracle.production_use_forbidden, true); + assert.equal(oracle.cases.length, 24, "retained oracle case count changed"); + const repositories = new Map(preparation.repositories.map((value) => [value.repository_id, value])); + const fragments = new Map(preparation.fragments.map((value) => [value.fragment_id, value])); + for (const binding of oracle.cases) { + const { value: retained } = await readBound(binding.path, binding.sha256); + assert.equal((await stat(binding.path)).size, binding.bytes, "retained oracle case length changed"); + assert.equal(retained.case_id, binding.case_id, "retained oracle case identity changed"); + const annotation = annotations.cases.find((value) => value.case_id === retained.case_id); + assert.ok(annotation, "retained oracle annotation missing"); + const repository = repositories.get(retained.repository_id); + assert.ok(repository, "retained oracle repository missing"); + const repositoryFragments = repository.fragment_ids.map((id) => fragments.get(id)); + const rowBytes = new Map(repositoryFragments.map((fragment, index) => [index, fragment.serialized_row_bytes])); + assert.equal(retained.alternatives.length, annotation.acceptable_sets.length, + "retained acceptable-set count changed"); + for (const alternative of retained.alternatives) { + const set = annotation.acceptable_sets.find((value) => value.set_id === alternative.set_id); + assert.ok(set, "retained acceptable set missing"); + const requirements = set.required_source_atoms.map(({ source_range }) => + requiredFragments(source_range, repositoryFragments)); + assert.deepEqual(requirements, alternative.requirements, "retained atom-to-fragment mapping changed"); + comparableOptimum(maximizeCoveredAtoms(requirements, rowBytes, repository.base_serialized_bytes), + alternative.optimum); + } + } + return { status: "reproduced", cases: oracle.cases.length }; +} + +function aggregateArm(cases, name) { + const groups = [...new Set(cases.map((value) => value.group))].sort(); + return { mean_recall: mean(cases.map((value) => value[name].recall)), + complete_set_rate: mean(cases.map((value) => value[name].complete_set_rate)), + groups: Object.fromEntries(groups.map((group) => [group, + { mean_recall: mean(cases.filter((value) => value.group === group).map((value) => value[name].recall)), + complete_set_rate: mean(cases.filter((value) => value.group === group) + .map((value) => value[name].complete_set_rate)) }])), + }; +} + +export function gateOne(aggregate) { + const gates = { mean_recall: aggregate.mean_recall >= 0.85, + every_group_recall: Object.values(aggregate.groups).every((group) => group.mean_recall >= 0.70), + complete_set_rate: aggregate.complete_set_rate >= 0.75, + source_address_authentication: true }; + return { pass: Object.values(gates).every(Boolean), gates }; +} + +function buildCases(scoredRows) { + const ids = [...new Set(scoredRows.map((value) => value.case_id))].sort(); + assert.equal(ids.length, 24, "evaluated case count changed"); + return ids.map((case_id) => { + const rows = scoredRows.filter((value) => value.case_id === case_id) + .toSorted((left, right) => left.phrasing_id.localeCompare(right.phrasing_id)); + assert.deepEqual(rows.map((value) => value.phrasing_id).sort(), + ["original", "paraphrase_1", "paraphrase_2"], "case phrasing set changed"); + const arm = (name) => ({ recall: mean(rows.map((value) => value[name].recall)), + complete_set_rate: mean(rows.map((value) => Number(value[name].complete_source_set))), + incomplete_phrasings: rows.filter((value) => !value[name].complete_source_set).length, + prepared_state_ns: rows.map((value) => value[name].prepared_state_ns) }); + const gainedPhrasings = rows.filter((row) => + row.candidate.reachable_atoms.some((atom) => !row.control.reachable_atoms.includes(atom))).length; + return { case_id, group: rows[0].group, control: arm("control"), candidate: arm("candidate"), + candidate_gained_atom_phrasings: gainedPhrasings, + control_incomplete_for_gain: rows.filter((value) => !value.control.complete_source_set).length >= 2, + candidate_gained_atom: gainedPhrasings >= 2 }; + }); +} + +export function gateTwo(cases, control, candidate, candidateGateOne) { + const recallDelta = candidate.mean_recall - control.mean_recall; + const completeDelta = candidate.complete_set_rate - control.complete_set_rate; + const groupLosses = Object.keys(control.groups).map((group) => + candidate.groups[group].mean_recall - control.groups[group].mean_recall); + const eligible = cases.filter((value) => value.control_incomplete_for_gain); + const gainRate = eligible.length + ? eligible.filter((value) => value.candidate_gained_atom).length / eligible.length : 0; + const gates = { candidate_sufficient: candidateGateOne, + material_recall_or_complete_gain: recallDelta >= 0.10 || completeDelta >= 0.10, + other_measure_not_regressed: recallDelta >= 0 && completeDelta >= 0, + no_group_loss_over_two_points: groupLosses.every((delta) => delta >= -0.02), + new_atom_in_half_of_eligible_cases: gainRate >= 0.5 }; + return { pass: Object.values(gates).every(Boolean), gates, + recall_delta: recallDelta, complete_set_rate_delta: completeDelta, + group_recall_deltas: Object.fromEntries(Object.keys(control.groups) + .map((group, index) => [group, groupLosses[index]])), + eligible_incomplete_cases: eligible.length, + gained_atom_cases: eligible.filter((value) => value.candidate_gained_atom).length, + gained_atom_case_rate: gainRate }; +} + +export function decision(controlGate, candidateGate, conditioningGate) { + if (!controlGate && !candidateGate) return { frontier: null, + decision: "stop_automatic_packet_compilation", reason: "neither_arm_sufficient" }; + if (controlGate && !conditioningGate) return { frontier: "control", + decision: "freeze_unconditioned_frontier_for_selector_experiment", + reason: candidateGate ? "conditioning_lacks_material_value" : "control_alone_sufficient" }; + if (candidateGate && conditioningGate) return { frontier: "candidate", + decision: "freeze_conditioned_frontier_for_selector_experiment", + reason: controlGate ? "conditioning_materially_better" : "conditioning_only_adequate_frontier" }; + return { frontier: null, decision: "stop_automatic_packet_compilation", + reason: "only_conditioned_arm_sufficient_without_material_conditioning_value" }; +} + +export async function evaluateEtr1({ validationPath, validationSha256, annotationsPath, + annotationsSha256, oraclePath, oracleSha256, sourceRoot }) { + const { value: validation } = await readBound(validationPath, validationSha256); + assert.equal(validation.contract, "codestory.etr1-validation/v1"); + assert.equal(validation.experiment_status, "valid", "validator did not authorize annotation access"); + assert.equal(validation.decision, "not_evaluated"); + assert.equal(validation.annotation_access, "not_accessed"); + const validated = await validateEtr1({ runBinding: validation.run, + runPath: validation.run.path, sourceRoot }); + assert.equal(validated.run.build.binary_sha256, validation.binary_sha256, + "validation receipt no longer binds the run binary"); + // This is the first annotation read in the authoritative path. + const { value: annotations } = await readBound(annotationsPath, annotationsSha256); + assert.equal(annotations.authority, "visible_development_only"); + assert.equal(annotations.questions_sha256, + validated.preparation.fixed_inputs.questions.sha256, "annotation question binding changed"); + const { value: oracle } = await readBound(oraclePath, oracleSha256); + const oracle_reproduction = await reproduceOracleFixtures({ oracle, + preparation: validated.preparation, annotations }); + const annotationByCase = new Map(annotations.cases.map((value) => [value.case_id, value])); + const repositoryById = new Map(validated.preparation.repositories + .map((value) => [value.repository_id, value])); + const fragmentById = new Map(validated.preparation.fragments.map((value) => [value.fragment_id, value])); + const scoredRows = validated.rows.map((row) => { + const annotation = annotationByCase.get(row.case_id), repository = repositoryById.get(row.repository_id); + assert.ok(annotation && repository, "evaluation binding missing"); + const repositoryFragments = repository.fragment_ids.map((id) => fragmentById.get(id)); + const score = (name) => ({ ...evaluateArm(annotation, repositoryFragments, + row[name].legally_selectable_pool, repository.base_serialized_bytes), + prepared_state_ns: row[name].timing.prepared_state_ns }); + return { case_id: row.case_id, phrasing_id: row.phrasing_id, group: row.group, + control: score("control"), candidate: score("candidate") }; + }); + const cases = buildCases(scoredRows), control = aggregateArm(cases, "control"), + candidate = aggregateArm(cases, "candidate"), controlGate = gateOne(control), + candidateGate = gateOne(candidate), conditioning = gateTwo(cases, control, candidate, candidateGate.pass); + let selected = decision(controlGate.pass, candidateGate.pass, conditioning.pass); + let latency = { status: "not_evaluated", p95_ns: null, threshold_ns: 1_250_000_000, pass: null }; + if (selected.frontier) { + const timings = scoredRows.map((value) => value[selected.frontier].prepared_state_ns); + const p95 = percentile(timings, 0.95); + latency = { status: "evaluated", p95_ns: p95, threshold_ns: 1_250_000_000, + pass: p95 <= 1_250_000_000 }; + if (!latency.pass) selected = { frontier: selected.frontier, + decision: "authorize_one_byte_identical_latency_repair", reason: "chosen_frontier_latency_failed" }; + } + return { contract: "codestory.etr1-evaluation/v1", authority: "visible_development_frontier_only", + experiment_status: "valid", packet_decision: "not_evaluated", source_address_validity: 1, + inputs: { validation: { path: validationPath, sha256: validationSha256 }, + annotations: { path: annotationsPath, sha256: annotationsSha256 }, + fragment_oracle: { path: oraclePath, sha256: oracleSha256 } }, + oracle_reproduction, aggregates: { control, candidate }, + gates: { control_frontier_sufficiency: controlGate, + candidate_frontier_sufficiency: candidateGate, conditioning_value: conditioning, + frontier_construction_latency: latency }, selected, cases, rows: scoredRows }; +} + +async function main() { + const { values } = parseArgs({ options: Object.fromEntries([ + "validation", "validation-sha256", "annotations", "annotations-sha256", + "fragment-oracle", "fragment-oracle-sha256", "output", + ].map((name) => [name, { type: "string" }])) }); + for (const name of ["validation", "validation-sha256", "annotations", "annotations-sha256", + "fragment-oracle", "fragment-oracle-sha256", "output"]) + assert.ok(values[name], `missing --${name}`); + const sourceRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + let report; + try { + report = await evaluateEtr1({ validationPath: values.validation, + validationSha256: values["validation-sha256"], annotationsPath: values.annotations, + annotationsSha256: values["annotations-sha256"], oraclePath: values["fragment-oracle"], + oracleSha256: values["fragment-oracle-sha256"], sourceRoot }); + } catch (error) { + report = { contract: "codestory.etr1-evaluation/v1", experiment_status: "invalid", + decision: "not_evaluated", packet_decision: "not_evaluated", error: error.message }; + } + const bytes = `${JSON.stringify(report, null, 2)}\n`; + await writeFile(values.output, bytes, { flag: "wx", mode: 0o600 }); + console.log(`${sha256(bytes)} ${values.output}`); + if (report.experiment_status !== "valid") process.exitCode = 1; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) + main().catch((error) => { console.error(error.message); process.exitCode = 1; }); diff --git a/scripts/codestory-etr1-validate.mjs b/scripts/codestory-etr1-validate.mjs new file mode 100644 index 000000000..99e253c6c --- /dev/null +++ b/scripts/codestory-etr1-validate.mjs @@ -0,0 +1,343 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { readFile, realpath, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { parseArgs } from "node:util"; +import { fileURLToPath } from "node:url"; +import { authenticateFragment, encodedCandidateInput, f32Dot, fragmentId, LIMITS, + scoreOrder, selectSuccessors, sha256, validateVector } from "./lib/etr1-evidence.mjs"; + +const FIXED = Object.freeze({ parent: "c9c935d87129a79f326b650bbf23d73191df8b4f", + fragment_diagnostic: "ca185ed13c635bbb4b64cc6760c5025799700359ebcc4dd3bcc53e34f8cf9194", + fragment_build: "2201780e1a752db4bfcceb047bf5cd0b5a854733c4050330ef87575b960f3baf", + lexical_membership_freeze: "e6867b5c79706160021ec5edf60792273345ca97cae8377e69934d5e2c9992ee", + questions: "8e7219a59c973c02f8ea93120bb680da46a75b8272153986c76e55bfb73ca3b6", + annotations: "52b0cc223292bc70f1e4fa3f52b67bf42a91e4d4b9ed997aa12c648c068e9ade", + model_contract: "cb0e3c00290f1eb21ecdcd873521d03331069b1efa766fcd1e493e6d4299b4b7", + model: "666db8df27c88570cdc07adca28646260038b8ca65354911d57b936ebf56efaa", + tokenizer: "7465b93c945b7a266481e6785aa13e505c625562c1c046c4b762bb4da4d46082", + lexical_policy_source: "43b2478d75abd3d5689d05e08c072e4148fd21ab29bcc55533d30a494edf986b" }); + +async function boundBytes(binding, expectedPath) { + assert.ok(binding && path.isAbsolute(binding.path), "bound path is not absolute"); + if (expectedPath) assert.equal(await realpath(binding.path), await realpath(expectedPath), "bound path differs"); + const metadata = await stat(binding.path); + assert.ok(metadata.isFile(), "bound path is not a regular file"); + const bytes = await readFile(binding.path); + assert.equal(bytes.length, binding.bytes, "bound length changed"); + assert.equal(sha256(bytes), binding.sha256, "bound digest changed"); + return bytes; +} + +async function boundJson(binding, expectedPath) { + return JSON.parse(await boundBytes(binding, expectedPath)); +} + +function renderSnippet(fragment) { + let result = "```text\n"; + const lines = fragment.source.match(/.*(?:\n|$)/gu).filter(Boolean); + lines.forEach((line, offset) => { + result += ` ${String(fragment.line_range.start + offset).padStart(5)} | ${line.replace(/[\r\n]+$/u, "")}\n`; + }); + return `${result}\`\`\``; +} + +function serializedRowBytes(fragment) { + return Buffer.byteLength(JSON.stringify({ kind: "source_range", path: fragment.path, + start_line: fragment.line_range.start, end_line: fragment.line_range.end, + snippet: renderSnippet(fragment), content_digest: fragment.content_digest, + byte_range: fragment.byte_range })); +} + +export function validateEngine(engine) { + assert.equal(engine.model_digest, FIXED.model, "engine model changed"); + assert.equal(engine.materialized_model_sha256, FIXED.model, "materialized model changed"); + assert.equal(engine.policy, "accelerated", "engine policy changed"); + assert.equal(engine.accelerator_execution_verified, true, "accelerator execution unverified"); + assert.equal(engine.worker_alive, true, "embedding worker is not alive"); + assert.equal(engine.embedded_model, true, "embedding model was not embedded"); + assert.equal(engine.load_error, null, "embedding engine recorded a load error"); +} + +export function validatePreAnnotationBoundary(run, preparation) { + assert.equal(run.annotation_access, "not_accessed", "run accessed annotations before validation"); + assert.equal(preparation.annotation_access, "not_accessed", + "preparation accessed annotations before validation"); + for (const key of ["graph_invocations", "bge_invocations", "symbol_document_invocations", + "host_query_invocations", "production_packet_invocations"]) + assert.equal(run[key], 0, `${key} is forbidden`); +} + +export function validateDocumentVectorRecord(record, fragment) { + assert.equal(record.id, fragment.fragment_id, "document vector order changed"); + assert.equal(record.purpose, "document", "symbol or query document substituted"); + assert.equal(record.text_sha256, sha256(fragment.source), "document vector text changed"); + validateVector(record.vector, "document vector"); +} + +function validateSourceAuthentication(arm, fragments, repository, sourceFiles) { + assert.deepEqual(arm.source_authentication.authenticated_fragment_ids, arm.hydrated_pool, + "source authentication differs from H"); + let fragmentBytes = 0; + const paths = new Set(); + for (const id of arm.hydrated_pool) { + const fragment = fragments.get(id); + assert.ok(fragment, "hydrated fragment is absent"); + assert.equal(fragment.project_id, repository.project_id, "hydrated fragment belongs to another project"); + authenticateFragment(fragment, sourceFiles.get(fragment.path)); + fragmentBytes += Buffer.byteLength(fragment.source); + paths.add(fragment.path); + } + assert.equal(arm.source_authentication.fragment_source_bytes, fragmentBytes, + "authenticated fragment byte total changed"); + const filesystemBytes = [...paths].reduce((sum, relative) => sum + sourceFiles.get(relative).length, 0); + assert.equal(arm.source_authentication.filesystem_bytes_read, filesystemBytes, + "filesystem byte total changed"); + assert.deepEqual(Object.keys(arm.source_authentication.file_digests).sort(), [...paths].sort(), + "authenticated file set changed"); + for (const relative of paths) + assert.equal(arm.source_authentication.file_digests[relative], sha256(sourceFiles.get(relative)), + "authenticated file digest changed"); +} + +export function validateArm({ arm, expectedName, wording, repository, fragments, documentVectors, + sourceFiles, batches }) { + assert.equal(arm.name, expectedName); + const seeds = wording.seed_fragment_ids; + assert.equal(arm.search_count, seeds.length, "logical search count changed"); + assert.equal(arm.query_receipts.length, seeds.length, "query receipt count changed"); + assert.ok(arm.successors.length <= LIMITS.successors, "successor ceiling exceeded"); + assert.equal(new Set(arm.successors).size, arm.successors.length, "successors are duplicated"); + assert.deepEqual(arm.descriptor_pool, [...seeds, ...arm.successors], "D equation changed"); + assert.ok(arm.descriptor_pool.length <= LIMITS.pool, "descriptor ceiling exceeded"); + assert.deepEqual(arm.hydrated_pool, arm.descriptor_pool, "H differs from fully authenticated D"); + assert.deepEqual(arm.legally_selectable_pool, [...new Set(arm.hydrated_pool)].filter((id) => { + const fragment = fragments.get(id); + return repository.base_serialized_bytes + fragment.serialized_row_bytes <= LIMITS.bytes; + }), "L equation changed"); + const seedSet = new Set(seeds), prior = new Set(); + for (let ordinal = 0; ordinal < arm.query_receipts.length; ordinal++) { + const query = arm.query_receipts[ordinal], seed = fragments.get(seeds[ordinal]); + assert.equal(query.query_ordinal, ordinal, "query ordinal changed"); + assert.equal(query.seed_fragment_id, seeds[ordinal], "query seed changed"); + const original = expectedName === "control" ? wording.question : `${wording.question}\n\n${seed.source}`; + assert.equal(query.original_input_sha256, sha256(original), "original query digest changed"); + const encoded = expectedName === "control" ? wording.question + : encodedCandidateInput(wording.question, seed.source, query.removed_trailing_source_lines); + assert.equal(query.encoded_input, encoded, "encoded query construction changed"); + assert.equal(query.encoded_input_sha256, sha256(encoded), "encoded query digest changed"); + if (expectedName === "control") { + assert.equal(query.removed_trailing_source_lines, 0, "control query was shortened"); + assert.equal(query.model_limit_rejections, 0, "control query exceeded the model limit"); + } else { + assert.ok(query.model_limit_rejections >= query.removed_trailing_source_lines, + "candidate shortening lacks a typed model-limit rejection"); + } + validateVector(query.query_vector, "query vector"); + assert.equal(query.score_order_sha256, repository.score_order_sha256, "score-order binding changed"); + assert.equal(query.scores.length, repository.fragment_ids.length, "complete score vector missing"); + query.scores.forEach((score, index) => { + assert.ok(Number.isFinite(score), "semantic score is nonfinite"); + const recomputed = f32Dot(query.query_vector, documentVectors.get(repository.fragment_ids[index])); + assert.ok(Math.abs(score - recomputed) <= 2e-6, "semantic score differs from normalized dot product"); + }); + const expectedExclusions = [...new Set([...seedSet, ...prior])].sort(); + assert.deepEqual(query.excluded_before, expectedExclusions, "cumulative exclusions changed"); + const selected = selectSuccessors(scoreOrder(repository.fragment_ids, query.scores), seedSet, prior); + assert.deepEqual(query.retained_successors, selected, "top-eight successor selection changed"); + selected.forEach((id) => prior.add(id)); + const batch = batches.get(query.global_batch_ordinal); + assert.ok(batch && batch.arm === expectedName, "query references the wrong successful batch"); + const position = batch.query_ordinals.indexOf(ordinal); + assert.ok(position >= 0, "query missing from its successful batch"); + assert.equal(batch.input_sha256[position], query.encoded_input_sha256, + "successful batch input digest changed"); + } + assert.deepEqual(arm.successors, [...prior], "successor pool equation changed"); + assert.ok(arm.batch_receipts.every((batch) => batch.query_ordinals.length >= 1 + && batch.query_ordinals.length <= 8 && batch.wall_ns > 0 && batch.completed_tokens > 0), + "batch contract invalid"); + const batchOrdinals = arm.batch_receipts.flatMap((batch) => batch.query_ordinals).toSorted((a, b) => a - b); + assert.deepEqual(batchOrdinals, Array.from({ length: seeds.length }, (_, index) => index), + "successful batches do not partition the queries"); + assert.equal(arm.token_total, + arm.batch_receipts.reduce((sum, batch) => sum + batch.completed_tokens, 0), "arm token total changed"); + const timingKeys = ["round_zero_bm25_ns", "seed_source_authentication_ns", "query_encoding_ns", + "vector_search_ns", "descriptor_mapping_ns", "remaining_source_authentication_ns"]; + timingKeys.forEach((key) => assert.ok(Number.isSafeInteger(arm.timing[key]) && arm.timing[key] >= 0, + `invalid timing ${key}`)); + assert.equal(arm.timing.prepared_state_ns, + timingKeys.reduce((sum, key) => sum + arm.timing[key], 0), "prepared timing does not reconcile"); + validateSourceAuthentication(arm, fragments, repository, sourceFiles); +} + +function parseEvents(bytes) { + assert.equal(bytes.at(-1), 10, "qualification event log is unterminated"); + let previous = 0; + return bytes.toString("utf8").trimEnd().split("\n").map(JSON.parse).filter((event) => { + assert.equal(event.schema_version, 1, "qualification event schema changed"); + assert.ok(event.sequence > previous, "qualification event sequence changed"); + previous = event.sequence; + return event.action === "completed_tokens"; + }); +} + +async function loadSourceFiles(repository, repositoryFragments) { + const root = await realpath(repository.local_root), result = new Map(); + for (const relative of new Set(repositoryFragments.map((fragment) => fragment.path))) { + assert.ok(relative && !path.isAbsolute(relative) && !relative.includes("\\") + && !relative.split("/").some((part) => !part || part === "." || part === ".."), + "source path escapes project"); + const absolute = await realpath(path.join(root, relative)); + assert.ok(absolute.startsWith(`${root}${path.sep}`), "source path escapes project"); + result.set(relative, await readFile(absolute)); + } + return result; +} + +export async function validateEtr1({ runBinding, runPath, sourceRoot }) { + const run = await boundJson(runBinding, runPath); + assert.equal(run.contract, "codestory.etr1-run/v1"); + assert.equal(run.authority, "visible_development_frontier_only"); + assert.equal(run.experiment_status, "awaiting_validation"); + assert.equal(run.decision, "not_evaluated"); + assert.equal(run.parent_head, FIXED.parent); + assert.equal(run.annotation_access, "not_accessed"); + assert.equal(run.vector_artifact_loaded_before_timing, true); + const sourceCommit = execFileSync("git", ["-C", sourceRoot, "rev-parse", "HEAD^{commit}"], + { encoding: "utf8", env: { ...process.env, GIT_OPTIONAL_LOCKS: "0", GIT_TERMINAL_PROMPT: "0" } }).trim(); + const sourceTree = execFileSync("git", ["-C", sourceRoot, "rev-parse", "HEAD^{tree}"], + { encoding: "utf8", env: { ...process.env, GIT_OPTIONAL_LOCKS: "0", GIT_TERMINAL_PROMPT: "0" } }).trim(); + assert.equal(run.build.source_commit, sourceCommit, "run source commit differs from validator checkout"); + assert.equal(run.build.source_tree, sourceTree, "run source tree differs from validator checkout"); + assert.equal(run.build.source_dirty, false, "run binary was built dirty"); + assert.equal(sha256(await readFile(run.build.binary_path)), run.build.binary_sha256, "run binary changed"); + const preparation = await boundJson(run.preparation); + assert.equal(preparation.contract, "codestory.etr1-preparation/v1"); + validatePreAnnotationBoundary(run, preparation); + assert.equal(preparation.annotations.sha256, FIXED.annotations); + assert.equal(preparation.model_sha256, FIXED.model); + assert.equal(preparation.tokenizer_sha256, FIXED.tokenizer); + assert.equal(preparation.build.source_commit, sourceCommit); + assert.equal(preparation.build.source_tree, sourceTree); + assert.equal(run.method_sha256, preparation.method.sha256); + for (const name of ["fragment_diagnostic", "fragment_build", "lexical_membership_freeze", + "questions", "model_contract", "lexical_policy_source"]) + assert.equal(preparation.fixed_inputs[name].sha256, FIXED[name], `fixed input changed: ${name}`); + await boundBytes(preparation.method); + await boundBytes(preparation.embedding_input); + for (const binding of Object.values(preparation.fixed_inputs)) await boundBytes(binding); + assert.equal(preparation.fragments.length, 10_369, "fragment count changed"); + assert.equal(preparation.wordings.length, 72, "wording count changed"); + const fragments = new Map(), fragmentsByRepository = new Map(); + for (const fragment of preparation.fragments) { + assert.equal(fragment.fragment_id, fragmentId(fragment), "prepared fragment identity changed"); + assert.equal(fragment.serialized_row_bytes, serializedRowBytes(fragment), "public row cost changed"); + assert.ok(!fragments.has(fragment.fragment_id), "duplicate prepared fragment"); + fragments.set(fragment.fragment_id, fragment); + const list = fragmentsByRepository.get(fragment.project_id) ?? []; + list.push(fragment); + fragmentsByRepository.set(fragment.project_id, list); + } + const repositories = new Map(preparation.repositories.map((repository) => [repository.repository_id, repository])); + for (const repository of repositories.values()) { + assert.equal(sha256(JSON.stringify(repository.fragment_ids)), repository.score_order_sha256, + "repository score order changed"); + assert.equal(new Set(repository.fragment_ids).size, repository.fragment_ids.length, + "repository fragment order contains duplicates"); + } + const vectorArtifact = await boundJson(run.fragment_vectors); + assert.equal(vectorArtifact.contract, "codestory.embedding-diagnostic-output/v1"); + assert.equal(vectorArtifact.input_sha256, preparation.embedding_input.sha256); + assert.equal(vectorArtifact.source_commit, sourceCommit); + assert.equal(vectorArtifact.source_tree, sourceTree); + validateEngine(vectorArtifact.initial_engine); + validateEngine(vectorArtifact.final_engine); + assert.equal(vectorArtifact.initial_engine.server_instance_id, vectorArtifact.final_engine.server_instance_id); + assert.equal(vectorArtifact.records.length, preparation.fragments.length, "document vector count changed"); + const documentVectors = new Map(); + vectorArtifact.records.forEach((record, index) => { + const fragment = preparation.fragments[index]; + validateDocumentVectorRecord(record, fragment); + assert.ok(!documentVectors.has(record.id), "duplicate document vector"); + documentVectors.set(record.id, record.vector); + }); + validateEngine(run.initial_engine); + validateEngine(run.final_engine); + assert.equal(run.initial_engine.server_instance_id, run.final_engine.server_instance_id, + "query engine changed during ETR-1"); + const eventBytes = await boundBytes(run.qualification_events), events = parseEvents(eventBytes); + const batches = new Map(), rows = []; + assert.equal(run.rows.length, preparation.wordings.length, "run row count changed"); + for (let rowIndex = 0; rowIndex < run.rows.length; rowIndex++) { + const row = await boundJson(run.rows[rowIndex]), wording = preparation.wordings[rowIndex]; + assert.equal(row.contract, "codestory.etr1-wording/v1"); + for (const key of ["case_id", "phrasing_id", "repository_id", "group", "question_sha256"]) + assert.equal(row[key], wording[key], `row ${key} changed`); + assert.deepEqual(row.seed_fragment_ids, wording.seed_fragment_ids, "arm seed manifest changed"); + const repository = repositories.get(wording.repository_id); + assert.ok(repository, "row repository missing"); + const repositoryFragments = repository.fragment_ids.map((id) => fragments.get(id)); + const sourceFiles = await loadSourceFiles(repository, repositoryFragments); + for (const fragment of repositoryFragments) authenticateFragment(fragment, sourceFiles.get(fragment.path)); + for (const arm of [row.control, row.candidate]) for (const batch of arm.batch_receipts) { + assert.ok(!batches.has(batch.global_batch_ordinal), "global batch ordinal duplicated"); + batches.set(batch.global_batch_ordinal, batch); + } + validateArm({ arm: row.control, expectedName: "control", wording, repository, fragments, + documentVectors, sourceFiles, batches }); + validateArm({ arm: row.candidate, expectedName: "candidate", wording, repository, fragments, + documentVectors, sourceFiles, batches }); + rows.push(row); + } + assert.deepEqual([...batches.keys()].sort((a, b) => a - b), + Array.from({ length: batches.size }, (_, index) => index), "global batch sequence has gaps"); + assert.equal(events.length, batches.size, "qualification event count differs from successful batches"); + for (const [ordinal, batch] of [...batches].sort((left, right) => left[0] - right[0])) { + const event = events[ordinal]; + assert.equal(event.sequence, batch.qualification_event_sequence, "batch event sequence changed"); + assert.equal(Number(event.details.completed_tokens), batch.completed_tokens, "batch token count changed"); + assert.ok(event.details.request_id, "batch request identity missing"); + assert.ok(Number(event.details.native_completion_sequence) > 0, "native completion identity missing"); + } + assert.equal(run.qualification_completed_token_total, + [...batches.values()].reduce((sum, batch) => sum + batch.completed_tokens, 0), + "qualification token total changed"); + return { run, preparation, rows, source_commit: sourceCommit, source_tree: sourceTree, + batch_count: batches.size, query_count: rows.reduce((sum, row) => + sum + row.control.search_count + row.candidate.search_count, 0) }; +} + +async function main() { + const { values } = parseArgs({ options: { + run: { type: "string" }, "run-sha256": { type: "string" }, output: { type: "string" }, + } }); + for (const name of ["run", "run-sha256", "output"]) assert.ok(values[name], `missing --${name}`); + assert.ok(path.isAbsolute(values.run) && path.isAbsolute(values.output), "paths must be absolute"); + const sourceRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + let receipt; + try { + const validated = await validateEtr1({ runBinding: { path: values.run, + sha256: values["run-sha256"], bytes: (await stat(values.run)).size }, runPath: values.run, sourceRoot }); + receipt = { contract: "codestory.etr1-validation/v1", experiment_status: "valid", + decision: "not_evaluated", annotation_access: "not_accessed", + run: { path: values.run, sha256: values["run-sha256"], bytes: (await stat(values.run)).size }, + source_commit: validated.source_commit, source_tree: validated.source_tree, + binary_sha256: validated.run.build.binary_sha256, + preparation_sha256: validated.run.preparation.sha256, + fragment_vectors_sha256: validated.run.fragment_vectors.sha256, + row_count: validated.rows.length, batch_count: validated.batch_count, + query_count: validated.query_count, source_address_validity: 1 }; + } catch (error) { + receipt = { contract: "codestory.etr1-validation/v1", experiment_status: "invalid", + decision: "not_evaluated", annotation_access: "not_accessed", + run: { path: values.run, sha256: values["run-sha256"] }, error: error.message }; + } + const bytes = `${JSON.stringify(receipt, null, 2)}\n`; + await writeFile(values.output, bytes, { flag: "wx", mode: 0o600 }); + console.log(`${sha256(bytes)} ${values.output}`); + if (receipt.experiment_status !== "valid") process.exitCode = 1; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) + main().catch((error) => { console.error(error.message); process.exitCode = 1; }); diff --git a/scripts/lib/etr1-evidence.mjs b/scripts/lib/etr1-evidence.mjs new file mode 100644 index 000000000..af678e653 --- /dev/null +++ b/scripts/lib/etr1-evidence.mjs @@ -0,0 +1,196 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; + +export const LIMITS = Object.freeze({ rows: 16, bytes: 16 * 1024, seeds: 16, + successorsPerQuery: 8, successors: 128, pool: 144, vectorDimension: 768 }); + +export const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex"); + +export function fragmentId(fragment) { + const framed = []; + for (const value of [fragment.project_id, fragment.path, fragment.content_digest]) { + const bytes = Buffer.from(value); + const length = Buffer.alloc(8); + length.writeBigUInt64LE(BigInt(bytes.length)); + framed.push(length, bytes); + } + const bounds = Buffer.alloc(16); + bounds.writeBigUInt64LE(BigInt(fragment.byte_range.start), 0); + bounds.writeBigUInt64LE(BigInt(fragment.byte_range.end), 8); + return sha256(Buffer.concat([Buffer.from("codestory.frozen-fragment/v1\0"), ...framed, bounds])); +} + +export function validateVector(vector, label = "vector") { + assert.equal(vector.length, LIMITS.vectorDimension, `${label} dimension changed`); + assert.ok(vector.every(Number.isFinite), `${label} contains a nonfinite value`); + const norm = vector.reduce((sum, value) => sum + value * value, 0); + assert.ok(Math.abs(norm - 1) < 0.001, `${label} is not normalized`); +} + +export function f32Dot(left, right) { + assert.equal(left.length, right.length, "dot-product dimensions differ"); + let sum = 0; + for (let index = 0; index < left.length; index++) + sum = Math.fround(sum + Math.fround(Math.fround(left[index]) * Math.fround(right[index]))); + return sum; +} + +export function scoreOrder(fragmentIds, scores) { + assert.equal(fragmentIds.length, scores.length, "score vector length differs from repository order"); + return fragmentIds.map((id, index) => ({ id, score: scores[index] })) + .toSorted((left, right) => right.score - left.score || left.id.localeCompare(right.id)); +} + +export function selectSuccessors(order, seeds, prior, limit = LIMITS.successorsPerQuery) { + const excluded = new Set([...seeds, ...prior]), seen = new Set(), selected = []; + for (const { id } of order) { + if (!excluded.has(id) && !seen.has(id)) { + seen.add(id); + selected.push(id); + if (selected.length === limit) break; + } + } + return selected; +} + +export function encodedCandidateInput(question, source, removedTrailingLines) { + assert.ok(Number.isSafeInteger(removedTrailingLines) && removedTrailingLines >= 0, + "invalid removed-line count"); + const lines = source.match(/.*(?:\n|$)/gu).filter(Boolean); + assert.ok(lines.length > removedTrailingLines, "all seed lines were removed"); + const retained = lines.slice(0, lines.length - removedTrailingLines).join(""); + assert.ok(retained.trim(), "retained seed source is empty"); + return `${question}\n\n${retained}`; +} + +function lineOffsets(bytes) { + const offsets = [0]; + for (let index = 0; index < bytes.length; index++) if (bytes[index] === 10) offsets.push(index + 1); + if (offsets.at(-1) !== bytes.length) offsets.push(bytes.length); + return offsets; +} + +export function authenticateFragment(fragment, bytes) { + assert.equal(fragment.fragment_id, fragmentId(fragment), "fragment identity changed"); + assert.equal(sha256(bytes), fragment.content_digest, "fragment file digest changed"); + const { start, end } = fragment.byte_range; + assert.ok(Number.isSafeInteger(start) && Number.isSafeInteger(end) + && start >= 0 && end > start && end <= bytes.length, "fragment range invalid"); + for (const offset of [start, end]) + assert.ok(offset === bytes.length || (bytes[offset] & 0xc0) !== 0x80, "fragment range splits UTF-8"); + assert.equal(bytes.subarray(start, end).toString("utf8"), fragment.source, "fragment source changed"); + const offsets = lineOffsets(bytes); + assert.equal(offsets.filter((offset) => offset <= start).length, fragment.line_range.start, + "fragment start line changed"); + assert.equal(offsets.filter((offset) => offset < end).length, fragment.line_range.end, + "fragment end line changed"); +} + +export function requiredFragments(atom, fragments) { + const matching = fragments.map((fragment, index) => ({ fragment, index })) + .filter(({ fragment }) => fragment.path === atom.path && fragment.content_digest === atom.content_digest) + .toSorted((left, right) => left.fragment.byte_range.start - right.fragment.byte_range.start); + for (let index = 1; index < matching.length; index++) + assert.ok(matching[index - 1].fragment.byte_range.end <= matching[index].fragment.byte_range.start, + "frozen fragments overlap"); + const required = []; + let cursor = atom.byte_range.start; + for (const { fragment, index } of matching) { + const range = fragment.byte_range; + if (range.end <= atom.byte_range.start || range.start >= atom.byte_range.end) continue; + if (range.start > cursor) return null; + required.push(index); + cursor = Math.max(cursor, range.end); + } + return cursor >= atom.byte_range.end ? required : null; +} + +export function exactPublicBytes(baseBytes, selected, rowBytes) { + return baseBytes + selected.reduce((sum, id) => sum + rowBytes.get(id), 0) + + Math.max(0, selected.length - 1); +} + +function compareIdentity(left, right) { + if (typeof left === "number" && typeof right === "number") return left - right; + assert.equal(typeof left, typeof right, "optimizer identity types differ"); + return left < right ? -1 : left > right ? 1 : 0; +} + +/** Exact union-of-whole-atom optimizer. Requirements are arrays of selectable IDs or null. */ +export function maximizeCoveredAtoms(requirements, rowBytes, baseBytes, + limits = { rows: LIMITS.rows, bytes: LIMITS.bytes }) { + assert.ok(Number.isSafeInteger(baseBytes) && baseBytes >= 0 && baseBytes <= limits.bytes, + "invalid fixed packet bytes"); + const ids = [...new Set(requirements.filter(Boolean).flat())].sort(compareIdentity); + ids.forEach((id) => assert.ok(Number.isSafeInteger(rowBytes.get(id)) && rowBytes.get(id) > 0, + `missing row cost for ${id}`)); + for (const requirement of requirements) if (requirement) + assert.ok(requirement.length > 0 && new Set(requirement).size === requirement.length, + "atom requirement is empty or duplicated"); + const bitFor = new Map(ids.map((id, index) => [id, 1n << BigInt(index)])); + const masks = requirements.map((requirement) => requirement == null ? null + : requirement.reduce((mask, id) => mask | bitFor.get(id), 0n)); + const states = new Map([[0n, { selected: [], bytes: baseBytes }]]); + for (const requirement of new Set(masks.filter((mask) => mask != null))) { + for (const [mask, state] of [...states]) { + const union = mask | requirement; + if (states.has(union)) continue; + const selected = ids.filter((id) => (union & bitFor.get(id)) !== 0n); + if (selected.length > limits.rows) continue; + const bytes = exactPublicBytes(baseBytes, selected, rowBytes); + if (bytes <= limits.bytes) states.set(union, { selected, bytes }); + } + } + let best = { mask: 0n, covered: 0, selected: [], rows: 0, public_bytes: baseBytes }; + for (const [mask, state] of states) { + const covered = masks.filter((required) => required != null && (required & mask) === required).length; + const candidate = { mask, covered, selected: state.selected, rows: state.selected.length, + public_bytes: state.bytes }; + if (candidate.covered > best.covered + || (candidate.covered === best.covered && (candidate.rows < best.rows + || (candidate.rows === best.rows && (candidate.public_bytes < best.public_bytes + || (candidate.public_bytes === best.public_bytes && candidate.mask < best.mask)))))) best = candidate; + } + return { covered: best.covered, selected: best.selected, rows: best.rows, + public_bytes: best.public_bytes, feasible_states: states.size }; +} + +export function evaluateAlternative(set, repositoryFragments, legalIds, baseBytes) { + const legal = new Set(legalIds), rowBytes = new Map(); + repositoryFragments.forEach((fragment, index) => rowBytes.set(index, fragment.serialized_row_bytes)); + const atoms = set.required_source_atoms; + const sourceRequirements = atoms.map(({ source_range }) => requiredFragments(source_range, repositoryFragments)); + const requirements = sourceRequirements.map((required) => required != null + && required.every((index) => legal.has(repositoryFragments[index].fragment_id)) ? required : null); + const optimum = maximizeCoveredAtoms(requirements, rowBytes, baseBytes); + const reachable_atoms = atoms.filter((_, index) => { + const required = requirements[index]; + if (required == null) return false; + return required.length <= LIMITS.rows + && exactPublicBytes(baseBytes, required, rowBytes) <= LIMITS.bytes; + }).map(({ atom_id }) => atom_id); + return { set_id: set.set_id, required_atoms: atoms.length, requirements, reachable_atoms, optimum, + recall: atoms.length ? optimum.covered / atoms.length : 0, + complete_source_set: optimum.covered === atoms.length }; +} + +export function evaluateArm(annotation, repositoryFragments, legalIds, baseBytes) { + assert.ok(annotation.acceptable_sets.length > 0, "case has no acceptable source set"); + const alternatives = annotation.acceptable_sets.map((set) => + evaluateAlternative(set, repositoryFragments, legalIds, baseBytes)); + const best = alternatives.toSorted((left, right) => right.recall - left.recall + || left.set_id.localeCompare(right.set_id))[0]; + const reachable_atoms = [...new Set(alternatives.flatMap((value) => value.reachable_atoms))].sort(); + return { best_set_id: best.set_id, recall: best.recall, + complete_source_set: best.complete_source_set, rows: best.optimum.rows, + public_bytes: best.optimum.public_bytes, selected_fragment_indexes: best.optimum.selected, + reachable_atoms, alternatives }; +} + +export const mean = (values) => values.reduce((sum, value) => sum + value, 0) / values.length; + +export function percentile(values, probability) { + assert.ok(values.length > 0 && probability > 0 && probability <= 1, "invalid percentile input"); + const ordered = values.toSorted((left, right) => left - right); + return ordered[Math.max(0, Math.ceil(probability * ordered.length) - 1)]; +} diff --git a/scripts/tests/etr1-evidence.test.mjs b/scripts/tests/etr1-evidence.test.mjs new file mode 100644 index 000000000..76852c0bf --- /dev/null +++ b/scripts/tests/etr1-evidence.test.mjs @@ -0,0 +1,207 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { authenticateFragment, encodedCandidateInput, evaluateArm, exactPublicBytes, fragmentId, + LIMITS, maximizeCoveredAtoms, scoreOrder, selectSuccessors, sha256 } from "../lib/etr1-evidence.mjs"; +import { validateArm, validateDocumentVectorRecord, validateEngine, + validatePreAnnotationBoundary } from "../codestory-etr1-validate.mjs"; +import { decision, evaluateEtr1, gateOne, gateTwo } from "../codestory-etr1-evaluate.mjs"; + +function unit(index) { + const vector = Array(LIMITS.vectorDimension).fill(0); + vector[index] = 1; + return vector; +} + +function fixture(expectedName = "control") { + const bytes = Buffer.from("seed\nsuccessor\n"), digest = sha256(bytes), project_id = "project"; + const make = (path, start, end, line, source) => { + const fragment = { project_id, path, content_digest: digest, byte_range: { start, end }, + line_range: { start: line, end: line }, source, serialized_row_bytes: 80 }; + fragment.fragment_id = fragmentId(fragment); + return fragment; + }; + const seed = make("src/lib.rs", 0, 5, 1, "seed\n"); + const successor = make("src/lib.rs", 5, bytes.length, 2, "successor\n"); + const repository = { project_id, fragment_ids: [seed.fragment_id, successor.fragment_id], + base_serialized_bytes: 100, + score_order_sha256: sha256(JSON.stringify([seed.fragment_id, successor.fragment_id])) }; + const question = "find successor", wording = { question, seed_fragment_ids: [seed.fragment_id] }; + const input = expectedName === "control" ? question : `${question}\n\n${seed.source}`; + const batch = { global_batch_ordinal: 0, arm: expectedName, query_ordinals: [0], + input_sha256: [sha256(input)], wall_ns: 1, completed_tokens: 2, + qualification_event_sequence: 1 }; + const arm = { name: expectedName, search_count: 1, + query_receipts: [{ query_ordinal: 0, seed_fragment_id: seed.fragment_id, + original_input_sha256: sha256(input), encoded_input_sha256: sha256(input), encoded_input: input, + removed_trailing_source_lines: 0, model_limit_rejections: 0, global_batch_ordinal: 0, + score_order_sha256: repository.score_order_sha256, query_vector: unit(0), scores: [0, 1], + excluded_before: [seed.fragment_id], retained_successors: [successor.fragment_id] }], + batch_receipts: [batch], successors: [successor.fragment_id], + descriptor_pool: [seed.fragment_id, successor.fragment_id], + hydrated_pool: [seed.fragment_id, successor.fragment_id], + legally_selectable_pool: [seed.fragment_id, successor.fragment_id], + source_authentication: { fragment_source_bytes: bytes.length, filesystem_bytes_read: bytes.length, + authenticated_fragment_ids: [seed.fragment_id, successor.fragment_id], + file_digests: { "src/lib.rs": digest } }, token_total: 2, + timing: { round_zero_bm25_ns: 1, seed_source_authentication_ns: 1, query_encoding_ns: 1, + vector_search_ns: 1, descriptor_mapping_ns: 1, remaining_source_authentication_ns: 1, + prepared_state_ns: 6 } }; + return { arm, expectedName, wording, repository, + fragments: new Map([[seed.fragment_id, seed], [successor.fragment_id, successor]]), + documentVectors: new Map([[seed.fragment_id, unit(1)], [successor.fragment_id, unit(0)]]), + sourceFiles: new Map([["src/lib.rs", bytes]]), batches: new Map([[0, batch]]), seed, successor }; +} + +test("fragment identity and source authentication bind every coordinate", () => { + const { seed, sourceFiles } = fixture(); + authenticateFragment(seed, sourceFiles.get(seed.path)); + for (const mutation of [ + { ...seed, project_id: "other" }, { ...seed, path: "src/other.rs" }, + { ...seed, byte_range: { start: 1, end: 5 } }, { ...seed, source: "fake\n" }, + ]) assert.throws(() => authenticateFragment(mutation, sourceFiles.get(seed.path))); +}); + +test("cumulative exclusions page deterministically through 128 unique successors", () => { + const seeds = new Set(Array.from({ length: 16 }, (_, index) => `seed-${index}`)); + const order = [...seeds, ...Array.from({ length: 140 }, (_, index) => `successor-${String(index).padStart(3, "0")}`)] + .map((id, index) => ({ id, score: 1 - index / 1000 })); + const prior = new Set(); + for (let query = 0; query < 16; query++) + selectSuccessors(order, seeds, prior).forEach((id) => prior.add(id)); + assert.equal(prior.size, 128); + assert.deepEqual([...prior].slice(0, 8), Array.from({ length: 8 }, (_, index) => + `successor-${String(index).padStart(3, "0")}`)); + const tied = scoreOrder(["z", "a", "m"], [0.5, 0.5, 0.4]); + assert.deepEqual(tied.map(({ id }) => id), ["a", "z", "m"]); +}); + +test("candidate input shortening preserves UTF-8 and complete trailing lines", () => { + const source = "first α\nsecond β\nthird γ\n"; + assert.equal(encodedCandidateInput("question", source, 1), "question\n\nfirst α\nsecond β\n"); + assert.throws(() => encodedCandidateInput("question", source, 3)); +}); + +test("validator reconstructs both arm query contracts and refuses hostile mutations", () => { + for (const name of ["control", "candidate"]) validateArm(fixture(name)); + const mutations = [ + (value) => { value.wording.question = "changed"; }, + (value) => { value.arm.query_receipts[0].encoded_input = "find successor\nseed\n"; }, + (value) => { value.arm.query_receipts[0].scores.pop(); }, + (value) => { value.arm.successors.push(value.successor.fragment_id); }, + (value) => { value.arm.query_receipts[0].excluded_before = []; }, + (value) => { value.arm.query_receipts[0].encoded_input = "find successor"; value.arm.query_receipts[0].removed_trailing_source_lines = 1; }, + ]; + for (const mutate of mutations) { + const value = fixture("candidate"); + mutate(value); + assert.throws(() => validateArm(value)); + } +}); + +test("vector, model, and pre-annotation boundaries refuse substitutions", () => { + const { seed } = fixture(); + validateDocumentVectorRecord({ id: seed.fragment_id, purpose: "document", + text_sha256: sha256(seed.source), vector: unit(0) }, seed); + assert.throws(() => validateDocumentVectorRecord({ id: seed.fragment_id, purpose: "symbol", + text_sha256: sha256(seed.source), vector: unit(0) }, seed)); + const engine = { model_digest: "666db8df27c88570cdc07adca28646260038b8ca65354911d57b936ebf56efaa", + materialized_model_sha256: "666db8df27c88570cdc07adca28646260038b8ca65354911d57b936ebf56efaa", + policy: "accelerated", accelerator_execution_verified: true, worker_alive: true, + embedded_model: true, load_error: null }; + validateEngine(engine); + assert.throws(() => validateEngine({ ...engine, model_digest: "f".repeat(64) })); + const run = { annotation_access: "not_accessed", graph_invocations: 0, bge_invocations: 0, + symbol_document_invocations: 0, host_query_invocations: 0, production_packet_invocations: 0 }; + validatePreAnnotationBoundary(run, { annotation_access: "not_accessed" }); + for (const mutate of [(value) => { value.annotation_access = "accessed"; }, + (value) => { value.graph_invocations = 1; }]) { + const changed = structuredClone(run); mutate(changed); + assert.throws(() => validatePreAnnotationBoundary(changed, { annotation_access: "not_accessed" })); + } +}); + +test("exact optimizer selects successors without compulsory discovery seeds", () => { + const rowBytes = new Map([["seed", 15_000], ["successor", 100]]); + const result = maximizeCoveredAtoms([["successor"]], rowBytes, 100); + assert.deepEqual(result.selected, ["successor"]); + assert.equal(result.covered, 1); +}); + +test("twelve independently discovered successors fit as twelve rows", () => { + const ids = Array.from({ length: 12 }, (_, index) => `successor-${index}`), + rowBytes = new Map(ids.map((id) => [id, 100])); + const result = maximizeCoveredAtoms(ids.map((id) => [id]), rowBytes, 100); + assert.equal(result.covered, 12); + assert.equal(result.rows, 12); + assert.equal(result.public_bytes, 100 + 1200 + 11); +}); + +test("shared rows are charged once and exact row and byte boundaries hold", () => { + const costs = new Map([["shared", 100], ["a", 50], ["b", 50]]); + const shared = maximizeCoveredAtoms([["shared", "a"], ["shared", "b"]], costs, 100); + assert.equal(shared.public_bytes, exactPublicBytes(100, ["a", "b", "shared"], costs)); + const sixteen = Array.from({ length: 16 }, (_, index) => `r${index}`), + exactCosts = new Map(sixteen.map((id) => [id, 1000])); + assert.equal(maximizeCoveredAtoms([sixteen], exactCosts, 369).covered, 1); + assert.equal(maximizeCoveredAtoms([sixteen], exactCosts, 370).covered, 0); + const seventeen = [...sixteen, "r16"], seventeenCosts = new Map([...exactCosts, ["r16", 1]]); + assert.equal(maximizeCoveredAtoms([seventeen], seventeenCosts, 1).covered, 0); +}); + +test("acceptable alternatives stay separate and relation-only evidence earns no credit", () => { + const { seed, successor } = fixture(), fragments = [seed, successor]; + const range = (fragment) => ({ path: fragment.path, content_digest: fragment.content_digest, + byte_range: fragment.byte_range, line_range: fragment.line_range }); + const annotation = { acceptable_sets: [ + { set_id: "seed-route", required_source_atoms: [{ atom_id: "seed", source_range: range(seed) }], + required_relation_atoms: [{ atom_id: "ignored-relation" }] }, + { set_id: "successor-route", required_source_atoms: [{ atom_id: "successor", + source_range: range(successor) }], required_relation_atoms: [] }, + ] }; + const result = evaluateArm(annotation, fragments, [successor.fragment_id], 100); + assert.equal(result.best_set_id, "successor-route"); + assert.equal(result.recall, 1); + assert.deepEqual(result.reachable_atoms, ["successor"]); +}); + +test("an atom spanning two fragments receives no partial credit", () => { + const { seed, successor } = fixture(), annotation = { acceptable_sets: [{ set_id: "whole", + required_source_atoms: [{ atom_id: "both", source_range: { path: seed.path, + content_digest: seed.content_digest, byte_range: { start: 0, end: successor.byte_range.end }, + line_range: { start: 1, end: 2 } } }], required_relation_atoms: [] }] }; + const partial = evaluateArm(annotation, [seed, successor], [seed.fragment_id], 100); + assert.equal(partial.recall, 0); + assert.equal(partial.complete_source_set, false); +}); + +test("the frozen decision table never authorizes production integration", () => { + const sufficient = { mean_recall: 0.9, complete_set_rate: 0.8, + groups: { a: { mean_recall: 0.8 }, b: { mean_recall: 0.9 } } }; + const weak = { mean_recall: 0.5, complete_set_rate: 0.4, + groups: { a: { mean_recall: 0.5 }, b: { mean_recall: 0.5 } } }; + assert.equal(gateOne(sufficient).pass, true); + assert.equal(gateOne(weak).pass, false); + assert.equal(decision(false, false, false).decision, "stop_automatic_packet_compilation"); + assert.equal(decision(true, true, false).decision, + "freeze_unconditioned_frontier_for_selector_experiment"); + assert.equal(decision(false, true, true).decision, + "freeze_conditioned_frontier_for_selector_experiment"); + const cases = [{ control_incomplete_for_gain: true, candidate_gained_atom: true }]; + assert.equal(gateTwo(cases, weak, sufficient, true).pass, true); +}); + +test("annotations cannot be opened before a valid validator receipt", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "etr1-boundary-")); + const validation = path.join(directory, "validation.json"), bytes = Buffer.from(JSON.stringify({ + contract: "codestory.etr1-validation/v1", experiment_status: "invalid", + decision: "not_evaluated", annotation_access: "not_accessed", + })); + await writeFile(validation, bytes); + await assert.rejects(() => evaluateEtr1({ validationPath: validation, + validationSha256: sha256(bytes), annotationsPath: path.join(directory, "must-not-open.json"), + annotationsSha256: "0".repeat(64), oraclePath: path.join(directory, "oracle.json"), + oracleSha256: "0".repeat(64), sourceRoot: directory }), /validator did not authorize/u); +}); From ef4901c112bf1eeb217a24bbb1fd8cd34e7ab410 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Fri, 4 Sep 2026 16:00:43 -0400 Subject: [PATCH 3/8] accept zero-based qualification events --- .../src/bin/codestory_etr1/run.rs | 35 +++++++++++++++++-- scripts/codestory-etr1-validate.mjs | 6 ++-- scripts/tests/etr1-evidence.test.mjs | 12 ++++++- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/crates/codestory-bench/src/bin/codestory_etr1/run.rs b/crates/codestory-bench/src/bin/codestory_etr1/run.rs index 6e1baa856..2d5ffd814 100644 --- a/crates/codestory-bench/src/bin/codestory_etr1/run.rs +++ b/crates/codestory-bench/src/bin/codestory_etr1/run.rs @@ -346,17 +346,17 @@ fn read_completed_events(path: &Path) -> Result> { "qualification_event_log_unterminated" ); let mut events = Vec::new(); - let mut previous = 0_u64; + let mut previous = None; for line in bytes .split(|byte| *byte == b'\n') .filter(|line| !line.is_empty()) { let event: QualificationEvent = serde_json::from_slice(line)?; ensure!( - event.schema_version == 1 && event.sequence > previous, + event.schema_version == 1 && previous.is_none_or(|previous| event.sequence > previous), "qualification_event_sequence_invalid" ); - previous = event.sequence; + previous = Some(event.sequence); if event.action == "completed_tokens" { ensure!( event.status == "completed" @@ -1101,6 +1101,35 @@ mod tests { assert!(shorten_single_query(&mut spec, "question", "a\nb\n").is_err()); } + #[test] + fn qualification_events_accept_the_native_zero_based_sequence() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("events.jsonl"); + fs::write( + &path, + concat!( + "{\"schema_version\":1,\"sequence\":0,\"action\":\"completed_tokens\",", + "\"status\":\"completed\",\"server_event_sequence\":10,\"clock\":{},", + "\"details\":{\"completed_tokens\":\"5\",\"native_completion_sequence\":\"1\",", + "\"request_id\":\"first\"}}\n", + "{\"schema_version\":1,\"sequence\":1,\"action\":\"completed_tokens\",", + "\"status\":\"completed\",\"server_event_sequence\":11,\"clock\":{},", + "\"details\":{\"completed_tokens\":\"7\",\"native_completion_sequence\":\"2\",", + "\"request_id\":\"second\"}}\n", + ), + ) + .unwrap(); + + let events = read_completed_events(&path).unwrap(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].sequence, 0); + assert_eq!(events[1].sequence, 1); + + let bytes = fs::read_to_string(&path).unwrap(); + fs::write(&path, bytes.replace("\"sequence\":1", "\"sequence\":0")).unwrap(); + assert!(read_completed_events(&path).is_err()); + } + #[test] fn legal_pool_deduplicates_exact_ids_without_making_seeds_compulsory() { let repository = PreparedRepositoryV1 { diff --git a/scripts/codestory-etr1-validate.mjs b/scripts/codestory-etr1-validate.mjs index 99e253c6c..9c9770651 100644 --- a/scripts/codestory-etr1-validate.mjs +++ b/scripts/codestory-etr1-validate.mjs @@ -171,12 +171,12 @@ export function validateArm({ arm, expectedName, wording, repository, fragments, validateSourceAuthentication(arm, fragments, repository, sourceFiles); } -function parseEvents(bytes) { +export function parseEvents(bytes) { assert.equal(bytes.at(-1), 10, "qualification event log is unterminated"); - let previous = 0; + let previous = null; return bytes.toString("utf8").trimEnd().split("\n").map(JSON.parse).filter((event) => { assert.equal(event.schema_version, 1, "qualification event schema changed"); - assert.ok(event.sequence > previous, "qualification event sequence changed"); + assert.ok(previous === null || event.sequence > previous, "qualification event sequence changed"); previous = event.sequence; return event.action === "completed_tokens"; }); diff --git a/scripts/tests/etr1-evidence.test.mjs b/scripts/tests/etr1-evidence.test.mjs index 76852c0bf..70320dff8 100644 --- a/scripts/tests/etr1-evidence.test.mjs +++ b/scripts/tests/etr1-evidence.test.mjs @@ -6,7 +6,7 @@ import path from "node:path"; import { authenticateFragment, encodedCandidateInput, evaluateArm, exactPublicBytes, fragmentId, LIMITS, maximizeCoveredAtoms, scoreOrder, selectSuccessors, sha256 } from "../lib/etr1-evidence.mjs"; import { validateArm, validateDocumentVectorRecord, validateEngine, - validatePreAnnotationBoundary } from "../codestory-etr1-validate.mjs"; + validatePreAnnotationBoundary, parseEvents } from "../codestory-etr1-validate.mjs"; import { decision, evaluateEtr1, gateOne, gateTwo } from "../codestory-etr1-evaluate.mjs"; function unit(index) { @@ -84,6 +84,16 @@ test("candidate input shortening preserves UTF-8 and complete trailing lines", ( assert.throws(() => encodedCandidateInput("question", source, 3)); }); +test("validator accepts the native zero-based qualification sequence", () => { + const event = (sequence) => JSON.stringify({ schema_version: 1, sequence, + action: "completed_tokens", status: "completed", server_event_sequence: 10 + sequence, + clock: {}, details: { completed_tokens: "5", native_completion_sequence: String(sequence + 1), + request_id: `request-${sequence}` } }); + const events = parseEvents(Buffer.from(`${event(0)}\n${event(1)}\n`)); + assert.deepEqual(events.map(({ sequence }) => sequence), [0, 1]); + assert.throws(() => parseEvents(Buffer.from(`${event(0)}\n${event(0)}\n`))); +}); + test("validator reconstructs both arm query contracts and refuses hostile mutations", () => { for (const name of ["control", "candidate"]) validateArm(fixture(name)); const mutations = [ From 871d163dde95c3bd65379f0a61a8949c912d4811 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Fri, 4 Sep 2026 17:18:26 -0400 Subject: [PATCH 4/8] repair etr1 execution and canary boundaries --- .../codestory-bench/src/bin/codestory_etr1.rs | 25 ++ .../src/bin/codestory_etr1/contract.rs | 6 +- .../src/bin/codestory_etr1/control.rs | 78 ++++++ .../src/bin/codestory_etr1/prepare.rs | 151 +++++++++++- .../src/bin/codestory_etr1/run.rs | 230 +++++++++++++----- scripts/codestory-etr1-canary.mjs | 149 ++++++------ scripts/codestory-etr1-evaluate.mjs | 52 ++-- scripts/codestory-etr1-validate.mjs | 87 +++++-- scripts/lib/etr1-execution.mjs | 143 +++++++++++ scripts/tests/etr1-evidence.test.mjs | 52 +++- 10 files changed, 786 insertions(+), 187 deletions(-) create mode 100644 crates/codestory-bench/src/bin/codestory_etr1/control.rs create mode 100644 scripts/lib/etr1-execution.mjs diff --git a/crates/codestory-bench/src/bin/codestory_etr1.rs b/crates/codestory-bench/src/bin/codestory_etr1.rs index 1fde157e5..9f3e326a6 100644 --- a/crates/codestory-bench/src/bin/codestory_etr1.rs +++ b/crates/codestory-bench/src/bin/codestory_etr1.rs @@ -8,6 +8,8 @@ use std::path::PathBuf; mod build_provenance; #[path = "codestory_etr1/contract.rs"] mod contract; +#[path = "codestory_etr1/control.rs"] +mod control; #[path = "codestory_etr1/prepare.rs"] mod prepare; #[path = "codestory_etr1/run.rs"] @@ -22,6 +24,13 @@ struct Args { #[derive(Subcommand)] enum Command { + /// Prepare the fixed synthetic canary, never a corpus experiment. + PrepareCanary { + #[arg(long)] + project_root: PathBuf, + #[arg(long)] + output_dir: PathBuf, + }, /// Authenticate the frozen corpus, rebuild BM25 memberships, and emit exact /// fragment documents for the existing embedding diagnostic. Prepare { @@ -35,6 +44,12 @@ enum Command { /// Build paired unconditioned and source-conditioned frontiers from a /// previously authenticated preparation and exact fragment vectors. Run { + #[arg(long)] + document_execution: PathBuf, + #[arg(long)] + document_execution_sha256: String, + #[arg(long)] + cancel_file: PathBuf, #[arg(long)] prepared: PathBuf, #[arg(long)] @@ -60,12 +75,19 @@ fn main() -> Result<()> { } let args = Args::parse(); match args.command { + Command::PrepareCanary { + project_root, + output_dir, + } => prepare::execute_canary(&project_root, &output_dir), Command::Prepare { evidence_root, corpus_root, output_dir, } => prepare::execute(&evidence_root, &corpus_root, &output_dir), Command::Run { + document_execution, + document_execution_sha256, + cancel_file, prepared, prepared_sha256, fragment_vectors, @@ -79,6 +101,9 @@ fn main() -> Result<()> { &fragment_vectors_sha256, &state_root, &output_dir, + &cancel_file, + &document_execution, + &document_execution_sha256, ), } } diff --git a/crates/codestory-bench/src/bin/codestory_etr1/contract.rs b/crates/codestory-bench/src/bin/codestory_etr1/contract.rs index 751e71887..23742cfa4 100644 --- a/crates/codestory-bench/src/bin/codestory_etr1/contract.rs +++ b/crates/codestory-bench/src/bin/codestory_etr1/contract.rs @@ -181,7 +181,9 @@ pub struct BatchReceiptV1 { pub input_sha256: Vec, pub wall_ns: u64, pub completed_tokens: u64, - pub qualification_event_sequence: u64, + pub qualification_native_completion_sequence: u64, + pub qualification_server_event_sequence: u64, + pub qualification_request_id_sha256: String, } #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -203,6 +205,7 @@ pub struct ArmTimingV1 { pub descriptor_mapping_ns: u64, pub remaining_source_authentication_ns: u64, pub prepared_state_ns: u64, + pub unaccounted_ns: u64, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -246,6 +249,7 @@ pub struct Etr1RunManifestV1 { pub build: BuildIdentity, pub preparation: FileBinding, pub fragment_vectors: FileBinding, + pub document_execution: FileBinding, pub method_sha256: String, pub annotation_access: String, pub vector_artifact_loaded_before_timing: bool, diff --git a/crates/codestory-bench/src/bin/codestory_etr1/control.rs b/crates/codestory-bench/src/bin/codestory_etr1/control.rs new file mode 100644 index 000000000..887b1b4cc --- /dev/null +++ b/crates/codestory-bench/src/bin/codestory_etr1/control.rs @@ -0,0 +1,78 @@ +//! One deadline and cancellation source for the complete paired experiment. +use anyhow::{Result, ensure}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +pub const PAIRED_RUN_LIMIT: Duration = Duration::from_secs(30 * 60); + +pub struct RunControl { + started: Instant, + cancel_file: PathBuf, + limit: Duration, +} + +impl RunControl { + pub fn new(cancel_file: &Path) -> Result { + ensure!(cancel_file.is_absolute(), "cancel_file_must_be_absolute"); + let control = Self { + started: Instant::now(), + cancel_file: cancel_file.into(), + limit: PAIRED_RUN_LIMIT, + }; + control.check()?; + Ok(control) + } + + pub fn cancelled(&self) -> bool { + self.cancel_file.try_exists().unwrap_or(true) || self.started.elapsed() >= self.limit + } + + pub fn check(&self) -> Result<()> { + ensure!(!self.cancel_file.try_exists()?, "etr1_cancelled"); + ensure!( + self.started.elapsed() < self.limit, + "etr1_deadline_exceeded" + ); + Ok(()) + } + + pub fn batch_timeout(&self) -> Result { + self.check()?; + Ok(self + .limit + .saturating_sub(self.started.elapsed()) + .min(Duration::from_secs(60))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cancellation_and_global_deadline_stop_every_later_batch() { + let root = tempfile::tempdir().unwrap(); + let cancel_file = root.path().join("cancel"); + let mut control = RunControl::new(&cancel_file).unwrap(); + assert!(!control.cancelled()); + std::fs::write(&cancel_file, b"cancel").unwrap(); + assert!(control.cancelled()); + assert!( + control + .batch_timeout() + .unwrap_err() + .to_string() + .contains("etr1_cancelled") + ); + std::fs::remove_file(&cancel_file).unwrap(); + control.limit = Duration::ZERO; + assert!(control.cancelled()); + assert!( + control + .batch_timeout() + .unwrap_err() + .to_string() + .contains("etr1_deadline_exceeded") + ); + } +} diff --git a/crates/codestory-bench/src/bin/codestory_etr1/prepare.rs b/crates/codestory-bench/src/bin/codestory_etr1/prepare.rs index 182cae505..fc5a9a77d 100644 --- a/crates/codestory-bench/src/bin/codestory_etr1/prepare.rs +++ b/crates/codestory-bench/src/bin/codestory_etr1/prepare.rs @@ -201,7 +201,7 @@ fn render_snippet(source: &str, start_line: u32) -> String { } fn authenticate_units( - repository: &SelectedRepository, + project_root: &Path, project_id: &str, units: &[LineUnit], ) -> Result> { @@ -210,7 +210,7 @@ fn authenticate_units( for unit in units { ensure!(unit.available, "frozen_fragment_unavailable"); if !sources.contains_key(&unit.path) { - let path = confined_source_path(&repository.project_root, &unit.path)?; + let path = confined_source_path(project_root, &unit.path)?; let bytes = std::fs::read(path)?; ensure!( sha256(&bytes) == unit.content_digest, @@ -327,6 +327,10 @@ fn method_freeze() -> Value { "query": {"control":"raw question", "candidate":"raw question + newline delimiter + seed source", "truncation":"none; complete trailing seed lines may be removed only after a typed model-limit rejection"}, "graph": false, "bge": false, "symbol_documents": false, "host_queries": false, "packet_decision": "not_evaluated" + ,"execution": {"paired_deadline_seconds":1800,"canary":"same_rust_runner_and_authoritative_validator", + "timing":"enclosing_request_wall_and_exclusive_phases_with_explicit_unaccounted_time", + "provenance":"independent_pre_dispatch_request_and_post_execution_result_bindings", + "native_completion_identity":"native_completion_sequence,server_event_sequence,request_id"} }) } @@ -431,7 +435,8 @@ pub fn execute(evidence_root: &Path, corpus_root: &Path, output: &Path) -> Resul .to_string(); let units_binding = external_binding(&selected_repository.units.line)?; let units: Vec = read_bound_json(&units_binding)?; - let repo_fragments = authenticate_units(selected_repository, &project_id, &units)?; + let repo_fragments = + authenticate_units(&selected_repository.project_root, &project_id, &units)?; let score_order_sha256 = sha256(serde_json::to_vec( &repo_fragments .iter() @@ -547,6 +552,32 @@ pub fn execute(evidence_root: &Path, corpus_root: &Path, output: &Path) -> Resul fixed_inputs.insert(name.to_string(), binding); } + finish_preparation( + output, + build, + fixed_inputs, + DeclaredBinding { + path: corpus_root.join("reconciled.json"), + sha256: ANNOTATIONS_SHA256.into(), + }, + repositories, + fragments, + wordings, + "visible_development_frontier_only", + ) +} + +#[allow(clippy::too_many_arguments)] +fn finish_preparation( + output: &Path, + build: BuildIdentity, + fixed_inputs: BTreeMap, + annotations: DeclaredBinding, + repositories: Vec, + fragments: Vec, + wordings: Vec, + authority: &str, +) -> Result<()> { let method_bytes = serialize_pretty(&method_freeze())?; let embedding_input = EmbeddingDiagnosticInput { contract: "codestory.embedding-diagnostic-input/v1".into(), @@ -574,16 +605,13 @@ pub fn execute(evidence_root: &Path, corpus_root: &Path, output: &Path) -> Resul }; let preparation = Etr1PreparationV1 { contract: "codestory.etr1-preparation/v1".into(), - authority: "visible_development_frontier_only".into(), + authority: authority.into(), packet_decision: "not_evaluated".into(), parent_head: PARENT_HEAD.into(), build, method, fixed_inputs, - annotations: DeclaredBinding { - path: corpus_root.join("reconciled.json"), - sha256: ANNOTATIONS_SHA256.into(), - }, + annotations, model_sha256: MODEL_SHA256.into(), tokenizer_sha256: TOKENIZER_SHA256.into(), embedding_input: embedding_binding, @@ -614,6 +642,113 @@ pub fn execute(evidence_root: &Path, corpus_root: &Path, output: &Path) -> Resul Ok(()) } +/// Only fixture construction differs from corpus preparation. Fragment +/// authentication, BM25, publication, encoding, and the paired runner are shared. +pub fn execute_canary(project_root: &Path, output: &Path) -> Result<()> { + ensure!( + project_root.is_absolute() && output.is_absolute(), + "canary_paths_must_be_absolute" + ); + let source = std::fs::read_to_string(project_root.join("canary.rs"))?; + let digest = sha256(source.as_bytes()); + let mut offset = 0; + let units = source + .split_inclusive('\n') + .enumerate() + .map(|(index, line)| { + let start = offset; + offset += line.len(); + LineUnit { + start_line: index as u32 + 1, + end_line: index as u32 + 1, + byte_range: ByteRangeV1 { + start: start as u64, + end: offset as u64, + }, + content: line.into(), + available: true, + snippet: render_snippet(line, index as u32 + 1), + path: "canary.rs".into(), + content_digest: digest.clone(), + } + }) + .collect::>(); + ensure!( + units.len() == 32 && units.iter().all(|unit| unit.content.len() <= 512), + "canary_fixture_shape_changed" + ); + let fragments = authenticate_units(project_root, "etr1-synthetic-canary", &units)?; + let fragment_ids = fragments + .iter() + .map(|fragment| fragment.fragment_id.clone()) + .collect::>(); + let publication = + json!({"project_id":"etr1-synthetic-canary", "source_commit":git_head(project_root)?}); + let empty = json!({"publication":publication,"answer_sufficiency":"not_asserted","support":[],"continuation":[]}); + let repository = PreparedRepositoryV1 { + repository_id: "canary".into(), + project_id: "etr1-synthetic-canary".into(), + commit: git_head(project_root)?, + local_root: project_root.into(), + publication, + score_order_sha256: sha256(serde_json::to_vec(&fragment_ids)?), + fragment_ids, + base_serialized_bytes: serde_json::to_vec(&empty)?.len() as u32, + }; + let source_binding = bind_file(&project_root.join("canary.rs"), Some(&digest))?; + let lexical = Etr1LexicalIndex::new(fragments.iter().map(|fragment| fragment.source.as_str()))?; + let mut wordings = Vec::new(); + for (index, query) in ["commonneedle", "raremarker", "absentmarker"] + .iter() + .enumerate() + { + let (terms, matches) = lexical.search(query)?; + let expected_seeds = [16, 1, 0][index]; + let seeds = natural_seed_prefix(&matches) + .iter() + .map(|item| repository.fragment_ids[item.rowid - 1].clone()) + .collect::>(); + ensure!( + seeds.len() == expected_seeds, + "canary_bm25_membership_changed" + ); + wordings.push(PreparedWordingV1 { + case_id: format!("canary-{index}"), + phrasing_id: "original".into(), + repository_id: "canary".into(), + group: "synthetic".into(), + question: (*query).into(), + question_sha256: sha256(query.as_bytes()), + membership: source_binding.clone(), + terms, + bm25_match_count: matches.len() as u32, + bm25_matches_sha256: sha256(serde_json::to_vec( + &matches + .iter() + .map(|item| MembershipMatch { + rowid: item.rowid, + score: item.score, + }) + .collect::>(), + )?), + seed_fragment_ids: seeds, + }); + } + finish_preparation( + output, + build_identity()?, + BTreeMap::from([("synthetic_source".into(), source_binding)]), + DeclaredBinding { + path: output.join("synthetic-annotations-unopened.json"), + sha256: "0".repeat(64), + }, + vec![repository], + fragments, + wordings, + "synthetic_canary_only", + ) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/codestory-bench/src/bin/codestory_etr1/run.rs b/crates/codestory-bench/src/bin/codestory_etr1/run.rs index 2d5ffd814..bc0175e3f 100644 --- a/crates/codestory-bench/src/bin/codestory_etr1/run.rs +++ b/crates/codestory-bench/src/bin/codestory_etr1/run.rs @@ -1,4 +1,5 @@ use super::contract::*; +use super::control::RunControl; use anyhow::{Context, Result, ensure}; use codestory_retrieval::benchmark_support::Etr1LexicalIndex; use codestory_retrieval::{ @@ -9,7 +10,7 @@ use serde_json::Value; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; -use std::time::{Duration, Instant}; +use std::time::Instant; const RUN_CONTRACT: &str = "codestory.etr1-run/v1"; const ROW_CONTRACT: &str = "codestory.etr1-wording/v1"; @@ -346,59 +347,103 @@ fn read_completed_events(path: &Path) -> Result> { "qualification_event_log_unterminated" ); let mut events = Vec::new(); - let mut previous = None; + let mut previous_native: Option = None; + let mut previous_server: Option = None; + let mut request_ids = HashSet::new(); for line in bytes .split(|byte| *byte == b'\n') .filter(|line| !line.is_empty()) { let event: QualificationEvent = serde_json::from_slice(line)?; ensure!( - event.schema_version == 1 && previous.is_none_or(|previous| event.sequence > previous), - "qualification_event_sequence_invalid" + event.schema_version == 1 + && event.sequence == 0 + && event.action == "completed_tokens" + && event.status == "completed" + && event.server_event_sequence > 0 + && previous_server.is_none_or(|previous| event.server_event_sequence > previous) + && event.clock.is_object() + && event.snapshot.is_none(), + "qualification_token_event_invalid" ); - previous = Some(event.sequence); - if event.action == "completed_tokens" { - ensure!( - event.status == "completed" - && event.server_event_sequence > 0 - && event.clock.is_object() - && event.snapshot.is_none(), - "qualification_token_event_invalid" - ); - events.push(event); - } + let details = event + .details + .as_ref() + .context("qualification_token_details_missing")?; + let request_id = details + .get("request_id") + .filter(|value| !value.is_empty()) + .context("qualification_token_request_id_missing")?; + let native_sequence = details + .get("native_completion_sequence") + .context("qualification_native_completion_sequence_missing")? + .parse::()?; + ensure!( + native_sequence > 0 + && previous_native + .is_none_or(|previous| { previous.checked_add(1) == Some(native_sequence) }) + && request_ids.insert(request_id.clone()), + "qualification_native_completion_identity_invalid" + ); + ensure!( + details + .get("completed_tokens") + .and_then(|value| value.parse::().ok()) + .is_some_and(|value| value > 0), + "qualification_completed_tokens_invalid" + ); + previous_native = Some(native_sequence); + previous_server = Some(event.server_event_sequence); + events.push(event); } + ensure!( + !events.is_empty(), + "qualification_completed_event_log_empty" + ); Ok(events) } -fn consume_completed_event(path: &Path, cursor: &mut EventCursor) -> Result<(u64, u64)> { - let events = read_completed_events(path)?; - ensure!( - events.len() == cursor.completed_events + 1, - "qualification_completed_event_count_invalid" - ); - let event = &events[cursor.completed_events]; - cursor.completed_events += 1; +fn completed_event_identity(event: &QualificationEvent) -> Result<(u64, u64, u64, String)> { let details = event .details .as_ref() .context("qualification_token_details_missing")?; - ensure!( - details - .get("request_id") - .is_some_and(|value| !value.is_empty()) - && details - .get("native_completion_sequence") - .and_then(|value| value.parse::().ok()) - .is_some_and(|value| value > 0), - "qualification_token_identity_missing" - ); + let request_id = details + .get("request_id") + .filter(|value| !value.is_empty()) + .context("qualification_token_request_id_missing")?; + let native_sequence = details + .get("native_completion_sequence") + .context("qualification_native_completion_sequence_missing")? + .parse::()?; let tokens = details .get("completed_tokens") .context("qualification_completed_tokens_missing")? .parse::()?; - ensure!(tokens > 0, "qualification_completed_tokens_zero"); - Ok((tokens, event.sequence)) + ensure!( + tokens > 0 && native_sequence > 0 && event.server_event_sequence > 0, + "qualification_completed_event_identity_invalid" + ); + Ok(( + tokens, + native_sequence, + event.server_event_sequence, + sha256(request_id.as_bytes()), + )) +} + +fn consume_completed_event( + path: &Path, + cursor: &mut EventCursor, +) -> Result<(u64, u64, u64, String)> { + let events = read_completed_events(path)?; + ensure!( + events.len() == cursor.completed_events + 1, + "qualification_completed_event_count_invalid" + ); + let identity = completed_event_identity(&events[cursor.completed_events])?; + cursor.completed_events += 1; + Ok(identity) } fn input_too_long(error: &anyhow::Error) -> bool { @@ -425,6 +470,7 @@ fn shorten_single_query(spec: &mut QuerySpec, question: &str, source: &str) -> R #[allow(clippy::too_many_arguments)] fn encode_partition( + control: &RunControl, client: &ProductEmbeddingClient, specs: &mut [QuerySpec], questions: &[String], @@ -445,14 +491,16 @@ fn encode_partition( .map(|spec| spec.encoded_input.clone()) .collect::>(); let started = Instant::now(); - match client.embed_queries_with_control(&inputs, Some(Duration::from_secs(60)), &|| false) { + match client.embed_queries_with_control(&inputs, Some(control.batch_timeout()?), &|| { + control.cancelled() + }) { Ok(vectors) => { let wall_ns = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX); ensure!(vectors.len() == specs.len(), "query_vector_count_mismatch"); for vector in &vectors { validate_vector(vector)?; } - let (completed_tokens, sequence) = + let (completed_tokens, native_sequence, server_sequence, request_id_sha256) = consume_completed_event(qualification_events, cursor)?; let ordinal = *next_batch_ordinal; *next_batch_ordinal = (*next_batch_ordinal).saturating_add(1); @@ -466,7 +514,9 @@ fn encode_partition( .collect(), wall_ns, completed_tokens, - qualification_event_sequence: sequence, + qualification_native_completion_sequence: native_sequence, + qualification_server_event_sequence: server_sequence, + qualification_request_id_sha256: request_id_sha256, }); for (spec, vector) in specs.iter().cloned().zip(vectors) { results.push(EncodedQuery { @@ -486,6 +536,7 @@ fn encode_partition( let middle = specs.len() / 2; let (left, right) = specs.split_at_mut(middle); encode_partition( + control, client, left, questions, @@ -498,6 +549,7 @@ fn encode_partition( arm, )?; encode_partition( + control, client, right, questions, @@ -513,6 +565,7 @@ fn encode_partition( let index = specs[0].ordinal; shorten_single_query(&mut specs[0], &questions[index], &seed_sources[index])?; encode_partition( + control, client, specs, questions, @@ -532,6 +585,7 @@ fn encode_partition( #[allow(clippy::too_many_arguments)] fn encode_queries( + control: &RunControl, client: &ProductEmbeddingClient, arm: &str, specs: &mut [QuerySpec], @@ -550,6 +604,7 @@ fn encode_queries( for start in (0..specs.len()).step_by(QUERY_BATCH_MAX) { let end = (start + QUERY_BATCH_MAX).min(specs.len()); encode_partition( + control, client, &mut specs[start..end], questions, @@ -631,6 +686,7 @@ fn exact_legally_selectable_pool( #[allow(clippy::too_many_arguments)] fn run_arm( + control: &RunControl, name: &str, wording: &PreparedWordingV1, repository: &PreparedRepositoryV1, @@ -642,6 +698,8 @@ fn run_arm( cursor: &mut EventCursor, next_batch_ordinal: &mut u32, ) -> Result { + let request_started = Instant::now(); + control.check()?; let bm25_started = Instant::now(); let (_, matches) = lexical.search(&wording.question)?; let observed_seeds = natural_seed_prefix(&matches) @@ -704,6 +762,7 @@ fn run_arm( .collect::>(); let encoding_started = Instant::now(); let (encoded, batch_receipts) = encode_queries( + control, client, name, &mut specs, @@ -719,6 +778,7 @@ fn run_arm( let vector_started = Instant::now(); let mut scored = Vec::with_capacity(encoded.len()); for query in &encoded { + control.check()?; scored.push(score_fragments( &query.vector, &repository.fragment_ids, @@ -732,7 +792,7 @@ fn run_arm( let mut successors = Vec::new(); let mut query_receipts = Vec::with_capacity(encoded.len()); for (query, (scores, ranked)) in encoded.into_iter().zip(scored) { - let excluded_before = seeds.iter().chain(prior.iter()).cloned().collect(); + let excluded_before = seeds.union(&prior).cloned().collect(); let selected = select_successors(&ranked, &seeds, &prior, SUCCESSORS_PER_QUERY); ensure!( selected.len() <= SUCCESSORS_PER_QUERY, @@ -772,6 +832,7 @@ fn run_arm( let remaining_auth_started = Instant::now(); for fragment_id in &successors { + control.check()?; authenticator.authenticate(fragment_id)?; } let remaining_source_authentication_ns = @@ -783,6 +844,14 @@ fn run_arm( ); let legally_selectable_pool = exact_legally_selectable_pool(&hydrated_pool, repository, fragments)?; + let prepared_state_ns = u64::try_from(request_started.elapsed().as_nanos()).unwrap_or(u64::MAX); + let accounted_ns = round_zero_bm25_ns + .saturating_add(seed_source_authentication_ns) + .saturating_add(query_encoding_ns) + .saturating_add(vector_search_ns) + .saturating_add(descriptor_mapping_ns) + .saturating_add(remaining_source_authentication_ns); + ensure!(accounted_ns <= prepared_state_ns, "request_timing_overlaps"); let timing = ArmTimingV1 { round_zero_bm25_ns, seed_source_authentication_ns, @@ -790,12 +859,8 @@ fn run_arm( vector_search_ns, descriptor_mapping_ns, remaining_source_authentication_ns, - prepared_state_ns: round_zero_bm25_ns - .saturating_add(seed_source_authentication_ns) - .saturating_add(query_encoding_ns) - .saturating_add(vector_search_ns) - .saturating_add(descriptor_mapping_ns) - .saturating_add(remaining_source_authentication_ns), + prepared_state_ns, + unaccounted_ns: prepared_state_ns - accounted_ns, }; Ok(ArmFrontierV1 { name: name.to_string(), @@ -828,7 +893,11 @@ pub fn execute( fragment_vectors_sha256: &str, state_root: &Path, output: &Path, + cancel_file: &Path, + document_execution: &Path, + document_execution_sha256: &str, ) -> Result<()> { + let run_control = RunControl::new(cancel_file)?; let source_root = Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .and_then(Path::parent) @@ -840,18 +909,19 @@ pub fn execute( let preparation_binding = bind_file(prepared, Some(prepared_sha256))?; let preparation: Etr1PreparationV1 = read_bound_json(&preparation_binding)?; let build = build_identity()?; + let canary = preparation.authority == "synthetic_canary_only"; ensure!( !build.source_dirty && preparation.contract == "codestory.etr1-preparation/v1" - && preparation.authority == "visible_development_frontier_only" + && (preparation.authority == "visible_development_frontier_only" || canary) && preparation.packet_decision == "not_evaluated" && preparation.parent_head == PARENT_HEAD && preparation.annotation_access == "not_accessed" - && preparation.annotations.sha256 == ANNOTATIONS_SHA256 + && (preparation.annotations.sha256 == ANNOTATIONS_SHA256 || canary) && preparation.model_sha256 == MODEL_SHA256 && preparation.tokenizer_sha256 == TOKENIZER_SHA256 - && preparation.fragments.len() == FRAGMENT_COUNT - && preparation.wordings.len() == WORDING_COUNT + && preparation.fragments.len() == if canary { 32 } else { FRAGMENT_COUNT } + && preparation.wordings.len() == if canary { 3 } else { WORDING_COUNT } && preparation.build.source_commit == build.source_commit && preparation.build.source_tree == build.source_tree, "etr1_preparation_identity_mismatch" @@ -872,6 +942,40 @@ pub fn execute( // request timing begins. It is never rebuilt or paged during ETR-1. let (vector_binding, vector_artifact, vectors) = load_vector_artifact(fragment_vectors, fragment_vectors_sha256, &preparation)?; + let document_execution = bind_file(document_execution, Some(document_execution_sha256))?; + let execution: Value = read_bound_json(&document_execution)?; + ensure!( + execution["contract"] == "codestory.etr1-execution/v1" + && execution["role"] == "documents" + && execution["experiment_status"] == "completed" + && execution["exit_code"] == 0 + && execution["signal"].is_null() + && execution["annotation_access"] == "not_accessed", + "document_execution_invalid" + ); + let request_binding: FileBinding = serde_json::from_value(execution["request"].clone())?; + let request: Value = read_bound_json(&request_binding)?; + let producer: FileBinding = serde_json::from_value(request["executable"].clone())?; + ensure!( + bind_file(&producer.path, Some(&producer.sha256))? == producer + && producer.sha256 == vector_artifact.binary_sha256, + "document_producer_changed" + ); + let inputs: Vec = serde_json::from_value(request["inputs"].clone())?; + let outputs: Vec = serde_json::from_value(execution["outputs"].clone())?; + ensure!( + inputs.contains(&preparation.embedding_input) && outputs.contains(&vector_binding), + "document_execution_artifact_not_bound" + ); + let document_events: FileBinding = serde_json::from_value(execution["events"].clone())?; + ensure!( + bind_file(&document_events.path, Some(&document_events.sha256))? == document_events, + "document_native_events_changed" + ); + ensure!( + !read_completed_events(&document_events.path)?.is_empty(), + "document_native_events_missing" + ); let runtime = SidecarRuntimeConfig::local(); let qualification_events = validate_isolated_state(state_root, &runtime)?; codestory_cli::install_native_embedding_client_transport()?; @@ -885,7 +989,7 @@ pub fn execute( .map(|fragment| (fragment.fragment_id.clone(), fragment)) .collect::>(); ensure!( - fragment_map.len() == FRAGMENT_COUNT, + fragment_map.len() == preparation.fragments.len(), "fragment_map_identity_collision" ); let repository_map = preparation @@ -923,6 +1027,7 @@ pub fn execute( .get(&wording.repository_id) .context("wording_lexical_index_missing")?; let control = finalize_arm(run_arm( + &run_control, "control", wording, repository, @@ -935,6 +1040,7 @@ pub fn execute( &mut next_batch_ordinal, )?); let candidate = finalize_arm(run_arm( + &run_control, "candidate", wording, repository, @@ -965,7 +1071,10 @@ pub fn execute( candidate, }); } - ensure!(rows.len() == WORDING_COUNT, "etr1_row_count_mismatch"); + ensure!( + rows.len() == preparation.wordings.len(), + "etr1_row_count_mismatch" + ); let final_engine = engine_receipt(&residency.revalidate()?)?; for key in [ "server_instance_id", @@ -997,6 +1106,7 @@ pub fn execute( .sum::(); let stage = stage_output_directory(output)?; + run_control.check()?; let rows_directory = stage.path().join("rows"); fs::create_dir(&rows_directory)?; let mut row_bindings = Vec::with_capacity(rows.len()); @@ -1021,13 +1131,14 @@ pub fn execute( }; let manifest = Etr1RunManifestV1 { contract: RUN_CONTRACT.into(), - authority: "visible_development_frontier_only".into(), + authority: preparation.authority.clone(), experiment_status: "awaiting_validation".into(), decision: "not_evaluated".into(), parent_head: PARENT_HEAD.into(), build, preparation: preparation_binding, fragment_vectors: vector_binding, + document_execution, method_sha256: preparation.method.sha256, annotation_access: "not_accessed".into(), vector_artifact_loaded_before_timing: true, @@ -1102,7 +1213,7 @@ mod tests { } #[test] - fn qualification_events_accept_the_native_zero_based_sequence() { + fn qualification_events_use_native_completion_identity() { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("events.jsonl"); fs::write( @@ -1112,7 +1223,7 @@ mod tests { "\"status\":\"completed\",\"server_event_sequence\":10,\"clock\":{},", "\"details\":{\"completed_tokens\":\"5\",\"native_completion_sequence\":\"1\",", "\"request_id\":\"first\"}}\n", - "{\"schema_version\":1,\"sequence\":1,\"action\":\"completed_tokens\",", + "{\"schema_version\":1,\"sequence\":0,\"action\":\"completed_tokens\",", "\"status\":\"completed\",\"server_event_sequence\":11,\"clock\":{},", "\"details\":{\"completed_tokens\":\"7\",\"native_completion_sequence\":\"2\",", "\"request_id\":\"second\"}}\n", @@ -1123,10 +1234,17 @@ mod tests { let events = read_completed_events(&path).unwrap(); assert_eq!(events.len(), 2); assert_eq!(events[0].sequence, 0); - assert_eq!(events[1].sequence, 1); + assert_eq!(events[1].sequence, 0); let bytes = fs::read_to_string(&path).unwrap(); - fs::write(&path, bytes.replace("\"sequence\":1", "\"sequence\":0")).unwrap(); + fs::write( + &path, + bytes.replace( + "\"native_completion_sequence\":\"2\"", + "\"native_completion_sequence\":\"1\"", + ), + ) + .unwrap(); assert!(read_completed_events(&path).is_err()); } diff --git a/scripts/codestory-etr1-canary.mjs b/scripts/codestory-etr1-canary.mjs index 8c984894f..e61280323 100644 --- a/scripts/codestory-etr1-canary.mjs +++ b/scripts/codestory-etr1-canary.mjs @@ -1,83 +1,90 @@ import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; -import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import { execFileSync } from "node:child_process"; +import { mkdir, readFile, writeFile, realpath } from "node:fs/promises"; import path from "node:path"; import { parseArgs } from "node:util"; import { fileURLToPath } from "node:url"; -import { maximizeCoveredAtoms, scoreOrder, selectSuccessors, sha256, - validateVector } from "./lib/etr1-evidence.mjs"; +import { randomUUID } from "node:crypto"; +import { executeRecorded, fileBinding } from "./lib/etr1-execution.mjs"; +import { validateEtr1 } from "./codestory-etr1-validate.mjs"; +import { evaluateEtr1 } from "./codestory-etr1-evaluate.mjs"; async function main() { const { values } = parseArgs({ options: { diagnostic: { type: "string" }, - "state-root": { type: "string" }, "output-dir": { type: "string" } } }); - for (const name of ["diagnostic", "state-root", "output-dir"]) + runner: { type: "string" }, "output-dir": { type: "string" } } }); + for (const name of ["diagnostic", "runner", "output-dir"]) assert.ok(values[name] && path.isAbsolute(values[name]), `missing absolute --${name}`); await mkdir(values["output-dir"], { mode: 0o700 }); - const documents = [ - { id: "seed", text: "pub fn process() { dispatch(); }\n" }, - { id: "target", text: "fn dispatch() { persist(); }\n" }, - { id: "noise-a", text: "fn render_banner() { paint(); }\n" }, - { id: "noise-b", text: "fn parse_flags() { validate(); }\n" }, - ]; - const question = "How does process reach persistence?"; - const input = { contract: "codestory.embedding-diagnostic-input/v1", records: [ - ...documents.map(({ id, text }) => ({ id, purpose: "document", text })), - { id: "control", purpose: "query", text: question }, - { id: "candidate", purpose: "query", text: `${question}\n\n${documents[0].text}` }, - ] }; - const inputBytes = Buffer.from(JSON.stringify(input)), inputPath = path.join(values["output-dir"], "input.json"); - await writeFile(inputPath, inputBytes, { flag: "wx", mode: 0o600 }); - const vectorPath = path.join(values["state-root"], "canary-vectors.json"); - const execution = spawnSync(values.diagnostic, ["--input", inputPath, "--input-sha256", sha256(inputBytes), - "--state-root", values["state-root"], "--output", vectorPath], - { encoding: "utf8", timeout: 180_000, maxBuffer: 4 * 1024 * 1024, env: process.env }); - assert.equal(execution.error, undefined, `embedding diagnostic failed to launch: ${execution.error}`); - assert.equal(execution.status, 0, `embedding diagnostic failed: ${execution.stderr}`); - const vectorBytes = await readFile(vectorPath), artifact = JSON.parse(vectorBytes); - assert.equal(artifact.contract, "codestory.embedding-diagnostic-output/v1"); - assert.equal(artifact.input_sha256, sha256(inputBytes)); - assert.equal(artifact.records.length, input.records.length); - const vectors = new Map(artifact.records.map((record, index) => { - assert.equal(record.id, input.records[index].id); - assert.equal(record.text_sha256, sha256(input.records[index].text)); - validateVector(record.vector, `canary vector ${record.id}`); - return [record.id, record.vector]; - })); - const dot = (left, right) => left.reduce((sum, value, index) => - Math.fround(sum + Math.fround(Math.fround(value) * Math.fround(right[index]))), 0); - const frontiers = {}; - for (const arm of ["control", "candidate"]) { - const scores = documents.map(({ id }) => dot(vectors.get(arm), vectors.get(id))); - const successors = selectSuccessors(scoreOrder(documents.map(({ id }) => id), scores), - new Set(["seed"]), new Set()); - frontiers[arm] = { scores, successors, legal_pool: ["seed", ...successors] }; - } - const frontierBytes = Buffer.from(JSON.stringify(frontiers)); - await writeFile(path.join(values["output-dir"], "frontiers.json"), frontierBytes, - { flag: "wx", mode: 0o600 }); - // Synthetic truth is constructed only after both frontier outputs are frozen. - const costs = new Map(documents.map(({ id, text }) => [id, Buffer.byteLength(text) + 128])); - const evaluated = Object.fromEntries(Object.entries(frontiers).map(([arm, frontier]) => { - const requirement = frontier.legal_pool.includes("target") ? [["target"]] : [null]; - return [arm, maximizeCoveredAtoms(requirement, costs, 256)]; - })); - const events = await readFile(path.join(values["state-root"], "ipc", - `${process.env.CODESTORY_EMBED_QUALIFICATION_NONCE}.events.jsonl`)); - const completed = events.toString("utf8").trimEnd().split("\n").map(JSON.parse) - .filter((event) => event.action === "completed_tokens"); - assert.ok(completed.length >= 2 && completed.every((event) => Number(event.details.completed_tokens) > 0), - "canary token completion evidence missing"); - const receipt = { contract: "codestory.etr1-synthetic-canary/v1", experiment_status: "valid", - packet_decision: "not_evaluated", input_sha256: sha256(inputBytes), - vectors_sha256: sha256(vectorBytes), frontiers_sha256: sha256(frontierBytes), - diagnostic_binary_sha256: sha256(await readFile(values.diagnostic)), - qualification_events_sha256: sha256(events), completed_token_events: completed.length, - vector_artifact_bytes: (await stat(vectorPath)).size, evaluated }; - const receiptBytes = Buffer.from(`${JSON.stringify(receipt, null, 2)}\n`); - const output = path.join(values["output-dir"], "receipt.json"); - await writeFile(output, receiptBytes, { flag: "wx", mode: 0o600 }); - console.log(`${sha256(receiptBytes)} ${output}`); + const root = await realpath(values["output-dir"]); + const sourceRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + const project = path.join(root, "repository"), prepared = path.join(root, "prepared"); + await mkdir(project, { mode: 0o700 }); + const source = Array.from({ length: 32 }, (_, i) => + `fn commonneedle_${i}() { commonneedle(); ${i === 0 ? "raremarker();" : ""} }\n`).join(""); + await writeFile(path.join(project, "canary.rs"), source, { flag: "wx", mode: 0o600 }); + const git = (...args) => execFileSync("git", ["-C", project, "-c", "core.hooksPath=/dev/null", ...args], + { encoding: "utf8", stdio: "pipe" }); + git("init", "--quiet"); git("add", "canary.rs"); + git("-c", "user.name=ETR canary", "-c", "user.email=canary@invalid.local", "-c", "commit.gpgsign=false", + "commit", "--quiet", "-m", "freeze synthetic source"); + execFileSync(values.runner, ["prepare-canary", "--project-root", project, "--output-dir", prepared], + { encoding: "utf8", stdio: "pipe", timeout: 60_000 }); + const preparationPath = path.join(prepared, "preparation.json"); + const preparation = JSON.parse(await readFile(preparationPath, "utf8")); + const preparationBinding = await fileBinding(preparationPath); + const makeState = async (name) => { + const state = path.join(root, name), ipc = path.join(state, "ipc"), cache = path.join(state, "cache"); + await mkdir(state, { mode: 0o700 }); + await mkdir(ipc, { mode: 0o700 }); await mkdir(cache, { mode: 0o700 }); + const nonce = `etr1-canary-${randomUUID()}`; + return { state, events: path.join(ipc, `${nonce}.events.jsonl`), + env: { ...process.env, CODESTORY_CACHE_ROOT: cache, CODESTORY_EMBED_ALLOW_CPU: "false", + CODESTORY_EMBED_QUALIFICATION_DIR: ipc, CODESTORY_EMBED_QUALIFICATION_NONCE: nonce } }; + }; + const documentState = await makeState("documents"), vectorPath = path.join(documentState.state, "vectors.json"); + const documents = await executeRecorded({ role: "documents", executable: values.diagnostic, + args: ["--input", preparation.embedding_input.path, "--input-sha256", preparation.embedding_input.sha256, + "--state-root", documentState.state, "--output", vectorPath], + inputs: [preparation.embedding_input.path], outputPaths: [vectorPath], eventsPath: documentState.events, + directory: path.join(root, "document-execution"), sourceRoot, env: documentState.env }); + assert.equal(documents.receipt.experiment_status, "completed", "canary document execution failed"); + const vectors = await fileBinding(vectorPath), queryState = await makeState("queries"); + const runDirectory = path.join(root, "run"), runPath = path.join(runDirectory, "run.json"); + const cancelFile = path.join(root, "cancel"); + const execution = await executeRecorded({ role: "paired_run", executable: values.runner, + args: ["run", "--prepared", preparationPath, "--prepared-sha256", preparationBinding.sha256, + "--fragment-vectors", vectorPath, "--fragment-vectors-sha256", vectors.sha256, + "--document-execution", documents.binding.path, "--document-execution-sha256", documents.binding.sha256, + "--state-root", queryState.state, "--output-dir", runDirectory, "--cancel-file", cancelFile], + inputs: [preparationPath, vectorPath, documents.binding.path], outputPaths: [runPath], + eventsPath: queryState.events, directory: path.join(root, "run-execution"), sourceRoot, + env: queryState.env, cancelFile }); + assert.equal(execution.receipt.experiment_status, "completed", "canary paired execution failed"); + const runBinding = await fileBinding(runPath); + const validated = await validateEtr1({ runBinding, sourceRoot, executionBinding: execution.binding, allowCanary: true }); + const validationPath = path.join(root, "validation.json"); + await writeFile(validationPath, JSON.stringify({ contract: "codestory.etr1-validation/v1", + authority: "synthetic_canary_only", experiment_status: "valid", decision: "not_evaluated", + annotation_access: "not_accessed", run: runBinding, execution: execution.binding, + binary_sha256: validated.run.build.binary_sha256 }), { flag: "wx", mode: 0o600 }); + // Synthetic truth enters only after the real paired run and validator finish. + const first = preparation.fragments[0]; + const annotationsPath = path.join(root, "annotations.json"); + const annotations = { authority: "synthetic_canary_only", cases: preparation.wordings.map((row) => ({ + case_id: row.case_id, acceptable_sets: [{ set_id: "first-fragment", required_relation_atoms: [], + required_source_atoms: [{ atom_id: "first", source_range: { path: first.path, + content_digest: first.content_digest, byte_range: first.byte_range, line_range: first.line_range } }] }] })) }; + await writeFile(annotationsPath, JSON.stringify(annotations), { flag: "wx", mode: 0o600 }); + const validation = await fileBinding(validationPath), annotationBinding = await fileBinding(annotationsPath); + const evaluated = await evaluateEtr1({ validationPath, validationSha256: validation.sha256, + annotationsPath, annotationsSha256: annotationBinding.sha256, sourceRoot, allowCanary: true }); + const receiptPath = path.join(root, "receipt.json"); + await writeFile(receiptPath, JSON.stringify({ contract: "codestory.etr1-synthetic-canary/v2", + authority: "synthetic_canary_only", experiment_status: "valid", packet_decision: "not_evaluated", + preparation: preparationBinding, documents: documents.binding, execution: execution.binding, + validation, evaluated }), { flag: "wx", mode: 0o600 }); + console.log(JSON.stringify(await fileBinding(receiptPath))); } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) - main().catch((error) => { console.error(error.message); process.exitCode = 1; }); + main().catch((error) => { console.error(error.stack); process.exitCode = 1; }); diff --git a/scripts/codestory-etr1-evaluate.mjs b/scripts/codestory-etr1-evaluate.mjs index 71a91457c..71de09e3b 100644 --- a/scripts/codestory-etr1-evaluate.mjs +++ b/scripts/codestory-etr1-evaluate.mjs @@ -115,50 +115,46 @@ export function gateTwo(cases, control, candidate, candidateGateOne) { export function decision(controlGate, candidateGate, conditioningGate) { if (!controlGate && !candidateGate) return { frontier: null, - decision: "stop_automatic_packet_compilation", reason: "neither_arm_sufficient" }; + decision: "no_frontier_selected", reason: "neither_arm_sufficient" }; if (controlGate && !conditioningGate) return { frontier: "control", - decision: "freeze_unconditioned_frontier_for_selector_experiment", + decision: "unconditioned_frontier_selected", reason: candidateGate ? "conditioning_lacks_material_value" : "control_alone_sufficient" }; if (candidateGate && conditioningGate) return { frontier: "candidate", - decision: "freeze_conditioned_frontier_for_selector_experiment", + decision: "conditioned_frontier_selected", reason: controlGate ? "conditioning_materially_better" : "conditioning_only_adequate_frontier" }; - return { frontier: null, decision: "stop_automatic_packet_compilation", + return { frontier: null, decision: "no_frontier_selected", reason: "only_conditioned_arm_sufficient_without_material_conditioning_value" }; } export async function evaluateEtr1({ validationPath, validationSha256, annotationsPath, - annotationsSha256, oraclePath, oracleSha256, sourceRoot }) { + annotationsSha256, oraclePath, oracleSha256, sourceRoot, allowCanary = false }) { const { value: validation } = await readBound(validationPath, validationSha256); assert.equal(validation.contract, "codestory.etr1-validation/v1"); assert.equal(validation.experiment_status, "valid", "validator did not authorize annotation access"); assert.equal(validation.decision, "not_evaluated"); assert.equal(validation.annotation_access, "not_accessed"); const validated = await validateEtr1({ runBinding: validation.run, - runPath: validation.run.path, sourceRoot }); + runPath: validation.run.path, sourceRoot, executionBinding: validation.execution, allowCanary }); assert.equal(validated.run.build.binary_sha256, validation.binary_sha256, "validation receipt no longer binds the run binary"); // This is the first annotation read in the authoritative path. const { value: annotations } = await readBound(annotationsPath, annotationsSha256); + const canary = validated.run.authority === "synthetic_canary_only"; + if (canary) { + assert.equal(annotations.authority, "synthetic_canary_only"); + const rows = scoreRows(validated, annotations); + assert.deepEqual(rows.map((row) => [row.control.recall, row.candidate.recall]), [[1, 1], [1, 1], [0, 0]], + "synthetic oracle output changed"); + return { contract: "codestory.etr1-evaluation/v1", authority: "synthetic_canary_only", + experiment_status: "valid", decision: "not_evaluated", packet_decision: "not_evaluated", rows }; + } assert.equal(annotations.authority, "visible_development_only"); assert.equal(annotations.questions_sha256, validated.preparation.fixed_inputs.questions.sha256, "annotation question binding changed"); const { value: oracle } = await readBound(oraclePath, oracleSha256); const oracle_reproduction = await reproduceOracleFixtures({ oracle, preparation: validated.preparation, annotations }); - const annotationByCase = new Map(annotations.cases.map((value) => [value.case_id, value])); - const repositoryById = new Map(validated.preparation.repositories - .map((value) => [value.repository_id, value])); - const fragmentById = new Map(validated.preparation.fragments.map((value) => [value.fragment_id, value])); - const scoredRows = validated.rows.map((row) => { - const annotation = annotationByCase.get(row.case_id), repository = repositoryById.get(row.repository_id); - assert.ok(annotation && repository, "evaluation binding missing"); - const repositoryFragments = repository.fragment_ids.map((id) => fragmentById.get(id)); - const score = (name) => ({ ...evaluateArm(annotation, repositoryFragments, - row[name].legally_selectable_pool, repository.base_serialized_bytes), - prepared_state_ns: row[name].timing.prepared_state_ns }); - return { case_id: row.case_id, phrasing_id: row.phrasing_id, group: row.group, - control: score("control"), candidate: score("candidate") }; - }); + const scoredRows = scoreRows(validated, annotations); const cases = buildCases(scoredRows), control = aggregateArm(cases, "control"), candidate = aggregateArm(cases, "candidate"), controlGate = gateOne(control), candidateGate = gateOne(candidate), conditioning = gateTwo(cases, control, candidate, candidateGate.pass); @@ -183,6 +179,22 @@ export async function evaluateEtr1({ validationPath, validationSha256, annotatio frontier_construction_latency: latency }, selected, cases, rows: scoredRows }; } +function scoreRows(validated, annotations) { + const annotationByCase = new Map(annotations.cases.map((value) => [value.case_id, value])); + const repositoryById = new Map(validated.preparation.repositories.map((value) => [value.repository_id, value])); + const fragmentById = new Map(validated.preparation.fragments.map((value) => [value.fragment_id, value])); + return validated.rows.map((row) => { + const annotation = annotationByCase.get(row.case_id), repository = repositoryById.get(row.repository_id); + assert.ok(annotation && repository, "evaluation binding missing"); + const fragments = repository.fragment_ids.map((id) => fragmentById.get(id)); + const score = (name) => ({ ...evaluateArm(annotation, fragments, + row[name].legally_selectable_pool, repository.base_serialized_bytes), + prepared_state_ns: row[name].timing.prepared_state_ns }); + return { case_id: row.case_id, phrasing_id: row.phrasing_id, group: row.group, + control: score("control"), candidate: score("candidate") }; + }); +} + async function main() { const { values } = parseArgs({ options: Object.fromEntries([ "validation", "validation-sha256", "annotations", "annotations-sha256", diff --git a/scripts/codestory-etr1-validate.mjs b/scripts/codestory-etr1-validate.mjs index 9c9770651..21133c3da 100644 --- a/scripts/codestory-etr1-validate.mjs +++ b/scripts/codestory-etr1-validate.mjs @@ -4,6 +4,7 @@ import { readFile, realpath, stat, writeFile } from "node:fs/promises"; import path from "node:path"; import { parseArgs } from "node:util"; import { fileURLToPath } from "node:url"; +import { validateExecution } from "./lib/etr1-execution.mjs"; import { authenticateFragment, encodedCandidateInput, f32Dot, fragmentId, LIMITS, scoreOrder, selectSuccessors, sha256, validateVector } from "./lib/etr1-evidence.mjs"; @@ -155,7 +156,12 @@ export function validateArm({ arm, expectedName, wording, repository, fragments, } assert.deepEqual(arm.successors, [...prior], "successor pool equation changed"); assert.ok(arm.batch_receipts.every((batch) => batch.query_ordinals.length >= 1 - && batch.query_ordinals.length <= 8 && batch.wall_ns > 0 && batch.completed_tokens > 0), + && batch.query_ordinals.length <= 8 && batch.wall_ns > 0 && batch.completed_tokens > 0 + && Number.isSafeInteger(batch.qualification_native_completion_sequence) + && batch.qualification_native_completion_sequence > 0 + && Number.isSafeInteger(batch.qualification_server_event_sequence) + && batch.qualification_server_event_sequence > 0 + && /^[0-9a-f]{64}$/u.test(batch.qualification_request_id_sha256)), "batch contract invalid"); const batchOrdinals = arm.batch_receipts.flatMap((batch) => batch.query_ordinals).toSorted((a, b) => a - b); assert.deepEqual(batchOrdinals, Array.from({ length: seeds.length }, (_, index) => index), @@ -167,18 +173,42 @@ export function validateArm({ arm, expectedName, wording, repository, fragments, timingKeys.forEach((key) => assert.ok(Number.isSafeInteger(arm.timing[key]) && arm.timing[key] >= 0, `invalid timing ${key}`)); assert.equal(arm.timing.prepared_state_ns, - timingKeys.reduce((sum, key) => sum + arm.timing[key], 0), "prepared timing does not reconcile"); + timingKeys.reduce((sum, key) => sum + arm.timing[key], 0) + arm.timing.unaccounted_ns, + "prepared timing does not reconcile"); + assert.ok(Number.isSafeInteger(arm.timing.unaccounted_ns) && arm.timing.unaccounted_ns >= 0, + "invalid unaccounted timing"); validateSourceAuthentication(arm, fragments, repository, sourceFiles); } export function parseEvents(bytes) { assert.equal(bytes.at(-1), 10, "qualification event log is unterminated"); - let previous = null; - return bytes.toString("utf8").trimEnd().split("\n").map(JSON.parse).filter((event) => { + let previousNative = null, previousServer = null; + const requestIds = new Set(); + return bytes.toString("utf8").trimEnd().split("\n").map(JSON.parse).map((event) => { assert.equal(event.schema_version, 1, "qualification event schema changed"); - assert.ok(previous === null || event.sequence > previous, "qualification event sequence changed"); - previous = event.sequence; - return event.action === "completed_tokens"; + assert.equal(event.sequence, 0, "automatic token event command sequence changed"); + assert.equal(event.action, "completed_tokens", "unexpected qualification action"); + assert.equal(event.status, "completed", "qualification token event failed"); + assert.ok(event.clock && typeof event.clock === "object" && event.snapshot === undefined, + "qualification token event shape changed"); + const nativeSequence = Number(event.details?.native_completion_sequence); + const serverSequence = Number(event.server_event_sequence); + const completedTokens = Number(event.details?.completed_tokens); + const requestId = event.details?.request_id; + assert.ok(Number.isSafeInteger(nativeSequence) && nativeSequence > 0 + && (previousNative === null || nativeSequence === previousNative + 1), + "qualification native completion sequence changed"); + assert.ok(Number.isSafeInteger(serverSequence) && serverSequence > 0 + && (previousServer === null || serverSequence > previousServer), + "qualification server event sequence changed"); + assert.ok(Number.isSafeInteger(completedTokens) && completedTokens > 0, + "qualification completed token count changed"); + assert.ok(typeof requestId === "string" && requestId && !requestIds.has(requestId), + "qualification request identity changed"); + requestIds.add(requestId); + previousNative = nativeSequence; + previousServer = serverSequence; + return event; }); } @@ -195,10 +225,12 @@ async function loadSourceFiles(repository, repositoryFragments) { return result; } -export async function validateEtr1({ runBinding, runPath, sourceRoot }) { +export async function validateEtr1({ runBinding, runPath, sourceRoot, executionBinding, allowCanary = false }) { const run = await boundJson(runBinding, runPath); assert.equal(run.contract, "codestory.etr1-run/v1"); - assert.equal(run.authority, "visible_development_frontier_only"); + const canary = run.authority === "synthetic_canary_only"; + assert.ok(canary ? allowCanary : run.authority === "visible_development_frontier_only", + "synthetic canary cannot authorize corpus evaluation"); assert.equal(run.experiment_status, "awaiting_validation"); assert.equal(run.decision, "not_evaluated"); assert.equal(run.parent_head, FIXED.parent); @@ -213,22 +245,29 @@ export async function validateEtr1({ runBinding, runPath, sourceRoot }) { assert.equal(run.build.source_dirty, false, "run binary was built dirty"); assert.equal(sha256(await readFile(run.build.binary_path)), run.build.binary_sha256, "run binary changed"); const preparation = await boundJson(run.preparation); + const { request: runnerRequest, receipt: runnerExecution } = await validateExecution(executionBinding, + { role: "paired_run", input: run.preparation, output: runBinding, sourceRoot }); + assert.equal(runnerRequest.executable.sha256, run.build.binary_sha256, "run producer changed"); + const { request: documentRequest } = await validateExecution(run.document_execution, + { role: "documents", input: preparation.embedding_input, output: run.fragment_vectors, sourceRoot }); assert.equal(preparation.contract, "codestory.etr1-preparation/v1"); + assert.equal(preparation.authority, run.authority); validatePreAnnotationBoundary(run, preparation); - assert.equal(preparation.annotations.sha256, FIXED.annotations); + assert.equal(preparation.annotations.sha256, canary ? "0".repeat(64) : FIXED.annotations); assert.equal(preparation.model_sha256, FIXED.model); assert.equal(preparation.tokenizer_sha256, FIXED.tokenizer); assert.equal(preparation.build.source_commit, sourceCommit); assert.equal(preparation.build.source_tree, sourceTree); assert.equal(run.method_sha256, preparation.method.sha256); - for (const name of ["fragment_diagnostic", "fragment_build", "lexical_membership_freeze", + for (const name of canary ? [] : ["fragment_diagnostic", "fragment_build", "lexical_membership_freeze", "questions", "model_contract", "lexical_policy_source"]) assert.equal(preparation.fixed_inputs[name].sha256, FIXED[name], `fixed input changed: ${name}`); await boundBytes(preparation.method); await boundBytes(preparation.embedding_input); for (const binding of Object.values(preparation.fixed_inputs)) await boundBytes(binding); - assert.equal(preparation.fragments.length, 10_369, "fragment count changed"); - assert.equal(preparation.wordings.length, 72, "wording count changed"); + assert.equal(preparation.fragments.length, canary ? 32 : 10_369, "fragment count changed"); + assert.equal(preparation.wordings.length, canary ? 3 : 72, "wording count changed"); + if (canary) assert.deepEqual(preparation.wordings.map((row) => row.seed_fragment_ids.length), [16, 1, 0]); const fragments = new Map(), fragmentsByRepository = new Map(); for (const fragment of preparation.fragments) { assert.equal(fragment.fragment_id, fragmentId(fragment), "prepared fragment identity changed"); @@ -251,6 +290,7 @@ export async function validateEtr1({ runBinding, runPath, sourceRoot }) { assert.equal(vectorArtifact.input_sha256, preparation.embedding_input.sha256); assert.equal(vectorArtifact.source_commit, sourceCommit); assert.equal(vectorArtifact.source_tree, sourceTree); + assert.equal(vectorArtifact.binary_sha256, documentRequest.executable.sha256, "document producer changed"); validateEngine(vectorArtifact.initial_engine); validateEngine(vectorArtifact.final_engine); assert.equal(vectorArtifact.initial_engine.server_instance_id, vectorArtifact.final_engine.server_instance_id); @@ -267,6 +307,7 @@ export async function validateEtr1({ runBinding, runPath, sourceRoot }) { assert.equal(run.initial_engine.server_instance_id, run.final_engine.server_instance_id, "query engine changed during ETR-1"); const eventBytes = await boundBytes(run.qualification_events), events = parseEvents(eventBytes); + assert.equal(run.qualification_events.sha256, runnerExecution.events.sha256, "runner native events changed"); const batches = new Map(), rows = []; assert.equal(run.rows.length, preparation.wordings.length, "run row count changed"); for (let rowIndex = 0; rowIndex < run.rows.length; rowIndex++) { @@ -295,10 +336,13 @@ export async function validateEtr1({ runBinding, runPath, sourceRoot }) { assert.equal(events.length, batches.size, "qualification event count differs from successful batches"); for (const [ordinal, batch] of [...batches].sort((left, right) => left[0] - right[0])) { const event = events[ordinal]; - assert.equal(event.sequence, batch.qualification_event_sequence, "batch event sequence changed"); + assert.equal(Number(event.details.native_completion_sequence), + batch.qualification_native_completion_sequence, "batch native completion sequence changed"); + assert.equal(event.server_event_sequence, batch.qualification_server_event_sequence, + "batch server event sequence changed"); + assert.equal(sha256(event.details.request_id), batch.qualification_request_id_sha256, + "batch request identity changed"); assert.equal(Number(event.details.completed_tokens), batch.completed_tokens, "batch token count changed"); - assert.ok(event.details.request_id, "batch request identity missing"); - assert.ok(Number(event.details.native_completion_sequence) > 0, "native completion identity missing"); } assert.equal(run.qualification_completed_token_total, [...batches.values()].reduce((sum, batch) => sum + batch.completed_tokens, 0), @@ -311,16 +355,23 @@ export async function validateEtr1({ runBinding, runPath, sourceRoot }) { async function main() { const { values } = parseArgs({ options: { run: { type: "string" }, "run-sha256": { type: "string" }, output: { type: "string" }, + execution: { type: "string" }, "execution-sha256": { type: "string" }, + "synthetic-canary": { type: "boolean", default: false }, } }); - for (const name of ["run", "run-sha256", "output"]) assert.ok(values[name], `missing --${name}`); + for (const name of ["run", "run-sha256", "output", "execution", "execution-sha256"]) + assert.ok(values[name], `missing --${name}`); assert.ok(path.isAbsolute(values.run) && path.isAbsolute(values.output), "paths must be absolute"); const sourceRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); let receipt; try { - const validated = await validateEtr1({ runBinding: { path: values.run, + const executionBinding = { path: values.execution, sha256: values["execution-sha256"], + bytes: (await stat(values.execution)).size }; + const validated = await validateEtr1({ executionBinding, allowCanary: values["synthetic-canary"], runBinding: { path: values.run, sha256: values["run-sha256"], bytes: (await stat(values.run)).size }, runPath: values.run, sourceRoot }); receipt = { contract: "codestory.etr1-validation/v1", experiment_status: "valid", decision: "not_evaluated", annotation_access: "not_accessed", + execution: executionBinding, + authority: validated.run.authority, run: { path: values.run, sha256: values["run-sha256"], bytes: (await stat(values.run)).size }, source_commit: validated.source_commit, source_tree: validated.source_tree, binary_sha256: validated.run.build.binary_sha256, diff --git a/scripts/lib/etr1-execution.mjs b/scripts/lib/etr1-execution.mjs new file mode 100644 index 000000000..60ebb6902 --- /dev/null +++ b/scripts/lib/etr1-execution.mjs @@ -0,0 +1,143 @@ +import assert from "node:assert/strict"; +import { spawn, execFileSync } from "node:child_process"; +import { mkdir, readFile, writeFile, realpath, lstat } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { sha256 } from "./etr1-evidence.mjs"; + +export const ANALYSIS_FILES = ["scripts/codestory-etr1-validate.mjs", + "scripts/codestory-etr1-evaluate.mjs", "scripts/lib/etr1-evidence.mjs", + "scripts/lib/etr1-execution.mjs"]; + +export async function fileBinding(file) { + const bytes = await readFile(file); + return { path: await realpath(file), sha256: sha256(bytes), bytes: bytes.length }; +} + +export async function readExecutionBinding(binding) { + const bytes = await readFile(binding.path); + assert.equal(bytes.length, binding.bytes, "execution artifact length changed"); + assert.equal(sha256(bytes), binding.sha256, "execution artifact digest changed"); + return JSON.parse(bytes); +} + +export async function analysisIdentity(sourceRoot) { + const git = (...args) => execFileSync("git", ["-C", sourceRoot, ...args], { + encoding: "utf8", env: { ...process.env, GIT_OPTIONAL_LOCKS: "0" } }).trim(); + assert.equal(git("status", "--porcelain", "--untracked-files=no"), "", "analysis checkout is dirty"); + return { source_commit: git("rev-parse", "HEAD"), source_tree: git("rev-parse", "HEAD^{tree}"), + files: await Promise.all(ANALYSIS_FILES.map((file) => fileBinding(path.join(sourceRoot, file)))), + node: await fileBinding(process.execPath), node_version: process.version }; +} + +export async function validateExecution(binding, { role, input, output, sourceRoot }) { + assert.ok(binding, "independent execution receipt required"); + const receipt = await readExecutionBinding(binding); + assert.equal(receipt.contract, "codestory.etr1-execution/v1"); + assert.equal(receipt.role, role, "execution role changed"); + assert.equal(receipt.experiment_status, "completed", "execution did not complete"); + assert.equal(receipt.exit_code, 0); + assert.equal(receipt.signal, null); + assert.equal(receipt.annotation_access, "not_accessed"); + const request = await readExecutionBinding(receipt.request); + assert.equal(request.contract, "codestory.etr1-execution-request/v1"); + assert.equal(request.role, role); + if (role === "paired_run") assert.equal(request.deadline_ms, 1_800_000, "paired deadline changed"); + assert.deepEqual(request.analysis, await analysisIdentity(sourceRoot), "analysis identity changed"); + assert.deepEqual(await fileBinding(request.executable.path), request.executable, "producer binary changed"); + for (const expected of request.inputs) + assert.deepEqual(await fileBinding(expected.path), expected, "execution input changed"); + assert.ok(request.inputs.some((item) => item.sha256 === input.sha256 && item.path === input.path), + "execution input not independently bound"); + assert.ok(request.output_paths.includes(output.path), "execution output not declared before launch"); + assert.ok(receipt.outputs.some((item) => item.sha256 === output.sha256 && item.path === output.path), + "execution output not independently bound"); + for (const expected of [...receipt.outputs, receipt.stdout, receipt.stderr, receipt.events]) + assert.deepEqual(await fileBinding(expected.path), expected, "execution result changed"); + assert.ok(Number.isSafeInteger(receipt.wall_ns) && receipt.wall_ns > 0); + assert.ok(Number.isSafeInteger(receipt.pid) && receipt.pid > 0); + return { request, receipt }; +} + +// The supervisor freezes inputs before launch and captures outputs before any +// evaluator can run. It does not claim to defend against a hostile host owner. +export async function executeRecorded({ role, executable, args, inputs, outputPaths, eventsPath, + directory, sourceRoot, env, cancelFile, deadlineMs = 30 * 60 * 1000 }) { + assert.ok(["documents", "paired_run"].includes(role), "unsupported ETR execution role"); + if (role === "paired_run") assert.equal(deadlineMs, 1_800_000, "paired deadline changed"); + for (const file of [...outputPaths, eventsPath]) { + const exists = await lstat(file).then(() => true, (error) => { + if (error.code === "ENOENT") return false; + throw error; + }); + assert.equal(exists, false, `execution output already exists: ${file}`); + } + await mkdir(directory, { mode: 0o700 }); + const recordedEnv = Object.fromEntries(Object.entries(env).filter(([key]) => + /^(CODESTORY_(CACHE_ROOT|EMBED_ALLOW_CPU|EMBED_QUALIFICATION_[A-Z_]+|EMBEDDING_[A-Z_]+)|XDG_CACHE_HOME|TMPDIR|DYLD_LIBRARY_PATH)$/u.test(key) + && !/TOKEN|SECRET|PASSWORD|CREDENTIAL/u.test(key))); + const request = { contract: "codestory.etr1-execution-request/v1", role, + executable: await fileBinding(executable), args, environment: recordedEnv, + inputs: await Promise.all(inputs.map(fileBinding)), output_paths: outputPaths, + events_path: eventsPath, cancel_file: cancelFile ?? null, deadline_ms: deadlineMs, + analysis: await analysisIdentity(sourceRoot) }; + const requestPath = path.join(directory, "request.json"); + await writeFile(requestPath, JSON.stringify(request), { flag: "wx", mode: 0o600 }); + const requestBinding = await fileBinding(requestPath); + const started = process.hrtime.bigint(); + const child = spawn(executable, args, { env, stdio: ["ignore", "pipe", "pipe"] }); + const stdout = [], stderr = []; + child.stdout.on("data", (value) => stdout.push(value)); + child.stderr.on("data", (value) => stderr.push(value)); + let cancelled = false, forcedKill; + const cancel = async () => { + cancelled = true; + if (cancelFile) await writeFile(cancelFile, "cancel\n", { flag: "wx", mode: 0o600 }).catch(() => {}); + else child.kill("SIGTERM"); + forcedKill ??= setTimeout(() => child.kill("SIGKILL"), 5000); + }; + const onSignal = () => { void cancel(); }; + process.once("SIGINT", onSignal); + process.once("SIGTERM", onSignal); + const timer = setTimeout(onSignal, deadlineMs); + const terminal = await new Promise((resolve) => { + child.once("error", (error) => resolve({ exit_code: null, signal: null, error: error.message })); + child.once("close", (code, signal) => resolve({ exit_code: code, signal })); + }); + clearTimeout(timer); + clearTimeout(forcedKill); + process.removeListener("SIGINT", onSignal); + process.removeListener("SIGTERM", onSignal); + const wall_ns = Number(process.hrtime.bigint() - started); + const stdoutPath = path.join(directory, "stdout.log"), stderrPath = path.join(directory, "stderr.log"); + await writeFile(stdoutPath, Buffer.concat(stdout), { flag: "wx", mode: 0o600 }); + await writeFile(stderrPath, Buffer.concat(stderr), { flag: "wx", mode: 0o600 }); + let outputs = [], events = null, bindingError; + try { + outputs = await Promise.all(outputPaths.map(fileBinding)); + events = await fileBinding(eventsPath); + } catch (error) { bindingError = error.message; } + const completed = terminal.exit_code === 0 && !terminal.signal && !cancelled && !bindingError; + const receipt = { contract: "codestory.etr1-execution/v1", role, request: requestBinding, + experiment_status: completed ? "completed" : "invalid", decision: "not_evaluated", + annotation_access: "not_accessed", pid: child.pid ?? null, ...terminal, cancelled, + error: terminal.error ?? bindingError ?? null, wall_ns, outputs, events, + stdout: await fileBinding(stdoutPath), stderr: await fileBinding(stderrPath) }; + const receiptPath = path.join(directory, "receipt.json"); + await writeFile(receiptPath, JSON.stringify(receipt), { flag: "wx", mode: 0o600 }); + return { binding: await fileBinding(receiptPath), receipt }; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const main = async () => { + assert.equal(process.argv.length, 3, "usage: node scripts/lib/etr1-execution.mjs /absolute/job.json"); + assert.ok(path.isAbsolute(process.argv[2])); + const job = JSON.parse(await readFile(process.argv[2], "utf8")); + const result = await executeRecorded({ ...job, + sourceRoot: path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."), + env: { ...process.env, ...job.environment } }); + console.log(JSON.stringify(result.binding)); + if (result.receipt.experiment_status !== "completed") process.exitCode = 1; + }; + main().catch((error) => { console.error(error.stack); process.exitCode = 1; }); +} diff --git a/scripts/tests/etr1-evidence.test.mjs b/scripts/tests/etr1-evidence.test.mjs index 70320dff8..9443a6231 100644 --- a/scripts/tests/etr1-evidence.test.mjs +++ b/scripts/tests/etr1-evidence.test.mjs @@ -8,6 +8,7 @@ import { authenticateFragment, encodedCandidateInput, evaluateArm, exactPublicBy import { validateArm, validateDocumentVectorRecord, validateEngine, validatePreAnnotationBoundary, parseEvents } from "../codestory-etr1-validate.mjs"; import { decision, evaluateEtr1, gateOne, gateTwo } from "../codestory-etr1-evaluate.mjs"; +import { fileBinding, readExecutionBinding, validateExecution } from "../lib/etr1-execution.mjs"; function unit(index) { const vector = Array(LIMITS.vectorDimension).fill(0); @@ -32,7 +33,8 @@ function fixture(expectedName = "control") { const input = expectedName === "control" ? question : `${question}\n\n${seed.source}`; const batch = { global_batch_ordinal: 0, arm: expectedName, query_ordinals: [0], input_sha256: [sha256(input)], wall_ns: 1, completed_tokens: 2, - qualification_event_sequence: 1 }; + qualification_native_completion_sequence: 1, qualification_server_event_sequence: 10, + qualification_request_id_sha256: sha256("request-1") }; const arm = { name: expectedName, search_count: 1, query_receipts: [{ query_ordinal: 0, seed_fragment_id: seed.fragment_id, original_input_sha256: sha256(input), encoded_input_sha256: sha256(input), encoded_input: input, @@ -48,7 +50,7 @@ function fixture(expectedName = "control") { file_digests: { "src/lib.rs": digest } }, token_total: 2, timing: { round_zero_bm25_ns: 1, seed_source_authentication_ns: 1, query_encoding_ns: 1, vector_search_ns: 1, descriptor_mapping_ns: 1, remaining_source_authentication_ns: 1, - prepared_state_ns: 6 } }; + prepared_state_ns: 6, unaccounted_ns: 0 } }; return { arm, expectedName, wording, repository, fragments: new Map([[seed.fragment_id, seed], [successor.fragment_id, successor]]), documentVectors: new Map([[seed.fragment_id, unit(1)], [successor.fragment_id, unit(0)]]), @@ -84,14 +86,14 @@ test("candidate input shortening preserves UTF-8 and complete trailing lines", ( assert.throws(() => encodedCandidateInput("question", source, 3)); }); -test("validator accepts the native zero-based qualification sequence", () => { - const event = (sequence) => JSON.stringify({ schema_version: 1, sequence, - action: "completed_tokens", status: "completed", server_event_sequence: 10 + sequence, - clock: {}, details: { completed_tokens: "5", native_completion_sequence: String(sequence + 1), - request_id: `request-${sequence}` } }); - const events = parseEvents(Buffer.from(`${event(0)}\n${event(1)}\n`)); - assert.deepEqual(events.map(({ sequence }) => sequence), [0, 1]); - assert.throws(() => parseEvents(Buffer.from(`${event(0)}\n${event(0)}\n`))); +test("validator uses native completion identity for automatic token events", () => { + const event = (nativeSequence) => JSON.stringify({ schema_version: 1, sequence: 0, + action: "completed_tokens", status: "completed", server_event_sequence: 3 + nativeSequence, + clock: {}, details: { completed_tokens: "5", native_completion_sequence: String(nativeSequence), + request_id: `request-${nativeSequence}` } }); + const events = parseEvents(Buffer.from(`${event(1)}\n${event(2)}\n`)); + assert.deepEqual(events.map(({ sequence }) => sequence), [0, 0]); + assert.throws(() => parseEvents(Buffer.from(`${event(1)}\n${event(1)}\n`))); }); test("validator reconstructs both arm query contracts and refuses hostile mutations", () => { @@ -111,6 +113,15 @@ test("validator reconstructs both arm query contracts and refuses hostile mutati } }); +test("request wall timing includes unaccounted work and rejects impossible phase totals", () => { + const value = fixture(); + value.arm.timing.prepared_state_ns = 10; + value.arm.timing.unaccounted_ns = 4; + validateArm(value); + value.arm.timing.unaccounted_ns = 0; + assert.throws(() => validateArm(value), /timing/u); +}); + test("vector, model, and pre-annotation boundaries refuse substitutions", () => { const { seed } = fixture(); validateDocumentVectorRecord({ id: seed.fragment_id, purpose: "document", @@ -194,11 +205,12 @@ test("the frozen decision table never authorizes production integration", () => groups: { a: { mean_recall: 0.5 }, b: { mean_recall: 0.5 } } }; assert.equal(gateOne(sufficient).pass, true); assert.equal(gateOne(weak).pass, false); - assert.equal(decision(false, false, false).decision, "stop_automatic_packet_compilation"); + assert.equal(decision(false, false, false).decision, "no_frontier_selected"); + assert.equal(decision(false, true, false).decision, "no_frontier_selected"); assert.equal(decision(true, true, false).decision, - "freeze_unconditioned_frontier_for_selector_experiment"); + "unconditioned_frontier_selected"); assert.equal(decision(false, true, true).decision, - "freeze_conditioned_frontier_for_selector_experiment"); + "conditioned_frontier_selected"); const cases = [{ control_incomplete_for_gain: true, candidate_gained_atom: true }]; assert.equal(gateTwo(cases, weak, sufficient, true).pass, true); }); @@ -215,3 +227,17 @@ test("annotations cannot be opened before a valid validator receipt", async () = annotationsSha256: "0".repeat(64), oraclePath: path.join(directory, "oracle.json"), oracleSha256: "0".repeat(64), sourceRoot: directory }), /validator did not authorize/u); }); + +test("independently frozen execution bindings reject rehashed substitute outputs", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "etr1-execution-")); + const output = path.join(directory, "vectors.json"); + await writeFile(output, JSON.stringify({ vector: unit(0) })); + const frozen = await fileBinding(output); + await readExecutionBinding(frozen); + await writeFile(output, JSON.stringify({ vector: unit(1) })); + const substitute = await fileBinding(output); + assert.notEqual(substitute.sha256, frozen.sha256); + await assert.rejects(() => readExecutionBinding(frozen), /execution artifact digest changed/u); + await assert.rejects(() => validateExecution(undefined, { role: "documents" }), + /independent execution receipt required/u); +}); From 8c7ba1e02f68b3cdbafd107ea21deb53affd6c3c Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Fri, 4 Sep 2026 17:31:31 -0400 Subject: [PATCH 5/8] bind etr process context and require the native canary --- scripts/codestory-etr1-canary.mjs | 8 ++-- scripts/codestory-etr1-validate.mjs | 6 +-- scripts/lib/etr1-execution.mjs | 70 ++++++++++++++++++++++++---- scripts/tests/etr1-evidence.test.mjs | 26 ++++++++++- 4 files changed, 94 insertions(+), 16 deletions(-) diff --git a/scripts/codestory-etr1-canary.mjs b/scripts/codestory-etr1-canary.mjs index e61280323..0be3c9ec6 100644 --- a/scripts/codestory-etr1-canary.mjs +++ b/scripts/codestory-etr1-canary.mjs @@ -5,7 +5,7 @@ import path from "node:path"; import { parseArgs } from "node:util"; import { fileURLToPath } from "node:url"; import { randomUUID } from "node:crypto"; -import { executeRecorded, fileBinding } from "./lib/etr1-execution.mjs"; +import { executeRecorded, fileBinding, analysisIdentity } from "./lib/etr1-execution.mjs"; import { validateEtr1 } from "./codestory-etr1-validate.mjs"; import { evaluateEtr1 } from "./codestory-etr1-evaluate.mjs"; @@ -42,7 +42,7 @@ async function main() { CODESTORY_EMBED_QUALIFICATION_DIR: ipc, CODESTORY_EMBED_QUALIFICATION_NONCE: nonce } }; }; const documentState = await makeState("documents"), vectorPath = path.join(documentState.state, "vectors.json"); - const documents = await executeRecorded({ role: "documents", executable: values.diagnostic, + const documents = await executeRecorded({ role: "documents", authority: "synthetic_canary_only", executable: values.diagnostic, args: ["--input", preparation.embedding_input.path, "--input-sha256", preparation.embedding_input.sha256, "--state-root", documentState.state, "--output", vectorPath], inputs: [preparation.embedding_input.path], outputPaths: [vectorPath], eventsPath: documentState.events, @@ -51,7 +51,7 @@ async function main() { const vectors = await fileBinding(vectorPath), queryState = await makeState("queries"); const runDirectory = path.join(root, "run"), runPath = path.join(runDirectory, "run.json"); const cancelFile = path.join(root, "cancel"); - const execution = await executeRecorded({ role: "paired_run", executable: values.runner, + const execution = await executeRecorded({ role: "paired_run", authority: "synthetic_canary_only", executable: values.runner, args: ["run", "--prepared", preparationPath, "--prepared-sha256", preparationBinding.sha256, "--fragment-vectors", vectorPath, "--fragment-vectors-sha256", vectors.sha256, "--document-execution", documents.binding.path, "--document-execution-sha256", documents.binding.sha256, @@ -82,7 +82,7 @@ async function main() { await writeFile(receiptPath, JSON.stringify({ contract: "codestory.etr1-synthetic-canary/v2", authority: "synthetic_canary_only", experiment_status: "valid", packet_decision: "not_evaluated", preparation: preparationBinding, documents: documents.binding, execution: execution.binding, - validation, evaluated }), { flag: "wx", mode: 0o600 }); + vectors, analysis: await analysisIdentity(sourceRoot), validation, evaluated }), { flag: "wx", mode: 0o600 }); console.log(JSON.stringify(await fileBinding(receiptPath))); } diff --git a/scripts/codestory-etr1-validate.mjs b/scripts/codestory-etr1-validate.mjs index 21133c3da..b1f086efb 100644 --- a/scripts/codestory-etr1-validate.mjs +++ b/scripts/codestory-etr1-validate.mjs @@ -57,7 +57,7 @@ export function validateEngine(engine) { assert.equal(engine.accelerator_execution_verified, true, "accelerator execution unverified"); assert.equal(engine.worker_alive, true, "embedding worker is not alive"); assert.equal(engine.embedded_model, true, "embedding model was not embedded"); - assert.equal(engine.load_error, null, "embedding engine recorded a load error"); + assert.equal(engine.load_error ?? null, null, "embedding engine recorded a load error"); } export function validatePreAnnotationBoundary(run, preparation) { @@ -246,10 +246,10 @@ export async function validateEtr1({ runBinding, runPath, sourceRoot, executionB assert.equal(sha256(await readFile(run.build.binary_path)), run.build.binary_sha256, "run binary changed"); const preparation = await boundJson(run.preparation); const { request: runnerRequest, receipt: runnerExecution } = await validateExecution(executionBinding, - { role: "paired_run", input: run.preparation, output: runBinding, sourceRoot }); + { role: "paired_run", input: run.preparation, output: runBinding, sourceRoot, authority: run.authority }); assert.equal(runnerRequest.executable.sha256, run.build.binary_sha256, "run producer changed"); const { request: documentRequest } = await validateExecution(run.document_execution, - { role: "documents", input: preparation.embedding_input, output: run.fragment_vectors, sourceRoot }); + { role: "documents", input: preparation.embedding_input, output: run.fragment_vectors, sourceRoot, authority: run.authority }); assert.equal(preparation.contract, "codestory.etr1-preparation/v1"); assert.equal(preparation.authority, run.authority); validatePreAnnotationBoundary(run, preparation); diff --git a/scripts/lib/etr1-execution.mjs b/scripts/lib/etr1-execution.mjs index 60ebb6902..028758f6c 100644 --- a/scripts/lib/etr1-execution.mjs +++ b/scripts/lib/etr1-execution.mjs @@ -7,7 +7,15 @@ import { sha256 } from "./etr1-evidence.mjs"; export const ANALYSIS_FILES = ["scripts/codestory-etr1-validate.mjs", "scripts/codestory-etr1-evaluate.mjs", "scripts/lib/etr1-evidence.mjs", - "scripts/lib/etr1-execution.mjs"]; + "scripts/lib/etr1-execution.mjs", "scripts/codestory-etr1-canary.mjs"]; + +// This is the complete child environment, not a filtered description of a +// larger inherited environment. Unlisted settings cannot affect the child. +export function executionEnvironment(env) { + return Object.fromEntries(Object.entries(env).filter(([key, value]) => typeof value === "string" + && /^(PATH|HOME|USER|LOGNAME|SHELL|TMPDIR|LANG|LC_[A-Z_]+|DEVELOPER_DIR|SDKROOT|OMP_NUM_THREADS|VECLIB_MAXIMUM_THREADS|CODESTORY_[A-Z_]+|XDG_CACHE_HOME|DYLD_LIBRARY_PATH)$/u.test(key) + && !/TOKEN|SECRET|PASSWORD|CREDENTIAL/u.test(key)).sort(([a], [b]) => a.localeCompare(b))); +} export async function fileBinding(file) { const bytes = await readFile(file); @@ -22,6 +30,8 @@ export async function readExecutionBinding(binding) { } export async function analysisIdentity(sourceRoot) { + assert.ok(!process.env.NODE_OPTIONS && !process.env.NODE_PATH, "analysis Node injection is forbidden"); + assert.equal(process.execArgv.length, 0, "analysis Node runtime options are forbidden"); const git = (...args) => execFileSync("git", ["-C", sourceRoot, ...args], { encoding: "utf8", env: { ...process.env, GIT_OPTIONAL_LOCKS: "0" } }).trim(); assert.equal(git("status", "--porcelain", "--untracked-files=no"), "", "analysis checkout is dirty"); @@ -30,7 +40,34 @@ export async function analysisIdentity(sourceRoot) { node: await fileBinding(process.execPath), node_version: process.version }; } -export async function validateExecution(binding, { role, input, output, sourceRoot }) { +export async function validateCanaryGate(binding, { role, executable, sourceRoot }) { + assert.ok(binding, "passing canary receipt required before corpus execution"); + const canary = await readExecutionBinding(binding); + assert.equal(canary.contract, "codestory.etr1-synthetic-canary/v2"); + assert.equal(canary.authority, "synthetic_canary_only"); + assert.equal(canary.experiment_status, "valid", "canary did not pass"); + assert.equal(canary.evaluated?.authority, "synthetic_canary_only"); + assert.equal(canary.evaluated?.experiment_status, "valid"); + assert.deepEqual(canary.analysis, await analysisIdentity(sourceRoot), "canary analysis identity changed"); + const validation = await readExecutionBinding(canary.validation); + assert.equal(validation.experiment_status, "valid"); + assert.equal(validation.authority, "synthetic_canary_only"); + assert.deepEqual(validation.execution, canary.execution); + const preparation = await readExecutionBinding(canary.preparation); + for (const [canaryRole, execution, input, output] of [ + ["documents", canary.documents, preparation.embedding_input, canary.vectors], + ["paired_run", canary.execution, canary.preparation, validation.run], + ]) { + const { request } = await validateExecution(execution, { role: canaryRole, input, output, + sourceRoot, authority: "synthetic_canary_only" }); + if (role === canaryRole) assert.deepEqual(request.executable, await fileBinding(executable), + "canary producer binary changed"); + } + return canary; +} + +export async function validateExecution(binding, { role, input, output, sourceRoot, + authority = "visible_development_frontier_only" }) { assert.ok(binding, "independent execution receipt required"); const receipt = await readExecutionBinding(binding); assert.equal(receipt.contract, "codestory.etr1-execution/v1"); @@ -42,6 +79,17 @@ export async function validateExecution(binding, { role, input, output, sourceRo const request = await readExecutionBinding(receipt.request); assert.equal(request.contract, "codestory.etr1-execution-request/v1"); assert.equal(request.role, role); + assert.equal(request.authority, authority, "execution authority changed"); + assert.equal(request.cwd, await realpath(sourceRoot), "execution cwd changed"); + assert.deepEqual(request.environment, executionEnvironment(request.environment), "execution environment is not canonical"); + assert.equal(request.context_sha256, sha256(JSON.stringify({ cwd: request.cwd, + environment: request.environment })), "execution context changed"); + if (authority === "visible_development_frontier_only") + await validateCanaryGate(request.canary, { role, executable: request.executable.path, sourceRoot }); + else { + assert.equal(authority, "synthetic_canary_only"); + assert.equal(request.canary, null); + } if (role === "paired_run") assert.equal(request.deadline_ms, 1_800_000, "paired deadline changed"); assert.deepEqual(request.analysis, await analysisIdentity(sourceRoot), "analysis identity changed"); assert.deepEqual(await fileBinding(request.executable.path), request.executable, "producer binary changed"); @@ -62,8 +110,15 @@ export async function validateExecution(binding, { role, input, output, sourceRo // The supervisor freezes inputs before launch and captures outputs before any // evaluator can run. It does not claim to defend against a hostile host owner. export async function executeRecorded({ role, executable, args, inputs, outputPaths, eventsPath, - directory, sourceRoot, env, cancelFile, deadlineMs = 30 * 60 * 1000 }) { + directory, sourceRoot, env, cancelFile, deadlineMs = 30 * 60 * 1000, + authority = "visible_development_frontier_only", canary = null }) { assert.ok(["documents", "paired_run"].includes(role), "unsupported ETR execution role"); + if (authority === "visible_development_frontier_only") + await validateCanaryGate(canary, { role, executable, sourceRoot }); + else { + assert.equal(authority, "synthetic_canary_only"); + assert.equal(canary, null); + } if (role === "paired_run") assert.equal(deadlineMs, 1_800_000, "paired deadline changed"); for (const file of [...outputPaths, eventsPath]) { const exists = await lstat(file).then(() => true, (error) => { @@ -73,10 +128,9 @@ export async function executeRecorded({ role, executable, args, inputs, outputPa assert.equal(exists, false, `execution output already exists: ${file}`); } await mkdir(directory, { mode: 0o700 }); - const recordedEnv = Object.fromEntries(Object.entries(env).filter(([key]) => - /^(CODESTORY_(CACHE_ROOT|EMBED_ALLOW_CPU|EMBED_QUALIFICATION_[A-Z_]+|EMBEDDING_[A-Z_]+)|XDG_CACHE_HOME|TMPDIR|DYLD_LIBRARY_PATH)$/u.test(key) - && !/TOKEN|SECRET|PASSWORD|CREDENTIAL/u.test(key))); - const request = { contract: "codestory.etr1-execution-request/v1", role, + const recordedEnv = executionEnvironment(env), cwd = await realpath(sourceRoot); + const request = { contract: "codestory.etr1-execution-request/v1", role, authority, canary, cwd, + context_sha256: sha256(JSON.stringify({ cwd, environment: recordedEnv })), executable: await fileBinding(executable), args, environment: recordedEnv, inputs: await Promise.all(inputs.map(fileBinding)), output_paths: outputPaths, events_path: eventsPath, cancel_file: cancelFile ?? null, deadline_ms: deadlineMs, @@ -85,7 +139,7 @@ export async function executeRecorded({ role, executable, args, inputs, outputPa await writeFile(requestPath, JSON.stringify(request), { flag: "wx", mode: 0o600 }); const requestBinding = await fileBinding(requestPath); const started = process.hrtime.bigint(); - const child = spawn(executable, args, { env, stdio: ["ignore", "pipe", "pipe"] }); + const child = spawn(executable, args, { cwd, env: recordedEnv, stdio: ["ignore", "pipe", "pipe"] }); const stdout = [], stderr = []; child.stdout.on("data", (value) => stdout.push(value)); child.stderr.on("data", (value) => stderr.push(value)); diff --git a/scripts/tests/etr1-evidence.test.mjs b/scripts/tests/etr1-evidence.test.mjs index 9443a6231..063ae8674 100644 --- a/scripts/tests/etr1-evidence.test.mjs +++ b/scripts/tests/etr1-evidence.test.mjs @@ -8,7 +8,8 @@ import { authenticateFragment, encodedCandidateInput, evaluateArm, exactPublicBy import { validateArm, validateDocumentVectorRecord, validateEngine, validatePreAnnotationBoundary, parseEvents } from "../codestory-etr1-validate.mjs"; import { decision, evaluateEtr1, gateOne, gateTwo } from "../codestory-etr1-evaluate.mjs"; -import { fileBinding, readExecutionBinding, validateExecution } from "../lib/etr1-execution.mjs"; +import { fileBinding, readExecutionBinding, validateExecution, executionEnvironment, + validateCanaryGate } from "../lib/etr1-execution.mjs"; function unit(index) { const vector = Array(LIMITS.vectorDimension).fill(0); @@ -133,6 +134,9 @@ test("vector, model, and pre-annotation boundaries refuse substitutions", () => policy: "accelerated", accelerator_execution_verified: true, worker_alive: true, embedded_model: true, load_error: null }; validateEngine(engine); + const omittedError = { ...engine }; delete omittedError.load_error; + validateEngine(omittedError); + assert.throws(() => validateEngine({ ...engine, load_error: "failed" })); assert.throws(() => validateEngine({ ...engine, model_digest: "f".repeat(64) })); const run = { annotation_access: "not_accessed", graph_invocations: 0, bge_invocations: 0, symbol_document_invocations: 0, host_query_invocations: 0, production_packet_invocations: 0 }; @@ -144,6 +148,26 @@ test("vector, model, and pre-annotation boundaries refuse substitutions", () => } }); +test("execution uses only its recorded secret-free process environment", () => { + const env = executionEnvironment({ PATH: "/bin", HOME: "/users/test", OMP_NUM_THREADS: "4", + CODESTORY_CACHE_ROOT: "/cache", API_TOKEN: "private", NODE_OPTIONS: "--require bad.cjs", + UNRELATED: "hidden" }); + assert.deepEqual(env, { CODESTORY_CACHE_ROOT: "/cache", HOME: "/users/test", + OMP_NUM_THREADS: "4", PATH: "/bin" }); +}); + +test("corpus launch requires a completed identity-matched canary before model access", async () => { + await assert.rejects(() => validateCanaryGate(undefined, {}), /passing canary receipt required/u); + const directory = await mkdtemp(path.join(os.tmpdir(), "etr1-canary-gate-")); + const file = path.join(directory, "receipt.json"); + await writeFile(file, JSON.stringify({ contract: "codestory.etr1-synthetic-canary/v2", + authority: "synthetic_canary_only", experiment_status: "invalid" })); + const binding = await fileBinding(file); + await assert.rejects(() => validateCanaryGate(binding, {}), /canary did not pass/u); + await writeFile(file, "{}"); + await assert.rejects(() => validateCanaryGate(binding, {}), /execution artifact/u); +}); + test("exact optimizer selects successors without compulsory discovery seeds", () => { const rowBytes = new Map([["seed", 15_000], ["successor", 100]]); const result = maximizeCoveredAtoms([["successor"]], rowBytes, 100); From f54b8e6c0f503f2f4a61eed35b303f9d72004e0d Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Fri, 4 Sep 2026 17:42:46 -0400 Subject: [PATCH 6/8] reconcile document completions and native process locale --- scripts/codestory-etr1-canary.mjs | 1 + scripts/codestory-etr1-validate.mjs | 26 ++++++++++++++++++++++++-- scripts/lib/etr1-execution.mjs | 6 ++++-- scripts/tests/etr1-evidence.test.mjs | 21 +++++++++++++++++++-- 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/scripts/codestory-etr1-canary.mjs b/scripts/codestory-etr1-canary.mjs index 0be3c9ec6..42df7ce9c 100644 --- a/scripts/codestory-etr1-canary.mjs +++ b/scripts/codestory-etr1-canary.mjs @@ -66,6 +66,7 @@ async function main() { await writeFile(validationPath, JSON.stringify({ contract: "codestory.etr1-validation/v1", authority: "synthetic_canary_only", experiment_status: "valid", decision: "not_evaluated", annotation_access: "not_accessed", run: runBinding, execution: execution.binding, + document_completion: validated.document_completion, binary_sha256: validated.run.build.binary_sha256 }), { flag: "wx", mode: 0o600 }); // Synthetic truth enters only after the real paired run and validator finish. const first = preparation.fragments[0]; diff --git a/scripts/codestory-etr1-validate.mjs b/scripts/codestory-etr1-validate.mjs index b1f086efb..0615314fe 100644 --- a/scripts/codestory-etr1-validate.mjs +++ b/scripts/codestory-etr1-validate.mjs @@ -212,6 +212,24 @@ export function parseEvents(bytes) { }); } +// The pinned diagnostic encodes consecutive document batches of sixteen. +// Reconcile its independently captured progress with every native completion; +// a digest alone cannot establish that a log contains the required events. +export function validateDocumentCompletions({ eventsBytes, stderrBytes, recordCount }) { + assert.ok(Number.isSafeInteger(recordCount) && recordCount > 0); + const events = parseEvents(eventsBytes); + const progress = [...stderrBytes.toString("utf8").matchAll(/^encoded (\d+)\/(\d+) records; batch_ms=(\d+)$/gmu)]; + const expectedBatches = Math.ceil(recordCount / 16); + assert.equal(events.length, expectedBatches, "document native completion count changed"); + assert.equal(progress.length, expectedBatches, "document batch progress count changed"); + progress.forEach((entry, index) => { + assert.equal(Number(entry[1]), Math.min((index + 1) * 16, recordCount), "document batch partition changed"); + assert.equal(Number(entry[2]), recordCount, "document record total changed"); + }); + return { batches: events.length, records: recordCount, + completed_tokens: events.reduce((sum, event) => sum + Number(event.details.completed_tokens), 0) }; +} + async function loadSourceFiles(repository, repositoryFragments) { const root = await realpath(repository.local_root), result = new Map(); for (const relative of new Set(repositoryFragments.map((fragment) => fragment.path))) { @@ -248,7 +266,7 @@ export async function validateEtr1({ runBinding, runPath, sourceRoot, executionB const { request: runnerRequest, receipt: runnerExecution } = await validateExecution(executionBinding, { role: "paired_run", input: run.preparation, output: runBinding, sourceRoot, authority: run.authority }); assert.equal(runnerRequest.executable.sha256, run.build.binary_sha256, "run producer changed"); - const { request: documentRequest } = await validateExecution(run.document_execution, + const { request: documentRequest, receipt: documentExecution } = await validateExecution(run.document_execution, { role: "documents", input: preparation.embedding_input, output: run.fragment_vectors, sourceRoot, authority: run.authority }); assert.equal(preparation.contract, "codestory.etr1-preparation/v1"); assert.equal(preparation.authority, run.authority); @@ -295,6 +313,9 @@ export async function validateEtr1({ runBinding, runPath, sourceRoot, executionB validateEngine(vectorArtifact.final_engine); assert.equal(vectorArtifact.initial_engine.server_instance_id, vectorArtifact.final_engine.server_instance_id); assert.equal(vectorArtifact.records.length, preparation.fragments.length, "document vector count changed"); + const document_completion = validateDocumentCompletions({ + eventsBytes: await boundBytes(documentExecution.events), + stderrBytes: await boundBytes(documentExecution.stderr), recordCount: preparation.fragments.length }); const documentVectors = new Map(); vectorArtifact.records.forEach((record, index) => { const fragment = preparation.fragments[index]; @@ -348,7 +369,7 @@ export async function validateEtr1({ runBinding, runPath, sourceRoot, executionB [...batches.values()].reduce((sum, batch) => sum + batch.completed_tokens, 0), "qualification token total changed"); return { run, preparation, rows, source_commit: sourceCommit, source_tree: sourceTree, - batch_count: batches.size, query_count: rows.reduce((sum, row) => + document_completion, batch_count: batches.size, query_count: rows.reduce((sum, row) => sum + row.control.search_count + row.candidate.search_count, 0) }; } @@ -377,6 +398,7 @@ async function main() { binary_sha256: validated.run.build.binary_sha256, preparation_sha256: validated.run.preparation.sha256, fragment_vectors_sha256: validated.run.fragment_vectors.sha256, + document_completion: validated.document_completion, row_count: validated.rows.length, batch_count: validated.batch_count, query_count: validated.query_count, source_address_validity: 1 }; } catch (error) { diff --git a/scripts/lib/etr1-execution.mjs b/scripts/lib/etr1-execution.mjs index 028758f6c..f7c13b5ea 100644 --- a/scripts/lib/etr1-execution.mjs +++ b/scripts/lib/etr1-execution.mjs @@ -11,9 +11,11 @@ export const ANALYSIS_FILES = ["scripts/codestory-etr1-validate.mjs", // This is the complete child environment, not a filtered description of a // larger inherited environment. Unlisted settings cannot affect the child. +const platformEnvironment = process.platform === "darwin" + ? { __CF_USER_TEXT_ENCODING: process.env.__CF_USER_TEXT_ENCODING } : {}; export function executionEnvironment(env) { - return Object.fromEntries(Object.entries(env).filter(([key, value]) => typeof value === "string" - && /^(PATH|HOME|USER|LOGNAME|SHELL|TMPDIR|LANG|LC_[A-Z_]+|DEVELOPER_DIR|SDKROOT|OMP_NUM_THREADS|VECLIB_MAXIMUM_THREADS|CODESTORY_[A-Z_]+|XDG_CACHE_HOME|DYLD_LIBRARY_PATH)$/u.test(key) + return Object.fromEntries(Object.entries({ ...platformEnvironment, ...env }).filter(([key, value]) => typeof value === "string" + && /^(PATH|HOME|USER|LOGNAME|SHELL|TMPDIR|LANG|LC_[A-Z_]+|DEVELOPER_DIR|SDKROOT|OMP_NUM_THREADS|VECLIB_MAXIMUM_THREADS|CODESTORY_[A-Z_]+|XDG_CACHE_HOME|DYLD_LIBRARY_PATH|__CF_USER_TEXT_ENCODING)$/u.test(key) && !/TOKEN|SECRET|PASSWORD|CREDENTIAL/u.test(key)).sort(([a], [b]) => a.localeCompare(b))); } diff --git a/scripts/tests/etr1-evidence.test.mjs b/scripts/tests/etr1-evidence.test.mjs index 063ae8674..8dba9d9eb 100644 --- a/scripts/tests/etr1-evidence.test.mjs +++ b/scripts/tests/etr1-evidence.test.mjs @@ -6,7 +6,7 @@ import path from "node:path"; import { authenticateFragment, encodedCandidateInput, evaluateArm, exactPublicBytes, fragmentId, LIMITS, maximizeCoveredAtoms, scoreOrder, selectSuccessors, sha256 } from "../lib/etr1-evidence.mjs"; import { validateArm, validateDocumentVectorRecord, validateEngine, - validatePreAnnotationBoundary, parseEvents } from "../codestory-etr1-validate.mjs"; + validatePreAnnotationBoundary, parseEvents, validateDocumentCompletions } from "../codestory-etr1-validate.mjs"; import { decision, evaluateEtr1, gateOne, gateTwo } from "../codestory-etr1-evaluate.mjs"; import { fileBinding, readExecutionBinding, validateExecution, executionEnvironment, validateCanaryGate } from "../lib/etr1-execution.mjs"; @@ -152,10 +152,27 @@ test("execution uses only its recorded secret-free process environment", () => { const env = executionEnvironment({ PATH: "/bin", HOME: "/users/test", OMP_NUM_THREADS: "4", CODESTORY_CACHE_ROOT: "/cache", API_TOKEN: "private", NODE_OPTIONS: "--require bad.cjs", UNRELATED: "hidden" }); - assert.deepEqual(env, { CODESTORY_CACHE_ROOT: "/cache", HOME: "/users/test", + assert.deepEqual(env, { ...executionEnvironment({}), CODESTORY_CACHE_ROOT: "/cache", HOME: "/users/test", OMP_NUM_THREADS: "4", PATH: "/bin" }); }); +test("document completions reconcile every fixed diagnostic batch before annotations", () => { + const event = (i) => JSON.stringify({ schema_version: 1, sequence: 0, + action: "completed_tokens", status: "completed", server_event_sequence: i + 10, clock: {}, + details: { native_completion_sequence: String(i), completed_tokens: "25", request_id: `doc-${i}` } }); + const eventsBytes = Buffer.from(`${event(1)}\n${event(2)}\n`); + const stderrBytes = Buffer.from("encoded 16/17 records; batch_ms=1\nencoded 17/17 records; batch_ms=2\n"); + const input = { eventsBytes, stderrBytes, recordCount: 17 }; + assert.deepEqual(validateDocumentCompletions(input), { batches: 2, records: 17, completed_tokens: 50 }); + for (const changed of [ + { eventsBytes: Buffer.from(`${event(1)}\n`) }, + { eventsBytes: Buffer.alloc(0) }, + { stderrBytes: Buffer.from("encoded 16/17 records; batch_ms=1\n") }, + { stderrBytes: Buffer.from("encoded 15/17 records; batch_ms=1\nencoded 17/17 records; batch_ms=2\n") }, + { recordCount: 18 }, + ]) assert.throws(() => validateDocumentCompletions({ ...input, ...changed })); +}); + test("corpus launch requires a completed identity-matched canary before model access", async () => { await assert.rejects(() => validateCanaryGate(undefined, {}), /passing canary receipt required/u); const directory = await mkdtemp(path.join(os.tmpdir(), "etr1-canary-gate-")); From d1405fa8f149f251ef0f60840e346f1c60c3e86c Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Fri, 4 Sep 2026 17:56:17 -0400 Subject: [PATCH 7/8] preserve complete source lines during evidence validation --- scripts/codestory-etr1-canary.mjs | 2 +- scripts/codestory-etr1-validate.mjs | 4 ++-- scripts/tests/etr1-evidence.test.mjs | 11 ++++++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/codestory-etr1-canary.mjs b/scripts/codestory-etr1-canary.mjs index 42df7ce9c..965f20858 100644 --- a/scripts/codestory-etr1-canary.mjs +++ b/scripts/codestory-etr1-canary.mjs @@ -20,7 +20,7 @@ async function main() { const project = path.join(root, "repository"), prepared = path.join(root, "prepared"); await mkdir(project, { mode: 0o700 }); const source = Array.from({ length: 32 }, (_, i) => - `fn commonneedle_${i}() { commonneedle(); ${i === 0 ? "raremarker();" : ""} }\n`).join(""); + `fn commonneedle_${i}() { commonneedle(); ${i === 0 ? "raremarker();" : ""} }${i === 1 ? " /* a\u2028b\u2029c */" : ""}${i % 3 === 0 ? "\r\n" : "\n"}`).join(""); await writeFile(path.join(project, "canary.rs"), source, { flag: "wx", mode: 0o600 }); const git = (...args) => execFileSync("git", ["-C", project, "-c", "core.hooksPath=/dev/null", ...args], { encoding: "utf8", stdio: "pipe" }); diff --git a/scripts/codestory-etr1-validate.mjs b/scripts/codestory-etr1-validate.mjs index 0615314fe..69947f47d 100644 --- a/scripts/codestory-etr1-validate.mjs +++ b/scripts/codestory-etr1-validate.mjs @@ -34,9 +34,9 @@ async function boundJson(binding, expectedPath) { return JSON.parse(await boundBytes(binding, expectedPath)); } -function renderSnippet(fragment) { +export function renderSnippet(fragment) { let result = "```text\n"; - const lines = fragment.source.match(/.*(?:\n|$)/gu).filter(Boolean); + const lines = fragment.source.match(/[^\n]*\n|[^\n]+$/gu) ?? []; lines.forEach((line, offset) => { result += ` ${String(fragment.line_range.start + offset).padStart(5)} | ${line.replace(/[\r\n]+$/u, "")}\n`; }); diff --git a/scripts/tests/etr1-evidence.test.mjs b/scripts/tests/etr1-evidence.test.mjs index 8dba9d9eb..7a2c773ae 100644 --- a/scripts/tests/etr1-evidence.test.mjs +++ b/scripts/tests/etr1-evidence.test.mjs @@ -6,7 +6,8 @@ import path from "node:path"; import { authenticateFragment, encodedCandidateInput, evaluateArm, exactPublicBytes, fragmentId, LIMITS, maximizeCoveredAtoms, scoreOrder, selectSuccessors, sha256 } from "../lib/etr1-evidence.mjs"; import { validateArm, validateDocumentVectorRecord, validateEngine, - validatePreAnnotationBoundary, parseEvents, validateDocumentCompletions } from "../codestory-etr1-validate.mjs"; + validatePreAnnotationBoundary, parseEvents, validateDocumentCompletions, + renderSnippet } from "../codestory-etr1-validate.mjs"; import { decision, evaluateEtr1, gateOne, gateTwo } from "../codestory-etr1-evaluate.mjs"; import { fileBinding, readExecutionBinding, validateExecution, executionEnvironment, validateCanaryGate } from "../lib/etr1-execution.mjs"; @@ -17,6 +18,14 @@ function unit(index) { return vector; } +test("source rendering splits only at LF and preserves all source inside a line", () => { + assert.equal(renderSnippet({ source: "first\r\nsecond\nthird\u2028inside\u2029line\rlast", + line_range: { start: 9 } }), + "```text\n 9 | first\n 10 | second\n 11 | third\u2028inside\u2029line\rlast\n```"); + assert.equal(renderSnippet({ source: "\n\r\n", line_range: { start: 1 } }), + "```text\n 1 | \n 2 | \n```"); +}); + function fixture(expectedName = "control") { const bytes = Buffer.from("seed\nsuccessor\n"), digest = sha256(bytes), project_id = "project"; const make = (path, start, end, line, source) => { From eb968e4f68e8999ce575ba25bee9d1c13b56cfdf Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Fri, 4 Sep 2026 17:59:17 -0400 Subject: [PATCH 8/8] share complete-line reconstruction across etr validation --- scripts/codestory-etr1-validate.mjs | 4 ++-- scripts/lib/etr1-evidence.mjs | 6 +++++- scripts/tests/etr1-evidence.test.mjs | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/codestory-etr1-validate.mjs b/scripts/codestory-etr1-validate.mjs index 69947f47d..e88e61223 100644 --- a/scripts/codestory-etr1-validate.mjs +++ b/scripts/codestory-etr1-validate.mjs @@ -6,7 +6,7 @@ import { parseArgs } from "node:util"; import { fileURLToPath } from "node:url"; import { validateExecution } from "./lib/etr1-execution.mjs"; import { authenticateFragment, encodedCandidateInput, f32Dot, fragmentId, LIMITS, - scoreOrder, selectSuccessors, sha256, validateVector } from "./lib/etr1-evidence.mjs"; + scoreOrder, selectSuccessors, sha256, validateVector, sourceLines } from "./lib/etr1-evidence.mjs"; const FIXED = Object.freeze({ parent: "c9c935d87129a79f326b650bbf23d73191df8b4f", fragment_diagnostic: "ca185ed13c635bbb4b64cc6760c5025799700359ebcc4dd3bcc53e34f8cf9194", @@ -36,7 +36,7 @@ async function boundJson(binding, expectedPath) { export function renderSnippet(fragment) { let result = "```text\n"; - const lines = fragment.source.match(/[^\n]*\n|[^\n]+$/gu) ?? []; + const lines = sourceLines(fragment.source); lines.forEach((line, offset) => { result += ` ${String(fragment.line_range.start + offset).padStart(5)} | ${line.replace(/[\r\n]+$/u, "")}\n`; }); diff --git a/scripts/lib/etr1-evidence.mjs b/scripts/lib/etr1-evidence.mjs index af678e653..f5473863d 100644 --- a/scripts/lib/etr1-evidence.mjs +++ b/scripts/lib/etr1-evidence.mjs @@ -6,6 +6,10 @@ export const LIMITS = Object.freeze({ rows: 16, bytes: 16 * 1024, seeds: 16, export const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex"); +// Match Rust split_inclusive('\n'): CR and Unicode separators remain source, +// never additional boundaries or reasons to discard text. +export const sourceLines = (source) => source.match(/[^\n]*\n|[^\n]+$/gu) ?? []; + export function fragmentId(fragment) { const framed = []; for (const value of [fragment.project_id, fragment.path, fragment.content_digest]) { @@ -56,7 +60,7 @@ export function selectSuccessors(order, seeds, prior, limit = LIMITS.successorsP export function encodedCandidateInput(question, source, removedTrailingLines) { assert.ok(Number.isSafeInteger(removedTrailingLines) && removedTrailingLines >= 0, "invalid removed-line count"); - const lines = source.match(/.*(?:\n|$)/gu).filter(Boolean); + const lines = sourceLines(source); assert.ok(lines.length > removedTrailingLines, "all seed lines were removed"); const retained = lines.slice(0, lines.length - removedTrailingLines).join(""); assert.ok(retained.trim(), "retained seed source is empty"); diff --git a/scripts/tests/etr1-evidence.test.mjs b/scripts/tests/etr1-evidence.test.mjs index 7a2c773ae..76bb1c448 100644 --- a/scripts/tests/etr1-evidence.test.mjs +++ b/scripts/tests/etr1-evidence.test.mjs @@ -94,6 +94,10 @@ test("candidate input shortening preserves UTF-8 and complete trailing lines", ( const source = "first α\nsecond β\nthird γ\n"; assert.equal(encodedCandidateInput("question", source, 1), "question\n\nfirst α\nsecond β\n"); assert.throws(() => encodedCandidateInput("question", source, 3)); + const mixed = "first α\r\nsecond\u2028β\u2029line\nlast\rinside"; + assert.equal(encodedCandidateInput("question", mixed, 0), `question\n\n${mixed}`); + assert.equal(encodedCandidateInput("question", mixed, 1), "question\n\nfirst α\r\nsecond\u2028β\u2029line\n"); + assert.equal(encodedCandidateInput("question", mixed, 2), "question\n\nfirst α\r\n"); }); test("validator uses native completion identity for automatic token events", () => {