From de491cd67eeb22e9decce44118f505118745c081 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Tue, 30 Jun 2026 12:07:15 -0400 Subject: [PATCH 1/6] {"schema":"decodex/commit/1","summary":"Split trace persistence modules","authority":"manual"} --- .../src/worker/trace_jobs/persistence.rs | 299 +----------------- .../trace_jobs/persistence/candidates.rs | 83 +++++ .../worker/trace_jobs/persistence/items.rs | 54 ++++ .../worker/trace_jobs/persistence/stages.rs | 90 ++++++ .../worker/trace_jobs/persistence/trace.rs | 69 ++++ 5 files changed, 304 insertions(+), 291 deletions(-) create mode 100644 apps/elf-worker/src/worker/trace_jobs/persistence/candidates.rs create mode 100644 apps/elf-worker/src/worker/trace_jobs/persistence/items.rs create mode 100644 apps/elf-worker/src/worker/trace_jobs/persistence/stages.rs create mode 100644 apps/elf-worker/src/worker/trace_jobs/persistence/trace.rs diff --git a/apps/elf-worker/src/worker/trace_jobs/persistence.rs b/apps/elf-worker/src/worker/trace_jobs/persistence.rs index 25891aef..c53fcc38 100644 --- a/apps/elf-worker/src/worker/trace_jobs/persistence.rs +++ b/apps/elf-worker/src/worker/trace_jobs/persistence.rs @@ -1,292 +1,9 @@ -use crate::worker::{ - PgConnection, PgExecutor, QueryBuilder, Result, TraceCandidateInsert, TraceCandidateRecord, - TraceItemInsert, TraceItemRecord, TraceRecord, TraceStageInsert, TraceStageItemInsert, - TraceTrajectoryStageRecord, Uuid, Value, +mod candidates; +mod items; +mod stages; +mod trace; + +pub(super) use self::{ + candidates::insert_trace_candidates_tx, items::insert_trace_items_tx, + stages::insert_trace_stages_tx, trace::insert_trace_tx, }; - -pub(super) async fn insert_trace_stages_tx( - executor: &mut PgConnection, - trace_id: Uuid, - stages: Vec, -) -> Result<()> { - if stages.is_empty() { - return Ok(()); - } - - let mut stage_inserts = Vec::with_capacity(stages.len()); - let mut item_inserts = Vec::new(); - - for stage in stages { - stage_inserts.push(TraceStageInsert { - stage_id: stage.stage_id, - stage_order: stage.stage_order as i32, - stage_name: stage.stage_name, - stage_payload: stage.stage_payload, - created_at: stage.created_at, - }); - - for item in stage.items { - item_inserts.push(TraceStageItemInsert { - id: item.id, - stage_id: stage.stage_id, - item_id: item.item_id, - note_id: item.note_id, - chunk_id: item.chunk_id, - metrics: item.metrics, - }); - } - } - - let mut stage_builder = QueryBuilder::new( - "\ - INSERT INTO search_trace_stages ( - stage_id, - trace_id, - stage_order, - stage_name, - stage_payload, - created_at - ) ", - ); - - stage_builder.push_values(stage_inserts, |mut b, stage| { - b.push_bind(stage.stage_id) - .push_bind(trace_id) - .push_bind(stage.stage_order) - .push_bind(stage.stage_name) - .push_bind(stage.stage_payload) - .push_bind(stage.created_at); - }); - stage_builder.push(" ON CONFLICT (stage_id) DO NOTHING"); - stage_builder.build().execute(&mut *executor).await?; - - if item_inserts.is_empty() { - return Ok(()); - } - - let mut item_builder = QueryBuilder::new( - "\ - INSERT INTO search_trace_stage_items ( - id, - stage_id, - item_id, - note_id, - chunk_id, - metrics - ) ", - ); - - item_builder.push_values(item_inserts, |mut b, item| { - b.push_bind(item.id) - .push_bind(item.stage_id) - .push_bind(item.item_id) - .push_bind(item.note_id) - .push_bind(item.chunk_id) - .push_bind(item.metrics); - }); - item_builder.push(" ON CONFLICT (id) DO NOTHING"); - item_builder.build().execute(executor).await?; - - Ok(()) -} - -pub(super) async fn insert_trace_tx<'e, E>( - executor: E, - trace_id: Uuid, - trace: &TraceRecord, - expanded_queries_json: Value, - allowed_scopes_json: Value, -) -> Result<()> -where - E: PgExecutor<'e>, -{ - sqlx::query( - "INSERT INTO search_traces ( - trace_id, - tenant_id, - project_id, - agent_id, - read_profile, - query, - expansion_mode, - expanded_queries, - allowed_scopes, - candidate_count, - top_k, - config_snapshot, - trace_version, - created_at, - expires_at -) -VALUES ( - $1, - $2, - $3, - $4, - $5, - $6, - $7, - $8, - $9, - $10, - $11, - $12, - $13, - $14, - $15 -) -ON CONFLICT (trace_id) DO NOTHING", - ) - .bind(trace_id) - .bind(trace.tenant_id.as_str()) - .bind(trace.project_id.as_str()) - .bind(trace.agent_id.as_str()) - .bind(trace.read_profile.as_str()) - .bind(trace.query.as_str()) - .bind(trace.expansion_mode.as_str()) - .bind(expanded_queries_json) - .bind(allowed_scopes_json) - .bind(trace.candidate_count as i32) - .bind(trace.top_k as i32) - .bind(trace.config_snapshot.clone()) - .bind(trace.trace_version) - .bind(trace.created_at) - .bind(trace.expires_at) - .execute(executor) - .await?; - - Ok(()) -} - -pub(super) async fn insert_trace_items_tx<'e, E>( - executor: E, - trace_id: Uuid, - items: Vec, -) -> Result<()> -where - E: PgExecutor<'e>, -{ - if items.is_empty() { - return Ok(()); - } - - let mut inserts = Vec::with_capacity(items.len()); - - for item in items { - inserts.push(TraceItemInsert { - item_id: item.item_id, - note_id: item.note_id, - chunk_id: item.chunk_id, - rank: item.rank as i32, - final_score: item.final_score, - explain: item.explain, - }); - } - - let mut builder = QueryBuilder::new( - "\ -INSERT INTO search_trace_items ( - item_id, - trace_id, - note_id, - chunk_id, - rank, - final_score, - explain -) ", - ); - - builder.push_values(inserts, |mut b, item| { - b.push_bind(item.item_id) - .push_bind(trace_id) - .push_bind(item.note_id) - .push_bind(item.chunk_id) - .push_bind(item.rank) - .push_bind(item.final_score) - .push_bind(item.explain); - }); - builder.push(" ON CONFLICT (item_id) DO NOTHING"); - builder.build().execute(executor).await?; - - Ok(()) -} - -pub(super) async fn insert_trace_candidates_tx<'e, E>( - executor: E, - trace_id: Uuid, - candidates: Vec, -) -> Result<()> -where - E: PgExecutor<'e>, -{ - if candidates.is_empty() { - return Ok(()); - } - - let mut inserts = Vec::with_capacity(candidates.len()); - - for candidate in candidates { - inserts.push(TraceCandidateInsert { - candidate_id: candidate.candidate_id, - note_id: candidate.note_id, - chunk_id: candidate.chunk_id, - chunk_index: candidate.chunk_index, - snippet: candidate.snippet, - candidate_snapshot: candidate.candidate_snapshot, - retrieval_rank: candidate.retrieval_rank as i32, - rerank_score: candidate.rerank_score, - note_scope: candidate.note_scope, - note_importance: candidate.note_importance, - note_updated_at: candidate.note_updated_at, - note_hit_count: candidate.note_hit_count, - note_last_hit_at: candidate.note_last_hit_at, - created_at: candidate.created_at, - expires_at: candidate.expires_at, - }); - } - - let mut builder = QueryBuilder::new( - "\ -INSERT INTO search_trace_candidates ( - candidate_id, - trace_id, - note_id, - chunk_id, - chunk_index, - snippet, - candidate_snapshot, - retrieval_rank, - rerank_score, - note_scope, - note_importance, - note_updated_at, - note_hit_count, - note_last_hit_at, - created_at, - expires_at -) ", - ); - - builder.push_values(inserts, |mut b, candidate| { - b.push_bind(candidate.candidate_id) - .push_bind(trace_id) - .push_bind(candidate.note_id) - .push_bind(candidate.chunk_id) - .push_bind(candidate.chunk_index) - .push_bind(candidate.snippet) - .push_bind(candidate.candidate_snapshot) - .push_bind(candidate.retrieval_rank) - .push_bind(candidate.rerank_score) - .push_bind(candidate.note_scope) - .push_bind(candidate.note_importance) - .push_bind(candidate.note_updated_at) - .push_bind(candidate.note_hit_count) - .push_bind(candidate.note_last_hit_at) - .push_bind(candidate.created_at) - .push_bind(candidate.expires_at); - }); - builder.push(" ON CONFLICT (candidate_id) DO NOTHING"); - builder.build().execute(executor).await?; - - Ok(()) -} diff --git a/apps/elf-worker/src/worker/trace_jobs/persistence/candidates.rs b/apps/elf-worker/src/worker/trace_jobs/persistence/candidates.rs new file mode 100644 index 00000000..82810914 --- /dev/null +++ b/apps/elf-worker/src/worker/trace_jobs/persistence/candidates.rs @@ -0,0 +1,83 @@ +use crate::worker::{ + PgExecutor, QueryBuilder, Result, TraceCandidateInsert, TraceCandidateRecord, Uuid, +}; + +pub(in crate::worker::trace_jobs) async fn insert_trace_candidates_tx<'e, E>( + executor: E, + trace_id: Uuid, + candidates: Vec, +) -> Result<()> +where + E: PgExecutor<'e>, +{ + if candidates.is_empty() { + return Ok(()); + } + + let mut inserts = Vec::with_capacity(candidates.len()); + + for candidate in candidates { + inserts.push(TraceCandidateInsert { + candidate_id: candidate.candidate_id, + note_id: candidate.note_id, + chunk_id: candidate.chunk_id, + chunk_index: candidate.chunk_index, + snippet: candidate.snippet, + candidate_snapshot: candidate.candidate_snapshot, + retrieval_rank: candidate.retrieval_rank as i32, + rerank_score: candidate.rerank_score, + note_scope: candidate.note_scope, + note_importance: candidate.note_importance, + note_updated_at: candidate.note_updated_at, + note_hit_count: candidate.note_hit_count, + note_last_hit_at: candidate.note_last_hit_at, + created_at: candidate.created_at, + expires_at: candidate.expires_at, + }); + } + + let mut builder = QueryBuilder::new( + "\ +INSERT INTO search_trace_candidates ( + candidate_id, + trace_id, + note_id, + chunk_id, + chunk_index, + snippet, + candidate_snapshot, + retrieval_rank, + rerank_score, + note_scope, + note_importance, + note_updated_at, + note_hit_count, + note_last_hit_at, + created_at, + expires_at +) ", + ); + + builder.push_values(inserts, |mut b, candidate| { + b.push_bind(candidate.candidate_id) + .push_bind(trace_id) + .push_bind(candidate.note_id) + .push_bind(candidate.chunk_id) + .push_bind(candidate.chunk_index) + .push_bind(candidate.snippet) + .push_bind(candidate.candidate_snapshot) + .push_bind(candidate.retrieval_rank) + .push_bind(candidate.rerank_score) + .push_bind(candidate.note_scope) + .push_bind(candidate.note_importance) + .push_bind(candidate.note_updated_at) + .push_bind(candidate.note_hit_count) + .push_bind(candidate.note_last_hit_at) + .push_bind(candidate.created_at) + .push_bind(candidate.expires_at); + }); + builder.push(" ON CONFLICT (candidate_id) DO NOTHING"); + builder.build().execute(executor).await?; + + Ok(()) +} diff --git a/apps/elf-worker/src/worker/trace_jobs/persistence/items.rs b/apps/elf-worker/src/worker/trace_jobs/persistence/items.rs new file mode 100644 index 00000000..50434f5b --- /dev/null +++ b/apps/elf-worker/src/worker/trace_jobs/persistence/items.rs @@ -0,0 +1,54 @@ +use crate::worker::{PgExecutor, QueryBuilder, Result, TraceItemInsert, TraceItemRecord, Uuid}; + +pub(in crate::worker::trace_jobs) async fn insert_trace_items_tx<'e, E>( + executor: E, + trace_id: Uuid, + items: Vec, +) -> Result<()> +where + E: PgExecutor<'e>, +{ + if items.is_empty() { + return Ok(()); + } + + let mut inserts = Vec::with_capacity(items.len()); + + for item in items { + inserts.push(TraceItemInsert { + item_id: item.item_id, + note_id: item.note_id, + chunk_id: item.chunk_id, + rank: item.rank as i32, + final_score: item.final_score, + explain: item.explain, + }); + } + + let mut builder = QueryBuilder::new( + "\ +INSERT INTO search_trace_items ( + item_id, + trace_id, + note_id, + chunk_id, + rank, + final_score, + explain +) ", + ); + + builder.push_values(inserts, |mut b, item| { + b.push_bind(item.item_id) + .push_bind(trace_id) + .push_bind(item.note_id) + .push_bind(item.chunk_id) + .push_bind(item.rank) + .push_bind(item.final_score) + .push_bind(item.explain); + }); + builder.push(" ON CONFLICT (item_id) DO NOTHING"); + builder.build().execute(executor).await?; + + Ok(()) +} diff --git a/apps/elf-worker/src/worker/trace_jobs/persistence/stages.rs b/apps/elf-worker/src/worker/trace_jobs/persistence/stages.rs new file mode 100644 index 00000000..15dcafa2 --- /dev/null +++ b/apps/elf-worker/src/worker/trace_jobs/persistence/stages.rs @@ -0,0 +1,90 @@ +use crate::worker::{ + PgConnection, QueryBuilder, Result, TraceStageInsert, TraceStageItemInsert, + TraceTrajectoryStageRecord, Uuid, +}; + +pub(in crate::worker::trace_jobs) async fn insert_trace_stages_tx( + executor: &mut PgConnection, + trace_id: Uuid, + stages: Vec, +) -> Result<()> { + if stages.is_empty() { + return Ok(()); + } + + let mut stage_inserts = Vec::with_capacity(stages.len()); + let mut item_inserts = Vec::new(); + + for stage in stages { + stage_inserts.push(TraceStageInsert { + stage_id: stage.stage_id, + stage_order: stage.stage_order as i32, + stage_name: stage.stage_name, + stage_payload: stage.stage_payload, + created_at: stage.created_at, + }); + + for item in stage.items { + item_inserts.push(TraceStageItemInsert { + id: item.id, + stage_id: stage.stage_id, + item_id: item.item_id, + note_id: item.note_id, + chunk_id: item.chunk_id, + metrics: item.metrics, + }); + } + } + + let mut stage_builder = QueryBuilder::new( + "\ + INSERT INTO search_trace_stages ( + stage_id, + trace_id, + stage_order, + stage_name, + stage_payload, + created_at + ) ", + ); + + stage_builder.push_values(stage_inserts, |mut b, stage| { + b.push_bind(stage.stage_id) + .push_bind(trace_id) + .push_bind(stage.stage_order) + .push_bind(stage.stage_name) + .push_bind(stage.stage_payload) + .push_bind(stage.created_at); + }); + stage_builder.push(" ON CONFLICT (stage_id) DO NOTHING"); + stage_builder.build().execute(&mut *executor).await?; + + if item_inserts.is_empty() { + return Ok(()); + } + + let mut item_builder = QueryBuilder::new( + "\ + INSERT INTO search_trace_stage_items ( + id, + stage_id, + item_id, + note_id, + chunk_id, + metrics + ) ", + ); + + item_builder.push_values(item_inserts, |mut b, item| { + b.push_bind(item.id) + .push_bind(item.stage_id) + .push_bind(item.item_id) + .push_bind(item.note_id) + .push_bind(item.chunk_id) + .push_bind(item.metrics); + }); + item_builder.push(" ON CONFLICT (id) DO NOTHING"); + item_builder.build().execute(executor).await?; + + Ok(()) +} diff --git a/apps/elf-worker/src/worker/trace_jobs/persistence/trace.rs b/apps/elf-worker/src/worker/trace_jobs/persistence/trace.rs new file mode 100644 index 00000000..9eb01a19 --- /dev/null +++ b/apps/elf-worker/src/worker/trace_jobs/persistence/trace.rs @@ -0,0 +1,69 @@ +use crate::worker::{PgExecutor, Result, TraceRecord, Uuid, Value}; + +pub(in crate::worker::trace_jobs) async fn insert_trace_tx<'e, E>( + executor: E, + trace_id: Uuid, + trace: &TraceRecord, + expanded_queries_json: Value, + allowed_scopes_json: Value, +) -> Result<()> +where + E: PgExecutor<'e>, +{ + sqlx::query( + "INSERT INTO search_traces ( + trace_id, + tenant_id, + project_id, + agent_id, + read_profile, + query, + expansion_mode, + expanded_queries, + allowed_scopes, + candidate_count, + top_k, + config_snapshot, + trace_version, + created_at, + expires_at +) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6, + $7, + $8, + $9, + $10, + $11, + $12, + $13, + $14, + $15 +) +ON CONFLICT (trace_id) DO NOTHING", + ) + .bind(trace_id) + .bind(trace.tenant_id.as_str()) + .bind(trace.project_id.as_str()) + .bind(trace.agent_id.as_str()) + .bind(trace.read_profile.as_str()) + .bind(trace.query.as_str()) + .bind(trace.expansion_mode.as_str()) + .bind(expanded_queries_json) + .bind(allowed_scopes_json) + .bind(trace.candidate_count as i32) + .bind(trace.top_k as i32) + .bind(trace.config_snapshot.clone()) + .bind(trace.trace_version) + .bind(trace.created_at) + .bind(trace.expires_at) + .execute(executor) + .await?; + + Ok(()) +} From dea0b9bafc23201046a3ddab0390a4cbabdda828 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Tue, 30 Jun 2026 12:11:06 -0400 Subject: [PATCH 2/6] {"schema":"decodex/commit/1","summary":"{\"schema\":\"decodex/commit/1\",\"summary\":\"Split API route builder modules\",\"authority\":\"manual\"}","authority":"manual"} --- apps/elf-api/src/routes/route_builder.rs | 258 +----------------- .../elf-api/src/routes/route_builder/admin.rs | 196 +++++++++++++ .../src/routes/route_builder/public.rs | 60 ++++ 3 files changed, 266 insertions(+), 248 deletions(-) create mode 100644 apps/elf-api/src/routes/route_builder/admin.rs create mode 100644 apps/elf-api/src/routes/route_builder/public.rs diff --git a/apps/elf-api/src/routes/route_builder.rs b/apps/elf-api/src/routes/route_builder.rs index 46a695c5..31a7ba78 100644 --- a/apps/elf-api/src/routes/route_builder.rs +++ b/apps/elf-api/src/routes/route_builder.rs @@ -1,7 +1,10 @@ -use axum::{Router, extract::DefaultBodyLimit, middleware, routing}; +mod admin; +mod public; + +use axum::{Router, extract::DefaultBodyLimit, middleware}; use crate::{ - routes::{self, ADMIN_VIEWER_PATH, MAX_DOC_REQUEST_BYTES, MAX_REQUEST_BYTES}, + routes::{self, MAX_DOC_REQUEST_BYTES, MAX_REQUEST_BYTES}, state::AppState, }; @@ -12,260 +15,19 @@ pub fn router(state: AppState) -> Router { Router::new() .merge(routes::contract_router()) .merge( - public_api_router() + public::public_api_router() .with_state(state.clone()) .layer(DefaultBodyLimit::max(MAX_REQUEST_BYTES)), ) .merge( - docs_api_router().with_state(state).layer(DefaultBodyLimit::max(MAX_DOC_REQUEST_BYTES)), + public::docs_api_router() + .with_state(state) + .layer(DefaultBodyLimit::max(MAX_DOC_REQUEST_BYTES)), ) .layer(middleware::from_fn_with_state(auth_state, routes::support::api_auth_middleware)) } /// Builds the authenticated admin API router. pub fn admin_router(state: AppState) -> Router { - let auth_state = state.clone(); - let protected_router = Router::new() - .merge(admin_search_routes()) - .merge(admin_core_routes()) - .merge(admin_docs_routes()) - .merge(admin_notes_routes()) - .merge(admin_ingestion_profile_routes()) - .merge(admin_consolidation_routes()) - .merge(admin_knowledge_routes()) - .merge(admin_trace_routes()) - .merge(admin_graph_routes()) - .merge(admin_ops_routes()) - .with_state(state) - .layer(DefaultBodyLimit::max(MAX_REQUEST_BYTES)) - .layer(middleware::from_fn_with_state(auth_state, routes::support::admin_auth_middleware)); - - Router::new() - .route(ADMIN_VIEWER_PATH, routing::get(routes::viewer::admin_viewer)) - .route("/", routing::get(routes::viewer::admin_viewer)) - .merge(protected_router) -} - -fn public_api_router() -> Router { - Router::new() - .route("/health", routing::get(routes::health::health)) - .route("/v2/notes/ingest", routing::post(routes::notes::notes_ingest)) - .route("/v2/events/ingest", routing::post(routes::events::events_ingest)) - .route("/v2/core-blocks", routing::get(routes::core_memory::core_blocks_get)) - .route("/v2/entity-memory", routing::get(routes::core_memory::entity_memory_get)) - .route("/v2/recall-debug/panel", routing::post(routes::recall::recall_debug_panel)) - .route("/v2/searches", routing::post(routes::search::searches_create)) - .route("/v2/searches/{search_id}", routing::get(routes::search::searches_get)) - .route("/v2/searches/{search_id}/timeline", routing::get(routes::search::searches_timeline)) - .route("/v2/searches/{search_id}/notes", routing::post(routes::search::searches_notes)) - .route("/v2/graph/query", routing::post(routes::graph::graph_query)) - .route("/v2/graph/report", routing::post(routes::graph::graph_report)) - .route("/v2/notes", routing::get(routes::notes::notes_list)) - .route( - "/v2/notes/{note_id}", - routing::get(routes::notes::notes_get) - .patch(routes::notes::notes_patch) - .delete(routes::notes::notes_delete), - ) - .route("/v2/notes/{note_id}/publish", routing::post(routes::notes::notes_publish)) - .route("/v2/notes/{note_id}/unpublish", routing::post(routes::notes::notes_unpublish)) - .route( - "/v2/work-journal/entries", - routing::post(routes::work_journal::work_journal_entry_create), - ) - .route( - "/v2/work-journal/entries/{entry_id}", - routing::get(routes::work_journal::work_journal_entry_get), - ) - .route( - "/v2/work-journal/readback", - routing::post(routes::work_journal::work_journal_session_readback), - ) - .route( - "/v2/spaces/{space}/grants", - routing::get(routes::sharing::space_grants_list) - .post(routes::sharing::space_grant_upsert), - ) - .route( - "/v2/spaces/{space}/grants/revoke", - routing::post(routes::sharing::space_grant_revoke), - ) -} - -fn docs_api_router() -> Router { - Router::new() - .route("/v2/docs", routing::post(routes::docs::docs_put)) - .route( - "/v2/docs/{doc_id}", - routing::get(routes::docs::docs_get).delete(routes::docs::docs_delete), - ) - .route("/v2/docs/search/l0", routing::post(routes::docs::docs_search_l0)) - .route("/v2/docs/excerpts", routing::post(routes::docs::docs_excerpts_get)) -} - -fn admin_search_routes() -> Router { - Router::new() - .route("/v2/admin/searches", routing::post(routes::search::searches_create)) - .route("/v2/admin/searches/raw", routing::post(routes::search::searches_raw)) - .route("/v2/admin/searches/{search_id}", routing::get(routes::search::searches_get)) - .route( - "/v2/admin/searches/{search_id}/timeline", - routing::get(routes::search::searches_timeline), - ) - .route( - "/v2/admin/searches/{search_id}/notes", - routing::post(routes::search::searches_notes), - ) -} - -fn admin_core_routes() -> Router { - Router::new() - .route("/v2/admin/core-blocks", routing::post(routes::core_memory::admin_core_block_upsert)) - .route( - "/v2/admin/core-blocks/{block_id}/attachments", - routing::post(routes::core_memory::admin_core_block_attach), - ) - .route( - "/v2/admin/core-blocks/attachments/{attachment_id}", - routing::delete(routes::core_memory::admin_core_block_detach), - ) - .route( - "/v2/admin/recall-debug/panel", - routing::post(routes::recall::admin_recall_debug_panel), - ) -} - -fn admin_docs_routes() -> Router { - Router::new() - .route("/v2/admin/docs/search/l0", routing::post(routes::docs::admin_docs_search_l0)) - .route("/v2/admin/docs/excerpts", routing::post(routes::docs::admin_docs_excerpts_get)) - .route("/v2/admin/docs/{doc_id}", routing::get(routes::docs::admin_docs_get)) -} - -fn admin_notes_routes() -> Router { - Router::new() - .route("/v2/admin/notes", routing::get(routes::notes::notes_list)) - .route("/v2/admin/notes/{note_id}", routing::get(routes::notes::notes_get)) - .route( - "/v2/admin/notes/{note_id}/provenance", - routing::get(routes::admin_notes::admin_note_provenance_get), - ) - .route( - "/v2/admin/notes/{note_id}/history", - routing::get(routes::admin_notes::admin_note_history_get), - ) - .route( - "/v2/admin/notes/{note_id}/corrections", - routing::post(routes::admin_notes::admin_note_correction_apply), - ) -} - -fn admin_ingestion_profile_routes() -> Router { - Router::new() - .route( - "/v2/admin/events/ingestion-profiles/default", - routing::get(routes::ingestion_profiles::admin_ingestion_profile_default_get) - .put(routes::ingestion_profiles::admin_ingestion_profile_default_set), - ) - .route( - "/v2/admin/events/ingestion-profiles/{profile_id}/versions", - routing::get(routes::ingestion_profiles::admin_ingestion_profile_versions_list), - ) - .route( - "/v2/admin/events/ingestion-profiles/{profile_id}", - routing::get(routes::ingestion_profiles::admin_ingestion_profile_get), - ) - .route( - "/v2/admin/events/ingestion-profiles", - routing::get(routes::ingestion_profiles::admin_ingestion_profiles_list) - .post(routes::ingestion_profiles::admin_ingestion_profile_create), - ) -} - -fn admin_consolidation_routes() -> Router { - Router::new() - .route( - "/v2/admin/consolidation/runs", - routing::get(routes::consolidation::consolidation_runs_list) - .post(routes::consolidation::consolidation_run_create), - ) - .route( - "/v2/admin/consolidation/runs/{run_id}", - routing::get(routes::consolidation::consolidation_run_get), - ) - .route( - "/v2/admin/consolidation/proposals", - routing::get(routes::consolidation::consolidation_proposals_list), - ) - .route( - "/v2/admin/consolidation/proposals/{proposal_id}", - routing::get(routes::consolidation::consolidation_proposal_get), - ) - .route( - "/v2/admin/consolidation/proposals/{proposal_id}/review", - routing::post(routes::consolidation::consolidation_proposal_review), - ) - .route( - "/v2/admin/dreaming/review-queue", - routing::get(routes::dreaming::dreaming_review_queue), - ) -} - -fn admin_knowledge_routes() -> Router { - Router::new() - .route("/v2/admin/knowledge/pages", routing::get(routes::knowledge::knowledge_pages_list)) - .route( - "/v2/admin/knowledge/pages/rebuild", - routing::post(routes::knowledge::knowledge_page_rebuild), - ) - .route( - "/v2/admin/knowledge/pages/rebuild-changed-sources", - routing::post(routes::knowledge::knowledge_pages_watch_rebuild), - ) - .route( - "/v2/admin/knowledge/pages/search", - routing::post(routes::knowledge::knowledge_pages_search), - ) - .route( - "/v2/admin/knowledge/pages/{page_id}", - routing::get(routes::knowledge::knowledge_page_get), - ) - .route( - "/v2/admin/knowledge/pages/{page_id}/lint", - routing::post(routes::knowledge::knowledge_page_lint), - ) -} - -fn admin_trace_routes() -> Router { - Router::new() - .route("/v2/admin/traces/recent", routing::get(routes::trace::trace_recent_list)) - .route("/v2/admin/traces/{trace_id}", routing::get(routes::trace::trace_get)) - .route("/v2/admin/traces/{trace_id}/bundle", routing::get(routes::trace::trace_bundle_get)) - .route( - "/v2/admin/trajectories/{trace_id}", - routing::get(routes::trace::trace_trajectory_get), - ) - .route("/v2/admin/trace-items/{item_id}", routing::get(routes::trace::trace_item_get)) -} - -fn admin_graph_routes() -> Router { - Router::new() - .route( - "/v2/admin/graph/predicates", - routing::get(routes::graph::admin_graph_predicates_list), - ) - .route( - "/v2/admin/graph/predicates/{predicate_id}", - routing::patch(routes::graph::admin_graph_predicate_patch), - ) - .route( - "/v2/admin/graph/predicates/{predicate_id}/aliases", - routing::post(routes::graph::admin_graph_predicate_alias_add) - .get(routes::graph::admin_graph_predicate_aliases_list), - ) -} - -fn admin_ops_routes() -> Router { - Router::new() - .route("/v2/admin/qdrant/rebuild", routing::post(routes::admin_ops::rebuild_qdrant)) + admin::admin_router(state) } diff --git a/apps/elf-api/src/routes/route_builder/admin.rs b/apps/elf-api/src/routes/route_builder/admin.rs new file mode 100644 index 00000000..b786ba78 --- /dev/null +++ b/apps/elf-api/src/routes/route_builder/admin.rs @@ -0,0 +1,196 @@ +use axum::{Router, extract::DefaultBodyLimit, middleware, routing}; + +use crate::{ + routes::{self, ADMIN_VIEWER_PATH, MAX_REQUEST_BYTES}, + state::AppState, +}; + +pub(super) fn admin_router(state: AppState) -> Router { + let auth_state = state.clone(); + let protected_router = Router::new() + .merge(admin_search_routes()) + .merge(admin_core_routes()) + .merge(admin_docs_routes()) + .merge(admin_notes_routes()) + .merge(admin_ingestion_profile_routes()) + .merge(admin_consolidation_routes()) + .merge(admin_knowledge_routes()) + .merge(admin_trace_routes()) + .merge(admin_graph_routes()) + .merge(admin_ops_routes()) + .with_state(state) + .layer(DefaultBodyLimit::max(MAX_REQUEST_BYTES)) + .layer(middleware::from_fn_with_state(auth_state, routes::support::admin_auth_middleware)); + + Router::new() + .route(ADMIN_VIEWER_PATH, routing::get(routes::viewer::admin_viewer)) + .route("/", routing::get(routes::viewer::admin_viewer)) + .merge(protected_router) +} + +fn admin_search_routes() -> Router { + Router::new() + .route("/v2/admin/searches", routing::post(routes::search::searches_create)) + .route("/v2/admin/searches/raw", routing::post(routes::search::searches_raw)) + .route("/v2/admin/searches/{search_id}", routing::get(routes::search::searches_get)) + .route( + "/v2/admin/searches/{search_id}/timeline", + routing::get(routes::search::searches_timeline), + ) + .route( + "/v2/admin/searches/{search_id}/notes", + routing::post(routes::search::searches_notes), + ) +} + +fn admin_core_routes() -> Router { + Router::new() + .route("/v2/admin/core-blocks", routing::post(routes::core_memory::admin_core_block_upsert)) + .route( + "/v2/admin/core-blocks/{block_id}/attachments", + routing::post(routes::core_memory::admin_core_block_attach), + ) + .route( + "/v2/admin/core-blocks/attachments/{attachment_id}", + routing::delete(routes::core_memory::admin_core_block_detach), + ) + .route( + "/v2/admin/recall-debug/panel", + routing::post(routes::recall::admin_recall_debug_panel), + ) +} + +fn admin_docs_routes() -> Router { + Router::new() + .route("/v2/admin/docs/search/l0", routing::post(routes::docs::admin_docs_search_l0)) + .route("/v2/admin/docs/excerpts", routing::post(routes::docs::admin_docs_excerpts_get)) + .route("/v2/admin/docs/{doc_id}", routing::get(routes::docs::admin_docs_get)) +} + +fn admin_notes_routes() -> Router { + Router::new() + .route("/v2/admin/notes", routing::get(routes::notes::notes_list)) + .route("/v2/admin/notes/{note_id}", routing::get(routes::notes::notes_get)) + .route( + "/v2/admin/notes/{note_id}/provenance", + routing::get(routes::admin_notes::admin_note_provenance_get), + ) + .route( + "/v2/admin/notes/{note_id}/history", + routing::get(routes::admin_notes::admin_note_history_get), + ) + .route( + "/v2/admin/notes/{note_id}/corrections", + routing::post(routes::admin_notes::admin_note_correction_apply), + ) +} + +fn admin_ingestion_profile_routes() -> Router { + Router::new() + .route( + "/v2/admin/events/ingestion-profiles/default", + routing::get(routes::ingestion_profiles::admin_ingestion_profile_default_get) + .put(routes::ingestion_profiles::admin_ingestion_profile_default_set), + ) + .route( + "/v2/admin/events/ingestion-profiles/{profile_id}/versions", + routing::get(routes::ingestion_profiles::admin_ingestion_profile_versions_list), + ) + .route( + "/v2/admin/events/ingestion-profiles/{profile_id}", + routing::get(routes::ingestion_profiles::admin_ingestion_profile_get), + ) + .route( + "/v2/admin/events/ingestion-profiles", + routing::get(routes::ingestion_profiles::admin_ingestion_profiles_list) + .post(routes::ingestion_profiles::admin_ingestion_profile_create), + ) +} + +fn admin_consolidation_routes() -> Router { + Router::new() + .route( + "/v2/admin/consolidation/runs", + routing::get(routes::consolidation::consolidation_runs_list) + .post(routes::consolidation::consolidation_run_create), + ) + .route( + "/v2/admin/consolidation/runs/{run_id}", + routing::get(routes::consolidation::consolidation_run_get), + ) + .route( + "/v2/admin/consolidation/proposals", + routing::get(routes::consolidation::consolidation_proposals_list), + ) + .route( + "/v2/admin/consolidation/proposals/{proposal_id}", + routing::get(routes::consolidation::consolidation_proposal_get), + ) + .route( + "/v2/admin/consolidation/proposals/{proposal_id}/review", + routing::post(routes::consolidation::consolidation_proposal_review), + ) + .route( + "/v2/admin/dreaming/review-queue", + routing::get(routes::dreaming::dreaming_review_queue), + ) +} + +fn admin_knowledge_routes() -> Router { + Router::new() + .route("/v2/admin/knowledge/pages", routing::get(routes::knowledge::knowledge_pages_list)) + .route( + "/v2/admin/knowledge/pages/rebuild", + routing::post(routes::knowledge::knowledge_page_rebuild), + ) + .route( + "/v2/admin/knowledge/pages/rebuild-changed-sources", + routing::post(routes::knowledge::knowledge_pages_watch_rebuild), + ) + .route( + "/v2/admin/knowledge/pages/search", + routing::post(routes::knowledge::knowledge_pages_search), + ) + .route( + "/v2/admin/knowledge/pages/{page_id}", + routing::get(routes::knowledge::knowledge_page_get), + ) + .route( + "/v2/admin/knowledge/pages/{page_id}/lint", + routing::post(routes::knowledge::knowledge_page_lint), + ) +} + +fn admin_trace_routes() -> Router { + Router::new() + .route("/v2/admin/traces/recent", routing::get(routes::trace::trace_recent_list)) + .route("/v2/admin/traces/{trace_id}", routing::get(routes::trace::trace_get)) + .route("/v2/admin/traces/{trace_id}/bundle", routing::get(routes::trace::trace_bundle_get)) + .route( + "/v2/admin/trajectories/{trace_id}", + routing::get(routes::trace::trace_trajectory_get), + ) + .route("/v2/admin/trace-items/{item_id}", routing::get(routes::trace::trace_item_get)) +} + +fn admin_graph_routes() -> Router { + Router::new() + .route( + "/v2/admin/graph/predicates", + routing::get(routes::graph::admin_graph_predicates_list), + ) + .route( + "/v2/admin/graph/predicates/{predicate_id}", + routing::patch(routes::graph::admin_graph_predicate_patch), + ) + .route( + "/v2/admin/graph/predicates/{predicate_id}/aliases", + routing::post(routes::graph::admin_graph_predicate_alias_add) + .get(routes::graph::admin_graph_predicate_aliases_list), + ) +} + +fn admin_ops_routes() -> Router { + Router::new() + .route("/v2/admin/qdrant/rebuild", routing::post(routes::admin_ops::rebuild_qdrant)) +} diff --git a/apps/elf-api/src/routes/route_builder/public.rs b/apps/elf-api/src/routes/route_builder/public.rs new file mode 100644 index 00000000..4124607a --- /dev/null +++ b/apps/elf-api/src/routes/route_builder/public.rs @@ -0,0 +1,60 @@ +use axum::{Router, routing}; + +use crate::{routes, state::AppState}; + +pub(super) fn public_api_router() -> Router { + Router::new() + .route("/health", routing::get(routes::health::health)) + .route("/v2/notes/ingest", routing::post(routes::notes::notes_ingest)) + .route("/v2/events/ingest", routing::post(routes::events::events_ingest)) + .route("/v2/core-blocks", routing::get(routes::core_memory::core_blocks_get)) + .route("/v2/entity-memory", routing::get(routes::core_memory::entity_memory_get)) + .route("/v2/recall-debug/panel", routing::post(routes::recall::recall_debug_panel)) + .route("/v2/searches", routing::post(routes::search::searches_create)) + .route("/v2/searches/{search_id}", routing::get(routes::search::searches_get)) + .route("/v2/searches/{search_id}/timeline", routing::get(routes::search::searches_timeline)) + .route("/v2/searches/{search_id}/notes", routing::post(routes::search::searches_notes)) + .route("/v2/graph/query", routing::post(routes::graph::graph_query)) + .route("/v2/graph/report", routing::post(routes::graph::graph_report)) + .route("/v2/notes", routing::get(routes::notes::notes_list)) + .route( + "/v2/notes/{note_id}", + routing::get(routes::notes::notes_get) + .patch(routes::notes::notes_patch) + .delete(routes::notes::notes_delete), + ) + .route("/v2/notes/{note_id}/publish", routing::post(routes::notes::notes_publish)) + .route("/v2/notes/{note_id}/unpublish", routing::post(routes::notes::notes_unpublish)) + .route( + "/v2/work-journal/entries", + routing::post(routes::work_journal::work_journal_entry_create), + ) + .route( + "/v2/work-journal/entries/{entry_id}", + routing::get(routes::work_journal::work_journal_entry_get), + ) + .route( + "/v2/work-journal/readback", + routing::post(routes::work_journal::work_journal_session_readback), + ) + .route( + "/v2/spaces/{space}/grants", + routing::get(routes::sharing::space_grants_list) + .post(routes::sharing::space_grant_upsert), + ) + .route( + "/v2/spaces/{space}/grants/revoke", + routing::post(routes::sharing::space_grant_revoke), + ) +} + +pub(super) fn docs_api_router() -> Router { + Router::new() + .route("/v2/docs", routing::post(routes::docs::docs_put)) + .route( + "/v2/docs/{doc_id}", + routing::get(routes::docs::docs_get).delete(routes::docs::docs_delete), + ) + .route("/v2/docs/search/l0", routing::post(routes::docs::docs_search_l0)) + .route("/v2/docs/excerpts", routing::post(routes::docs::docs_excerpts_get)) +} From 579d32893f6b772b2f322e7438a3ec22315918aa Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Tue, 30 Jun 2026 12:18:03 -0400 Subject: [PATCH 3/6] {"schema":"decodex/commit/1","summary":"{\"schema\":\"decodex/commit/1\",\"summary\":\"Split benchmark domain metrics markdown modules\",\"authority\":\"manual\"}","authority":"manual"} --- .../markdown/domain_metrics.rs | 296 +----------------- .../markdown/domain_metrics/consolidation.rs | 70 +++++ .../markdown/domain_metrics/knowledge.rs | 40 +++ .../markdown/domain_metrics/memory.rs | 44 +++ .../markdown/domain_metrics/proactive.rs | 45 +++ .../markdown/domain_metrics/scheduled.rs | 45 +++ .../markdown/domain_metrics/work.rs | 60 ++++ 7 files changed, 316 insertions(+), 284 deletions(-) create mode 100644 apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/consolidation.rs create mode 100644 apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/knowledge.rs create mode 100644 apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/memory.rs create mode 100644 apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/proactive.rs create mode 100644 apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/scheduled.rs create mode 100644 apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/work.rs diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics.rs index 76e0cdf1..89848231 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics.rs @@ -1,284 +1,12 @@ -use crate::markdown::{self, RealWorldReport}; - -pub(super) fn render_markdown_consolidation(out: &mut String, report: &RealWorldReport) { - if report.summary.consolidation.proposal_count == 0 { - return; - } - - out.push_str("## Consolidation\n\n"); - out.push_str("| Job | Proposals | Usefulness | Lineage | Review Actions | Source Mutations | Proposal Unsupported Claims | Executable Gaps |\n"); - out.push_str("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n"); - - for job in &report.jobs { - let Some(consolidation) = &job.consolidation else { - continue; - }; - - out.push_str(&format!( - "| {} | {} | `{}` | `{}` | `{}` | {} | {} | {} |\n", - markdown::md_cell(job.job_id.as_str()), - consolidation.proposal_count, - markdown::optional_f64(consolidation.proposal_usefulness, ""), - markdown::optional_f64(consolidation.lineage_completeness, ""), - markdown::optional_f64(consolidation.review_action_correctness, ""), - consolidation.source_mutation_count, - consolidation.proposal_unsupported_claim_count, - consolidation.executable_gaps.len() - )); - } - - out.push_str( - "\nSource mutation count must remain `0` for proposal-only consolidation cases.\n\n", - ); - - render_markdown_consolidation_gaps(out, report); -} - -pub(super) fn render_markdown_knowledge(out: &mut String, report: &RealWorldReport) { - let knowledge_jobs = - report.jobs.iter().filter(|job| job.knowledge.is_some()).collect::>(); - - if knowledge_jobs.is_empty() { - return; - } - - out.push_str("## Knowledge Page Metrics\n\n"); - out.push_str("| Job | Pages | Sections | Citation Coverage | Stale Claim Detection | Rebuild Determinism | Version Diff Coverage | Page Usefulness | Backlinks | Unsupported Summaries | Untraced Sections | Allowed Variance |\n"); - out.push_str( - "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n", - ); - - for job in knowledge_jobs { - let Some(knowledge) = &job.knowledge else { - continue; - }; - - out.push_str(&format!( - "| {} | {} | {} | `{:.3}` | `{:.3}` | `{:.3}` | `{:.3}` | `{:.3}` | {} | {} | {} | {} |\n", - markdown::md_cell(job.job_id.as_str()), - knowledge.page_count, - knowledge.section_count, - knowledge.citation_coverage, - knowledge.stale_claim_detection, - knowledge.rebuild_determinism, - knowledge.version_diff_coverage, - knowledge.page_usefulness, - knowledge.backlink_count, - knowledge.unsupported_summary_count, - knowledge.untraced_section_count, - knowledge.allowed_variance_count - )); - } - - out.push('\n'); -} - -pub(super) fn render_markdown_memory_summary(out: &mut String, report: &RealWorldReport) { - let memory_jobs = - report.jobs.iter().filter(|job| job.memory_summary.is_some()).collect::>(); - - if memory_jobs.is_empty() { - return; - } - - out.push_str("## Memory Summary Metrics\n\n"); - out.push_str("| Job | Summaries | Entries | Categories | Source Coverage | Freshness | Rationale | Invalid Top-of-Mind | Untraced | Derived Unsupported | Unsupported Current | Tombstone Refs |\n"); - out.push_str( - "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n", - ); - - for job in memory_jobs { - let Some(metrics) = &job.memory_summary else { - continue; - }; - - out.push_str(&format!( - "| {} | {} | {} | `{}/{}` | `{:.3}` | `{:.3}` | `{:.3}` | {} | {} | {} | {} | {} |\n", - markdown::md_cell(job.job_id.as_str()), - metrics.summary_count, - metrics.entry_count, - metrics.covered_required_category_count, - metrics.required_category_count, - metrics.source_ref_coverage, - metrics.freshness_coverage, - metrics.rationale_coverage, - metrics.invalid_top_of_mind_count, - metrics.untraced_entry_count, - metrics.unsupported_derived_entry_count, - metrics.unsupported_current_entry_count, - metrics.tombstone_ref_count - )); - } - - out.push('\n'); -} - -pub(super) fn render_markdown_proactive_brief(out: &mut String, report: &RealWorldReport) { - let proactive_jobs = - report.jobs.iter().filter(|job| job.proactive_brief.is_some()).collect::>(); - - if proactive_jobs.is_empty() { - return; - } - - out.push_str("## Proactive Brief Metrics\n\n"); - out.push_str("| Job | Briefs | Suggestions | Kinds | Evidence Coverage | Freshness | Action Rationale | Invalid Current | Untraced | Unsupported Current | Tombstone Violations | Rejected | Deferred |\n"); - out.push_str( - "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n", - ); - - for job in proactive_jobs { - let Some(metrics) = &job.proactive_brief else { - continue; - }; - - out.push_str(&format!( - "| {} | {} | {} | `{}/{}` | `{:.3}` | `{:.3}` | `{:.3}` | {} | {} | {} | {} | {} | {} |\n", - markdown::md_cell(job.job_id.as_str()), - metrics.brief_count, - metrics.suggestion_count, - metrics.covered_required_suggestion_kind_count, - metrics.required_suggestion_kind_count, - metrics.evidence_ref_coverage, - metrics.freshness_coverage, - metrics.action_rationale_coverage, - metrics.invalid_current_suggestion_count, - metrics.untraced_suggestion_count, - metrics.unsupported_current_suggestion_count, - metrics.tombstone_violation_count, - metrics.rejected_count, - metrics.deferred_count - )); - } - - out.push('\n'); -} - -pub(super) fn render_markdown_scheduled_memory(out: &mut String, report: &RealWorldReport) { - let scheduled_jobs = - report.jobs.iter().filter(|job| job.scheduled_memory.is_some()).collect::>(); - - if scheduled_jobs.is_empty() { - return; - } - - out.push_str("## Scheduled Memory Metrics\n\n"); - out.push_str("| Job | Task Runs | Outputs | Kinds | Evidence Coverage | Freshness | Action Rationale | Trace Coverage | Invalid Current | Untraced | Unsupported Current | Tombstone Violations | Source Mutations |\n"); - out.push_str( - "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n", - ); - - for job in scheduled_jobs { - let Some(metrics) = &job.scheduled_memory else { - continue; - }; - - out.push_str(&format!( - "| {} | {} | {} | `{}/{}` | `{:.3}` | `{:.3}` | `{:.3}` | `{:.3}` | {} | {} | {} | {} | {} |\n", - markdown::md_cell(job.job_id.as_str()), - metrics.task_run_count, - metrics.output_count, - metrics.covered_required_task_kind_count, - metrics.required_task_kind_count, - metrics.evidence_ref_coverage, - metrics.freshness_coverage, - metrics.action_rationale_coverage, - metrics.trace_coverage, - metrics.invalid_current_output_count, - metrics.untraced_output_count, - metrics.unsupported_current_output_count, - metrics.tombstone_violation_count, - metrics.source_mutation_count - )); - } - - out.push('\n'); -} - -pub(super) fn render_markdown_work_continuity(out: &mut String, report: &RealWorldReport) { - let work_jobs = - report.jobs.iter().filter(|job| job.work_continuity.is_some()).collect::>(); - - if work_jobs.is_empty() { - return; - } - - out.push_str("## Work Continuity Metrics\n\n"); - out.push_str("| Job | Readbacks | Entries | Reset/Resume | Decision Rationale | Rejected Suppression | Explicit Precision | Inferred Labeling | Handoff Sources | Redaction | Janitor False Promotion | Sensitive Persistence | Journal Authority Claims |\n"); - out.push_str( - "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n", - ); - - for job in work_jobs { - let Some(metrics) = &job.work_continuity else { - continue; - }; - - out.push_str(&format!( - "| {} | {} | {} | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | {} | {} |\n", - markdown::md_cell(job.job_id.as_str()), - metrics.readback_count, - metrics.entry_count, - metrics.reset_resume_success_count, - metrics.reset_resume_required_count, - metrics.reset_resume_success_rate, - metrics.decision_rationale_recalled_count, - metrics.decision_rationale_required_count, - metrics.decision_rationale_recall_rate, - metrics.rejected_option_suppressed_count, - metrics.rejected_option_required_count, - metrics.rejected_option_suppression_rate, - metrics.explicit_next_step_correct_count, - metrics.explicit_next_step_returned_count, - metrics.explicit_next_step_precision, - metrics.inferred_next_step_labeled_count, - metrics.inferred_next_step_required_count, - metrics.inferred_next_step_labeling_rate, - metrics.handoff_source_ref_covered_count, - metrics.handoff_source_ref_required_count, - metrics.handoff_source_ref_coverage, - metrics.redaction_applied_count, - metrics.redaction_required_count, - metrics.redaction_rate, - metrics.janitor_false_promotion_count, - metrics.janitor_candidate_count, - metrics.janitor_false_promotion_rate, - metrics.sensitive_marker_persistence_count, - metrics.journal_only_authority_claim_count - )); - } - - out.push('\n'); -} - -fn render_markdown_consolidation_gaps(out: &mut String, report: &RealWorldReport) { - let gaps = report - .jobs - .iter() - .filter_map(|job| job.consolidation.as_ref().map(|consolidation| (job, consolidation))) - .flat_map(|(job, consolidation)| { - consolidation.executable_gaps.iter().map(move |gap| (job.job_id.as_str(), gap)) - }) - .collect::>(); - - if gaps.is_empty() { - return; - } - - out.push_str("### Executable Gaps\n\n"); - out.push_str("| Job | Primitive | Follow-Up Issue | Blocks Fixture Pass | Reason |\n"); - out.push_str("| --- | --- | --- | --- | --- |\n"); - - for (job_id, gap) in gaps { - out.push_str(&format!( - "| {} | {} | {} | `{}` | {} |\n", - markdown::md_cell(job_id), - markdown::md_cell(gap.primitive.as_str()), - markdown::md_cell(gap.follow_up_issue.as_str()), - gap.blocks_fixture_pass, - markdown::md_cell(gap.reason.as_str()) - )); - } - - out.push('\n'); -} +mod consolidation; +mod knowledge; +mod memory; +mod proactive; +mod scheduled; +mod work; + +pub(super) use self::{ + consolidation::render_markdown_consolidation, knowledge::render_markdown_knowledge, + memory::render_markdown_memory_summary, proactive::render_markdown_proactive_brief, + scheduled::render_markdown_scheduled_memory, work::render_markdown_work_continuity, +}; diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/consolidation.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/consolidation.rs new file mode 100644 index 00000000..2d8fac98 --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/consolidation.rs @@ -0,0 +1,70 @@ +use crate::markdown::{self, RealWorldReport}; + +pub(in crate::markdown) fn render_markdown_consolidation( + out: &mut String, + report: &RealWorldReport, +) { + if report.summary.consolidation.proposal_count == 0 { + return; + } + + out.push_str("## Consolidation\n\n"); + out.push_str("| Job | Proposals | Usefulness | Lineage | Review Actions | Source Mutations | Proposal Unsupported Claims | Executable Gaps |\n"); + out.push_str("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n"); + + for job in &report.jobs { + let Some(consolidation) = &job.consolidation else { + continue; + }; + + out.push_str(&format!( + "| {} | {} | `{}` | `{}` | `{}` | {} | {} | {} |\n", + markdown::md_cell(job.job_id.as_str()), + consolidation.proposal_count, + markdown::optional_f64(consolidation.proposal_usefulness, ""), + markdown::optional_f64(consolidation.lineage_completeness, ""), + markdown::optional_f64(consolidation.review_action_correctness, ""), + consolidation.source_mutation_count, + consolidation.proposal_unsupported_claim_count, + consolidation.executable_gaps.len() + )); + } + + out.push_str( + "\nSource mutation count must remain `0` for proposal-only consolidation cases.\n\n", + ); + + render_markdown_consolidation_gaps(out, report); +} + +fn render_markdown_consolidation_gaps(out: &mut String, report: &RealWorldReport) { + let gaps = report + .jobs + .iter() + .filter_map(|job| job.consolidation.as_ref().map(|consolidation| (job, consolidation))) + .flat_map(|(job, consolidation)| { + consolidation.executable_gaps.iter().map(move |gap| (job.job_id.as_str(), gap)) + }) + .collect::>(); + + if gaps.is_empty() { + return; + } + + out.push_str("### Executable Gaps\n\n"); + out.push_str("| Job | Primitive | Follow-Up Issue | Blocks Fixture Pass | Reason |\n"); + out.push_str("| --- | --- | --- | --- | --- |\n"); + + for (job_id, gap) in gaps { + out.push_str(&format!( + "| {} | {} | {} | `{}` | {} |\n", + markdown::md_cell(job_id), + markdown::md_cell(gap.primitive.as_str()), + markdown::md_cell(gap.follow_up_issue.as_str()), + gap.blocks_fixture_pass, + markdown::md_cell(gap.reason.as_str()) + )); + } + + out.push('\n'); +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/knowledge.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/knowledge.rs new file mode 100644 index 00000000..30cf3706 --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/knowledge.rs @@ -0,0 +1,40 @@ +use crate::markdown::{self, RealWorldReport}; + +pub(in crate::markdown) fn render_markdown_knowledge(out: &mut String, report: &RealWorldReport) { + let knowledge_jobs = + report.jobs.iter().filter(|job| job.knowledge.is_some()).collect::>(); + + if knowledge_jobs.is_empty() { + return; + } + + out.push_str("## Knowledge Page Metrics\n\n"); + out.push_str("| Job | Pages | Sections | Citation Coverage | Stale Claim Detection | Rebuild Determinism | Version Diff Coverage | Page Usefulness | Backlinks | Unsupported Summaries | Untraced Sections | Allowed Variance |\n"); + out.push_str( + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n", + ); + + for job in knowledge_jobs { + let Some(knowledge) = &job.knowledge else { + continue; + }; + + out.push_str(&format!( + "| {} | {} | {} | `{:.3}` | `{:.3}` | `{:.3}` | `{:.3}` | `{:.3}` | {} | {} | {} | {} |\n", + markdown::md_cell(job.job_id.as_str()), + knowledge.page_count, + knowledge.section_count, + knowledge.citation_coverage, + knowledge.stale_claim_detection, + knowledge.rebuild_determinism, + knowledge.version_diff_coverage, + knowledge.page_usefulness, + knowledge.backlink_count, + knowledge.unsupported_summary_count, + knowledge.untraced_section_count, + knowledge.allowed_variance_count + )); + } + + out.push('\n'); +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/memory.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/memory.rs new file mode 100644 index 00000000..586eadc1 --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/memory.rs @@ -0,0 +1,44 @@ +use crate::markdown::{self, RealWorldReport}; + +pub(in crate::markdown) fn render_markdown_memory_summary( + out: &mut String, + report: &RealWorldReport, +) { + let memory_jobs = + report.jobs.iter().filter(|job| job.memory_summary.is_some()).collect::>(); + + if memory_jobs.is_empty() { + return; + } + + out.push_str("## Memory Summary Metrics\n\n"); + out.push_str("| Job | Summaries | Entries | Categories | Source Coverage | Freshness | Rationale | Invalid Top-of-Mind | Untraced | Derived Unsupported | Unsupported Current | Tombstone Refs |\n"); + out.push_str( + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n", + ); + + for job in memory_jobs { + let Some(metrics) = &job.memory_summary else { + continue; + }; + + out.push_str(&format!( + "| {} | {} | {} | `{}/{}` | `{:.3}` | `{:.3}` | `{:.3}` | {} | {} | {} | {} | {} |\n", + markdown::md_cell(job.job_id.as_str()), + metrics.summary_count, + metrics.entry_count, + metrics.covered_required_category_count, + metrics.required_category_count, + metrics.source_ref_coverage, + metrics.freshness_coverage, + metrics.rationale_coverage, + metrics.invalid_top_of_mind_count, + metrics.untraced_entry_count, + metrics.unsupported_derived_entry_count, + metrics.unsupported_current_entry_count, + metrics.tombstone_ref_count + )); + } + + out.push('\n'); +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/proactive.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/proactive.rs new file mode 100644 index 00000000..9e9f490d --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/proactive.rs @@ -0,0 +1,45 @@ +use crate::markdown::{self, RealWorldReport}; + +pub(in crate::markdown) fn render_markdown_proactive_brief( + out: &mut String, + report: &RealWorldReport, +) { + let proactive_jobs = + report.jobs.iter().filter(|job| job.proactive_brief.is_some()).collect::>(); + + if proactive_jobs.is_empty() { + return; + } + + out.push_str("## Proactive Brief Metrics\n\n"); + out.push_str("| Job | Briefs | Suggestions | Kinds | Evidence Coverage | Freshness | Action Rationale | Invalid Current | Untraced | Unsupported Current | Tombstone Violations | Rejected | Deferred |\n"); + out.push_str( + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n", + ); + + for job in proactive_jobs { + let Some(metrics) = &job.proactive_brief else { + continue; + }; + + out.push_str(&format!( + "| {} | {} | {} | `{}/{}` | `{:.3}` | `{:.3}` | `{:.3}` | {} | {} | {} | {} | {} | {} |\n", + markdown::md_cell(job.job_id.as_str()), + metrics.brief_count, + metrics.suggestion_count, + metrics.covered_required_suggestion_kind_count, + metrics.required_suggestion_kind_count, + metrics.evidence_ref_coverage, + metrics.freshness_coverage, + metrics.action_rationale_coverage, + metrics.invalid_current_suggestion_count, + metrics.untraced_suggestion_count, + metrics.unsupported_current_suggestion_count, + metrics.tombstone_violation_count, + metrics.rejected_count, + metrics.deferred_count + )); + } + + out.push('\n'); +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/scheduled.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/scheduled.rs new file mode 100644 index 00000000..d8ca0d35 --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/scheduled.rs @@ -0,0 +1,45 @@ +use crate::markdown::{self, RealWorldReport}; + +pub(in crate::markdown) fn render_markdown_scheduled_memory( + out: &mut String, + report: &RealWorldReport, +) { + let scheduled_jobs = + report.jobs.iter().filter(|job| job.scheduled_memory.is_some()).collect::>(); + + if scheduled_jobs.is_empty() { + return; + } + + out.push_str("## Scheduled Memory Metrics\n\n"); + out.push_str("| Job | Task Runs | Outputs | Kinds | Evidence Coverage | Freshness | Action Rationale | Trace Coverage | Invalid Current | Untraced | Unsupported Current | Tombstone Violations | Source Mutations |\n"); + out.push_str( + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n", + ); + + for job in scheduled_jobs { + let Some(metrics) = &job.scheduled_memory else { + continue; + }; + + out.push_str(&format!( + "| {} | {} | {} | `{}/{}` | `{:.3}` | `{:.3}` | `{:.3}` | `{:.3}` | {} | {} | {} | {} | {} |\n", + markdown::md_cell(job.job_id.as_str()), + metrics.task_run_count, + metrics.output_count, + metrics.covered_required_task_kind_count, + metrics.required_task_kind_count, + metrics.evidence_ref_coverage, + metrics.freshness_coverage, + metrics.action_rationale_coverage, + metrics.trace_coverage, + metrics.invalid_current_output_count, + metrics.untraced_output_count, + metrics.unsupported_current_output_count, + metrics.tombstone_violation_count, + metrics.source_mutation_count + )); + } + + out.push('\n'); +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/work.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/work.rs new file mode 100644 index 00000000..48c1d878 --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/work.rs @@ -0,0 +1,60 @@ +use crate::markdown::{self, RealWorldReport}; + +pub(in crate::markdown) fn render_markdown_work_continuity( + out: &mut String, + report: &RealWorldReport, +) { + let work_jobs = + report.jobs.iter().filter(|job| job.work_continuity.is_some()).collect::>(); + + if work_jobs.is_empty() { + return; + } + + out.push_str("## Work Continuity Metrics\n\n"); + out.push_str("| Job | Readbacks | Entries | Reset/Resume | Decision Rationale | Rejected Suppression | Explicit Precision | Inferred Labeling | Handoff Sources | Redaction | Janitor False Promotion | Sensitive Persistence | Journal Authority Claims |\n"); + out.push_str( + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n", + ); + + for job in work_jobs { + let Some(metrics) = &job.work_continuity else { + continue; + }; + + out.push_str(&format!( + "| {} | {} | {} | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | {} | {} |\n", + markdown::md_cell(job.job_id.as_str()), + metrics.readback_count, + metrics.entry_count, + metrics.reset_resume_success_count, + metrics.reset_resume_required_count, + metrics.reset_resume_success_rate, + metrics.decision_rationale_recalled_count, + metrics.decision_rationale_required_count, + metrics.decision_rationale_recall_rate, + metrics.rejected_option_suppressed_count, + metrics.rejected_option_required_count, + metrics.rejected_option_suppression_rate, + metrics.explicit_next_step_correct_count, + metrics.explicit_next_step_returned_count, + metrics.explicit_next_step_precision, + metrics.inferred_next_step_labeled_count, + metrics.inferred_next_step_required_count, + metrics.inferred_next_step_labeling_rate, + metrics.handoff_source_ref_covered_count, + metrics.handoff_source_ref_required_count, + metrics.handoff_source_ref_coverage, + metrics.redaction_applied_count, + metrics.redaction_required_count, + metrics.redaction_rate, + metrics.janitor_false_promotion_count, + metrics.janitor_candidate_count, + metrics.janitor_false_promotion_rate, + metrics.sensitive_marker_persistence_count, + metrics.journal_only_authority_claim_count + )); + } + + out.push('\n'); +} From e37712f59a281843aca328d4cd65c874e1618afa Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Tue, 30 Jun 2026 12:21:11 -0400 Subject: [PATCH 4/6] {"schema":"decodex/commit/1","summary":"{\"schema\":\"decodex/commit/1\",\"summary\":\"Split benchmark scoring count modules\",\"authority\":\"manual\"}","authority":"manual"} --- .../scoring/counts.rs | 276 +----------------- .../scoring/counts/declared.rs | 47 +++ .../scoring/counts/metric_failures.rs | 104 +++++++ .../scoring/counts/operator_debug.rs | 17 ++ .../scoring/counts/wrong_results.rs | 101 +++++++ 5 files changed, 282 insertions(+), 263 deletions(-) create mode 100644 apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/declared.rs create mode 100644 apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/metric_failures.rs create mode 100644 apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/operator_debug.rs create mode 100644 apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/wrong_results.rs diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts.rs index e61b0100..7b886dcc 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts.rs @@ -1,264 +1,14 @@ -use crate::scoring::{ - ConsolidationJobReport, DimensionScoreReport, EvolutionJobReport, FailureCounts, JobScoring, - MemorySummaryJobMetrics, ProactiveBriefJobMetrics, RealWorldJob, ScheduledMemoryJobMetrics, - TypedStatus, WorkContinuityJobMetrics, +mod declared; +mod metric_failures; +mod operator_debug; +mod wrong_results; + +pub(super) use self::{ + declared::score_declared_job, + metric_failures::{ + apply_memory_summary_failure_counts, apply_proactive_brief_failure_counts, + apply_scheduled_memory_failure_counts, apply_work_continuity_failure_counts, + }, + operator_debug::operator_debug_failure_counts, + wrong_results::{wrong_result_count, wrong_result_signal_count}, }; - -pub(super) fn apply_memory_summary_failure_counts( - counts: &mut FailureCounts, - metrics: Option<&MemorySummaryJobMetrics>, -) { - let Some(metrics) = metrics else { - return; - }; - - counts.memory_summary_invalid_current_entries = metrics.invalid_top_of_mind_count; - counts.memory_summary_untraced_entries = metrics.untraced_entry_count; - counts.memory_summary_missing_freshness = - metrics.entry_count.saturating_sub(metrics.freshness_marker_count); - counts.memory_summary_missing_rationale = - metrics.entry_count.saturating_sub(metrics.rationale_count); - counts.memory_summary_missing_categories = metrics.missing_required_category_count; - counts.memory_summary_unsupported_current_entries = metrics.unsupported_current_entry_count; -} - -pub(super) fn apply_proactive_brief_failure_counts( - counts: &mut FailureCounts, - metrics: Option<&ProactiveBriefJobMetrics>, -) { - let Some(metrics) = metrics else { - return; - }; - - counts.proactive_brief_invalid_current_suggestions = metrics.invalid_current_suggestion_count; - counts.proactive_brief_untraced_suggestions = metrics.untraced_suggestion_count; - counts.proactive_brief_missing_freshness = - metrics.suggestion_count.saturating_sub(metrics.freshness_marker_count); - counts.proactive_brief_missing_action_rationale = - metrics.suggestion_count.saturating_sub(metrics.action_rationale_count); - counts.proactive_brief_missing_kinds = metrics.missing_required_suggestion_kind_count; - counts.proactive_brief_unsupported_current_suggestions = - metrics.unsupported_current_suggestion_count; - counts.proactive_brief_tombstone_violations = metrics.tombstone_violation_count; -} - -pub(super) fn apply_scheduled_memory_failure_counts( - counts: &mut FailureCounts, - metrics: Option<&ScheduledMemoryJobMetrics>, -) { - let Some(metrics) = metrics else { - return; - }; - - counts.scheduled_memory_invalid_current_outputs = metrics.invalid_current_output_count; - counts.scheduled_memory_untraced_outputs = metrics.untraced_output_count; - counts.scheduled_memory_missing_freshness = - metrics.output_count.saturating_sub(metrics.freshness_marker_count); - counts.scheduled_memory_missing_action_rationale = - metrics.output_count.saturating_sub(metrics.action_rationale_count); - counts.scheduled_memory_missing_task_kinds = metrics.missing_required_task_kind_count; - counts.scheduled_memory_unsupported_current_outputs = metrics.unsupported_current_output_count; - counts.scheduled_memory_tombstone_violations = metrics.tombstone_violation_count; - counts.scheduled_memory_missing_trace = - metrics.trace_required_count.saturating_sub(metrics.trace_complete_count); - counts.source_mutations += metrics.source_mutation_count; -} - -pub(super) fn apply_work_continuity_failure_counts( - counts: &mut FailureCounts, - metrics: Option<&WorkContinuityJobMetrics>, -) { - let Some(metrics) = metrics else { - return; - }; - - counts.work_continuity_reset_resume_missing = - metrics.reset_resume_required_count.saturating_sub(metrics.reset_resume_success_count); - counts.work_continuity_decision_rationale_missing = metrics - .decision_rationale_required_count - .saturating_sub(metrics.decision_rationale_recalled_count); - counts.work_continuity_rejected_option_unsuppressed = metrics - .rejected_option_required_count - .saturating_sub(metrics.rejected_option_suppressed_count); - counts.work_continuity_rejected_option_resurrection = - metrics.rejected_option_resurrection_count; - counts.work_continuity_explicit_next_step_missing = metrics - .explicit_next_step_required_count - .saturating_sub(metrics.explicit_next_step_correct_count); - counts.work_continuity_explicit_next_step_extra = metrics - .explicit_next_step_returned_count - .saturating_sub(metrics.explicit_next_step_correct_count); - counts.work_continuity_inferred_step_unlabeled = metrics - .inferred_next_step_required_count - .saturating_sub(metrics.inferred_next_step_labeled_count); - counts.work_continuity_inferred_step_as_instruction = metrics.inferred_step_instruction_count; - counts.work_continuity_handoff_source_ref_missing = metrics - .handoff_source_ref_required_count - .saturating_sub(metrics.handoff_source_ref_covered_count); - counts.work_continuity_redaction_missing = - metrics.redaction_required_count.saturating_sub(metrics.redaction_applied_count); - counts.work_continuity_sensitive_marker_persistence = - metrics.sensitive_marker_persistence_count; - counts.work_continuity_janitor_false_promotion = metrics.janitor_false_promotion_count; - counts.work_continuity_journal_only_authority_claim = - metrics.journal_only_authority_claim_count; -} - -pub(super) fn score_declared_job( - job: &RealWorldJob, - status: TypedStatus, - trap_ids_used: Vec, - evolution: Option, - consolidation: Option, -) -> JobScoring { - JobScoring { - status, - normalized_score: 0.0, - hard_fail_hits: Vec::new(), - unsupported_claims: Vec::new(), - wrong_result_count: 0, - knowledge: None, - trap_ids_used, - dimension_scores: declared_not_encoded_dimension_scores(job), - reason: job - .encoding - .reason - .clone() - .unwrap_or_else(|| "Job did not reach a runnable scoring state.".to_string()), - evolution, - consolidation, - memory_summary: None, - proactive_brief: None, - scheduled_memory: None, - work_continuity: None, - } -} - -pub(super) fn wrong_result_count(counts: &FailureCounts) -> usize { - counts.missing_claims - + counts.forbidden_claims - + counts.missing_evidence - + counts.trap_uses - + counts.operator_debug_missing - + counts.operator_debug_raw_sql - + counts.operator_debug_trace_gaps - + counts.operator_debug_repair_unclear - + counts.conflict_detection_missing - + counts.update_rationale_missing - + counts.proposal_usefulness_failures - + counts.lineage_failures - + counts.review_action_failures - + counts.memory_summary_invalid_current_entries - + counts.memory_summary_untraced_entries - + counts.memory_summary_missing_freshness - + counts.memory_summary_missing_rationale - + counts.memory_summary_missing_categories - + counts.memory_summary_unsupported_current_entries - + counts.proactive_brief_invalid_current_suggestions - + counts.proactive_brief_untraced_suggestions - + counts.proactive_brief_missing_freshness - + counts.proactive_brief_missing_action_rationale - + counts.proactive_brief_missing_kinds - + counts.proactive_brief_unsupported_current_suggestions - + counts.proactive_brief_tombstone_violations - + counts.scheduled_memory_invalid_current_outputs - + counts.scheduled_memory_untraced_outputs - + counts.scheduled_memory_missing_freshness - + counts.scheduled_memory_missing_action_rationale - + counts.scheduled_memory_missing_task_kinds - + counts.scheduled_memory_unsupported_current_outputs - + counts.scheduled_memory_tombstone_violations - + counts.scheduled_memory_missing_trace - + work_continuity_wrong_result_count(counts) - + counts.untraced_page_sections - + counts.missed_stale_findings - + counts.rebuild_failures - + counts.page_usefulness_failures -} - -pub(super) fn operator_debug_failure_counts(job: &RealWorldJob) -> FailureCounts { - let Some(debug) = &job.operator_debug else { - return FailureCounts { - operator_debug_missing: usize::from(job.suite == "operator_debugging_ux"), - ..FailureCounts::default() - }; - }; - - FailureCounts { - operator_debug_raw_sql: usize::from(debug.raw_sql_needed), - operator_debug_trace_gaps: usize::from(debug.trace_completeness != "complete"), - operator_debug_repair_unclear: usize::from(debug.repair_action_clarity != "clear"), - ..FailureCounts::default() - } -} - -pub(super) fn wrong_result_signal_count(counts: &FailureCounts) -> usize { - counts.missing_claims - + counts.forbidden_claims - + counts.missing_evidence - + counts.trap_uses - + counts.operator_debug_missing - + counts.operator_debug_raw_sql - + counts.operator_debug_trace_gaps - + counts.operator_debug_repair_unclear - + counts.conflict_detection_missing - + counts.update_rationale_missing - + counts.proposal_usefulness_failures - + counts.lineage_failures - + counts.review_action_failures - + counts.memory_summary_invalid_current_entries - + counts.memory_summary_untraced_entries - + counts.memory_summary_missing_freshness - + counts.memory_summary_missing_rationale - + counts.memory_summary_missing_categories - + counts.memory_summary_unsupported_current_entries - + counts.proactive_brief_invalid_current_suggestions - + counts.proactive_brief_untraced_suggestions - + counts.proactive_brief_missing_freshness - + counts.proactive_brief_missing_action_rationale - + counts.proactive_brief_missing_kinds - + counts.proactive_brief_unsupported_current_suggestions - + counts.proactive_brief_tombstone_violations - + counts.scheduled_memory_invalid_current_outputs - + counts.scheduled_memory_untraced_outputs - + counts.scheduled_memory_missing_freshness - + counts.scheduled_memory_missing_action_rationale - + counts.scheduled_memory_missing_task_kinds - + counts.scheduled_memory_unsupported_current_outputs - + counts.scheduled_memory_tombstone_violations - + counts.scheduled_memory_missing_trace - + work_continuity_wrong_result_count(counts) - + counts.untraced_page_sections - + counts.missed_stale_findings - + counts.rebuild_failures - + counts.page_usefulness_failures -} - -pub(super) fn work_continuity_wrong_result_count(counts: &FailureCounts) -> usize { - counts.work_continuity_reset_resume_missing - + counts.work_continuity_decision_rationale_missing - + counts.work_continuity_rejected_option_unsuppressed - + counts.work_continuity_rejected_option_resurrection - + counts.work_continuity_explicit_next_step_missing - + counts.work_continuity_explicit_next_step_extra - + counts.work_continuity_inferred_step_unlabeled - + counts.work_continuity_inferred_step_as_instruction - + counts.work_continuity_handoff_source_ref_missing - + counts.work_continuity_redaction_missing - + counts.work_continuity_sensitive_marker_persistence - + counts.work_continuity_janitor_false_promotion - + counts.work_continuity_journal_only_authority_claim -} - -fn declared_not_encoded_dimension_scores(job: &RealWorldJob) -> Vec { - job.scoring_rubric - .dimensions - .iter() - .map(|(dimension_id, dimension)| DimensionScoreReport { - dimension: dimension_id.clone(), - score: 0.0, - max_points: dimension.max_points, - weight: dimension.weight, - }) - .collect() -} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/declared.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/declared.rs new file mode 100644 index 00000000..6d2ae868 --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/declared.rs @@ -0,0 +1,47 @@ +use crate::scoring::{ + ConsolidationJobReport, DimensionScoreReport, EvolutionJobReport, JobScoring, RealWorldJob, + TypedStatus, +}; + +pub(in crate::scoring) fn score_declared_job( + job: &RealWorldJob, + status: TypedStatus, + trap_ids_used: Vec, + evolution: Option, + consolidation: Option, +) -> JobScoring { + JobScoring { + status, + normalized_score: 0.0, + hard_fail_hits: Vec::new(), + unsupported_claims: Vec::new(), + wrong_result_count: 0, + knowledge: None, + trap_ids_used, + dimension_scores: declared_not_encoded_dimension_scores(job), + reason: job + .encoding + .reason + .clone() + .unwrap_or_else(|| "Job did not reach a runnable scoring state.".to_string()), + evolution, + consolidation, + memory_summary: None, + proactive_brief: None, + scheduled_memory: None, + work_continuity: None, + } +} + +fn declared_not_encoded_dimension_scores(job: &RealWorldJob) -> Vec { + job.scoring_rubric + .dimensions + .iter() + .map(|(dimension_id, dimension)| DimensionScoreReport { + dimension: dimension_id.clone(), + score: 0.0, + max_points: dimension.max_points, + weight: dimension.weight, + }) + .collect() +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/metric_failures.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/metric_failures.rs new file mode 100644 index 00000000..9b755c49 --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/metric_failures.rs @@ -0,0 +1,104 @@ +use crate::scoring::{ + FailureCounts, MemorySummaryJobMetrics, ProactiveBriefJobMetrics, ScheduledMemoryJobMetrics, + WorkContinuityJobMetrics, +}; + +pub(in crate::scoring) fn apply_memory_summary_failure_counts( + counts: &mut FailureCounts, + metrics: Option<&MemorySummaryJobMetrics>, +) { + let Some(metrics) = metrics else { + return; + }; + + counts.memory_summary_invalid_current_entries = metrics.invalid_top_of_mind_count; + counts.memory_summary_untraced_entries = metrics.untraced_entry_count; + counts.memory_summary_missing_freshness = + metrics.entry_count.saturating_sub(metrics.freshness_marker_count); + counts.memory_summary_missing_rationale = + metrics.entry_count.saturating_sub(metrics.rationale_count); + counts.memory_summary_missing_categories = metrics.missing_required_category_count; + counts.memory_summary_unsupported_current_entries = metrics.unsupported_current_entry_count; +} + +pub(in crate::scoring) fn apply_proactive_brief_failure_counts( + counts: &mut FailureCounts, + metrics: Option<&ProactiveBriefJobMetrics>, +) { + let Some(metrics) = metrics else { + return; + }; + + counts.proactive_brief_invalid_current_suggestions = metrics.invalid_current_suggestion_count; + counts.proactive_brief_untraced_suggestions = metrics.untraced_suggestion_count; + counts.proactive_brief_missing_freshness = + metrics.suggestion_count.saturating_sub(metrics.freshness_marker_count); + counts.proactive_brief_missing_action_rationale = + metrics.suggestion_count.saturating_sub(metrics.action_rationale_count); + counts.proactive_brief_missing_kinds = metrics.missing_required_suggestion_kind_count; + counts.proactive_brief_unsupported_current_suggestions = + metrics.unsupported_current_suggestion_count; + counts.proactive_brief_tombstone_violations = metrics.tombstone_violation_count; +} + +pub(in crate::scoring) fn apply_scheduled_memory_failure_counts( + counts: &mut FailureCounts, + metrics: Option<&ScheduledMemoryJobMetrics>, +) { + let Some(metrics) = metrics else { + return; + }; + + counts.scheduled_memory_invalid_current_outputs = metrics.invalid_current_output_count; + counts.scheduled_memory_untraced_outputs = metrics.untraced_output_count; + counts.scheduled_memory_missing_freshness = + metrics.output_count.saturating_sub(metrics.freshness_marker_count); + counts.scheduled_memory_missing_action_rationale = + metrics.output_count.saturating_sub(metrics.action_rationale_count); + counts.scheduled_memory_missing_task_kinds = metrics.missing_required_task_kind_count; + counts.scheduled_memory_unsupported_current_outputs = metrics.unsupported_current_output_count; + counts.scheduled_memory_tombstone_violations = metrics.tombstone_violation_count; + counts.scheduled_memory_missing_trace = + metrics.trace_required_count.saturating_sub(metrics.trace_complete_count); + counts.source_mutations += metrics.source_mutation_count; +} + +pub(in crate::scoring) fn apply_work_continuity_failure_counts( + counts: &mut FailureCounts, + metrics: Option<&WorkContinuityJobMetrics>, +) { + let Some(metrics) = metrics else { + return; + }; + + counts.work_continuity_reset_resume_missing = + metrics.reset_resume_required_count.saturating_sub(metrics.reset_resume_success_count); + counts.work_continuity_decision_rationale_missing = metrics + .decision_rationale_required_count + .saturating_sub(metrics.decision_rationale_recalled_count); + counts.work_continuity_rejected_option_unsuppressed = metrics + .rejected_option_required_count + .saturating_sub(metrics.rejected_option_suppressed_count); + counts.work_continuity_rejected_option_resurrection = + metrics.rejected_option_resurrection_count; + counts.work_continuity_explicit_next_step_missing = metrics + .explicit_next_step_required_count + .saturating_sub(metrics.explicit_next_step_correct_count); + counts.work_continuity_explicit_next_step_extra = metrics + .explicit_next_step_returned_count + .saturating_sub(metrics.explicit_next_step_correct_count); + counts.work_continuity_inferred_step_unlabeled = metrics + .inferred_next_step_required_count + .saturating_sub(metrics.inferred_next_step_labeled_count); + counts.work_continuity_inferred_step_as_instruction = metrics.inferred_step_instruction_count; + counts.work_continuity_handoff_source_ref_missing = metrics + .handoff_source_ref_required_count + .saturating_sub(metrics.handoff_source_ref_covered_count); + counts.work_continuity_redaction_missing = + metrics.redaction_required_count.saturating_sub(metrics.redaction_applied_count); + counts.work_continuity_sensitive_marker_persistence = + metrics.sensitive_marker_persistence_count; + counts.work_continuity_janitor_false_promotion = metrics.janitor_false_promotion_count; + counts.work_continuity_journal_only_authority_claim = + metrics.journal_only_authority_claim_count; +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/operator_debug.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/operator_debug.rs new file mode 100644 index 00000000..4066181a --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/operator_debug.rs @@ -0,0 +1,17 @@ +use crate::scoring::{FailureCounts, RealWorldJob}; + +pub(in crate::scoring) fn operator_debug_failure_counts(job: &RealWorldJob) -> FailureCounts { + let Some(debug) = &job.operator_debug else { + return FailureCounts { + operator_debug_missing: usize::from(job.suite == "operator_debugging_ux"), + ..FailureCounts::default() + }; + }; + + FailureCounts { + operator_debug_raw_sql: usize::from(debug.raw_sql_needed), + operator_debug_trace_gaps: usize::from(debug.trace_completeness != "complete"), + operator_debug_repair_unclear: usize::from(debug.repair_action_clarity != "clear"), + ..FailureCounts::default() + } +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/wrong_results.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/wrong_results.rs new file mode 100644 index 00000000..57da2dbb --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/wrong_results.rs @@ -0,0 +1,101 @@ +use crate::scoring::FailureCounts; + +pub(in crate::scoring) fn wrong_result_count(counts: &FailureCounts) -> usize { + counts.missing_claims + + counts.forbidden_claims + + counts.missing_evidence + + counts.trap_uses + + counts.operator_debug_missing + + counts.operator_debug_raw_sql + + counts.operator_debug_trace_gaps + + counts.operator_debug_repair_unclear + + counts.conflict_detection_missing + + counts.update_rationale_missing + + counts.proposal_usefulness_failures + + counts.lineage_failures + + counts.review_action_failures + + counts.memory_summary_invalid_current_entries + + counts.memory_summary_untraced_entries + + counts.memory_summary_missing_freshness + + counts.memory_summary_missing_rationale + + counts.memory_summary_missing_categories + + counts.memory_summary_unsupported_current_entries + + counts.proactive_brief_invalid_current_suggestions + + counts.proactive_brief_untraced_suggestions + + counts.proactive_brief_missing_freshness + + counts.proactive_brief_missing_action_rationale + + counts.proactive_brief_missing_kinds + + counts.proactive_brief_unsupported_current_suggestions + + counts.proactive_brief_tombstone_violations + + counts.scheduled_memory_invalid_current_outputs + + counts.scheduled_memory_untraced_outputs + + counts.scheduled_memory_missing_freshness + + counts.scheduled_memory_missing_action_rationale + + counts.scheduled_memory_missing_task_kinds + + counts.scheduled_memory_unsupported_current_outputs + + counts.scheduled_memory_tombstone_violations + + counts.scheduled_memory_missing_trace + + work_continuity_wrong_result_count(counts) + + counts.untraced_page_sections + + counts.missed_stale_findings + + counts.rebuild_failures + + counts.page_usefulness_failures +} + +pub(in crate::scoring) fn wrong_result_signal_count(counts: &FailureCounts) -> usize { + counts.missing_claims + + counts.forbidden_claims + + counts.missing_evidence + + counts.trap_uses + + counts.operator_debug_missing + + counts.operator_debug_raw_sql + + counts.operator_debug_trace_gaps + + counts.operator_debug_repair_unclear + + counts.conflict_detection_missing + + counts.update_rationale_missing + + counts.proposal_usefulness_failures + + counts.lineage_failures + + counts.review_action_failures + + counts.memory_summary_invalid_current_entries + + counts.memory_summary_untraced_entries + + counts.memory_summary_missing_freshness + + counts.memory_summary_missing_rationale + + counts.memory_summary_missing_categories + + counts.memory_summary_unsupported_current_entries + + counts.proactive_brief_invalid_current_suggestions + + counts.proactive_brief_untraced_suggestions + + counts.proactive_brief_missing_freshness + + counts.proactive_brief_missing_action_rationale + + counts.proactive_brief_missing_kinds + + counts.proactive_brief_unsupported_current_suggestions + + counts.proactive_brief_tombstone_violations + + counts.scheduled_memory_invalid_current_outputs + + counts.scheduled_memory_untraced_outputs + + counts.scheduled_memory_missing_freshness + + counts.scheduled_memory_missing_action_rationale + + counts.scheduled_memory_missing_task_kinds + + counts.scheduled_memory_unsupported_current_outputs + + counts.scheduled_memory_tombstone_violations + + counts.scheduled_memory_missing_trace + + work_continuity_wrong_result_count(counts) + + counts.untraced_page_sections + + counts.missed_stale_findings + + counts.rebuild_failures + + counts.page_usefulness_failures +} + +fn work_continuity_wrong_result_count(counts: &FailureCounts) -> usize { + counts.work_continuity_reset_resume_missing + + counts.work_continuity_decision_rationale_missing + + counts.work_continuity_rejected_option_unsuppressed + + counts.work_continuity_rejected_option_resurrection + + counts.work_continuity_explicit_next_step_missing + + counts.work_continuity_explicit_next_step_extra + + counts.work_continuity_inferred_step_unlabeled + + counts.work_continuity_inferred_step_as_instruction + + counts.work_continuity_handoff_source_ref_missing + + counts.work_continuity_redaction_missing + + counts.work_continuity_sensitive_marker_persistence + + counts.work_continuity_janitor_false_promotion + + counts.work_continuity_journal_only_authority_claim +} From d604fc994324bd6a3f565f7d48379b26125b7fd3 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Tue, 30 Jun 2026 12:25:32 -0400 Subject: [PATCH 5/6] {"schema":"decodex/commit/1","summary":"{\"schema\":\"decodex/commit/1\",\"summary\":\"Split recovery artifact validation modules\",\"authority\":\"manual\"}","authority":"manual"} --- .../validation/recovery_artifact.rs | 273 +----------------- .../validation/recovery_artifact/checks.rs | 123 ++++++++ .../validation/recovery_artifact/core.rs | 78 +++++ .../recovery_artifact/measurements.rs | 22 ++ .../validation/recovery_artifact/records.rs | 63 ++++ 5 files changed, 291 insertions(+), 268 deletions(-) create mode 100644 apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/checks.rs create mode 100644 apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/core.rs create mode 100644 apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/measurements.rs create mode 100644 apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/records.rs diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact.rs index d14b88e1..591fce3b 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact.rs @@ -1,269 +1,6 @@ -use crate::validation::{ - self, AUTHORITY_RECOVERY_DRILL_SCHEMA, AuthorityRecoveryDrillArtifact, BTreeSet, Path, - REQUIRED_AUTHORITY_PLANES, RecoveryBackupPitr, RecoveryDeadLetterHandling, - RecoveryDegradedRead, RecoveryDrillTopology, RecoveryMeasurement, RecoveryMigrationRepair, - RecoveryOutboxReplay, RecoveryQdrantRebuild, Result, eyre, -}; +mod checks; +mod core; +mod measurements; +mod records; -pub(super) fn validate_authority_recovery_drill_artifact( - drill: &AuthorityRecoveryDrillArtifact, - path: &Path, - evidence_ids: &BTreeSet, -) -> Result<()> { - if drill.drill_id.trim().is_empty() - || drill.contract_schema != AUTHORITY_RECOVERY_DRILL_SCHEMA - || drill.generated_at.trim().is_empty() - { - return Err(eyre::eyre!("{} has an incomplete authority recovery drill.", path.display())); - } - - validation::validate_optional_rfc3339(&drill.generated_at, path, drill.drill_id.as_str())?; - - validate_recovery_topology(&drill.topology, path, drill.drill_id.as_str())?; - validate_recovery_backup_pitr(&drill.backup_pitr, path, evidence_ids)?; - validate_recovery_degraded_read(&drill.degraded_read, path, evidence_ids)?; - validate_recovery_measurement("rpo", &drill.rpo, path, evidence_ids)?; - validate_recovery_measurement("rto", &drill.rto, path, evidence_ids)?; - validate_recovery_authority_record_counts(drill, path, evidence_ids)?; - validate_recovery_outbox_replay(&drill.outbox_replay, path, evidence_ids)?; - validate_recovery_qdrant_rebuild(&drill.qdrant_rebuild, path, evidence_ids)?; - validate_recovery_migration_repair(&drill.migration_repair, path, evidence_ids)?; - validate_recovery_dead_letter(&drill.dead_letter, path, evidence_ids)?; - - for injection in &drill.failure_injections { - if injection.injection_id.trim().is_empty() - || injection.target.trim().is_empty() - || injection.fault.trim().is_empty() - || injection.started_at.trim().is_empty() - || injection.completed_at.trim().is_empty() - || injection.evidence_refs.is_empty() - { - return Err(eyre::eyre!( - "{} authority recovery drill {} has an incomplete failure injection.", - path.display(), - drill.drill_id - )); - } - - validation::validate_optional_rfc3339( - &injection.started_at, - path, - injection.injection_id.as_str(), - )?; - validation::validate_optional_rfc3339( - &injection.completed_at, - path, - injection.injection_id.as_str(), - )?; - validation::ensure_known_evidence_refs(path, evidence_ids, &injection.evidence_refs)?; - } - - if drill.failure_injections.is_empty() { - return Err(eyre::eyre!( - "{} authority recovery drill {} must include failure injection evidence.", - path.display(), - drill.drill_id - )); - } - - Ok(()) -} - -fn validate_recovery_topology( - topology: &RecoveryDrillTopology, - path: &Path, - drill_id: &str, -) -> Result<()> { - if topology.authority_store.trim().is_empty() - || topology.derived_indexes.is_empty() - || topology.failover.trim().is_empty() - { - return Err(eyre::eyre!( - "{} authority recovery drill {} has incomplete topology.", - path.display(), - drill_id - )); - } - - Ok(()) -} - -fn validate_recovery_backup_pitr( - backup_pitr: &RecoveryBackupPitr, - path: &Path, - evidence_ids: &BTreeSet, -) -> Result<()> { - if backup_pitr.backup_ref.trim().is_empty() - || backup_pitr.pitr_target.trim().is_empty() - || backup_pitr.evidence_refs.is_empty() - || !backup_pitr.restored - { - return Err(eyre::eyre!("{} has incomplete backup/PITR drill evidence.", path.display())); - } - - validation::validate_optional_rfc3339( - &backup_pitr.pitr_target, - path, - backup_pitr.backup_ref.as_str(), - )?; - - validation::ensure_known_evidence_refs(path, evidence_ids, &backup_pitr.evidence_refs) -} - -fn validate_recovery_degraded_read( - degraded_read: &RecoveryDegradedRead, - path: &Path, - evidence_ids: &BTreeSet, -) -> Result<()> { - if degraded_read.unavailable_labels.is_empty() || degraded_read.evidence_refs.is_empty() { - return Err(eyre::eyre!("{} has incomplete degraded-read drill evidence.", path.display())); - } - if !degraded_read.source_of_truth_visible { - return Err(eyre::eyre!( - "{} has hidden source-of-truth records during degraded read.", - path.display() - )); - } - - validation::ensure_known_evidence_refs(path, evidence_ids, °raded_read.evidence_refs) -} - -fn validate_recovery_measurement( - label: &str, - measurement: &RecoveryMeasurement, - path: &Path, - evidence_ids: &BTreeSet, -) -> Result<()> { - if !measurement.target_seconds.is_finite() - || !measurement.measured_seconds.is_finite() - || measurement.target_seconds < 0.0 - || measurement.measured_seconds < 0.0 - || measurement.evidence_refs.is_empty() - { - return Err(eyre::eyre!("{} has invalid {label} recovery measurement.", path.display())); - } - if !validation::recovery_measurement_met(measurement) { - return Err(eyre::eyre!("{} exceeded {label} recovery target.", path.display())); - } - - validation::ensure_known_evidence_refs(path, evidence_ids, &measurement.evidence_refs) -} - -fn validate_recovery_authority_record_counts( - drill: &AuthorityRecoveryDrillArtifact, - path: &Path, - evidence_ids: &BTreeSet, -) -> Result<()> { - let present_planes = drill - .authority_record_counts - .iter() - .map(|count| count.plane.as_str()) - .collect::>(); - - for plane in REQUIRED_AUTHORITY_PLANES { - if !present_planes.contains(plane) { - return Err(eyre::eyre!( - "{} authority recovery drill {} is missing {} authority counts.", - path.display(), - drill.drill_id, - plane - )); - } - } - for count in &drill.authority_record_counts { - if count.plane.trim().is_empty() || count.evidence_refs.is_empty() { - return Err(eyre::eyre!( - "{} authority recovery drill {} has incomplete authority record counts.", - path.display(), - drill.drill_id - )); - } - if count.before_count != count.after_count { - return Err(eyre::eyre!( - "{} authority recovery drill {} lost or gained {} authority records.", - path.display(), - drill.drill_id, - count.plane - )); - } - if !count.source_refs_preserved { - return Err(eyre::eyre!( - "{} authority recovery drill {} did not preserve {} authority source refs.", - path.display(), - drill.drill_id, - count.plane - )); - } - if !count.lifecycle_history_preserved { - return Err(eyre::eyre!( - "{} authority recovery drill {} did not preserve {} authority lifecycle history.", - path.display(), - drill.drill_id, - count.plane - )); - } - - validation::ensure_known_evidence_refs(path, evidence_ids, &count.evidence_refs)?; - } - - Ok(()) -} - -fn validate_recovery_outbox_replay( - replay: &RecoveryOutboxReplay, - path: &Path, - evidence_ids: &BTreeSet, -) -> Result<()> { - if replay.evidence_refs.is_empty() || !validation::recovery_outbox_replay_succeeded(replay) { - return Err(eyre::eyre!("{} has incomplete outbox replay drill evidence.", path.display())); - } - - validation::ensure_known_evidence_refs(path, evidence_ids, &replay.evidence_refs) -} - -fn validate_recovery_qdrant_rebuild( - rebuild: &RecoveryQdrantRebuild, - path: &Path, - evidence_ids: &BTreeSet, -) -> Result<()> { - if rebuild.evidence_refs.is_empty() || !validation::recovery_qdrant_rebuild_succeeded(rebuild) { - return Err(eyre::eyre!( - "{} has incomplete Qdrant rebuild drill evidence.", - path.display() - )); - } - - validation::ensure_known_evidence_refs(path, evidence_ids, &rebuild.evidence_refs) -} - -fn validate_recovery_migration_repair( - repair: &RecoveryMigrationRepair, - path: &Path, - evidence_ids: &BTreeSet, -) -> Result<()> { - if repair.evidence_refs.is_empty() || !validation::recovery_migration_repair_succeeded(repair) { - return Err(eyre::eyre!( - "{} has incomplete migration repair drill evidence.", - path.display() - )); - } - - validation::ensure_known_evidence_refs(path, evidence_ids, &repair.evidence_refs) -} - -fn validate_recovery_dead_letter( - dead_letter: &RecoveryDeadLetterHandling, - path: &Path, - evidence_ids: &BTreeSet, -) -> Result<()> { - if dead_letter.evidence_refs.is_empty() - || !validation::recovery_dead_letter_succeeded(dead_letter) - { - return Err(eyre::eyre!( - "{} has incomplete dead-letter handling drill evidence.", - path.display() - )); - } - - validation::ensure_known_evidence_refs(path, evidence_ids, &dead_letter.evidence_refs) -} +pub(super) use self::core::validate_authority_recovery_drill_artifact; diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/checks.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/checks.rs new file mode 100644 index 00000000..fea4cde1 --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/checks.rs @@ -0,0 +1,123 @@ +use crate::validation::{ + self, BTreeSet, Path, RecoveryBackupPitr, RecoveryDeadLetterHandling, RecoveryDegradedRead, + RecoveryDrillTopology, RecoveryMigrationRepair, RecoveryOutboxReplay, RecoveryQdrantRebuild, + Result, eyre, +}; + +pub(in crate::validation::recovery_artifact) fn validate_recovery_topology( + topology: &RecoveryDrillTopology, + path: &Path, + drill_id: &str, +) -> Result<()> { + if topology.authority_store.trim().is_empty() + || topology.derived_indexes.is_empty() + || topology.failover.trim().is_empty() + { + return Err(eyre::eyre!( + "{} authority recovery drill {} has incomplete topology.", + path.display(), + drill_id + )); + } + + Ok(()) +} + +pub(in crate::validation::recovery_artifact) fn validate_recovery_backup_pitr( + backup_pitr: &RecoveryBackupPitr, + path: &Path, + evidence_ids: &BTreeSet, +) -> Result<()> { + if backup_pitr.backup_ref.trim().is_empty() + || backup_pitr.pitr_target.trim().is_empty() + || backup_pitr.evidence_refs.is_empty() + || !backup_pitr.restored + { + return Err(eyre::eyre!("{} has incomplete backup/PITR drill evidence.", path.display())); + } + + validation::validate_optional_rfc3339( + &backup_pitr.pitr_target, + path, + backup_pitr.backup_ref.as_str(), + )?; + + validation::ensure_known_evidence_refs(path, evidence_ids, &backup_pitr.evidence_refs) +} + +pub(in crate::validation::recovery_artifact) fn validate_recovery_degraded_read( + degraded_read: &RecoveryDegradedRead, + path: &Path, + evidence_ids: &BTreeSet, +) -> Result<()> { + if degraded_read.unavailable_labels.is_empty() || degraded_read.evidence_refs.is_empty() { + return Err(eyre::eyre!("{} has incomplete degraded-read drill evidence.", path.display())); + } + if !degraded_read.source_of_truth_visible { + return Err(eyre::eyre!( + "{} has hidden source-of-truth records during degraded read.", + path.display() + )); + } + + validation::ensure_known_evidence_refs(path, evidence_ids, °raded_read.evidence_refs) +} + +pub(in crate::validation::recovery_artifact) fn validate_recovery_outbox_replay( + replay: &RecoveryOutboxReplay, + path: &Path, + evidence_ids: &BTreeSet, +) -> Result<()> { + if replay.evidence_refs.is_empty() || !validation::recovery_outbox_replay_succeeded(replay) { + return Err(eyre::eyre!("{} has incomplete outbox replay drill evidence.", path.display())); + } + + validation::ensure_known_evidence_refs(path, evidence_ids, &replay.evidence_refs) +} + +pub(in crate::validation::recovery_artifact) fn validate_recovery_qdrant_rebuild( + rebuild: &RecoveryQdrantRebuild, + path: &Path, + evidence_ids: &BTreeSet, +) -> Result<()> { + if rebuild.evidence_refs.is_empty() || !validation::recovery_qdrant_rebuild_succeeded(rebuild) { + return Err(eyre::eyre!( + "{} has incomplete Qdrant rebuild drill evidence.", + path.display() + )); + } + + validation::ensure_known_evidence_refs(path, evidence_ids, &rebuild.evidence_refs) +} + +pub(in crate::validation::recovery_artifact) fn validate_recovery_migration_repair( + repair: &RecoveryMigrationRepair, + path: &Path, + evidence_ids: &BTreeSet, +) -> Result<()> { + if repair.evidence_refs.is_empty() || !validation::recovery_migration_repair_succeeded(repair) { + return Err(eyre::eyre!( + "{} has incomplete migration repair drill evidence.", + path.display() + )); + } + + validation::ensure_known_evidence_refs(path, evidence_ids, &repair.evidence_refs) +} + +pub(in crate::validation::recovery_artifact) fn validate_recovery_dead_letter( + dead_letter: &RecoveryDeadLetterHandling, + path: &Path, + evidence_ids: &BTreeSet, +) -> Result<()> { + if dead_letter.evidence_refs.is_empty() + || !validation::recovery_dead_letter_succeeded(dead_letter) + { + return Err(eyre::eyre!( + "{} has incomplete dead-letter handling drill evidence.", + path.display() + )); + } + + validation::ensure_known_evidence_refs(path, evidence_ids, &dead_letter.evidence_refs) +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/core.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/core.rs new file mode 100644 index 00000000..e59d92d5 --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/core.rs @@ -0,0 +1,78 @@ +use crate::validation::{ + self, AUTHORITY_RECOVERY_DRILL_SCHEMA, AuthorityRecoveryDrillArtifact, BTreeSet, Path, Result, + eyre, + recovery_artifact::{checks, measurements, records}, +}; + +pub(in crate::validation) fn validate_authority_recovery_drill_artifact( + drill: &AuthorityRecoveryDrillArtifact, + path: &Path, + evidence_ids: &BTreeSet, +) -> Result<()> { + if drill.drill_id.trim().is_empty() + || drill.contract_schema != AUTHORITY_RECOVERY_DRILL_SCHEMA + || drill.generated_at.trim().is_empty() + { + return Err(eyre::eyre!("{} has an incomplete authority recovery drill.", path.display())); + } + + validation::validate_optional_rfc3339(&drill.generated_at, path, drill.drill_id.as_str())?; + checks::validate_recovery_topology(&drill.topology, path, drill.drill_id.as_str())?; + checks::validate_recovery_backup_pitr(&drill.backup_pitr, path, evidence_ids)?; + checks::validate_recovery_degraded_read(&drill.degraded_read, path, evidence_ids)?; + measurements::validate_recovery_measurement("rpo", &drill.rpo, path, evidence_ids)?; + measurements::validate_recovery_measurement("rto", &drill.rto, path, evidence_ids)?; + records::validate_recovery_authority_record_counts(drill, path, evidence_ids)?; + checks::validate_recovery_outbox_replay(&drill.outbox_replay, path, evidence_ids)?; + checks::validate_recovery_qdrant_rebuild(&drill.qdrant_rebuild, path, evidence_ids)?; + checks::validate_recovery_migration_repair(&drill.migration_repair, path, evidence_ids)?; + checks::validate_recovery_dead_letter(&drill.dead_letter, path, evidence_ids)?; + + validate_recovery_failure_injections(drill, path, evidence_ids)?; + + if drill.failure_injections.is_empty() { + return Err(eyre::eyre!( + "{} authority recovery drill {} must include failure injection evidence.", + path.display(), + drill.drill_id + )); + } + + Ok(()) +} + +fn validate_recovery_failure_injections( + drill: &AuthorityRecoveryDrillArtifact, + path: &Path, + evidence_ids: &BTreeSet, +) -> Result<()> { + for injection in &drill.failure_injections { + if injection.injection_id.trim().is_empty() + || injection.target.trim().is_empty() + || injection.fault.trim().is_empty() + || injection.started_at.trim().is_empty() + || injection.completed_at.trim().is_empty() + || injection.evidence_refs.is_empty() + { + return Err(eyre::eyre!( + "{} authority recovery drill {} has an incomplete failure injection.", + path.display(), + drill.drill_id + )); + } + + validation::validate_optional_rfc3339( + &injection.started_at, + path, + injection.injection_id.as_str(), + )?; + validation::validate_optional_rfc3339( + &injection.completed_at, + path, + injection.injection_id.as_str(), + )?; + validation::ensure_known_evidence_refs(path, evidence_ids, &injection.evidence_refs)?; + } + + Ok(()) +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/measurements.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/measurements.rs new file mode 100644 index 00000000..6e3b0bb0 --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/measurements.rs @@ -0,0 +1,22 @@ +use crate::validation::{self, BTreeSet, Path, RecoveryMeasurement, Result, eyre}; + +pub(in crate::validation::recovery_artifact) fn validate_recovery_measurement( + label: &str, + measurement: &RecoveryMeasurement, + path: &Path, + evidence_ids: &BTreeSet, +) -> Result<()> { + if !measurement.target_seconds.is_finite() + || !measurement.measured_seconds.is_finite() + || measurement.target_seconds < 0.0 + || measurement.measured_seconds < 0.0 + || measurement.evidence_refs.is_empty() + { + return Err(eyre::eyre!("{} has invalid {label} recovery measurement.", path.display())); + } + if !validation::recovery_measurement_met(measurement) { + return Err(eyre::eyre!("{} exceeded {label} recovery target.", path.display())); + } + + validation::ensure_known_evidence_refs(path, evidence_ids, &measurement.evidence_refs) +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/records.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/records.rs new file mode 100644 index 00000000..3ab21f63 --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/validation/recovery_artifact/records.rs @@ -0,0 +1,63 @@ +use crate::validation::{ + self, AuthorityRecoveryDrillArtifact, BTreeSet, Path, REQUIRED_AUTHORITY_PLANES, Result, eyre, +}; + +pub(in crate::validation::recovery_artifact) fn validate_recovery_authority_record_counts( + drill: &AuthorityRecoveryDrillArtifact, + path: &Path, + evidence_ids: &BTreeSet, +) -> Result<()> { + let present_planes = drill + .authority_record_counts + .iter() + .map(|count| count.plane.as_str()) + .collect::>(); + + for plane in REQUIRED_AUTHORITY_PLANES { + if !present_planes.contains(plane) { + return Err(eyre::eyre!( + "{} authority recovery drill {} is missing {} authority counts.", + path.display(), + drill.drill_id, + plane + )); + } + } + for count in &drill.authority_record_counts { + if count.plane.trim().is_empty() || count.evidence_refs.is_empty() { + return Err(eyre::eyre!( + "{} authority recovery drill {} has incomplete authority record counts.", + path.display(), + drill.drill_id + )); + } + if count.before_count != count.after_count { + return Err(eyre::eyre!( + "{} authority recovery drill {} lost or gained {} authority records.", + path.display(), + drill.drill_id, + count.plane + )); + } + if !count.source_refs_preserved { + return Err(eyre::eyre!( + "{} authority recovery drill {} did not preserve {} authority source refs.", + path.display(), + drill.drill_id, + count.plane + )); + } + if !count.lifecycle_history_preserved { + return Err(eyre::eyre!( + "{} authority recovery drill {} did not preserve {} authority lifecycle history.", + path.display(), + drill.drill_id, + count.plane + )); + } + + validation::ensure_known_evidence_refs(path, evidence_ids, &count.evidence_refs)?; + } + + Ok(()) +} From 269011af6643b01640f77586952462cdb270ad36 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Tue, 30 Jun 2026 12:29:42 -0400 Subject: [PATCH 6/6] {"schema":"decodex/commit/1","summary":"{\"schema\":\"decodex/commit/1\",\"summary\":\"Split elf testkit helper modules\",\"authority\":\"manual\"}","authority":"manual"} --- packages/elf-testkit/src/cleanup.rs | 119 ++++++++++++ packages/elf-testkit/src/database.rs | 124 ++++++++++++ packages/elf-testkit/src/env.rs | 11 ++ packages/elf-testkit/src/harness.rs | 24 +++ packages/elf-testkit/src/lib.rs | 278 +-------------------------- 5 files changed, 287 insertions(+), 269 deletions(-) create mode 100644 packages/elf-testkit/src/cleanup.rs create mode 100644 packages/elf-testkit/src/database.rs create mode 100644 packages/elf-testkit/src/env.rs create mode 100644 packages/elf-testkit/src/harness.rs diff --git a/packages/elf-testkit/src/cleanup.rs b/packages/elf-testkit/src/cleanup.rs new file mode 100644 index 00000000..3a624fc3 --- /dev/null +++ b/packages/elf-testkit/src/cleanup.rs @@ -0,0 +1,119 @@ +use std::{collections::HashSet, time::Duration}; + +use qdrant_client::Qdrant; +use sqlx::{ + AssertSqlSafe, Connection, + postgres::{PgConnectOptions, PgConnection}, +}; +use tokio::time; + +use crate::{Error, Result}; + +const ADMIN_DATABASES: [&str; 2] = ["postgres", "template1"]; + +pub(crate) async fn connect_admin( + base_options: &PgConnectOptions, +) -> Result<(PgConnectOptions, PgConnection)> { + let mut last_err = None; + + for database in ADMIN_DATABASES { + let options = base_options.clone().database(database); + + match PgConnection::connect_with(&options).await { + Ok(conn) => return Ok((options, conn)), + Err(err) => { + last_err = Some(err); + }, + } + } + + Err(Error::Message(format!("Failed to connect to an admin database: {last_err:?}."))) +} + +pub(crate) async fn cleanup_database(name: &str, admin_options: &PgConnectOptions) -> Result<()> { + let conn = PgConnection::connect_with(admin_options).await.map_err(|err| { + Error::Message(format!("Failed to connect to admin database for cleanup: {err}.")) + })?; + let drop_sql = format!(r#"DROP DATABASE IF EXISTS "{}""#, name); + let mut conn = conn; + let _ = sqlx::query( + "\ +SELECT pg_terminate_backend(pid) +FROM pg_stat_activity +WHERE datname = $1 AND pid <> pg_backend_pid()", + ) + .bind(name) + .fetch_all(&mut conn) + .await; + + sqlx::raw_sql(AssertSqlSafe(drop_sql)) + .execute(&mut conn) + .await + .map_err(|err| Error::Message(format!("Failed to drop test database: {err}.")))?; + + Ok(()) +} + +pub(crate) async fn cleanup_qdrant_collections(collections: &[String]) -> Result<()> { + if collections.is_empty() { + return Ok(()); + } + + let Some(qdrant_url) = crate::env_qdrant_url() else { + eprintln!( + "Skipping Qdrant cleanup; set ELF_QDRANT_GRPC_URL (or ELF_QDRANT_URL) to delete test collections." + ); + + return Ok(()); + }; + let client = Qdrant::from_url(&qdrant_url) + .build() + .map_err(|err| Error::Message(format!("Failed to build Qdrant client: {err}.")))?; + let max_attempts = 6; + let mut remaining = collections.iter().cloned().collect::>(); + let mut backoff = Duration::from_millis(100); + + for attempt in 1..=max_attempts { + let existing = time::timeout(Duration::from_secs(10), client.list_collections()) + .await + .map_err(|_| Error::Message("Qdrant list_collections timed out.".to_string()))? + .map_err(|err| Error::Message(format!("Failed to list Qdrant collections: {err}.")))?; + let existing = existing.collections.into_iter().map(|c| c.name).collect::>(); + + remaining.retain(|collection| existing.contains(collection)); + + if remaining.is_empty() { + return Ok(()); + } + + for collection in remaining.iter().cloned().collect::>() { + let result = time::timeout( + Duration::from_secs(10), + client.delete_collection(collection.clone()), + ) + .await; + + match result { + Ok(Ok(_)) => {}, + Ok(Err(err)) => + if attempt == max_attempts { + return Err(Error::Message(format!( + "Failed to delete Qdrant collection {collection:?} after {attempt} attempts: {err}." + ))); + }, + Err(_) => + if attempt == max_attempts { + return Err(Error::Message(format!( + "Timed out deleting Qdrant collection {collection:?} after {attempt} attempts." + ))); + }, + } + } + + time::sleep(backoff).await; + + backoff = backoff.saturating_mul(2).min(Duration::from_secs(2)); + } + + Ok(()) +} diff --git a/packages/elf-testkit/src/database.rs b/packages/elf-testkit/src/database.rs new file mode 100644 index 00000000..a8421c8e --- /dev/null +++ b/packages/elf-testkit/src/database.rs @@ -0,0 +1,124 @@ +use std::{collections::HashSet, str::FromStr, sync::Mutex, thread}; + +use sqlx::{AssertSqlSafe, ConnectOptions, postgres::PgConnectOptions}; +use tokio::runtime::Builder; +use uuid::Uuid; + +use crate::{Error, Result, cleanup}; + +/// Ephemeral test database handle with tracked Qdrant collections for cleanup. +pub struct TestDatabase { + name: String, + dsn: String, + admin_options: PgConnectOptions, + cleaned: bool, + collections: Mutex>, +} +impl TestDatabase { + /// Creates a fresh temporary Postgres database from a base admin DSN. + pub async fn new(base_dsn: &str) -> Result { + let base_options: PgConnectOptions = PgConnectOptions::from_str(base_dsn) + .map_err(|err| Error::Message(format!("Failed to parse ELF_PG_DSN: {err}.")))?; + let (admin_options, mut admin_conn) = cleanup::connect_admin(&base_options).await?; + let name = format!("elf_test_{}", Uuid::new_v4().simple()); + let create_sql = format!(r#"CREATE DATABASE "{}""#, name); + + sqlx::raw_sql(AssertSqlSafe(create_sql)) + .execute(&mut admin_conn) + .await + .map_err(|err| Error::Message(format!("Failed to create test database: {err}.")))?; + + let dsn = base_options.clone().database(&name).to_url_lossy().to_string(); + + Ok(Self { + name, + dsn, + admin_options, + cleaned: false, + collections: Mutex::new(HashSet::new()), + }) + } + + /// Returns the DSN for the temporary test database. + pub fn dsn(&self) -> &str { + &self.dsn + } + + /// Returns the generated database name. + pub fn name(&self) -> &str { + &self.name + } + + /// Returns a unique collection prefix and tracks the related Qdrant collections. + pub fn collection_name(&self, prefix: &str) -> String { + let collection = format!("{prefix}_{}", self.name); + let docs_collection = format!("{collection}_docs"); + let mut tracked = self.collections.lock().unwrap_or_else(|err| err.into_inner()); + + tracked.insert(collection.clone()); + tracked.insert(docs_collection); + + collection + } + + /// Drops the temporary database and any tracked Qdrant collections. + pub async fn cleanup(mut self) -> Result<()> { + self.cleanup_inner().await + } + + pub(crate) async fn cleanup_inner(&mut self) -> Result<()> { + if self.cleaned { + return Ok(()); + } + + let collections = { + let tracked = self.collections.lock().unwrap_or_else(|err| err.into_inner()); + + tracked.iter().cloned().collect::>() + }; + let db_result = cleanup::cleanup_database(&self.name, &self.admin_options).await; + let qdrant_result = cleanup::cleanup_qdrant_collections(&collections).await; + + db_result?; + qdrant_result?; + + self.cleaned = true; + + Ok(()) + } +} +impl Drop for TestDatabase { + fn drop(&mut self) { + if self.cleaned { + return; + } + + let name = self.name.clone(); + let admin_options = self.admin_options.clone(); + let collections = self + .collections + .lock() + .unwrap_or_else(|err| err.into_inner()) + .iter() + .cloned() + .collect::>(); + let cleanup_thread = thread::spawn(move || { + let runtime = match Builder::new_current_thread().enable_all().build() { + Ok(runtime) => runtime, + Err(err) => { + eprintln!("Test database cleanup failed: {err}."); + + return; + }, + }; + + if let Err(err) = runtime.block_on(cleanup::cleanup_qdrant_collections(&collections)) { + eprintln!("Test Qdrant cleanup failed: {err}."); + } + if let Err(err) = runtime.block_on(cleanup::cleanup_database(&name, &admin_options)) { + eprintln!("Test database cleanup failed: {err}."); + } + }); + let _ = cleanup_thread.join(); + } +} diff --git a/packages/elf-testkit/src/env.rs b/packages/elf-testkit/src/env.rs new file mode 100644 index 00000000..cdac4205 --- /dev/null +++ b/packages/elf-testkit/src/env.rs @@ -0,0 +1,11 @@ +use std::env; + +/// Returns `ELF_PG_DSN` when it is available for integration tests. +pub fn env_dsn() -> Option { + env::var("ELF_PG_DSN").ok() +} + +/// Returns the configured Qdrant URL for integration tests. +pub fn env_qdrant_url() -> Option { + env::var("ELF_QDRANT_GRPC_URL").or_else(|_| env::var("ELF_QDRANT_URL")).ok() +} diff --git a/packages/elf-testkit/src/harness.rs b/packages/elf-testkit/src/harness.rs new file mode 100644 index 00000000..ebf5cdbf --- /dev/null +++ b/packages/elf-testkit/src/harness.rs @@ -0,0 +1,24 @@ +use std::future::Future; + +use crate::{Result, TestDatabase}; + +/// Runs an async test closure with a temporary database and guaranteed cleanup. +pub async fn with_test_db(base_dsn: &str, f: F) -> Result +where + F: FnOnce(&TestDatabase) -> Fut, + Fut: Future>, +{ + let db = TestDatabase::new(base_dsn).await?; + let result = f(&db).await; + let mut db = db; + + if let Err(err) = db.cleanup_inner().await { + eprintln!("Test database cleanup warning: {err}."); + + if result.is_ok() { + return Err(err); + } + } + + result +} diff --git a/packages/elf-testkit/src/lib.rs b/packages/elf-testkit/src/lib.rs index 591e3d29..c51de6aa 100644 --- a/packages/elf-testkit/src/lib.rs +++ b/packages/elf-testkit/src/lib.rs @@ -1,274 +1,14 @@ //! Test helpers for ephemeral Postgres databases and Qdrant collections. +mod cleanup; +mod database; +mod env; mod error; +mod harness; -pub use error::{Error, Result}; - -use std::{ - collections::HashSet, env, future::Future, str::FromStr, sync::Mutex, thread, time::Duration, -}; - -use qdrant_client::Qdrant; -use sqlx::{ - AssertSqlSafe, ConnectOptions, Connection, - postgres::{PgConnectOptions, PgConnection}, +pub use self::{ + database::TestDatabase, + env::{env_dsn, env_qdrant_url}, + error::{Error, Result}, + harness::with_test_db, }; -use tokio::{runtime::Builder, time}; -use uuid::Uuid; - -const ADMIN_DATABASES: [&str; 2] = ["postgres", "template1"]; - -/// Ephemeral test database handle with tracked Qdrant collections for cleanup. -pub struct TestDatabase { - name: String, - dsn: String, - admin_options: PgConnectOptions, - cleaned: bool, - collections: Mutex>, -} -impl TestDatabase { - /// Creates a fresh temporary Postgres database from a base admin DSN. - pub async fn new(base_dsn: &str) -> Result { - let base_options: PgConnectOptions = PgConnectOptions::from_str(base_dsn) - .map_err(|err| Error::Message(format!("Failed to parse ELF_PG_DSN: {err}.")))?; - let (admin_options, mut admin_conn) = connect_admin(&base_options).await?; - let name = format!("elf_test_{}", Uuid::new_v4().simple()); - let create_sql = format!(r#"CREATE DATABASE "{}""#, name); - - sqlx::raw_sql(AssertSqlSafe(create_sql)) - .execute(&mut admin_conn) - .await - .map_err(|err| Error::Message(format!("Failed to create test database: {err}.")))?; - - let dsn = base_options.clone().database(&name).to_url_lossy().to_string(); - - Ok(Self { - name, - dsn, - admin_options, - cleaned: false, - collections: Mutex::new(HashSet::new()), - }) - } - - /// Returns the DSN for the temporary test database. - pub fn dsn(&self) -> &str { - &self.dsn - } - - /// Returns the generated database name. - pub fn name(&self) -> &str { - &self.name - } - - /// Returns a unique collection prefix and tracks the related Qdrant collections. - pub fn collection_name(&self, prefix: &str) -> String { - let collection = format!("{prefix}_{}", self.name); - let docs_collection = format!("{collection}_docs"); - let mut tracked = self.collections.lock().unwrap_or_else(|err| err.into_inner()); - - tracked.insert(collection.clone()); - tracked.insert(docs_collection); - - collection - } - - /// Drops the temporary database and any tracked Qdrant collections. - pub async fn cleanup(mut self) -> Result<()> { - self.cleanup_inner().await - } - - async fn cleanup_inner(&mut self) -> Result<()> { - if self.cleaned { - return Ok(()); - } - - let collections = { - let tracked = self.collections.lock().unwrap_or_else(|err| err.into_inner()); - - tracked.iter().cloned().collect::>() - }; - let db_result = cleanup_database(&self.name, &self.admin_options).await; - let qdrant_result = cleanup_qdrant_collections(&collections).await; - - db_result?; - qdrant_result?; - - self.cleaned = true; - - Ok(()) - } -} -impl Drop for TestDatabase { - fn drop(&mut self) { - if self.cleaned { - return; - } - - let name = self.name.clone(); - let admin_options = self.admin_options.clone(); - let collections = self - .collections - .lock() - .unwrap_or_else(|err| err.into_inner()) - .iter() - .cloned() - .collect::>(); - let cleanup_thread = thread::spawn(move || { - let runtime = match Builder::new_current_thread().enable_all().build() { - Ok(runtime) => runtime, - Err(err) => { - eprintln!("Test database cleanup failed: {err}."); - - return; - }, - }; - - if let Err(err) = runtime.block_on(cleanup_qdrant_collections(&collections)) { - eprintln!("Test Qdrant cleanup failed: {err}."); - } - if let Err(err) = runtime.block_on(cleanup_database(&name, &admin_options)) { - eprintln!("Test database cleanup failed: {err}."); - } - }); - let _ = cleanup_thread.join(); - } -} - -/// Returns `ELF_PG_DSN` when it is available for integration tests. -pub fn env_dsn() -> Option { - env::var("ELF_PG_DSN").ok() -} - -/// Returns the configured Qdrant URL for integration tests. -pub fn env_qdrant_url() -> Option { - env::var("ELF_QDRANT_GRPC_URL").or_else(|_| env::var("ELF_QDRANT_URL")).ok() -} - -/// Runs an async test closure with a temporary database and guaranteed cleanup. -pub async fn with_test_db(base_dsn: &str, f: F) -> Result -where - F: FnOnce(&TestDatabase) -> Fut, - Fut: Future>, -{ - let db = TestDatabase::new(base_dsn).await?; - let result = f(&db).await; - let mut db = db; - - if let Err(err) = db.cleanup_inner().await { - eprintln!("Test database cleanup warning: {err}."); - - if result.is_ok() { - return Err(err); - } - } - - result -} - -async fn connect_admin( - base_options: &PgConnectOptions, -) -> Result<(PgConnectOptions, PgConnection)> { - let mut last_err = None; - - for database in ADMIN_DATABASES { - let options = base_options.clone().database(database); - - match PgConnection::connect_with(&options).await { - Ok(conn) => return Ok((options, conn)), - Err(err) => { - last_err = Some(err); - }, - } - } - - Err(Error::Message(format!("Failed to connect to an admin database: {last_err:?}."))) -} - -async fn cleanup_database(name: &str, admin_options: &PgConnectOptions) -> Result<()> { - let conn = PgConnection::connect_with(admin_options).await.map_err(|err| { - Error::Message(format!("Failed to connect to admin database for cleanup: {err}.")) - })?; - let drop_sql = format!(r#"DROP DATABASE IF EXISTS "{}""#, name); - let mut conn = conn; - let _ = sqlx::query( - "\ -SELECT pg_terminate_backend(pid) -FROM pg_stat_activity -WHERE datname = $1 AND pid <> pg_backend_pid()", - ) - .bind(name) - .fetch_all(&mut conn) - .await; - - sqlx::raw_sql(AssertSqlSafe(drop_sql)) - .execute(&mut conn) - .await - .map_err(|err| Error::Message(format!("Failed to drop test database: {err}.")))?; - - Ok(()) -} - -async fn cleanup_qdrant_collections(collections: &[String]) -> Result<()> { - if collections.is_empty() { - return Ok(()); - } - - let Some(qdrant_url) = env_qdrant_url() else { - eprintln!( - "Skipping Qdrant cleanup; set ELF_QDRANT_GRPC_URL (or ELF_QDRANT_URL) to delete test collections." - ); - - return Ok(()); - }; - let client = Qdrant::from_url(&qdrant_url) - .build() - .map_err(|err| Error::Message(format!("Failed to build Qdrant client: {err}.")))?; - let max_attempts = 6; - let mut remaining = collections.iter().cloned().collect::>(); - let mut backoff = Duration::from_millis(100); - - for attempt in 1..=max_attempts { - let existing = time::timeout(Duration::from_secs(10), client.list_collections()) - .await - .map_err(|_| Error::Message("Qdrant list_collections timed out.".to_string()))? - .map_err(|err| Error::Message(format!("Failed to list Qdrant collections: {err}.")))?; - let existing = existing.collections.into_iter().map(|c| c.name).collect::>(); - - remaining.retain(|collection| existing.contains(collection)); - - if remaining.is_empty() { - return Ok(()); - } - - for collection in remaining.iter().cloned().collect::>() { - let result = time::timeout( - Duration::from_secs(10), - client.delete_collection(collection.clone()), - ) - .await; - - match result { - Ok(Ok(_)) => {}, - Ok(Err(err)) => - if attempt == max_attempts { - return Err(Error::Message(format!( - "Failed to delete Qdrant collection {collection:?} after {attempt} attempts: {err}." - ))); - }, - Err(_) => - if attempt == max_attempts { - return Err(Error::Message(format!( - "Timed out deleting Qdrant collection {collection:?} after {attempt} attempts." - ))); - }, - } - } - - time::sleep(backoff).await; - - backoff = backoff.saturating_mul(2).min(Duration::from_secs(2)); - } - - Ok(()) -}