diff --git a/apps/elf-api/src/routes.rs b/apps/elf-api/src/routes.rs index 8722da64..e97fb83f 100644 --- a/apps/elf-api/src/routes.rs +++ b/apps/elf-api/src/routes.rs @@ -59,7 +59,8 @@ use elf_service::{ KnowledgePageLintRequest, KnowledgePageLintResponse, KnowledgePageRebuildRequest, KnowledgePageRebuildResponse, KnowledgePageResponse, KnowledgePageSearchRequest, KnowledgePageSearchResponse, KnowledgePagesListRequest, KnowledgePagesListResponse, - ListRequest, ListResponse, MemoryHistoryGetRequest, MemoryHistoryResponse, NoteFetchRequest, + ListRequest, ListResponse, MemoryCorrectionAction, MemoryCorrectionRequest, + MemoryCorrectionResponse, MemoryHistoryGetRequest, MemoryHistoryResponse, NoteFetchRequest, NoteFetchResponse, NoteProvenanceBundleResponse, NoteProvenanceGetRequest, PayloadLevel, PublishNoteRequest, QueryPlan, RankingRequestOverride, RebuildReport, RecallDebugPanelRequest, RecallDebugPanelResponse, SearchDetailsRequest, SearchDetailsResult, SearchExplainRequest, @@ -163,11 +164,12 @@ const VIEWER_HTML: &str = include_str!("../static/viewer.html"); trace_item_get, admin_graph_predicates_list, admin_graph_predicate_patch, - admin_graph_predicate_alias_add, - admin_graph_predicate_aliases_list, - admin_note_provenance_get, - admin_note_history_get, - ), + admin_graph_predicate_alias_add, + admin_graph_predicate_aliases_list, + admin_note_provenance_get, + admin_note_history_get, + admin_note_correction_apply, + ), components(schemas( AdminIngestionProfileDefaultResponseV2, AdminIngestionProfileDefaultSetBody, @@ -483,6 +485,14 @@ struct NotePatchRequest { ttl_days: Option, } +#[derive(Clone, Debug, Deserialize)] +struct AdminNoteCorrectionBody { + action: MemoryCorrectionAction, + reason: String, + source_ref: Value, + restore_version_id: Option, +} + #[derive(Clone, Debug, Deserialize)] struct AdminGraphPredicatesListQuery { scope: Option, @@ -791,6 +801,7 @@ pub fn admin_router(state: AppState) -> Router { ) .route("/v2/admin/notes/{note_id}/provenance", routing::get(admin_note_provenance_get)) .route("/v2/admin/notes/{note_id}/history", routing::get(admin_note_history_get)) + .route("/v2/admin/notes/{note_id}/corrections", routing::post(admin_note_correction_apply)) .with_state(state) .layer(DefaultBodyLimit::max(MAX_REQUEST_BYTES)) .layer(middleware::from_fn_with_state(auth_state, admin_auth_middleware)); @@ -2851,6 +2862,50 @@ async fn admin_note_history_get( Ok(Json(response)) } +#[utoipa::path( + post, + path = "/v2/admin/notes/{note_id}/corrections", + tag = "admin", + params(("note_id" = Uuid, Path, description = "Note ID.")), + request_body = Value, + responses( + (status = 200, description = "Memory correction was applied.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 404, description = "Note was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] +async fn admin_note_correction_apply( + State(state): State, + headers: HeaderMap, + Path(note_id): Path, + payload: Result, JsonRejection>, +) -> Result, ApiError> { + let ctx = RequestContext::from_headers(&headers)?; + let Json(payload) = payload.map_err(|err| { + tracing::warn!(error = %err, "Invalid request payload."); + + json_error(StatusCode::BAD_REQUEST, "INVALID_REQUEST", "Invalid request payload.", None) + })?; + let response = state + .service + .memory_correction_apply(MemoryCorrectionRequest { + tenant_id: ctx.tenant_id, + project_id: ctx.project_id, + actor_agent_id: ctx.agent_id, + note_id, + action: payload.action, + reason: payload.reason, + source_ref: payload.source_ref, + restore_version_id: payload.restore_version_id, + }) + .await?; + + Ok(Json(response)) +} + #[utoipa::path( post, path = "/v2/admin/consolidation/runs", diff --git a/apps/elf-api/tests/http.rs b/apps/elf-api/tests/http.rs index 841bebb5..f8127187 100644 --- a/apps/elf-api/tests/http.rs +++ b/apps/elf-api/tests/http.rs @@ -960,6 +960,7 @@ async fn openapi_json_route_serves_generated_contract() { assert_openapi_method(&spec, "/v2/admin/consolidation/proposals", "get"); assert_openapi_method(&spec, "/v2/admin/consolidation/proposals/{proposal_id}", "get"); assert_openapi_method(&spec, "/v2/admin/consolidation/proposals/{proposal_id}/review", "post"); + assert_openapi_method(&spec, "/v2/admin/notes/{note_id}/corrections", "post"); assert_openapi_method(&spec, "/v2/admin/knowledge/pages/rebuild", "post"); assert_openapi_method(&spec, "/v2/admin/knowledge/pages", "get"); assert_openapi_method(&spec, "/v2/admin/knowledge/pages/search", "post"); diff --git a/docs/spec/system_consolidation_proposals_v1.md b/docs/spec/system_consolidation_proposals_v1.md index 7fd25875..105d2e3b 100644 --- a/docs/spec/system_consolidation_proposals_v1.md +++ b/docs/spec/system_consolidation_proposals_v1.md @@ -6,7 +6,7 @@ resource: docs/spec/system_consolidation_proposals_v1.md status: active authority: normative owner: spec -last_verified: 2026-06-18 +last_verified: 2026-06-22 tags: - docs - spec @@ -37,10 +37,11 @@ Related inputs: Consolidation output is derived and reviewable. It must never destructively rewrite authoritative source notes, events, docs, traces, graph facts, or search traces. -The authoritative source-of-truth remains the ELF Core storage defined by +The raw evidence source-of-truth remains the ELF Core storage defined by `docs/spec/system_elf_memory_service_v2.md`. Consolidation stores proposals over -immutable input snapshots. A proposal may later create or update a derived artifact, -but source evidence remains inspectable and unchanged. +immutable input snapshots. A reviewed proposal may later create or update an approved +operational memory record or another derived artifact, but source evidence remains +inspectable and unchanged. ## Contract Schema @@ -261,8 +262,10 @@ Allowed review transitions: Terminal states are `rejected`, `applied`, and `archived`. -`applied` means the proposal has been approved and marked as applied to the derived -target. It does not mean authoritative source memory was changed. +`applied` means the proposal has been approved and marked as applied to the target. +For `create_derived_note` and `update_derived_note`, applying the proposal creates or +updates an approved operational memory note. It never mutates the raw source notes, +docs, events, traces, graph facts, or source pointers that justified the proposal. Operator review actions map to the lifecycle states: @@ -275,6 +278,29 @@ Operator review actions map to the lifecycle states: Every review transition must write an append-only audit event with proposal id, run id, reviewer agent id, action, prior state, next state, optional comment, and timestamp. +## Memory Promotion Apply Contract + +`create_derived_note` and `update_derived_note` are the only proposal apply intents +that may write to `memory_notes`. They are the Memory Inbox promotion path: + +- The proposal must first receive an explicit `apply` review action. If it starts from + `proposed`, the service records both `approve` and `apply` review events. +- The proposed payload must validate as a normal memory note payload with English text, + allowed type, allowed writable scope, and finite confidence/importance scores. +- The promoted memory note must be `active`, write `memory_note_versions`, and enqueue + the normal indexing outbox `UPSERT`. +- The promoted note `source_ref` must use `elf.memory_promotion/v1` and include + proposal id, run id, apply intent, source refs, source snapshot, lineage, + unsupported-claim flags, review actor/comment/timestamp, and any payload-level + source ref. +- The proposal `target_ref` must be updated to `elf.memory_record_ref/v1` with + `kind = "note"` and the accepted memory `id`. +- `update_derived_note` may update only an active target note identified by + `target_ref.id` or `target_ref.note_id`. + +Promoted memory records are approved operational memory. They do not replace or edit +the Source Library/raw evidence that supplied `source_refs`. + ## Service Boundary The first implementation exposes fixture-driven service flows: @@ -286,6 +312,8 @@ The first implementation exposes fixture-driven service flows: - get a consolidation proposal - transition proposal review state through `approve`, `apply`, `discard`, and `defer` actions with review-event readback +- apply reviewed `create_derived_note` and `update_derived_note` proposals into + approved memory notes with `target_ref` readback These flows must not call LLM, embedding, rerank, or external provider adapters. diff --git a/docs/spec/system_elf_memory_service_v2.md b/docs/spec/system_elf_memory_service_v2.md index 5dc0a928..aaae9204 100644 --- a/docs/spec/system_elf_memory_service_v2.md +++ b/docs/spec/system_elf_memory_service_v2.md @@ -6,7 +6,7 @@ resource: docs/spec/system_elf_memory_service_v2.md status: active authority: normative owner: spec -last_verified: 2026-06-18 +last_verified: 2026-06-22 tags: - docs - spec @@ -1107,6 +1107,11 @@ Behavior: - Review action values are `approve`, `apply`, `discard`, and `defer`. - `apply` records an approval transition before the applied transition when a proposal starts from `proposed`. +- For `create_derived_note` and `update_derived_note`, `apply` promotes the reviewed + proposal into an approved operational memory note. The promoted note source ref + must use `elf.memory_promotion/v1`, include the proposal source refs and review + actor/timestamp, write `memory_note_versions`, enqueue `UPSERT`, and update the + proposal `target_ref` to `elf.memory_record_ref/v1` with `kind = "note"`. - Every review action writes append-only review audit events returned by proposal detail readback. - `GET /v2/admin/dreaming/review-queue` exposes @@ -1120,8 +1125,37 @@ Behavior: limited to approved tag or duplicate-merge candidates with no lint or source mutation request. - These endpoints must not call LLM, embedding, rerank, or external provider adapters. -- They must not mutate authoritative source notes, docs, events, traces, graph facts, - or search traces. +- They must not mutate raw source notes, docs, events, traces, graph facts, or search + traces. Applied note proposals may create or update approved operational memory + records only through the explicit review path above. + +Admin memory correction loop: +- POST /v2/admin/notes/{note_id}/corrections + +Body: +{ + "action": "supersede|delete|restore", + "reason": "non-empty reviewer or policy reason", + "source_ref": { "...": "non-empty source or review reference" }, + "restore_version_id": "uuid|null" +} + +Behavior: +- `supersede` sets the note status to `deprecated`, writes a `DEPRECATE` + `memory_note_versions` row, stores `elf.memory_correction/v1` source-ref evidence, + and enqueues an indexing `DELETE` so normal recall suppresses the note. +- `delete` sets the note status to `deleted`, writes a `DELETE` + `memory_note_versions` row, stores `elf.memory_correction/v1` source-ref evidence, + and enqueues an indexing `DELETE`. +- `restore` restores the latest prior active snapshot from a `DELETE` or `DEPRECATE` + version, or the supplied `restore_version_id`, writes a `RESTORE` + `memory_note_versions` row, stores `elf.memory_correction/v1` source-ref evidence, + and enqueues an indexing `UPSERT`. +- Correction actions require a non-empty reason and non-empty JSON object + `source_ref`. They do not mutate raw source notes, docs, events, traces, graph + facts, or source pointers. +- Normal recall remains active-only; `deprecated` and `deleted` notes are visible + through provenance/history or explicit non-active list filters, not ordinary search. Admin recall/debug panel: - POST /v2/admin/recall-debug/panel @@ -1580,7 +1614,7 @@ Response: "events": [ { "event_id": "string", - "event_type": "add|update|ignore|reject|expire|delete|derived|applied|invalidated|related", + "event_type": "add|update|ignore|reject|expire|delete|derived|applied|superseded|restored|defer|invalidated|related", "subject_type": "note", "note_id": "uuid", "source_table": "string", @@ -1602,8 +1636,11 @@ Notes: - History events are a chronological read-only projection over durable source tables. - Ingest decisions that produce note versions should set `note_version_id` so history can link the decision to the resulting note version. -- Derived, applied, and invalidated events come from consolidation proposals and - review events that reference the note in `source_refs`. +- Derived, applied, rejected, and deferred events come from consolidation proposals + and review events that reference the note in `source_refs` or the proposal + `target_ref`. +- Superseded, deleted, and restored events come from correction-backed note version + rows. ============================================================ 15. HTTP API (PUBLIC) diff --git a/packages/elf-service/src/consolidation.rs b/packages/elf-service/src/consolidation.rs index 9ac2a32f..43307b0d 100644 --- a/packages/elf-service/src/consolidation.rs +++ b/packages/elf-service/src/consolidation.rs @@ -2,23 +2,35 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +use sqlx::{Postgres, Transaction}; use time::{Duration, OffsetDateTime}; use uuid::Uuid; -use crate::{ElfService, Error, Result}; -use elf_domain::consolidation::{ - self, CONSOLIDATION_CONTRACT_SCHEMA_V1, ConsolidationApplyIntent, ConsolidationInputRef, - ConsolidationJobPayload, ConsolidationLineage, ConsolidationMarkers, - ConsolidationProposalContract, ConsolidationProposalDiff, ConsolidationReviewAction, - ConsolidationReviewState, ConsolidationRunState, ConsolidationUnsupportedClaimFlag, - ConsolidationValidationError, +use crate::{ + ElfService, Error, InsertVersionArgs, Result, + access::{self, ORG_PROJECT_ID}, +}; +use elf_config::Config; +use elf_domain::{ + consolidation::{ + self, CONSOLIDATION_CONTRACT_SCHEMA_V1, ConsolidationApplyIntent, ConsolidationInputRef, + ConsolidationJobPayload, ConsolidationLineage, ConsolidationMarkers, + ConsolidationProposalContract, ConsolidationProposalDiff, ConsolidationReviewAction, + ConsolidationReviewState, ConsolidationRunState, ConsolidationUnsupportedClaimFlag, + ConsolidationValidationError, + }, + ttl, + writegate::{self, NoteInput}, }; use elf_storage::{ consolidation::{ ConsolidationProposalReviewEventInsert, ConsolidationProposalReviewUpdate, - ConsolidationRunJobInsert, + ConsolidationProposalTargetRefUpdate, ConsolidationRunJobInsert, + }, + models::{ + ConsolidationProposal, ConsolidationProposalReviewEvent, ConsolidationRun, MemoryNote, }, - models::{ConsolidationProposal, ConsolidationProposalReviewEvent, ConsolidationRun}, + queries, }; const DEFAULT_LIST_LIMIT: i64 = 50; @@ -369,6 +381,20 @@ impl From for ConsolidationProposalResponse { } } +#[derive(Clone, Debug, Deserialize)] +struct PromotedMemoryPayload { + #[serde(rename = "type")] + note_type: String, + text: String, + scope: Option, + key: Option, + importance: Option, + confidence: Option, + ttl_days: Option, + #[serde(default = "empty_object")] + source_ref: Value, +} + impl ElfService { /// Creates a fixture-backed consolidation run and optional proposals. pub async fn consolidation_run_create( @@ -537,8 +563,10 @@ impl ElfService { req.reviewer_agent_id.as_str(), )?; - let existing = elf_storage::consolidation::get_consolidation_proposal( - &self.db.pool, + let now = OffsetDateTime::now_utc(); + let mut tx = self.db.pool.begin().await?; + let existing = elf_storage::consolidation::lock_consolidation_proposal( + &mut *tx, req.tenant_id.as_str(), req.project_id.as_str(), req.proposal_id, @@ -553,9 +581,7 @@ impl ElfService { message: "stored proposal review_state is invalid".to_string(), } })?; - let now = OffsetDateTime::now_utc(); let steps = review_steps(current, req.review_action)?; - let mut tx = self.db.pool.begin().await?; let mut last_state = current; let mut updated = existing; @@ -598,6 +624,19 @@ impl ElfService { .ok_or_else(|| Error::NotFound { message: "consolidation proposal not found".to_string(), })?; + + if action == ConsolidationReviewAction::Apply { + updated = self + .apply_consolidation_proposal_to_memory( + &mut tx, + updated, + req.reviewer_agent_id.as_str(), + req.review_comment.as_deref(), + transition_time, + ) + .await?; + } + last_state = next_state; } @@ -617,6 +656,53 @@ impl ElfService { Ok(response) } + async fn apply_consolidation_proposal_to_memory( + &self, + tx: &mut Transaction<'_, Postgres>, + proposal: ConsolidationProposal, + reviewer_agent_id: &str, + review_comment: Option<&str>, + now: OffsetDateTime, + ) -> Result { + let note_id = match proposal.apply_intent.as_str() { + "create_derived_note" => + create_promoted_memory_note( + tx, + &proposal, + reviewer_agent_id, + review_comment, + &self.cfg, + now, + ) + .await?, + "update_derived_note" => + update_promoted_memory_note( + tx, + &proposal, + reviewer_agent_id, + review_comment, + &self.cfg, + now, + ) + .await?, + _ => return Ok(proposal), + }; + let target_ref = promoted_memory_target_ref(note_id, now); + + elf_storage::consolidation::update_consolidation_proposal_target_ref( + &mut **tx, + ConsolidationProposalTargetRefUpdate { + tenant_id: proposal.tenant_id.as_str(), + project_id: proposal.project_id.as_str(), + proposal_id: proposal.proposal_id, + target_ref: &target_ref, + now, + }, + ) + .await? + .ok_or_else(|| Error::NotFound { message: "consolidation proposal not found".to_string() }) + } + async fn consolidation_proposal_review_events( &self, tenant_id: &str, @@ -708,6 +794,135 @@ fn review_steps( Ok(steps) } +fn decode_promoted_memory_payload( + proposal: &ConsolidationProposal, +) -> Result { + let payload: PromotedMemoryPayload = serde_json::from_value(proposal.proposed_payload.clone()) + .map_err(|err| Error::InvalidRequest { + message: format!("proposed_payload is not a memory note payload: {err}"), + })?; + + if !matches!(payload.source_ref, Value::Object(_)) { + return Err(Error::InvalidRequest { + message: "proposed_payload.source_ref must be a JSON object when provided.".to_string(), + }); + } + if payload.importance.is_some_and(invalid_score) + || payload.confidence.is_some_and(invalid_score) + { + return Err(Error::InvalidRequest { + message: "proposed memory scores must be finite values in 0.0..=1.0.".to_string(), + }); + } + + Ok(payload) +} + +fn validate_promoted_memory_payload( + payload: &PromotedMemoryPayload, + effective_scope: &str, + cfg: &Config, +) -> Result<()> { + let gate = NoteInput { + note_type: payload.note_type.clone(), + scope: effective_scope.to_string(), + text: payload.text.clone(), + }; + + if let Err(code) = writegate::writegate(&gate, cfg) { + return Err(Error::InvalidRequest { + message: format!( + "proposed memory failed writegate: {}", + crate::writegate_reason_code(code) + ), + }); + } + + Ok(()) +} + +fn invalid_score(score: f32) -> bool { + !score.is_finite() || !(0.0..=1.0).contains(&score) +} + +fn target_note_id(proposal: &ConsolidationProposal) -> Result { + let raw = proposal + .target_ref + .get("id") + .or_else(|| proposal.target_ref.get("note_id")) + .and_then(Value::as_str) + .ok_or_else(|| Error::InvalidRequest { + message: "update_derived_note requires target_ref.id or target_ref.note_id." + .to_string(), + })?; + + Uuid::parse_str(raw).map_err(|err| Error::InvalidRequest { + message: format!("target_ref note id is invalid: {err}"), + }) +} + +fn normalized_optional_string(value: Option) -> Option { + value.map(|raw| raw.trim().to_string()).filter(|trimmed| !trimmed.is_empty()) +} + +fn promoted_memory_scope(payload: &PromotedMemoryPayload, default_scope: &str) -> Result { + match payload.scope.as_deref() { + Some(raw) => { + let scope = raw.trim(); + + if scope.is_empty() { + return Err(Error::InvalidRequest { + message: "proposed_payload.scope must not be empty when provided.".to_string(), + }); + } + + Ok(scope.to_string()) + }, + None => Ok(default_scope.to_string()), + } +} + +fn promoted_memory_project_id<'a>(proposal_project_id: &'a str, scope: &str) -> &'a str { + if scope == "org_shared" { ORG_PROJECT_ID } else { proposal_project_id } +} + +fn promotion_source_ref( + proposal: &ConsolidationProposal, + proposed_source_ref: &Value, + reviewer_agent_id: &str, + review_comment: Option<&str>, + now: OffsetDateTime, +) -> Value { + serde_json::json!({ + "schema": "elf.memory_promotion/v1", + "proposal_id": proposal.proposal_id, + "run_id": proposal.run_id, + "proposal_kind": proposal.proposal_kind, + "apply_intent": proposal.apply_intent, + "source_refs": proposal.source_refs, + "source_snapshot": proposal.source_snapshot, + "lineage": proposal.lineage, + "unsupported_claim_flags": proposal.unsupported_claim_flags, + "review": { + "action": "apply", + "reviewer_agent_id": reviewer_agent_id, + "review_comment": review_comment, + "applied_at": now, + }, + "proposed_source_ref": proposed_source_ref, + }) +} + +fn promoted_memory_target_ref(note_id: Uuid, now: OffsetDateTime) -> Value { + serde_json::json!({ + "schema": "elf.memory_record_ref/v1", + "kind": "note", + "id": note_id, + "status": "active", + "applied_at": now, + }) +} + fn bounded_limit(limit: Option) -> i64 { limit.map(i64::from).unwrap_or(DEFAULT_LIST_LIMIT).clamp(1, MAX_LIST_LIMIT) } @@ -733,3 +948,238 @@ fn terminal_time(state: ConsolidationRunState, now: OffsetDateTime) -> Option None, } } + +async fn create_promoted_memory_note( + tx: &mut Transaction<'_, Postgres>, + proposal: &ConsolidationProposal, + reviewer_agent_id: &str, + review_comment: Option<&str>, + cfg: &Config, + now: OffsetDateTime, +) -> Result { + let payload = decode_promoted_memory_payload(proposal)?; + let scope = promoted_memory_scope(&payload, "agent_private")?; + + validate_promoted_memory_payload(&payload, &scope, cfg)?; + + let project_id = promoted_memory_project_id(proposal.project_id.as_str(), &scope); + let note_type = payload.note_type; + let expires_at = ttl::compute_expires_at(payload.ttl_days, ¬e_type, cfg, now); + let source_ref = + promotion_source_ref(proposal, &payload.source_ref, reviewer_agent_id, review_comment, now); + let note_id = Uuid::new_v4(); + + access::ensure_active_project_scope_grant( + &mut **tx, + proposal.tenant_id.as_str(), + project_id, + scope.as_str(), + proposal.agent_id.as_str(), + ) + .await?; + + let note = MemoryNote { + note_id, + tenant_id: proposal.tenant_id.clone(), + project_id: project_id.to_string(), + agent_id: proposal.agent_id.clone(), + scope, + r#type: note_type, + key: normalized_optional_string(payload.key), + text: payload.text, + importance: payload.importance.unwrap_or(proposal.confidence), + confidence: payload.confidence.unwrap_or(proposal.confidence), + status: "active".to_string(), + created_at: now, + updated_at: now, + expires_at, + embedding_version: crate::embedding_version(cfg), + source_ref, + hit_count: 0, + last_hit_at: None, + }; + + queries::insert_note(&mut **tx, ¬e).await?; + crate::insert_version( + &mut **tx, + InsertVersionArgs { + note_id, + op: "ADD", + prev_snapshot: None, + new_snapshot: Some(crate::note_snapshot(¬e)), + reason: "consolidation_apply.create_derived_note", + actor: reviewer_agent_id, + ts: now, + }, + ) + .await?; + crate::enqueue_outbox_tx(&mut **tx, note_id, "UPSERT", ¬e.embedding_version, now).await?; + + Ok(note_id) +} + +async fn update_promoted_memory_note( + tx: &mut Transaction<'_, Postgres>, + proposal: &ConsolidationProposal, + reviewer_agent_id: &str, + review_comment: Option<&str>, + cfg: &Config, + now: OffsetDateTime, +) -> Result { + let payload = decode_promoted_memory_payload(proposal)?; + let note_id = target_note_id(proposal)?; + let mut note = sqlx::query_as::<_, MemoryNote>( + "\ +SELECT * +FROM memory_notes +WHERE note_id = $1 AND tenant_id = $2 AND project_id IN ($3, $4) +FOR UPDATE", + ) + .bind(note_id) + .bind(proposal.tenant_id.as_str()) + .bind(proposal.project_id.as_str()) + .bind(ORG_PROJECT_ID) + .fetch_optional(&mut **tx) + .await? + .ok_or_else(|| Error::InvalidRequest { + message: "Target memory note was not found.".to_string(), + })?; + + if note.status != "active" { + return Err(Error::InvalidRequest { + message: "Only active target memory can be updated by proposal apply.".to_string(), + }); + } + if note.agent_id != proposal.agent_id { + return Err(Error::InvalidRequest { + message: "Target memory note owner does not match the proposal owner.".to_string(), + }); + } + + let scope = promoted_memory_scope(&payload, note.scope.as_str())?; + + validate_promoted_memory_payload(&payload, &scope, cfg)?; + + let project_id = promoted_memory_project_id(proposal.project_id.as_str(), &scope); + let prev_snapshot = crate::note_snapshot(¬e); + + note.project_id = project_id.to_string(); + note.scope = scope; + note.r#type = payload.note_type; + note.key = normalized_optional_string(payload.key); + note.text = payload.text; + note.importance = payload.importance.unwrap_or(note.importance); + note.confidence = payload.confidence.unwrap_or(note.confidence); + + if payload.ttl_days.is_some() { + note.expires_at = ttl::compute_expires_at(payload.ttl_days, ¬e.r#type, cfg, now); + } + + note.updated_at = now; + note.source_ref = + promotion_source_ref(proposal, &payload.source_ref, reviewer_agent_id, review_comment, now); + + access::ensure_active_project_scope_grant( + &mut **tx, + note.tenant_id.as_str(), + note.project_id.as_str(), + note.scope.as_str(), + note.agent_id.as_str(), + ) + .await?; + + update_promoted_note_row(tx, ¬e).await?; + + crate::insert_version( + &mut **tx, + InsertVersionArgs { + note_id, + op: "UPDATE", + prev_snapshot: Some(prev_snapshot), + new_snapshot: Some(crate::note_snapshot(¬e)), + reason: "consolidation_apply.update_derived_note", + actor: reviewer_agent_id, + ts: now, + }, + ) + .await?; + crate::enqueue_outbox_tx(&mut **tx, note_id, "UPSERT", ¬e.embedding_version, now).await?; + + Ok(note_id) +} + +async fn update_promoted_note_row( + tx: &mut Transaction<'_, Postgres>, + note: &MemoryNote, +) -> Result<()> { + sqlx::query( + "\ +UPDATE memory_notes +SET + project_id = $1, + scope = $2, + type = $3, + key = $4, + text = $5, + importance = $6, + confidence = $7, + updated_at = $8, + expires_at = $9, + source_ref = $10 +WHERE note_id = $11", + ) + .bind(note.project_id.as_str()) + .bind(note.scope.as_str()) + .bind(note.r#type.as_str()) + .bind(note.key.as_deref()) + .bind(note.text.as_str()) + .bind(note.importance) + .bind(note.confidence) + .bind(note.updated_at) + .bind(note.expires_at) + .bind(¬e.source_ref) + .bind(note.note_id) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + fn payload_with_scope(scope: Option<&str>) -> super::PromotedMemoryPayload { + super::PromotedMemoryPayload { + note_type: "fact".to_string(), + text: "Fact: Reviewed memory promotion is explicit.".to_string(), + scope: scope.map(str::to_string), + key: None, + importance: None, + confidence: None, + ttl_days: None, + source_ref: serde_json::json!({}), + } + } + + #[test] + fn promoted_memory_scope_uses_default_and_rejects_blank_override() { + let defaulted = super::promoted_memory_scope(&payload_with_scope(None), "project_shared") + .expect("missing scope should use target default"); + + assert_eq!(defaulted, "project_shared"); + assert!( + super::promoted_memory_scope(&payload_with_scope(Some(" ")), "agent_private").is_err() + ); + } + + #[test] + fn promoted_memory_project_id_normalizes_org_shared_scope() { + assert_eq!( + super::promoted_memory_project_id("source-project", "project_shared"), + "source-project" + ); + assert_eq!( + super::promoted_memory_project_id("source-project", "org_shared"), + crate::access::ORG_PROJECT_ID + ); + } +} diff --git a/packages/elf-service/src/lib.rs b/packages/elf-service/src/lib.rs index c6910b50..012faf48 100644 --- a/packages/elf-service/src/lib.rs +++ b/packages/elf-service/src/lib.rs @@ -17,6 +17,7 @@ pub mod graph_query; pub mod graph_report; pub mod knowledge; pub mod list; +pub mod memory_corrections; pub mod notes; pub mod progressive_search; pub mod provenance; @@ -102,6 +103,9 @@ pub use self::{ KnowledgePagesListRequest, KnowledgePagesListResponse, }, list::{ListItem, ListRequest, ListResponse}, + memory_corrections::{ + MemoryCorrectionAction, MemoryCorrectionRequest, MemoryCorrectionResponse, + }, notes::{NoteFetchRequest, NoteFetchResponse}, progressive_search::{ SearchDetailsError, SearchDetailsRequest, SearchDetailsResponse, SearchDetailsResult, diff --git a/packages/elf-service/src/memory_corrections.rs b/packages/elf-service/src/memory_corrections.rs new file mode 100644 index 00000000..b95f9c20 --- /dev/null +++ b/packages/elf-service/src/memory_corrections.rs @@ -0,0 +1,693 @@ +//! Review-backed memory correction and rollback APIs. + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use sqlx::{Postgres, Transaction}; +use time::OffsetDateTime; +use uuid::Uuid; + +use crate::{ElfService, Error, InsertVersionArgs, NoteOp, Result, access::ORG_PROJECT_ID}; +use elf_config::Scopes; +use elf_storage::models::MemoryNote; + +/// Review-backed correction action for an approved memory record. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum MemoryCorrectionAction { + /// Mark the memory as superseded while retaining historical readback. + Supersede, + /// Tombstone the memory while retaining historical readback. + Delete, + /// Restore the latest prior active snapshot from the memory ledger. + Restore, +} +impl MemoryCorrectionAction { + /// Returns the canonical action string. + pub fn as_str(self) -> &'static str { + match self { + Self::Supersede => "supersede", + Self::Delete => "delete", + Self::Restore => "restore", + } + } +} + +impl ElfService { + /// Applies a review-backed memory correction and writes an audit version row. + pub async fn memory_correction_apply( + &self, + req: MemoryCorrectionRequest, + ) -> Result { + let tenant_id = req.tenant_id.trim(); + let project_id = req.project_id.trim(); + let actor_agent_id = req.actor_agent_id.trim(); + let reason = req.reason.trim(); + + validate_correction_request( + tenant_id, + project_id, + actor_agent_id, + reason, + &req.source_ref, + )?; + + let now = OffsetDateTime::now_utc(); + let mut tx = self.db.pool.begin().await?; + let mut note = + load_note_for_correction(&mut tx, req.note_id, tenant_id, project_id).await?; + + validate_write_scope(¬e, &self.cfg.scopes)?; + + let version_id = match req.action { + MemoryCorrectionAction::Supersede => + supersede_note(&mut tx, &mut note, actor_agent_id, reason, &req.source_ref, now) + .await?, + MemoryCorrectionAction::Delete => + delete_note(&mut tx, &mut note, actor_agent_id, reason, &req.source_ref, now) + .await?, + MemoryCorrectionAction::Restore => { + let embed_version = crate::embedding_version(&self.cfg); + + restore_note( + &mut tx, + &mut note, + RestoreNoteArgs { + actor_agent_id, + reason, + correction_source_ref: &req.source_ref, + restore_version_id: req.restore_version_id, + embedding_version: embed_version.as_str(), + now, + }, + ) + .await? + }, + }; + let op = match (req.action, version_id) { + (_, None) => NoteOp::None, + (MemoryCorrectionAction::Delete, Some(_)) => NoteOp::Delete, + (MemoryCorrectionAction::Supersede | MemoryCorrectionAction::Restore, Some(_)) => + NoteOp::Update, + }; + + tx.commit().await?; + + Ok(MemoryCorrectionResponse { + note_id: note.note_id, + action: req.action, + op, + status: note.status, + version_id, + }) + } +} + +/// Request payload for applying a memory correction. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct MemoryCorrectionRequest { + /// Tenant that owns the memory. + pub tenant_id: String, + /// Project that owns the memory. + pub project_id: String, + /// Reviewer or policy actor applying the correction. + pub actor_agent_id: String, + /// Identifier of the memory note being corrected. + pub note_id: Uuid, + /// Correction action to apply. + pub action: MemoryCorrectionAction, + /// Reviewer or policy reason for the correction. + pub reason: String, + /// Source reference or review record that justifies the correction. + pub source_ref: Value, + /// Optional ledger version to restore from. Defaults to the latest supersede/delete snapshot. + pub restore_version_id: Option, +} + +/// Response returned after applying a memory correction. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct MemoryCorrectionResponse { + /// Identifier of the corrected memory note. + pub note_id: Uuid, + /// Correction action that was requested. + pub action: MemoryCorrectionAction, + /// Storage operation applied to the memory record. + pub op: NoteOp, + /// Current lifecycle status after the correction. + pub status: String, + /// Version row written for this correction, when a change occurred. + pub version_id: Option, +} + +struct RestoreNoteArgs<'a> { + actor_agent_id: &'a str, + reason: &'a str, + correction_source_ref: &'a Value, + restore_version_id: Option, + embedding_version: &'a str, + now: OffsetDateTime, +} + +fn validate_correction_request( + tenant_id: &str, + project_id: &str, + actor_agent_id: &str, + reason: &str, + source_ref: &Value, +) -> Result<()> { + if tenant_id.is_empty() || project_id.is_empty() || actor_agent_id.is_empty() { + return Err(Error::InvalidRequest { + message: "tenant_id, project_id, and actor_agent_id are required.".to_string(), + }); + } + if reason.is_empty() { + return Err(Error::InvalidRequest { message: "reason must not be empty.".to_string() }); + } + if !is_non_empty_object(source_ref) { + return Err(Error::InvalidRequest { + message: "source_ref must be a non-empty JSON object.".to_string(), + }); + } + + Ok(()) +} + +fn validate_write_scope(note: &MemoryNote, scopes: &Scopes) -> Result<()> { + if !scopes.allowed.iter().any(|scope| scope == ¬e.scope) { + return Err(Error::ScopeDenied { message: "Scope is not allowed.".to_string() }); + } + + let write_allowed = match note.scope.as_str() { + "agent_private" => scopes.write_allowed.agent_private, + "project_shared" => scopes.write_allowed.project_shared, + "org_shared" => scopes.write_allowed.org_shared, + _ => false, + }; + + if write_allowed { + Ok(()) + } else { + Err(Error::ScopeDenied { message: "Scope is not writable.".to_string() }) + } +} + +fn apply_restore_snapshot( + note: &mut MemoryNote, + snapshot: &Value, + now: OffsetDateTime, +) -> Result<()> { + let status = required_string(snapshot, "status")?; + + if status != "active" { + return Err(Error::InvalidRequest { + message: "Restore snapshot must represent an active memory.".to_string(), + }); + } + + note.scope = required_string(snapshot, "scope")?; + note.r#type = required_string(snapshot, "type")?; + note.key = optional_string(snapshot, "key")?; + note.text = required_string(snapshot, "text")?; + note.importance = required_f32(snapshot, "importance")?; + note.confidence = required_f32(snapshot, "confidence")?; + note.status = status; + note.updated_at = now; + note.expires_at = optional_offset_datetime(snapshot, "expires_at")?; + + Ok(()) +} + +fn correction_source_ref_for( + action: MemoryCorrectionAction, + prior_snapshot: &Value, + correction_source_ref: &Value, + reason: &str, + actor_agent_id: &str, + now: OffsetDateTime, + restore_version_id: Option, +) -> Value { + serde_json::json!({ + "schema": "elf.memory_correction/v1", + "action": action.as_str(), + "reason": reason, + "actor_agent_id": actor_agent_id, + "ts": now, + "restore_version_id": restore_version_id, + "prior_source_ref": prior_snapshot.get("source_ref").cloned().unwrap_or_else(empty_object), + "prior_snapshot": prior_snapshot, + "correction_source_ref": correction_source_ref, + }) +} + +fn is_non_empty_object(value: &Value) -> bool { + matches!(value, Value::Object(map) if !map.is_empty()) +} + +fn required_string(snapshot: &Value, field: &'static str) -> Result { + snapshot + .get(field) + .and_then(Value::as_str) + .map(str::to_string) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| Error::InvalidRequest { + message: format!("Restore snapshot field {field} must be a non-empty string."), + }) +} + +fn optional_string(snapshot: &Value, field: &'static str) -> Result> { + match snapshot.get(field) { + None | Some(Value::Null) => Ok(None), + Some(Value::String(value)) => Ok(Some(value.clone())), + _ => Err(Error::InvalidRequest { + message: format!("Restore snapshot field {field} must be a string or null."), + }), + } +} + +fn required_f32(snapshot: &Value, field: &'static str) -> Result { + let Some(value) = snapshot.get(field).and_then(Value::as_f64) else { + return Err(Error::InvalidRequest { + message: format!("Restore snapshot field {field} must be a number."), + }); + }; + + if !value.is_finite() || value < f64::from(f32::MIN) || value > f64::from(f32::MAX) { + return Err(Error::InvalidRequest { + message: format!("Restore snapshot field {field} is out of range."), + }); + } + + Ok(value as f32) +} + +fn optional_offset_datetime( + snapshot: &Value, + field: &'static str, +) -> Result> { + let Some(value) = snapshot.get(field) else { + return Ok(None); + }; + + serde_json::from_value(value.clone()).map_err(|err| Error::InvalidRequest { + message: format!("Restore snapshot field {field} is not a valid timestamp: {err}."), + }) +} + +fn empty_object() -> Value { + Value::Object(Map::new()) +} + +async fn load_note_for_correction( + tx: &mut Transaction<'_, Postgres>, + note_id: Uuid, + tenant_id: &str, + project_id: &str, +) -> Result { + sqlx::query_as::<_, MemoryNote>( + "\ +SELECT * +FROM memory_notes +WHERE note_id = $1 AND tenant_id = $2 AND project_id IN ($3, $4) +FOR UPDATE", + ) + .bind(note_id) + .bind(tenant_id) + .bind(project_id) + .bind(ORG_PROJECT_ID) + .fetch_optional(&mut **tx) + .await? + .ok_or_else(|| Error::InvalidRequest { message: "Note not found.".to_string() }) +} + +async fn supersede_note( + tx: &mut Transaction<'_, Postgres>, + note: &mut MemoryNote, + actor_agent_id: &str, + reason: &str, + correction_source_ref: &Value, + now: OffsetDateTime, +) -> Result> { + if note.status == "deprecated" { + return Ok(None); + } + if note.status == "deleted" { + return Err(Error::InvalidRequest { + message: "Deleted memory must be restored before it can be superseded.".to_string(), + }); + } + + let prev_snapshot = crate::note_snapshot(note); + + note.status = "deprecated".to_string(); + note.updated_at = now; + note.source_ref = correction_source_ref_for( + MemoryCorrectionAction::Supersede, + &prev_snapshot, + correction_source_ref, + reason, + actor_agent_id, + now, + None, + ); + + update_note_lifecycle(tx, note).await?; + + let version_id = insert_correction_version( + tx, + note, + "DEPRECATE", + prev_snapshot, + actor_agent_id, + reason, + now, + ) + .await?; + + crate::enqueue_outbox_tx(&mut **tx, note.note_id, "DELETE", ¬e.embedding_version, now) + .await?; + + Ok(Some(version_id)) +} + +async fn delete_note( + tx: &mut Transaction<'_, Postgres>, + note: &mut MemoryNote, + actor_agent_id: &str, + reason: &str, + correction_source_ref: &Value, + now: OffsetDateTime, +) -> Result> { + if note.status == "deleted" { + return Ok(None); + } + + let prev_snapshot = crate::note_snapshot(note); + + note.status = "deleted".to_string(); + note.updated_at = now; + note.source_ref = correction_source_ref_for( + MemoryCorrectionAction::Delete, + &prev_snapshot, + correction_source_ref, + reason, + actor_agent_id, + now, + None, + ); + + update_note_lifecycle(tx, note).await?; + + let version_id = + insert_correction_version(tx, note, "DELETE", prev_snapshot, actor_agent_id, reason, now) + .await?; + + crate::enqueue_outbox_tx(&mut **tx, note.note_id, "DELETE", ¬e.embedding_version, now) + .await?; + + Ok(Some(version_id)) +} + +async fn restore_note( + tx: &mut Transaction<'_, Postgres>, + note: &mut MemoryNote, + args: RestoreNoteArgs<'_>, +) -> Result> { + if note.status == "active" { + return Ok(None); + } + + let (restore_version_id, restore_snapshot) = + load_restore_snapshot(tx, note.note_id, args.restore_version_id).await?; + let prev_snapshot = crate::note_snapshot(note); + + apply_restore_snapshot(note, &restore_snapshot, args.now)?; + + note.embedding_version = args.embedding_version.to_string(); + note.source_ref = correction_source_ref_for( + MemoryCorrectionAction::Restore, + &restore_snapshot, + args.correction_source_ref, + args.reason, + args.actor_agent_id, + args.now, + Some(restore_version_id), + ); + + update_note_restored(tx, note).await?; + + let version_id = insert_correction_version( + tx, + note, + "RESTORE", + prev_snapshot, + args.actor_agent_id, + args.reason, + args.now, + ) + .await?; + + crate::enqueue_outbox_tx(&mut **tx, note.note_id, "UPSERT", ¬e.embedding_version, args.now) + .await?; + + Ok(Some(version_id)) +} + +async fn update_note_lifecycle( + tx: &mut Transaction<'_, Postgres>, + note: &MemoryNote, +) -> Result<()> { + sqlx::query( + "\ +UPDATE memory_notes +SET status = $1, updated_at = $2, source_ref = $3 +WHERE note_id = $4", + ) + .bind(note.status.as_str()) + .bind(note.updated_at) + .bind(¬e.source_ref) + .bind(note.note_id) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +async fn update_note_restored(tx: &mut Transaction<'_, Postgres>, note: &MemoryNote) -> Result<()> { + sqlx::query( + "\ +UPDATE memory_notes +SET + scope = $1, + type = $2, + key = $3, + text = $4, + importance = $5, + confidence = $6, + status = $7, + updated_at = $8, + expires_at = $9, + embedding_version = $10, + source_ref = $11 +WHERE note_id = $12", + ) + .bind(note.scope.as_str()) + .bind(note.r#type.as_str()) + .bind(note.key.as_deref()) + .bind(note.text.as_str()) + .bind(note.importance) + .bind(note.confidence) + .bind(note.status.as_str()) + .bind(note.updated_at) + .bind(note.expires_at) + .bind(note.embedding_version.as_str()) + .bind(¬e.source_ref) + .bind(note.note_id) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +async fn insert_correction_version( + tx: &mut Transaction<'_, Postgres>, + note: &MemoryNote, + op: &str, + prev_snapshot: Value, + actor_agent_id: &str, + reason: &str, + now: OffsetDateTime, +) -> Result { + let reason = format!("memory_correction.{}: {reason}", op.to_ascii_lowercase()); + + crate::insert_version( + &mut **tx, + InsertVersionArgs { + note_id: note.note_id, + op, + prev_snapshot: Some(prev_snapshot), + new_snapshot: Some(crate::note_snapshot(note)), + reason: reason.as_str(), + actor: actor_agent_id, + ts: now, + }, + ) + .await +} + +async fn load_restore_snapshot( + tx: &mut Transaction<'_, Postgres>, + note_id: Uuid, + restore_version_id: Option, +) -> Result<(Uuid, Value)> { + let row: Option<(Uuid, Value)> = if let Some(version_id) = restore_version_id { + sqlx::query_as( + "\ +SELECT version_id, prev_snapshot +FROM memory_note_versions +WHERE note_id = $1 AND version_id = $2 AND prev_snapshot IS NOT NULL +LIMIT 1", + ) + .bind(note_id) + .bind(version_id) + .fetch_optional(&mut **tx) + .await? + } else { + sqlx::query_as( + "\ +SELECT version_id, prev_snapshot +FROM memory_note_versions +WHERE note_id = $1 + AND op IN ('DELETE', 'DEPRECATE') + AND prev_snapshot IS NOT NULL + AND prev_snapshot ->> 'status' = 'active' +ORDER BY ts DESC, version_id DESC +LIMIT 1", + ) + .bind(note_id) + .fetch_optional(&mut **tx) + .await? + }; + + row.ok_or_else(|| Error::InvalidRequest { + message: "No restorable memory snapshot was found.".to_string(), + }) +} + +#[cfg(test)] +mod tests { + use time::OffsetDateTime; + use uuid::Uuid; + + use crate::memory_corrections::{self, MemoryCorrectionAction}; + use elf_storage::models::MemoryNote; + + fn note(status: &str) -> MemoryNote { + MemoryNote { + note_id: Uuid::new_v4(), + tenant_id: "tenant".to_string(), + project_id: "project".to_string(), + agent_id: "agent".to_string(), + scope: "agent_private".to_string(), + r#type: "fact".to_string(), + key: Some("target".to_string()), + text: "Fact: Original memory.".to_string(), + importance: 0.7, + confidence: 0.9, + status: status.to_string(), + created_at: OffsetDateTime::UNIX_EPOCH, + updated_at: OffsetDateTime::UNIX_EPOCH, + expires_at: None, + embedding_version: "test:test:4".to_string(), + source_ref: serde_json::json!({ "schema": "test/source" }), + hit_count: 0, + last_hit_at: None, + } + } + + #[test] + fn correction_request_requires_non_empty_reason_and_source() { + assert!( + memory_corrections::validate_correction_request( + "tenant", + "project", + "actor", + "because", + &serde_json::json!({ + "schema": "review" + }) + ) + .is_ok() + ); + assert!( + memory_corrections::validate_correction_request( + "tenant", + "project", + "actor", + "", + &serde_json::json!({ + "schema": "review" + }) + ) + .is_err() + ); + assert!( + memory_corrections::validate_correction_request( + "tenant", + "project", + "actor", + "because", + &serde_json::json!({}) + ) + .is_err() + ); + } + + #[test] + fn restore_snapshot_must_be_active_and_restores_memory_fields() { + let snapshot = serde_json::json!({ + "scope": "project_shared", + "type": "decision", + "key": null, + "text": "Decision: Restore the reviewed memory.", + "importance": 0.8, + "confidence": 0.95, + "status": "active", + "expires_at": null + }); + let mut note = note("deleted"); + + memory_corrections::apply_restore_snapshot( + &mut note, + &snapshot, + OffsetDateTime::UNIX_EPOCH, + ) + .expect("snapshot should restore"); + + assert_eq!(note.status, "active"); + assert_eq!(note.scope, "project_shared"); + assert_eq!(note.r#type, "decision"); + assert_eq!(note.key, None); + assert_eq!(note.text, "Decision: Restore the reviewed memory."); + } + + #[test] + fn correction_source_ref_preserves_prior_and_review_evidence() { + let prior = serde_json::json!({ + "source_ref": { "schema": "prior" }, + "text": "Fact: Prior memory." + }); + let correction = memory_corrections::correction_source_ref_for( + MemoryCorrectionAction::Supersede, + &prior, + &serde_json::json!({ "schema": "review" }), + "newer source wins", + "reviewer", + OffsetDateTime::UNIX_EPOCH, + None, + ); + + assert_eq!(correction["schema"], "elf.memory_correction/v1"); + assert_eq!(correction["action"], "supersede"); + assert_eq!(correction["prior_source_ref"]["schema"], "prior"); + assert_eq!(correction["correction_source_ref"]["schema"], "review"); + } +} diff --git a/packages/elf-service/src/provenance.rs b/packages/elf-service/src/provenance.rs index c39af394..f0e925c5 100644 --- a/packages/elf-service/src/provenance.rs +++ b/packages/elf-service/src/provenance.rs @@ -748,7 +748,9 @@ fn version_event_type(op: &str, reason: &str) -> &'static str { "DELETE" if reason.contains("expire") => "expire", "DELETE" => "delete", "PUBLISH" | "UNPUBLISH" => "related", - "DEPRECATE" | "INVALIDATE" => "invalidated", + "DEPRECATE" => "superseded", + "RESTORE" => "restored", + "INVALIDATE" => "invalidated", _ => "related", } } @@ -772,7 +774,8 @@ fn decision_event_type(decision: &NoteProvenanceIngestDecision) -> &'static str fn proposal_review_event_type(action: &str) -> &'static str { match action { "apply" => "applied", - "discard" | "defer" => "invalidated", + "discard" => "reject", + "defer" => "defer", "approve" => "related", _ => "related", } @@ -784,6 +787,8 @@ fn version_summary(event_type: &str, reason: &str) -> String { "update" => format!("Note was updated by {reason}."), "delete" => format!("Note was deleted by {reason}."), "expire" => format!("Note expired through {reason}."), + "superseded" => format!("Note was superseded by {reason}."), + "restored" => format!("Note was restored by {reason}."), "invalidated" => format!("Note was invalidated by {reason}."), _ => format!("Note recorded related transition {reason}."), } @@ -953,8 +958,9 @@ async fn load_memory_history_events( let decisions = load_ingest_decisions(pool, req).await?; let versions = load_note_versions(pool, &req.tenant_id, &req.project_id, req.note_id).await?; let proposal_ref = serde_json::json!([{ "kind": "note", "id": req.note_id }]); - let proposals = load_derived_proposals_for_note(pool, req, &proposal_ref).await?; - let reviews = load_proposal_reviews_for_note(pool, req, &proposal_ref).await?; + let target_ref = serde_json::json!({ "kind": "note", "id": req.note_id }); + let proposals = load_derived_proposals_for_note(pool, req, &proposal_ref, &target_ref).await?; + let reviews = load_proposal_reviews_for_note(pool, req, &proposal_ref, &target_ref).await?; let mut decision_by_version = HashMap::new(); for decision in &decisions { @@ -1007,6 +1013,7 @@ async fn load_derived_proposals_for_note( pool: &PgPool, req: &ValidatedNoteProvenanceRequest, proposal_ref: &Value, + target_ref: &Value, ) -> Result> { let rows = sqlx::query_as::<_, NoteDerivedProposalRow>( "\ @@ -1028,13 +1035,14 @@ SELECT FROM consolidation_proposals WHERE tenant_id = $1 AND project_id = $2 - AND source_refs @> $3 + AND (source_refs @> $3 OR target_ref @> $4) ORDER BY created_at DESC, proposal_id DESC -LIMIT $4", +LIMIT $5", ) .bind(&req.tenant_id) .bind(&req.project_id) .bind(proposal_ref) + .bind(target_ref) .bind(NOTE_PROVENANCE_HISTORY_LIMIT) .fetch_all(pool) .await?; @@ -1046,6 +1054,7 @@ async fn load_proposal_reviews_for_note( pool: &PgPool, req: &ValidatedNoteProvenanceRequest, proposal_ref: &Value, + target_ref: &Value, ) -> Result> { let rows = sqlx::query_as::<_, NoteProposalReviewRow>( "\ @@ -1067,13 +1076,14 @@ JOIN consolidation_proposals proposals ON proposals.proposal_id = reviews.proposal_id WHERE reviews.tenant_id = $1 AND reviews.project_id = $2 - AND proposals.source_refs @> $3 + AND (proposals.source_refs @> $3 OR proposals.target_ref @> $4) ORDER BY reviews.created_at DESC, reviews.review_id DESC -LIMIT $4", +LIMIT $5", ) .bind(&req.tenant_id) .bind(&req.project_id) .bind(proposal_ref) + .bind(target_ref) .bind(NOTE_PROVENANCE_HISTORY_LIMIT) .fetch_all(pool) .await?; diff --git a/packages/elf-service/tests/acceptance/consolidation.rs b/packages/elf-service/tests/acceptance/consolidation.rs index 696776e0..d986ac2a 100644 --- a/packages/elf-service/tests/acceptance/consolidation.rs +++ b/packages/elf-service/tests/acceptance/consolidation.rs @@ -15,7 +15,9 @@ use elf_service::{ AddNoteInput, AddNoteRequest, ConsolidationProposalGetRequest, ConsolidationProposalInput, ConsolidationProposalReviewRequest, ConsolidationProposalsListRequest, ConsolidationProposalsListResponse, ConsolidationRunCreateRequest, - ConsolidationRunCreateResponse, ConsolidationRunGetRequest, ElfService, Providers, + ConsolidationRunCreateResponse, ConsolidationRunGetRequest, ElfService, ListRequest, + MemoryCorrectionAction, MemoryCorrectionRequest, MemoryCorrectionResponse, + MemoryHistoryGetRequest, Providers, }; use elf_storage::{db::Db, qdrant::QdrantStore}; use elf_testkit::TestDatabase; @@ -24,6 +26,7 @@ use elf_worker::worker::{self, WorkerState}; const TENANT_ID: &str = "tenant_consolidation"; const PROJECT_ID: &str = "project_consolidation"; const AGENT_ID: &str = "agent_consolidation"; +const REVIEWER_ID: &str = "reviewer_consolidation"; struct ConsolidationFixture { service: ElfService, @@ -55,6 +58,21 @@ fn lineage(source: &ConsolidationInputRef) -> ConsolidationLineage { } fn proposal_input(source: &ConsolidationInputRef, kind: &str) -> ConsolidationProposalInput { + proposal_input_with_payload( + source, + kind, + serde_json::json!({ + "type": "fact", + "text": "Fact: Consolidation proposals are derived and reviewable." + }), + ) +} + +fn proposal_input_with_payload( + source: &ConsolidationInputRef, + kind: &str, + proposed_payload: serde_json::Value, +) -> ConsolidationProposalInput { ConsolidationProposalInput { proposal_kind: kind.to_string(), apply_intent: ConsolidationApplyIntent::CreateDerivedNote, @@ -85,10 +103,7 @@ fn proposal_input(source: &ConsolidationInputRef, kind: &str) -> ConsolidationPr }), }, target_ref: serde_json::json!({}), - proposed_payload: serde_json::json!({ - "type": "fact", - "text": "Fact: Consolidation proposals are derived and reviewable." - }), + proposed_payload, } } @@ -226,6 +241,100 @@ async fn materialized_proposals( .expect("consolidation proposals should be listed") } +async fn promote_reviewed_memory(service: &ElfService) -> Uuid { + let note_id = insert_source_note( + service, + "memory_authority_source", + "Fact: Reviewed memories require source-linked approval.", + ) + .await; + let source = source_ref(note_id); + let created = + create_run_with_proposals(service, &source, vec![proposal_input(&source, "derived_note")]) + .await; + + process_consolidation_worker(service).await; + + let materialized = materialized_proposals(service, created.run.run_id).await; + let proposal_id = materialized.proposals[0].proposal_id; + let reviewed = service + .consolidation_proposal_review(ConsolidationProposalReviewRequest { + tenant_id: TENANT_ID.to_string(), + project_id: PROJECT_ID.to_string(), + reviewer_agent_id: AGENT_ID.to_string(), + proposal_id, + review_action: ConsolidationReviewAction::Apply, + review_comment: Some("Approve memory authority candidate.".to_string()), + }) + .await + .expect("review action should promote memory"); + + reviewed + .target_ref + .get("id") + .and_then(serde_json::Value::as_str) + .and_then(|value| Uuid::parse_str(value).ok()) + .expect("applied proposal should point at promoted note") +} + +async fn active_list_contains(service: &ElfService, note_id: Uuid) -> bool { + service + .list(ListRequest { + tenant_id: TENANT_ID.to_string(), + project_id: PROJECT_ID.to_string(), + agent_id: Some(AGENT_ID.to_string()), + scope: Some("agent_private".to_string()), + status: None, + r#type: None, + }) + .await + .expect("active notes should list") + .items + .iter() + .any(|item| item.note_id == note_id) +} + +async fn apply_memory_correction( + service: &ElfService, + note_id: Uuid, + action: MemoryCorrectionAction, + reason: &str, + source: &str, + restore_version_id: Option, +) -> MemoryCorrectionResponse { + service + .memory_correction_apply(MemoryCorrectionRequest { + tenant_id: TENANT_ID.to_string(), + project_id: PROJECT_ID.to_string(), + actor_agent_id: AGENT_ID.to_string(), + note_id, + action, + reason: reason.to_string(), + source_ref: serde_json::json!({ + "schema": "acceptance/review", + "source": source + }), + restore_version_id, + }) + .await + .expect("memory correction should persist") +} + +async fn memory_history_event_types(service: &ElfService, note_id: Uuid) -> Vec { + service + .memory_history_get(MemoryHistoryGetRequest { + tenant_id: TENANT_ID.to_string(), + project_id: PROJECT_ID.to_string(), + note_id, + }) + .await + .expect("promoted memory history should be readable") + .events + .into_iter() + .map(|event| event.event_type) + .collect() +} + #[tokio::test] #[ignore = "Requires external Postgres and Qdrant. Set ELF_PG_DSN and ELF_QDRANT_URL to run this test."] async fn apply_action_is_audited_without_source_rewrite() { @@ -275,7 +384,7 @@ async fn apply_action_is_audited_without_source_rewrite() { .consolidation_proposal_review(ConsolidationProposalReviewRequest { tenant_id: TENANT_ID.to_string(), project_id: PROJECT_ID.to_string(), - reviewer_agent_id: AGENT_ID.to_string(), + reviewer_agent_id: REVIEWER_ID.to_string(), proposal_id: proposal.proposal_id, review_action: ConsolidationReviewAction::Apply, review_comment: Some("Apply reviewed derived proposal.".to_string()), @@ -292,6 +401,40 @@ async fn apply_action_is_audited_without_source_rewrite() { assert_eq!(reviewed.review_events[1].from_review_state, "approved"); assert_eq!(reviewed.review_events[1].to_review_state, "applied"); + let promoted_note_id = reviewed + .target_ref + .get("id") + .and_then(serde_json::Value::as_str) + .and_then(|value| Uuid::parse_str(value).ok()) + .expect("applied proposal should point at promoted note"); + let promoted_source_ref: serde_json::Value = + sqlx::query_scalar("SELECT source_ref FROM memory_notes WHERE note_id = $1") + .bind(promoted_note_id) + .fetch_one(&service.db.pool) + .await + .expect("promoted memory source ref should be queryable"); + let promoted_status: String = + sqlx::query_scalar("SELECT status FROM memory_notes WHERE note_id = $1") + .bind(promoted_note_id) + .fetch_one(&service.db.pool) + .await + .expect("promoted memory status should be queryable"); + let promoted_agent_id: String = + sqlx::query_scalar("SELECT agent_id FROM memory_notes WHERE note_id = $1") + .bind(promoted_note_id) + .fetch_one(&service.db.pool) + .await + .expect("promoted memory owner should be queryable"); + + assert_eq!(promoted_status, "active"); + assert_eq!(promoted_agent_id, AGENT_ID); + assert_eq!(promoted_source_ref["schema"], "elf.memory_promotion/v1"); + assert_eq!( + promoted_source_ref["proposal_id"].as_str().map(str::to_string), + Some(proposal.proposal_id.to_string()) + ); + assert_eq!(promoted_source_ref["review"]["reviewer_agent_id"], REVIEWER_ID); + let stored_text: String = sqlx::query_scalar("SELECT text FROM memory_notes WHERE note_id = $1") .bind(note_id) @@ -309,6 +452,139 @@ async fn apply_action_is_audited_without_source_rewrite() { assert_eq!(version_count, 1); } +#[tokio::test] +#[ignore = "Requires external Postgres and Qdrant. Set ELF_PG_DSN and ELF_QDRANT_URL to run this test."] +async fn apply_project_shared_memory_creates_owner_grant() { + let Some(fixture) = setup_service("apply_project_shared_memory_creates_owner_grant").await + else { + return; + }; + let service = &fixture.service; + let note_id = insert_source_note( + service, + "consolidation_project_shared_source", + "Fact: Shared memory promotions must preserve project grant semantics.", + ) + .await; + let source = source_ref(note_id); + let proposal = proposal_input_with_payload( + &source, + "derived_note", + serde_json::json!({ + "type": "fact", + "scope": "project_shared", + "text": "Fact: Project-shared promoted memories keep project grants." + }), + ); + let created = create_run_with_proposals(service, &source, vec![proposal]).await; + + process_consolidation_worker(service).await; + + let materialized = materialized_proposals(service, created.run.run_id).await; + let reviewed = service + .consolidation_proposal_review(ConsolidationProposalReviewRequest { + tenant_id: TENANT_ID.to_string(), + project_id: PROJECT_ID.to_string(), + reviewer_agent_id: REVIEWER_ID.to_string(), + proposal_id: materialized.proposals[0].proposal_id, + review_action: ConsolidationReviewAction::Apply, + review_comment: Some("Apply reviewed project-shared memory.".to_string()), + }) + .await + .expect("project-shared review action should promote memory"); + let promoted_note_id = reviewed + .target_ref + .get("id") + .and_then(serde_json::Value::as_str) + .and_then(|value| Uuid::parse_str(value).ok()) + .expect("applied proposal should point at promoted note"); + let promoted: (String, String, String) = + sqlx::query_as("SELECT project_id, agent_id, scope FROM memory_notes WHERE note_id = $1") + .bind(promoted_note_id) + .fetch_one(&service.db.pool) + .await + .expect("promoted memory should be queryable"); + let grant_count: i64 = sqlx::query_scalar( + "\ +SELECT count(*) +FROM memory_space_grants +WHERE tenant_id = $1 + AND project_id = $2 + AND scope = 'project_shared' + AND space_owner_agent_id = $3 + AND grantee_kind = 'project' + AND revoked_at IS NULL", + ) + .bind(TENANT_ID) + .bind(PROJECT_ID) + .bind(AGENT_ID) + .fetch_one(&service.db.pool) + .await + .expect("project grant should be queryable"); + + assert_eq!( + promoted, + (PROJECT_ID.to_string(), AGENT_ID.to_string(), "project_shared".to_string()) + ); + assert_eq!(grant_count, 1); +} + +#[tokio::test] +#[ignore = "Requires external Postgres and Qdrant. Set ELF_PG_DSN and ELF_QDRANT_URL to run this test."] +async fn promoted_memory_corrections_suppress_and_restore_recall() { + let Some(fixture) = + setup_service("promoted_memory_corrections_suppress_and_restore_recall").await + else { + return; + }; + let service = &fixture.service; + let promoted_note_id = promote_reviewed_memory(service).await; + let superseded = apply_memory_correction( + service, + promoted_note_id, + MemoryCorrectionAction::Supersede, + "Newer reviewed source supersedes the derived memory.", + "supersede", + None, + ) + .await; + + assert_eq!(superseded.status, "deprecated"); + assert!(!active_list_contains(service, promoted_note_id).await); + + let restored = apply_memory_correction( + service, + promoted_note_id, + MemoryCorrectionAction::Restore, + "Rollback to prior approved memory after reviewer audit.", + "restore", + superseded.version_id, + ) + .await; + + assert_eq!(restored.status, "active"); + assert!(active_list_contains(service, promoted_note_id).await); + + let deleted = apply_memory_correction( + service, + promoted_note_id, + MemoryCorrectionAction::Delete, + "Reviewer removed the restored memory from normal recall.", + "delete", + None, + ) + .await; + + assert_eq!(deleted.status, "deleted"); + assert!(!active_list_contains(service, promoted_note_id).await); + + let event_types = memory_history_event_types(service, promoted_note_id).await; + + for expected in ["add", "derived", "applied", "superseded", "restored", "delete"] { + assert!(event_types.iter().any(|event_type| event_type == expected)); + } +} + #[tokio::test] #[ignore = "Requires external Postgres and Qdrant. Set ELF_PG_DSN and ELF_QDRANT_URL to run this test."] async fn discard_and_defer_actions_remain_auditable() { diff --git a/packages/elf-storage/src/consolidation.rs b/packages/elf-storage/src/consolidation.rs index d6699e36..baee8fff 100644 --- a/packages/elf-storage/src/consolidation.rs +++ b/packages/elf-storage/src/consolidation.rs @@ -97,6 +97,20 @@ pub struct ConsolidationProposalReviewUpdate<'a> { pub now: OffsetDateTime, } +/// Arguments for updating a consolidation proposal target reference. +pub struct ConsolidationProposalTargetRefUpdate<'a> { + /// Tenant that owns the proposal. + pub tenant_id: &'a str, + /// Project that owns the proposal. + pub project_id: &'a str, + /// Proposal identifier. + pub proposal_id: Uuid, + /// New target reference. + pub target_ref: &'a Value, + /// Update timestamp. + pub now: OffsetDateTime, +} + /// Arguments for inserting a consolidation proposal review event. pub struct ConsolidationProposalReviewEventInsert<'a> { /// Review event identifier. @@ -538,6 +552,57 @@ where Ok(row) } +/// Locks one consolidation proposal by tenant and proposal identifier. +pub async fn lock_consolidation_proposal<'e, E>( + executor: E, + tenant_id: &str, + project_id: &str, + proposal_id: Uuid, +) -> Result> +where + E: PgExecutor<'e>, +{ + let row = sqlx::query_as::<_, ConsolidationProposal>( + "\ +SELECT + proposal_id, + run_id, + tenant_id, + project_id, + agent_id, + contract_schema, + proposal_kind, + apply_intent, + review_state, + source_refs, + source_snapshot, + lineage, + diff, + confidence, + COALESCE(unsupported_claim_flags, '[]'::jsonb) AS unsupported_claim_flags, + COALESCE(contradiction_markers, '[]'::jsonb) AS contradiction_markers, + COALESCE(staleness_markers, '[]'::jsonb) AS staleness_markers, + COALESCE(target_ref, '{}'::jsonb) AS target_ref, + COALESCE(proposed_payload, '{}'::jsonb) AS proposed_payload, + reviewer_agent_id, + review_comment, + reviewed_at, + created_at, + updated_at +FROM consolidation_proposals +WHERE tenant_id = $1 AND project_id = $2 AND proposal_id = $3 +LIMIT 1 +FOR UPDATE", + ) + .bind(tenant_id) + .bind(project_id) + .bind(proposal_id) + .fetch_optional(executor) + .await?; + + Ok(row) +} + /// Lists consolidation proposals for one tenant and project. pub async fn list_consolidation_proposals<'e, E>( executor: E, @@ -653,6 +718,56 @@ RETURNING Ok(row) } +/// Updates one proposal target reference. +pub async fn update_consolidation_proposal_target_ref<'e, E>( + executor: E, + args: ConsolidationProposalTargetRefUpdate<'_>, +) -> Result> +where + E: PgExecutor<'e>, +{ + let row = sqlx::query_as::<_, ConsolidationProposal>( + "\ +UPDATE consolidation_proposals +SET target_ref = $1, updated_at = $2 +WHERE tenant_id = $3 AND project_id = $4 AND proposal_id = $5 +RETURNING + proposal_id, + run_id, + tenant_id, + project_id, + agent_id, + contract_schema, + proposal_kind, + apply_intent, + review_state, + source_refs, + source_snapshot, + lineage, + diff, + confidence, + COALESCE(unsupported_claim_flags, '[]'::jsonb) AS unsupported_claim_flags, + COALESCE(contradiction_markers, '[]'::jsonb) AS contradiction_markers, + COALESCE(staleness_markers, '[]'::jsonb) AS staleness_markers, + COALESCE(target_ref, '{}'::jsonb) AS target_ref, + COALESCE(proposed_payload, '{}'::jsonb) AS proposed_payload, + reviewer_agent_id, + review_comment, + reviewed_at, + created_at, + updated_at", + ) + .bind(args.target_ref) + .bind(args.now) + .bind(args.tenant_id) + .bind(args.project_id) + .bind(args.proposal_id) + .fetch_optional(executor) + .await?; + + Ok(row) +} + /// Inserts one proposal review audit event. pub async fn insert_consolidation_proposal_review_event<'e, E>( executor: E,