diff --git a/src/api/router/snapshot_chunk_map_retention_tests.rs b/src/api/router/snapshot_chunk_map_retention_tests.rs new file mode 100644 index 00000000..587e4196 --- /dev/null +++ b/src/api/router/snapshot_chunk_map_retention_tests.rs @@ -0,0 +1,1405 @@ +use sea_orm::{DatabaseConnection, DbBackend, Statement}; +use tokio::{sync::Notify, time::timeout}; + +use super::*; +use crate::{ + ceres::snapshot::{ + chunks::{ChunkMapSource, VerifiedSourceChunkMap}, + error::SnapshotErrorCode, + }, + jupiter::storage::{ + native_chunk_map::PostgresChunkMapRepository, object_storage::mock_object_storage, + }, +}; + +fn statement(sql: &str, values: [sea_orm::Value; N]) -> Statement { + Statement::from_sql_and_values(DbBackend::Postgres, sql, values) +} + +fn router(state: &MonoApiServiceState) -> Router { + Router::new().nest("/api/v2", routers(state.clone()).with_state(state.clone())) +} + +fn budgeted_chunks_app( + fixture: &Fixture, + response_budget: &Arc, + scratch_budget: &Arc, +) -> Router { + use axum::{ + extract::{Path as AxumPath, State}, + routing::post, + }; + + use crate::api::router::snapshot_router::{ + content, request::Mst2Bytes, snapshot_auth_middleware, + }; + + let response_budget = response_budget.clone(); + let scratch_budget = scratch_budget.clone(); + Router::new() + .route( + "/api/v2/snapshots/{snapshot_id}/chunks", + post( + move |state: State, + path: AxumPath, + body: Mst2Bytes| { + content::chunks_with_budgets( + state, + path, + body, + content::ChunksBudgets { + response: response_budget.clone(), + scratch: scratch_budget.clone(), + }, + ) + }, + ), + ) + .route_layer(axum::middleware::from_fn_with_state( + fixture.state.clone(), + snapshot_auth_middleware, + )) + .with_state(fixture.state.clone()) +} + +async fn count(db: &DatabaseConnection, table: &str) -> i64 { + db.query_one_raw(statement( + &format!("SELECT count(*) AS count FROM {table}"), + [], + )) + .await + .unwrap() + .unwrap() + .try_get("", "count") + .unwrap() +} + +async fn wait_count(db: &DatabaseConnection, table: &str, expected: i64) { + timeout(Duration::from_secs(10), async { + loop { + if count(db, table).await == expected { + return; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("actual durable owner release did not complete"); +} + +async fn age_unowned_candidates(db: &DatabaseConnection) { + db.execute_unprepared("UPDATE mst2_chunk_receipt_generation SET last_progress=pg_catalog.clock_timestamp()-interval '2 hours' WHERE state='LIVE'; UPDATE mst2_chunk_map_lifetime SET last_used=pg_catalog.clock_timestamp()-interval '2 hours' WHERE state='LIVE'").await.unwrap(); +} + +async fn oid_for(fixture: &Fixture, path: &str) -> String { + let handler = MonoApiService::from(&fixture.state); + let main = fixture + .state + .storage + .mono_storage() + .get_main_ref("/project") + .await + .unwrap() + .unwrap(); + let tree = handler.get_tree_by_hash(&main.ref_tree_hash).await.unwrap(); + match resolve_abs_metadata(&handler, &tree, path).await.unwrap() { + MetadataWalkOutcome::FoundFile { oid, .. } => oid, + other => panic!("retention fixture did not resolve: {other:?}"), + } +} + +#[tokio::test] +async fn shared_map_survives_other_source_retirement_while_actual_reader_is_live() { + let fixture = + Fixture::new_in_metadata_family(false, 0, &[("other".to_string(), vec![19; 4096])], true) + .await; + let original = fixture.map("/file").await; + let other = oid_for(&fixture, "/other").await; + let db = fixture.state.storage.mono_storage().get_connection(); + // This G fixture isolates independent source receipts without changing a + // Q-certified file tuple. Two exact facts consume the same actual raw + // bytes independently; map sharing never admits the second source. + db.execute_raw(statement( + "UPDATE mst2_verified_object SET size=$1,raw_sha256=$2 WHERE git_oid=$3", + [ + (fixture.raw.len() as i64).into(), + fixture.digest.to_vec().into(), + other.clone().into(), + ], + )) + .await + .unwrap(); + *fixture.counts.object_size_override.lock().unwrap() = Some(fixture.raw.len() as i64); + *fixture.counts.object_fault.lock().unwrap() = Some(bounded_objects::StreamFault { + oid: other.clone(), + kind: bounded_objects::FaultKind::Parts( + fixture + .raw + .chunks(CHUNK_SIZE as usize) + .map(Bytes::copy_from_slice) + .collect(), + ), + }); + fixture.counts.reset(); + assert_eq!(fixture.map("/other").await["map"], original["map"]); + fixture.counts.assert(1, fixture.raw.len()); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 1); + wait_count(db, "mst2_chunk_reader", 0).await; + assert_eq!(count(db, "mst2_chunk_map_source").await, 2); + assert_eq!(count(db, "mst2_chunk_map").await, 1); + assert_eq!(count(db, "mst2_chunk_map_lifetime").await, 1); + let fact = fixture + .state + .storage + .mono_storage() + .get_verified_blobs(vec![other.clone()]) + .await + .unwrap() + .remove(&other) + .unwrap(); + let source = ChunkMapSource::from_fact(fact, &other).unwrap(); + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + let objects = &fixture.state.storage.git_service.obj_storage; + let reader = repository.read(&source, objects).await.unwrap().unwrap(); + age_unowned_candidates(db).await; + repository.maintain(objects, 64).await.unwrap(); + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + assert_eq!(count(db, "mst2_chunk_map").await, 1); + assert_eq!(count(db, "mst2_chunk_map_leaf").await, 1); + assert_eq!(count(db, "mst2_chunk_map_node").await, 1); + assert_eq!(fixture.counts.receipt_deletes.load(Ordering::SeqCst), 1); + repository + .selected_page(&reader, 0) + .await + .unwrap() + .verify_chunk(&reader.map, 0, &fixture.raw[..CHUNK_SIZE as usize]) + .unwrap(); + drop(reader); + wait_count(db, "mst2_chunk_reader", 0).await; + age_unowned_candidates(db).await; + repository.maintain(objects, 64).await.unwrap(); + assert_eq!(count(db, "mst2_chunk_map_source").await, 0); + assert_eq!(count(db, "mst2_chunk_map").await, 0); + assert_eq!(fixture.counts.receipt_deletes.load(Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn pending_receipt_is_replayed_before_another_aged_live_victim() { + let fixture = Fixture::new_with_pg_config_directories_and_objects( + false, + 0, + &[("other".to_string(), vec![19; 4096])], + ) + .await; + fixture.map("/file").await; + let other = fixture.map("/other").await; + let db = fixture.state.storage.mono_storage().get_connection(); + wait_count(db, "mst2_chunk_reader", 0).await; + age_unowned_candidates(db).await; + db.execute_raw(statement( + "UPDATE mst2_chunk_receipt_generation g SET state='DELETING' WHERE EXISTS(SELECT 1 FROM mst2_chunk_map_source s WHERE s.receipt_key=g.receipt_key AND s.git_oid=$1)", + [fixture.oid.clone().into()], + )) + .await + .unwrap(); + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + repository + .maintain(&fixture.state.storage.git_service.obj_storage, 1) + .await + .unwrap(); + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + assert_eq!(count(db, "mst2_chunk_map").await, 1); + assert_eq!(fixture.counts.receipt_deletes.load(Ordering::SeqCst), 1); + fixture.counts.reset(); + assert_eq!(fixture.map("/other").await["map"], other["map"]); + fixture.counts.assert(0, 0); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn actual_http_replays_a_retiring_source_row_before_rebuilding_its_next_generation() { + let fixture = Fixture::new().await; + let original = fixture.map("/file").await; + let db = fixture.state.storage.mono_storage().get_connection(); + wait_count(db, "mst2_chunk_reader", 0).await; + let old_key: String = db + .query_one_raw(statement( + "SELECT receipt_key FROM mst2_chunk_map_source", + [], + )) + .await + .unwrap() + .unwrap() + .try_get("", "receipt_key") + .unwrap(); + db.execute_unprepared( + "UPDATE mst2_chunk_receipt_generation SET state='DELETING' WHERE state='LIVE'", + ) + .await + .unwrap(); + // The source row is deliberately still present; inline replay cannot + // depend on the source-index delete having already finished. + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + fixture.counts.reset(); + assert_eq!(fixture.map("/file").await["map"], original["map"]); + fixture.counts.assert(1, fixture.raw.len()); + assert_eq!(fixture.counts.receipt_deletes.load(Ordering::SeqCst), 1); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 1); + let new_key: String = db + .query_one_raw(statement( + "SELECT receipt_key FROM mst2_chunk_map_source", + [], + )) + .await + .unwrap() + .unwrap() + .try_get("", "receipt_key") + .unwrap(); + assert_ne!(old_key, new_key); + assert_eq!(count(db, "mst2_chunk_map").await, 1); + fixture.counts.reset(); + fixture.map("/alias").await; + fixture.counts.assert(0, 0); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn actual_warm_body_owner_cancels_never_returning_open_and_next_without_backend_release() { + use crate::ceres::snapshot::content_budget::{MemoryBudget, RANGE_WORK_BYTES}; + + for raw_route in [true, false] { + for stage in 0..3 { + let held_open = stage == 0; + let complete_without_eof = stage == 2; + let fixture = Fixture::new().await; + let map = fixture.map("/file").await; + let db = fixture.state.storage.mono_storage().get_connection(); + wait_count(db, "mst2_chunk_reader", 0).await; + fixture.counts.reset(); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let drops = Arc::new(AtomicUsize::new(0)); + let tail_polls = Arc::new(AtomicUsize::new(0)); + if held_open { + let holds = Some((entered.clone(), release.clone(), drops.clone())); + if raw_route { + *fixture.counts.whole_open_holds.lock().unwrap() = holds; + } else { + *fixture.counts.range_open_holds.lock().unwrap() = holds; + } + } else { + *fixture.counts.object_fault.lock().unwrap() = Some(bounded_objects::StreamFault { + oid: fixture.oid.clone(), + kind: bounded_objects::FaultKind::HeldFragment { + prefix: if complete_without_eof { + Bytes::copy_from_slice(if raw_route { + &fixture.raw + } else { + &fixture.raw[..CHUNK_SIZE as usize] + }) + } else { + Bytes::new() + }, + fragment: None, + entered: entered.clone(), + release: release.clone(), + drops: drops.clone(), + tail_polls: tail_polls.clone(), + }, + }); + } + let response_bytes = if raw_route { + CHUNK_SIZE as usize + } else { + CHUNK_SIZE as usize + 2048 + }; + let response_budget = MemoryBudget::new(response_bytes); + let scratch_budget = MemoryBudget::new(RANGE_WORK_BYTES); + let app = if raw_route { + super::raw_blob::budgeted_app(&fixture, &response_budget, &scratch_budget) + } else { + budgeted_chunks_app(&fixture, &response_budget, &scratch_budget) + }; + let request = if raw_route { + fixture.request("GET", "blob?path=/file", Body::empty()) + } else { + fixture.request( + "POST", + "chunks", + Body::from( + fixture + .chunk_body("/file", map["map"]["map_id"].as_str().unwrap(), "0") + .to_string(), + ), + ) + }; + let expected_raw = fixture.raw.clone(); + let mut task = tokio::spawn(async move { + let response = app.oneshot(request).await.unwrap(); + if raw_route && !held_open { + assert_eq!(response.status(), 200); + let mut body = response.into_body().into_data_stream(); + if complete_without_eof { + let first = body.next().await.unwrap().unwrap(); + assert_eq!(first.as_ref(), &expected_raw[..CHUNK_SIZE as usize]); + drop(first); + } + assert!(body.next().await.unwrap().is_err()); + assert!(body.next().await.is_none()); + } else { + error(response, 410, "LEASE_EXPIRED", false).await; + } + }); + timeout(Duration::from_secs(10), entered.notified()) + .await + .unwrap(); + assert_eq!(count(db, "mst2_chunk_reader").await, 1); + assert_eq!(scratch_budget.used(), RANGE_WORK_BYTES); + assert_eq!( + response_budget.used(), + if raw_route && complete_without_eof { + fixture.raw.len() - CHUNK_SIZE as usize + } else { + response_bytes + } + ); + db.execute_unprepared("UPDATE mst2_chunk_reader SET deadline=pg_catalog.clock_timestamp()+interval '20 milliseconds'").await.unwrap(); + // Never release the backend and never cancel the caller. The + // request must observe actual expiry during its pending await. + let finished = timeout(Duration::from_secs(12), &mut task).await; + if finished.is_err() { + task.abort(); + let _ = task.await; + panic!("actual body owner did not cancel its permanently held backend"); + } + finished.unwrap().unwrap(); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert_eq!(tail_polls.load(Ordering::SeqCst), 0); + let consumed = if complete_without_eof { + if raw_route { + fixture.raw.len() + } else { + CHUNK_SIZE as usize + } + } else { + 0 + }; + assert_eq!(fixture.counts.bytes.load(Ordering::SeqCst), consumed); + assert_eq!(scratch_budget.used(), 0); + assert_eq!(response_budget.used(), 0); + if raw_route { + fixture.counts.assert(1, consumed); + } else { + assert_eq!(fixture.counts.whole.load(Ordering::SeqCst), 0); + assert_eq!(fixture.counts.range.load(Ordering::SeqCst), 1); + } + wait_count(db, "mst2_chunk_reader", 0).await; + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + drop(release); + } + } +} + +#[tokio::test] +async fn actual_cold_builder_uses_remaining_database_deadline_for_open_and_next_and_refunds_credit() +{ + use crate::ceres::snapshot::content_budget::MemoryBudget; + + for stage in 0..3 { + let held_open = stage == 0; + let complete_without_eof = stage == 2; + let fixture = Fixture::new().await; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + let source = ChunkMapSource::from_fact(fixture.fact().await, &fixture.oid).unwrap(); + let admission = repository + .admit_install(&source, &fixture.state.storage.git_service.obj_storage) + .await + .unwrap(); + let db = fixture.state.storage.mono_storage().get_connection(); + db.execute_unprepared("UPDATE mst2_chunk_receipt_generation SET deadline=pg_catalog.clock_timestamp()+interval '3 seconds' WHERE state='RESERVED'").await.unwrap(); + admission.test_check_next_owner_operation().await; + fixture.counts.reset(); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let drops = Arc::new(AtomicUsize::new(0)); + let tail_polls = Arc::new(AtomicUsize::new(0)); + if held_open { + *fixture.counts.whole_open_holds.lock().unwrap() = + Some((entered.clone(), release.clone(), drops.clone())); + } else { + *fixture.counts.object_fault.lock().unwrap() = Some(bounded_objects::StreamFault { + oid: fixture.oid.clone(), + kind: bounded_objects::FaultKind::HeldFragment { + prefix: if complete_without_eof { + Bytes::copy_from_slice(&fixture.raw) + } else { + Bytes::new() + }, + fragment: None, + entered: entered.clone(), + release: release.clone(), + drops: drops.clone(), + tail_polls: tail_polls.clone(), + }, + }); + } + let budget = MemoryBudget::new(512 * 1024 * 1024); + let task_budget = budget.clone(); + let handler = MonoApiService::from(&fixture.state); + let mut task = tokio::spawn(async move { + VerifiedSourceChunkMap::verify(&handler, source, &task_budget, &admission).await + }); + timeout(Duration::from_secs(2), entered.notified()) + .await + .unwrap(); + assert!(budget.used() > 0); + let result = timeout(Duration::from_secs(5), &mut task).await; + if result.is_err() { + task.abort(); + let _ = task.await; + panic!("cold source ignored its actual remaining database deadline"); + } + let result = result.unwrap().unwrap(); + let error = match result { + Ok(_) => panic!("incomplete cold source produced trust"), + Err(error) => error, + }; + assert_eq!(error.code, SnapshotErrorCode::LeaseExpired); + assert_eq!(budget.used(), 0); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert_eq!(tail_polls.load(Ordering::SeqCst), 0); + fixture.counts.assert( + 1, + if complete_without_eof { + fixture.raw.len() + } else { + 0 + }, + ); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 0); + assert_eq!(count(db, "mst2_chunk_map").await, 0); + assert_eq!(count(db, "mst2_chunk_map_source").await, 0); + drop(release); + } +} + +#[tokio::test] +async fn actual_receipt_create_and_read_waits_cancel_without_backend_release_or_source_publication() +{ + use crate::ceres::snapshot::content_budget::MemoryBudget; + + for creating in [true, false] { + let fixture = Fixture::new().await; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + let source = ChunkMapSource::from_fact(fixture.fact().await, &fixture.oid).unwrap(); + let objects = fixture.state.storage.git_service.obj_storage.clone(); + let admission = repository.admit_install(&source, &objects).await.unwrap(); + let budget = MemoryBudget::new(512 * 1024 * 1024); + let handler = MonoApiService::from(&fixture.state); + let verified = VerifiedSourceChunkMap::verify(&handler, source, &budget, &admission) + .await + .unwrap(); + assert!(budget.used() > 0); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let drops = if creating { + *fixture.counts.receipt_write_holds.lock().unwrap() = + Some((entered.clone(), release.clone())); + fixture.counts.receipt_write_wait_drops.clone() + } else { + let drops = Arc::new(AtomicUsize::new(0)); + *fixture.counts.receipt_open_holds.lock().unwrap() = + Some((entered.clone(), release.clone(), drops.clone())); + drops + }; + let mut task = + tokio::spawn(async move { repository.install(verified, &objects, &admission).await }); + timeout(Duration::from_secs(10), entered.notified()) + .await + .unwrap(); + let db = fixture.state.storage.mono_storage().get_connection(); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 1); + if creating { + db.execute_unprepared("UPDATE mst2_chunk_receipt_generation SET deadline=pg_catalog.clock_timestamp()+interval '20 milliseconds' WHERE state='CREATING'").await.unwrap(); + } + // Atomic create uses the actual owner expiry; receipt input also keeps + // its stricter five-second cap. Neither wait needs backend release. + let finished = timeout( + Duration::from_secs(if creating { 12 } else { 7 }), + &mut task, + ) + .await; + if finished.is_err() { + task.abort(); + let _ = task.await; + panic!("receipt wait retained its install workspace indefinitely"); + } + let error = finished.unwrap().unwrap().unwrap_err(); + assert_eq!( + error.code, + if creating { + SnapshotErrorCode::LeaseExpired + } else { + SnapshotErrorCode::TemporaryUnavailable + } + ); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert_eq!(budget.used(), 0); + fixture.counts.assert(1, fixture.raw.len()); + assert_eq!( + fixture.counts.receipt_reads.load(Ordering::SeqCst), + usize::from(!creating) + ); + assert_eq!(count(db, "mst2_chunk_reader").await, 0); + assert_eq!(count(db, "mst2_chunk_map_source").await, 0); + assert_eq!(count(db, "mst2_chunk_map").await, 0); + *fixture.counts.receipt_open_holds.lock().unwrap() = None; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + repository + .maintain(&fixture.state.storage.git_service.obj_storage, 64) + .await + .unwrap(); + assert_eq!(fixture.counts.receipt_deletes.load(Ordering::SeqCst), 1); + drop(release); + } +} + +#[tokio::test] +async fn held_raw_and_exact_range_do_not_resume_after_actual_reader_expiry() { + for raw_route in [true, false] { + let fixture = Fixture::new().await; + let map = fixture.map("/file").await; + let db = fixture.state.storage.mono_storage().get_connection(); + wait_count(db, "mst2_chunk_reader", 0).await; + fixture.counts.reset(); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let drops = Arc::new(AtomicUsize::new(0)); + let tail_polls = Arc::new(AtomicUsize::new(0)); + *fixture.counts.object_fault.lock().unwrap() = Some(bounded_objects::StreamFault { + oid: fixture.oid.clone(), + kind: bounded_objects::FaultKind::HeldFragment { + prefix: Bytes::new(), + fragment: Some(Bytes::new()), + entered: entered.clone(), + release: release.clone(), + tail_polls: tail_polls.clone(), + drops: drops.clone(), + }, + }); + let app = fixture.app.clone(); + let request = if raw_route { + fixture.request("GET", "blob?path=/file", Body::empty()) + } else { + fixture.request( + "POST", + "chunks", + Body::from( + fixture + .chunk_body("/file", map["map"]["map_id"].as_str().unwrap(), "0") + .to_string(), + ), + ) + }; + let mut task = tokio::spawn(async move { + let response = app.oneshot(request).await.unwrap(); + if raw_route { + assert_eq!(response.status(), 200); + let mut body = response.into_body().into_data_stream(); + assert!(body.next().await.unwrap().is_err()); + assert!(body.next().await.is_none()); + } else { + error(response, 410, "LEASE_EXPIRED", false).await; + } + }); + timeout(Duration::from_secs(10), entered.notified()) + .await + .unwrap(); + assert_eq!(count(db, "mst2_chunk_reader").await, 1); + db.execute_unprepared("UPDATE mst2_chunk_reader SET deadline=pg_catalog.clock_timestamp()+interval '20 milliseconds'").await.unwrap(); + // Exercise the real request-held reader without reaching into it: + // let the existing ten-second local check interval elapse too. + tokio::time::sleep(Duration::from_secs(11)).await; + release.notify_one(); + let completed = timeout(Duration::from_secs(5), &mut task).await; + if completed.is_err() { + task.abort(); + let _ = task.await; + panic!("expired reader continued into another held backend poll"); + } + completed.unwrap().unwrap(); + assert_eq!(tail_polls.load(Ordering::SeqCst), 0); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert_eq!(fixture.counts.bytes.load(Ordering::SeqCst), 0); + wait_count(db, "mst2_chunk_reader", 0).await; + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + } +} + +#[tokio::test] +async fn production_q_bootstrap_after_retention_preserves_catalog_and_reconstructed_warm_routes() { + use crate::jupiter::storage::qualified_metadata_family::{ + SnapshotMetadataFamily, provision_or_verify_rooted_qualified_family, + }; + + let fixture = Fixture::new_with_pg_config(true).await; + let db = fixture.state.storage.mono_storage().get_connection(); + let namespace = provision_or_verify_rooted_qualified_family(db) + .await + .unwrap(); + assert_eq!( + fixture + .state + .storage + .snapshot_metadata_family(&fixture.lease, true) + .await + .unwrap(), + Some(SnapshotMetadataFamily::Rooted) + ); + let applied: i64 = db + .query_one_raw(statement( + "SELECT count(*) AS count FROM seaql_migrations WHERE version='m20261008_000300_add_mst2_chunk_map_retention'", + [], + )) + .await + .unwrap() + .unwrap() + .try_get("", "count") + .unwrap(); + assert_eq!(applied, 1); + let original = fixture.map("/file").await; + fixture.counts.assert(1, fixture.raw.len()); + wait_count(db, "mst2_chunk_reader", 0).await; + + // This is the production connection path: it reapplies migration checks + // and verifies the complete existing Q authority catalog without a test + // exemption or a refreshed registration fingerprint. + let config = fixture.state.storage.config(); + let connection = crate::jupiter::storage::init::database_connection(&config.database) + .await + .unwrap(); + assert_eq!( + provision_or_verify_rooted_qualified_family(&connection) + .await + .unwrap(), + namespace + ); + let storage = crate::jupiter::storage::Storage::new_with_connection( + config, + Arc::new(connection), + fixture.state.storage.git_service.obj_storage.clone(), + ) + .await + .unwrap(); + let state = MonoApiServiceState { + storage, + ..fixture.state.clone() + }; + assert_eq!( + state + .storage + .snapshot_metadata_family(&fixture.lease, true) + .await + .unwrap(), + Some(SnapshotMetadataFamily::Rooted) + ); + let app = router(&state); + fixture.counts.reset(); + let warm = success_json( + app.clone() + .oneshot(fixture.request("GET", "chunk-map?path=/alias", Body::empty())) + .await + .unwrap(), + ) + .await; + assert_eq!(warm["map"], original["map"]); + fixture.counts.assert(0, 0); + assert_eq!(fixture.counts.receipt_reads.load(Ordering::SeqCst), 1); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 0); + + fixture.counts.reset(); + let response = app + .oneshot( + fixture.request( + "POST", + "chunks", + Body::from( + fixture + .chunk_body("/alias", original["map"]["map_id"].as_str().unwrap(), "0") + .to_string(), + ), + ), + ) + .await + .unwrap(); + assert_eq!(response.status(), 200); + let wire = to_bytes(response.into_body(), 2 * 1024 * 1024) + .await + .unwrap(); + let frames = parse_stream(&wire).unwrap(); + let [Frame::Chunk(chunk), Frame::End(end)] = frames.as_slice() else { + panic!("reconstructed rooted route must emit CHUNK and terminal END"); + }; + assert_eq!(chunk.chunk_bytes, &fixture.raw[..CHUNK_SIZE as usize]); + assert_eq!(chunk.file_content_id, fixture.digest); + assert_eq!(chunk.chunk_index, 0); + assert_eq!(end.request_item_count, 1); + assert_eq!(end.logical_bytes, u64::from(CHUNK_SIZE)); + assert_eq!(fixture.counts.whole.load(Ordering::SeqCst), 0); + assert_eq!(fixture.counts.range.load(Ordering::SeqCst), 1); + assert_eq!( + fixture.counts.bytes.load(Ordering::SeqCst), + CHUNK_SIZE as usize + ); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 0); + drop(wire); + wait_count(db, "mst2_chunk_reader", 0).await; + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + assert_eq!(count(db, "mst2_chunk_map").await, 1); +} + +#[tokio::test] +async fn one_connection_install_commits_bounded_stages_and_reconstructed_warm_reads() { + let fixture = Fixture::new_with_pg_config(true).await; + let mut config = fixture.state.storage.config().database.clone(); + config.max_connection = 1; + config.min_connection = 1; + let connection = crate::jupiter::storage::init::database_connection(&config) + .await + .unwrap(); + let repository = PostgresChunkMapRepository::new(connection.clone()) + .await + .unwrap(); + assert!( + fixture + .state + .storage + .native_chunk_maps + .set(repository) + .is_ok() + ); + connection.execute_unprepared("CREATE TABLE chunk_stage_audit(relation text NOT NULL, writer_xid bigint NOT NULL); CREATE FUNCTION chunk_stage_audit() RETURNS trigger LANGUAGE plpgsql AS $audit$ BEGIN INSERT INTO chunk_stage_audit VALUES(TG_TABLE_NAME,pg_catalog.txid_current()); RETURN NEW; END $audit$; CREATE TRIGGER chunk_stage_audit AFTER INSERT ON mst2_chunk_map FOR EACH ROW EXECUTE FUNCTION chunk_stage_audit(); CREATE TRIGGER chunk_stage_audit AFTER INSERT ON mst2_chunk_map_leaf FOR EACH ROW EXECUTE FUNCTION chunk_stage_audit(); CREATE TRIGGER chunk_stage_audit AFTER INSERT ON mst2_chunk_map_node FOR EACH ROW EXECUTE FUNCTION chunk_stage_audit(); CREATE TRIGGER chunk_stage_audit AFTER INSERT ON mst2_chunk_map_source FOR EACH ROW EXECUTE FUNCTION chunk_stage_audit()").await.unwrap(); + let map = timeout(Duration::from_secs(10), fixture.map("/file")) + .await + .expect("single-connection staging waited for its own held connection"); + fixture.counts.assert(1, fixture.raw.len()); + let transactions: i64 = connection + .query_one_raw(statement( + "SELECT count(DISTINCT writer_xid) AS count FROM chunk_stage_audit", + [], + )) + .await + .unwrap() + .unwrap() + .try_get("", "count") + .unwrap(); + assert!(transactions >= 4); + for table in [ + "mst2_chunk_map", + "mst2_chunk_map_leaf", + "mst2_chunk_map_node", + "mst2_chunk_map_source", + ] { + assert_eq!(count(&connection, table).await, 1); + } + fixture.counts.reset(); + let reconstructed = PostgresChunkMapRepository::new(connection.clone()) + .await + .unwrap(); + let source = ChunkMapSource::from_fact(fixture.fact().await, &fixture.oid).unwrap(); + let persisted = timeout( + Duration::from_secs(10), + reconstructed.read(&source, &fixture.state.storage.git_service.obj_storage), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + format!("sha256:{}", hex_of(&persisted.map_id)), + map["map"]["map_id"] + ); + let page = reconstructed.selected_page(&persisted, 0).await.unwrap(); + page.verify_chunk(&persisted.map, 0, &fixture.raw[..CHUNK_SIZE as usize]) + .unwrap(); + fixture.counts.assert(0, 0); + assert_eq!(fixture.counts.receipt_reads.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn cached_inventory_is_coalesced_and_bound_to_the_actual_backend_arc() { + let fixture = Fixture::new().await; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + let objects = &fixture.state.storage.git_service.obj_storage; + repository.maintain(objects, 8).await.unwrap(); + let first = fixture + .counts + .receipt_inventory_calls + .load(Ordering::SeqCst); + repository.maintain(objects, 8).await.unwrap(); + assert_eq!( + fixture + .counts + .receipt_inventory_calls + .load(Ordering::SeqCst), + first + ); + let counts = Arc::new(ReadCounts::default()); + counts + .receipt_retention_unsupported + .store(true, Ordering::SeqCst); + let other = MegaObjectStorageWrapper::new(Arc::new(CountingStorage { + inner: objects.clone(), + counts: counts.clone(), + })); + let error = repository.maintain(&other, 8).await.unwrap_err(); + assert_eq!(error.code, SnapshotErrorCode::TemporaryUnavailable); + assert_eq!(counts.receipt_inventory_calls.load(Ordering::SeqCst), 1); + fixture.counts.assert(0, 0); + assert_eq!( + count( + fixture.state.storage.mono_storage().get_connection(), + "mst2_chunk_receipt_generation" + ) + .await, + 0 + ); +} + +#[tokio::test] +async fn completion_during_real_inventory_keeps_backing_credit_after_pending_install_disappears() { + let mut fixture = Fixture::new().await; + let backing = mock_object_storage(); + backing + .inner + .put_stream( + &ObjectKey { + namespace: ObjectNamespace::Git, + key: fixture.oid.clone(), + }, + Box::pin(futures::stream::iter([Ok(Bytes::copy_from_slice( + &fixture.raw, + ))])), + ObjectMeta::default(), + ) + .await + .unwrap(); + fixture.state.storage.git_service = GitService { + obj_storage: MegaObjectStorageWrapper::new(Arc::new(CountingStorage { + inner: backing.clone(), + counts: fixture.counts.clone(), + })), + }; + fixture.app = router(&fixture.state); + let db = fixture.state.storage.mono_storage().get_connection(); + let create_entered = Arc::new(Notify::new()); + let create_release = Arc::new(Notify::new()); + *fixture.counts.receipt_late_create_holds.lock().unwrap() = + Some((create_entered.clone(), create_release.clone())); + let app = fixture.app.clone(); + let request = fixture.request("GET", "chunk-map?path=/file", Body::empty()); + let leader = tokio::spawn(async move { app.oneshot(request).await.unwrap() }); + timeout(Duration::from_secs(10), create_entered.notified()) + .await + .unwrap(); + *fixture.counts.receipt_late_create_holds.lock().unwrap() = None; + for index in 0..crate::orbit_api::object_storage::MAX_CHUNK_MAP_RECEIPTS - 1 { + backing + .inner + .put_metadata_atomic_create( + &ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: format!("{index:064x}"), + }, + Bytes::from_static(&[1; 129]), + ObjectMeta::default(), + ) + .await + .unwrap(); + } + // Synthetic occupants test physical capacity; they confer no source trust. + let actual = &fixture.state.storage.git_service.obj_storage; + let inventory_entered = Arc::new(Notify::new()); + let inventory_release = Arc::new(Notify::new()); + let observer_counts = Arc::new(ReadCounts::default()); + *observer_counts.receipt_inventory_holds.lock().unwrap() = + Some((inventory_entered.clone(), inventory_release.clone())); + let observer = PostgresChunkMapRepository::new(db.clone()).await.unwrap(); + let observed = MegaObjectStorageWrapper::new(Arc::new(CountingStorage { + inner: actual.clone(), + counts: observer_counts, + })); + let capacity = tokio::spawn(async move { observer.test_new_install_capacity(&observed).await }); + timeout(Duration::from_secs(30), inventory_entered.notified()) + .await + .unwrap(); + create_release.notify_one(); + success_json( + timeout(Duration::from_secs(30), leader) + .await + .unwrap() + .unwrap(), + ) + .await; + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + let pending:i64=db.query_one_raw(statement("SELECT count(*) AS count FROM mst2_chunk_receipt_generation WHERE state IN ('RESERVED','CREATING')",[])).await.unwrap().unwrap().try_get("","count").unwrap(); + assert_eq!(pending, 0); + inventory_release.notify_one(); + let error = timeout(Duration::from_secs(10), capacity) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_eq!(error.code, SnapshotErrorCode::LimitExceeded); + fixture.counts.assert(1, fixture.raw.len()); + assert_eq!( + actual + .inner + .chunk_map_receipt_inventory() + .await + .unwrap() + .objects + .len(), + crate::orbit_api::object_storage::MAX_CHUNK_MAP_RECEIPTS + ); +} + +#[tokio::test] +async fn json_chunk_and_raw_last_transport_clones_block_actual_collection() { + use crate::ceres::snapshot::content_budget::{MemoryBudget, RANGE_WORK_BYTES}; + + for route in 0..3 { + let fixture = Fixture::new().await; + let map = fixture.map("/file").await; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + let db = fixture.state.storage.mono_storage().get_connection(); + wait_count(db, "mst2_chunk_reader", 0).await; + fixture.counts.reset(); + let response_budget = MemoryBudget::new(CHUNK_SIZE as usize + 2048); + let scratch_budget = MemoryBudget::new(RANGE_WORK_BYTES); + let response = match route { + 0 => { + fixture + .send("GET", "chunk-map?path=/file", Body::empty()) + .await + } + 1 => budgeted_chunks_app(&fixture, &response_budget, &scratch_budget) + .oneshot( + fixture.request( + "POST", + "chunks", + Body::from( + json!({"items":[{ + "path":"/file", "expected_digest":fixture.digest_string(), + "map_id":map["map"]["map_id"], "chunk_index":"0" + }],"encoding":"identity"}) + .to_string(), + ), + ), + ) + .await + .unwrap(), + _ => super::raw_blob::budgeted_app(&fixture, &response_budget, &scratch_budget) + .oneshot(fixture.request("GET", "blob?path=/file", Body::empty())) + .await + .unwrap(), + }; + assert_eq!(response.status(), 200); + let mut body = response.into_body().into_data_stream(); + let mut wire = Vec::new(); + let mut transport = Vec::new(); + while let Some(frame) = body.next().await { + let bytes = frame.unwrap(); + wire.extend_from_slice(&bytes); + transport.push(bytes.clone()); + } + drop(body); + if route == 2 { + assert_eq!(wire, fixture.raw); + } + if route == 1 { + let frames = parse_stream(&wire).unwrap(); + assert!(frames.iter().any(|frame| matches!(frame, Frame::End(_)))); + } + assert!(!transport.is_empty()); + let last = transport.pop().unwrap(); + let last_transport = last.clone(); + drop(last); + drop(transport); + assert_eq!(scratch_budget.used(), 0); + if route == 1 { + assert_eq!(response_budget.used(), CHUNK_SIZE as usize + 2048); + } else if route == 2 { + assert_eq!( + response_budget.used(), + fixture.raw.len() - CHUNK_SIZE as usize + ); + } + assert_eq!(count(db, "mst2_chunk_reader").await, 1); + age_unowned_candidates(db).await; + repository + .maintain(&fixture.state.storage.git_service.obj_storage, 64) + .await + .unwrap(); + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + assert_eq!(count(db, "mst2_chunk_map").await, 1); + assert_eq!(fixture.counts.receipt_deletes.load(Ordering::SeqCst), 0); + drop(last_transport); + assert_eq!(response_budget.used(), 0); + wait_count(db, "mst2_chunk_reader", 0).await; + repository + .maintain(&fixture.state.storage.git_service.obj_storage, 64) + .await + .unwrap(); + for table in [ + "mst2_chunk_map_source", + "mst2_chunk_map", + "mst2_chunk_map_leaf", + "mst2_chunk_map_node", + "mst2_chunk_map_lifetime", + ] { + assert_eq!(count(db, table).await, 0); + } + assert_eq!(fixture.counts.receipt_deletes.load(Ordering::SeqCst), 1); + assert_eq!(count(db, "mst2_chunk_receipt_generation").await, 1); + assert_eq!(count(db, "mst2_chunk_map_gc").await, 1); + } +} + +#[tokio::test] +async fn expired_reader_arc_cannot_resurrect_or_authenticate_a_page() { + let fixture = Fixture::new().await; + fixture.map("/file").await; + let db = fixture.state.storage.mono_storage().get_connection(); + wait_count(db, "mst2_chunk_reader", 0).await; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + let source = ChunkMapSource::from_fact(fixture.fact().await, &fixture.oid).unwrap(); + let map = repository + .read(&source, &fixture.state.storage.git_service.obj_storage) + .await + .unwrap() + .unwrap(); + let old_generation: i64 = db + .query_one_raw(statement( + "SELECT generation FROM mst2_chunk_map_lifetime WHERE map_id=$1", + [map.map_id.to_vec().into()], + )) + .await + .unwrap() + .unwrap() + .try_get("", "generation") + .unwrap(); + db.execute_unprepared("UPDATE mst2_chunk_reader SET deadline=pg_catalog.clock_timestamp()+interval '20 milliseconds'").await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + map.test_check_next_owner_operation().await; + assert_eq!( + map.ensure_live().await.unwrap_err().code, + SnapshotErrorCode::LeaseExpired + ); + assert_eq!( + map.record_progress().await.unwrap_err().code, + SnapshotErrorCode::LeaseExpired + ); + assert_eq!( + repository.selected_page(&map, 0).await.err().unwrap().code, + SnapshotErrorCode::LeaseExpired + ); + assert!(db.execute_unprepared("UPDATE mst2_chunk_reader SET deadline=pg_catalog.clock_timestamp()+interval '59 seconds'").await.is_err()); + age_unowned_candidates(db).await; + repository + .maintain(&fixture.state.storage.git_service.obj_storage, 64) + .await + .unwrap(); + assert_eq!(count(db, "mst2_chunk_map").await, 0); + assert_eq!( + map.ensure_live().await.unwrap_err().code, + SnapshotErrorCode::LeaseExpired + ); + fixture.counts.reset(); + assert_eq!( + fixture.map("/file").await["map"]["file_size"], + fixture.raw.len().to_string() + ); + fixture.counts.assert(1, fixture.raw.len()); + let new_generation: i64 = db + .query_one_raw(statement( + "SELECT generation FROM mst2_chunk_map_lifetime WHERE map_id=$1", + [map.map_id.to_vec().into()], + )) + .await + .unwrap() + .unwrap() + .try_get("", "generation") + .unwrap(); + assert!(new_generation > old_generation); + assert!( + db.execute_raw(statement( + "UPDATE mst2_chunk_map_lifetime SET state='DELETING' WHERE map_id=$1", + [map.map_id.to_vec().into()] + )) + .await + .is_err() + ); + assert!( + db.execute_raw(statement( + "DELETE FROM mst2_chunk_map_leaf WHERE map_id=$1", + [map.map_id.to_vec().into()] + )) + .await + .is_err() + ); + fixture.counts.reset(); + fixture.map("/alias").await; + fixture.counts.assert(0, 0); + assert_eq!( + map.ensure_live().await.unwrap_err().code, + SnapshotErrorCode::LeaseExpired + ); +} + +#[tokio::test] +async fn late_cancelled_create_is_reserved_and_old_key_replay_preserves_new_generation() { + let fixture = Fixture::new().await; + let db = fixture.state.storage.mono_storage().get_connection(); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + *fixture.counts.receipt_late_create_holds.lock().unwrap() = + Some((entered.clone(), release.clone())); + let app = fixture.app.clone(); + let request = fixture.request("GET", "chunk-map?path=/file", Body::empty()); + let cancelled = tokio::spawn(async move { app.oneshot(request).await }); + timeout(Duration::from_secs(10), entered.notified()) + .await + .unwrap(); + let old_key: String = db + .query_one_raw(statement( + "SELECT receipt_key FROM mst2_chunk_receipt_generation", + [], + )) + .await + .unwrap() + .unwrap() + .try_get("", "receipt_key") + .unwrap(); + assert!( + !fixture + .state + .storage + .git_service + .obj_storage + .inner + .exists(&ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: old_key.clone() + }) + .await + .unwrap() + ); + cancelled.abort(); + assert!(cancelled.await.err().unwrap().is_cancelled()); + *fixture.counts.receipt_late_create_holds.lock().unwrap() = None; + let current = fixture.map("/file").await; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + wait_count(db, "mst2_chunk_reader", 0).await; + let history = db + .query_one_raw(statement( + "SELECT state,create_completed FROM mst2_chunk_receipt_generation WHERE receipt_key=$1", + [old_key.clone().into()], + )) + .await + .unwrap() + .unwrap(); + assert_eq!(history.try_get::("", "state").unwrap(), "APPLIED"); + assert!(!history.try_get::("", "create_completed").unwrap()); + let new_key: String = db + .query_one_raw(statement( + "SELECT receipt_key FROM mst2_chunk_map_source", + [], + )) + .await + .unwrap() + .unwrap() + .try_get("", "receipt_key") + .unwrap(); + assert_ne!(old_key, new_key); + release.notify_one(); + timeout(Duration::from_secs(10), async { + loop { + if fixture + .state + .storage + .git_service + .obj_storage + .inner + .exists(&ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: old_key.clone(), + }) + .await + .unwrap() + && fixture.counts.receipt_writes.load(Ordering::SeqCst) == 2 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + repository + .maintain(&fixture.state.storage.git_service.obj_storage, 64) + .await + .unwrap(); + assert!( + !fixture + .state + .storage + .git_service + .obj_storage + .inner + .exists(&ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: old_key.clone() + }) + .await + .unwrap() + ); + assert!( + fixture + .state + .storage + .git_service + .obj_storage + .inner + .exists(&ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: new_key + }) + .await + .unwrap() + ); + fixture.counts.reset(); + assert_eq!(fixture.map("/file").await["map"], current["map"]); + fixture.counts.assert(0, 0); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 0); + let history = db + .query_one_raw(statement( + "SELECT create_completed FROM mst2_chunk_receipt_generation WHERE receipt_key=$1", + [old_key.into()], + )) + .await + .unwrap() + .unwrap(); + assert!(!history.try_get::("", "create_completed").unwrap()); +} + +#[tokio::test] +async fn actual_backing_quota_and_unsupported_capability_reject_before_source_open() { + for unsupported in [true, false] { + let fixture = Fixture::new().await; + let counts = Arc::new(ReadCounts::default()); + counts + .receipt_retention_unsupported + .store(unsupported, Ordering::SeqCst); + let backend = mock_object_storage(); + if !unsupported { + for index in 0..=crate::orbit_api::object_storage::MAX_CHUNK_MAP_RECEIPTS { + backend + .inner + .put_metadata_atomic_create( + &ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: format!("{index:064x}"), + }, + Bytes::from_static(&[1; 129]), + ObjectMeta::default(), + ) + .await + .unwrap(); + } + } + let mut state = fixture.state.clone(); + state.storage.git_service = GitService { + obj_storage: MegaObjectStorageWrapper::new(Arc::new(CountingStorage { + inner: backend, + counts: counts.clone(), + })), + }; + let response = router(&state) + .oneshot(fixture.request("GET", "chunk-map?path=/file", Body::empty())) + .await + .unwrap(); + if unsupported { + error(response, 503, "TEMPORARY_UNAVAILABLE", true).await; + } else { + error(response, 413, "LIMIT_EXCEEDED", false).await; + } + counts.assert(0, 0); + assert_eq!(counts.receipt_writes.load(Ordering::SeqCst), 0); + let db = fixture.state.storage.mono_storage().get_connection(); + assert_eq!(count(db, "mst2_chunk_receipt_generation").await, 0); + assert_eq!(count(db, "mst2_chunk_map").await, 0); + } +} + +#[tokio::test] +async fn empty_source_fragment_after_deadline_does_not_renew_install_or_publish() { + let fixture = Fixture::new().await; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + let source = ChunkMapSource::from_fact(fixture.fact().await, &fixture.oid).unwrap(); + let admission = Arc::new( + repository + .admit_install(&source, &fixture.state.storage.git_service.obj_storage) + .await + .unwrap(), + ); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let drops = Arc::new(AtomicUsize::new(0)); + let tail_polls = Arc::new(AtomicUsize::new(0)); + *fixture.counts.object_fault.lock().unwrap() = Some(bounded_objects::StreamFault { + oid: fixture.oid.clone(), + kind: bounded_objects::FaultKind::HeldFragment { + prefix: Bytes::new(), + fragment: Some(Bytes::new()), + entered: entered.clone(), + release: release.clone(), + tail_polls: tail_polls.clone(), + drops: drops.clone(), + }, + }); + let handler = MonoApiService::from(&fixture.state); + let budget = crate::ceres::snapshot::content_budget::MemoryBudget::new(8 * 1024 * 1024); + let verifier_budget = budget.clone(); + let owner = admission.clone(); + let task = tokio::spawn(async move { + VerifiedSourceChunkMap::verify(&handler, source, &verifier_budget, &owner).await + }); + timeout(Duration::from_secs(10), entered.notified()) + .await + .unwrap(); + let db = fixture.state.storage.mono_storage().get_connection(); + db.execute_unprepared("UPDATE mst2_chunk_receipt_generation SET deadline=pg_catalog.clock_timestamp()+interval '20 milliseconds' WHERE state='RESERVED'").await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + admission.test_check_next_owner_operation().await; + release.notify_one(); + let error = timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap() + .err() + .unwrap(); + assert_eq!(error.code, SnapshotErrorCode::LeaseExpired); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert_eq!(tail_polls.load(Ordering::SeqCst), 0); + assert_eq!(fixture.counts.bytes.load(Ordering::SeqCst), 0); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 0); + assert_eq!(count(db, "mst2_chunk_map_source").await, 0); + assert_eq!(budget.used(), 0); + drop(admission); + repository + .maintain(&fixture.state.storage.git_service.obj_storage, 64) + .await + .unwrap(); + assert_eq!(count(db, "mst2_chunk_map").await, 0); +} diff --git a/src/api/router/snapshot_content.rs b/src/api/router/snapshot_content.rs index 275f7dd7..961d1b00 100644 --- a/src/api/router/snapshot_content.rs +++ b/src/api/router/snapshot_content.rs @@ -19,7 +19,7 @@ use super::{ }; use crate::ceres::snapshot::{ chunks::{ChunkMapSource, VerifiedSourceChunkMap, map_build_reservation_bytes}, - content_budget::{BudgetedFrame, MemoryLease, reserve_range_work, reserve_response}, + content_budget::{BudgetedFrame, MemoryLease, reserve_response}, error::{SnapshotError, SnapshotErrorCode}, pages::{MetadataWalkOutcome, base64_of, hex_of, resolve_abs_metadata}, resolver::FsKind, @@ -640,21 +640,29 @@ pub(super) async fn project_resolved, ) -> Result { let bytes = map_json_bytes(value, memory)?; let headers = super::REQUEST_HEADERS.try_with(Clone::clone).map_err(|_| { @@ -865,11 +874,17 @@ async fn guarded_map_json_response( ) })?; super::revalidate_access(state, context, &headers).await?; + map.ensure_live().await?; let state = state.clone(); let context = context.clone(); let stream = futures::stream::once(async move { super::revalidate_access(&state, &context, &headers).await?; - Ok::<_, SnapshotError>(bytes) + map.ensure_live().await?; + map.record_progress().await?; + Ok::<_, SnapshotError>(bytes::Bytes::from_owner(RetainedChunkFrame { + bytes, + _maps: std::sync::Arc::new(vec![map]), + })) }); Response::builder() .header("content-type", "application/json") @@ -879,6 +894,43 @@ async fn guarded_map_json_response( .map_err(|_| internal("chunk map JSON response build failed")) } +struct RetainedChunkFrame { + bytes: bytes::Bytes, + _maps: std::sync::Arc< + Vec>, + >, +} +impl AsRef<[u8]> for RetainedChunkFrame { + fn as_ref(&self) -> &[u8] { + &self.bytes + } +} + +fn retain_chunk_maps( + response: Response, + maps: Vec>, +) -> Response { + let maps = std::sync::Arc::new(maps); + let (parts, body) = response.into_parts(); + let stream = body.into_data_stream().then(move |frame| { + let maps = maps.clone(); + async move { + let bytes = frame.map_err(std::io::Error::other)?; + for map in maps.iter() { + map.ensure_live().await.map_err(std::io::Error::other)?; + if !bytes.is_empty() { + map.record_progress().await.map_err(std::io::Error::other)?; + } + } + Ok::<_, std::io::Error>(bytes::Bytes::from_owner(RetainedChunkFrame { + bytes, + _maps: maps, + })) + } + }); + Response::from_parts(parts, axum::body::Body::from_stream(stream)) +} + #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] struct ChunksRequest { @@ -899,6 +951,20 @@ struct ChunkItem { const CHUNKS_MAX_ITEMS: usize = 128; const CHUNKS_TOTAL_MAX: u64 = 128 * 1024 * 1024; +pub(super) struct ChunksBudgets { + pub(super) response: std::sync::Arc, + pub(super) scratch: std::sync::Arc, +} + +impl Default for ChunksBudgets { + fn default() -> Self { + Self { + response: crate::ceres::snapshot::content_budget::response_budget().clone(), + scratch: crate::ceres::snapshot::content_budget::range_budget().clone(), + } + } +} + struct Planned { projection: std::sync::Arc, page: std::sync::Arc, @@ -932,6 +998,10 @@ async fn read_chunk_range( planned: &Planned, ) -> Result, Response> { let projection = &planned.projection; + projection + .ensure_live() + .await + .map_err(mst2_error_response)?; let len = projection.map.chunk_len(planned.index).map_err(|error| { mst2_error_response(internal(format!("invalid admitted chunk: {error}"))) })?; @@ -942,9 +1012,10 @@ async fn read_chunk_range( let end = start .checked_add(len) .ok_or_else(|| mst2_error_response(internal("chunk end overflow")))?; - let (mut input, meta) = handler - .get_raw_blob_range_stream_exact(&planned.oid, start, end) + let (mut input, meta) = projection + .await_backend(handler.get_raw_blob_range_stream_exact(&planned.oid, start, end)) .await + .map_err(mst2_error_response)? .map_err(|error| mst2_error_response(content_read_error(error)))? .ok_or_else(|| { mst2_error_response(SnapshotError::new( @@ -952,6 +1023,10 @@ async fn read_chunk_range( "fixed source does not support exact raw ranges", )) })?; + projection + .ensure_live() + .await + .map_err(mst2_error_response)?; if u64::try_from(meta.size).ok() != Some(projection.map.file_size) { return Err(mst2_error_response(SnapshotError::new( SnapshotErrorCode::IntegrityError, @@ -965,7 +1040,21 @@ async fn read_chunk_range( "range allocation could not be admitted", )) })?; - while let Some(part) = input.next().await { + let mut empty_parts = 0usize; + loop { + projection + .ensure_live() + .await + .map_err(mst2_error_response)?; + let part = projection + .await_backend(input.next()) + .await + .map_err(mst2_error_response)?; + projection + .ensure_live() + .await + .map_err(mst2_error_response)?; + let Some(part) = part else { break }; let bytes = part.map_err(|error| { tracing::warn!(error = %error, "fixed-view range stream failed"); mst2_error_response(SnapshotError::new( @@ -980,19 +1069,53 @@ async fn read_chunk_range( ))); } raw.extend_from_slice(&bytes); + if !bytes.is_empty() { + projection + .record_progress() + .await + .map_err(mst2_error_response)?; + } else { + empty_parts += 1; + if empty_parts == 32 { + tokio::task::yield_now().await; + projection + .ensure_live() + .await + .map_err(mst2_error_response)?; + empty_parts = 0; + } + } } planned .page .verify_chunk(&projection.map, planned.index, &raw) .map_err(mst2_error_response)?; + projection + .ensure_live() + .await + .map_err(mst2_error_response)?; + projection + .record_progress() + .await + .map_err(mst2_error_response)?; Ok(raw) } #[allow(clippy::result_large_err)] pub(super) async fn chunks( + state: State, + path: AxumPath, + body: Mst2Bytes, +) -> Result { + chunks_with_budgets(state, path, body, ChunksBudgets::default()).await +} + +#[allow(clippy::result_large_err)] +pub(super) async fn chunks_with_budgets( state: State, AxumPath(snapshot_id): AxumPath, Mst2Bytes(body): Mst2Bytes, + budgets: ChunksBudgets, ) -> Result { ensure(&state)?; let ctx = super::request_context(&state, &snapshot_id).map_err(mst2_error_response)?; @@ -1090,7 +1213,10 @@ pub(super) async fn chunks( .ok() .and_then(|bytes| bytes.checked_add(req.items.len() * 1024 + 1024)) .ok_or_else(|| mst2_error_response(internal("chunk response memory overflow")))?; - let response_memory = reserve_response(response_bytes).map_err(mst2_error_response)?; + let response_memory = budgets + .response + .reserve(response_bytes) + .map_err(mst2_error_response)?; let mut maps = std::collections::HashMap::new(); let mut pages = std::collections::HashMap::new(); for item in resolved { @@ -1141,7 +1267,10 @@ pub(super) async fn chunks( let mut stream = FrameStream::new(1, encoding); let mut out: Vec> = Vec::new(); for p in planned.iter() { - let _work_memory = reserve_range_work().map_err(mst2_error_response)?; + let _work_memory = budgets + .scratch + .reserve(crate::ceres::snapshot::content_budget::RANGE_WORK_BYTES) + .map_err(mst2_error_response)?; // The source is the current request's fixed OID, never a cached // handler/backend/credential from a different scope. let bytes = read_chunk_range(handler.as_ref(), p).await?; @@ -1163,7 +1292,7 @@ pub(super) async fn chunks( ); out.push(end); - guarded_treeframe_response_with_budget( + let response = guarded_treeframe_response_with_budget( &state, &ctx, &snapshot_id, @@ -1171,7 +1300,8 @@ pub(super) async fn chunks( out, Some(response_memory), ) - .map_err(mst2_error_response) + .map_err(mst2_error_response)?; + Ok(retain_chunk_maps(response, maps.into_values().collect())) } /// Strict decimal-string parse for unsigned counts (spec 04 §1: no leading diff --git a/src/api/router/snapshot_content_tests.rs b/src/api/router/snapshot_content_tests.rs index 2dc14015..052c8369 100644 --- a/src/api/router/snapshot_content_tests.rs +++ b/src/api/router/snapshot_content_tests.rs @@ -88,6 +88,9 @@ mod bounded_chunks; #[path = "snapshot_persisted_chunk_map_tests.rs"] mod persisted_chunk_maps; +#[path = "snapshot_chunk_map_retention_tests.rs"] +mod chunk_map_retention; + #[path = "snapshot_raw_blob_tests.rs"] mod raw_blob; @@ -107,6 +110,27 @@ mod generation_qualified_fixture; mod install_capability_fixture; type ReceiptWriteHold = (Arc, Arc); +type ReadOpenHold = ( + Arc, + Arc, + Arc, +); + +struct ReadOpenDrop(Arc); + +impl Drop for ReadOpenDrop { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} + +async fn await_read_open_hold(hold: Option) { + if let Some((entered, release, drops)) = hold { + let _owner = ReadOpenDrop(drops); + entered.notify_one(); + release.notified().await; + } +} #[path = "snapshot_rooted_metadata_tests.rs"] mod rooted_metadata; @@ -126,6 +150,15 @@ struct ReadCounts { receipt_read_meta_size: std::sync::atomic::AtomicI64, receipt_read_late_error: AtomicBool, receipt_write_holds: std::sync::Mutex>, + receipt_write_wait_drops: Arc, + receipt_late_create_holds: std::sync::Mutex>, + receipt_inventory_holds: std::sync::Mutex>, + receipt_inventory_calls: AtomicUsize, + receipt_retention_unsupported: AtomicBool, + receipt_deletes: AtomicUsize, + whole_open_holds: std::sync::Mutex>, + range_open_holds: std::sync::Mutex>, + receipt_open_holds: std::sync::Mutex>, } impl ReadCounts { @@ -151,12 +184,71 @@ struct CountingStorage { #[async_trait::async_trait] impl MegaObjectStorage for CountingStorage { + async fn chunk_map_receipt_inventory( + &self, + ) -> OrbitResult { + self.counts + .receipt_inventory_calls + .fetch_add(1, Ordering::SeqCst); + if self + .counts + .receipt_retention_unsupported + .load(Ordering::SeqCst) + { + return Err(crate::orbit_api::error::IoOrbitError::ChunkMapRetentionUnsupported); + } + let inventory = self.inner.inner.chunk_map_receipt_inventory().await?; + let holds = self.counts.receipt_inventory_holds.lock().unwrap().clone(); + if let Some((entered, release)) = holds { + entered.notify_one(); + release.notified().await; + } + Ok(inventory) + } + + async fn delete_chunk_map_receipt( + &self, + authority: &crate::orbit_api::object_storage::ChunkMapReceiptDeletion, + ) -> OrbitResult { + let deleted = self.inner.inner.delete_chunk_map_receipt(authority).await?; + if deleted { + self.counts.receipt_deletes.fetch_add(1, Ordering::SeqCst); + } + Ok(deleted) + } + async fn put_metadata_atomic_create( &self, key: &ObjectKey, bytes: Bytes, meta: ObjectMeta, ) -> OrbitResult<()> { + let late = if key.namespace == ObjectNamespace::ChunkMapReceipt { + self.counts + .receipt_late_create_holds + .lock() + .unwrap() + .clone() + } else { + None + }; + if let Some((entered, release)) = late { + let inner = self.inner.clone(); + let key = key.clone(); + let counts = self.counts.clone(); + return tokio::spawn(async move { + entered.notify_one(); + release.notified().await; + inner + .inner + .put_metadata_atomic_create(&key, bytes, meta) + .await?; + counts.receipt_writes.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .await + .unwrap(); + } self.inner .inner .put_metadata_atomic_create(key, bytes, meta) @@ -165,6 +257,7 @@ impl MegaObjectStorage for CountingStorage { self.counts.receipt_writes.fetch_add(1, Ordering::SeqCst); let holds = self.counts.receipt_write_holds.lock().unwrap().clone(); if let Some((entered, release)) = holds { + let _hold = ReadOpenDrop(self.counts.receipt_write_wait_drops.clone()); entered.notify_one(); release.notified().await; } @@ -193,6 +286,8 @@ impl MegaObjectStorage for CountingStorage { async fn get_stream(&self, key: &ObjectKey) -> OrbitResult<(ObjectByteStream, ObjectMeta)> { if key.namespace == ObjectNamespace::ChunkMapReceipt { self.counts.receipt_reads.fetch_add(1, Ordering::SeqCst); + let holds = self.counts.receipt_open_holds.lock().unwrap().clone(); + await_read_open_hold(holds).await; if self.counts.receipt_read_failure.load(Ordering::SeqCst) { return Err( crate::orbit_api::error::IoOrbitError::object_store_not_found( @@ -225,6 +320,8 @@ impl MegaObjectStorage for CountingStorage { return self.inner.inner.get_stream(key).await; } self.counts.whole.fetch_add(1, Ordering::SeqCst); + let holds = self.counts.whole_open_holds.lock().unwrap().clone(); + await_read_open_hold(holds).await; let chunk_fault = self .counts .chunk_faults @@ -286,6 +383,8 @@ impl MegaObjectStorage for CountingStorage { end: u64, ) -> OrbitResult> { self.counts.range.fetch_add(1, Ordering::SeqCst); + let holds = self.counts.range_open_holds.lock().unwrap().clone(); + await_read_open_hold(holds).await; let fault = self .counts .chunk_faults @@ -312,7 +411,12 @@ impl MegaObjectStorage for CountingStorage { .inner .get_range_stream_exact(key, start, end) .await?; + let fault = self.counts.object_fault.lock().unwrap().clone(); Ok(result.map(|(stream, meta)| { + let stream = match fault { + Some(fault) if fault.oid == key.key => fault.stream(), + _ => stream, + }; let counts = self.counts.clone(); let stream = stream.map(move |part| { if let Ok(bytes) = &part { diff --git a/src/api/router/snapshot_persisted_chunk_map_tests.rs b/src/api/router/snapshot_persisted_chunk_map_tests.rs index 6b261115..23e1808b 100644 --- a/src/api/router/snapshot_persisted_chunk_map_tests.rs +++ b/src/api/router/snapshot_persisted_chunk_map_tests.rs @@ -832,7 +832,7 @@ pub(super) async fn assert_three_page_proofs_and_selected_sibling_faults( .await .unwrap(); } else { - db.execute_raw(statement("UPDATE mst2_chunk_map_node SET digest=$2 WHERE map_id=$1 AND first_page=2 AND page_count=1", [id.clone().into(), vec![9; 32].into()])).await.unwrap(); + db.execute_raw(statement("UPDATE mst2_chunk_map_node SET digest=$2 WHERE map_id=$1 AND first_page=2 AND page_count=1", [id.clone().into(), vec![9u8; 32].into()])).await.unwrap(); } db.execute_unprepared("ALTER TABLE mst2_chunk_map_node ENABLE TRIGGER USER") .await @@ -1160,7 +1160,7 @@ async fn ordinary_incomplete_source_dml_cannot_commit_an_admitted_partial_map() )) .await .unwrap(); - txn.execute_raw(statement("INSERT INTO mst2_chunk_map_source(storage_domain,git_oid,object_kind,fact_id,source_id,source_bytes,primary_scope,map_id,receipt_digest) VALUES('git',$1,'blob',$2,$3,$4,$5,$6,$7)", [fixture.oid.clone().into(), source.fact().id.into(), repository.source_identity(&source).unwrap().to_vec().into(), source.canonical_bytes().unwrap().into(), scope.to_vec().into(), map.map_id.to_vec().into(), vec![9;32].into()])).await.unwrap(); + txn.execute_raw(statement("INSERT INTO mst2_chunk_map_source(storage_domain,git_oid,object_kind,fact_id,source_id,source_bytes,primary_scope,map_id,receipt_digest) VALUES('git',$1,'blob',$2,$3,$4,$5,$6,$7)", [fixture.oid.clone().into(), source.fact().id.into(), repository.source_identity(&source).unwrap().to_vec().into(), source.canonical_bytes().unwrap().into(), scope.to_vec().into(), map.map_id.to_vec().into(), vec![9u8;32].into()])).await.unwrap(); assert!(txn.commit().await.is_err()); for table in [ "mst2_chunk_map", diff --git a/src/api/router/snapshot_raw_blob.rs b/src/api/router/snapshot_raw_blob.rs index ea438ec1..016310cd 100644 --- a/src/api/router/snapshot_raw_blob.rs +++ b/src/api/router/snapshot_raw_blob.rs @@ -133,10 +133,20 @@ pub(super) async fn blob_with_budgets( } else { None }; - let (input, meta) = handler - .get_raw_blob_stream_with_meta(&file.oid) - .await - .map_err(|error| mst2_error_response(content::content_read_error(error)))?; + if let Some(map) = map.as_ref() { + map.ensure_live().await.map_err(mst2_error_response)?; + } + let open = handler.get_raw_blob_stream_with_meta(&file.oid); + let opened = if let Some(map) = map.as_ref() { + map.await_backend(open).await.map_err(mst2_error_response)? + } else { + open.await + }; + let (input, meta) = + opened.map_err(|error| mst2_error_response(content::content_read_error(error)))?; + if let Some(map) = map.as_ref() { + map.ensure_live().await.map_err(mst2_error_response)?; + } if u64::try_from(meta.size).ok() != Some(file.size) { return Err(mst2_error_response(integrity( "raw source physical size disagrees with its fixed verified fact", @@ -231,7 +241,11 @@ impl AsRef<[u8]> for RawBlobChunk { impl RawBlobReader { async fn validate(&mut self) -> Result<(), SnapshotError> { - revalidate_access(&self.state, &self.context, &self.headers).await + revalidate_access(&self.state, &self.context, &self.headers).await?; + if let Some(map) = self.map.as_ref() { + map.ensure_live().await?; + } + Ok(()) } async fn next_chunk(&mut self) -> Result, SnapshotError> { @@ -327,6 +341,9 @@ impl RawBlobReader { } } self.validate().await?; + if let Some(map) = self.map.as_ref() { + map.record_progress().await?; + } if raw.capacity() > credit.bytes { return Err(integrity("raw chunk allocation exceeds its owned credit")); } @@ -342,19 +359,30 @@ impl RawBlobReader { } async fn poll_source(&mut self) -> Result, SnapshotError> { + self.validate().await?; let input = self .input .as_mut() .ok_or_else(|| integrity("raw source stream was already closed"))?; - let result = input.next().await; + let result = if let Some(map) = self.map.as_ref() { + map.await_backend(input.next()).await? + } else { + input.next().await + }; self.validate().await?; - result.transpose().map_err(|error| { + let part = result.transpose().map_err(|error| { tracing::warn!(%error, "fixed-view raw stream failed"); SnapshotError::new( SnapshotErrorCode::ObjectUnavailable, "fixed-view raw stream failed", ) - }) + })?; + if part.as_ref().is_some_and(|bytes| !bytes.is_empty()) { + if let Some(map) = self.map.as_ref() { + map.record_progress().await?; + } + } + Ok(part) } async fn require_eof(&mut self) -> Result<(), SnapshotError> { diff --git a/src/api/router/snapshot_raw_blob_tests.rs b/src/api/router/snapshot_raw_blob_tests.rs index 1c4e70f9..79a38c79 100644 --- a/src/api/router/snapshot_raw_blob_tests.rs +++ b/src/api/router/snapshot_raw_blob_tests.rs @@ -11,7 +11,7 @@ use crate::{ ceres::snapshot::content_budget::{MemoryBudget, RANGE_WORK_BYTES}, }; -fn budgeted_app( +pub(super) fn budgeted_app( fixture: &Fixture, response_budget: &Arc, scratch_budget: &Arc, diff --git a/src/ceres/snapshot/chunks.rs b/src/ceres/snapshot/chunks.rs index 8346cec2..8018c245 100644 --- a/src/ceres/snapshot/chunks.rs +++ b/src/ceres/snapshot/chunks.rs @@ -78,6 +78,7 @@ impl VerifiedSourceChunkMap { handler: &T, source: ChunkMapSource, budget: &Arc, + admission: &crate::jupiter::storage::native_chunk_map::retention::ChunkMapInstall, ) -> Result { let digest: [u8; 32] = source .fact @@ -106,6 +107,7 @@ impl VerifiedSourceChunkMap { }) }, budget, + admission, ) .await?; Ok(Self { source, projection }) diff --git a/src/ceres/snapshot/chunks_stream.rs b/src/ceres/snapshot/chunks_stream.rs index ab19e5b5..35e33e97 100644 --- a/src/ceres/snapshot/chunks_stream.rs +++ b/src/ceres/snapshot/chunks_stream.rs @@ -63,6 +63,7 @@ async fn build_stream( input, lease, size <= STAGED_CAP_BYTES as u64, + None, ) .await } @@ -72,6 +73,7 @@ pub(super) async fn build_source_stream( size: u64, open: F, budget: &Arc, + admission: &crate::jupiter::storage::native_chunk_map::retention::ChunkMapInstall, ) -> Result where F: FnOnce() -> Fut, @@ -84,6 +86,7 @@ where open, budget, BUILDERS.get_or_init(|| Arc::new(Semaphore::new(MAX_BUILDERS))), + Some(admission), ) .await } @@ -94,6 +97,7 @@ async fn build_source_with_resources( open: F, budget: &Arc, builders: &Arc, + admission: Option<&crate::jupiter::storage::native_chunk_map::retention::ChunkMapInstall>, ) -> Result where F: FnOnce() -> Fut, @@ -106,8 +110,15 @@ where ) })?; let lease = budget.reserve(source_reservation_bytes(size)?)?; - let input = open().await?; - build_stream_with_inline(content_id, size, input, lease, false).await + let input = if let Some(admission) = admission { + admission.ensure_live().await?; + let result = admission.await_backend(open()).await?; + admission.ensure_live().await?; + result? + } else { + open().await? + }; + build_stream_with_inline(content_id, size, input, lease, false, admission).await } pub(super) fn source_reservation_bytes(size: u64) -> Result { @@ -135,6 +146,7 @@ async fn build_stream_with_inline( mut input: ObjectByteStream, lease: MemoryLease, inline: bool, + admission: Option<&crate::jupiter::storage::native_chunk_map::retention::ChunkMapInstall>, ) -> Result { let chunk_count = size.div_ceil(CHUNK_SIZE as u64); let page_count = chunk_count.div_ceil(CHUNKS_PER_PAGE as u64); @@ -161,7 +173,17 @@ async fn build_stream_with_inline( let mut chunk_bytes = 0usize; let mut digests = 0u64; let mut since_yield = 0usize; - while let Some(part) = input.next().await { + let mut empty_parts = 0usize; + loop { + let next = if let Some(admission) = admission { + admission.ensure_live().await?; + let next = admission.await_backend(input.next()).await?; + admission.ensure_live().await?; + next + } else { + input.next().await + }; + let Some(part) = next else { break }; let bytes = part.map_err(|error| { tracing::warn!(error = %error, "fixed-view chunk projection stream failed"); SnapshotError::new( @@ -185,6 +207,18 @@ async fn build_stream_with_inline( raw.extend_from_slice(&bytes); } received += bytes.len() as u64; + if let Some(admission) = admission { + if !bytes.is_empty() { + admission.record_progress(received).await?; + } else { + empty_parts += 1; + if empty_parts == 32 { + tokio::task::yield_now().await; + admission.ensure_live().await?; + empty_parts = 0; + } + } + } let mut rest = bytes.as_ref(); while !rest.is_empty() { let len = rest.len().min(CHUNK_SIZE as usize - chunk_bytes); diff --git a/src/ceres/snapshot/chunks_stream_tests.rs b/src/ceres/snapshot/chunks_stream_tests.rs index 61de6453..d6f7e042 100644 --- a/src/ceres/snapshot/chunks_stream_tests.rs +++ b/src/ceres/snapshot/chunks_stream_tests.rs @@ -34,6 +34,7 @@ async fn exact_source_builder_admits_all_install_workspace_before_open_and_cance }, &budget, &builders, + None, ) .await .err() @@ -52,7 +53,8 @@ async fn exact_source_builder_admits_all_install_workspace_before_open_and_cance Ok(stream(vec![Ok(raw.clone())])) }, &budget, - &builders + &builders, + None ) .await .err() @@ -88,6 +90,7 @@ async fn exact_source_builder_admits_all_install_workspace_before_open_and_cance }, &task_budget, &task_builders, + None, ) .await }); @@ -110,6 +113,7 @@ async fn exact_source_builder_admits_all_install_workspace_before_open_and_cance }, &budget, &builders, + None, ) .await .unwrap(); diff --git a/src/ceres/snapshot/content_budget.rs b/src/ceres/snapshot/content_budget.rs index 6bb8d32b..18e16a51 100644 --- a/src/ceres/snapshot/content_budget.rs +++ b/src/ceres/snapshot/content_budget.rs @@ -89,10 +89,6 @@ pub(crate) fn response_budget() -> &'static Arc { BUDGET.get_or_init(|| MemoryBudget::new(RESPONSE_LIVE_BYTES)) } -pub(crate) fn reserve_range_work() -> Result { - range_budget().reserve(RANGE_WORK_BYTES) -} - pub(crate) fn range_budget() -> &'static Arc { static BUDGET: OnceLock> = OnceLock::new(); BUDGET.get_or_init(|| MemoryBudget::new(RANGE_SCRATCH_BYTES)) diff --git a/src/jupiter/migration/m20261008_000300_add_mst2_chunk_map_retention.rs b/src/jupiter/migration/m20261008_000300_add_mst2_chunk_map_retention.rs new file mode 100644 index 00000000..81e58229 --- /dev/null +++ b/src/jupiter/migration/m20261008_000300_add_mst2_chunk_map_retention.rs @@ -0,0 +1,26 @@ +use sea_orm::{ConnectionTrait, DbBackend}; +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + if manager.get_database_backend() != DbBackend::Postgres { + return Err(DbErr::Custom( + "chunk-map retention requires primary PostgreSQL".into(), + )); + } + manager + .get_connection() + .execute_unprepared(include_str!("m20261008_000300_chunk_map_retention.sql")) + .await?; + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + // Never discard generation fences or late-create reconciliation history. + Ok(()) + } +} diff --git a/src/jupiter/migration/m20261008_000300_chunk_map_retention.sql b/src/jupiter/migration/m20261008_000300_chunk_map_retention.sql new file mode 100644 index 00000000..46a75aa3 --- /dev/null +++ b/src/jupiter/migration/m20261008_000300_chunk_map_retention.sql @@ -0,0 +1,262 @@ +-- Physical receipt generations are never reused. APPLIED history remains +-- bounded and is rechecked: a timed-out backend create can complete late. +CREATE SEQUENCE mst2_chunk_generation_sequence START 2; +ALTER TABLE mst2_chunk_map_source ADD COLUMN receipt_generation bigint NOT NULL DEFAULT 1 CHECK(receipt_generation>0); +ALTER TABLE mst2_chunk_map_source ADD COLUMN receipt_key text GENERATED ALWAYS AS + (CASE WHEN receipt_generation=1 THEN pg_catalog.encode(source_id,'hex') + ELSE pg_catalog.encode(source_id,'hex')||'-'||pg_catalog.lpad(pg_catalog.to_hex(receipt_generation),16,'0') END) STORED; +CREATE UNIQUE INDEX mst2_chunk_map_source_receipt_key ON mst2_chunk_map_source(receipt_key); + +CREATE TABLE mst2_chunk_receipt_generation ( + receipt_key text PRIMARY KEY CHECK(receipt_key ~ '^[0-9a-f]{64}(-[0-9a-f]{16})?$'), + source_id bytea NOT NULL CHECK(pg_catalog.octet_length(source_id)=32), + generation bigint NOT NULL CHECK(generation>0), + source_bytes bytea NOT NULL CHECK(pg_catalog.octet_length(source_bytes) BETWEEN 1 AND 2048), + primary_scope bytea NOT NULL CHECK(pg_catalog.octet_length(primary_scope) BETWEEN 1 AND 1024), + map_id bytea CHECK(map_id IS NULL OR pg_catalog.octet_length(map_id)=32), + map_generation bigint CHECK(map_generation IS NULL OR map_generation>0), + receipt_bytes bytea CHECK(receipt_bytes IS NULL OR pg_catalog.octet_length(receipt_bytes) BETWEEN 129 AND 4096), + create_completed boolean NOT NULL DEFAULT false, + create_completed_at timestamptz, + owner uuid, + state text NOT NULL CHECK(state IN ('RESERVED','CREATING','LIVE','DELETING','APPLIED')), + reserved_pages integer NOT NULL CHECK(reserved_pages BETWEEN 0 AND 32768), + reserved_nodes integer NOT NULL CHECK(reserved_nodes BETWEEN 0 AND 65535), + reserved_bytes bigint NOT NULL CHECK(reserved_bytes BETWEEN 0 AND 536870912), + deadline timestamptz, + created_at timestamptz NOT NULL DEFAULT pg_catalog.clock_timestamp(), + last_progress timestamptz NOT NULL DEFAULT pg_catalog.clock_timestamp(), + checked_at timestamptz NOT NULL DEFAULT '-infinity', + UNIQUE(source_id,generation), + CHECK(receipt_key=CASE WHEN generation=1 THEN pg_catalog.encode(source_id,'hex') ELSE pg_catalog.encode(source_id,'hex')||'-'||pg_catalog.lpad(pg_catalog.to_hex(generation),16,'0') END), + CHECK(state NOT IN ('CREATING','LIVE') OR (map_id IS NOT NULL AND receipt_bytes IS NOT NULL)), + CHECK(state NOT IN ('RESERVED','CREATING') OR (owner IS NOT NULL AND deadline IS NOT NULL)) +); +CREATE INDEX mst2_chunk_receipt_generation_maintenance ON mst2_chunk_receipt_generation(state,checked_at,receipt_key); +CREATE INDEX mst2_chunk_receipt_generation_created ON mst2_chunk_receipt_generation(created_at) WHERE receipt_bytes IS NOT NULL; +CREATE INDEX mst2_chunk_receipt_generation_completed ON mst2_chunk_receipt_generation(create_completed_at) WHERE receipt_bytes IS NOT NULL; +CREATE INDEX mst2_chunk_receipt_generation_install ON mst2_chunk_receipt_generation(map_id,map_generation,deadline) WHERE state='CREATING'; +CREATE TABLE mst2_chunk_map_lifetime ( + map_id bytea PRIMARY KEY REFERENCES mst2_chunk_map(map_id) DEFERRABLE INITIALLY DEFERRED, + generation bigint NOT NULL CHECK(generation>0), + state text NOT NULL CHECK(state IN ('LIVE','DELETING')), + last_used timestamptz NOT NULL DEFAULT pg_catalog.clock_timestamp(), + UNIQUE(map_id,generation) +); +CREATE TABLE mst2_chunk_reader ( + owner uuid PRIMARY KEY, + receipt_key text NOT NULL REFERENCES mst2_chunk_receipt_generation(receipt_key), + source_generation bigint NOT NULL CHECK(source_generation>0), + map_id bytea NOT NULL CHECK(pg_catalog.octet_length(map_id)=32), + map_generation bigint NOT NULL CHECK(map_generation>0), + deadline timestamptz NOT NULL, + last_progress timestamptz NOT NULL DEFAULT pg_catalog.clock_timestamp() +); +CREATE INDEX mst2_chunk_reader_receipt ON mst2_chunk_reader(receipt_key,source_generation,deadline); +CREATE INDEX mst2_chunk_reader_map ON mst2_chunk_reader(map_id,map_generation,deadline); +CREATE TABLE mst2_chunk_map_gc ( + map_id bytea NOT NULL CHECK(pg_catalog.octet_length(map_id)=32), + generation bigint NOT NULL CHECK(generation>0), + state text NOT NULL CHECK(state IN ('PENDING','APPLIED')), + created_at timestamptz NOT NULL DEFAULT pg_catalog.clock_timestamp(), + applied_at timestamptz, + PRIMARY KEY(map_id,generation) +); + +-- Bootstrap existing #77 data from actual bounded rows. SQL does not infer +-- backing receipt presence; repository admission separately inventories it. +DO $$ DECLARE maps bigint; sources bigint; leaves bigint; nodes bigint; bytes bigint; BEGIN + SELECT pg_catalog.count(*) INTO maps FROM mst2_chunk_map; + SELECT pg_catalog.count(*) INTO sources FROM mst2_chunk_map_source; + SELECT pg_catalog.count(*) INTO leaves FROM mst2_chunk_map_leaf; + SELECT pg_catalog.count(*) INTO nodes FROM mst2_chunk_map_node; + SELECT COALESCE(pg_catalog.sum(pg_catalog.pg_total_relation_size(c.oid)),0) INTO bytes + FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname=pg_catalog.current_schema() AND c.relkind='r' + AND c.relname IN ('mst2_chunk_map','mst2_chunk_map_source','mst2_chunk_map_leaf','mst2_chunk_map_node'); + IF maps>4096 OR sources>16384 OR leaves>65536 OR nodes>131072 OR bytes>536870912 THEN + RAISE EXCEPTION 'existing chunk-map indexes exceed bounded retention bootstrap'; + END IF; +END $$; + +INSERT INTO mst2_chunk_map_lifetime(map_id,generation,state) SELECT map_id,1,'LIVE' FROM mst2_chunk_map; +INSERT INTO mst2_chunk_receipt_generation(receipt_key,source_id,generation,source_bytes,primary_scope,map_id,map_generation,receipt_bytes,create_completed,create_completed_at,state,reserved_pages,reserved_nodes,reserved_bytes) + SELECT s.receipt_key,s.source_id,1,s.source_bytes,s.primary_scope,s.map_id,1, + pg_catalog.convert_to('MST2-CHUNK-MAP-RECEIPT','UTF8')||pg_catalog.decode('00','hex')|| + pg_catalog.int4send(pg_catalog.octet_length(s.primary_scope))||s.primary_scope|| + pg_catalog.int4send(pg_catalog.octet_length(s.source_bytes))||s.source_bytes||m.descriptor, + true,pg_catalog.clock_timestamp(),'LIVE',0,0,0 FROM mst2_chunk_map_source s JOIN mst2_chunk_map m ON m.map_id=s.map_id; + +CREATE FUNCTION mst2_chunk_retention_barrier() RETURNS void LANGUAGE plpgsql AS $$ +BEGIN + IF pg_catalog.pg_is_in_recovery() OR pg_catalog.current_setting('transaction_isolation')<>'read committed' THEN + RAISE EXCEPTION 'chunk retention requires primary READ COMMITTED'; + END IF; + PERFORM pg_catalog.pg_advisory_xact_lock(1296717363,pg_catalog.hashtext(pg_catalog.current_schema())); +END $$; + +CREATE FUNCTION mst2_chunk_retention_bytes() RETURNS bigint LANGUAGE sql AS $$ + SELECT COALESCE(pg_catalog.sum(pg_catalog.pg_total_relation_size(c.oid)),0)::bigint + FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname=pg_catalog.current_schema() AND c.relkind='r' + AND c.relname IN ('mst2_chunk_map','mst2_chunk_map_source','mst2_chunk_map_leaf','mst2_chunk_map_node', + 'mst2_chunk_receipt_generation','mst2_chunk_map_lifetime','mst2_chunk_reader','mst2_chunk_map_gc') +$$; +DO $$ BEGIN + IF mst2_chunk_retention_bytes()>536870912 THEN RAISE EXCEPTION 'bounded chunk retention bootstrap exceeds actual durable byte quota'; END IF; +END $$; + +CREATE OR REPLACE FUNCTION mst2_chunk_map_primary() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF pg_catalog.current_schema() IS DISTINCT FROM TG_TABLE_SCHEMA THEN + RAISE EXCEPTION 'chunk resident insertion requires its actual primary schema'; + END IF; + PERFORM mst2_chunk_retention_barrier(); + IF mst2_chunk_retention_bytes()+1048576>536870912 THEN + RAISE EXCEPTION 'chunk resident insertion lacks actual durable byte headroom'; + END IF; + RETURN NULL; +END $$; + +-- Preserve the original immutable/source-completeness oracles. Only exact +-- terminal generations with no live owners may lose their resident indexes. +CREATE OR REPLACE FUNCTION mst2_chunk_map_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE permitted boolean:=false; +BEGIN + IF TG_OP<>'DELETE' OR pg_catalog.current_schema() IS DISTINCT FROM TG_TABLE_SCHEMA THEN + RAISE EXCEPTION 'chunk map indexes and source receipts are immutable'; + END IF; + PERFORM mst2_chunk_retention_barrier(); + IF TG_TABLE_NAME='mst2_chunk_map_source' THEN + EXECUTE pg_catalog.format('SELECT EXISTS(SELECT 1 FROM %I.mst2_chunk_receipt_generation g WHERE g.receipt_key=$1 AND g.source_id=$2 AND g.generation=$3 AND g.map_id=$4 AND g.state=''APPLIED'') AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_reader r WHERE r.receipt_key=$1 AND r.source_generation=$3 AND r.deadline>pg_catalog.clock_timestamp())',TG_TABLE_SCHEMA,TG_TABLE_SCHEMA) + INTO permitted USING OLD.receipt_key,OLD.source_id,OLD.receipt_generation,OLD.map_id; + ELSE + EXECUTE pg_catalog.format('SELECT EXISTS(SELECT 1 FROM %I.mst2_chunk_map_lifetime l JOIN %I.mst2_chunk_map_gc g ON g.map_id=l.map_id AND g.generation=l.generation WHERE l.map_id=$1 AND l.state=''DELETING'' AND g.state=''PENDING'') AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_map_source s WHERE s.map_id=$1) AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_reader r JOIN %I.mst2_chunk_map_lifetime l ON l.map_id=r.map_id AND l.generation=r.map_generation WHERE r.map_id=$1 AND r.deadline>pg_catalog.clock_timestamp())',TG_TABLE_SCHEMA,TG_TABLE_SCHEMA,TG_TABLE_SCHEMA,TG_TABLE_SCHEMA,TG_TABLE_SCHEMA) + INTO permitted USING OLD.map_id; + END IF; + IF NOT permitted THEN RAISE EXCEPTION 'chunk map deletion lacks an exact retired generation'; END IF; + RETURN OLD; +END $$; + +CREATE FUNCTION mst2_chunk_generation_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE owned boolean; +BEGIN + IF pg_catalog.current_schema() IS DISTINCT FROM TG_TABLE_SCHEMA THEN + RAISE EXCEPTION 'chunk retention mutation requires its actual primary schema'; + END IF; + PERFORM mst2_chunk_retention_barrier(); + IF TG_OP='INSERT' THEN + EXECUTE pg_catalog.format('SELECT pg_catalog.count(*)>=131072 FROM %I.mst2_chunk_receipt_generation',TG_TABLE_SCHEMA) INTO owned; + IF owned THEN RAISE EXCEPTION 'chunk receipt history quota exhausted'; END IF; + END IF; + IF TG_OP='DELETE' OR TG_OP='TRUNCATE' THEN + RAISE EXCEPTION 'chunk receipt generations and collection history are retained'; + END IF; + IF mst2_chunk_retention_bytes()+CASE WHEN NEW.state IN ('DELETING','APPLIED') THEN 262144 ELSE 1048576 END>536870912 THEN + RAISE EXCEPTION 'chunk receipt mutation lacks actual durable byte headroom'; + END IF; + IF TG_OP='UPDATE' AND (NEW.receipt_key IS DISTINCT FROM OLD.receipt_key OR NEW.source_id IS DISTINCT FROM OLD.source_id + OR NEW.generation IS DISTINCT FROM OLD.generation OR NEW.source_bytes IS DISTINCT FROM OLD.source_bytes + OR NEW.primary_scope IS DISTINCT FROM OLD.primary_scope OR NEW.created_at IS DISTINCT FROM OLD.created_at + OR (OLD.receipt_bytes IS NOT NULL AND NEW.receipt_bytes IS DISTINCT FROM OLD.receipt_bytes) + OR (OLD.map_id IS NOT NULL AND NEW.map_id IS DISTINCT FROM OLD.map_id) + OR (OLD.map_generation IS NOT NULL AND NEW.map_generation IS DISTINCT FROM OLD.map_generation) + OR (OLD.create_completed AND NOT NEW.create_completed) + OR (OLD.create_completed_at IS NOT NULL AND NEW.create_completed_at IS DISTINCT FROM OLD.create_completed_at) + OR (OLD.state='APPLIED' AND NEW.state<>'APPLIED') + OR (OLD.state='DELETING' AND NEW.state NOT IN ('DELETING','APPLIED')) + OR (OLD.state='LIVE' AND NEW.state NOT IN ('LIVE','DELETING')) + OR (OLD.state='CREATING' AND NEW.state NOT IN ('CREATING','LIVE','DELETING')) + OR (OLD.state='RESERVED' AND NEW.state NOT IN ('RESERVED','CREATING','DELETING','APPLIED'))) THEN + RAISE EXCEPTION 'chunk receipt generation identity or transition changed'; + END IF; + IF NEW.state='DELETING' AND (TG_OP='INSERT' OR OLD.state<>'DELETING') THEN + EXECUTE pg_catalog.format('SELECT EXISTS(SELECT 1 FROM %I.mst2_chunk_reader WHERE receipt_key=$1 AND source_generation=$2 AND deadline>pg_catalog.clock_timestamp())',TG_TABLE_SCHEMA) + INTO owned USING NEW.receipt_key,NEW.generation; + IF owned THEN RAISE EXCEPTION 'receipt retirement cannot cross a live reader'; END IF; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_chunk_generation_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_chunk_receipt_generation + FOR EACH ROW EXECUTE FUNCTION mst2_chunk_generation_guard(); +CREATE TRIGGER mst2_chunk_generation_no_truncate BEFORE TRUNCATE ON mst2_chunk_receipt_generation + FOR EACH STATEMENT EXECUTE FUNCTION mst2_chunk_generation_guard(); + +CREATE FUNCTION mst2_chunk_gc_history_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE full_history boolean; +BEGIN + IF pg_catalog.current_schema() IS DISTINCT FROM TG_TABLE_SCHEMA THEN RAISE EXCEPTION 'wrong chunk history schema'; END IF; + PERFORM mst2_chunk_retention_barrier(); + IF TG_OP='INSERT' THEN + EXECUTE pg_catalog.format('SELECT pg_catalog.count(*)>=131072 FROM %I.mst2_chunk_map_gc',TG_TABLE_SCHEMA) INTO full_history; + IF full_history THEN RAISE EXCEPTION 'chunk collection history quota exhausted'; END IF; + END IF; + IF TG_OP='DELETE' OR TG_OP='TRUNCATE' THEN RAISE EXCEPTION 'chunk collection history is retained'; END IF; + IF mst2_chunk_retention_bytes()+262144>536870912 THEN RAISE EXCEPTION 'chunk collection history lacks actual durable byte headroom'; END IF; + IF TG_OP='UPDATE' AND (NEW.map_id IS DISTINCT FROM OLD.map_id OR NEW.generation IS DISTINCT FROM OLD.generation + OR NEW.created_at IS DISTINCT FROM OLD.created_at OR OLD.state='APPLIED' OR NEW.state<>'APPLIED') THEN + RAISE EXCEPTION 'chunk collection history is immutable except terminal completion'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_chunk_gc_history_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_chunk_map_gc + FOR EACH ROW EXECUTE FUNCTION mst2_chunk_gc_history_guard(); +CREATE TRIGGER mst2_chunk_gc_no_truncate BEFORE TRUNCATE ON mst2_chunk_map_gc + FOR EACH STATEMENT EXECUTE FUNCTION mst2_chunk_gc_history_guard(); + +CREATE FUNCTION mst2_chunk_reader_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE live boolean; +BEGIN + IF pg_catalog.current_schema() IS DISTINCT FROM TG_TABLE_SCHEMA THEN RAISE EXCEPTION 'wrong chunk reader schema'; END IF; + PERFORM mst2_chunk_retention_barrier(); + IF TG_OP='TRUNCATE' THEN RAISE EXCEPTION 'chunk readers cannot be truncated'; END IF; + IF TG_OP='DELETE' THEN RETURN OLD; END IF; + IF mst2_chunk_retention_bytes()+1048576>536870912 THEN RAISE EXCEPTION 'chunk reader mutation lacks actual durable byte headroom'; END IF; + IF TG_OP='INSERT' THEN + EXECUTE pg_catalog.format('SELECT pg_catalog.count(*)>=4096 FROM %I.mst2_chunk_reader',TG_TABLE_SCHEMA) INTO live; + IF live THEN RAISE EXCEPTION 'chunk reader quota exhausted'; END IF; + END IF; + IF NEW.deadline>pg_catalog.clock_timestamp()+interval '60 seconds' OR NEW.deadline<=pg_catalog.clock_timestamp() + OR (TG_OP='UPDATE' AND (OLD.deadline<=pg_catalog.clock_timestamp() OR NEW.owner IS DISTINCT FROM OLD.owner + OR NEW.receipt_key IS DISTINCT FROM OLD.receipt_key OR NEW.source_generation IS DISTINCT FROM OLD.source_generation + OR NEW.map_id IS DISTINCT FROM OLD.map_id OR NEW.map_generation IS DISTINCT FROM OLD.map_generation)) THEN + RAISE EXCEPTION 'chunk reader cannot renew an expired or different owner'; + END IF; + EXECUTE pg_catalog.format('SELECT EXISTS(SELECT 1 FROM %I.mst2_chunk_receipt_generation g JOIN %I.mst2_chunk_map_lifetime l ON l.map_id=g.map_id AND l.generation=g.map_generation JOIN %I.mst2_chunk_map_source s ON s.receipt_key=g.receipt_key AND s.receipt_generation=g.generation WHERE g.receipt_key=$1 AND g.generation=$2 AND g.map_id=$3 AND g.map_generation=$4 AND g.state=''LIVE'' AND l.state=''LIVE'')',TG_TABLE_SCHEMA,TG_TABLE_SCHEMA,TG_TABLE_SCHEMA) + INTO live USING NEW.receipt_key,NEW.source_generation,NEW.map_id,NEW.map_generation; + IF NOT live THEN RAISE EXCEPTION 'chunk reader lacks exact live source and map generations'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_chunk_reader_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_chunk_reader + FOR EACH ROW EXECUTE FUNCTION mst2_chunk_reader_guard(); +CREATE TRIGGER mst2_chunk_reader_no_truncate BEFORE TRUNCATE ON mst2_chunk_reader + FOR EACH STATEMENT EXECUTE FUNCTION mst2_chunk_reader_guard(); + +CREATE FUNCTION mst2_chunk_map_lifetime_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE permitted boolean; +BEGIN + IF pg_catalog.current_schema() IS DISTINCT FROM TG_TABLE_SCHEMA THEN RAISE EXCEPTION 'wrong chunk map lifetime schema'; END IF; + PERFORM mst2_chunk_retention_barrier(); + IF TG_OP='TRUNCATE' THEN RAISE EXCEPTION 'chunk map lifetimes cannot be truncated'; END IF; + IF TG_OP='DELETE' THEN + EXECUTE pg_catalog.format('SELECT $2=''DELETING'' AND EXISTS(SELECT 1 FROM %I.mst2_chunk_map_gc g WHERE g.map_id=$1 AND g.generation=$3 AND g.state=''PENDING'') AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_map_source s WHERE s.map_id=$1) AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_reader r WHERE r.map_id=$1 AND r.map_generation=$3 AND r.deadline>pg_catalog.clock_timestamp())',TG_TABLE_SCHEMA,TG_TABLE_SCHEMA,TG_TABLE_SCHEMA) + INTO permitted USING OLD.map_id,OLD.state,OLD.generation; + IF NOT permitted THEN RAISE EXCEPTION 'map lifetime deletion lacks exact reader-free collection claim'; END IF; + RETURN OLD; + END IF; + IF mst2_chunk_retention_bytes()+CASE WHEN NEW.state='DELETING' THEN 262144 ELSE 1048576 END>536870912 THEN + RAISE EXCEPTION 'chunk map lifetime mutation lacks actual durable byte headroom'; + END IF; + IF TG_OP='UPDATE' AND (NEW.map_id IS DISTINCT FROM OLD.map_id OR NEW.generation IS DISTINCT FROM OLD.generation + OR (OLD.state='DELETING' AND NEW.state<>'DELETING')) THEN RAISE EXCEPTION 'map lifetime cannot reincarnate in place'; END IF; + IF TG_OP='UPDATE' AND NEW.state='DELETING' AND OLD.state='LIVE' THEN + EXECUTE pg_catalog.format('SELECT EXISTS(SELECT 1 FROM %I.mst2_chunk_map_gc g WHERE g.map_id=$1 AND g.generation=$2 AND g.state=''PENDING'') AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_map_source WHERE map_id=$1) AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_reader WHERE map_id=$1 AND map_generation=$2 AND deadline>pg_catalog.clock_timestamp()) AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_receipt_generation WHERE map_id=$1 AND map_generation=$2 AND state=''CREATING'' AND deadline>pg_catalog.clock_timestamp())',TG_TABLE_SCHEMA,TG_TABLE_SCHEMA,TG_TABLE_SCHEMA,TG_TABLE_SCHEMA) + INTO permitted USING OLD.map_id,OLD.generation; + IF NOT permitted THEN RAISE EXCEPTION 'map collection cannot cross a reader or active partial installation'; END IF; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_chunk_map_lifetime_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_chunk_map_lifetime + FOR EACH ROW EXECUTE FUNCTION mst2_chunk_map_lifetime_guard(); +CREATE TRIGGER mst2_chunk_map_lifetime_no_truncate BEFORE TRUNCATE ON mst2_chunk_map_lifetime + FOR EACH STATEMENT EXECUTE FUNCTION mst2_chunk_map_lifetime_guard(); diff --git a/src/jupiter/migration/mod.rs b/src/jupiter/migration/mod.rs index 2fbc20ce..018716bc 100644 --- a/src/jupiter/migration/mod.rs +++ b/src/jupiter/migration/mod.rs @@ -153,6 +153,7 @@ mod m20261007_000500_add_mst2_install_capability; mod m20261007_000600_add_mst2_storage_routes; mod m20261008_000100_add_mst2_chunk_maps; mod m20261008_000200_add_mst2_rooted_qualified_family; +mod m20261008_000300_add_mst2_chunk_map_retention; mod runner; pub use m20260905_000100_add_push_queue::ensure_queue_control_seed; pub use runner::apply_migrations; @@ -294,6 +295,7 @@ impl MigratorTrait for Migrator { Box::new(m20261007_000600_add_mst2_storage_routes::Migration), Box::new(m20261008_000100_add_mst2_chunk_maps::Migration), Box::new(m20261008_000200_add_mst2_rooted_qualified_family::Migration), + Box::new(m20261008_000300_add_mst2_chunk_map_retention::Migration), ] } } @@ -1204,7 +1206,7 @@ mod tests { async fn import_repo_alias_rows_canonicalized() { let names = migration_names(); assert_eq!( - &names[names.len() - 16..names.len() - 7], + &names[names.len() - 17..names.len() - 8], &[ "m20260923_000200_canonicalize_import_repo_paths".to_string(), "m20260925_000100_media_paging".to_string(), @@ -1219,34 +1221,38 @@ mod tests { "native retention, metadata installation and view tables follow media paging" ); assert_eq!( - &names[names.len() - 7], + &names[names.len() - 8], "m20261007_000200_add_mst2_metadata_generations" ); assert_eq!( - &names[names.len() - 6], + &names[names.len() - 7], "m20261007_000300_add_mst2_metadata_lifetime_history" ); assert_eq!( - &names[names.len() - 5], + &names[names.len() - 6], "m20261007_000400_add_mst2_qualified_metadata_gc" ); assert_eq!( - &names[names.len() - 4], + &names[names.len() - 5], "m20261007_000500_add_mst2_install_capability" ); assert_eq!( - &names[names.len() - 3], + &names[names.len() - 4], "m20261007_000600_add_mst2_storage_routes" ); assert_eq!( - &names[names.len() - 2], + &names[names.len() - 3], "m20261008_000100_add_mst2_chunk_maps" ); assert_eq!( - names.last().unwrap(), + &names[names.len() - 2], "m20261008_000200_add_mst2_rooted_qualified_family" ); + assert_eq!( + names.last().unwrap(), + "m20261008_000300_add_mst2_chunk_map_retention" + ); let db = alias_db().await; insert_repo(&db, 1, "/third-party//a").await; insert_repo(&db, 2, "/third-party/b/").await; diff --git a/src/jupiter/storage/native_chunk_map.rs b/src/jupiter/storage/native_chunk_map.rs index 5ce849a8..1d5e5666 100644 --- a/src/jupiter/storage/native_chunk_map.rs +++ b/src/jupiter/storage/native_chunk_map.rs @@ -2,8 +2,8 @@ //! //! The object writer is trusted like the verified-object writer. A database //! row, its checksum or a map's presence is never a full-source proof. Only -//! the opaque full-stream verifier can install a source receipt. This initial -//! store is append-only; bounded retention and collection remain separate work. +//! the opaque full-stream verifier can install a source receipt. Persistent +//! owners, reservations and physical generations bound its retention. use std::sync::Arc; @@ -33,11 +33,17 @@ const DESCRIPTOR_CREDIT: usize = 32 * 1024; const PAGE_CREDIT: usize = 64 * 1024; const RECEIPT_DOMAIN: &[u8] = b"MST2-CHUNK-MAP-RECEIPT\0"; +pub(crate) mod retention; + +#[derive(Clone)] pub(crate) struct PostgresChunkMapRepository { connection: DatabaseConnection, primary_scope: Vec, budget: Arc, schema: String, + receipt_observation: Arc>>>, + cancelled_installs: Arc>>, + admission_maintenance: Arc>>, } pub(crate) struct PersistedChunkMap { @@ -45,6 +51,7 @@ pub(crate) struct PersistedChunkMap { pub map_id: [u8; 32], source: ChunkMapSource, source_id: [u8; 32], + reader: retention::ChunkMapReader, _memory: MemoryLease, } @@ -52,6 +59,26 @@ impl PersistedChunkMap { pub(crate) fn source_id(&self) -> [u8; 32] { self.source_id } + + pub(crate) async fn ensure_live(&self) -> Result<(), SnapshotError> { + self.reader.ensure_live().await + } + + pub(crate) async fn record_progress(&self) -> Result<(), SnapshotError> { + self.reader.record_progress().await + } + + pub(crate) async fn await_backend( + &self, + future: F, + ) -> Result { + self.reader.await_backend(future).await + } + + #[cfg(test)] + pub(crate) async fn test_check_next_owner_operation(&self) { + self.reader.test_check_next_owner_operation().await; + } } pub(crate) struct AuthenticatedChunkPage { @@ -100,6 +127,9 @@ impl PostgresChunkMapRepository { primary_scope, budget: projection_budget().clone(), schema, + receipt_observation: Arc::default(), + cancelled_installs: Arc::default(), + admission_maintenance: Arc::default(), }) } @@ -108,27 +138,78 @@ impl PostgresChunkMapRepository { source: &ChunkMapSource, objects: &MegaObjectStorageWrapper, ) -> Result>, SnapshotError> { + for pass in 0..2 { + let (map, pending) = self.read_current(source, objects).await?; + if !pending { + return Ok(map); + } + if pass == 0 { + self.maintain(objects, 8).await?; + } + } + Err(SnapshotError::new( + SnapshotErrorCode::TemporaryUnavailable, + "previous source receipt deletion is still pending", + )) + } + + async fn read_current( + &self, + source: &ChunkMapSource, + objects: &MegaObjectStorageWrapper, + ) -> Result<(Option>, bool), SnapshotError> { let memory = self.budget.reserve(DESCRIPTOR_CREDIT)?; let txn = self.transaction().await?; let result = async { self.require_scope(&txn).await?; require_current_source(&txn, source, &self.schema).await?; let Some(row) = source_row(&txn, source, &self.schema).await? else { - return Ok(None); + let pending = self.require_missing_source_retired(&txn, source).await?; + return Ok((None, pending)); }; let (map, source_id) = self.validate_source_row(source, &row)?; + let generation_state: Option = + row.try_get("", "generation_state").map_err(db_error)?; + if generation_state.as_deref() == Some("DELETING") { + // A legitimate retirement can still retain its source row + // until the physical delete completes. Replay it just like + // a missing retired row, after checking every source fact; + // damaged LIVE rows never enter this cold-rebuild path. + return Ok((None, true)); + } let bytes = receipt(&self.primary_scope, source, &map)?; - read_receipt(objects, &receipt_key(source_id), &bytes).await?; - Ok(Some(Arc::new(PersistedChunkMap { - map_id: map.map_id(), - map, - source: source.clone(), - source_id, - _memory: memory, - }))) + let reader = self.admit_reader(&txn, source, &row, &map).await?; + let key = ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: row.try_get("", "receipt_key").map_err(db_error)?, + }; + Ok(( + Some(( + Arc::new(PersistedChunkMap { + map_id: map.map_id(), + map, + source: source.clone(), + source_id, + reader, + _memory: memory, + }), + key, + bytes, + )), + false, + )) } .await; - finish(txn, result).await + let (admitted, pending) = finish(txn, result).await?; + let Some((map, key, bytes)) = admitted else { + return Ok((None, pending)); + }; + // Admission is durable before I/O. No completion barrier or fact row + // lock is held while the independent backend receipt is consumed. + map.await_backend(read_receipt(objects, &key, &bytes)) + .await??; + map.reader.validate_after_receipt().await?; + Ok((Some(map), false)) } /// Publication accepts no caller-provided map or "verified" flag. @@ -136,75 +217,96 @@ impl PostgresChunkMapRepository { &self, verified: VerifiedSourceChunkMap, objects: &MegaObjectStorageWrapper, + admission: &retention::ChunkMapInstall, + ) -> Result<(), SnapshotError> { + let result = self.install_reserved(verified, objects, admission).await; + if result.is_err() { + if let Err(error) = admission.retire().await { + tracing::warn!( + ?error, + "failed chunk-map installation will retire by database deadline" + ); + } else if let Err(error) = self.collect_maps(64).await { + tracing::warn!( + ?error, + "retired partial chunk-map indexes await bounded collection replay" + ); + } + } + result + } + + async fn install_reserved( + &self, + verified: VerifiedSourceChunkMap, + objects: &MegaObjectStorageWrapper, + admission: &retention::ChunkMapInstall, ) -> Result<(), SnapshotError> { let source = verified.source(); let map = verified.map(); - // The verifier reserved index and bounded SQL parameter workspace - // before opening the body. It owns those credits through publication. let nodes = indexed_nodes(verified.leaf_hashes())?; - if nodes.last().map(|n| n.digest) != Some(map.pages_root) { + if nodes.last().map(|node| node.digest) != Some(map.pages_root) { return Err(integrity( "verified chunk map index disagrees with canonical root", )); } - let source_bytes = source.canonical_bytes()?; - let source_id = source_id(&self.primary_scope, &source_bytes); - let bytes = receipt(&self.primary_scope, source, map)?; - let key = receipt_key(source_id); + let bytes = admission.prepare(&verified).await?; + let key = admission.key(); + admission + .await_backend(objects.inner.put_metadata_atomic_create( + key, + Bytes::copy_from_slice(&bytes), + ObjectMeta { + size: bytes.len() as i64, + ..Default::default() + }, + )) + .await? + .map_err(storage_error)?; + admission.acknowledge_create().await?; + admission + .await_backend(read_receipt(objects, key, &bytes)) + .await??; + let observation = retention::inventory(self, objects).await?; + let map_generation = self + .stage_map_start(&verified, admission, &observation) + .await?; + for batch in verified.leaves().chunks(64) { + self.stage_map_batch(&verified, admission, map_generation, Some(batch), None) + .await?; + } + for batch in nodes.chunks(1024) { + self.stage_map_batch(&verified, admission, map_generation, None, Some(batch)) + .await?; + } + self.compare_staged_map(&verified, &nodes, admission, map_generation) + .await?; let txn = self.transaction().await?; - let result = async { + let result=async { self.require_scope(&txn).await?; - require_current_source(&txn, source, &self.schema).await?; - if let Some(row) = source_row(&txn, source, &self.schema).await? { - let (stored, _) = self.validate_source_row(source, &row)?; - if stored != *map { return Err(integrity("immutable source map conflicts with reverified content")); } - read_receipt(objects, &key, &bytes).await?; - compare_complete_map(&txn, &verified, &nodes, &self.schema).await?; - return Ok(()); - } - // The complete source pass happened before this atomic create. - // A lost DB commit leaves an orphan trusted receipt, never an - // admitted partial map; exact re-verification can replay it. - objects.inner.put_metadata_atomic_create(&key, Bytes::copy_from_slice(&bytes), ObjectMeta { - size: bytes.len() as i64, ..Default::default() - }).await.map_err(storage_error)?; - read_receipt(objects, &key, &bytes).await?; - txn.execute_raw(stmt(&self.schema, "INSERT INTO mst2_chunk_map(map_id,descriptor,page_count,pages_root) VALUES($1,$2,$3,$4) ON CONFLICT(map_id) DO NOTHING", - [map.map_id().to_vec().into(), map.encode().into(), (map.page_count as i32).into(), map.pages_root.to_vec().into()])).await.map_err(db_error)?; - // Existing indexes must be compared, never repaired or overwritten. - let present = txn.query_one_raw(stmt(&self.schema, "SELECT EXISTS(SELECT 1 FROM mst2_chunk_map_source WHERE map_id=$1) AS sealed", [map.map_id().to_vec().into()])) - .await.map_err(db_error)?.ok_or_else(|| integrity("chunk map seal observation is missing"))? - .try_get::("", "sealed").map_err(db_error)?; - if !present { - for batch in verified.leaves().chunks(64) { - let encoded = encode_leaves(batch)?; - txn.execute_raw(stmt(&self.schema, "INSERT INTO mst2_chunk_map_leaf(map_id,page_index,payload) SELECT $1,p.page_index,pg_catalog.decode(p.payload,'hex') FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(page_index integer,payload text) ON CONFLICT(map_id,page_index) DO NOTHING", - [map.map_id().to_vec().into(), encoded.into()])).await.map_err(db_error)?; - } - for batch in nodes.chunks(1024) { - let encoded = encode_nodes(batch)?; - txn.execute_raw(stmt(&self.schema, "INSERT INTO mst2_chunk_map_node(map_id,first_page,page_count,digest) SELECT $1,p.first_page,p.page_count,pg_catalog.decode(p.digest,'hex') FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(first_page integer,page_count integer,digest text) ON CONFLICT(map_id,first_page,page_count) DO NOTHING", - [map.map_id().to_vec().into(), encoded.into()])).await.map_err(db_error)?; - } - } - compare_complete_map(&txn, &verified, &nodes, &self.schema).await?; - let fact = source.fact(); - txn.execute_raw(stmt(&self.schema, "INSERT INTO mst2_chunk_map_source(storage_domain,git_oid,object_kind,fact_id,source_id,source_bytes,primary_scope,map_id,receipt_digest) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT(storage_domain,git_oid,object_kind) DO NOTHING", - [fact.storage_domain.clone().into(), fact.git_oid.clone().into(), fact.object_kind.clone().into(), fact.id.into(), - source_id.to_vec().into(), source_bytes.into(), self.primary_scope.clone().into(), map.map_id().to_vec().into(), Sha256::digest(&bytes).to_vec().into()])).await.map_err(db_error)?; - let row = source_row(&txn, source, &self.schema).await?.ok_or_else(|| integrity("installed source receipt is missing"))?; - let (stored, _) = self.validate_source_row(source, &row)?; - if stored != *map { return Err(integrity("concurrent source installation conflicts")); } + require_current_source(&txn,source,&self.schema).await?; + retention::map_barrier(&txn,map.map_id()).await?; + retention::barrier(&txn,&self.schema).await?; + retention::ensure_quotas(&txn,&self.schema,&observation,0,0,0,Some(admission.owner())).await?; + let admitted=txn.query_one_raw(stmt(&self.schema,"UPDATE mst2_chunk_receipt_generation SET state='LIVE',owner=NULL,deadline=NULL,reserved_pages=0,reserved_nodes=0,reserved_bytes=0,last_progress=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state='CREATING' AND deadline>pg_catalog.clock_timestamp() AND create_completed AND map_id=$4 AND map_generation=$5 AND receipt_bytes=$6 AND EXISTS(SELECT 1 FROM mst2_chunk_map_lifetime WHERE map_id=$4 AND generation=$5 AND state='LIVE') RETURNING generation",[key.key.clone().into(),admission.generation().into(),admission.owner().to_string().into(),map.map_id().to_vec().into(),map_generation.into(),bytes.clone().into()])).await.map_err(db_error)?; + if admitted.is_none() {return Err(SnapshotError::new(SnapshotErrorCode::LeaseExpired,"chunk-map publication lost its exact live install reservation"));} + let fact=source.fact(); + let source_bytes=source.canonical_bytes()?; + let source_id=source_id(&self.primary_scope,&source_bytes); + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_map_source(storage_domain,git_oid,object_kind,fact_id,source_id,source_bytes,primary_scope,map_id,receipt_digest,receipt_generation) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT(storage_domain,git_oid,object_kind) DO NOTHING",[fact.storage_domain.clone().into(),fact.git_oid.clone().into(),fact.object_kind.clone().into(),fact.id.into(),source_id.to_vec().into(),source_bytes.into(),self.primary_scope.clone().into(),map.map_id().to_vec().into(),Sha256::digest(&bytes).to_vec().into(),admission.generation().into()])).await.map_err(db_error)?; + let row=source_row(&txn,source,&self.schema).await?.ok_or_else(||integrity("installed source receipt is missing"))?; + let (stored,_)=self.validate_source_row(source,&row)?; + if stored!=*map || row.try_get::("","receipt_key").map_err(db_error)?!=key.key {return Err(integrity("concurrent source installation conflicts"));} Ok(()) }.await; finish(txn, result).await } - pub(crate) async fn selected_page( &self, map: &PersistedChunkMap, page_index: u64, ) -> Result, SnapshotError> { + map.ensure_live().await?; let intervals = proof_intervals(map.map.page_count, page_index)?; let memory = self.budget.reserve(PAGE_CREDIT)?; let txn = self.transaction().await?; @@ -240,7 +342,10 @@ impl PostgresChunkMapRepository { tracing::debug!(target:"mst2::chunk_map", selected_leaf_bytes = bytes.len(), selected_sibling_rows = nodes.len(), page_count = map.map.page_count, "authenticated persisted chunk map selected page"); Ok(Arc::new(AuthenticatedChunkPage { leaf, proof, _memory: memory })) }.await; - finish(txn, result).await + let page = finish(txn, result).await?; + map.ensure_live().await?; + map.record_progress().await?; + Ok(page) } async fn transaction(&self) -> Result { @@ -412,7 +517,7 @@ async fn source_row( source: &ChunkMapSource, schema: &str, ) -> Result, SnapshotError> { - connection.query_one_raw(stmt(schema, "SELECT s.fact_id,CASE WHEN pg_catalog.octet_length(s.source_id)=32 THEN s.source_id ELSE NULL END AS source_id,CASE WHEN pg_catalog.octet_length(s.source_bytes)<=2048 THEN s.source_bytes ELSE NULL END AS source_bytes,CASE WHEN pg_catalog.octet_length(s.primary_scope)<=1024 THEN s.primary_scope ELSE NULL END AS primary_scope,CASE WHEN pg_catalog.octet_length(s.map_id)=32 THEN s.map_id ELSE NULL END AS map_id,CASE WHEN pg_catalog.octet_length(s.receipt_digest)=32 THEN s.receipt_digest ELSE NULL END AS receipt_digest,CASE WHEN pg_catalog.octet_length(m.descriptor)=100 THEN m.descriptor ELSE NULL END AS descriptor,m.page_count,CASE WHEN pg_catalog.octet_length(m.pages_root)=32 THEN m.pages_root ELSE NULL END AS pages_root FROM mst2_chunk_map_source s LEFT JOIN mst2_chunk_map m ON m.map_id=s.map_id WHERE s.storage_domain=$1 AND s.git_oid=$2 AND s.object_kind=$3", + connection.query_one_raw(stmt(schema, "SELECT s.fact_id,s.receipt_key,s.receipt_generation,g.state AS generation_state,l.state AS map_state,l.generation AS map_generation,CASE WHEN pg_catalog.octet_length(g.receipt_bytes)<=4096 THEN g.receipt_bytes ELSE NULL END AS generation_receipt_bytes,CASE WHEN pg_catalog.octet_length(s.source_id)=32 THEN s.source_id ELSE NULL END AS source_id,CASE WHEN pg_catalog.octet_length(s.source_bytes)<=2048 THEN s.source_bytes ELSE NULL END AS source_bytes,CASE WHEN pg_catalog.octet_length(s.primary_scope)<=1024 THEN s.primary_scope ELSE NULL END AS primary_scope,CASE WHEN pg_catalog.octet_length(s.map_id)=32 THEN s.map_id ELSE NULL END AS map_id,CASE WHEN pg_catalog.octet_length(s.receipt_digest)=32 THEN s.receipt_digest ELSE NULL END AS receipt_digest,CASE WHEN pg_catalog.octet_length(m.descriptor)=100 THEN m.descriptor ELSE NULL END AS descriptor,m.page_count,CASE WHEN pg_catalog.octet_length(m.pages_root)=32 THEN m.pages_root ELSE NULL END AS pages_root FROM mst2_chunk_map_source s LEFT JOIN mst2_chunk_map m ON m.map_id=s.map_id LEFT JOIN mst2_chunk_receipt_generation g ON g.receipt_key=s.receipt_key AND g.generation=s.receipt_generation AND g.map_id=s.map_id LEFT JOIN mst2_chunk_map_lifetime l ON l.map_id=s.map_id AND l.generation=g.map_generation WHERE s.storage_domain=$1 AND s.git_oid=$2 AND s.object_kind=$3", [source.fact().storage_domain.clone().into(), source.fact().git_oid.clone().into(), source.fact().object_kind.clone().into()])).await.map_err(db_error) } @@ -447,13 +552,6 @@ fn receipt( Ok(bytes) } -fn receipt_key(source_id: [u8; 32]) -> ObjectKey { - ObjectKey { - namespace: ObjectNamespace::ChunkMapReceipt, - key: hex::encode(source_id), - } -} - async fn read_receipt( objects: &MegaObjectStorageWrapper, key: &ObjectKey, @@ -505,46 +603,6 @@ fn encode_nodes(nodes: &[ChunkMapNode]) -> Result { serde_json::to_string(&nodes.iter().map(|n| json!({"first_page":n.start,"page_count":n.pages,"digest":hex::encode(n.digest)})).collect::>()).map_err(db_error) } -async fn compare_complete_map( - txn: &DatabaseTransaction, - verified: &VerifiedSourceChunkMap, - nodes: &[ChunkMapNode], - schema: &str, -) -> Result<(), SnapshotError> { - let map = verified.map(); - let row = txn.query_one_raw(stmt(schema, "SELECT CASE WHEN pg_catalog.octet_length(descriptor)=100 THEN descriptor ELSE NULL END AS descriptor,page_count,CASE WHEN pg_catalog.octet_length(pages_root)=32 THEN pages_root ELSE NULL END AS pages_root,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_leaf WHERE map_id=$1) AS leaves,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_node WHERE map_id=$1) AS nodes FROM mst2_chunk_map WHERE map_id=$1", [map.map_id().to_vec().into()])) - .await.map_err(db_error)?.ok_or_else(|| integrity("chunk map installation descriptor is missing"))?; - if bounded_bytes(&row, "descriptor")? != map.encode() - || row.try_get::("", "page_count").map_err(db_error)? as u64 != map.page_count - || bounded_bytes(&row, "pages_root")? != map.pages_root - || row.try_get::("", "leaves").map_err(db_error)? as u64 != map.page_count - || row.try_get::("", "nodes").map_err(db_error)? as usize != nodes.len() - { - return Err(integrity( - "chunk map installation has conflicting descriptor or incomplete coverage", - )); - } - for batch in verified.leaves().chunks(64) { - let bad = txn.query_one_raw(stmt(schema, "SELECT p.page_index FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(page_index integer,payload text) LEFT JOIN mst2_chunk_map_leaf l ON l.map_id=$1 AND l.page_index=p.page_index WHERE l.payload IS DISTINCT FROM pg_catalog.decode(p.payload,'hex') LIMIT 1", - [map.map_id().to_vec().into(), encode_leaves(batch)?.into()])).await.map_err(db_error)?; - if bad.is_some() { - return Err(integrity( - "immutable stored chunk map leaf conflicts with verified source", - )); - } - } - for batch in nodes.chunks(1024) { - let bad = txn.query_one_raw(stmt(schema, "SELECT p.first_page FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(first_page integer,page_count integer,digest text) LEFT JOIN mst2_chunk_map_node n ON n.map_id=$1 AND n.first_page=p.first_page AND n.page_count=p.page_count WHERE n.digest IS DISTINCT FROM pg_catalog.decode(p.digest,'hex') LIMIT 1", - [map.map_id().to_vec().into(), encode_nodes(batch)?.into()])).await.map_err(db_error)?; - if bad.is_some() { - return Err(integrity( - "immutable stored chunk map node conflicts with verified source", - )); - } - } - Ok(()) -} - fn quoted(schema: &str) -> String { format!("\"{}\"", schema.replace('"', "\"\"")) } @@ -591,6 +649,11 @@ fn qualify_relations(schema: &str, sql: &str) -> String { "mst2_chunk_map_source", "mst2_chunk_map_leaf", "mst2_chunk_map_node", + "mst2_chunk_receipt_generation", + "mst2_chunk_map_lifetime", + "mst2_chunk_reader", + "mst2_chunk_map_gc", + "mst2_chunk_retention_barrier", ] .contains(&&sql[start..cursor]) { diff --git a/src/jupiter/storage/native_chunk_map/retention.rs b/src/jupiter/storage/native_chunk_map/retention.rs new file mode 100644 index 00000000..82a74da2 --- /dev/null +++ b/src/jupiter/storage/native_chunk_map/retention.rs @@ -0,0 +1,1220 @@ +//! Bounded primary owners and exact physical-generation reconciliation. + +use std::{ + sync::OnceLock, + time::{Duration, Instant}, +}; + +use tokio::sync::Mutex; +use uuid::Uuid; + +use super::*; +use crate::orbit_api::{ + error::IoOrbitError, + object_storage::{ChunkMapReceiptDeletion, ChunkMapReceiptInventory}, +}; + +const RENEW_INTERVAL: Duration = Duration::from_secs(10); +const HISTORY_CAP: i64 = 131_072; +const MAINTENANCE_MAX: usize = 64; +const OWNER_TTL: Duration = Duration::from_secs(59); + +struct OwnerDeadline(std::sync::Mutex); + +impl OwnerDeadline { + fn from_sql_start(started: Instant) -> Self { + // The SQL INSERT/renew sets clock_timestamp()+59s after this sample. + // Counting SQL and commit latency against the TTL is conservative. + Self(std::sync::Mutex::new(started + OWNER_TTL)) + } + + fn get(&self) -> Result { + let deadline = *self + .0 + .lock() + .map_err(|_| integrity("owner deadline is poisoned"))?; + if deadline <= Instant::now() { + return Err(expired("chunk-map ownership deadline elapsed")); + } + Ok(deadline) + } + + fn observe(&self, deadline: Instant, renewed: bool) -> Result<(), SnapshotError> { + let mut previous = self + .0 + .lock() + .map_err(|_| integrity("owner deadline is poisoned"))?; + *previous = if renewed { + deadline + } else { + (*previous).min(deadline) + }; + if *previous <= Instant::now() { + return Err(expired("chunk-map ownership deadline elapsed")); + } + Ok(()) + } + + async fn bound(&self, future: F) -> Result { + let deadline = self.get()?; + let result = tokio::time::timeout_at(tokio::time::Instant::from_std(deadline), future) + .await + .map_err(|_| expired("chunk-map ownership deadline elapsed"))?; + self.get()?; + Ok(result) + } +} + +fn remaining_deadline(started: Instant, row: &QueryResult) -> Result { + let remaining = row.try_get::("", "remaining_us").map_err(db_error)?; + let remaining = u64::try_from(remaining) + .ok() + .filter(|value| *value > 0) + .ok_or_else(|| expired("chunk-map database ownership deadline elapsed"))?; + started + .checked_add(Duration::from_micros(remaining)) + .ok_or_else(|| integrity("chunk-map ownership deadline is not representable")) +} + +fn retention_budget() -> &'static Arc { + // Inventory survives requests. Its separate process-wide credits bound + // independent repositories without retaining source or response credit. + static BUDGET: OnceLock> = OnceLock::new(); + BUDGET.get_or_init(|| MemoryBudget::new(128 * 1024 * 1024)) +} + +pub(super) struct BackingObservation { + inventory: ChunkMapReceiptInventory, + started_at: String, + captured: Instant, + backing: MegaObjectStorageWrapper, + _memory: MemoryLease, +} + +pub(crate) struct ChunkMapReader { + repository: PostgresChunkMapRepository, + source: ChunkMapSource, + owner: Uuid, + receipt_key: String, + generation: i64, + map_id: [u8; 32], + map_generation: i64, + confirmed: Mutex<(Instant, Instant)>, + deadline: OwnerDeadline, +} + +impl ChunkMapReader { + #[cfg(test)] + pub(crate) async fn test_check_next_owner_operation(&self) { + let before = Instant::now() - Duration::from_secs(11); + *self.confirmed.lock().await = (before, before); + } + pub(crate) async fn ensure_live(&self) -> Result<(), SnapshotError> { + self.deadline.get()?; + let mut confirmed = self.confirmed.lock().await; + self.deadline.get()?; + if confirmed.0.elapsed() < RENEW_INTERVAL { + return Ok(()); + } + let deadline = self.deadline.bound(self.check(false)).await??; + self.deadline.observe(deadline, false)?; + confirmed.0 = Instant::now(); + Ok(()) + } + + pub(super) async fn record_progress(&self) -> Result<(), SnapshotError> { + self.deadline.get()?; + let mut confirmed = self.confirmed.lock().await; + self.deadline.get()?; + if confirmed.1.elapsed() < RENEW_INTERVAL { + return Ok(()); + } + let deadline = self.deadline.bound(self.check(true)).await??; + self.deadline.observe(deadline, true)?; + *confirmed = (Instant::now(), Instant::now()); + Ok(()) + } + + pub(super) async fn validate_after_receipt(&self) -> Result<(), SnapshotError> { + let mut confirmed = self.confirmed.lock().await; + self.deadline.get()?; + let deadline = self.deadline.bound(self.check(true)).await??; + self.deadline.observe(deadline, true)?; + *confirmed = (Instant::now(), Instant::now()); + Ok(()) + } + + pub(crate) async fn await_backend( + &self, + future: F, + ) -> Result { + self.ensure_live().await?; + tokio::pin!(future); + loop { + let deadline = self.deadline.get()?; + let check_at = self.confirmed.lock().await.0 + RENEW_INTERVAL; + tokio::select! { + biased; + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(deadline)) => { + return Err(expired("chunk-map reader backend wait outlived its ownership")); + } + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(check_at)) => { + self.ensure_live().await?; + } + value = &mut future => { + self.deadline.get()?; + return Ok(value); + } + } + } + } + + async fn check(&self, renew: bool) -> Result { + let started = Instant::now(); + let txn = self.repository.transaction().await?; + let result = async { + self.repository.require_scope(&txn).await?; + require_current_source(&txn, &self.source, &self.repository.schema).await?; + barrier(&txn, &self.repository.schema).await?; + let row = if renew {txn.query_one_raw(stmt(&self.repository.schema, + "UPDATE mst2_chunk_reader SET deadline=pg_catalog.clock_timestamp()+interval '59 seconds',last_progress=pg_catalog.clock_timestamp() WHERE owner=$1::uuid AND receipt_key=$2 AND source_generation=$3 AND map_id=$4 AND map_generation=$5 AND deadline>pg_catalog.clock_timestamp() RETURNING owner", + [self.owner.to_string().into(),self.receipt_key.clone().into(),self.generation.into(),self.map_id.to_vec().into(),self.map_generation.into()])).await.map_err(db_error)?} else { + txn.query_one_raw(stmt(&self.repository.schema,"SELECT r.owner FROM mst2_chunk_reader r JOIN mst2_chunk_receipt_generation g ON g.receipt_key=r.receipt_key AND g.generation=r.source_generation JOIN mst2_chunk_map_lifetime l ON l.map_id=r.map_id AND l.generation=r.map_generation WHERE r.owner=$1::uuid AND r.receipt_key=$2 AND r.source_generation=$3 AND r.map_id=$4 AND r.map_generation=$5 AND r.deadline>pg_catalog.clock_timestamp() AND g.state='LIVE' AND l.state='LIVE'",[self.owner.to_string().into(),self.receipt_key.clone().into(),self.generation.into(),self.map_id.to_vec().into(),self.map_generation.into()])).await.map_err(db_error)? + }; + if row.is_none() { return Err(expired("chunk-map reader expired before progress resumed")); } + if renew {txn.execute_raw(stmt(&self.repository.schema, + "UPDATE mst2_chunk_map_lifetime SET last_used=pg_catalog.clock_timestamp() WHERE map_id=$1 AND generation=$2 AND state='LIVE'", + [self.map_id.to_vec().into(),self.map_generation.into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.repository.schema,"UPDATE mst2_chunk_receipt_generation SET last_progress=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND state='LIVE'",[self.receipt_key.clone().into(),self.generation.into()])).await.map_err(db_error)?; + } + let remaining = txn.query_one_raw(stmt(&self.repository.schema, + "SELECT pg_catalog.floor(EXTRACT(EPOCH FROM (deadline-pg_catalog.clock_timestamp()))*1000000)::bigint AS remaining_us FROM mst2_chunk_reader WHERE owner=$1::uuid", + [self.owner.to_string().into()])).await.map_err(db_error)? + .ok_or_else(|| expired("chunk-map reader deadline disappeared"))?; + remaining_deadline(started, &remaining) + }.await; + finish(txn, result).await + } +} + +impl Drop for ChunkMapReader { + fn drop(&mut self) { + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return; + }; + let repository = self.repository.clone(); + let owner = self.owner; + runtime.spawn(async move { + let result = async { + let txn = repository.transaction().await?; + let result = async { + repository.require_scope(&txn).await?; + barrier(&txn, &repository.schema).await?; + txn.execute_raw(stmt( + &repository.schema, + "DELETE FROM mst2_chunk_reader WHERE owner=$1::uuid", + [owner.to_string().into()], + )) + .await + .map_err(db_error)?; + Ok(()) + } + .await; + finish(txn, result).await + } + .await; + if let Err(error) = result { + tracing::warn!( + ?error, + "chunk-map reader release will expire by database deadline" + ); + } + }); + } +} + +/// A capacity reservation exists before the verifier opens the source body. +/// It cannot publish source trust; installation still consumes the opaque +/// complete-stream verifier and independently checks the backend receipt. +pub(crate) struct ChunkMapInstall { + repository: PostgresChunkMapRepository, + source: ChunkMapSource, + owner: Uuid, + key: ObjectKey, + generation: i64, + progress: Mutex<(Instant, Instant, u64)>, + deadline: OwnerDeadline, +} + +#[derive(Clone)] +pub(super) struct CancelledInstall { + owner: Uuid, + key: String, + generation: i64, +} + +impl ChunkMapInstall { + #[cfg(test)] + pub(crate) async fn test_check_next_owner_operation(&self) { + let mut progress = self.progress.lock().await; + let before = Instant::now() - Duration::from_secs(11); + progress.0 = before; + } + pub(crate) async fn ensure_live(&self) -> Result<(), SnapshotError> { + self.renew(false, 0).await + } + + pub(crate) async fn record_progress(&self, completed: u64) -> Result<(), SnapshotError> { + self.renew(true, completed).await + } + + pub(crate) async fn await_backend( + &self, + future: F, + ) -> Result { + self.ensure_live().await?; + tokio::pin!(future); + loop { + let deadline = self.deadline.get()?; + let check_at = self.progress.lock().await.0 + RENEW_INTERVAL; + tokio::select! { + biased; + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(deadline)) => { + return Err(expired("chunk-map install backend wait outlived its ownership")); + } + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(check_at)) => { + self.ensure_live().await?; + } + value = &mut future => { + self.deadline.get()?; + return Ok(value); + } + } + } + } + + async fn renew(&self, progress: bool, completed: u64) -> Result<(), SnapshotError> { + self.deadline.get()?; + let mut confirmed = self.progress.lock().await; + self.deadline.get()?; + if progress && completed <= confirmed.2 { + return Ok(()); + } + if (if progress { + confirmed.1.elapsed() + } else { + confirmed.0.elapsed() + }) < RENEW_INTERVAL + { + if progress { + confirmed.2 = completed; + } + return Ok(()); + } + // Calling ensure_live after a long stalled await checks the old + // deadline without extending it. Only new source progress renews it. + let started = Instant::now(); + let deadline = self.deadline.bound(async { + let txn = self.repository.transaction().await?; + let result=async { + self.repository.require_scope(&txn).await?; + require_current_source(&txn,&self.source,&self.repository.schema).await?; + barrier(&txn,&self.repository.schema).await?; + let row=if progress { + txn.query_one_raw(stmt(&self.repository.schema,"UPDATE mst2_chunk_receipt_generation SET deadline=pg_catalog.clock_timestamp()+interval '59 seconds',last_progress=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state IN ('RESERVED','CREATING') AND deadline>pg_catalog.clock_timestamp() RETURNING generation",[self.key.key.clone().into(),self.generation.into(),self.owner.to_string().into()])).await.map_err(db_error)? + } else { + txn.query_one_raw(stmt(&self.repository.schema,"SELECT generation FROM mst2_chunk_receipt_generation WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state IN ('RESERVED','CREATING') AND deadline>pg_catalog.clock_timestamp()",[self.key.key.clone().into(),self.generation.into(),self.owner.to_string().into()])).await.map_err(db_error)? + }; + if row.is_none() { return Err(expired("chunk-map install reservation expired or was retired")); } + let remaining = txn.query_one_raw(stmt(&self.repository.schema, + "SELECT pg_catalog.floor(EXTRACT(EPOCH FROM (deadline-pg_catalog.clock_timestamp()))*1000000)::bigint AS remaining_us FROM mst2_chunk_receipt_generation WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state IN ('RESERVED','CREATING')", + [self.key.key.clone().into(),self.generation.into(),self.owner.to_string().into()])).await.map_err(db_error)? + .ok_or_else(|| expired("chunk-map install deadline disappeared"))?; + remaining_deadline(started, &remaining) + }.await; + finish(txn, result).await + }).await??; + self.deadline.observe(deadline, progress)?; + if progress { + *confirmed = (Instant::now(), Instant::now(), completed); + } else { + confirmed.0 = Instant::now(); + } + Ok(()) + } + + pub(super) async fn prepare( + &self, + verified: &VerifiedSourceChunkMap, + ) -> Result, SnapshotError> { + if verified.source() != &self.source { + return Err(integrity( + "verified source differs from install reservation", + )); + } + let bytes = receipt(&self.repository.primary_scope, &self.source, verified.map())?; + let started = Instant::now(); + self.deadline.bound(async { + let txn = self.repository.transaction().await?; + let result=async { + self.repository.require_scope(&txn).await?; + require_current_source(&txn,&self.source,&self.repository.schema).await?; + barrier(&txn,&self.repository.schema).await?; + let row=txn.query_one_raw(stmt(&self.repository.schema,"UPDATE mst2_chunk_receipt_generation SET state='CREATING',map_id=$4,receipt_bytes=$5,deadline=pg_catalog.clock_timestamp()+interval '59 seconds',last_progress=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state='RESERVED' AND deadline>pg_catalog.clock_timestamp() RETURNING generation",[self.key.key.clone().into(),self.generation.into(),self.owner.to_string().into(),verified.map().map_id().to_vec().into(),bytes.clone().into()])).await.map_err(db_error)?; + if row.is_none() { return Err(expired("chunk-map install lost its exact reservation before receipt creation")); } + Ok(()) + }.await; + finish(txn, result).await?; + Ok::<_, SnapshotError>(()) + }).await??; + self.deadline.observe(started + OWNER_TTL, true)?; + Ok(bytes) + } + + pub(super) fn key(&self) -> &ObjectKey { + &self.key + } + pub(super) fn generation(&self) -> i64 { + self.generation + } + pub(super) fn owner(&self) -> Uuid { + self.owner + } + + pub(super) async fn acknowledge_create(&self) -> Result<(), SnapshotError> { + // This method is called only after the actual atomic create future + // returns success. Absence, a checksum or inventory cannot call it. + let started = Instant::now(); + let renewed = self.deadline.bound(async { + let txn = self.repository.transaction().await?; + let result=async { + self.repository.require_scope(&txn).await?; + barrier(&txn,&self.repository.schema).await?; + let row=txn.query_one_raw(stmt(&self.repository.schema,"UPDATE mst2_chunk_receipt_generation SET create_completed=true,create_completed_at=COALESCE(create_completed_at,pg_catalog.clock_timestamp()),deadline=CASE WHEN state='CREATING' AND owner=$3::uuid AND deadline>pg_catalog.clock_timestamp() THEN pg_catalog.clock_timestamp()+interval '59 seconds' ELSE deadline END WHERE receipt_key=$1 AND generation=$2 AND receipt_bytes IS NOT NULL RETURNING generation,CASE WHEN state='CREATING' AND owner=$3::uuid AND deadline>pg_catalog.clock_timestamp() THEN true ELSE false END AS deadline_renewed",[self.key.key.clone().into(),self.generation.into(),self.owner.to_string().into()])).await.map_err(db_error)? + .ok_or_else(|| integrity("completed create lost its persistent generation fence"))?; + row.try_get::("", "deadline_renewed").map_err(db_error) + }.await; + finish(txn, result).await + }).await??; + if renewed { + self.deadline.observe(started + OWNER_TTL, true)?; + } + Ok(()) + } + + pub(super) async fn retire(&self) -> Result<(), SnapshotError> { + retire_install(&self.repository, self.owner, &self.key.key, self.generation).await + } +} + +impl Drop for ChunkMapInstall { + fn drop(&mut self) { + if let Ok(mut cancelled) = self.repository.cancelled_installs.lock() { + if cancelled.len() < 64 { + cancelled.push(CancelledInstall { + owner: self.owner, + key: self.key.key.clone(), + generation: self.generation, + }); + } + } + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return; + }; + let repository = self.repository.clone(); + let owner = self.owner; + let key = self.key.key.clone(); + let generation = self.generation; + runtime.spawn(async move { + let result = retire_install(&repository, owner, &key, generation).await; + if result.is_ok() { + forget_cancelled_install(&repository, owner); + } + if let Err(error) = result { + tracing::warn!( + ?error, + "chunk-map install cancellation will reconcile by database deadline" + ); + } + }); + } +} + +fn forget_cancelled_install(repository: &PostgresChunkMapRepository, owner: Uuid) { + if let Ok(mut cancelled) = repository.cancelled_installs.lock() { + cancelled.retain(|install| install.owner != owner); + } +} + +async fn retire_install( + repository: &PostgresChunkMapRepository, + owner: Uuid, + key: &str, + generation: i64, +) -> Result<(), SnapshotError> { + let txn = repository.transaction().await?; + let result = async { + repository.require_scope(&txn).await?; + barrier(&txn, &repository.schema).await?; + txn.execute_raw(stmt(&repository.schema,"UPDATE mst2_chunk_receipt_generation SET state=CASE WHEN receipt_bytes IS NULL THEN 'APPLIED' ELSE 'DELETING' END,owner=NULL,deadline=NULL,reserved_pages=0,reserved_nodes=0,reserved_bytes=0 WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state IN ('RESERVED','CREATING')",[key.into(),generation.into(),owner.to_string().into()])).await.map_err(db_error)?; + Ok(()) + }.await; + finish(txn, result).await +} + +/// Constructor stays in the collector module. Its private fields carry a +/// real primary claim and the independently read exact backend body. +pub(crate) struct VerifiedReceiptDeletionClaim { + key: ObjectKey, + expected: Vec, + _generation: i64, +} +impl VerifiedReceiptDeletionClaim { + pub(crate) fn key(&self) -> &ObjectKey { + &self.key + } + pub(crate) fn expected_bytes(&self) -> &[u8] { + &self.expected + } +} + +impl PostgresChunkMapRepository { + async fn replay_cancelled_installs(&self) -> Result<(), SnapshotError> { + let cancelled = self + .cancelled_installs + .lock() + .map_err(|_| integrity("cancelled install replay ownership is poisoned"))? + .clone(); + for install in cancelled { + retire_install(self, install.owner, &install.key, install.generation).await?; + forget_cancelled_install(self, install.owner); + } + Ok(()) + } + + async fn maintain_before_admission( + &self, + objects: &MegaObjectStorageWrapper, + ) -> Result<(), SnapshotError> { + let mut completed = self.admission_maintenance.lock().await; + self.replay_cancelled_installs().await?; + if completed + .as_ref() + .is_some_and(|last| last.elapsed() < RENEW_INTERVAL) + { + let txn = self.transaction().await?; + self.require_scope(&txn).await?; + let pending=txn.query_one_raw(stmt(&self.schema,"SELECT EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation WHERE state='DELETING' OR (state IN ('RESERVED','CREATING') AND deadline<=pg_catalog.clock_timestamp())) AS pending",[])).await.map_err(db_error)?.ok_or_else(||integrity("pending chunk-map admission replay observation missing"))?.try_get::("","pending").map_err(db_error)?; + txn.commit().await.map_err(db_error)?; + if !pending { + return Ok(()); + } + } + self.maintain(objects, 8).await?; + *completed = Some(Instant::now()); + Ok(()) + } + #[cfg(test)] + pub(crate) async fn test_new_install_capacity( + &self, + objects: &MegaObjectStorageWrapper, + ) -> Result<(), SnapshotError> { + let observation = inventory(self, objects).await?; + let txn = self.transaction().await?; + let result = async { + self.require_scope(&txn).await?; + barrier(&txn, &self.schema).await?; + ensure_quotas(&txn, &self.schema, &observation, 1, 1, 1024 * 1024, None).await + } + .await; + finish(txn, result).await + } + pub(super) async fn stage_map_start( + &self, + verified: &VerifiedSourceChunkMap, + admission: &ChunkMapInstall, + observation: &BackingObservation, + ) -> Result { + let map = verified.map(); + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + require_current_source(&txn,verified.source(),&self.schema).await?; + map_barrier(&txn,map.map_id()).await?; + barrier(&txn,&self.schema).await?; + let inserted=txn.query_one_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_map(map_id,descriptor,page_count,pages_root) VALUES($1,$2,$3,$4) ON CONFLICT(map_id) DO NOTHING RETURNING map_id",[map.map_id().to_vec().into(),map.encode().into(),(map.page_count as i32).into(),map.pages_root.to_vec().into()])).await.map_err(db_error)?.is_some(); + let lifetime=txn.query_one_raw(stmt(&self.schema,"SELECT generation,state FROM mst2_chunk_map_lifetime WHERE map_id=$1",[map.map_id().to_vec().into()])).await.map_err(db_error)?; + let generation=if let Some(row)=lifetime { + if row.try_get::("","state").map_err(db_error)?!="LIVE" {return Err(unavailable("shared chunk map collection is pending"));} + row.try_get::("","generation").map_err(db_error)? + } else if inserted { + let generation=next_generation(&txn,&self.schema).await?; + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_map_lifetime(map_id,generation,state) VALUES($1,$2,'LIVE')",[map.map_id().to_vec().into(),generation.into()])).await.map_err(db_error)?; + generation + } else {return Err(integrity("existing chunk map has no exact lifetime; never repair it"));}; + let row=txn.query_one_raw(stmt(&self.schema,"SELECT CASE WHEN pg_catalog.octet_length(descriptor)=100 THEN descriptor ELSE NULL END AS descriptor,page_count,CASE WHEN pg_catalog.octet_length(pages_root)=32 THEN pages_root ELSE NULL END AS pages_root FROM mst2_chunk_map WHERE map_id=$1",[map.map_id().to_vec().into()])).await.map_err(db_error)?.ok_or_else(||integrity("staged map descriptor missing"))?; + if bounded_bytes(&row,"descriptor")?!=map.encode() || row.try_get::("","page_count").map_err(db_error)? as u64!=map.page_count || bounded_bytes(&row,"pages_root")?!=map.pages_root {return Err(integrity("immutable staged map descriptor conflicts with verified source"));} + let assigned=txn.query_one_raw(stmt(&self.schema,"UPDATE mst2_chunk_receipt_generation SET map_generation=$4,deadline=pg_catalog.clock_timestamp()+interval '59 seconds',last_progress=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state='CREATING' AND deadline>pg_catalog.clock_timestamp() AND map_id=$5 AND create_completed RETURNING generation",[admission.key.key.clone().into(),admission.generation.into(),admission.owner.to_string().into(),generation.into(),map.map_id().to_vec().into()])).await.map_err(db_error)?; + if assigned.is_none() {return Err(expired("map staging lost the exact completed-create reservation"));} + ensure_quotas(&txn,&self.schema,observation,0,0,0,Some(admission.owner)).await?; + Ok(generation) + }.await; + finish(txn, result).await + } + + pub(super) async fn stage_map_batch( + &self, + verified: &VerifiedSourceChunkMap, + admission: &ChunkMapInstall, + generation: i64, + leaves: Option<&[ChunkLeaf]>, + nodes: Option<&[ChunkMapNode]>, + ) -> Result<(), SnapshotError> { + let map_id = verified.map().map_id(); + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + require_current_source(&txn,verified.source(),&self.schema).await?; + map_barrier(&txn,map_id).await?; + let sealed=txn.query_one_raw(stmt(&self.schema,"SELECT EXISTS(SELECT 1 FROM mst2_chunk_map_source WHERE map_id=$1) AS sealed",[map_id.to_vec().into()])).await.map_err(db_error)?.ok_or_else(||integrity("staged map seal observation missing"))?.try_get::("","sealed").map_err(db_error)?; + if !sealed { + if let Some(batch)=leaves { + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_map_leaf(map_id,page_index,payload) SELECT $1,p.page_index,pg_catalog.decode(p.payload,'hex') FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(page_index integer,payload text) ON CONFLICT(map_id,page_index) DO NOTHING",[map_id.to_vec().into(),encode_leaves(batch)?.into()])).await.map_err(db_error)?; + } + if let Some(batch)=nodes { + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_map_node(map_id,first_page,page_count,digest) SELECT $1,p.first_page,p.page_count,pg_catalog.decode(p.digest,'hex') FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(first_page integer,page_count integer,digest text) ON CONFLICT(map_id,first_page,page_count) DO NOTHING",[map_id.to_vec().into(),encode_nodes(batch)?.into()])).await.map_err(db_error)?; + } + } + self.checkpoint_staging(&txn,admission,map_id,generation).await + }.await; + finish(txn, result).await + } + + async fn checkpoint_staging( + &self, + txn: &DatabaseTransaction, + admission: &ChunkMapInstall, + map_id: [u8; 32], + generation: i64, + ) -> Result<(), SnapshotError> { + barrier(txn, &self.schema).await?; + let progress=txn.query_one_raw(stmt(&self.schema,"UPDATE mst2_chunk_receipt_generation SET deadline=pg_catalog.clock_timestamp()+interval '59 seconds',last_progress=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state='CREATING' AND deadline>pg_catalog.clock_timestamp() AND map_id=$4 AND map_generation=$5 AND EXISTS(SELECT 1 FROM mst2_chunk_map_lifetime WHERE map_id=$4 AND generation=$5 AND state='LIVE') RETURNING generation",[admission.key.key.clone().into(),admission.generation.into(),admission.owner.to_string().into(),map_id.to_vec().into(),generation.into()])).await.map_err(db_error)?; + if progress.is_none() { + return Err(expired( + "bounded map publication batch lost its exact install generation", + )); + } + Ok(()) + } + + pub(super) async fn compare_staged_map( + &self, + verified: &VerifiedSourceChunkMap, + nodes: &[ChunkMapNode], + admission: &ChunkMapInstall, + generation: i64, + ) -> Result<(), SnapshotError> { + let map = verified.map(); + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + require_current_source(&txn,verified.source(),&self.schema).await?; + map_barrier(&txn,map.map_id()).await?; + let row=txn.query_one_raw(stmt(&self.schema,"SELECT CASE WHEN pg_catalog.octet_length(descriptor)=100 THEN descriptor ELSE NULL END AS descriptor,page_count,CASE WHEN pg_catalog.octet_length(pages_root)=32 THEN pages_root ELSE NULL END AS pages_root,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_leaf WHERE map_id=$1) AS leaves,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_node WHERE map_id=$1) AS nodes FROM mst2_chunk_map WHERE map_id=$1",[map.map_id().to_vec().into()])).await.map_err(db_error)?.ok_or_else(||integrity("staged map descriptor missing"))?; + if bounded_bytes(&row,"descriptor")?!=map.encode() || row.try_get::("","page_count").map_err(db_error)? as u64!=map.page_count || bounded_bytes(&row,"pages_root")?!=map.pages_root || row.try_get::("","leaves").map_err(db_error)? as u64!=map.page_count || row.try_get::("","nodes").map_err(db_error)? as usize!=nodes.len() {return Err(integrity("staged map has conflicting descriptor or incomplete coverage"));} + self.checkpoint_staging(&txn,admission,map.map_id(),generation).await + }.await; + finish(txn, result).await?; + for batch in verified.leaves().chunks(64) { + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + require_current_source(&txn,verified.source(),&self.schema).await?; + map_barrier(&txn,map.map_id()).await?; + let bad=txn.query_one_raw(stmt(&self.schema,"SELECT p.page_index FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(page_index integer,payload text) LEFT JOIN mst2_chunk_map_leaf l ON l.map_id=$1 AND l.page_index=p.page_index WHERE l.payload IS DISTINCT FROM pg_catalog.decode(p.payload,'hex') LIMIT 1",[map.map_id().to_vec().into(),encode_leaves(batch)?.into()])).await.map_err(db_error)?; + if bad.is_some() {return Err(integrity("immutable stored chunk map leaf conflicts with verified source"));} + self.checkpoint_staging(&txn,admission,map.map_id(),generation).await + }.await; + finish(txn, result).await?; + } + for batch in nodes.chunks(1024) { + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + require_current_source(&txn,verified.source(),&self.schema).await?; + map_barrier(&txn,map.map_id()).await?; + let bad=txn.query_one_raw(stmt(&self.schema,"SELECT p.first_page FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(first_page integer,page_count integer,digest text) LEFT JOIN mst2_chunk_map_node n ON n.map_id=$1 AND n.first_page=p.first_page AND n.page_count=p.page_count WHERE n.digest IS DISTINCT FROM pg_catalog.decode(p.digest,'hex') LIMIT 1",[map.map_id().to_vec().into(),encode_nodes(batch)?.into()])).await.map_err(db_error)?; + if bad.is_some() {return Err(integrity("immutable stored chunk map node conflicts with verified source"));} + self.checkpoint_staging(&txn,admission,map.map_id(),generation).await + }.await; + finish(txn, result).await?; + } + Ok(()) + } + + pub(crate) async fn maintain( + &self, + objects: &MegaObjectStorageWrapper, + limit: usize, + ) -> Result<(), SnapshotError> { + if !(1..=MAINTENANCE_MAX).contains(&limit) { + return Err(SnapshotError::new( + SnapshotErrorCode::InvalidRequest, + "chunk-map maintenance limit must be 1..64", + )); + } + // Unsupported enumeration must fail before changing any resident + // generation. Overquota stores may still replay known deletions. + match inventory(self, objects).await { + Ok(_) => {} + Err(error) if error.code == SnapshotErrorCode::LimitExceeded => {} + Err(error) => return Err(error), + } + self.replay_cancelled_installs().await?; + let _memory = retention_budget().reserve(8 * 1024 * 1024)?; + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + barrier(&txn,&self.schema).await?; + txn.execute_raw(stmt(&self.schema,"DELETE FROM mst2_chunk_reader WHERE owner IN (SELECT owner FROM mst2_chunk_reader WHERE deadline<=pg_catalog.clock_timestamp() ORDER BY deadline,owner LIMIT $1)",[(limit as i64).into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.schema,"UPDATE mst2_chunk_receipt_generation SET state=CASE WHEN receipt_bytes IS NULL THEN 'APPLIED' ELSE 'DELETING' END,owner=NULL,deadline=NULL,reserved_pages=0,reserved_nodes=0,reserved_bytes=0 WHERE receipt_key IN (SELECT receipt_key FROM mst2_chunk_receipt_generation WHERE state IN ('RESERVED','CREATING') AND deadline<=pg_catalog.clock_timestamp() ORDER BY deadline,receipt_key LIMIT $1)",[(limit as i64).into()])).await.map_err(db_error)?; + // Replay unfinished receipt claims before admitting new victims. + let pending=txn.query_one_raw(stmt(&self.schema,"SELECT EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation WHERE state='DELETING') AS pending",[])).await.map_err(db_error)?.ok_or_else(||integrity("chunk receipt replay observation missing"))?.try_get::("","pending").map_err(db_error)?; + if !pending { + txn.execute_raw(stmt(&self.schema,"UPDATE mst2_chunk_receipt_generation SET state='DELETING' WHERE receipt_key IN (SELECT g.receipt_key FROM mst2_chunk_receipt_generation g WHERE g.state='LIVE' AND g.last_progress<=pg_catalog.clock_timestamp()-interval '1 hour' AND NOT EXISTS(SELECT 1 FROM mst2_chunk_reader r WHERE r.receipt_key=g.receipt_key AND r.source_generation=g.generation AND r.deadline>pg_catalog.clock_timestamp()) ORDER BY g.last_progress,g.receipt_key LIMIT $1)",[(limit as i64).into()])).await.map_err(db_error)?; + } + // Rotate APPLIED claims too: absence is never a promise that a + // cancelled remote create cannot restore this retired key later. + txn.query_all_raw(stmt(&self.schema,"SELECT receipt_key,generation,CASE WHEN pg_catalog.octet_length(receipt_bytes)<=4096 THEN receipt_bytes ELSE NULL END AS receipt_bytes,CASE WHEN pg_catalog.octet_length(primary_scope)<=1024 THEN primary_scope ELSE NULL END AS primary_scope,map_id,map_generation,state FROM mst2_chunk_receipt_generation WHERE state IN ('DELETING','APPLIED') ORDER BY (state='DELETING') DESC,checked_at,receipt_key LIMIT $1",[(limit as i64).into()])).await.map_err(db_error) + }.await; + let claims = finish(txn, result).await?; + for row in claims { + self.reconcile_receipt(objects, row).await?; + } + self.collect_maps(limit).await?; + let observation = inventory(self, objects).await?; + let requested = serde_json::to_string( + &observation + .inventory + .objects + .iter() + .map(|(key, _)| &key.key) + .collect::>(), + ) + .map_err(db_error)?; + let txn = self.transaction().await?; + self.require_scope(&txn).await?; + let unknown=txn.query_all_raw(stmt(&self.schema,"SELECT p.key FROM pg_catalog.jsonb_array_elements_text($1::jsonb) AS p(key) LEFT JOIN mst2_chunk_receipt_generation g ON g.receipt_key=p.key WHERE g.receipt_key IS NULL ORDER BY p.key LIMIT $2",[requested.into(),(limit as i64).into()])).await.map_err(db_error)?; + txn.commit().await.map_err(db_error)?; + for row in unknown { + let key = ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: row.try_get("", "key").map_err(db_error)?, + }; + self.claim_orphan(objects, &key).await?; + } + tracing::debug!(target:"mst2::chunk_map",maintenance_limit=limit,actual_backing_receipts=observation.inventory.objects.len(),actual_backing_bytes=observation.inventory.bytes,"bounded chunk-map retention tick"); + Ok(()) + } + + async fn reconcile_receipt( + &self, + objects: &MegaObjectStorageWrapper, + row: QueryResult, + ) -> Result<(), SnapshotError> { + if bounded_bytes(&row, "primary_scope")? != self.primary_scope { + return Err(integrity( + "retired receipt belongs to a different primary scope", + )); + } + let key = ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: row.try_get("", "receipt_key").map_err(db_error)?, + }; + let generation: i64 = row.try_get("", "generation").map_err(db_error)?; + let expected: Option> = row.try_get("", "receipt_bytes").map_err(db_error)?; + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + barrier(&txn,&self.schema).await?; + txn.query_one_raw(stmt(&self.schema,"SELECT generation FROM mst2_chunk_receipt_generation WHERE receipt_key=$1 AND generation=$2 AND receipt_bytes IS NOT DISTINCT FROM $3::bytea AND primary_scope=$4 AND state IN ('DELETING','APPLIED') AND NOT EXISTS(SELECT 1 FROM mst2_chunk_reader WHERE receipt_key=$1 AND source_generation=$2 AND deadline>pg_catalog.clock_timestamp())",[key.key.clone().into(),generation.into(),expected.clone().into(),self.primary_scope.clone().into()])).await.map_err(db_error) + }.await; + if finish(txn, result).await?.is_none() { + return Ok(()); + } + if let Some(bytes) = expected.as_ref() { + let (scope, _, map, physical_generation) = decode_orphan(&key, bytes)?; + let stored_map: Option> = row.try_get("", "map_id").map_err(db_error)?; + if scope != self.primary_scope + || physical_generation != generation + || stored_map.as_deref() != Some(map.map_id().as_slice()) + { + return Err(integrity( + "receipt deletion claim disagrees with the independent physical generation", + )); + } + } + if let Some(expected) = expected { + let present = tokio::time::timeout(Duration::from_secs(5), objects.inner.exists(&key)) + .await + .map_err(|_| unavailable("retired receipt existence check timed out"))? + .map_err(storage_error)?; + if present { + read_receipt(objects, &key, &expected).await?; + let authority = ChunkMapReceiptDeletion::from_claim(VerifiedReceiptDeletionClaim { + key: key.clone(), + expected, + _generation: generation, + }); + tokio::time::timeout( + Duration::from_secs(5), + objects.inner.delete_chunk_map_receipt(&authority), + ) + .await + .map_err(|_| unavailable("retired receipt deletion timed out"))? + .map_err(storage_error)?; + } + } else if tokio::time::timeout(Duration::from_secs(5), objects.inner.exists(&key)) + .await + .map_err(|_| unavailable("unused receipt check timed out"))? + .map_err(storage_error)? + { + return Err(integrity( + "receipt exists for a generation that never prepared a create", + )); + } + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + barrier(&txn,&self.schema).await?; + let applied=txn.query_one_raw(stmt(&self.schema,"UPDATE mst2_chunk_receipt_generation SET state='APPLIED',owner=NULL,deadline=NULL,reserved_pages=0,reserved_nodes=0,reserved_bytes=0,checked_at=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND state IN ('DELETING','APPLIED') AND NOT EXISTS(SELECT 1 FROM mst2_chunk_reader r WHERE r.receipt_key=$1 AND r.source_generation=$2 AND r.deadline>pg_catalog.clock_timestamp()) RETURNING generation",[key.key.clone().into(),generation.into()])).await.map_err(db_error)?; + if applied.is_none() { return Err(integrity("retired receipt lost its exact reader-free completion claim")); } + txn.execute_raw(stmt(&self.schema,"DELETE FROM mst2_chunk_map_source WHERE receipt_key=$1 AND receipt_generation=$2",[key.key.into(),generation.into()])).await.map_err(db_error)?; + Ok(()) + }.await; + finish(txn, result).await + } + + async fn claim_orphan( + &self, + objects: &MegaObjectStorageWrapper, + key: &ObjectKey, + ) -> Result<(), SnapshotError> { + let actual = read_bounded_orphan(objects, key).await?; + let Some(actual) = actual else { return Ok(()) }; + let (scope, source_bytes, map, generation) = decode_orphan(key, &actual)?; + if scope != self.primary_scope { + return Err(integrity( + "orphan receipt belongs to a different captured primary", + )); + } + let identity = source_id(&scope, &source_bytes); + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + barrier(&txn,&self.schema).await?; + let count=txn.query_one_raw(stmt(&self.schema,"SELECT pg_catalog.count(*) AS history FROM mst2_chunk_receipt_generation",[])).await.map_err(db_error)?.ok_or_else(||integrity("orphan receipt history count missing"))?.try_get::("","history").map_err(db_error)?; + if count>=HISTORY_CAP { return Err(SnapshotError::new(SnapshotErrorCode::LimitExceeded,"chunk receipt reconciliation history quota exhausted")); } + // An orphan never admits a source or map. Its actual body only + // identifies a retired physical key for collection. + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_receipt_generation(receipt_key,source_id,generation,source_bytes,primary_scope,map_id,receipt_bytes,state,reserved_pages,reserved_nodes,reserved_bytes) VALUES($1,$2,$3,$4,$5,$6,$7,'DELETING',0,0,0) ON CONFLICT(receipt_key) DO NOTHING",[key.key.clone().into(),identity.to_vec().into(),generation.into(),source_bytes.into(),scope.into(),map.map_id().to_vec().into(),actual.into()])).await.map_err(db_error)?; + Ok(()) + }.await; + finish(txn, result).await + } + + pub(super) async fn collect_maps(&self, limit: usize) -> Result<(), SnapshotError> { + let txn = self.transaction().await?; + self.require_scope(&txn).await?; + let rows=txn.query_all_raw(stmt(&self.schema,"SELECT l.map_id,l.generation,l.state FROM mst2_chunk_map_lifetime l WHERE l.state='DELETING' OR (NOT EXISTS(SELECT 1 FROM mst2_chunk_map_lifetime WHERE state='DELETING') AND (l.last_used<=pg_catalog.clock_timestamp()-interval '1 hour' OR EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation g WHERE g.map_id=l.map_id AND g.map_generation=l.generation AND g.state IN ('DELETING','APPLIED'))) AND NOT EXISTS(SELECT 1 FROM mst2_chunk_map_source s WHERE s.map_id=l.map_id) AND NOT EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation g WHERE g.map_id=l.map_id AND g.map_generation=l.generation AND g.state='CREATING' AND g.deadline>pg_catalog.clock_timestamp())) ORDER BY (l.state='DELETING') DESC,l.last_used,l.map_id LIMIT $1",[(limit as i64).into()])).await.map_err(db_error)?; + txn.commit().await.map_err(db_error)?; + for row in rows { + let map_id: Vec = row.try_get("", "map_id").map_err(db_error)?; + let generation: i64 = row.try_get("", "generation").map_err(db_error)?; + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + let acquired=txn.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres,"SELECT pg_catalog.pg_try_advisory_xact_lock(1296717364,pg_catalog.hashtext($1)) AS acquired",[hex::encode(&map_id).into()])).await.map_err(db_error)?.ok_or_else(||integrity("map collection lock observation missing"))?.try_get::("","acquired").map_err(db_error)?; + if !acquired { return Ok(()); } + barrier(&txn,&self.schema).await?; + let eligible=txn.query_one_raw(stmt(&self.schema,"SELECT state FROM mst2_chunk_map_lifetime l WHERE map_id=$1 AND generation=$2 AND (state='DELETING' OR last_used<=pg_catalog.clock_timestamp()-interval '1 hour' OR EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation g WHERE g.map_id=l.map_id AND g.map_generation=l.generation AND g.state IN ('DELETING','APPLIED'))) AND NOT EXISTS(SELECT 1 FROM mst2_chunk_map_source s WHERE s.map_id=$1) AND NOT EXISTS(SELECT 1 FROM mst2_chunk_reader r WHERE r.map_id=$1 AND r.map_generation=$2 AND r.deadline>pg_catalog.clock_timestamp()) AND NOT EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation g WHERE g.map_id=$1 AND g.map_generation=$2 AND g.state='CREATING' AND g.deadline>pg_catalog.clock_timestamp())",[map_id.clone().into(),generation.into()])).await.map_err(db_error)?; + let Some(eligible)=eligible else { return Ok(()) }; + if eligible.try_get::("","state").map_err(db_error)?=="LIVE" { + let count=txn.query_one_raw(stmt(&self.schema,"SELECT pg_catalog.count(*) AS history FROM mst2_chunk_map_gc",[])).await.map_err(db_error)?.ok_or_else(||integrity("map collection history count missing"))?.try_get::("","history").map_err(db_error)?; + if count>=HISTORY_CAP { return Err(SnapshotError::new(SnapshotErrorCode::LimitExceeded,"chunk map collection history quota exhausted")); } + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_map_gc(map_id,generation,state) VALUES($1,$2,'PENDING') ON CONFLICT(map_id,generation) DO NOTHING",[map_id.clone().into(),generation.into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.schema,"UPDATE mst2_chunk_map_lifetime SET state='DELETING' WHERE map_id=$1 AND generation=$2 AND state='LIVE'",[map_id.clone().into(),generation.into()])).await.map_err(db_error)?; + } + // One bounded resident batch per exact map generation. Large + // maps stay PENDING and are replayed before new map victims. + txn.execute_raw(stmt(&self.schema,"DELETE FROM mst2_chunk_map_leaf WHERE map_id=$1 AND page_index IN (SELECT page_index FROM mst2_chunk_map_leaf WHERE map_id=$1 ORDER BY page_index LIMIT 1024)",[map_id.clone().into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.schema,"DELETE FROM mst2_chunk_map_node WHERE (map_id,first_page,page_count) IN (SELECT map_id,first_page,page_count FROM mst2_chunk_map_node WHERE map_id=$1 ORDER BY first_page,page_count LIMIT 1024)",[map_id.clone().into()])).await.map_err(db_error)?; + let empty=txn.query_one_raw(stmt(&self.schema,"SELECT NOT EXISTS(SELECT 1 FROM mst2_chunk_map_leaf WHERE map_id=$1) AND NOT EXISTS(SELECT 1 FROM mst2_chunk_map_node WHERE map_id=$1) AS empty",[map_id.clone().into()])).await.map_err(db_error)?.ok_or_else(||integrity("map collection completion observation missing"))?.try_get::("","empty").map_err(db_error)?; + if empty { + // Deferred lifetime FK permits map deletion while its + // exact PENDING generation is still available to guards. + txn.execute_raw(stmt(&self.schema,"DELETE FROM mst2_chunk_map WHERE map_id=$1",[map_id.clone().into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.schema,"DELETE FROM mst2_chunk_map_lifetime WHERE map_id=$1 AND generation=$2 AND state='DELETING'",[map_id.clone().into(),generation.into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.schema,"UPDATE mst2_chunk_map_gc SET state='APPLIED',applied_at=pg_catalog.clock_timestamp() WHERE map_id=$1 AND generation=$2 AND state='PENDING'",[map_id.clone().into(),generation.into()])).await.map_err(db_error)?; + } + Ok(()) + }.await; + finish(txn, result).await?; + } + Ok(()) + } + + pub(crate) async fn admit_install( + &self, + source: &ChunkMapSource, + objects: &MegaObjectStorageWrapper, + ) -> Result { + // Replay first. Every backend await happens outside the primary + // completion barrier; active reservations count concurrent creates. + self.maintain_before_admission(objects).await?; + let inventory = inventory(self, objects).await?; + let source_bytes = source.canonical_bytes()?; + let identity = source_id(&self.primary_scope, &source_bytes); + let pages = (source.fact().size as u64) + .div_ceil(mst2_codec::chunkmap::CHUNK_SIZE as u64) + .div_ceil(CHUNKS_PER_PAGE as u64) as i32; + let nodes = pages * 2 - 1; + let reserved_bytes = pages as i64 * 12288 + nodes as i64 * 512 + 1024 * 1024; + let owner = Uuid::new_v4(); + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + require_current_source(&txn,source,&self.schema).await?; + barrier(&txn,&self.schema).await?; + ensure_quotas(&txn,&self.schema,&inventory,pages,nodes,reserved_bytes,None).await?; + let row=txn.query_one_raw(stmt(&self.schema,"SELECT EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation WHERE source_id=$1) AS used,EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation WHERE source_id=$1 AND state IN ('RESERVED','CREATING','LIVE','DELETING')) AS occupied",[identity.to_vec().into()])).await.map_err(db_error)?.ok_or_else(|| integrity("chunk-map reservation observation missing"))?; + if row.try_get::("","occupied").map_err(db_error)? { return Err(unavailable("exact source already has a live or pending chunk-map generation")); } + let generation=if row.try_get::("","used").map_err(db_error)? { next_generation(&txn,&self.schema).await? } else { 1 }; + let key=physical_key(identity,generation); + let owner_started=Instant::now(); + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_receipt_generation(receipt_key,source_id,generation,source_bytes,primary_scope,owner,state,reserved_pages,reserved_nodes,reserved_bytes,deadline) VALUES($1,$2,$3,$4,$5,$6::uuid,'RESERVED',$7,$8,$9,pg_catalog.clock_timestamp()+interval '59 seconds')",[key.key.clone().into(),identity.to_vec().into(),generation.into(),source_bytes.clone().into(),self.primary_scope.clone().into(),owner.to_string().into(),pages.into(),nodes.into(),reserved_bytes.into()])).await.map_err(db_error)?; + Ok(ChunkMapInstall{repository:self.clone(),source:source.clone(),owner,key,generation,progress:Mutex::new((Instant::now(),Instant::now(),0)),deadline:OwnerDeadline::from_sql_start(owner_started)}) + }.await; + // An uncertain commit is an error. Its UUID is never reused to mint + // another reservation; the durable deadline resolves any lost reply. + finish(txn, result).await + } + + pub(super) async fn admit_reader( + &self, + txn: &DatabaseTransaction, + source: &ChunkMapSource, + row: &QueryResult, + map: &ChunkMap, + ) -> Result { + barrier(txn, &self.schema).await?; + let key: String = row.try_get("", "receipt_key").map_err(db_error)?; + let generation: i64 = row.try_get("", "receipt_generation").map_err(db_error)?; + let map_generation: i64 = row + .try_get::>("", "map_generation") + .map_err(db_error)? + .ok_or_else(|| integrity("admitted chunk map lifetime is missing"))?; + let state: Option = row.try_get("", "generation_state").map_err(db_error)?; + let map_state: Option = row.try_get("", "map_state").map_err(db_error)?; + if state.as_deref() == Some("DELETING") { + return Err(unavailable("admitted chunk map generation is retiring")); + } + if state.as_deref() != Some("LIVE") || map_state.as_deref() != Some("LIVE") { + return Err(integrity( + "admitted source has a missing or terminal map lifetime", + )); + } + let identity = self.source_identity(source)?; + if key != physical_key(identity, generation).key + || bounded_bytes(row, "generation_receipt_bytes")? + != receipt(&self.primary_scope, source, map)? + { + return Err(integrity( + "chunk map lifetime and independent receipt identity disagree", + )); + } + txn.execute_raw(stmt(&self.schema,"DELETE FROM mst2_chunk_reader WHERE owner IN (SELECT owner FROM mst2_chunk_reader WHERE deadline<=pg_catalog.clock_timestamp() ORDER BY deadline,owner LIMIT 64)",[])).await.map_err(db_error)?; + let count = txn + .query_one_raw(stmt( + &self.schema, + "SELECT pg_catalog.count(*) AS readers FROM mst2_chunk_reader", + [], + )) + .await + .map_err(db_error)? + .ok_or_else(|| integrity("chunk reader count missing"))? + .try_get::("", "readers") + .map_err(db_error)?; + if count >= 4096 { + return Err(unavailable("chunk-map reader quota is occupied")); + } + let owner = Uuid::new_v4(); + let owner_started = Instant::now(); + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_reader(owner,receipt_key,source_generation,map_id,map_generation,deadline) VALUES($1::uuid,$2,$3,$4,$5,pg_catalog.clock_timestamp()+interval '59 seconds')",[owner.to_string().into(),key.clone().into(),generation.into(),map.map_id().to_vec().into(),map_generation.into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.schema,"UPDATE mst2_chunk_map_lifetime SET last_used=pg_catalog.clock_timestamp() WHERE map_id=$1 AND generation=$2 AND state='LIVE'",[map.map_id().to_vec().into(),map_generation.into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.schema,"UPDATE mst2_chunk_receipt_generation SET last_progress=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND state='LIVE'",[key.clone().into(),generation.into()])).await.map_err(db_error)?; + Ok(ChunkMapReader { + repository: self.clone(), + source: source.clone(), + owner, + receipt_key: key, + generation, + map_id: map.map_id(), + map_generation, + confirmed: Mutex::new((Instant::now(), Instant::now())), + deadline: OwnerDeadline::from_sql_start(owner_started), + }) + } + + pub(super) async fn require_missing_source_retired( + &self, + txn: &DatabaseTransaction, + source: &ChunkMapSource, + ) -> Result { + let identity = self.source_identity(source)?; + let row=txn.query_one_raw(stmt(&self.schema,"SELECT state FROM mst2_chunk_receipt_generation WHERE source_id=$1 AND state IN ('LIVE','DELETING') ORDER BY generation DESC LIMIT 1",[identity.to_vec().into()])).await.map_err(db_error)?; + if let Some(row) = row { + if row.try_get::("", "state").map_err(db_error)? == "LIVE" { + return Err(integrity( + "live chunk-map source row is missing; never fall back to the cold body", + )); + } + return Ok(true); + } + Ok(false) + } +} + +pub(super) async fn barrier(txn: &DatabaseTransaction, schema: &str) -> Result<(), SnapshotError> { + txn.query_one_raw(stmt(schema, "SELECT mst2_chunk_retention_barrier()", [])) + .await + .map_err(db_error)?; + Ok(()) +} + +pub(super) async fn map_barrier( + txn: &DatabaseTransaction, + map_id: [u8; 32], +) -> Result<(), SnapshotError> { + // Per-map lock precedes the short global completion barrier. Collection + // never holds the global barrier while waiting for a map writer. + txn.query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT pg_catalog.pg_advisory_xact_lock(1296717364,pg_catalog.hashtext($1))", + [hex::encode(map_id).into()], + )) + .await + .map_err(db_error)?; + Ok(()) +} + +pub(super) async fn next_generation( + txn: &DatabaseTransaction, + schema: &str, +) -> Result { + txn.query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT pg_catalog.nextval($1::regclass) AS generation", + [format!("{}.mst2_chunk_generation_sequence", quoted(schema)).into()], + )) + .await + .map_err(db_error)? + .ok_or_else(|| integrity("chunk generation sequence missing"))? + .try_get("", "generation") + .map_err(db_error) +} + +fn physical_key(source: [u8; 32], generation: i64) -> ObjectKey { + ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: if generation == 1 { + hex::encode(source) + } else { + format!("{}-{generation:016x}", hex::encode(source)) + }, + } +} + +pub(super) async fn inventory( + repository: &PostgresChunkMapRepository, + objects: &MegaObjectStorageWrapper, +) -> Result, SnapshotError> { + let mut cached = repository.receipt_observation.lock().await; + if let Some(observation) = cached.as_ref().filter(|observation| { + Arc::ptr_eq(&observation.backing.inner, &objects.inner) + && observation.captured.elapsed() < RENEW_INTERVAL + }) { + return Ok(observation.clone()); + } + let memory = retention_budget().reserve(4 * 1024 * 1024)?; + let txn = repository.transaction().await?; + repository.require_scope(&txn).await?; + let started_at = txn + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT pg_catalog.clock_timestamp()::text AS started_at", + )) + .await + .map_err(db_error)? + .ok_or_else(|| integrity("backing inventory database clock missing"))? + .try_get("", "started_at") + .map_err(db_error)?; + txn.commit().await.map_err(db_error)?; + let inventory = tokio::time::timeout( + Duration::from_secs(30), + objects.inner.chunk_map_receipt_inventory(), + ) + .await + .map_err(|_| unavailable("bounded chunk receipt inventory timed out"))? + .map_err(|error| match error { + IoOrbitError::ChunkMapRetentionUnsupported => SnapshotError::new( + SnapshotErrorCode::TemporaryUnavailable, + "backing store does not support bounded chunk-map retention", + ), + IoOrbitError::ChunkMapRetentionCapacityExceeded => SnapshotError::new( + SnapshotErrorCode::LimitExceeded, + "actual backing chunk-map receipts exceed retention quota", + ), + error => storage_error(error), + })?; + let observation = Arc::new(BackingObservation { + inventory, + started_at, + captured: Instant::now(), + backing: objects.clone(), + _memory: memory, + }); + *cached = Some(observation.clone()); + Ok(observation) +} + +pub(super) async fn ensure_quotas( + txn: &DatabaseTransaction, + schema: &str, + observation: &BackingObservation, + pages: i32, + nodes: i32, + bytes: i64, + exclude: Option, +) -> Result<(), SnapshotError> { + let inventory = &observation.inventory; + let row=txn.query_one_raw(stmt(schema,"SELECT (SELECT pg_catalog.count(*) FROM mst2_chunk_map) AS maps,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_source) AS sources,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_leaf) AS leaves,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_node) AS nodes,(SELECT pg_catalog.count(*) FROM mst2_chunk_receipt_generation) AS history,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_gc) AS map_history,(SELECT pg_catalog.count(*) FROM mst2_chunk_receipt_generation WHERE receipt_bytes IS NOT NULL AND NOT create_completed) AS uncertain,(SELECT pg_catalog.count(*) FROM mst2_chunk_receipt_generation WHERE receipt_bytes IS NOT NULL AND (created_at>=$2::timestamptz OR create_completed_at>=$2::timestamptz)) AS recent,pg_catalog.count(*) AS pending,COALESCE(pg_catalog.sum(reserved_pages),0)::bigint AS pending_pages,COALESCE(pg_catalog.sum(reserved_nodes),0)::bigint AS pending_nodes,COALESCE(pg_catalog.sum(reserved_bytes),0)::bigint AS pending_bytes FROM mst2_chunk_receipt_generation WHERE state IN ('RESERVED','CREATING') AND ($1::uuid IS NULL OR owner IS DISTINCT FROM $1::uuid)",[exclude.map(|id|id.to_string()).into(),observation.started_at.clone().into()])).await.map_err(db_error)?.ok_or_else(|| integrity("chunk-map actual quota observation missing"))?; + let get = |name: &str| row.try_get::("", name).map_err(db_error); + let pending = get("pending")?; + let relation_bytes=txn.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres,"SELECT COALESCE(pg_catalog.sum(pg_catalog.pg_total_relation_size(c.oid)),0)::bigint AS bytes FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 AND c.relkind='r' AND c.relname IN ('mst2_chunk_map','mst2_chunk_map_source','mst2_chunk_map_leaf','mst2_chunk_map_node','mst2_chunk_receipt_generation','mst2_chunk_map_lifetime','mst2_chunk_reader','mst2_chunk_map_gc')",[schema.into()])).await.map_err(db_error)?.ok_or_else(||integrity("chunk relation byte observation missing"))?.try_get::("","bytes").map_err(db_error)?; + let extra = if exclude.is_none() { 1 } else { 0 }; + if get("maps")? + pending + extra > 4096 + || get("sources")? + pending + extra > 16384 + || get("leaves")? + get("pending_pages")? + pages as i64 > 65536 + || get("nodes")? + get("pending_nodes")? + nodes as i64 > 131072 + || relation_bytes + get("pending_bytes")? + bytes + 1024 * 1024 > 536870912 + || get("history")? + extra > HISTORY_CAP + || get("map_history")? > HISTORY_CAP + || pending + extra > 64 + || inventory.objects.len() as i64 + pending + get("uncertain")? + get("recent")? + extra + > 16384 + || inventory.bytes + ((pending + get("uncertain")? + get("recent")? + extra) as u64) * 4096 + > 64 * 1024 * 1024 + { + return Err(SnapshotError::new( + SnapshotErrorCode::LimitExceeded, + "actual chunk-map rows, history, backing or reserved capacity exceed retention quota", + )); + } + Ok(()) +} + +fn unavailable(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::TemporaryUnavailable, message) +} +fn expired(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::LeaseExpired, message) +} + +async fn read_bounded_orphan( + objects: &MegaObjectStorageWrapper, + key: &ObjectKey, +) -> Result>, SnapshotError> { + tokio::time::timeout(Duration::from_secs(5), async { + let (mut stream, meta) = match objects.inner.get_stream(key).await { + Ok(value) => value, + Err(error) if error.is_not_found() => return Ok(None), + Err(error) => return Err(storage_error(error)), + }; + if !(129..=4096).contains(&meta.size) { + return Err(integrity("orphan receipt exceeds its byte profile")); + } + let mut bytes = Vec::with_capacity(meta.size as usize); + while let Some(part) = stream.next().await { + let part = part.map_err(storage_error)?; + if part.len() > meta.size as usize - bytes.len() { + return Err(integrity( + "orphan receipt exceeds its advertised exact size", + )); + } + bytes.extend_from_slice(&part); + } + if bytes.len() != meta.size as usize { + return Err(integrity("orphan receipt is truncated")); + } + Ok(Some(bytes)) + }) + .await + .map_err(|_| unavailable("orphan receipt read timed out"))? +} + +fn decode_orphan( + key: &ObjectKey, + bytes: &[u8], +) -> Result<(Vec, Vec, ChunkMap, i64), SnapshotError> { + if !bytes.starts_with(RECEIPT_DOMAIN) { + return Err(integrity("orphan is not a canonical trusted receipt")); + } + let mut offset = RECEIPT_DOMAIN.len(); + let take = |offset: &mut usize, max: usize| -> Result, SnapshotError> { + let len_bytes = bytes + .get(*offset..*offset + 4) + .ok_or_else(|| integrity("orphan receipt length is truncated"))?; + let len = u32::from_be_bytes( + len_bytes + .try_into() + .map_err(|_| integrity("invalid orphan receipt length"))?, + ) as usize; + *offset += 4; + if len == 0 || len > max { + return Err(integrity("orphan receipt field exceeds its profile")); + } + let field = bytes + .get(*offset..*offset + len) + .ok_or_else(|| integrity("orphan receipt field is truncated"))? + .to_vec(); + *offset += len; + Ok(field) + }; + let scope = take(&mut offset, 1024)?; + let source = take(&mut offset, 2048)?; + let descriptor = bytes + .get(offset..) + .ok_or_else(|| integrity("orphan map descriptor is missing"))?; + let map = ChunkMap::decode(descriptor) + .map_err(|_| integrity("orphan map descriptor is not canonical MCM2"))?; + if map.encode() != descriptor { + return Err(integrity("orphan receipt has noncanonical map bytes")); + } + let generation = if key.key.len() == 64 { + 1 + } else if key.key.len() == 81 && key.key.as_bytes()[64] == b'-' { + i64::from_str_radix(&key.key[65..], 16) + .map_err(|_| integrity("orphan physical generation is invalid"))? + } else { + return Err(integrity( + "orphan receipt key has invalid generation profile", + )); + }; + if generation <= 0 || physical_key(source_id(&scope, &source), generation) != *key { + return Err(integrity( + "orphan physical receipt key disagrees with its exact body", + )); + } + Ok((scope, source, map, generation)) +} diff --git a/src/jupiter/storage/object_storage.rs b/src/jupiter/storage/object_storage.rs index f65a33ab..3fbe45d8 100644 --- a/src/jupiter/storage/object_storage.rs +++ b/src/jupiter/storage/object_storage.rs @@ -278,6 +278,56 @@ impl MegaObjectStorage for InMemoryObjectStorage { Ok((Self::stream_bytes(bytes), meta)) } + async fn chunk_map_receipt_inventory( + &self, + ) -> OrbitResult { + use crate::orbit_api::object_storage::{ + ChunkMapReceiptInventory, MAX_CHUNK_MAP_RECEIPT_BYTES, MAX_CHUNK_MAP_RECEIPTS, + ObjectNamespace, + }; + let objects = self + .objects + .lock() + .map_err(|_| IoOrbitError::Other("object storage lock poisoned".into()))?; + let mut inventory = ChunkMapReceiptInventory { + objects: Vec::new(), + bytes: 0, + }; + for (key, (bytes, _)) in objects + .iter() + .filter(|(key, _)| key.namespace == ObjectNamespace::ChunkMapReceipt) + { + if inventory.objects.len() == MAX_CHUNK_MAP_RECEIPTS { + return Err(IoOrbitError::ChunkMapRetentionCapacityExceeded); + } + inventory.bytes = inventory + .bytes + .checked_add(bytes.len() as u64) + .filter(|bytes| *bytes <= MAX_CHUNK_MAP_RECEIPT_BYTES) + .ok_or(IoOrbitError::ChunkMapRetentionCapacityExceeded)?; + inventory.objects.push((key.clone(), bytes.len() as u64)); + } + Ok(inventory) + } + + async fn delete_chunk_map_receipt( + &self, + authority: &crate::orbit_api::object_storage::ChunkMapReceiptDeletion, + ) -> OrbitResult { + let mut objects = self + .objects + .lock() + .map_err(|_| IoOrbitError::Other("object storage lock poisoned".into()))?; + let Some((bytes, _)) = objects.get(authority.key()) else { + return Ok(false); + }; + if bytes.as_ref() != authority.expected_bytes() { + return Err(IoOrbitError::Other("retired receipt body changed".into())); + } + objects.remove(authority.key()); + Ok(true) + } + async fn get_range_stream( &self, key: &ObjectKey, diff --git a/src/orbit/adapter/object.rs b/src/orbit/adapter/object.rs index 87327a39..4ae923b0 100644 --- a/src/orbit/adapter/object.rs +++ b/src/orbit/adapter/object.rs @@ -104,6 +104,111 @@ impl MegaObjectStorage for ObjectStoreAdapter { Ok((Box::pin(stream), meta)) } + async fn chunk_map_receipt_inventory( + &self, + ) -> OrbitResult { + use crate::orbit_api::object_storage::{ + ChunkMapReceiptInventory, MAX_CHUNK_MAP_RECEIPT_BYTES, MAX_CHUNK_MAP_RECEIPTS, + }; + // object_store 0.14 cannot inventory/delete retained cloud versions. + // A current-object listing is not a physical backing-byte quota. + if !matches!(&self.store, BackendStore::Local(_)) { + return Err(IoOrbitError::ChunkMapRetentionUnsupported); + } + let prefix = object_store::path::Path::from("chunk-map-receipt"); + let mut listing = self.to_store().list(Some(&prefix)); + let mut inventory = ChunkMapReceiptInventory { + objects: Vec::new(), + bytes: 0, + }; + while let Some(entry) = listing.next().await { + let meta = entry.map_err(IoOrbitError::from)?; + if inventory.objects.len() == MAX_CHUNK_MAP_RECEIPTS { + return Err(IoOrbitError::ChunkMapRetentionCapacityExceeded); + } + inventory.bytes = inventory + .bytes + .checked_add(meta.size) + .filter(|bytes| *bytes <= MAX_CHUNK_MAP_RECEIPT_BYTES) + .ok_or(IoOrbitError::ChunkMapRetentionCapacityExceeded)?; + let location = meta.location.as_ref(); + let parts: Vec<_> = location.split('/').collect(); + if parts.len() != 5 + || parts[0] != "chunk-map-receipt" + || parts[1..4].iter().any(|part| part.len() != 2) + || parts[4].len() > 128 + { + return Err(IoOrbitError::Other( + "invalid physical chunk-map receipt path".into(), + )); + } + let key = ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: parts[1..].concat(), + }; + key.validate()?; + if key.default_sharding() != location { + return Err(IoOrbitError::Other( + "noncanonical physical chunk-map receipt path".into(), + )); + } + inventory.objects.push((key, meta.size)); + } + Ok(inventory) + } + + async fn delete_chunk_map_receipt( + &self, + authority: &crate::orbit_api::object_storage::ChunkMapReceiptDeletion, + ) -> OrbitResult { + if !matches!(&self.store, BackendStore::Local(_)) { + return Err(IoOrbitError::ChunkMapRetentionUnsupported); + } + let key = authority.key(); + if key.namespace != ObjectNamespace::ChunkMapReceipt { + return Err(IoOrbitError::Other( + "invalid sealed receipt namespace".into(), + )); + } + let (mut stream, meta) = match self.get_stream(key).await { + Ok(object) => object, + Err(error) if error.is_not_found() => return Ok(false), + Err(error) => return Err(error), + }; + let expected = authority.expected_bytes(); + if meta.size != expected.len() as i64 { + return Err(IoOrbitError::Other( + "retired receipt body size changed".into(), + )); + } + let mut offset: usize = 0; + while let Some(part) = stream.next().await { + let bytes = part?; + let end = offset + .checked_add(bytes.len()) + .filter(|end| *end <= expected.len()) + .ok_or_else(|| { + IoOrbitError::Other("retired receipt body exceeds its exact profile".into()) + })?; + if expected[offset..end] != bytes[..] { + return Err(IoOrbitError::Other("retired receipt body changed".into())); + } + offset = end; + } + if offset != expected.len() { + return Err(IoOrbitError::Other( + "retired receipt body is truncated".into(), + )); + } + // Physical generation keys are never reused. A late create can only + // restore this same retired body; persistent history will reconcile it. + match self.to_store().delete(&Self::checked_path(key)?).await { + Ok(()) => Ok(true), + Err(object_store::Error::NotFound { .. }) => Ok(false), + Err(error) => Err(IoOrbitError::from(error)), + } + } + async fn get_range_stream( &self, key: &ObjectKey, diff --git a/src/orbit_api/error.rs b/src/orbit_api/error.rs index 365d2e4d..9de51a0c 100644 --- a/src/orbit_api/error.rs +++ b/src/orbit_api/error.rs @@ -17,6 +17,12 @@ pub enum IoOrbitError { #[error("write manifest precondition failed")] WriteManifestPreconditionFailed, + #[error("chunk-map retention is not supported by this storage backend")] + ChunkMapRetentionUnsupported, + + #[error("chunk-map backing receipts exceed the fixed retention quota")] + ChunkMapRetentionCapacityExceeded, + #[error("other error: {0}")] Other(String), } diff --git a/src/orbit_api/object_storage.rs b/src/orbit_api/object_storage.rs index 90539315..9264d944 100644 --- a/src/orbit_api/object_storage.rs +++ b/src/orbit_api/object_storage.rs @@ -171,6 +171,39 @@ pub type ObjectByteStream = Pin, + pub bytes: u64, +} + +/// Sealed by the primary retention repository after claiming one exact +/// retired generation and independently reading its immutable backend body. +/// Ordinary object callers cannot construct a receipt deletion authority. +pub struct ChunkMapReceiptDeletion { + claim: crate::jupiter::storage::native_chunk_map::retention::VerifiedReceiptDeletionClaim, +} + +impl ChunkMapReceiptDeletion { + pub(crate) fn from_claim( + claim: crate::jupiter::storage::native_chunk_map::retention::VerifiedReceiptDeletionClaim, + ) -> Self { + Self { claim } + } + + pub(crate) fn key(&self) -> &ObjectKey { + self.claim.key() + } + + pub(crate) fn expected_bytes(&self) -> &[u8] { + self.claim.expected_bytes() + } +} + /// A streaming source of multiple objects. /// /// Each item yields: @@ -278,6 +311,22 @@ pub trait MegaObjectStorage: Send + Sync { )) } + /// Enumerate the real receipt namespace with fixed count/byte bounds. + /// Unsupported stores fail before a cold source body can be opened. + async fn chunk_map_receipt_inventory(&self) -> OrbitResult { + Err(IoOrbitError::ChunkMapRetentionUnsupported) + } + + /// Delete only the independently checked body of a sealed retired + /// physical generation. The key is never reused by a later installation. + /// Missing is idempotent; transport/authentication errors remain errors. + async fn delete_chunk_map_receipt( + &self, + _authority: &ChunkMapReceiptDeletion, + ) -> OrbitResult { + Err(IoOrbitError::ChunkMapRetentionUnsupported) + } + /// Retrieve a single object from the storage backend. /// /// # Returns diff --git a/src/server/http_server.rs b/src/server/http_server.rs index 04be7f77..777e732a 100644 --- a/src/server/http_server.rs +++ b/src/server/http_server.rs @@ -380,6 +380,35 @@ fn spawn_view_worker_task( spawn_view_worker_with_round(storage, token, production_round()) } +fn spawn_chunk_map_retention_task( + storage: Storage, + token: CancellationToken, + available: bool, +) -> Option> { + if !available { + return None; + } + Some(tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(30)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tokio::select! { + _=token.cancelled()=>break, + _=interval.tick()=>{ + // Cancellation drops the real maintenance future. Claims + // are durable, so interrupted backend operations replay. + tokio::select! { + _=token.cancelled()=>break, + result=async { + storage.chunk_maps().await?.maintain(&storage.git_service.obj_storage,64).await + }=>if let Err(error)=result {tracing::warn!(?error,"bounded chunk-map retention tick failed");} + } + } + } + } + })) +} + /// Returns a future that completes when the cancellation token is triggered. async fn shutdown_signal(token: CancellationToken) { token.cancelled().await; @@ -450,6 +479,46 @@ pub async fn start_http(ctx: AppContext, options: CommonHttpOptions) -> MegaResu // TP-15 / 4.1 ②③⑥: open CLs, last_policy vs non-terminal rows, watermark reset. ctx.storage.prepare_push_policy_startup().await?; ctx.storage.check_view_reserved_startup().await?; + let chunk_map_retention_available = if ctx.storage.config().mst2.enabled { + // Detect the actual backend capability. Unsupported CHUNK retention + // must not disable other snapshot objects or generic HTTP serving. + match tokio::time::timeout( + std::time::Duration::from_secs(30), + ctx.storage + .git_service + .obj_storage + .inner + .chunk_map_receipt_inventory(), + ) + .await + { + Ok(Err(crate::orbit_api::error::IoOrbitError::ChunkMapRetentionUnsupported)) => { + tracing::info!( + "CHUNK cold admission and maintenance unavailable for this backing store; other snapshot routes remain available" + ); + false + } + _ => { + let result = async { + ctx.storage + .chunk_maps() + .await? + .maintain(&ctx.storage.git_service.obj_storage, 64) + .await + } + .await; + if let Err(error) = result { + tracing::warn!( + ?error, + "initial CHUNK retention check failed; cold admission remains fail closed" + ); + } + true + } + } + } else { + false + }; // First-build the shared authorization snapshot before the listener binds // (UN-02). `off` is a no-op; a failed first build fails startup. @@ -479,6 +548,11 @@ pub async fn start_http(ctx: AppContext, options: CommonHttpOptions) -> MegaResu spawn_artifact_gc_task(shutdown_context.clone(), shutdown_token.clone())?; let view_worker_handle = spawn_view_worker_task(shutdown_context.storage.clone(), shutdown_token.clone())?; + let chunk_map_retention_handle = spawn_chunk_map_retention_task( + shutdown_context.storage.clone(), + shutdown_token.clone(), + chunk_map_retention_available, + ); let notification_shutdown = shutdown_context.notification_shutdown.clone(); let server_token = shutdown_token.clone(); @@ -541,7 +615,7 @@ pub async fn start_http(ctx: AppContext, options: CommonHttpOptions) -> MegaResu }, }; - let (cleanup_result, artifact_gc_result, view_worker_result) = tokio::join!( + let (cleanup_result, artifact_gc_result, view_worker_result, chunk_map_retention_result) = tokio::join!( async { if let Some(handle) = cleanup_handle { match tokio::time::timeout(std::time::Duration::from_secs(30), handle).await { @@ -612,11 +686,34 @@ pub async fn start_http(ctx: AppContext, options: CommonHttpOptions) -> MegaResu } else { Ok(()) } + }, + async { + if let Some(mut handle) = chunk_map_retention_handle { + match tokio::time::timeout(std::time::Duration::from_secs(30), &mut handle).await { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => { + tracing::error!(%error,"chunk-map retention task panicked"); + Err(()) + } + Err(_) => { + handle.abort(); + let _ = handle.await; + tracing::error!( + "chunk-map retention task exceeded shutdown deadline and was aborted" + ); + Err(()) + } + } + } else { + Ok(()) + } } ); - let shutdown_failed = - cleanup_result.is_err() || artifact_gc_result.is_err() || view_worker_result.is_err(); + let shutdown_failed = cleanup_result.is_err() + || artifact_gc_result.is_err() + || view_worker_result.is_err() + || chunk_map_retention_result.is_err(); match (shutdown_failed, &server_result) { (false, Ok(())) => { tracing::info!("Graceful shutdown completed successfully");