diff --git a/.github/workflows/docs-quality.yml b/.github/workflows/docs-quality.yml index eae33b97..8a41f64a 100644 --- a/.github/workflows/docs-quality.yml +++ b/.github/workflows/docs-quality.yml @@ -2,11 +2,17 @@ name: Documentation Quality on: pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review paths: - "**/*.md" - "**/*.json" - ".github/workflows/**" - "scripts/validate_documentation.py" + - "crates/compute_backend/**" push: branches: - main diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ffe514db..e8702e1e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,6 +61,7 @@ boundaries above remain the target modular MSA architecture. | `tepp_simulation` | known-truth temporal/event data generation | | `validation_core` | RMSE, bias, coverage, graph, and Monte Carlo metrics | | `tepp_api` | versioned DTO, schema, and export contracts | +| `compute_backend` | VRAM-budgeted streamed planning, executable OOM retry plans, and a compensated CPU `f64` reference | No crate exposes placeholder production behavior in Task 1. This prevents an empty façade from becoming a de facto public API before its invariants and tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 36c2e8dd..3cc2e6c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added +- `compute_backend` ADR 0006 first slice: VRAM profiles and reserve-aware micro-batching, executable successive OOM retry plans, CPU fallback, compensated `f64` reference arithmetic with scale-aware parity tolerance, grouped adaptation policies, and fail-closed estimand-preserving memory policies. - `tepp_api` naruon live loopback HTTP/1.1 listener: `serve_one` installs a read/write deadline, requires a loopback `Host`, refuses `Transfer-Encoding` and NIM/proxy credential headers, parses `knowledge_cutoff` as RFC 3339 and refuses a future cutoff, keys analysis-run idempotency by tenant plus key, and proves both analysis-run and export POSTs over a real `TcpStream`. Not a production TLS/`$PORT` service (ADR 0011). - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. @@ -75,6 +76,9 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Changed +- Removed the completed one-shot PR #51 repair job from `docs-quality.yml`; the + workflow no longer invokes deleted repair scripts or requests write authority + after the executable compute implementation is already present. - Clarified ADR 0001 so it owns Rust-first numerical/reference-backend authority while ADR 0011 owns cross-service MSA/service authority. - Clarified ADR 0006 so it owns GPU/VRAM and model-credential boundaries; ADR 0010 now owns LLM orchestration policy and ADR 0015 owns autonomous repository-write/review/merge authority. - Expanded ADR 0002–0005 and 0009–0011 with explicit implementation maturity, alternatives, failure/recovery, compatibility/migration, verification, and rollback/supersession boundaries where they were previously implicit. diff --git a/Cargo.lock b/Cargo.lock index fb502b9c..1d918902 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,6 +114,10 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "compute_backend" +version = "0.1.0" + [[package]] name = "corpus_split" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 92565940..071f3560 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "crates/tepp_simulation", "crates/validation_core", "crates/tepp_api", + "crates/compute_backend", ] default-members = [ "crates/evidence_core", @@ -23,6 +24,7 @@ default-members = [ "crates/tepp_simulation", "crates/validation_core", "crates/tepp_api", + "crates/compute_backend", ] [workspace.package] diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 3f094947..54cd65be 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -33,6 +33,7 @@ TEPP's approved PRD v0.4 and implementation plan are the primary product baselin | Hourly NIM product-development operations | [`docs/operations/HOURLY_NIM_PRODUCT_DEVELOPMENT.md`](docs/operations/HOURLY_NIM_PRODUCT_DEVELOPMENT.md) | | Actions workflow fleet audit | [`docs/operations/ACTIONS_WORKFLOW_FLEET.md`](docs/operations/ACTIONS_WORKFLOW_FLEET.md) | | Actions fleet research doctoring | [`docs/research/actions-workflow-fleet.md`](docs/research/actions-workflow-fleet.md) | +| VRAM budget / GPU fallback doctoring | [`docs/research/vram-budget-types.md`](docs/research/vram-budget-types.md) | | Retention/deletion/legal-hold doctoring | [`docs/research/retention-deletion-legal-hold.md`](docs/research/retention-deletion-legal-hold.md) | | Provider-payload minimization doctoring | [`docs/research/provider-payload-minimization.md`](docs/research/provider-payload-minimization.md) | | Adaptive orchestration router doctoring | [`docs/research/adaptive-orchestration-router.md`](docs/research/adaptive-orchestration-router.md) | diff --git a/crates/compute_backend/Cargo.toml b/crates/compute_backend/Cargo.toml new file mode 100644 index 00000000..503caaac --- /dev/null +++ b/crates/compute_backend/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "compute_backend" +description = "VRAM-budgeted GPU planning with a CPU f64 reference and fail-closed OOM fallback." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true +publish = false + +[dependencies] + +[lints] +workspace = true diff --git a/crates/compute_backend/src/controller.rs b/crates/compute_backend/src/controller.rs new file mode 100644 index 00000000..b9c81402 --- /dev/null +++ b/crates/compute_backend/src/controller.rs @@ -0,0 +1,393 @@ +//! VRAM controller: reserve, predict, autotune, retry, and fall back. + +use crate::error::ComputeBackendError; +use crate::inventory::{DeviceInventory, SafetyReserve, VramBudget}; +use crate::plan::{ComputeBackendKind, FallbackReason, MicroBatchPlan, predicted_peak_bytes}; +use crate::request::{ + CorpusPlacement, CutoffPolicy, ModelComplexity, ObservationRetention, PrecisionMode, + WorkloadRequest, +}; + +/// Plans streamed work under a VRAM budget without changing the estimand. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct VramController { + inventory: DeviceInventory, + max_retries: u32, +} + +impl VramController { + /// Construct a controller with a bounded OOM retry budget. + /// + /// # Errors + /// + /// This constructor is currently infallible for valid inventories. It + /// returns [`Result`] so callers can share the crate error type. + pub const fn new( + inventory: DeviceInventory, + max_retries: u32, + ) -> Result { + Ok(Self { + inventory, + max_retries, + }) + } + + /// Return the reserved safety headroom. + #[must_use] + pub const fn safety_reserve(self) -> SafetyReserve { + self.inventory.safety_reserve() + } + + /// Return the usable VRAM budget. + #[must_use] + pub const fn budget(self) -> VramBudget { + self.inventory.budget() + } + + /// Return the bounded OOM retry budget. + #[must_use] + pub const fn max_retries(self) -> u32 { + self.max_retries + } + + /// Plan a micro-batch or CPU fallback without dropping observations. + /// + /// # Errors + /// + /// Returns a fail-closed [`ComputeBackendError`] when the caller requests a + /// forbidden memory adaptation, mixed-precision finals, or an overflowing + /// peak prediction. + pub fn plan(&self, request: &WorkloadRequest) -> Result { + Self::validate_request(request)?; + + if !self.inventory.device_present() { + return Ok(Self::cpu_plan( + request.requested_batch(), + 0, + FallbackReason::DeviceUnavailable, + )); + } + + if self.inventory.budget().usable_bytes() == 0 { + return Ok(Self::cpu_plan( + request.requested_batch(), + 0, + FallbackReason::InsufficientVram, + )); + } + + Ok(self.gpu_plan_or_cpu( + request, + request.requested_batch(), + 0, + FallbackReason::InsufficientVram, + )) + } + + /// Return the next executable plan after one observed device OOM. + /// + /// Each accepted retry halves the current micro-batch and recomputes its + /// peak estimate from the original workload. Once the configured retry + /// budget is exhausted, or a unit batch fails, the plan switches to the CPU + /// `f64` reference without dropping any observation. + /// + /// # Errors + /// + /// Returns [`ComputeBackendError::RetryBudgetExceeded`] when the supplied + /// plan is already on the CPU path, and validation/overflow errors for an + /// invalid workload or retry counter. + pub fn recover_from_oom( + &self, + request: &WorkloadRequest, + plan: &MicroBatchPlan, + ) -> Result { + Self::validate_request(request)?; + if plan.backend() != ComputeBackendKind::GpuStreamed { + return Err(ComputeBackendError::RetryBudgetExceeded); + } + let next_retry = plan + .oom_retry_count() + .checked_add(1) + .ok_or(ComputeBackendError::InvalidBudget)?; + if next_retry <= self.max_retries && plan.batch_size() > 1 { + return Ok(self.gpu_plan_or_cpu( + request, + plan.batch_size() / 2, + next_retry, + FallbackReason::OutOfMemoryRetryExhausted, + )); + } + Ok(Self::cpu_plan( + request.requested_batch(), + next_retry, + FallbackReason::OutOfMemoryRetryExhausted, + )) + } + + fn validate_request(request: &WorkloadRequest) -> Result<(), ComputeBackendError> { + if request.corpus_placement() == CorpusPlacement::FullCorpusOnDevice { + return Err(ComputeBackendError::FullCorpusTensorRefused); + } + if request.observation_retention() == ObservationRetention::DropToFit { + return Err(ComputeBackendError::ObservationDropForbidden); + } + if request.model_complexity() == ModelComplexity::ReduceToFit { + return Err(ComputeBackendError::ComplexityReductionForbidden); + } + if request.cutoff_policy() == CutoffPolicy::MoveToFit { + return Err(ComputeBackendError::CutoffMutationForbidden); + } + if request.final_quantity_precision() != PrecisionMode::ReferenceF64 { + return Err(ComputeBackendError::UnsupportedPrecision); + } + Ok(()) + } + + fn gpu_plan_or_cpu( + &self, + request: &WorkloadRequest, + mut batch: u32, + oom_retry_count: u32, + fallback_reason: FallbackReason, + ) -> MicroBatchPlan { + loop { + if let Some(plan) = self.gpu_plan_if_fits(request, batch, oom_retry_count) { + return plan; + } + if batch == 1 { + return Self::cpu_plan(request.requested_batch(), oom_retry_count, fallback_reason); + } + batch /= 2; + } + } + + fn gpu_plan_if_fits( + &self, + request: &WorkloadRequest, + batch: u32, + oom_retry_count: u32, + ) -> Option { + let peak = predicted_peak_bytes( + batch, + request.bytes_per_observation(), + request.working_set_bytes(), + ) + .ok()?; + if peak > self.inventory.budget().usable_bytes() { + return None; + } + Some(MicroBatchPlan::new( + ComputeBackendKind::GpuStreamed, + batch, + peak, + PrecisionMode::ReferenceF64, + oom_retry_count, + None, + )) + } + + const fn cpu_plan( + batch_size: u32, + oom_retry_count: u32, + reason: FallbackReason, + ) -> MicroBatchPlan { + MicroBatchPlan::new( + ComputeBackendKind::CpuF64Reference, + batch_size, + 0, + PrecisionMode::ReferenceF64, + oom_retry_count, + Some(reason), + ) + } +} + +#[cfg(test)] +mod tests { + use super::VramController; + use crate::error::ComputeBackendError; + use crate::inventory::DeviceInventory; + use crate::plan::{ComputeBackendKind, FallbackReason, MicroBatchPlan}; + use crate::profile::VramProfile; + use crate::request::{ + AdaptationPolicy, CorpusPlacement, CutoffPolicy, ModelComplexity, ObservationRetention, + PrecisionMode, WorkloadRequest, + }; + + fn request(batch: u32, bytes_per_observation: u64) -> WorkloadRequest { + WorkloadRequest::new( + 4, + 2, + bytes_per_observation, + 8, + batch, + AdaptationPolicy { + corpus_placement: CorpusPlacement::StreamedMicroBatches, + observation_retention: ObservationRetention::KeepAll, + model_complexity: ModelComplexity::KeepSpecified, + cutoff_policy: CutoffPolicy::KeepCutoff, + }, + PrecisionMode::ReferenceF64, + ) + .expect("valid") + } + + #[test] + fn cpu_only_and_unusable_vram_fall_back() { + let cpu = VramController::new(DeviceInventory::cpu_only(VramProfile::Gib4), 1) + .expect("cpu controller"); + assert_eq!(cpu.max_retries(), 1); + assert_eq!( + cpu.safety_reserve().bytes(), + VramProfile::Gib4.safety_bytes() + ); + assert_eq!(cpu.budget().usable_bytes(), 0); + let planned = cpu.plan(&request(4, 8)).expect("cpu plan"); + assert_eq!(planned.backend(), ComputeBackendKind::CpuF64Reference); + assert_eq!(planned.fallback(), Some(FallbackReason::DeviceUnavailable)); + assert_eq!( + cpu.recover_from_oom(&request(4, 8), &planned), + Err(ComputeBackendError::RetryBudgetExceeded) + ); + + let tight = DeviceInventory::gpu(VramProfile::Gib4, VramProfile::Gib4.safety_bytes()) + .expect("tight"); + let controller = VramController::new(tight, 0).expect("tight controller"); + let planned = controller.plan(&request(2, 8)).expect("unusable"); + assert_eq!(planned.fallback(), Some(FallbackReason::InsufficientVram)); + } + + #[test] + fn unit_batch_that_still_exceeds_usable_vram_falls_back() { + let available = VramProfile::Gib4.safety_bytes() + 16; + let inventory = DeviceInventory::gpu(VramProfile::Gib4, available).expect("small usable"); + let controller = VramController::new(inventory, 1).expect("controller"); + let planned = controller.plan(&request(8, 64)).expect("fallback"); + assert_eq!(planned.backend(), ComputeBackendKind::CpuF64Reference); + assert_eq!(planned.fallback(), Some(FallbackReason::InsufficientVram)); + assert_eq!(planned.batch_size(), 8); + assert_eq!(planned.precision(), PrecisionMode::ReferenceF64); + assert_eq!(planned.predicted_peak_bytes(), 0); + assert_eq!(planned.oom_retry_count(), 0); + } + + #[test] + fn overflowing_peak_falls_back_to_cpu() { + let inventory = + DeviceInventory::gpu(VramProfile::Gib24, VramProfile::Gib24.bytes()).expect("24"); + let controller = VramController::new(inventory, 1).expect("controller"); + let huge = WorkloadRequest::new( + 1, + 1, + u64::MAX, + u64::MAX, + 2, + AdaptationPolicy { + corpus_placement: CorpusPlacement::StreamedMicroBatches, + observation_retention: ObservationRetention::KeepAll, + model_complexity: ModelComplexity::KeepSpecified, + cutoff_policy: CutoffPolicy::KeepCutoff, + }, + PrecisionMode::ReferenceF64, + ) + .expect("request"); + let plan = controller.plan(&huge).expect("overflow falls back"); + assert_eq!(plan.backend(), ComputeBackendKind::CpuF64Reference); + assert_eq!(plan.fallback(), Some(FallbackReason::InsufficientVram)); + } + + #[test] + fn overflowing_oom_retry_peak_falls_back_to_cpu() { + let inventory = + DeviceInventory::gpu(VramProfile::Gib24, VramProfile::Gib24.bytes()).expect("24"); + let controller = VramController::new(inventory, 1).expect("controller"); + let huge = WorkloadRequest::new( + 1, + 1, + u64::MAX, + u64::MAX, + 2, + AdaptationPolicy { + corpus_placement: CorpusPlacement::StreamedMicroBatches, + observation_retention: ObservationRetention::KeepAll, + model_complexity: ModelComplexity::KeepSpecified, + cutoff_policy: CutoffPolicy::KeepCutoff, + }, + PrecisionMode::ReferenceF64, + ) + .expect("request"); + let initial = MicroBatchPlan::new( + ComputeBackendKind::GpuStreamed, + 2, + 0, + PrecisionMode::ReferenceF64, + 0, + None, + ); + let plan = controller + .recover_from_oom(&huge, &initial) + .expect("overflow falls back"); + assert_eq!(plan.backend(), ComputeBackendKind::CpuF64Reference); + assert_eq!( + plan.fallback(), + Some(FallbackReason::OutOfMemoryRetryExhausted) + ); + } + + #[test] + fn oom_recovery_emits_retries_then_falls_back() { + let inventory = + DeviceInventory::gpu(VramProfile::Gib12, VramProfile::Gib12.bytes()).expect("12"); + let controller = VramController::new(inventory, 1).expect("controller"); + let workload = request(4, 8); + let initial = controller.plan(&workload).expect("gpu"); + let retry = controller + .recover_from_oom(&workload, &initial) + .expect("retry"); + assert_eq!(retry.backend(), ComputeBackendKind::GpuStreamed); + assert_eq!(retry.batch_size(), 2); + assert_eq!(retry.oom_retry_count(), 1); + let fallback = controller + .recover_from_oom(&workload, &retry) + .expect("fallback"); + assert_eq!(fallback.backend(), ComputeBackendKind::CpuF64Reference); + // CPU `f64` fallback restores the requested batch; it is not a GPU retry plan. + assert_eq!(fallback.batch_size(), 4); + assert_eq!(fallback.oom_retry_count(), 2); + + let zero_retry = VramController::new(inventory, 0).expect("zero retry"); + let immediate = zero_retry + .recover_from_oom(&workload, &initial) + .expect("immediate fallback"); + assert_eq!(immediate.backend(), ComputeBackendKind::CpuF64Reference); + assert_eq!(immediate.oom_retry_count(), 1); + + let unit_workload = request(1, 8); + let unit_plan = controller.plan(&unit_workload).expect("unit gpu"); + let unit_fallback = controller + .recover_from_oom(&unit_workload, &unit_plan) + .expect("unit fallback"); + assert_eq!(unit_fallback.backend(), ComputeBackendKind::CpuF64Reference); + assert_eq!(unit_fallback.batch_size(), 1); + } + + #[test] + fn overflowing_retry_counter_fails_closed() { + let inventory = + DeviceInventory::gpu(VramProfile::Gib12, VramProfile::Gib12.bytes()).expect("12"); + let controller = VramController::new(inventory, u32::MAX).expect("controller"); + let workload = request(4, 8); + let invalid = MicroBatchPlan::new( + ComputeBackendKind::GpuStreamed, + 4, + 40, + PrecisionMode::ReferenceF64, + u32::MAX, + None, + ); + assert_eq!( + controller.recover_from_oom(&workload, &invalid), + Err(ComputeBackendError::InvalidBudget) + ); + } +} diff --git a/crates/compute_backend/src/error.rs b/crates/compute_backend/src/error.rs new file mode 100644 index 00000000..5db3b3e5 --- /dev/null +++ b/crates/compute_backend/src/error.rs @@ -0,0 +1,181 @@ +//! Fail-closed VRAM and compute-backend errors. + +use std::fmt; + +/// A fail-closed compute-backend error. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum ComputeBackendError { + /// Device allocation failed. This is an expected operating state. + OutOfMemory, + /// The accelerator disappeared after planning. + DeviceLoss, + /// A reference or diagnostic quantity was non-finite. + NonFiniteOutput, + /// CPU `f64` and candidate outputs diverged beyond tolerance. + ParityFailure, + /// A parity tolerance was negative. + InvalidTolerance, + /// Mixed precision was requested for a final diagnostic quantity. + UnsupportedPrecision, + /// A claimed accelerator could not be initialized. + BackendInitFailure, + /// A full document-by-topic tensor was requested on device memory. + FullCorpusTensorRefused, + /// Observations would be dropped to fit memory. + ObservationDropForbidden, + /// Topic or model complexity would be reduced to fit memory. + ComplexityReductionForbidden, + /// A knowledge cutoff would change to fit memory. + CutoffMutationForbidden, + /// A budget, inventory, or workload field was empty or overflowed. + InvalidBudget, + /// Telemetry attempted to carry raw source text. + SourceTextInTelemetry, + /// Further OOM retries were requested after the bounded budget. + RetryBudgetExceeded, +} + +impl ComputeBackendError { + /// Return whether the error is a tested operating state rather than a bug. + #[must_use] + pub const fn is_expected_operating_state(self) -> bool { + matches!(self, Self::OutOfMemory) + } +} + +impl fmt::Display for ComputeBackendError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + Self::OutOfMemory => "device out of memory", + Self::DeviceLoss => "compute device lost", + Self::NonFiniteOutput => "non-finite compute output", + Self::ParityFailure => "cpu gpu parity failure", + Self::InvalidTolerance => "invalid parity tolerance", + Self::UnsupportedPrecision => "mixed precision cannot finalize diagnostics", + Self::BackendInitFailure => "compute backend initialization failed", + Self::FullCorpusTensorRefused => "full-corpus device tensor is refused", + Self::ObservationDropForbidden => "observations cannot be dropped to fit memory", + Self::ComplexityReductionForbidden => { + "model complexity cannot be reduced to fit memory" + } + Self::CutoffMutationForbidden => "knowledge cutoff cannot change to fit memory", + Self::InvalidBudget => "invalid compute budget", + Self::SourceTextInTelemetry => "telemetry cannot carry source text", + Self::RetryBudgetExceeded => "oom retry budget exceeded", + }; + formatter.write_str(message) + } +} + +impl std::error::Error for ComputeBackendError {} + +/// Return the typed out-of-memory operating state. +#[must_use] +pub const fn report_out_of_memory() -> ComputeBackendError { + ComputeBackendError::OutOfMemory +} + +/// Return the typed device-loss failure. +#[must_use] +pub const fn report_device_loss() -> ComputeBackendError { + ComputeBackendError::DeviceLoss +} + +/// Return the typed backend-initialization failure. +#[must_use] +pub const fn refuse_uninitialized_backend() -> ComputeBackendError { + ComputeBackendError::BackendInitFailure +} + +#[cfg(test)] +mod tests { + use super::{ + ComputeBackendError, refuse_uninitialized_backend, report_device_loss, report_out_of_memory, + }; + + #[test] + fn messages_and_operating_states_are_stable() { + for (error, message, expected) in [ + ( + ComputeBackendError::OutOfMemory, + "device out of memory", + true, + ), + ( + ComputeBackendError::DeviceLoss, + "compute device lost", + false, + ), + ( + ComputeBackendError::NonFiniteOutput, + "non-finite compute output", + false, + ), + ( + ComputeBackendError::ParityFailure, + "cpu gpu parity failure", + false, + ), + ( + ComputeBackendError::InvalidTolerance, + "invalid parity tolerance", + false, + ), + ( + ComputeBackendError::UnsupportedPrecision, + "mixed precision cannot finalize diagnostics", + false, + ), + ( + ComputeBackendError::BackendInitFailure, + "compute backend initialization failed", + false, + ), + ( + ComputeBackendError::FullCorpusTensorRefused, + "full-corpus device tensor is refused", + false, + ), + ( + ComputeBackendError::ObservationDropForbidden, + "observations cannot be dropped to fit memory", + false, + ), + ( + ComputeBackendError::ComplexityReductionForbidden, + "model complexity cannot be reduced to fit memory", + false, + ), + ( + ComputeBackendError::CutoffMutationForbidden, + "knowledge cutoff cannot change to fit memory", + false, + ), + ( + ComputeBackendError::InvalidBudget, + "invalid compute budget", + false, + ), + ( + ComputeBackendError::SourceTextInTelemetry, + "telemetry cannot carry source text", + false, + ), + ( + ComputeBackendError::RetryBudgetExceeded, + "oom retry budget exceeded", + false, + ), + ] { + assert_eq!(error.to_string(), message); + assert_eq!(error.is_expected_operating_state(), expected); + } + assert_eq!(report_out_of_memory(), ComputeBackendError::OutOfMemory); + assert_eq!(report_device_loss(), ComputeBackendError::DeviceLoss); + assert_eq!( + refuse_uninitialized_backend(), + ComputeBackendError::BackendInitFailure + ); + } +} diff --git a/crates/compute_backend/src/inventory.rs b/crates/compute_backend/src/inventory.rs new file mode 100644 index 00000000..3e5ead73 --- /dev/null +++ b/crates/compute_backend/src/inventory.rs @@ -0,0 +1,187 @@ +//! Measured device inventory and usable VRAM budget. + +use crate::error::ComputeBackendError; +use crate::profile::VramProfile; + +/// Observed accelerator inventory for one planning decision. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct DeviceInventory { + profile: VramProfile, + available_bytes: u64, + device_present: bool, +} + +/// Reserved bytes that working tensors must not consume. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SafetyReserve { + bytes: u64, +} + +/// Usable VRAM remaining after the safety reserve. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct VramBudget { + profile: VramProfile, + available_bytes: u64, + safety_bytes: u64, + usable_bytes: u64, +} + +impl DeviceInventory { + /// Construct a present GPU inventory that cannot exceed its profile. + /// + /// # Errors + /// + /// Returns [`ComputeBackendError::InvalidBudget`] when `available_bytes` is + /// zero or larger than the profile capacity. + pub const fn gpu( + profile: VramProfile, + available_bytes: u64, + ) -> Result { + if available_bytes == 0 || available_bytes > profile.bytes() { + return Err(ComputeBackendError::InvalidBudget); + } + Ok(Self { + profile, + available_bytes, + device_present: true, + }) + } + + /// Construct a CPU-only inventory with no accelerator. + #[must_use] + pub const fn cpu_only(profile: VramProfile) -> Self { + Self { + profile, + available_bytes: 0, + device_present: false, + } + } + + /// Return the governing profile. + #[must_use] + pub const fn profile(self) -> VramProfile { + self.profile + } + + /// Return currently free device bytes. + #[must_use] + pub const fn available_bytes(self) -> u64 { + self.available_bytes + } + + /// Return whether an accelerator is present. + #[must_use] + pub const fn device_present(self) -> bool { + self.device_present + } + + /// Return the reserved safety headroom. + #[must_use] + pub const fn safety_reserve(self) -> SafetyReserve { + SafetyReserve { + bytes: self.profile.safety_bytes(), + } + } + + /// Return the usable budget after reserving safety memory. + #[must_use] + pub const fn budget(self) -> VramBudget { + let safety_bytes = self.profile.safety_bytes(); + let usable_bytes = if self.device_present && self.available_bytes > safety_bytes { + self.available_bytes - safety_bytes + } else { + 0 + }; + VramBudget { + profile: self.profile, + available_bytes: self.available_bytes, + safety_bytes, + usable_bytes, + } + } +} + +impl SafetyReserve { + /// Return reserved bytes. + #[must_use] + pub const fn bytes(self) -> u64 { + self.bytes + } +} + +impl VramBudget { + /// Return the governing profile. + #[must_use] + pub const fn profile(self) -> VramProfile { + self.profile + } + + /// Return observed free bytes. + #[must_use] + pub const fn available_bytes(self) -> u64 { + self.available_bytes + } + + /// Return reserved safety bytes. + #[must_use] + pub const fn safety_bytes(self) -> u64 { + self.safety_bytes + } + + /// Return bytes available for working tensors. + #[must_use] + pub const fn usable_bytes(self) -> u64 { + self.usable_bytes + } +} + +#[cfg(test)] +mod tests { + use super::DeviceInventory; + use crate::error::ComputeBackendError; + use crate::profile::VramProfile; + + #[test] + fn gpu_inventory_rejects_empty_and_oversize_availability() { + assert_eq!( + DeviceInventory::gpu(VramProfile::Gib4, 0), + Err(ComputeBackendError::InvalidBudget) + ); + assert_eq!( + DeviceInventory::gpu(VramProfile::Gib4, VramProfile::Gib4.bytes() + 1), + Err(ComputeBackendError::InvalidBudget) + ); + let inventory = + DeviceInventory::gpu(VramProfile::Gib4, VramProfile::Gib4.bytes()).expect("full 4 GiB"); + assert!(inventory.device_present()); + assert_eq!(inventory.profile(), VramProfile::Gib4); + assert_eq!(inventory.available_bytes(), VramProfile::Gib4.bytes()); + assert_eq!( + inventory.safety_reserve().bytes(), + VramProfile::Gib4.safety_bytes() + ); + let budget = inventory.budget(); + assert_eq!(budget.profile(), VramProfile::Gib4); + assert_eq!(budget.available_bytes(), VramProfile::Gib4.bytes()); + assert_eq!(budget.safety_bytes(), VramProfile::Gib4.safety_bytes()); + assert_eq!( + budget.usable_bytes(), + VramProfile::Gib4.bytes() - VramProfile::Gib4.safety_bytes() + ); + } + + #[test] + fn cpu_only_inventory_has_zero_usable_bytes() { + let inventory = DeviceInventory::cpu_only(VramProfile::Gib8); + assert!(!inventory.device_present()); + assert_eq!(inventory.available_bytes(), 0); + assert_eq!(inventory.budget().usable_bytes(), 0); + } + + #[test] + fn availability_at_or_below_reserve_is_unusable() { + let reserve = VramProfile::Gib6.safety_bytes(); + let inventory = DeviceInventory::gpu(VramProfile::Gib6, reserve).expect("at reserve"); + assert_eq!(inventory.budget().usable_bytes(), 0); + } +} diff --git a/crates/compute_backend/src/lib.rs b/crates/compute_backend/src/lib.rs new file mode 100644 index 00000000..9431056b --- /dev/null +++ b/crates/compute_backend/src/lib.rs @@ -0,0 +1,67 @@ +#![forbid(unsafe_code)] +#![deny(missing_docs)] +//! VRAM-budgeted compute planning with a CPU `f64` reference path. +//! +//! This crate plans streamed GPU work under 4/6/8/12/24-GiB profiles and +//! treats out-of-memory as an expected operating state. It does not claim a +//! live accelerator lane. Full document-by-topic tensors are refused, and +//! memory pressure may not drop observations, shrink the model, or move a +//! knowledge cutoff. + +mod controller; +mod error; +mod inventory; +mod plan; +mod profile; +mod reference; +mod request; +mod telemetry; + +/// VRAM controller that autotunes batches and falls back to CPU. +pub use controller::VramController; +/// Fail-closed compute-backend errors. +pub use error::ComputeBackendError; +/// Typed backend-initialization failure. +pub use error::refuse_uninitialized_backend; +/// Typed device-loss failure. +pub use error::report_device_loss; +/// Typed out-of-memory operating state. +pub use error::report_out_of_memory; +/// Observed accelerator inventory. +pub use inventory::DeviceInventory; +/// Reserved unused device bytes. +pub use inventory::SafetyReserve; +/// Usable bytes after the safety reserve. +pub use inventory::VramBudget; +/// Selected execution backend. +pub use plan::ComputeBackendKind; +/// Why a plan left the accelerator. +pub use plan::FallbackReason; +/// Planned micro-batch. +pub use plan::MicroBatchPlan; +/// Predict peak bytes for a micro-batch. +pub use plan::predicted_peak_bytes; +/// Accepted device-class profile. +pub use profile::VramProfile; +/// Compare a candidate quantity to the CPU `f64` reference. +pub use reference::require_cpu_gpu_parity; +/// Reject a non-finite diagnostic quantity. +pub use reference::require_finite; +/// CPU `f64` streamed weighted sum. +pub use reference::streamed_weighted_sum; +/// Memory-adaptation policies grouped for safe workload construction. +pub use request::AdaptationPolicy; +/// Corpus placement policy. +pub use request::CorpusPlacement; +/// Cutoff-mutation policy. +pub use request::CutoffPolicy; +/// Model-complexity policy. +pub use request::ModelComplexity; +/// Observation-retention policy. +pub use request::ObservationRetention; +/// Transient versus diagnostic precision. +pub use request::PrecisionMode; +/// Streamed workload request. +pub use request::WorkloadRequest; +/// Resource telemetry without source text. +pub use telemetry::AllocationTelemetry; diff --git a/crates/compute_backend/src/plan.rs b/crates/compute_backend/src/plan.rs new file mode 100644 index 00000000..e85976dd --- /dev/null +++ b/crates/compute_backend/src/plan.rs @@ -0,0 +1,145 @@ +//! Planned backend, micro-batch, and fallback reason. + +use crate::request::PrecisionMode; + +/// Executable backend selected by the VRAM controller. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ComputeBackendKind { + /// CPU `f64` numerical reference and universal fallback. + CpuF64Reference, + /// Streamed GPU plan that still finalizes diagnostics on CPU `f64`. + GpuStreamed, +} + +/// Why a plan left the accelerator or reduced a batch. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FallbackReason { + /// Usable VRAM could not hold even a unit micro-batch. + InsufficientVram, + /// Bounded OOM retries still could not keep the work on device. + OutOfMemoryRetryExhausted, + /// No accelerator was present. + DeviceUnavailable, + /// A non-finite guard forced the CPU reference path. + NonFiniteGuard, +} + +/// A planned micro-batch that preserves the full observation set. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct MicroBatchPlan { + backend: ComputeBackendKind, + batch_size: u32, + predicted_peak_bytes: u64, + precision: PrecisionMode, + oom_retry_count: u32, + fallback: Option, +} + +impl MicroBatchPlan { + pub(crate) const fn new( + backend: ComputeBackendKind, + batch_size: u32, + predicted_peak_bytes: u64, + precision: PrecisionMode, + oom_retry_count: u32, + fallback: Option, + ) -> Self { + Self { + backend, + batch_size, + predicted_peak_bytes, + precision, + oom_retry_count, + fallback, + } + } + + /// Return the selected backend. + #[must_use] + pub const fn backend(self) -> ComputeBackendKind { + self.backend + } + + /// Return the planned micro-batch size. + #[must_use] + pub const fn batch_size(self) -> u32 { + self.batch_size + } + + /// Return the predicted peak working-set plus batch charge. + #[must_use] + pub const fn predicted_peak_bytes(self) -> u64 { + self.predicted_peak_bytes + } + + /// Return the precision used for final diagnostics. + #[must_use] + pub const fn precision(self) -> PrecisionMode { + self.precision + } + + /// Return how many observed OOMs led to this plan. + #[must_use] + pub const fn oom_retry_count(self) -> u32 { + self.oom_retry_count + } + + /// Return the fallback reason, if the accelerator was not used. + #[must_use] + pub const fn fallback(self) -> Option { + self.fallback + } +} + +/// Predict peak bytes for a micro-batch plus fixed working set. +/// +/// # Errors +/// +/// Returns [`crate::ComputeBackendError::InvalidBudget`] on overflow. +pub const fn predicted_peak_bytes( + batch_size: u32, + bytes_per_observation: u64, + working_set_bytes: u64, +) -> Result { + let Some(batch_bytes) = bytes_per_observation.checked_mul(batch_size as u64) else { + return Err(crate::ComputeBackendError::InvalidBudget); + }; + match batch_bytes.checked_add(working_set_bytes) { + Some(peak) => Ok(peak), + None => Err(crate::ComputeBackendError::InvalidBudget), + } +} + +#[cfg(test)] +mod tests { + use super::{ComputeBackendKind, FallbackReason, MicroBatchPlan, predicted_peak_bytes}; + use crate::error::ComputeBackendError; + use crate::request::PrecisionMode; + + #[test] + fn peak_prediction_and_plan_accessors() { + assert_eq!(predicted_peak_bytes(2, 8, 16).expect("peak"), 32); + assert_eq!( + predicted_peak_bytes(2, u64::MAX, 1), + Err(ComputeBackendError::InvalidBudget) + ); + assert_eq!( + predicted_peak_bytes(1, u64::MAX, 1), + Err(ComputeBackendError::InvalidBudget) + ); + let plan = MicroBatchPlan::new( + ComputeBackendKind::CpuF64Reference, + 3, + 24, + PrecisionMode::ReferenceF64, + 2, + Some(FallbackReason::NonFiniteGuard), + ); + assert_eq!(plan.backend(), ComputeBackendKind::CpuF64Reference); + assert_eq!(plan.batch_size(), 3); + assert_eq!(plan.predicted_peak_bytes(), 24); + assert_eq!(plan.precision(), PrecisionMode::ReferenceF64); + assert_eq!(plan.oom_retry_count(), 2); + assert_eq!(plan.fallback(), Some(FallbackReason::NonFiniteGuard)); + } +} diff --git a/crates/compute_backend/src/profile.rs b/crates/compute_backend/src/profile.rs new file mode 100644 index 00000000..d0f20291 --- /dev/null +++ b/crates/compute_backend/src/profile.rs @@ -0,0 +1,68 @@ +//! Accepted VRAM device-class profiles. + +/// Binary-gigabyte VRAM profile used to select micro-batches. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum VramProfile { + /// 4 GiB class. + Gib4, + /// 6 GiB class. + Gib6, + /// 8 GiB class. + Gib8, + /// 12 GiB class. + Gib12, + /// 24 GiB class. + Gib24, +} + +impl VramProfile { + /// One binary gigabyte in bytes. + pub const GIBIBYTE: u64 = 1 << 30; + + /// Return every accepted profile in increasing capacity order. + #[must_use] + pub const fn all() -> [Self; 5] { + [Self::Gib4, Self::Gib6, Self::Gib8, Self::Gib12, Self::Gib24] + } + + /// Return the profile capacity in binary gigabytes. + #[must_use] + pub const fn gibibytes(self) -> u64 { + match self { + Self::Gib4 => 4, + Self::Gib6 => 6, + Self::Gib8 => 8, + Self::Gib12 => 12, + Self::Gib24 => 24, + } + } + + /// Return the profile capacity in bytes. + #[must_use] + pub const fn bytes(self) -> u64 { + self.gibibytes() * Self::GIBIBYTE + } + + /// Return the reserved safety headroom (one eighth of capacity). + #[must_use] + pub const fn safety_bytes(self) -> u64 { + self.bytes() / 8 + } +} + +#[cfg(test)] +mod tests { + use super::VramProfile; + + #[test] + fn capacities_match_accepted_profiles() { + assert_eq!(VramProfile::GIBIBYTE, 1_073_741_824); + assert_eq!(VramProfile::Gib6.gibibytes(), 6); + assert_eq!(VramProfile::Gib8.bytes(), 8 * VramProfile::GIBIBYTE); + assert_eq!( + VramProfile::Gib12.safety_bytes(), + VramProfile::Gib12.bytes() / 8 + ); + assert_eq!(VramProfile::all().len(), 5); + } +} diff --git a/crates/compute_backend/src/reference.rs b/crates/compute_backend/src/reference.rs new file mode 100644 index 00000000..cc5eb57d --- /dev/null +++ b/crates/compute_backend/src/reference.rs @@ -0,0 +1,156 @@ +//! CPU `f64` streamed reference arithmetic. + +use crate::error::ComputeBackendError; + +/// Stream a compensated weighted sum on the CPU `f64` reference path. +/// +/// Neumaier-style compensation preserves low-order terms in cancellation-heavy +/// inputs while keeping deterministic input order. This sequential function is +/// the numerical reference for later fixed-pool CPU and GPU implementations. +/// +/// # Errors +/// +/// Returns [`ComputeBackendError::InvalidBudget`] when the slices are empty or +/// unequal, and [`ComputeBackendError::NonFiniteOutput`] when any term or +/// accumulator is non-finite. +pub fn streamed_weighted_sum(weights: &[f64], values: &[f64]) -> Result { + if weights.is_empty() || weights.len() != values.len() { + return Err(ComputeBackendError::InvalidBudget); + } + let mut total = 0.0_f64; + let mut compensation = 0.0_f64; + for (weight, value) in weights.iter().zip(values) { + let term = require_finite(*weight)? * require_finite(*value)?; + let term = require_finite(term)?; + let next = require_finite(total + term)?; + let correction = if total.abs() >= term.abs() { + (total - next) + term + } else { + (term - next) + total + }; + compensation = require_finite(compensation + correction)?; + total = next; + } + require_finite(total + compensation) +} + +/// Reject a non-finite diagnostic quantity. +/// +/// # Errors +/// +/// Returns [`ComputeBackendError::NonFiniteOutput`] when `value` is NaN or +/// infinite. +pub fn require_finite(value: f64) -> Result { + if value.is_finite() { + Ok(value) + } else { + Err(ComputeBackendError::NonFiniteOutput) + } +} + +/// Compare a candidate quantity against the CPU `f64` reference. +/// +/// # Errors +/// +/// Returns [`ComputeBackendError::NonFiniteOutput`] when either value or the +/// tolerance is non-finite, [`ComputeBackendError::InvalidTolerance`] for a +/// negative tolerance, and [`ComputeBackendError::ParityFailure`] when the +/// normalized gap exceeds `tolerance`, where the gap is divided by +/// `max(1, |reference|, |candidate|)`. Computing the normalized gap first +/// prevents a finite tolerance bound from overflowing before comparison. +pub fn require_cpu_gpu_parity( + cpu_reference: f64, + candidate: f64, + tolerance: f64, +) -> Result<(), ComputeBackendError> { + let left = require_finite(cpu_reference)?; + let right = require_finite(candidate)?; + let bound = require_finite(tolerance)?; + if bound < 0.0 { + return Err(ComputeBackendError::InvalidTolerance); + } + let scale = left.abs().max(right.abs()).max(1.0); + let normalized_gap = require_finite((left - right).abs() / scale)?; + if normalized_gap <= bound { + Ok(()) + } else { + Err(ComputeBackendError::ParityFailure) + } +} + +#[cfg(test)] +mod tests { + use super::{require_cpu_gpu_parity, require_finite, streamed_weighted_sum}; + use crate::error::ComputeBackendError; + + #[test] + fn compensated_reference_recovers_low_order_cancellation_term() { + let result = + streamed_weighted_sum(&[1.0, 1.0, 1.0], &[1e16, 1.0, -1e16]).expect("compensated sum"); + assert!((result - 1.0).abs() < 1e-15); + let reverse = streamed_weighted_sum(&[1.0, 1.0, 1.0], &[-1e16, 1.0, 1e16]) + .expect("reverse compensation branch"); + assert!((reverse - 1.0).abs() < 1e-15); + } + + #[test] + fn reference_path_rejects_invalid_and_non_finite_input() { + assert_eq!( + streamed_weighted_sum(&[], &[1.0]), + Err(ComputeBackendError::InvalidBudget) + ); + assert_eq!( + streamed_weighted_sum(&[1.0], &[1.0, 2.0]), + Err(ComputeBackendError::InvalidBudget) + ); + assert_eq!( + streamed_weighted_sum(&[f64::NAN], &[1.0]), + Err(ComputeBackendError::NonFiniteOutput) + ); + assert_eq!( + streamed_weighted_sum(&[1.0], &[f64::INFINITY]), + Err(ComputeBackendError::NonFiniteOutput) + ); + assert_eq!( + streamed_weighted_sum(&[1e308], &[1e308]), + Err(ComputeBackendError::NonFiniteOutput) + ); + assert_eq!( + require_finite(f64::NEG_INFINITY), + Err(ComputeBackendError::NonFiniteOutput) + ); + let finite = require_finite(1.5).expect("finite"); + assert!((finite - 1.5).abs() < 1e-15); + require_cpu_gpu_parity(1.0, 1.0, 0.0).expect("exact parity"); + assert_eq!( + require_cpu_gpu_parity(1.0, 2.0, 0.1), + Err(ComputeBackendError::ParityFailure) + ); + require_cpu_gpu_parity(1.0e12, 1.0e12 + 1.0e6, 1.0e-6) + .expect("relative parity at large scale"); + assert_eq!( + require_cpu_gpu_parity(1.0e12, 1.0e12 + 2.0e6, 1.0e-6), + Err(ComputeBackendError::ParityFailure) + ); + assert_eq!( + require_cpu_gpu_parity(1.0, 1.0, -0.1), + Err(ComputeBackendError::InvalidTolerance) + ); + assert_eq!( + require_cpu_gpu_parity(f64::NAN, 1.0, 0.1), + Err(ComputeBackendError::NonFiniteOutput) + ); + assert_eq!( + require_cpu_gpu_parity(1.0, f64::NAN, 0.1), + Err(ComputeBackendError::NonFiniteOutput) + ); + assert_eq!( + require_cpu_gpu_parity(1.0, 1.0, f64::NAN), + Err(ComputeBackendError::NonFiniteOutput) + ); + assert_eq!( + require_cpu_gpu_parity(f64::MAX, -f64::MAX, f64::MAX), + Err(ComputeBackendError::NonFiniteOutput) + ); + } +} diff --git a/crates/compute_backend/src/request.rs b/crates/compute_backend/src/request.rs new file mode 100644 index 00000000..8189cabc --- /dev/null +++ b/crates/compute_backend/src/request.rs @@ -0,0 +1,262 @@ +//! Workload request and precision policy. + +use crate::error::ComputeBackendError; + +/// Arithmetic mode for transient kernels versus final diagnostics. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PrecisionMode { + /// CPU `f64` reference precision required for diagnostics. + ReferenceF64, + /// Approved mixed precision for transient device computation only. + TransientMixed, +} + +/// Whether a full document-by-topic tensor may reside on device. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CorpusPlacement { + /// Stream micro-batches only. + StreamedMicroBatches, + /// Pin the full corpus responsibility tensor on the device. + FullCorpusOnDevice, +} + +/// Whether observations may be dropped under memory pressure. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ObservationRetention { + /// Keep every observation. + KeepAll, + /// Drop observations so a batch fits. + DropToFit, +} + +/// Whether topic or model complexity may shrink to fit memory. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ModelComplexity { + /// Keep the requested topic/model complexity. + KeepSpecified, + /// Reduce complexity so a batch fits. + ReduceToFit, +} + +/// Whether a knowledge cutoff may move to fit memory. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CutoffPolicy { + /// Keep the requested cutoff. + KeepCutoff, + /// Move the cutoff so a batch fits. + MoveToFit, +} + +/// Memory-adaptation policies that must remain explicit at workload creation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct AdaptationPolicy { + /// Whether the corpus may be fully resident on the device. + pub corpus_placement: CorpusPlacement, + /// Whether observations may be dropped to fit the device budget. + pub observation_retention: ObservationRetention, + /// Whether model complexity may be reduced to fit the device budget. + pub model_complexity: ModelComplexity, + /// Whether the knowledge cutoff may move to fit the device budget. + pub cutoff_policy: CutoffPolicy, +} + +/// A streamed workload that must never pin a full document-by-topic tensor. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct WorkloadRequest { + document_count: u64, + topic_count: u64, + bytes_per_observation: u64, + working_set_bytes: u64, + requested_batch: u32, + corpus_placement: CorpusPlacement, + observation_retention: ObservationRetention, + model_complexity: ModelComplexity, + cutoff_policy: CutoffPolicy, + final_quantity_precision: PrecisionMode, +} + +impl WorkloadRequest { + /// Construct a fail-closed workload request. + /// + /// Streamed document and topic cardinalities are stored independently; the + /// constructor deliberately does not materialize or size a hypothetical + /// full-corpus tensor that the controller refuses to allocate. + /// + /// # Errors + /// + /// Returns [`ComputeBackendError::InvalidBudget`] when counts, batch size, + /// or per-observation bytes are zero. + pub const fn new( + document_count: u64, + topic_count: u64, + bytes_per_observation: u64, + working_set_bytes: u64, + requested_batch: u32, + policy: AdaptationPolicy, + final_quantity_precision: PrecisionMode, + ) -> Result { + if document_count == 0 + || topic_count == 0 + || bytes_per_observation == 0 + || requested_batch == 0 + { + return Err(ComputeBackendError::InvalidBudget); + } + Ok(Self { + document_count, + topic_count, + bytes_per_observation, + working_set_bytes, + requested_batch, + corpus_placement: policy.corpus_placement, + observation_retention: policy.observation_retention, + model_complexity: policy.model_complexity, + cutoff_policy: policy.cutoff_policy, + final_quantity_precision, + }) + } + + /// Return the document count. + #[must_use] + pub const fn document_count(self) -> u64 { + self.document_count + } + + /// Return the topic count. + #[must_use] + pub const fn topic_count(self) -> u64 { + self.topic_count + } + + /// Return bytes charged per streamed observation. + #[must_use] + pub const fn bytes_per_observation(self) -> u64 { + self.bytes_per_observation + } + + /// Return the fixed working-set charge. + #[must_use] + pub const fn working_set_bytes(self) -> u64 { + self.working_set_bytes + } + + /// Return the caller-requested micro-batch. + #[must_use] + pub const fn requested_batch(self) -> u32 { + self.requested_batch + } + + /// Return the corpus placement policy. + #[must_use] + pub const fn corpus_placement(self) -> CorpusPlacement { + self.corpus_placement + } + + /// Return the observation-retention policy. + #[must_use] + pub const fn observation_retention(self) -> ObservationRetention { + self.observation_retention + } + + /// Return the model-complexity policy. + #[must_use] + pub const fn model_complexity(self) -> ModelComplexity { + self.model_complexity + } + + /// Return the cutoff policy. + #[must_use] + pub const fn cutoff_policy(self) -> CutoffPolicy { + self.cutoff_policy + } + + /// Return the precision required for final diagnostics. + #[must_use] + pub const fn final_quantity_precision(self) -> PrecisionMode { + self.final_quantity_precision + } +} + +#[cfg(test)] +mod tests { + use super::{ + AdaptationPolicy, CorpusPlacement, CutoffPolicy, ModelComplexity, ObservationRetention, + PrecisionMode, WorkloadRequest, + }; + use crate::error::ComputeBackendError; + + fn request( + documents: u64, + topics: u64, + bytes_per_observation: u64, + batch: u32, + ) -> Result { + WorkloadRequest::new( + documents, + topics, + bytes_per_observation, + 0, + batch, + AdaptationPolicy { + corpus_placement: CorpusPlacement::StreamedMicroBatches, + observation_retention: ObservationRetention::KeepAll, + model_complexity: ModelComplexity::KeepSpecified, + cutoff_policy: CutoffPolicy::KeepCutoff, + }, + PrecisionMode::ReferenceF64, + ) + } + + #[test] + fn request_rejects_zero_counts() { + assert_eq!(request(0, 1, 8, 1), Err(ComputeBackendError::InvalidBudget)); + assert_eq!(request(1, 0, 8, 1), Err(ComputeBackendError::InvalidBudget)); + assert_eq!(request(1, 1, 0, 1), Err(ComputeBackendError::InvalidBudget)); + assert_eq!(request(1, 1, 8, 0), Err(ComputeBackendError::InvalidBudget)); + } + + #[test] + fn streamed_dimensions_are_not_multiplied_into_a_full_tensor() { + let request = request(u64::MAX, u64::MAX, 8, 1).expect("streamed cardinality"); + assert_eq!(request.document_count(), u64::MAX); + assert_eq!(request.topic_count(), u64::MAX); + } + + #[test] + fn request_accessors_preserve_policy_enums() { + let request = WorkloadRequest::new( + 2, + 3, + 8, + 16, + 4, + AdaptationPolicy { + corpus_placement: CorpusPlacement::StreamedMicroBatches, + observation_retention: ObservationRetention::KeepAll, + model_complexity: ModelComplexity::KeepSpecified, + cutoff_policy: CutoffPolicy::KeepCutoff, + }, + PrecisionMode::TransientMixed, + ) + .expect("valid"); + assert_eq!(request.document_count(), 2); + assert_eq!(request.topic_count(), 3); + assert_eq!(request.bytes_per_observation(), 8); + assert_eq!(request.working_set_bytes(), 16); + assert_eq!(request.requested_batch(), 4); + assert_eq!( + request.corpus_placement(), + CorpusPlacement::StreamedMicroBatches + ); + assert_eq!( + request.observation_retention(), + ObservationRetention::KeepAll + ); + assert_eq!(request.model_complexity(), ModelComplexity::KeepSpecified); + assert_eq!(request.cutoff_policy(), CutoffPolicy::KeepCutoff); + assert_eq!( + request.final_quantity_precision(), + PrecisionMode::TransientMixed + ); + } +} diff --git a/crates/compute_backend/src/telemetry.rs b/crates/compute_backend/src/telemetry.rs new file mode 100644 index 00000000..324df894 --- /dev/null +++ b/crates/compute_backend/src/telemetry.rs @@ -0,0 +1,112 @@ +//! Allocation telemetry that must not carry source text. + +use crate::error::ComputeBackendError; +use crate::plan::FallbackReason; +use crate::request::PrecisionMode; + +/// Resource telemetry for one planning or retry decision. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct AllocationTelemetry { + allocated_bytes: u64, + transfer_bytes: u64, + retry_count: u32, + kernel_launches: u32, + precision: PrecisionMode, + fallback: Option, +} + +impl AllocationTelemetry { + /// Record allocation, transfer, retry, kernel, precision, and fallback. + #[must_use] + pub const fn new( + allocated_bytes: u64, + transfer_bytes: u64, + retry_count: u32, + kernel_launches: u32, + precision: PrecisionMode, + fallback: Option, + ) -> Self { + Self { + allocated_bytes, + transfer_bytes, + retry_count, + kernel_launches, + precision, + fallback, + } + } + + /// Refuse to attach raw source text to telemetry. + /// + /// # Errors + /// + /// Always returns [`ComputeBackendError::SourceTextInTelemetry`]. + #[allow(clippy::unused_self)] + pub fn attach_source_text(&self, _source_text: &str) -> Result<(), ComputeBackendError> { + Err(ComputeBackendError::SourceTextInTelemetry) + } + + /// Return allocated bytes. + #[must_use] + pub const fn allocated_bytes(self) -> u64 { + self.allocated_bytes + } + + /// Return transfer bytes. + #[must_use] + pub const fn transfer_bytes(self) -> u64 { + self.transfer_bytes + } + + /// Return OOM retry count. + #[must_use] + pub const fn retry_count(self) -> u32 { + self.retry_count + } + + /// Return recorded kernel launches. + #[must_use] + pub const fn kernel_launches(self) -> u32 { + self.kernel_launches + } + + /// Return recorded precision. + #[must_use] + pub const fn precision(self) -> PrecisionMode { + self.precision + } + + /// Return recorded fallback reason. + #[must_use] + pub const fn fallback(self) -> Option { + self.fallback + } +} + +#[cfg(test)] +mod tests { + use super::AllocationTelemetry; + use crate::plan::FallbackReason; + use crate::request::PrecisionMode; + + #[test] + fn telemetry_accessors_exclude_source_text() { + let telemetry = AllocationTelemetry::new( + 8, + 4, + 2, + 1, + PrecisionMode::TransientMixed, + Some(FallbackReason::DeviceUnavailable), + ); + assert_eq!(telemetry.allocated_bytes(), 8); + assert_eq!(telemetry.transfer_bytes(), 4); + assert_eq!(telemetry.retry_count(), 2); + assert_eq!(telemetry.kernel_launches(), 1); + assert_eq!(telemetry.precision(), PrecisionMode::TransientMixed); + assert_eq!( + telemetry.fallback(), + Some(FallbackReason::DeviceUnavailable) + ); + } +} diff --git a/crates/compute_backend/tests/crate_contract.rs b/crates/compute_backend/tests/crate_contract.rs new file mode 100644 index 00000000..56d994f6 --- /dev/null +++ b/crates/compute_backend/tests/crate_contract.rs @@ -0,0 +1,7 @@ +//! Integration contract for the `compute_backend` package identity. + +#[test] +fn package_identity_is_stable() { + let observed = std::hint::black_box(env!("CARGO_PKG_NAME")); + assert_eq!(observed, "compute_backend"); +} diff --git a/crates/compute_backend/tests/vram_budget_contract.rs b/crates/compute_backend/tests/vram_budget_contract.rs new file mode 100644 index 00000000..32569fcf --- /dev/null +++ b/crates/compute_backend/tests/vram_budget_contract.rs @@ -0,0 +1,267 @@ +//! VRAM budget, executable OOM retry, and CPU `f64` reference contracts. +#![allow(clippy::cast_precision_loss)] + +use compute_backend::{ + AdaptationPolicy, AllocationTelemetry, ComputeBackendError, ComputeBackendKind, + CorpusPlacement, CutoffPolicy, DeviceInventory, FallbackReason, ModelComplexity, + ObservationRetention, PrecisionMode, VramController, VramProfile, WorkloadRequest, + require_cpu_gpu_parity, streamed_weighted_sum, +}; + +fn rmse(truth: &[f64], recovered: &[f64]) -> f64 { + let n = truth.len() as f64; + let sum_sq: f64 = truth + .iter() + .zip(recovered) + .map(|(left, right)| { + let residual = left - right; + residual * residual + }) + .sum(); + (sum_sq / n).sqrt() +} + +fn base_request(batch: u32, bytes_per_observation: u64) -> WorkloadRequest { + WorkloadRequest::new( + 1_024, + 64, + bytes_per_observation, + 1_048_576, + batch, + AdaptationPolicy { + corpus_placement: CorpusPlacement::StreamedMicroBatches, + observation_retention: ObservationRetention::KeepAll, + model_complexity: ModelComplexity::KeepSpecified, + cutoff_policy: CutoffPolicy::KeepCutoff, + }, + PrecisionMode::ReferenceF64, + ) + .expect("valid workload") +} + +#[test] +fn profiles_cover_the_adr_device_classes() { + let profiles = VramProfile::all(); + assert_eq!(profiles.map(VramProfile::gibibytes), [4, 6, 8, 12, 24]); + assert_eq!(VramProfile::Gib4.bytes(), 4 * (1 << 30)); + assert_eq!(VramProfile::Gib24.bytes(), 24 * (1 << 30)); +} + +#[test] +fn compensated_reference_recovers_cancellation_and_known_total() { + // CPU-reference evidence only; this crate has no GPU execution path yet. + let weights = [0.25_f64, 0.25, 0.25, 0.25]; + let values = [4.0_f64, 8.0, 12.0, 16.0]; + let recovered = streamed_weighted_sum(&weights, &values).expect("finite reference"); + let error = rmse(&[10.0], &[recovered]); + assert!(error < 1e-12, "CPU f64 RMSE {error} exceeded bound"); + + let cancellation = streamed_weighted_sum(&[1.0, 1.0, 1.0], &[1e16, 1.0, -1e16]) + .expect("compensated cancellation"); + assert!((cancellation - 1.0).abs() < 1e-15); +} + +#[test] +fn larger_vram_profiles_admit_larger_micro_batches() { + let request = base_request(1_024, 4_194_304); + let small = VramController::new( + DeviceInventory::gpu(VramProfile::Gib4, VramProfile::Gib4.bytes()).expect("4 GiB"), + 3, + ) + .expect("controller") + .plan(&request) + .expect("4 GiB plan"); + let large = VramController::new( + DeviceInventory::gpu(VramProfile::Gib24, VramProfile::Gib24.bytes()).expect("24 GiB"), + 3, + ) + .expect("controller") + .plan(&request) + .expect("24 GiB plan"); + + assert_eq!(small.backend(), ComputeBackendKind::GpuStreamed); + assert_eq!(large.backend(), ComputeBackendKind::GpuStreamed); + assert!(large.batch_size() > small.batch_size()); + assert_eq!(small.oom_retry_count(), 0); + assert_eq!(large.oom_retry_count(), 0); +} + +#[test] +fn each_oom_returns_a_smaller_gpu_plan_before_cpu_fallback() { + let controller = VramController::new( + DeviceInventory::gpu(VramProfile::Gib6, VramProfile::Gib6.bytes()).expect("6 GiB"), + 2, + ) + .expect("controller"); + let request = base_request(64, 1_048_576); + let initial = controller.plan(&request).expect("initial plan"); + let retry_one = controller + .recover_from_oom(&request, &initial) + .expect("first retry plan"); + assert_eq!(retry_one.backend(), ComputeBackendKind::GpuStreamed); + assert_eq!(retry_one.batch_size(), initial.batch_size() / 2); + assert_eq!(retry_one.oom_retry_count(), 1); + assert!(retry_one.predicted_peak_bytes() < initial.predicted_peak_bytes()); + + let retry_two = controller + .recover_from_oom(&request, &retry_one) + .expect("second retry plan"); + assert_eq!(retry_two.backend(), ComputeBackendKind::GpuStreamed); + assert_eq!(retry_two.batch_size(), retry_one.batch_size() / 2); + assert_eq!(retry_two.oom_retry_count(), 2); + + let fallback = controller + .recover_from_oom(&request, &retry_two) + .expect("bounded fallback"); + assert_eq!(fallback.backend(), ComputeBackendKind::CpuF64Reference); + assert_eq!( + fallback.fallback(), + Some(FallbackReason::OutOfMemoryRetryExhausted) + ); + // CPU f64 fallback is not VRAM-limited, so it restores the requested batch. + assert_eq!(fallback.batch_size(), request.requested_batch()); + assert_eq!(fallback.oom_retry_count(), 3); +} + +#[test] +fn streamed_cardinality_does_not_require_a_hypothetical_full_tensor() { + let request = WorkloadRequest::new( + u64::MAX, + u64::MAX, + 8, + 0, + 1, + AdaptationPolicy { + corpus_placement: CorpusPlacement::StreamedMicroBatches, + observation_retention: ObservationRetention::KeepAll, + model_complexity: ModelComplexity::KeepSpecified, + cutoff_policy: CutoffPolicy::KeepCutoff, + }, + PrecisionMode::ReferenceF64, + ) + .expect("streamed dimensions are independently representable"); + assert_eq!(request.document_count(), u64::MAX); + assert_eq!(request.topic_count(), u64::MAX); + + let controller = VramController::new( + DeviceInventory::gpu(VramProfile::Gib4, VramProfile::Gib4.bytes()).expect("4 GiB"), + 1, + ) + .expect("controller"); + let plan = controller.plan(&request).expect("streamed plan"); + assert_eq!(plan.backend(), ComputeBackendKind::GpuStreamed); + assert_eq!(plan.batch_size(), 1); + assert_eq!(plan.predicted_peak_bytes(), 8); +} + +#[test] +fn parity_rejects_negative_tolerance() { + assert_eq!( + require_cpu_gpu_parity(1.0, 1.0, -0.1), + Err(ComputeBackendError::InvalidTolerance) + ); +} + +fn forbidden_request( + placement: CorpusPlacement, + retention: ObservationRetention, + complexity: ModelComplexity, + cutoff: CutoffPolicy, + precision: PrecisionMode, +) -> WorkloadRequest { + WorkloadRequest::new( + 8, + 4, + 8, + 64, + 2, + AdaptationPolicy { + corpus_placement: placement, + observation_retention: retention, + model_complexity: complexity, + cutoff_policy: cutoff, + }, + precision, + ) + .expect("request") +} + +#[test] +fn forbidden_memory_adaptations_fail_closed() { + let controller = VramController::new( + DeviceInventory::gpu(VramProfile::Gib8, VramProfile::Gib8.bytes()).expect("8 GiB"), + 1, + ) + .expect("controller"); + + for (request, expected) in [ + ( + forbidden_request( + CorpusPlacement::FullCorpusOnDevice, + ObservationRetention::KeepAll, + ModelComplexity::KeepSpecified, + CutoffPolicy::KeepCutoff, + PrecisionMode::ReferenceF64, + ), + ComputeBackendError::FullCorpusTensorRefused, + ), + ( + forbidden_request( + CorpusPlacement::StreamedMicroBatches, + ObservationRetention::DropToFit, + ModelComplexity::KeepSpecified, + CutoffPolicy::KeepCutoff, + PrecisionMode::ReferenceF64, + ), + ComputeBackendError::ObservationDropForbidden, + ), + ( + forbidden_request( + CorpusPlacement::StreamedMicroBatches, + ObservationRetention::KeepAll, + ModelComplexity::ReduceToFit, + CutoffPolicy::KeepCutoff, + PrecisionMode::ReferenceF64, + ), + ComputeBackendError::ComplexityReductionForbidden, + ), + ( + forbidden_request( + CorpusPlacement::StreamedMicroBatches, + ObservationRetention::KeepAll, + ModelComplexity::KeepSpecified, + CutoffPolicy::MoveToFit, + PrecisionMode::ReferenceF64, + ), + ComputeBackendError::CutoffMutationForbidden, + ), + ( + forbidden_request( + CorpusPlacement::StreamedMicroBatches, + ObservationRetention::KeepAll, + ModelComplexity::KeepSpecified, + CutoffPolicy::KeepCutoff, + PrecisionMode::TransientMixed, + ), + ComputeBackendError::UnsupportedPrecision, + ), + ] { + assert_eq!(controller.plan(&request), Err(expected)); + } +} + +#[test] +fn telemetry_refuses_raw_source_text() { + let telemetry = AllocationTelemetry::new( + 1_024, + 256, + 1, + 0, + PrecisionMode::ReferenceF64, + Some(FallbackReason::InsufficientVram), + ); + assert_eq!( + telemetry.attach_source_text("secret document body"), + Err(ComputeBackendError::SourceTextInTelemetry) + ); +} diff --git a/docs/adr/0006-vram-gpu-nvidia-orchestration.md b/docs/adr/0006-vram-gpu-nvidia-orchestration.md index b1b8d1ae..f1d97862 100644 --- a/docs/adr/0006-vram-gpu-nvidia-orchestration.md +++ b/docs/adr/0006-vram-gpu-nvidia-orchestration.md @@ -1,7 +1,7 @@ # ADR 0006 — VRAM-adaptive GPU compute and model-credential boundary **Decision status:** Accepted -**Implementation maturity:** accepted-target +**Implementation maturity:** active-PR — VRAM profiles, safety reserve, peak prediction, micro-batch autotune, typed OOM with bounded CPU `f64` fallback, and forbidden-adaptation refusal are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; live GPU kernels, mixed-precision device lanes, and hardware parity remain accepted-target **Date:** 2026-08-05 **Supersession:** LLM orchestration-selection and test-time-compute policy is superseded by ADR 0010. Autonomous development/review/merge authority separation is governed by ADR 0015. This ADR remains authoritative for GPU/VRAM execution and the model-credential boundary. diff --git a/docs/adr/0011-standalone-modular-msa-boundary.md b/docs/adr/0011-standalone-modular-msa-boundary.md index d545ee23..04181fb3 100644 --- a/docs/adr/0011-standalone-modular-msa-boundary.md +++ b/docs/adr/0011-standalone-modular-msa-boundary.md @@ -1,7 +1,7 @@ # ADR 0011 — Standalone operation and modular CWL MSA boundary **Decision status:** Accepted -**Implementation maturity:** partial — Rust crates are independently usable; naruon HTTP interchange and loopback live listener (`POST /v1/analysis-runs` and `/v1/exports`, fail-closed table-access, NIM/proxy headers, RFC 3339 cutoff, stream deadline) are on the active PR (not implemented-main); production TLS/`$PORT` and remaining persistence integrations remain accepted-target +**Implementation maturity:** partial — Rust crates are independently usable; naruon HTTP interchange and loopback live listener (`POST /v1/analysis-runs` and `/v1/exports`, fail-closed table-access, NIM/proxy headers, RFC 3339 cutoff, stream deadline) are on the active PR (not implemented-main); production TLS/`$PORT` and remaining persistence integrations remain accepted-target **Date:** 2026-08-10 **Supersedes:** The broad cross-service ownership wording in ADR 0001. ADR 0001 remains authoritative for Rust-first numerical architecture. diff --git a/docs/connectors/naruon-artifact-consumer.md b/docs/connectors/naruon-artifact-consumer.md index 2e4f4d6c..5fe0424c 100644 --- a/docs/connectors/naruon-artifact-consumer.md +++ b/docs/connectors/naruon-artifact-consumer.md @@ -1,6 +1,6 @@ # naruon modular consumer contract for TEPP artifacts -**Status:** Partial — versioned DTO, HTTP interchange, and loopback live listener on the active PR; production TLS/`$PORT` remaining +**Status:** Partial — versioned DTO, HTTP interchange, and loopback live listener on the active PR; production TLS/`$PORT` remaining **Last reviewed:** 2026-08-16 ## Boundary diff --git a/docs/research/vram-budget-types.md b/docs/research/vram-budget-types.md new file mode 100644 index 00000000..af4e0cf6 --- /dev/null +++ b/docs/research/vram-budget-types.md @@ -0,0 +1,47 @@ +# VRAM budget types, executable OOM retries, and CPU `f64` reference + +## Scope + +This slice delivers the first executable ADR 0006 contract in `compute_backend`: + +1. classify devices into the accepted 4/6/8/12/24-GiB profiles; +2. reserve one eighth of profile capacity as unused safety memory; +3. predict peak bytes as `batch × bytes_per_observation + working_set`; +4. autotune the micro-batch by successive halving until the predicted peak fits usable VRAM; +5. after each observed OOM, emit a smaller executable GPU plan with an incremented retry count, then fall back to the CPU `f64` reference after the bounded retry budget or a failed unit batch; +6. refuse full-corpus document-by-topic device tensors and refuse dropping observations, shrinking topic/model complexity, or moving a knowledge cutoff to fit memory; +7. keep mixed precision out of final diagnostic quantities and compare CPU/candidate outputs with a normalized parity tolerance; +8. keep raw source text out of allocation telemetry; +9. use compensated deterministic summation for the sequential CPU `f64` numerical reference. + +Live CUDA/WGPU kernels, deterministic fixed-pool CPU multithreading, mixed-precision device lanes, and hardware CPU/GPU parity remain accepted-target. This slice does not claim an accelerator or a multithreaded production estimator. + +## Authoritative sources + +IEEE. (2019). *IEEE standard for floating-point arithmetic* (IEEE Std 754-2019). https://standards.ieee.org/ieee/754/6210/ + +Micikevicius, P., Narang, S., Alben, J., Diamos, G., Elsen, E., Garcia, D., Ginsburg, B., Houston, M., Kuchaiev, O., Venkatesh, G., & Wu, H. (2018). Mixed precision training. In *International Conference on Learning Representations*. https://openreview.net/forum?id=r1gs9JgRZ + +NVIDIA Corporation. (2024). *CUDA C++ programming guide*. https://docs.nvidia.com/cuda/cuda-c-programming-guide/ + +Ogita, T., Rump, S. M., & Oishi, S. (2005). Accurate sum and dot product. *SIAM Journal on Scientific Computing, 26*(6), 1955–1988. https://doi.org/10.1137/030601818 + +Rhu, M., Gimelshein, N., Clemons, J., Zulfiqar, A., & Keckler, S. W. (2016). vDNN: Virtualized deep neural networks for scalable, memory-efficient neural network design. In *2016 49th Annual IEEE/ACM International Symposium on Microarchitecture (MICRO)* (pp. 1–13). IEEE. https://doi.org/10.1109/MICRO.2016.7783721 + +## Formula notes + +- **Profile capacity** is \(p \times 2^{30}\) bytes for \(p \in \{4,6,8,12,24\}\). +- **Safety reserve** is \(p \times 2^{30} / 8\). Usable VRAM is \(\max(0, a - s)\) for available bytes \(a\) and reserve \(s\). +- **Peak** is \(b \cdot c + w\) for batch \(b\), per-observation charge \(c\), and working set \(w\). Overflow fails closed. +- **OOM retry** is stateful: retry count \(r\) increments after each observed OOM, batch is halved when \(r\leq r_{max}\), and the peak is recomputed from the original workload. No loop is counted as a retry unless an executable plan is returned to the caller. +- **CPU `f64` reference** uses deterministic compensated summation in IEEE 754 binary64 so cancellation-heavy low-order terms are not needlessly discarded (IEEE, 2019; Ogita et al., 2005). +- Streamed document/topic cardinalities are not multiplied into a hypothetical full-corpus allocation; the forbidden full-corpus policy is rejected by the controller. +- Mixed precision may be recorded as a transient mode only; final diagnostics remain binary64 (Micikevicius et al., 2018). + +## Verification + +- cancellation-heavy CPU `f64` weighted sums recover the low-order term and known totals with computed RMSE; +- 24-GiB profiles admit a larger autotuned micro-batch than 4-GiB profiles for the same workload; +- each accepted OOM retry returns a smaller GPU plan and an exact retry count before CPU fallback; +- streamed extreme cardinalities remain valid because no full tensor is sized; +- negative parity tolerances, full-corpus placement, observation drop, complexity reduction, cutoff mutation, mixed-final precision, and source-text telemetry fail closed. diff --git a/scripts/check_workspace_contract.py b/scripts/check_workspace_contract.py index c7b1ecf5..114af5bc 100644 --- a/scripts/check_workspace_contract.py +++ b/scripts/check_workspace_contract.py @@ -23,6 +23,7 @@ "tepp_simulation", "validation_core", "tepp_api", + "compute_backend", ) REQUIRED_CI_SNIPPETS: tuple[str, ...] = ( diff --git a/tests/quality/test_check_docstrings.py b/tests/quality/test_check_docstrings.py index 2c11f7a5..56d553d2 100644 --- a/tests/quality/test_check_docstrings.py +++ b/tests/quality/test_check_docstrings.py @@ -11,6 +11,7 @@ from unittest import mock from scripts import check_docstrings as docstrings +from scripts import check_workspace_contract as contract REPOSITORY_ROOT = Path(__file__).resolve().parents[2] @@ -24,7 +25,7 @@ def test_live_repository_is_documented(self) -> None: sources = docstrings.rust_sources(REPOSITORY_ROOT) crate_roots = sorted(REPOSITORY_ROOT.glob("crates/*/src/lib.rs")) - self.assertEqual(len(crate_roots), 10) + self.assertEqual(len(crate_roots), len(contract.EXPECTED_CRATES)) self.assertTrue(set(crate_roots).issubset(sources)) self.assertGreaterEqual(len(sources), len(crate_roots)) self.assertEqual(docstrings.validate_repository(REPOSITORY_ROOT), [])