diff --git a/CHANGELOG.md b/CHANGELOG.md index 24b9e7f..ef48c50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,13 @@ - Add executable deterministic Rust HMM provider baseline and provider-host benchmark. - Add bounded investigator context packing, explicit workspace access, candidate transaction, identity envelope, quarantine, and morning-report contracts. +- Add executable local-harness request/observation normalization with timeout and output + bounds, authority-expansion rejection, and explicit proposal-only semantics. +- Add Git-validated detached proposal worktree materialization, source commit identities, + reproducible metadata, and explicit preserve-or-cleanup behavior. +- Add the reviewed Rust `mnel-provider-loader` unsafe boundary, SHA-256 artifact admission, + v1 descriptor/query/result validation, host-owned output copying, native cdylib fixtures, + and existing-host quarantine integration tests. ABI v1 is unchanged. ## 0.1.0a0 — unreleased diff --git a/Cargo.lock b/Cargo.lock index be79b0f..5396460 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,76 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "mnel-provider-api" version = "0.1.0-alpha.0" @@ -14,6 +84,24 @@ dependencies = [ "mnel-provider-sdk", ] +[[package]] +name = "mnel-provider-fixture" +version = "0.1.0-alpha.0" +dependencies = [ + "mnel-provider-api", +] + +[[package]] +name = "mnel-provider-fixture-invalid" +version = "0.1.0-alpha.0" +dependencies = [ + "mnel-provider-api", +] + +[[package]] +name = "mnel-provider-fixture-no-entry" +version = "0.1.0-alpha.0" + [[package]] name = "mnel-provider-host" version = "0.1.0-alpha.0" @@ -23,9 +111,52 @@ dependencies = [ "mnel-provider-sdk", ] +[[package]] +name = "mnel-provider-loader" +version = "0.1.0-alpha.0" +dependencies = [ + "libloading", + "mnel-provider-api", + "mnel-provider-fixture", + "mnel-provider-fixture-invalid", + "mnel-provider-fixture-no-entry", + "mnel-provider-host", + "mnel-provider-sdk", + "sha2", +] + [[package]] name = "mnel-provider-sdk" version = "0.1.0-alpha.0" dependencies = [ "mnel-provider-api", ] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" diff --git a/Cargo.toml b/Cargo.toml index a49a0d3..23f7d9c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,10 @@ members = [ "crates/mnel-provider-classical", "crates/mnel-provider-sdk", "crates/mnel-provider-host", + "crates/mnel-provider-loader", + "crates/mnel-provider-fixture", + "crates/mnel-provider-fixture-invalid", + "crates/mnel-provider-fixture-no-entry", ] resolver = "2" diff --git a/README.md b/README.md index 7ccaa5e..d5f9972 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,8 @@ conventional neural-weight training. > process-local persistent Rust host, reusable identity-bound snapshots, bounded and > normalized diagnostic results, failure quarantine, an executable Rust HMM baseline, > deterministic runtime measurements, and bounded investigator context/workspace -> contracts. It still does not provide dynamic library loading, process isolation, +> contracts, an executable local-harness/worktree path, and a validated Rust v1 dynamic +> provider loader. It still does not provide process isolation, > unattended model execution, distributed scheduling, protected final custody, formal > MNCS/MNCDS conformance, or automatic RAVEL promotion. @@ -96,6 +97,13 @@ copy their authority or silently create substitute implementations. - executable `mnel-provider-classical` HMM diagnostic provider and host integration tests; - eligible-context packing, read-only/proposal workspace models, identity envelopes, candidate transactions, quarantine queues, and deterministic morning-report records; +- bounded local-harness JSON-line execution with timeout/output ceilings, authority + rejection, deterministic observations, and detached Git proposal worktrees; +- SHA-256 artifact-bound Rust dynamic loading for `mnel_provider_entry_v1`, descriptor and + pointer/length validation, host-owned output copying, clean unload, and quarantine + integration through the existing provider host; +- initial immutable transition, tabular, and pair diagnostic snapshot producers with + compact binary payloads and dependency-bound content identities; - deterministic reference workflow, JSON schemas, mutation-oriented tests, and CI. ## Install @@ -296,8 +304,11 @@ operating-system sandbox. Untrusted experiment execution belongs in a hardened r with network restrictions, resource controls, immutable verifiers, and disposable workspaces. -The provider runtime crates establish contracts and admission policy; they do not yet -implement a hardened dynamic loader or operating-system sandbox. +The provider runtime includes a small native-trusted dynamic loader with explicit ABI, +artifact, pointer/length, output, and diagnostic-authority checks. It is not an +operating-system sandbox: malformed native code can still crash the host process, so +untrusted experiment execution belongs in a hardened runner with network restrictions, +resource controls, immutable verifiers, and disposable workspaces. A local MNEL result or learned-provider observation can describe bounded development context. It cannot by itself establish independent evaluation, protected custody, diff --git a/crates/mnel-provider-fixture-invalid/Cargo.toml b/crates/mnel-provider-fixture-invalid/Cargo.toml new file mode 100644 index 0000000..6bc2b82 --- /dev/null +++ b/crates/mnel-provider-fixture-invalid/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "mnel-provider-fixture-invalid" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Malformed ABI descriptor fixture for MNEL loader integration tests" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +mnel-provider-api = { path = "../mnel-provider-api" } + +[lints] +workspace = true diff --git a/crates/mnel-provider-fixture-invalid/src/lib.rs b/crates/mnel-provider-fixture-invalid/src/lib.rs new file mode 100644 index 0000000..5ff576c --- /dev/null +++ b/crates/mnel-provider-fixture-invalid/src/lib.rs @@ -0,0 +1,28 @@ +//! A deterministic descriptor that must be rejected before invocation. + +#![allow(unsafe_code)] + +use mnel_provider_api::{ByteView, Digest32, ProviderDescriptorV1, ABI_VERSION_V1}; + +static ID: &[u8] = b"fixture.invalid"; +static VERSION: &[u8] = b"0.1.0"; +static mut DESCRIPTOR: ProviderDescriptorV1 = ProviderDescriptorV1 { + abi_version: ABI_VERSION_V1 + 1, + reserved: 0, + provider_id: ByteView { + data: ID.as_ptr(), + len: ID.len(), + }, + provider_version: ByteView { + data: VERSION.as_ptr(), + len: VERSION.len(), + }, + declaration_identity: Digest32 { bytes: [8; 32] }, + implementation_context: core::ptr::null_mut(), + infer: None, +}; + +#[no_mangle] +pub extern "C" fn mnel_provider_entry_v1() -> *const ProviderDescriptorV1 { + &raw const DESCRIPTOR +} diff --git a/crates/mnel-provider-fixture-no-entry/Cargo.toml b/crates/mnel-provider-fixture-no-entry/Cargo.toml new file mode 100644 index 0000000..de3a2e8 --- /dev/null +++ b/crates/mnel-provider-fixture-no-entry/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "mnel-provider-fixture-no-entry" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "No-entry-symbol cdylib fixture for MNEL loader integration tests" + +[lib] +crate-type = ["cdylib"] + +[lints] +workspace = true diff --git a/crates/mnel-provider-fixture-no-entry/src/lib.rs b/crates/mnel-provider-fixture-no-entry/src/lib.rs new file mode 100644 index 0000000..ba727c2 --- /dev/null +++ b/crates/mnel-provider-fixture-no-entry/src/lib.rs @@ -0,0 +1,6 @@ +//! A shared library without the versioned MNEL entry symbol. + +#![allow(unsafe_code)] + +#[no_mangle] +pub extern "C" fn unrelated_fixture_symbol() {} diff --git a/crates/mnel-provider-fixture/Cargo.toml b/crates/mnel-provider-fixture/Cargo.toml new file mode 100644 index 0000000..b723258 --- /dev/null +++ b/crates/mnel-provider-fixture/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "mnel-provider-fixture" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Deterministic native cdylib fixture for MNEL loader integration tests" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +mnel-provider-api = { path = "../mnel-provider-api" } + +[lints] +workspace = true diff --git a/crates/mnel-provider-fixture/src/lib.rs b/crates/mnel-provider-fixture/src/lib.rs new file mode 100644 index 0000000..d9766c9 --- /dev/null +++ b/crates/mnel-provider-fixture/src/lib.rs @@ -0,0 +1,86 @@ +//! Deterministic cdylib fixture. Query identity byte zero selects bounded test behavior: +//! 1 success, 2 oversized length, 3 provider runtime error, and 4 invalid output pointer. + +#![allow(unsafe_code)] + +use core::ptr; + +use mnel_provider_api::{ + ByteView, Digest32, ProviderDescriptorV1, ProviderQueryV1, ProviderResultV1, ABI_VERSION_V1, + AUTHORITY_DIAGNOSTIC_ONLY, OUTPUT_ANOMALY_SCORE, PROVIDER_STATUS_COMPLETED, + PROVIDER_STATUS_RUNTIME_ERROR, VERDICT_SEMANTICS_NOT_A_VERDICT, +}; + +static PROVIDER_ID: &[u8] = b"fixture.dynamic-provider"; +static PROVIDER_VERSION: &[u8] = b"0.1.0"; +static DECLARATION: Digest32 = Digest32 { bytes: [9; 32] }; + +static mut DESCRIPTOR: ProviderDescriptorV1 = ProviderDescriptorV1 { + abi_version: ABI_VERSION_V1, + reserved: 0, + provider_id: ByteView { + data: PROVIDER_ID.as_ptr(), + len: PROVIDER_ID.len(), + }, + provider_version: ByteView { + data: PROVIDER_VERSION.as_ptr(), + len: PROVIDER_VERSION.len(), + }, + declaration_identity: DECLARATION, + implementation_context: ptr::null_mut(), + infer: Some(infer), +}; + +#[no_mangle] +pub extern "C" fn mnel_provider_entry_v1() -> *const ProviderDescriptorV1 { + &raw const DESCRIPTOR +} + +extern "C" fn infer( + _context: *mut core::ffi::c_void, + query: *const ProviderQueryV1, + result: *mut ProviderResultV1, +) -> i32 { + // This fixture is intentionally a trusted native test artifact. The loader tests + // malformed output metadata without asking the fixture to dereference invalid memory. + unsafe { + if query.is_null() || result.is_null() { + return -1; + } + let mode = (*query).query_identity.bytes[0]; + (*result).abi_version = ABI_VERSION_V1; + (*result).authority = AUTHORITY_DIAGNOSTIC_ONLY; + (*result).verdict_semantics = VERDICT_SEMANTICS_NOT_A_VERDICT; + (*result).output_kind = OUTPUT_ANOMALY_SCORE; + (*result).scalar_value = 0.75; + (*result).calibration_band = 1; + (*result).flags = 0; + if mode == 3 { + (*result).status = PROVIDER_STATUS_RUNTIME_ERROR; + return -7; + } + if mode == 4 { + (*result).status = PROVIDER_STATUS_COMPLETED; + (*result).observation_payload.data = 1 as *mut u8; + (*result).observation_payload.len = 1; + return 0; + } + (*result).status = PROVIDER_STATUS_COMPLETED; + if mode == 2 { + (*result).observation_payload.len = (*result).observation_payload.capacity + 1; + return 0; + } + let payload = b"fixture-observation"; + if (*result).observation_payload.capacity < payload.len() { + (*result).status = PROVIDER_STATUS_RUNTIME_ERROR; + return -2; + } + ptr::copy_nonoverlapping( + payload.as_ptr(), + (*result).observation_payload.data, + payload.len(), + ); + (*result).observation_payload.len = payload.len(); + 0 + } +} diff --git a/crates/mnel-provider-loader/Cargo.toml b/crates/mnel-provider-loader/Cargo.toml new file mode 100644 index 0000000..cb7bd69 --- /dev/null +++ b/crates/mnel-provider-loader/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "mnel-provider-loader" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Auditable v1 dynamic-library loader for MNEL diagnostic providers" + +[dependencies] +libloading = "0.8" +mnel-provider-api = { path = "../mnel-provider-api" } +mnel-provider-sdk = { path = "../mnel-provider-sdk" } +sha2 = "0.10" + +[dev-dependencies] +mnel-provider-fixture = { path = "../mnel-provider-fixture" } +mnel-provider-fixture-invalid = { path = "../mnel-provider-fixture-invalid" } +mnel-provider-fixture-no-entry = { path = "../mnel-provider-fixture-no-entry" } +mnel-provider-host = { path = "../mnel-provider-host" } + +[lints] +workspace = true diff --git a/crates/mnel-provider-loader/src/lib.rs b/crates/mnel-provider-loader/src/lib.rs new file mode 100644 index 0000000..b125606 --- /dev/null +++ b/crates/mnel-provider-loader/src/lib.rs @@ -0,0 +1,472 @@ +//! Small reviewed unsafe boundary for the versioned MNEL provider shared-library ABI. +//! +//! The loader is for native-trusted providers. It validates the descriptor and all +//! host-owned result metadata, copies output before returning, serializes calls, and +//! keeps the dynamic library alive for the entire provider handle lifetime. A malformed +//! native provider can still crash its own process while executing arbitrary code; a +//! process sandbox remains a separate future boundary. + +#![deny(unsafe_code)] + +use std::ffi::c_void; +use std::fmt::{Display, Formatter}; +use std::fs::File; +use std::io::{Read, Seek}; +use std::path::Path; +use std::slice; +use std::sync::Mutex; + +use libloading::{Library, Symbol}; +use mnel_provider_api::{ + ByteView, Digest32, ProviderDescriptorV1, ProviderInferV1, ProviderQueryV1, ProviderResultV1, + ProviderStatusV1, ABI_VERSION_V1, AUTHORITY_DIAGNOSTIC_ONLY, OUTPUT_ANOMALY_SCORE, + PROVIDER_STATUS_ABSTAINED, PROVIDER_STATUS_BUDGET_EXCEEDED, PROVIDER_STATUS_COMPLETED, + PROVIDER_STATUS_INVALID_INPUT, PROVIDER_STATUS_OUT_OF_DISTRIBUTION, + PROVIDER_STATUS_RUNTIME_ERROR, RESULT_FLAG_OUT_OF_DISTRIBUTION, + VERDICT_SEMANTICS_NOT_A_VERDICT, +}; +use mnel_provider_sdk::{DiagnosticResult, Invocation, LearnedProvider, ProviderError}; +use sha2::{Digest as Sha2Digest, Sha256}; + +const MAX_IDENTIFIER_BYTES: usize = 256; +const MAX_SNAPSHOTS: usize = 1024; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProviderExpectation { + pub provider_id: String, + pub provider_version: String, + pub declaration_identity: Digest32, + pub artifact_identity: Digest32, +} + +impl ProviderExpectation { + pub fn validate(&self) -> Result<(), LoaderError> { + if self.provider_id.trim().is_empty() || self.provider_version.trim().is_empty() { + return Err(LoaderError::InvalidExpectation( + "missing provider identity".to_owned(), + )); + } + if self.artifact_identity == Digest32::ZERO { + return Err(LoaderError::InvalidExpectation( + "artifact identity must be non-zero".to_owned(), + )); + } + Ok(()) + } +} + +#[derive(Debug)] +pub enum LoaderError { + InvalidExpectation(String), + Library(String), + MissingEntrySymbol, + NullDescriptor, + UnsupportedAbi(u32), + MalformedDescriptor(String), + ProviderIdentityMismatch, + DeclarationIdentityMismatch, + ArtifactIdentityMismatch, + InvalidQuery(String), + InvalidResult(String), + ProviderReturnCode(i32), + ProviderStatus(ProviderStatusV1), + CallLockPoisoned, + Io(String), +} + +impl Display for LoaderError { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidExpectation(reason) => { + write!(formatter, "invalid provider expectation: {reason}") + } + Self::Library(reason) => write!(formatter, "provider library error: {reason}"), + Self::MissingEntrySymbol => write!(formatter, "missing mnel_provider_entry_v1 symbol"), + Self::NullDescriptor => write!(formatter, "provider entry returned a null descriptor"), + Self::UnsupportedAbi(version) => { + write!(formatter, "unsupported provider ABI: {version}") + } + Self::MalformedDescriptor(reason) => { + write!(formatter, "malformed provider descriptor: {reason}") + } + Self::ProviderIdentityMismatch => { + write!(formatter, "provider descriptor identity mismatch") + } + Self::DeclarationIdentityMismatch => { + write!(formatter, "provider declaration identity mismatch") + } + Self::ArtifactIdentityMismatch => { + write!(formatter, "provider artifact identity mismatch") + } + Self::InvalidQuery(reason) => write!(formatter, "invalid provider query: {reason}"), + Self::InvalidResult(reason) => write!(formatter, "invalid provider result: {reason}"), + Self::ProviderReturnCode(code) => { + write!(formatter, "provider returned error code {code}") + } + Self::ProviderStatus(status) => write!(formatter, "invalid provider status: {status}"), + Self::CallLockPoisoned => write!(formatter, "provider call lock is poisoned"), + Self::Io(reason) => write!(formatter, "provider artifact I/O error: {reason}"), + } + } +} + +impl std::error::Error for LoaderError {} + +pub fn artifact_identity(path: &Path) -> Result { + let mut file = File::open(path).map_err(|error| LoaderError::Io(error.to_string()))?; + let length = file + .seek(std::io::SeekFrom::End(0)) + .map_err(|error| LoaderError::Io(error.to_string()))?; + if length == 0 { + return Err(LoaderError::Io("empty provider artifact".to_owned())); + } + file.rewind() + .map_err(|error| LoaderError::Io(error.to_string()))?; + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let count = file + .read(&mut buffer) + .map_err(|error| LoaderError::Io(error.to_string()))?; + if count == 0 { + break; + } + hasher.update(&buffer[..count]); + } + Ok(Digest32 { + bytes: hasher.finalize().into(), + }) +} + +pub struct LoadedProvider { + _library: Library, + provider_id: String, + provider_version: String, + declaration_identity: Digest32, + artifact_identity: Digest32, + implementation_context: *mut c_void, + infer: ProviderInferV1, + call_lock: Mutex<()>, + max_output_bytes: usize, + max_snapshot_bytes: usize, +} + +// The ABI contract forbids borrowed provider memory from escaping an invocation. +// Calls are serialized, so a provider context is never accessed concurrently through +// this adapter. The library field keeps all descriptor/function/context addresses valid. +#[allow(unsafe_code)] +unsafe impl Send for LoadedProvider {} +#[allow(unsafe_code)] +unsafe impl Sync for LoadedProvider {} + +impl LoadedProvider { + #[allow(unsafe_code)] + pub fn load( + path: &Path, + expectation: &ProviderExpectation, + max_output_bytes: usize, + max_snapshot_bytes: usize, + ) -> Result { + expectation.validate()?; + if max_output_bytes == 0 || max_snapshot_bytes == 0 { + return Err(LoaderError::InvalidExpectation( + "provider byte limits must be positive".to_owned(), + )); + } + let canonical = path + .canonicalize() + .map_err(|error| LoaderError::Library(error.to_string()))?; + let actual_artifact = artifact_identity(&canonical)?; + if actual_artifact != expectation.artifact_identity { + return Err(LoaderError::ArtifactIdentityMismatch); + } + // All operations involving the foreign library and raw pointers are confined to + // this constructor and the `infer` method below. + let library = unsafe { Library::new(&canonical) } + .map_err(|error| LoaderError::Library(error.to_string()))?; + let (provider_id, provider_version, declaration_identity, context, infer) = unsafe { + let entry: Symbol<'_, mnel_provider_api::ProviderEntryV1> = library + .get(mnel_provider_api::ENTRY_SYMBOL_V1.as_bytes()) + .map_err(|_| LoaderError::MissingEntrySymbol)?; + let descriptor = entry(); + validate_descriptor(descriptor, expectation)? + }; + Ok(Self { + _library: library, + provider_id, + provider_version, + declaration_identity, + artifact_identity: actual_artifact, + implementation_context: context, + infer, + call_lock: Mutex::new(()), + max_output_bytes, + max_snapshot_bytes, + }) + } + + pub fn provider_id(&self) -> &str { + &self.provider_id + } + + pub fn provider_version(&self) -> &str { + &self.provider_version + } + + pub fn declaration_identity(&self) -> Digest32 { + self.declaration_identity + } + + pub fn artifact_identity(&self) -> Digest32 { + self.artifact_identity + } + + fn invoke_abi(&self, invocation: &Invocation<'_>) -> Result { + let _guard = self + .call_lock + .lock() + .map_err(|_| LoaderError::CallLockPoisoned)?; + let query = invocation.as_raw(); + validate_query(&query, self.max_snapshot_bytes)?; + let mut result = HostResultBuffer::new(self.max_output_bytes); + let return_code = (self.infer)(self.implementation_context, &query, result.as_raw_mut()); + result.finish(return_code) + } +} + +impl LearnedProvider for LoadedProvider { + fn infer(&self, invocation: &Invocation<'_>) -> Result { + self.invoke_abi(invocation).map_err(|error| match error { + LoaderError::ProviderStatus(PROVIDER_STATUS_ABSTAINED) => ProviderError::Abstained, + LoaderError::ProviderStatus(PROVIDER_STATUS_INVALID_INPUT) => { + ProviderError::InvalidBudget + } + LoaderError::ProviderStatus(PROVIDER_STATUS_BUDGET_EXCEEDED) => { + ProviderError::BudgetExceeded + } + LoaderError::ProviderStatus(PROVIDER_STATUS_OUT_OF_DISTRIBUTION) => { + ProviderError::OutOfDistribution + } + LoaderError::ProviderStatus(PROVIDER_STATUS_RUNTIME_ERROR) + | LoaderError::ProviderReturnCode(_) + | LoaderError::ProviderStatus(_) + | LoaderError::InvalidResult(_) + | LoaderError::InvalidQuery(_) + | LoaderError::CallLockPoisoned + | LoaderError::Library(_) + | LoaderError::MissingEntrySymbol + | LoaderError::NullDescriptor + | LoaderError::UnsupportedAbi(_) + | LoaderError::MalformedDescriptor(_) + | LoaderError::ProviderIdentityMismatch + | LoaderError::DeclarationIdentityMismatch + | LoaderError::ArtifactIdentityMismatch + | LoaderError::InvalidExpectation(_) + | LoaderError::Io(_) => ProviderError::RuntimeFailure, + }) + } +} + +struct HostResultBuffer { + storage: Vec, + result: ProviderResultV1, +} + +impl HostResultBuffer { + fn new(max_output_bytes: usize) -> Self { + let mut storage = vec![0_u8; max_output_bytes]; + let result = ProviderResultV1 { + abi_version: ABI_VERSION_V1, + status: PROVIDER_STATUS_RUNTIME_ERROR, + output_kind: OUTPUT_ANOMALY_SCORE, + calibration_band: 0, + scalar_value: 0.0, + flags: 0, + observation_payload: mnel_provider_api::MutableByteBuffer { + data: storage.as_mut_ptr(), + capacity: storage.len(), + len: 0, + }, + authority: AUTHORITY_DIAGNOSTIC_ONLY, + verdict_semantics: VERDICT_SEMANTICS_NOT_A_VERDICT, + }; + Self { storage, result } + } + + fn as_raw_mut(&mut self) -> &mut ProviderResultV1 { + &mut self.result + } + + fn finish(self, return_code: i32) -> Result { + if return_code != 0 { + return Err(LoaderError::ProviderReturnCode(return_code)); + } + if self.result.abi_version != ABI_VERSION_V1 + || self.result.authority != AUTHORITY_DIAGNOSTIC_ONLY + || self.result.verdict_semantics != VERDICT_SEMANTICS_NOT_A_VERDICT + { + return Err(LoaderError::InvalidResult( + "provider changed ABI or diagnostic authority fields".to_owned(), + )); + } + let buffer = self.result.observation_payload; + if !std::ptr::eq(buffer.data.cast_const(), self.storage.as_ptr()) + || buffer.capacity != self.storage.len() + || buffer.len > buffer.capacity + { + return Err(LoaderError::InvalidResult( + "provider returned an invalid host buffer pointer or length".to_owned(), + )); + } + let payload = self.storage[..buffer.len].to_vec(); + if !(1..=7).contains(&self.result.output_kind) { + return Err(LoaderError::InvalidResult( + "provider returned an unknown output kind".to_owned(), + )); + } + let result = match self.result.status { + PROVIDER_STATUS_COMPLETED => DiagnosticResult { + output_kind: self.result.output_kind, + value: self.result.scalar_value, + calibration_band: self.result.calibration_band, + out_of_distribution: self.result.flags & RESULT_FLAG_OUT_OF_DISTRIBUTION != 0, + payload, + }, + status => return Err(LoaderError::ProviderStatus(status)), + }; + result.validate().map_err(|error| match error { + ProviderError::NonFiniteResult => { + LoaderError::InvalidResult("non-finite scalar".to_owned()) + } + _ => LoaderError::InvalidResult("provider result validation failed".to_owned()), + }) + } +} + +#[allow(unsafe_code)] +unsafe fn validate_descriptor( + pointer: *const ProviderDescriptorV1, + expectation: &ProviderExpectation, +) -> Result<(String, String, Digest32, *mut c_void, ProviderInferV1), LoaderError> { + if pointer.is_null() { + return Err(LoaderError::NullDescriptor); + } + let descriptor = &*pointer; + if descriptor.abi_version != ABI_VERSION_V1 { + return Err(LoaderError::UnsupportedAbi(descriptor.abi_version)); + } + if descriptor.reserved != 0 { + return Err(LoaderError::MalformedDescriptor( + "reserved descriptor field is non-zero".to_owned(), + )); + } + let provider_id = view_to_string(descriptor.provider_id, "provider_id")?; + let provider_version = view_to_string(descriptor.provider_version, "provider_version")?; + if provider_id != expectation.provider_id || provider_version != expectation.provider_version { + return Err(LoaderError::ProviderIdentityMismatch); + } + if descriptor.declaration_identity != expectation.declaration_identity { + return Err(LoaderError::DeclarationIdentityMismatch); + } + let infer = descriptor + .infer + .ok_or_else(|| LoaderError::MalformedDescriptor("missing infer function".to_owned()))?; + Ok(( + provider_id, + provider_version, + descriptor.declaration_identity, + descriptor.implementation_context, + infer, + )) +} + +#[allow(unsafe_code)] +unsafe fn view_to_string(view: ByteView, label: &str) -> Result { + if view.len == 0 || view.len > MAX_IDENTIFIER_BYTES || view.data.is_null() { + return Err(LoaderError::MalformedDescriptor(format!( + "{label} has an invalid pointer or length" + ))); + } + let bytes = slice::from_raw_parts(view.data, view.len); + let value = std::str::from_utf8(bytes) + .map_err(|_| LoaderError::MalformedDescriptor(format!("{label} is not UTF-8")))?; + if value.trim().is_empty() { + return Err(LoaderError::MalformedDescriptor(format!( + "{label} is empty" + ))); + } + Ok(value.to_owned()) +} + +#[allow(unsafe_code)] +fn validate_query(query: &ProviderQueryV1, max_snapshot_bytes: usize) -> Result<(), LoaderError> { + if query.abi_version != ABI_VERSION_V1 { + return Err(LoaderError::InvalidQuery( + "unsupported ABI version".to_owned(), + )); + } + if query.reserved != 0 { + return Err(LoaderError::InvalidQuery( + "reserved query field is non-zero".to_owned(), + )); + } + if query.budget.wall_time_ns == 0 + || query.budget.operation_limit == 0 + || query.budget.memory_bytes == 0 + { + return Err(LoaderError::InvalidQuery("zero resource budget".to_owned())); + } + if query.snapshot_count == 0 + || query.snapshot_count > MAX_SNAPSHOTS + || query.snapshots.is_null() + { + return Err(LoaderError::InvalidQuery( + "invalid snapshot pointer or count".to_owned(), + )); + } + let snapshots = unsafe { slice::from_raw_parts(query.snapshots, query.snapshot_count) }; + for snapshot in snapshots { + if snapshot.reserved != 0 { + return Err(LoaderError::InvalidQuery( + "reserved snapshot field is non-zero".to_owned(), + )); + } + if snapshot.payload.len == 0 + || snapshot.payload.len > max_snapshot_bytes + || snapshot.payload.data.is_null() + { + return Err(LoaderError::InvalidQuery( + "invalid snapshot payload view".to_owned(), + )); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[allow(unsafe_code)] + fn null_descriptor_is_rejected_without_dereference() { + let expectation = ProviderExpectation { + provider_id: "fixture".to_owned(), + provider_version: "0.1".to_owned(), + declaration_identity: Digest32 { bytes: [1; 32] }, + artifact_identity: Digest32 { bytes: [2; 32] }, + }; + let result = unsafe { validate_descriptor(std::ptr::null(), &expectation) }; + assert!(matches!(result, Err(LoaderError::NullDescriptor))); + } + + #[test] + fn invalid_result_pointer_is_rejected_before_copy() { + let mut buffer = HostResultBuffer::new(8); + buffer.result.observation_payload.data = 1 as *mut u8; + assert!(matches!( + buffer.finish(0), + Err(LoaderError::InvalidResult(_)) + )); + } +} diff --git a/crates/mnel-provider-loader/tests/dynamic_fixture.rs b/crates/mnel-provider-loader/tests/dynamic_fixture.rs new file mode 100644 index 0000000..5e7ce69 --- /dev/null +++ b/crates/mnel-provider-loader/tests/dynamic_fixture.rs @@ -0,0 +1,191 @@ +use std::path::PathBuf; +use std::sync::Arc; + +use mnel_provider_api::Digest32; +use mnel_provider_host::{ExecutionTier, ImplementationLanguage, ProviderHost, ProviderManifest}; +use mnel_provider_loader::{artifact_identity, LoadedProvider, ProviderExpectation}; +use mnel_provider_sdk::{ + Invocation, InvocationIdentity, LearnedProvider, ResourceBudget, SnapshotRef, +}; + +fn fixture_path(package: &str) -> PathBuf { + let target = std::env::var_os("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target")); + let stem = package.replace('-', "_"); + let filename = if cfg!(target_os = "windows") { + format!("{stem}.dll") + } else if cfg!(target_os = "macos") { + format!("lib{stem}.dylib") + } else { + format!("lib{stem}.so") + }; + let direct = target.join("debug").join(&filename); + if direct.is_file() { + direct + } else { + target.join("debug").join("deps").join(filename) + } +} + +fn expectation(path: &std::path::Path, provider_id: &str, declaration: u8) -> ProviderExpectation { + ProviderExpectation { + provider_id: provider_id.to_owned(), + provider_version: "0.1.0".to_owned(), + declaration_identity: Digest32 { + bytes: [declaration; 32], + }, + artifact_identity: ok(artifact_identity(path)), + } +} + +fn ok(result: Result) -> T { + match result { + Ok(value) => value, + Err(error) => panic!("unexpected fixture test error: {error:?}"), + } +} + +fn invocation(mode: u8) -> Invocation<'static> { + let payload: &'static [u8] = Box::leak(vec![1_u8, 2, 3].into_boxed_slice()); + ok(Invocation::new( + InvocationIdentity { + declaration: Digest32 { bytes: [9; 32] }, + model: Digest32 { bytes: [2; 32] }, + calibration: Digest32 { bytes: [3; 32] }, + query: Digest32 { bytes: [mode; 32] }, + }, + ResourceBudget { + wall_time_ns: 1_000_000, + operation_limit: 100, + memory_bytes: 1_024, + }, + vec![SnapshotRef { + schema_version: 1, + identity: Digest32 { bytes: [4; 32] }, + feature_extractor_identity: Digest32::ZERO, + payload, + }], + )) +} + +#[test] +fn native_fixture_loads_invokes_repeatedly_and_unloads_cleanly() { + let path = fixture_path("mnel-provider-fixture"); + let provider = ok(LoadedProvider::load( + &path, + &expectation(&path, "fixture.dynamic-provider", 9), + 64, + 64, + )); + for _ in 0..2 { + let result = ok(provider.infer(&invocation(1))); + assert_eq!(result.payload, b"fixture-observation"); + } + drop(provider); + assert!(path.is_file()); +} + +#[test] +fn malformed_abi_and_missing_symbol_are_rejected() { + let invalid = fixture_path("mnel-provider-fixture-invalid"); + assert!(matches!( + LoadedProvider::load( + &invalid, + &expectation(&invalid, "fixture.invalid", 8), + 64, + 64, + ), + Err(mnel_provider_loader::LoaderError::UnsupportedAbi(_)) + )); + let no_entry = fixture_path("mnel-provider-fixture-no-entry"); + assert!(matches!( + LoadedProvider::load( + &no_entry, + &ProviderExpectation { + provider_id: "missing".to_owned(), + provider_version: "0.1.0".to_owned(), + declaration_identity: Digest32 { bytes: [1; 32] }, + artifact_identity: ok(artifact_identity(&no_entry)), + }, + 64, + 64, + ), + Err(mnel_provider_loader::LoaderError::MissingEntrySymbol) + )); +} + +#[test] +fn oversized_and_invalid_output_are_bounded_errors() { + let path = fixture_path("mnel-provider-fixture"); + let provider = ok(LoadedProvider::load( + &path, + &expectation(&path, "fixture.dynamic-provider", 9), + 8, + 64, + )); + assert_eq!( + provider.infer(&invocation(2)), + Err(mnel_provider_sdk::ProviderError::RuntimeFailure) + ); + assert_eq!( + provider.infer(&invocation(4)), + Err(mnel_provider_sdk::ProviderError::RuntimeFailure) + ); +} + +#[test] +fn runtime_error_quarantines_through_the_existing_host() { + let path = fixture_path("mnel-provider-fixture"); + let provider = ok(LoadedProvider::load( + &path, + &expectation(&path, "fixture.dynamic-provider", 9), + 64, + 64, + )); + let mut host = ProviderHost::new(1, 64); + ok(host.admit( + ProviderManifest { + provider_id: "fixture.dynamic-provider".to_owned(), + provider_version: "0.1.0".to_owned(), + declaration_identity: Digest32 { bytes: [9; 32] }, + artifact_identity: ok(artifact_identity(&path)), + language: ImplementationLanguage::Rust, + tier: ExecutionTier::NativeTrusted, + abi_version: 1, + persistent_host: true, + language_exception: None, + placement_policy: Default::default(), + placement_capabilities: Default::default(), + }, + Arc::new(provider), + )); + host.register_snapshot(mnel_provider_host::CachedSnapshot { + identity: Digest32 { bytes: [4; 32] }, + feature_extractor_identity: Digest32::ZERO, + payload: Arc::from([1_u8, 2, 3]), + }); + let result = ok(host.invoke( + "fixture.dynamic-provider", + InvocationIdentity { + declaration: Digest32 { bytes: [9; 32] }, + model: Digest32 { bytes: [2; 32] }, + calibration: Digest32 { bytes: [3; 32] }, + query: Digest32 { bytes: [3; 32] }, + }, + &[Digest32 { bytes: [4; 32] }], + ResourceBudget { + wall_time_ns: 1_000_000, + operation_limit: 100, + memory_bytes: 1_024, + }, + )); + assert_eq!( + result.status, + mnel_provider_api::PROVIDER_STATUS_RUNTIME_ERROR + ); + assert!(matches!( + host.state("fixture.dynamic-provider"), + Some(mnel_provider_host::ProviderState::Quarantined { .. }) + )); +} diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md index 3d9dc9c..55db372 100644 --- a/docs/INTEGRATIONS.md +++ b/docs/INTEGRATIONS.md @@ -2,10 +2,23 @@ ## epi13-local-harness -The local harness is the intended model-routing and bounded-tool substrate. MNEL sends a -role contract, eligible record identities, task prompt, allowed tools, and workspace. -The response remains a proposal. The local harness must not receive hidden partitions or -promotion authority. +The local harness is the intended model-routing and bounded-tool substrate. MNEL's +`LocalHarnessAdapter` sends the sibling JSON-line `chat/start` protocol with a role +contract, eligible-context identity and record identities, runtime identity envelope, task +prompt, allowed tools, and a detached proposal workspace. The command is an explicit +argument vector (`shell=False`), and the adapter enforces a timeout and output ceiling. + +Responses must contain the expected protocol/method/request identity and bounded route, +attempt, and model-output fields. Verdict, conformance, promotion, evaluator, hidden +transfer, and future-final fields are rejected. A harness `successful` flag is retained +only as a diagnostic execution observation; MNEL emits no evaluator verdict. Malformed, +failed, or timed-out runs become quarantined or `UNKNOWN` observations and retain their +worktree until explicit cleanup. + +`run_local_investigator` composes context packing, source commit identification, detached +Git worktree materialization, request execution, and append-only observation records. The +authoritative checkout is never used as the proposal mutation workspace. Worktree roots +must be configured outside the source checkout, and cleanup is an explicit operator call. ## MNCS Forge diff --git a/docs/LEARNED_PROVIDER_RUNTIME.md b/docs/LEARNED_PROVIDER_RUNTIME.md index 70840d8..a124203 100644 --- a/docs/LEARNED_PROVIDER_RUNTIME.md +++ b/docs/LEARNED_PROVIDER_RUNTIME.md @@ -137,15 +137,21 @@ An admitted host must: The Rust host now provides a process-local lifecycle for admitted `LearnedProvider` trait objects, reusable snapshot storage, bounded result normalization, timing/copy measurements, -clean unload, and deterministic failure quarantine. Dynamic shared-library loading, OS -sandboxing, and a production accelerator backend remain future work. The C ABI v1 remains -unchanged. +clean unload, and deterministic failure quarantine. `mnel-provider-loader` adds the +native-trusted dynamic-library increment: it hashes the artifact, resolves +`mnel_provider_entry_v1`, validates descriptor identities and ABI version, checks query and +result pointer/length metadata, copies output into host-owned memory, serializes calls, and +unloads the library with the provider handle. Native code is not sandboxed; OS isolation, +and a production accelerator backend remain future work. The C ABI v1 remains unchanged. ## Snapshot transport -Forge or another identified producer should construct an AST, graph, trace, transition, -pair, tabular, or composite snapshot once. Compatible providers consume borrowed views -of that immutable payload. +The initial Python snapshot producers construct bounded transition, pair, or tabular +snapshots once. Each immutable payload is binary-friendly and carries producer, source, +dependency, feature-extractor, schema, and payload identities. Compatible deterministic +probes and learned providers can consume the same payload boundary; changing a material +dependency changes the content identity and prevents silent reuse. Forge or another +identified producer can extend this vocabulary to AST, graph, trace, or composite views. The durable ledger may describe the snapshot with canonical JSON, but the hot path uses compact binary bytes with explicit schema and feature-extractor identities. Any material @@ -168,7 +174,8 @@ artifact and does not establish a general language preference. 1. Freeze and test the v1 manifest and ABI vocabulary. 2. Use the executable Rust HMM reference provider as the classical baseline. -3. Keep dynamic loading and host-owned ABI output enforcement behind a reviewed boundary. +3. Keep dynamic loading and host-owned ABI output enforcement behind the reviewed + `mnel-provider-loader` boundary and its native cdylib fixtures. 4. Add Forge snapshot producers and reuse measurements. 5. Export one Python-trained neural provider and compare it with the baseline. 6. Add WASM quarantine only after native measurements establish the overhead budget. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index ed27e0c..1f5f7f8 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -28,8 +28,15 @@ - **Implemented:** first native Rust HMM classical provider baseline; - **Implemented:** warm/cold timing, copied-byte, output, placement, and snapshot-reuse measurement harness; -- native adapter to `MNEL-local-harness` and Git worktree materialization remain open; -- ABI dynamic-loader validation remains open; ABI v1 itself is unchanged. +- **Implemented:** bounded adapter to the local JSON-line harness protocol, including + timeout/output limits, machine-readable response validation, authority-expansion + rejection, deterministic diagnostic observations, and explicit proposal-only records; +- **Implemented:** Git-validated detached proposal worktrees with source commit identity, + root confinement, reproducible metadata, immutable authoritative checkout behavior, and + explicit preservation or cleanup; +- **Implemented:** ABI v1 dynamic-library loading and validation in a small Rust unsafe + boundary, including artifact hashing, descriptor identity checks, pointer/length checks, + host-owned output copying, clean unload, and quarantine integration. ABI v1 is unchanged. ## 0.3 — Forge experiment lifecycle @@ -39,8 +46,10 @@ - independent-probe comparison; - verifier health and coverage records; - skeptic-driven omitted-question discovery; -- identity-bound graph, trace, transition, tabular, pair, and composite diagnostic - snapshots suitable for both deterministic probes and learned micro-providers; +- **Started:** identity-bound transition, tabular, and pair diagnostic snapshots with + immutable compact binary payloads, producer/source/dependency/extractor identities, and + deterministic content identities suitable for deterministic probes and learned + micro-providers; - compact binary snapshot views shared across compatible providers; - learned observations normalized as diagnostic events without verifier status. diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index 3492eac..c1f8784 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -58,6 +58,11 @@ probe before CUDA placement, keep explicit choices fail-closed, record reserve/c math, and mark sequential offload verified only from completed inference plus observed hooks and parameter residency. +Snapshot reuse is also identity-gated. The current snapshot producers include source, +dependency, extractor, producer, schema, and payload identities in the content identity; +material dependency changes therefore invalidate reuse rather than silently transferring +stale diagnostic context. + ### Apparent independence Multiple local machines run the same operator-controlled stack. This is replication, @@ -65,7 +70,8 @@ not independent evaluation or protected custody. ## Current residual risks -The foundation does not provide dynamic library loading, process isolation, network enforcement, cgroups, -hardware attestation, authenticated Fabric transport, protected custody, or immutable -remote verifier nodes. Those remain roadmap requirements before unattended operation on -untrusted workloads. +The foundation now validates and loads identified native provider libraries through a +small Rust boundary, but it does not provide process isolation for that native code. +Network enforcement, cgroups, hardware attestation, authenticated Fabric transport, +protected custody, and immutable remote verifier nodes remain roadmap requirements before +unattended operation on untrusted workloads. diff --git a/docs/decisions/0001-rust-provider-runtime.md b/docs/decisions/0001-rust-provider-runtime.md index b075f7d..6763d4f 100644 --- a/docs/decisions/0001-rust-provider-runtime.md +++ b/docs/decisions/0001-rust-provider-runtime.md @@ -63,6 +63,10 @@ This decision is enforced by repository artifacts rather than prose alone: - `mnel-provider-host::placement` mirrors the backend-neutral CPU/CUDA/offload policy; physical accelerator adapters remain outside the trusted ABI boundary. - `include/mnel_provider_v1.h` is the language-neutral ABI header. +- `mnel-provider-loader` is the reviewed native-trusted dynamic-library boundary. It + hashes the admitted artifact, validates v1 descriptors and pointer/length metadata, + copies results into host-owned memory, serializes calls, and preserves diagnostic-only + semantics before handing a provider to the existing host. - `ProviderRuntimeManifest` mirrors the admission contract in the Python control plane. - `learned-provider-runtime-manifest.schema.json` makes the durable manifest testable. - CI runs Rust formatting, linting, and tests alongside the Python suite. @@ -73,7 +77,7 @@ bounded output normalization, timing measurements, and quarantine without changi v1 ABI. Sequential CPU offload is an optional external/backend capability, not a reason to replace the Rust host with a Python daemon. -A future loader may not weaken these requirements. It must reject unsupported ABI +A loader may not weaken these requirements. It must reject unsupported ABI versions, missing identities, unbounded queries, process-per-invocation providers, invalid tier/language combinations, or attempts to grant learned output evaluator semantics. diff --git a/schemas/mnel-diagnostic-snapshot.schema.json b/schemas/mnel-diagnostic-snapshot.schema.json new file mode 100644 index 0000000..57040db --- /dev/null +++ b/schemas/mnel-diagnostic-snapshot.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/epi13/Machine-Native-Experimental-Learning/schemas/mnel-diagnostic-snapshot.schema.json", + "title": "MNEL identity-bound diagnostic snapshot metadata", + "type": "object", + "additionalProperties": false, + "required": ["schema", "snapshot_type", "schema_version", "producer_identity", "source_identity", "dependency_identity", "feature_extractor_identity", "payload_identity", "payload_bytes", "snapshot_identity", "authority", "semantics"], + "properties": { + "schema": {"const": "mnel-diagnostic-snapshot/0.3"}, + "snapshot_type": {"enum": ["transition", "pair", "tabular"]}, + "schema_version": {"type": "integer", "minimum": 1}, + "producer_identity": {"type": "string", "minLength": 1}, + "source_identity": {"type": "string", "minLength": 1}, + "dependency_identity": {"type": "string", "minLength": 1}, + "feature_extractor_identity": {"type": "string", "minLength": 1}, + "payload_identity": {"type": "string", "minLength": 1}, + "payload_bytes": {"type": "integer", "minimum": 1, "maximum": 1048576}, + "snapshot_identity": {"type": "string", "minLength": 1}, + "authority": {"const": "diagnostic-only"}, + "semantics": {"const": "not-a-verdict"} + } +} diff --git a/schemas/mnel-investigator-runtime.schema.json b/schemas/mnel-investigator-runtime.schema.json index 370a7bc..cce42e2 100644 --- a/schemas/mnel-investigator-runtime.schema.json +++ b/schemas/mnel-investigator-runtime.schema.json @@ -5,7 +5,11 @@ "oneOf": [ {"$ref": "#/$defs/context"}, {"$ref": "#/$defs/transaction"}, - {"$ref": "#/$defs/report"} + {"$ref": "#/$defs/report"}, + {"$ref": "#/$defs/request"}, + {"$ref": "#/$defs/observation"}, + {"$ref": "#/$defs/materialized"}, + {"$ref": "#/$defs/run"} ], "$defs": { "context": { @@ -49,6 +53,81 @@ "authority": {"const": "proposal-only"}, "report_identity": {"type": "string", "minLength": 1} } + }, + "request": { + "type": "object", + "additionalProperties": false, + "required": ["schema", "task_id", "role", "prompt", "eligible_context_identity", "eligible_record_ids", "allowed_tools", "workspace", "workspace_access", "proposal_only", "timeout_seconds", "authority", "request_identity"], + "properties": { + "schema": {"const": "mnel-local-investigator-request/0.2"}, + "task_id": {"type": "string", "minLength": 1}, + "role": {"type": "string", "minLength": 1}, + "prompt": {"type": "string", "minLength": 1}, + "eligible_context_identity": {"type": "string", "minLength": 1}, + "eligible_record_ids": {"type": "array", "items": {"type": "string"}, "minItems": 1, "uniqueItems": true}, + "allowed_tools": {"type": "array", "items": {"type": "string"}}, + "workspace": {"type": "string", "minLength": 1}, + "workspace_access": {"enum": ["read-only", "proposal"]}, + "proposal_only": {"const": true}, + "timeout_seconds": {"type": "integer", "minimum": 1, "maximum": 3600}, + "runtime_identity": {"type": "object", "additionalProperties": {"type": "string"}}, + "authority": {"const": "proposal-only"}, + "request_identity": {"type": "string", "minLength": 1} + } + }, + "observation": { + "type": "object", + "additionalProperties": false, + "required": ["schema", "request_identity", "status", "returncode", "timed_out", "duration_ns", "stdout_bytes", "stderr_bytes", "route", "model_output", "attempts", "harness_successful", "response_identity", "error", "authority", "semantics", "observation_identity"], + "properties": { + "schema": {"const": "mnel-local-investigator-observation/0.2"}, + "request_identity": {"type": "string", "minLength": 1}, + "status": {"enum": ["completed", "unknown", "quarantined"]}, + "returncode": {"type": ["integer", "null"]}, + "timed_out": {"type": "boolean"}, + "duration_ns": {"type": ["integer", "null"], "minimum": 0}, + "stdout_bytes": {"type": "integer", "minimum": 0}, + "stderr_bytes": {"type": "integer", "minimum": 0}, + "route": {"type": ["object", "null"]}, + "model_output": {"type": "string"}, + "attempts": {"type": "array", "items": {"type": "object"}, "maxItems": 32}, + "harness_successful": {"type": ["boolean", "null"]}, + "response_identity": {"type": ["string", "null"]}, + "error": {"type": ["string", "null"]}, + "authority": {"const": "proposal-only"}, + "semantics": {"const": "diagnostic-only; not-a-verdict"}, + "observation_identity": {"type": "string", "minLength": 1} + } + }, + "materialized": { + "type": "object", + "additionalProperties": false, + "required": ["schema", "source", "path", "transaction", "access", "proposal_only", "materialization_identity"], + "properties": { + "schema": {"const": "mnel-materialized-worktree/0.2"}, + "source": {"type": "object"}, + "path": {"type": "string", "minLength": 1}, + "transaction": {"$ref": "#/$defs/transaction"}, + "access": {"const": "proposal"}, + "proposal_only": {"const": true}, + "materialization_identity": {"type": "string", "minLength": 1} + } + }, + "run": { + "type": "object", + "additionalProperties": false, + "required": ["schema", "source", "context", "worktree", "request", "observation", "authority", "semantics", "run_identity"], + "properties": { + "schema": {"const": "mnel-local-investigator-run/0.2"}, + "source": {"type": "object"}, + "context": {"$ref": "#/$defs/context"}, + "worktree": {"$ref": "#/$defs/materialized"}, + "request": {"$ref": "#/$defs/request"}, + "observation": {"$ref": "#/$defs/observation"}, + "authority": {"const": "proposal-only"}, + "semantics": {"const": "diagnostic-only; not-a-verdict"}, + "run_identity": {"type": "string", "minLength": 1} + } } } } diff --git a/src/mnel/integrations.py b/src/mnel/integrations.py index d0d5714..daad46e 100644 --- a/src/mnel/integrations.py +++ b/src/mnel/integrations.py @@ -4,19 +4,24 @@ import json import subprocess +import time from dataclasses import asdict, dataclass from pathlib import Path from typing import Any, Protocol, Sequence from .core import canonical_digest +from .investigator_harness import RuntimeIdentityEnvelope, WorkspaceAccess @dataclass(frozen=True) class AdapterResult: - returncode: int + returncode: int | None stdout: str stderr: str parsed: dict[str, Any] | None + timed_out: bool = False + error: str | None = None + duration_ns: int | None = None class JSONCommandAdapter: @@ -28,33 +33,97 @@ def __init__( *, cwd: str | Path | None = None, timeout_seconds: int = 300, + max_output_bytes: int = 256 * 1024, ) -> None: if not command or any(not isinstance(part, str) or not part for part in command): raise ValueError("command must contain non-empty strings") + if timeout_seconds < 1 or timeout_seconds > 3600: + raise ValueError("timeout_seconds must be between 1 and 3600") + if max_output_bytes < 1: + raise ValueError("max_output_bytes must be positive") self.command = tuple(command) self.cwd = Path(cwd).resolve() if cwd is not None else None self.timeout_seconds = timeout_seconds + self.max_output_bytes = max_output_bytes - def run(self, request: dict[str, Any]) -> AdapterResult: - completed = subprocess.run( - self.command, - input=json.dumps(request, sort_keys=True), - text=True, - capture_output=True, - cwd=self.cwd, - timeout=self.timeout_seconds, - check=False, - shell=False, - ) + def run( + self, + request: dict[str, Any], + *, + cwd: str | Path | None = None, + timeout_seconds: int | None = None, + ) -> AdapterResult: + selected_cwd = Path(cwd).resolve() if cwd is not None else self.cwd + if selected_cwd is not None and not selected_cwd.is_dir(): + raise ValueError(f"adapter cwd is not a directory: {selected_cwd}") + selected_timeout = timeout_seconds if timeout_seconds is not None else self.timeout_seconds + if selected_timeout < 1 or selected_timeout > 3600: + raise ValueError("timeout_seconds must be between 1 and 3600") + started = time.monotonic_ns() + try: + completed = subprocess.run( + self.command, + input=json.dumps(request, sort_keys=True) + "\n", + text=True, + capture_output=True, + cwd=selected_cwd, + timeout=selected_timeout, + check=False, + shell=False, + ) + except subprocess.TimeoutExpired as exc: + stdout = _decode_process_output(exc.stdout) + stderr = _decode_process_output(exc.stderr) + return AdapterResult( + None, + stdout[: self.max_output_bytes], + stderr[: self.max_output_bytes], + None, + timed_out=True, + error="timeout", + duration_ns=time.monotonic_ns() - started, + ) + duration_ns = time.monotonic_ns() - started + if len(completed.stdout.encode("utf-8")) > self.max_output_bytes or len( + completed.stderr.encode("utf-8") + ) > self.max_output_bytes: + return AdapterResult( + completed.returncode, + completed.stdout[: self.max_output_bytes], + completed.stderr[: self.max_output_bytes], + None, + error="output-size-limit", + duration_ns=duration_ns, + ) parsed = None + error = None if completed.stdout.strip(): try: value = json.loads(completed.stdout) if isinstance(value, dict): parsed = value + else: + error = "response-not-object" except json.JSONDecodeError: - pass - return AdapterResult(completed.returncode, completed.stdout, completed.stderr, parsed) + error = "malformed-json" + else: + error = "empty-response" + return AdapterResult( + completed.returncode, + completed.stdout, + completed.stderr, + parsed, + error=error, + duration_ns=duration_ns, + ) + + +def _decode_process_output(value: str | bytes | None) -> str: + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return value @dataclass(frozen=True) @@ -65,13 +134,257 @@ class LocalHarnessRequest: eligible_record_ids: tuple[str, ...] allowed_tools: tuple[str, ...] workspace: str + eligible_context_identity: str = "" + runtime_identity: RuntimeIdentityEnvelope | None = None + workspace_access: WorkspaceAccess = WorkspaceAccess.PROPOSAL + proposal_only: bool = True + timeout_seconds: int = 300 + + def __post_init__(self) -> None: + if not self.task_id.strip() or not self.role.strip() or not self.prompt.strip(): + raise ValueError("local harness requests require task, role, and prompt") + if not self.eligible_record_ids: + raise ValueError("local harness requests require eligible context records") + if any(not item.strip() for item in self.eligible_record_ids): + raise ValueError("eligible record identities must be non-empty") + if any(not item.strip() for item in self.allowed_tools): + raise ValueError("allowed tool identities must be non-empty") + if not Path(self.workspace).is_absolute(): + raise ValueError("local harness workspace must be absolute") + if not self.eligible_context_identity.strip(): + raise ValueError("eligible context identity is required") + if self.workspace_access not in {WorkspaceAccess.READ_ONLY, WorkspaceAccess.PROPOSAL}: + raise ValueError("workspace access must be read-only or proposal") + if not self.proposal_only: + raise ValueError("MNEL local harness requests are always proposal-only") + if self.timeout_seconds < 1 or self.timeout_seconds > 3600: + raise ValueError("timeout_seconds must be between 1 and 3600") + forbidden = {"hidden-transfer", "future-final", "promotion", "evaluator"} + if forbidden.intersection(self.allowed_tools): + raise ValueError("local harness tools may not expand authority") def to_dict(self) -> dict[str, Any]: - return { - "schema": "mnel-local-investigator-request/0.1", - **asdict(self), + value = { + "schema": "mnel-local-investigator-request/0.2", + "task_id": self.task_id, + "role": self.role, + "prompt": self.prompt, + "eligible_context_identity": self.eligible_context_identity, + "eligible_record_ids": list(self.eligible_record_ids), + "allowed_tools": list(self.allowed_tools), + "workspace": self.workspace, + "workspace_access": self.workspace_access.value, + "proposal_only": self.proposal_only, + "timeout_seconds": self.timeout_seconds, "authority": "proposal-only", } + if self.runtime_identity is not None: + value["runtime_identity"] = self.runtime_identity.to_dict() + value["request_identity"] = canonical_digest(value) + return value + + +@dataclass(frozen=True) +class LocalHarnessObservation: + """A bounded harness observation; it intentionally has no verdict field.""" + + request_identity: str + status: str + returncode: int | None + timed_out: bool + duration_ns: int | None + stdout_bytes: int + stderr_bytes: int + route: dict[str, Any] | None + model_output: str + attempts: tuple[dict[str, Any], ...] + harness_successful: bool | None + response_identity: str | None + error: str | None = None + authority: str = "proposal-only" + semantics: str = "diagnostic-only; not-a-verdict" + + def to_dict(self) -> dict[str, Any]: + value = { + "schema": "mnel-local-investigator-observation/0.2", + "request_identity": self.request_identity, + "status": self.status, + "returncode": self.returncode, + "timed_out": self.timed_out, + "duration_ns": self.duration_ns, + "stdout_bytes": self.stdout_bytes, + "stderr_bytes": self.stderr_bytes, + "route": self.route, + "model_output": self.model_output, + "attempts": list(self.attempts), + "harness_successful": self.harness_successful, + "response_identity": self.response_identity, + "error": self.error, + "authority": self.authority, + "semantics": self.semantics, + } + value["observation_identity"] = canonical_digest(value) + return value + + +class LocalHarnessAdapter: + """Execute a configured local harness bridge without granting it MNEL authority.""" + + FORBIDDEN_KEYS = frozenset( + { + "verdict", + "conformance", + "promotion_authorized", + "promotion", + "evaluator_verdict", + "evaluator_eligible", + "hidden_transfer", + "future_final", + "ravel_promotion", + } + ) + + def __init__( + self, + command: Sequence[str], + *, + timeout_seconds: int = 300, + max_output_bytes: int = 256 * 1024, + ) -> None: + self.command_adapter = JSONCommandAdapter( + command, + timeout_seconds=timeout_seconds, + max_output_bytes=max_output_bytes, + ) + + def execute(self, request: LocalHarnessRequest) -> LocalHarnessObservation: + envelope = { + "protocolVersion": 1, + "requestId": request.task_id, + "method": "chat/start", + "params": { + "messages": [{"role": "user", "content": request.prompt}], + "lane": request.role, + "mnel_request": request.to_dict(), + }, + } + result = self.command_adapter.run( + envelope, + cwd=request.workspace, + timeout_seconds=request.timeout_seconds, + ) + request_identity = request.to_dict()["request_identity"] + base = { + "request_identity": request_identity, + "returncode": result.returncode, + "timed_out": result.timed_out, + "duration_ns": result.duration_ns, + "stdout_bytes": len(result.stdout.encode("utf-8")), + "stderr_bytes": len(result.stderr.encode("utf-8")), + } + if result.timed_out: + return LocalHarnessObservation( + **base, + status="unknown", + route=None, + model_output="", + attempts=(), + harness_successful=None, + response_identity=None, + error="timeout", + ) + if result.returncode != 0 or result.parsed is None: + return LocalHarnessObservation( + **base, + status="quarantined", + route=None, + model_output="", + attempts=(), + harness_successful=None, + response_identity=None, + error=result.error or "command-failed", + ) + try: + response = self._validate_response(result.parsed, request) + except ValueError as exc: + return LocalHarnessObservation( + **base, + status="quarantined", + route=None, + model_output="", + attempts=(), + harness_successful=None, + response_identity=None, + error=str(exc), + ) + return LocalHarnessObservation( + **base, + status="completed", + route=response["route"], + model_output=response["final_content"], + attempts=tuple(response["attempts"]), + harness_successful=response["successful"], + response_identity=canonical_digest(response), + ) + + @classmethod + def _validate_response( + cls, response: dict[str, Any], request: LocalHarnessRequest + ) -> dict[str, Any]: + cls._reject_authority(response) + if response.get("protocolVersion") != 1 or response.get("method") != "chat/start": + raise ValueError("unexpected local harness protocol or method") + if response.get("requestId") != request.task_id: + raise ValueError("local harness response request identity mismatch") + result = response.get("result") + if not isinstance(result, dict): + raise ValueError("local harness result must be an object") + cls._reject_authority(result) + if result.get("ok") is False: + raise ValueError("local harness rejected the request") + route = result.get("route") + final_content = result.get("final_content") + successful = result.get("successful") + attempts = result.get("attempts", []) + if not isinstance(route, dict) or not isinstance(final_content, str): + raise ValueError("local harness result is missing bounded diagnostic content") + if not isinstance(successful, bool) or not isinstance(attempts, list): + raise ValueError("local harness result has invalid execution fields") + normalized_attempts: list[dict[str, Any]] = [] + for attempt in attempts[:32]: + if not isinstance(attempt, dict): + raise ValueError("local harness attempt must be an object") + cls._reject_authority(attempt) + normalized_attempts.append( + { + key: attempt[key] + for key in ("role", "model", "content", "error", "verification") + if key in attempt + } + ) + if len(final_content.encode("utf-8")) > 128 * 1024: + raise ValueError("local harness model output exceeds the bounded limit") + return { + "route": route, + "final_content": final_content, + "successful": successful, + "attempts": normalized_attempts, + } + + @classmethod + def _reject_authority(cls, value: Any) -> None: + if isinstance(value, dict): + for key, child in value.items(): + if key in cls.FORBIDDEN_KEYS: + raise ValueError(f"local harness response contains forbidden authority field: {key}") + if key == "authority" and child != "proposal-only": + raise ValueError("local harness response attempted to change authority") + if key == "visibility" and child in {"transfer-hidden", "future-final"}: + raise ValueError("local harness response attempted hidden visibility") + cls._reject_authority(child) + elif isinstance(value, list): + for child in value: + cls._reject_authority(child) @dataclass(frozen=True) diff --git a/src/mnel/local_runtime.py b/src/mnel/local_runtime.py new file mode 100644 index 0000000..8f53f11 --- /dev/null +++ b/src/mnel/local_runtime.py @@ -0,0 +1,111 @@ +"""Executable bounded local-investigator path composed from existing MNEL contracts.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + +from .core import EvidenceLedger, canonical_digest +from .integrations import LocalHarnessAdapter, LocalHarnessObservation, LocalHarnessRequest +from .investigator_harness import ( + CandidateTransaction, + PackedContext, + RuntimeIdentityEnvelope, + pack_eligible_context, +) +from .worktrees import GitWorktreeMaterializer, MaterializedWorktree, SourceRevision + + +@dataclass(frozen=True, slots=True) +class LocalInvestigatorRun: + source: SourceRevision + context: PackedContext + worktree: MaterializedWorktree + request: LocalHarnessRequest + observation: LocalHarnessObservation + + def to_dict(self) -> dict[str, Any]: + value = { + "schema": "mnel-local-investigator-run/0.2", + "source": self.source.to_dict(), + "context": self.context.to_dict(), + "worktree": self.worktree.to_dict(), + "request": self.request.to_dict(), + "observation": self.observation.to_dict(), + "authority": "proposal-only", + "semantics": "diagnostic-only; not-a-verdict", + } + value["run_identity"] = canonical_digest(value) + return value + + def append_to_ledger(self, ledger: EvidenceLedger) -> tuple[dict[str, Any], ...]: + """Record reproducible run components without creating evaluator state.""" + + return ( + ledger.append("investigator-materialization", self.worktree.to_dict(), actor="investigator"), + ledger.append("investigator-request", self.request.to_dict(), actor="investigator"), + ledger.append("investigator-observation", self.observation.to_dict(), actor="local-harness"), + ) + + +def run_local_investigator( + *, + repository: str | Path, + base_ref: str, + workspace_root: str | Path, + task_id: str, + parent_candidate_id: str, + prompt: str, + role: str, + allowed_tools: tuple[str, ...], + runtime_identity: RuntimeIdentityEnvelope, + records: Iterable[dict[str, Any]], + adapter: LocalHarnessAdapter, + max_records: int = 32, + max_context_bytes: int = 256 * 1024, +) -> LocalInvestigatorRun: + """Materialize, invoke, and normalize one explicit investigator experiment. + + Failed or malformed runs intentionally leave their worktree available for + diagnosis. Call ``GitWorktreeMaterializer.cleanup(..., preserve=False)`` only + after an operator has decided that the material is no longer needed. + """ + + context = pack_eligible_context( + records, + max_records=max_records, + max_bytes=max_context_bytes, + ) + materializer = GitWorktreeMaterializer(workspace_root) + source = materializer.identify(repository, base_ref) + candidate_name = "mnel-" + canonical_digest( + {"task_id": task_id, "parent_candidate_id": parent_candidate_id, "context": context.snapshot_identity} + ).removeprefix("sha256:")[:24] + workspace = Path(workspace_root).resolve() / candidate_name + transaction = CandidateTransaction.create( + parent_candidate_id=parent_candidate_id, + context_snapshot_identity=context.snapshot_identity, + workspace=workspace, + ) + worktree = materializer.materialize(source, transaction, name=candidate_name) + materializer.write_metadata( + worktree, + { + "context_snapshot_identity": context.snapshot_identity, + "runtime_identity": runtime_identity.to_dict(), + "task_id": task_id, + }, + ) + request = LocalHarnessRequest( + task_id=task_id, + role=role, + prompt=prompt, + eligible_context_identity=context.snapshot_identity, + eligible_record_ids=context.record_ids, + allowed_tools=allowed_tools, + workspace=worktree.path, + runtime_identity=runtime_identity, + ) + observation = adapter.execute(request) + return LocalInvestigatorRun(source, context, worktree, request, observation) diff --git a/src/mnel/snapshots.py b/src/mnel/snapshots.py new file mode 100644 index 0000000..bfd4a18 --- /dev/null +++ b/src/mnel/snapshots.py @@ -0,0 +1,177 @@ +"""Compact identity-bound diagnostic snapshot producers. + +Snapshots are immutable transport objects, not verifier results. Their content identity +includes source, dependency, extractor, schema, and payload identities so reuse is +invalidated when any material producer dependency changes. +""" + +from __future__ import annotations + +import hashlib +import math +import struct +from dataclasses import dataclass +from typing import Sequence + +from .core import canonical_digest + + +class SnapshotError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class DiagnosticSnapshot: + snapshot_type: str + schema_version: int + producer_identity: str + source_identity: str + dependency_identity: str + feature_extractor_identity: str + payload: bytes + payload_identity: str + snapshot_identity: str + + @classmethod + def build( + cls, + *, + snapshot_type: str, + schema_version: int, + producer_identity: str, + source_identity: str, + dependency_identity: str, + feature_extractor_identity: str, + payload: bytes, + ) -> "DiagnosticSnapshot": + if not snapshot_type.strip() or schema_version < 1: + raise SnapshotError("snapshot type and positive schema version are required") + identities = ( + producer_identity, + source_identity, + dependency_identity, + feature_extractor_identity, + ) + if any(not identity.strip() for identity in identities): + raise SnapshotError("snapshot producer and dependency identities are required") + if not payload or len(payload) > 1024 * 1024: + raise SnapshotError("snapshot payload must be non-empty and bounded") + payload_identity = "sha256:" + hashlib.sha256(payload).hexdigest() + identity_body = { + "snapshot_type": snapshot_type, + "schema_version": schema_version, + "producer_identity": producer_identity, + "source_identity": source_identity, + "dependency_identity": dependency_identity, + "feature_extractor_identity": feature_extractor_identity, + "payload_identity": payload_identity, + } + return cls( + snapshot_type, + schema_version, + producer_identity, + source_identity, + dependency_identity, + feature_extractor_identity, + bytes(payload), + payload_identity, + canonical_digest(identity_body), + ) + + def to_dict(self) -> dict[str, object]: + return { + "schema": "mnel-diagnostic-snapshot/0.3", + "snapshot_type": self.snapshot_type, + "schema_version": self.schema_version, + "producer_identity": self.producer_identity, + "source_identity": self.source_identity, + "dependency_identity": self.dependency_identity, + "feature_extractor_identity": self.feature_extractor_identity, + "payload_identity": self.payload_identity, + "payload_bytes": len(self.payload), + "snapshot_identity": self.snapshot_identity, + "authority": "diagnostic-only", + "semantics": "not-a-verdict", + } + + +def transition_snapshot( + previous_state: bytes, + next_state: bytes, + *, + producer_identity: str, + source_identity: str, + dependency_identity: str, + feature_extractor_identity: str, + schema_version: int = 1, +) -> DiagnosticSnapshot: + return DiagnosticSnapshot.build( + snapshot_type="transition", + schema_version=schema_version, + producer_identity=producer_identity, + source_identity=source_identity, + dependency_identity=dependency_identity, + feature_extractor_identity=feature_extractor_identity, + payload=_pair_payload(b"MNEL-T1", previous_state, next_state), + ) + + +def pair_snapshot( + left: bytes, + right: bytes, + *, + producer_identity: str, + source_identity: str, + dependency_identity: str, + feature_extractor_identity: str, + schema_version: int = 1, +) -> DiagnosticSnapshot: + return DiagnosticSnapshot.build( + snapshot_type="pair", + schema_version=schema_version, + producer_identity=producer_identity, + source_identity=source_identity, + dependency_identity=dependency_identity, + feature_extractor_identity=feature_extractor_identity, + payload=_pair_payload(b"MNEL-P1", left, right), + ) + + +def tabular_snapshot( + rows: Sequence[Sequence[float]], + *, + producer_identity: str, + source_identity: str, + dependency_identity: str, + feature_extractor_identity: str, + schema_version: int = 1, +) -> DiagnosticSnapshot: + if not rows or not rows[0]: + raise SnapshotError("tabular snapshot requires non-empty rows and columns") + column_count = len(rows[0]) + if len(rows) > 65535 or column_count > 65535 or any(len(row) != column_count for row in rows): + raise SnapshotError("tabular shape is invalid or exceeds the bounded format") + values: list[float] = [] + for row in rows: + for value in row: + if not math.isfinite(value): + raise SnapshotError("tabular values must be finite") + values.append(float(value)) + payload = struct.pack(">4sHH", b"MNET", len(rows), column_count) + struct.pack( + f">{len(values)}d", *values + ) + return DiagnosticSnapshot.build( + snapshot_type="tabular", + schema_version=schema_version, + producer_identity=producer_identity, + source_identity=source_identity, + dependency_identity=dependency_identity, + feature_extractor_identity=feature_extractor_identity, + payload=payload, + ) + + +def _pair_payload(header: bytes, left: bytes, right: bytes) -> bytes: + if not left or not right or len(left) > 65535 or len(right) > 65535: + raise SnapshotError("pair members must be non-empty and bounded") + return header + struct.pack(">HH", len(left), len(right)) + left + right diff --git a/src/mnel/worktrees.py b/src/mnel/worktrees.py new file mode 100644 index 0000000..c1a9381 --- /dev/null +++ b/src/mnel/worktrees.py @@ -0,0 +1,207 @@ +"""Bounded Git worktree materialization for proposal-only investigator runs.""" + +from __future__ import annotations + +import json +import re +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Sequence + +from .core import canonical_digest +from .investigator_harness import CandidateTransaction, InvestigatorWorkspace + + +class WorktreeError(ValueError): + """A source identity, path, or Git operation was not safe to use.""" + + +@dataclass(frozen=True, slots=True) +class SourceRevision: + repository: str + requested_ref: str + commit: str + source_identity: str + + def to_dict(self) -> dict[str, str]: + return { + "repository": self.repository, + "requested_ref": self.requested_ref, + "commit": self.commit, + "source_identity": self.source_identity, + } + + +@dataclass(frozen=True, slots=True) +class MaterializedWorktree: + source: SourceRevision + path: str + transaction: CandidateTransaction + + def workspace(self) -> InvestigatorWorkspace: + return InvestigatorWorkspace.proposal(self.path) + + def to_dict(self) -> dict[str, object]: + value = { + "schema": "mnel-materialized-worktree/0.2", + "source": self.source.to_dict(), + "path": self.path, + "transaction": self.transaction.to_dict(), + "access": "proposal", + "proposal_only": True, + } + value["materialization_identity"] = canonical_digest(value) + return value + + +class GitWorktreeMaterializer: + """Materialize detached proposal worktrees under one configured root. + + The implementation never interpolates a shell command and never removes a + worktree implicitly. Git's worktree metadata is the only intentional + mutation outside the configured materialization root. + """ + + _SAFE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$") + + def __init__( + self, + workspace_root: str | Path, + *, + git_executable: Sequence[str] = ("git",), + timeout_seconds: int = 30, + ) -> None: + if not git_executable or any(not part for part in git_executable): + raise ValueError("git_executable must contain non-empty command parts") + if timeout_seconds < 1 or timeout_seconds > 300: + raise ValueError("timeout_seconds must be between 1 and 300") + self.workspace_root = Path(workspace_root).resolve() + self.git_executable = tuple(git_executable) + self.timeout_seconds = timeout_seconds + + def identify(self, repository: str | Path, ref: str) -> SourceRevision: + repository_root = self._repository_root(repository) + self._validate_ref(ref) + commit = self._git( + repository_root, + "rev-parse", + "--verify", + "--quiet", + "--end-of-options", + f"{ref}^{{commit}}", + ).strip() + if not re.fullmatch(r"[0-9a-fA-F]{40,64}", commit): + raise WorktreeError("Git did not return a concrete commit identity") + source_identity = canonical_digest( + {"repository": str(repository_root), "commit": commit.lower()} + ) + return SourceRevision(str(repository_root), ref, commit.lower(), source_identity) + + def materialize( + self, + source: SourceRevision, + transaction: CandidateTransaction, + *, + name: str | None = None, + ) -> MaterializedWorktree: + source_root = Path(source.repository).resolve() + if not source_root.is_dir() or not self._is_git_root(source_root): + raise WorktreeError("source repository is no longer a valid Git checkout") + self._validate_root(source_root) + if transaction.access.value != "proposal" or not transaction.proposal_only: + raise WorktreeError("materialized investigator worktrees must be proposal-only") + selected_name = name or f"mnel-{transaction.transaction_identity.removeprefix('sha256:')[:24]}" + if not self._SAFE_NAME.fullmatch(selected_name): + raise WorktreeError("unsafe worktree name") + self.workspace_root.mkdir(parents=True, exist_ok=True) + path = (self.workspace_root / selected_name).resolve() + if not self._within(path, self.workspace_root) or path == self.workspace_root: + raise WorktreeError("worktree path escapes the configured root") + if path.exists(): + raise WorktreeError("refusing to overwrite an existing experiment worktree") + self._git( + source_root, + "worktree", + "add", + "--detach", + str(path), + source.commit, + ) + return MaterializedWorktree(source, str(path), transaction) + + def write_metadata(self, worktree: MaterializedWorktree, extra: dict[str, object] | None = None) -> Path: + path = Path(worktree.path).resolve() + self._assert_materialized_path(path) + value: dict[str, object] = {"worktree": worktree.to_dict()} + if extra: + value["context"] = dict(extra) + metadata = path / ".mnel-worktree.json" + metadata.write_text(json.dumps(value, sort_keys=True, indent=2) + "\n", encoding="utf-8") + return metadata + + def cleanup(self, worktree: MaterializedWorktree, *, preserve: bool = True) -> bool: + """Remove only when explicitly requested with ``preserve=False``.""" + + path = Path(worktree.path).resolve() + self._assert_materialized_path(path) + if preserve: + return False + if not path.exists(): + return False + self._git(Path(worktree.source.repository), "worktree", "remove", "--force", str(path)) + return True + + def _repository_root(self, repository: str | Path) -> Path: + candidate = Path(repository).expanduser().resolve() + if not candidate.is_dir(): + raise WorktreeError("repository is not a directory") + output = self._git(candidate, "rev-parse", "--show-toplevel").strip() + root = Path(output).resolve() + if not root.is_dir(): + raise WorktreeError("Git returned an invalid repository root") + return root + + def _is_git_root(self, repository: Path) -> bool: + try: + return Path(self._git(repository, "rev-parse", "--show-toplevel").strip()).resolve() == repository + except WorktreeError: + return False + + def _validate_root(self, source_root: Path) -> None: + if self.workspace_root == source_root or self._within(self.workspace_root, source_root): + raise WorktreeError("materialization root must not be inside the authoritative checkout") + + def _assert_materialized_path(self, path: Path) -> None: + if not self._within(path, self.workspace_root) or path == self.workspace_root: + raise WorktreeError("worktree path escapes the configured root") + + @staticmethod + def _within(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + return True + except ValueError: + return False + + @staticmethod + def _validate_ref(ref: str) -> None: + if not ref or not ref.strip() or ref.startswith("-") or "\x00" in ref: + raise WorktreeError("invalid Git ref") + + def _git(self, repository: Path, *arguments: str) -> str: + try: + completed = subprocess.run( + (*self.git_executable, "-C", str(repository), *arguments), + capture_output=True, + text=True, + timeout=self.timeout_seconds, + check=False, + shell=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise WorktreeError(f"Git command failed: {exc}") from exc + if completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() or "unknown Git error" + raise WorktreeError(f"Git command failed: {detail}") + return completed.stdout diff --git a/tests/test_local_runtime.py b/tests/test_local_runtime.py new file mode 100644 index 0000000..eda3c33 --- /dev/null +++ b/tests/test_local_runtime.py @@ -0,0 +1,179 @@ +import subprocess +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + +from mnel.core import EvidenceLedger +from mnel.integrations import LocalHarnessAdapter, LocalHarnessRequest +from mnel.investigator_harness import RuntimeIdentityEnvelope +from mnel.local_runtime import run_local_investigator +from mnel.worktrees import GitWorktreeMaterializer, WorktreeError + + +class LocalRuntimeTests(unittest.TestCase): + def _fixture(self, directory: Path) -> Path: + path = directory / "fake_harness.py" + path.write_text( + textwrap.dedent( + """ + import json + import sys + import time + + request = json.loads(sys.stdin.read()) + mode = sys.argv[1] + if mode == "timeout": + time.sleep(2) + if mode == "malformed": + print("not-json") + raise SystemExit(0) + result = { + "route": {"primary_role": "investigator", "escalation_roles": [], "reasons": []}, + "final_content": "bounded proposal", + "successful": True, + "attempts": [{"role": "investigator", "model": "fixture", "content": "bounded proposal"}], + } + if mode == "authority": + result["verdict"] = "PASS" + print(json.dumps({ + "protocolVersion": 1, + "requestId": request["requestId"], + "method": "chat/start", + "result": result, + })) + """ + ), + encoding="utf-8", + ) + return path + + @staticmethod + def _request(workspace: Path) -> LocalHarnessRequest: + return LocalHarnessRequest( + task_id="task-1", + role="investigator", + prompt="propose a bounded diagnostic question", + eligible_context_identity="sha256:context", + eligible_record_ids=("record-1",), + allowed_tools=("read",), + workspace=str(workspace.resolve()), + runtime_identity=RuntimeIdentityEnvelope("model", "q", "runtime", "prompt", "tools"), + timeout_seconds=1, + ) + + def test_adapter_accepts_bounded_diagnostic_response(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + command = (sys.executable, str(self._fixture(root)), "success") + observation = LocalHarnessAdapter(command).execute(self._request(root)) + self.assertEqual(observation.status, "completed") + self.assertEqual(observation.model_output, "bounded proposal") + self.assertIsNone(observation.to_dict().get("verdict")) + self.assertEqual(observation.authority, "proposal-only") + + def test_adapter_fails_closed_for_malformed_authority_and_timeout(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for mode, expected in (("malformed", "quarantined"), ("authority", "quarantined"), ("timeout", "unknown")): + observation = LocalHarnessAdapter( + (sys.executable, str(self._fixture(root)), mode), timeout_seconds=1 + ).execute(self._request(root)) + self.assertEqual(observation.status, expected) + self.assertEqual(observation.model_output, "") + + def test_request_rejects_authority_expanding_tools(self) -> None: + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(ValueError): + LocalHarnessRequest( + task_id="task-1", + role="investigator", + prompt="bounded", + eligible_context_identity="sha256:context", + eligible_record_ids=("record-1",), + allowed_tools=("promotion",), + workspace=str(Path(directory).resolve()), + ) + + def test_git_materialization_validates_refs_paths_and_explicit_cleanup(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + repository = root / "source" + workspace_root = root / "experiments" + repository.mkdir() + self._git(repository, "init", "-q") + self._git(repository, "config", "user.email", "fixture@example.invalid") + self._git(repository, "config", "user.name", "MNEL fixture") + (repository / "source.txt").write_text("source\n", encoding="utf-8") + self._git(repository, "add", "source.txt") + self._git(repository, "commit", "-qm", "initial") + materializer = GitWorktreeMaterializer(workspace_root) + source = materializer.identify(repository, "HEAD") + transaction_path = workspace_root / "candidate" + from mnel.investigator_harness import CandidateTransaction + + transaction = CandidateTransaction.create( + parent_candidate_id="candidate-1", + context_snapshot_identity="sha256:context", + workspace=transaction_path, + ) + with self.assertRaises(WorktreeError): + materializer.identify(repository, "--bad-ref") + with self.assertRaises(WorktreeError): + materializer.materialize(source, transaction, name="../escape") + worktree = materializer.materialize(source, transaction, name="candidate") + self.assertEqual((Path(worktree.path) / "source.txt").read_text(encoding="utf-8"), "source\n") + self.assertFalse(materializer.cleanup(worktree, preserve=True)) + self.assertTrue(Path(worktree.path).exists()) + self.assertTrue(materializer.cleanup(worktree, preserve=False)) + self.assertFalse(Path(worktree.path).exists()) + + def test_end_to_end_run_keeps_source_immutable_and_records_observation(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + repository = root / "source" + repository.mkdir() + self._git(repository, "init", "-q") + self._git(repository, "config", "user.email", "fixture@example.invalid") + self._git(repository, "config", "user.name", "MNEL fixture") + (repository / "source.txt").write_text("authoritative\n", encoding="utf-8") + self._git(repository, "add", "source.txt") + self._git(repository, "commit", "-qm", "initial") + fixture = self._fixture(root) + run = run_local_investigator( + repository=repository, + base_ref="HEAD", + workspace_root=root / "experiments", + task_id="task-e2e", + parent_candidate_id="candidate-1", + prompt="propose one bounded probe", + role="investigator", + allowed_tools=("read",), + runtime_identity=RuntimeIdentityEnvelope("model", "q", "runtime", "prompt", "tools"), + records=({"record_id": "record-1", "value": "visible"},), + adapter=LocalHarnessAdapter((sys.executable, str(fixture), "success")), + ) + self.assertEqual(run.observation.status, "completed") + self.assertEqual((repository / "source.txt").read_text(encoding="utf-8"), "authoritative\n") + ledger = EvidenceLedger(root / "evidence.jsonl") + run.append_to_ledger(ledger) + self.assertTrue(ledger.verify().valid) + self.assertEqual(ledger.verify().record_count, 3) + self.assertTrue((Path(run.worktree.path) / ".mnel-worktree.json").exists()) + GitWorktreeMaterializer(root / "experiments").cleanup(run.worktree, preserve=False) + + @staticmethod + def _git(repository: Path, *args: str) -> None: + completed = subprocess.run( + ("git", "-C", str(repository), *args), + check=False, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + raise AssertionError(completed.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_snapshots.py b/tests/test_snapshots.py new file mode 100644 index 0000000..e0505f1 --- /dev/null +++ b/tests/test_snapshots.py @@ -0,0 +1,42 @@ +import unittest + +from mnel.snapshots import SnapshotError, pair_snapshot, tabular_snapshot, transition_snapshot + + +class SnapshotTests(unittest.TestCase): + def _kwargs(self) -> dict[str, str]: + return { + "producer_identity": "producer:v1", + "source_identity": "source:v1", + "dependency_identity": "dependency:v1", + "feature_extractor_identity": "extractor:v1", + } + + def test_transition_and_pair_snapshots_are_immutable_and_diagnostic_only(self) -> None: + transition = transition_snapshot(b"a", b"b", **self._kwargs()) + pair = pair_snapshot(b"left", b"right", **self._kwargs()) + self.assertEqual(transition.snapshot_type, "transition") + self.assertNotEqual(transition.snapshot_identity, pair.snapshot_identity) + self.assertEqual(transition.to_dict()["authority"], "diagnostic-only") + self.assertEqual(transition.to_dict()["semantics"], "not-a-verdict") + with self.assertRaises(AttributeError): + transition.payload = b"changed" # type: ignore[misc] + + def test_material_dependency_changes_invalidate_identity(self) -> None: + first = transition_snapshot(b"a", b"b", **self._kwargs()) + changed = transition_snapshot( + b"a", b"b", **{**self._kwargs(), "dependency_identity": "dependency:v2"} + ) + self.assertNotEqual(first.snapshot_identity, changed.snapshot_identity) + self.assertNotEqual(first.payload_identity, "") + + def test_tabular_payload_is_binary_bounded_and_rejects_nonfinite_values(self) -> None: + snapshot = tabular_snapshot(((1.0, 2.0), (3.0, 4.0)), **self._kwargs()) + self.assertEqual(snapshot.payload[:4], b"MNET") + self.assertGreater(len(snapshot.payload), 4) + with self.assertRaises(SnapshotError): + tabular_snapshot(((float("nan"),),), **self._kwargs()) + + +if __name__ == "__main__": + unittest.main()