diff --git a/docs/reference/mmcg.md b/docs/reference/mmcg.md index 8fd961e5..1631d3c1 100644 --- a/docs/reference/mmcg.md +++ b/docs/reference/mmcg.md @@ -1419,9 +1419,10 @@ Run `mmcg watch` in a separate terminal so the index stays current while you wor Structural MCP queries also refresh the managed `.mastermind/mmcg.db` on demand when source files or the extractor contract drift. The repository root is derived from the canonical database path and checked against the stored index -identity before any refresh. Automatic refresh admits at most 20,000 source +identity before any refresh. Automatic refresh admits at most 100,000 source candidates and 512 MiB of declared source bytes. Exceeding either cap returns -`refresh_limit_exceeded` without a partial refresh. A custom external `--index` +`refresh_limit_exceeded` without a partial refresh. Status freshness scans have +a 30-second deadline. A custom external `--index` is opened read-only by `serve`: it remains query-compatible when fresh, requires an explicit `mmcg index` when stale, and is never created, migrated, truncated, or given a WAL by the server. Reading an existing active WAL may create or diff --git a/evals/benchmark/rubrics/callees-definition-boundaries-01.json b/evals/benchmark/rubrics/callees-definition-boundaries-01.json index e44a5a99..7af78092 100644 --- a/evals/benchmark/rubrics/callees-definition-boundaries-01.json +++ b/evals/benchmark/rubrics/callees-definition-boundaries-01.json @@ -1,6 +1,6 @@ { "task_id": "callees-definition-boundaries-01", - "source_revision": "cbc8179320d815c07609f859810b3e64ebc74d48", + "source_revision": "f220777fd1fbd0ad03ec9a4f8ed2f52b2464e890", "status": "calibration_key_source_reviewed", "required_knowns": [ { diff --git a/evals/benchmark/rubrics/document-evidence-boundaries-01.json b/evals/benchmark/rubrics/document-evidence-boundaries-01.json index c1bc5faa..c5147c61 100644 --- a/evals/benchmark/rubrics/document-evidence-boundaries-01.json +++ b/evals/benchmark/rubrics/document-evidence-boundaries-01.json @@ -1,6 +1,6 @@ { "task_id": "document-evidence-boundaries-01", - "source_revision": "1bc7c3c51e9e9a06799062fbd0dee83899ff3299", + "source_revision": "ad17b1aad3dcf13bb97c8731f93efc4761b01fd8", "status": "calibration_key_source_reviewed", "required_knowns": [ { diff --git a/evals/benchmark/rubrics/reference-removal-evidence-01.json b/evals/benchmark/rubrics/reference-removal-evidence-01.json index dbf393cb..c8d5d966 100644 --- a/evals/benchmark/rubrics/reference-removal-evidence-01.json +++ b/evals/benchmark/rubrics/reference-removal-evidence-01.json @@ -1,6 +1,6 @@ { "task_id": "reference-removal-evidence-01", - "source_revision": "cbc8179320d815c07609f859810b3e64ebc74d48", + "source_revision": "f220777fd1fbd0ad03ec9a4f8ed2f52b2464e890", "status": "calibration_key_source_reviewed", "required_knowns": [ { diff --git a/evals/benchmark/rubrics/task-phase-continuity-01.json b/evals/benchmark/rubrics/task-phase-continuity-01.json index 13429acd..c7ff7095 100644 --- a/evals/benchmark/rubrics/task-phase-continuity-01.json +++ b/evals/benchmark/rubrics/task-phase-continuity-01.json @@ -1,7 +1,7 @@ { "task_id": "task-phase-continuity-01", "status": "calibration_key_source_reviewed", - "source_revision": "1bc7c3c51e9e9a06799062fbd0dee83899ff3299", + "source_revision": "ad17b1aad3dcf13bb97c8731f93efc4761b01fd8", "required_knowns": [ { "claim": "Pre-flight verifies the parsed spec, then persists approved state, its spec hash and the captured HEAD baseline.", diff --git a/evals/benchmark/tasks/callees-definition-boundaries-01.json b/evals/benchmark/tasks/callees-definition-boundaries-01.json index d61f7d37..897c8fcf 100644 --- a/evals/benchmark/tasks/callees-definition-boundaries-01.json +++ b/evals/benchmark/tasks/callees-definition-boundaries-01.json @@ -1,7 +1,7 @@ { "id": "callees-definition-boundaries-01", "kind": "research", - "revision": "cbc8179320d815c07609f859810b3e64ebc74d48", + "revision": "f220777fd1fbd0ad03ec9a4f8ed2f52b2464e890", "source_allowlist": [ "mcp/servers/mmcg/src/mcp.rs", "mcp/servers/mmcg/src/queries.rs", diff --git a/evals/benchmark/tasks/document-evidence-boundaries-01.json b/evals/benchmark/tasks/document-evidence-boundaries-01.json index dde8aba1..eae50880 100644 --- a/evals/benchmark/tasks/document-evidence-boundaries-01.json +++ b/evals/benchmark/tasks/document-evidence-boundaries-01.json @@ -1,7 +1,7 @@ { "id": "document-evidence-boundaries-01", "kind": "research", - "revision": "1bc7c3c51e9e9a06799062fbd0dee83899ff3299", + "revision": "ad17b1aad3dcf13bb97c8731f93efc4761b01fd8", "source_allowlist": [ "skills/workflow/mastermind-project-history/scripts/document_graph.py", "skills/workflow/mastermind-project-history/SKILL.md", diff --git a/evals/benchmark/tasks/reference-removal-evidence-01.json b/evals/benchmark/tasks/reference-removal-evidence-01.json index b821cbaa..a15f7958 100644 --- a/evals/benchmark/tasks/reference-removal-evidence-01.json +++ b/evals/benchmark/tasks/reference-removal-evidence-01.json @@ -1,7 +1,7 @@ { "id": "reference-removal-evidence-01", "kind": "research", - "revision": "cbc8179320d815c07609f859810b3e64ebc74d48", + "revision": "f220777fd1fbd0ad03ec9a4f8ed2f52b2464e890", "source_allowlist": [ "mcp/servers/mmcg/src/indexer/rust_lang.rs", "mcp/servers/mmcg/src/store.rs", diff --git a/evals/benchmark/tasks/task-phase-continuity-01.json b/evals/benchmark/tasks/task-phase-continuity-01.json index 69deda73..006817a6 100644 --- a/evals/benchmark/tasks/task-phase-continuity-01.json +++ b/evals/benchmark/tasks/task-phase-continuity-01.json @@ -1,7 +1,7 @@ { "id": "task-phase-continuity-01", "kind": "research", - "revision": "1bc7c3c51e9e9a06799062fbd0dee83899ff3299", + "revision": "ad17b1aad3dcf13bb97c8731f93efc4761b01fd8", "source_allowlist": [ "mcp/servers/mmcg/src/run_task.rs", "mcp/servers/mmcg/src/verify_spec.rs", diff --git a/mcp/servers/mmcg/src/indexer.rs b/mcp/servers/mmcg/src/indexer.rs index 97b085e7..2d90214c 100644 --- a/mcp/servers/mmcg/src/indexer.rs +++ b/mcp/servers/mmcg/src/indexer.rs @@ -96,7 +96,7 @@ pub(crate) const PROJECT_DECISION_DIRS: [&str; 6] = [ "adrs", ".mastermind/decisions", ]; -pub const AUTO_REFRESH_SOURCE_CANDIDATE_LIMIT: usize = 20_000; +pub const AUTO_REFRESH_SOURCE_CANDIDATE_LIMIT: usize = 100_000; pub const AUTO_REFRESH_SOURCE_AGGREGATE_BYTES: u64 = 512 * 1024 * 1024; /// Parsed files waiting for the single SQLite writer at once. This bounds peak @@ -244,10 +244,12 @@ fn assigned_secret_like(line: &str) -> bool { let left = if separator == '=' { let trimmed = left.trim_end(); let start = trimmed - .rfind(|character: char| { + .char_indices() + .rev() + .find(|(_, character)| { character.is_whitespace() || matches!(character, ',' | ';' | '(' | '{') }) - .map_or(0, |position| position + 1); + .map_or(0, |(position, character)| position + character.len_utf8()); &trimmed[start..] } else { left @@ -1613,6 +1615,19 @@ fn git_relative_path(raw: &[u8]) -> Result { .map_err(index_error_from_read) } +fn unportable_non_source_git_path(raw: &[u8]) -> bool { + let Ok(text) = std::str::from_utf8(raw) else { + return false; + }; + let path = Path::new(text); + text.contains('\\') + && !path.is_absolute() + && path + .components() + .all(|component| matches!(component, std::path::Component::Normal(_))) + && extractor_for_path(path).is_none() +} + /// Tracked files remain source-of-truth even when a later or overly broad /// ignore rule matches them. Git itself applies ignore rules to untracked /// discovery, not to entries already present in the index. Failure to query Git @@ -1652,6 +1667,12 @@ fn tracked_relative_paths_controlled( if raw.is_empty() { continue; } + // Unix repositories can track literal backslashes in filenames. Such + // paths cannot enter the portable index, but an unsupported file type + // should not prevent unrelated source files from being indexed. + if unportable_non_source_git_path(raw) { + continue; + } let path = git_relative_path(raw).map_err(|_| crate::diff::WorkingTreeDiffError::IndexStale)?; if !has_skipped_component(&path) { @@ -1721,6 +1742,9 @@ pub(crate) fn source_candidates_bounded( if raw.is_empty() { continue; } + if unportable_non_source_git_path(raw) { + continue; + } let relative = git_relative_path(raw)?; if has_skipped_component(&relative) { continue; @@ -1997,6 +2021,24 @@ pub(crate) fn source_admission_with_capability( Ok(prefix) } +pub(crate) fn source_is_binary_after_admission( + root: &RootCapability, + path: &Path, + expected_identity: StableFileIdentity, + control: ReadControl<'_>, +) -> Result { + let source = read_regular_file_expected( + root, + path, + MAX_INDEXABLE_FILE_SIZE, + MAX_INDEXABLE_FILE_SIZE, + control, + Some(expected_identity), + ) + .map_err(index_error_from_read)?; + Ok(is_binary_content(&source.bytes)) +} + fn read_source_bounded( path: &Path, root: &Path, @@ -2475,6 +2517,9 @@ def password_material(): def api_key_material(): """apiassignmentcanary stripe_api_key = liveexamplecredential""" +def unicode_space_material(): + """nbspcanary description{NBSP}stripe_api_key = liveunicodecredential""" + def prose(): """rotationquartz explains token buckets and password rotation""" @@ -2483,7 +2528,8 @@ def bearer_prose(): def placeholder(): """placeholderquartz token = placeholder""" -"#, +"# + .replace("{NBSP}", "\u{00a0}"), ) .unwrap(); let mut store = Store::open(&db).unwrap(); @@ -2496,9 +2542,11 @@ def placeholder(): "jsonbearercanary", "passwordcanary", "apiassignmentcanary", + "nbspcanary", "livecredential123", "livecredential456", "liveexamplecredential", + "liveunicodecredential", "jsoncredential789", ] { assert!( @@ -2517,13 +2565,13 @@ def placeholder(): ); let stats = store.concept_documentation_stats().unwrap(); assert_eq!(stats.indexed_documents, 3); - assert_eq!(stats.secret_omitted, 6); + assert_eq!(stats.secret_omitted, 7); assert_eq!( store .meta_value(crate::store::CONCEPT_DOCUMENTATION_SECRET_OMITTED_META_KEY) .unwrap() .as_deref(), - Some("6") + Some("7") ); fs::remove_dir_all(&dir).ok(); } @@ -3603,6 +3651,32 @@ def candidate(value: ImportantType) -> ResultType"# fs::remove_dir_all(&dir).ok(); } + #[cfg(unix)] + #[test] + fn tracked_inventory_skips_unportable_non_source_files() { + let (dir, db) = setup("tracked_backslash_non_source"); + fs::write(dir.join("good.rs"), "pub fn good() {}\n").unwrap(); + fs::write(dir.join("allure-results\\executor.json"), "{}\n").unwrap(); + git(&dir, &["init", "-q", "--initial-branch=main"]); + git(&dir, &["add", "-A"]); + + assert_eq!( + tracked_relative_paths(&dir).unwrap(), + [PathBuf::from("good.rs")] + ); + let root = RootCapability::open(&dir).unwrap(); + assert_eq!( + source_candidates_bounded(&root, 10, ReadControl::default()).unwrap(), + vec![root.canonical_root().join("good.rs")] + ); + let mut store = Store::open(&db).unwrap(); + let stats = Indexer::new(&dir).index_all(&mut store, false).unwrap(); + assert_eq!(stats.files_indexed, 1); + assert_eq!(stats.files_failed, 0); + assert_eq!(store.indexed_paths().unwrap(), vec!["good.rs"]); + fs::remove_dir_all(&dir).ok(); + } + #[cfg(all(unix, not(target_os = "macos")))] #[test] fn tracked_inventory_rejects_non_utf8_source_paths() { diff --git a/mcp/servers/mmcg/src/lens.rs b/mcp/servers/mmcg/src/lens.rs index 6e4cf8e6..5cc228ed 100644 --- a/mcp/servers/mmcg/src/lens.rs +++ b/mcp/servers/mmcg/src/lens.rs @@ -1063,7 +1063,24 @@ fn validated_index_paths( interrupted: None, }, ) { - Ok(_) => return Err(LensError::IndexStale), + Ok(admitted) => { + let binary = crate::indexer::source_is_binary_after_admission( + &root_capability, + &root.join(&relative), + admitted.identity, + crate::bounded_fs::ReadControl { + deadline, + interrupted: None, + }, + ) + .map_err(|error| match error { + crate::indexer::IndexError::DeadlineExceeded => LensError::AnalysisTimeout, + _ => LensError::IndexStale, + })?; + if !binary { + return Err(LensError::IndexStale); + } + } Err(crate::indexer::IndexError::Skipped(_)) => {} Err(crate::indexer::IndexError::Missing) => { let path = root.join(&relative); diff --git a/mcp/servers/mmcg/src/queries.rs b/mcp/servers/mmcg/src/queries.rs index 13544456..773a406d 100644 --- a/mcp/servers/mmcg/src/queries.rs +++ b/mcp/servers/mmcg/src/queries.rs @@ -5090,7 +5090,7 @@ fn stale_count(store: &Store, index_root: &std::path::Path) -> (usize, bool, Opt let Ok(root) = index_root.canonicalize() else { return (1, false, Some("index_root_unavailable")); }; - let hard_deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let hard_deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); let deadline = store .request_deadline() .map_or(hard_deadline, |deadline| deadline.min(hard_deadline)); diff --git a/mcp/servers/mmcg/src/workflow_status.rs b/mcp/servers/mmcg/src/workflow_status.rs index ba2f1ff9..384e6426 100644 --- a/mcp/servers/mmcg/src/workflow_status.rs +++ b/mcp/servers/mmcg/src/workflow_status.rs @@ -28,7 +28,7 @@ const MAX_WORKFLOW_DIAGNOSTICS: usize = 4_096; const MAX_WORKFLOW_CONTEXT_ESTIMATES: usize = 16_384; const MAX_STATUS_TASKS: usize = 4_096; const MAX_TASK_STATE_BYTES: u64 = 1024 * 1024; -const STATUS_FRESHNESS_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +const STATUS_FRESHNESS_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); const STATUS_STALE_FILE_LIMIT: usize = 10; const STATUS_STALE_FILE_PROBE_LIMIT: usize = STATUS_STALE_FILE_LIMIT + 1; @@ -4815,6 +4815,19 @@ pub(crate) fn stale_paths_controlled( cap: source_bytes_cap, }); } + // Parsing rejects NUL bytes anywhere in a source file. Admission only + // sniffs the prefix, so verify the whole file for paths the index omitted. + if !indexed.contains_key(&relative) + && crate::indexer::source_is_binary_after_admission( + &root_capability, + &path, + admitted.identity, + control, + )? + { + seen.insert(relative); + continue; + } seen.insert(relative.clone()); if indexed .get(&relative) @@ -7462,6 +7475,44 @@ mod tests { fs::remove_dir_all(root).ok(); } + #[test] + fn stale_paths_ignore_binary_source_with_nul_after_admission_prefix() { + let root = tempfile::tempdir().unwrap(); + init_git_repository(root.path()); + let source = root.path().join("late_nul.rs"); + let mut bytes = vec![b'a'; 9_000]; + bytes.push(0); + fs::write(&source, bytes).unwrap(); + let output = Command::new("git") + .args(["add", "late_nul.rs"]) + .current_dir(root.path()) + .output() + .unwrap(); + assert!(output.status.success()); + let db = root.path().join("mmcg.db"); + let mut store = crate::store::Store::open(&db).unwrap(); + let stats = crate::indexer::Indexer::new(root.path()) + .index_all(&mut store, false) + .unwrap(); + assert_eq!(stats.files_skipped_binary, 1); + assert!(stale_paths_controlled( + &store, + root.path(), + 10, + crate::indexer::AUTO_REFRESH_SOURCE_CANDIDATE_LIMIT, + crate::indexer::AUTO_REFRESH_SOURCE_AGGREGATE_BYTES, + crate::bounded_fs::ReadControl::default(), + ) + .unwrap() + .is_empty()); + assert!(crate::lens::validate_index_snapshot( + &store, + &root.path().canonicalize().unwrap(), + None, + ) + .is_ok()); + } + #[test] fn stale_paths_treat_a_stably_deleted_tracked_source_as_current_after_purge() { let root = tempfile::tempdir().unwrap();