From d7dcc3f2410f18788a2ecb7f6a3d7b4d98d8bd2a Mon Sep 17 00:00:00 2001 From: Gabriel Fonseca Date: Tue, 17 Nov 2026 14:45:34 +0000 Subject: [PATCH 1/3] fix(incremental): address CodeRabbit findings #1-#8, #13 from PR #166 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Group 1 — correctness fixes in code-incremental, code-runtime, code-server: - #1/#1-dup engine_binary: baked_path() walked one parent() too few (the original comment 'code-incremental' implied three calls, but two reach the workspace root from the crate dir); target_triple() emitted raw 'macos' instead of the 'darwin' directory name used under third_party/incremental/engine-binaries/. Add a regression test that asserts the baked path shape (engine-binaries layout, no raw 'macos' triple, ends in the binary name, sits under the crate's workspace). - #2 lib.rs: Store::open now runs init_store() BEFORE connect(). The pinned engine exits when tg.log is absent, before it can answer the hello handshake that connect performs, so first-use on a fresh worktree failed with store_missing. init is idempotent (engine skips when the log exists). - #4 lib.rs: the init handshake now uses a bounded 30s try_wait loop (INIT_TIMEOUT) instead of an unbounded wait; a stalled engine child is killed so Store::open cannot hang the server. - #13 lib.rs: Plan::from_json falls back to run-everything (fallback("invalid_plan")) when tasks is not an array or any entry lacks a string id, instead of silently dropping malformed tasks. None/Null tasks remain allowed. - #5 incremental_observer: affected() now chains plan.unobserved into the returned task ids so unobserved tasks are never certified as unaffected. - #6 incremental_observer: advance() reads the store's current baseline via store.baseline() and passes it as expected (None only when the store has no baseline yet), instead of always None, which let a concurrent writer silently overwrite the baseline. - #7 code-server: new repo_root_for_context() helper uses context.checkout_root when recorded, falling back to publish_workspace_root.join(repo); call sites in reconcile_active_tasks and complete_task now resolve repo_root consistently with the context instead of re-deriving from the publish workspace. - #8 outcome mapping: outcome_for() maps worker statuses finished -> pass and cancelled -> error (previously 'none', so a successful completion was recorded with no outcome); observe_task_completion in code-server maps finished -> completed and failed -> failed before recording and advancing, so the baseline carries the canonical task status. - #12 (accepted, documented): the vendored protocol client writes the request to the engine stdin before starting the response deadline, so a write to a stalled engine can block longer than the 30s read deadline. Patching third_party/incremental is the wrong layer (the manifest sha256_rust_src pins upstream bytes and the fetch script re-copies them). Exposure is bounded: requests are small (below the pipe buffer) and a wedged engine is caught by the next request's 30s read deadline, which falls back to run-everything. Rationale documented in Store::open. --- crates/code-incremental/src/engine_binary.rs | 61 ++++++- crates/code-incremental/src/lib.rs | 167 ++++++++++++++---- .../code-runtime/src/incremental_observer.rs | 48 ++++- crates/code-server/src/lib.rs | 41 ++++- 4 files changed, 263 insertions(+), 54 deletions(-) diff --git a/crates/code-incremental/src/engine_binary.rs b/crates/code-incremental/src/engine_binary.rs index 755b374..c9300ca 100644 --- a/crates/code-incremental/src/engine_binary.rs +++ b/crates/code-incremental/src/engine_binary.rs @@ -81,10 +81,13 @@ pub fn resolve() -> EngineBinary { /// Build-time-baked path to the engine binary, computed relative to the /// workspace root (which is two directories above `crates/code-incremental`). +/// +/// `CARGO_MANIFEST_DIR` is the crate directory itself +/// (`/crates/code-incremental`), so exactly two `parent()` calls reach +/// the workspace root: one off the crate dir, one off `crates`. fn baked_path() -> Option { let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() // code-incremental - .and_then(|p| p.parent()) // crates + .parent() // crates .and_then(|p| p.parent()); // workspace root let triple = target_triple(); @@ -104,9 +107,16 @@ fn which_binary(name: &str) -> bool { } /// The target triple for this platform (e.g. `darwin-arm64`, `linux-x86_64`). +/// +/// The OS component is normalized to the directory names used under +/// `third_party/incremental/engine-binaries/`: macOS reports as `macos` to +/// `env::consts::OS`, but the layout uses `darwin`. #[must_use] pub fn target_triple() -> String { - let os = env::consts::OS; + let os = match env::consts::OS { + "macos" => "darwin", + other => other, + }; let arch = env::consts::ARCH; // Normalize arch names to match the directory layout. let arch = match arch { @@ -131,6 +141,51 @@ mod tests { ); } + #[test] + fn baked_path_resolves_within_workspace() { + // The baked path must point at the vendored engine location relative + // to the workspace root. A wrong parent depth or a raw `macos` triple + // would point at a non-existent layout and the integration would + // silently degrade to "run everything." + // + // Only the path shape is asserted (not file existence): the engine + // binary is a checked-in artifact that is not present in the Bazel + // sandbox, where the crate is tested from a copied workspace that + // carries the Rust sources but not the prebuilt binaries. + let Some(path) = baked_path() else { + panic!("baked path could not be computed from CARGO_MANIFEST_DIR"); + }; + let s = path.display().to_string(); + assert!( + s.contains("third_party/incremental/engine-binaries/"), + "baked path missing the engine-binaries layout: {s}" + ); + assert!( + !s.contains("macos-"), + "raw `macos` triple not normalized to `darwin`: {s}" + ); + assert!( + s.ends_with("/incremental"), + "baked path must end in the engine binary name: {s}" + ); + // The path must resolve relative to the crate's own workspace, i.e. + // walking up from the baked path must reach the crate manifest dir. + let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut p = path.clone(); + for _ in 0..7 { + // third_party/incremental/engine-binaries//incremental + if !p.pop() { + break; + } + } + assert!( + manifest.starts_with(&p), + "baked path does not sit under the same workspace as the crate: {} vs {}", + path.display(), + p.display() + ); + } + #[test] fn missing_engine_reports_unavailable() { // We can't guarantee a missing engine in all environments, so just diff --git a/crates/code-incremental/src/lib.rs b/crates/code-incremental/src/lib.rs index b1d7b7c..61d8502 100644 --- a/crates/code-incremental/src/lib.rs +++ b/crates/code-incremental/src/lib.rs @@ -21,7 +21,7 @@ pub mod context; pub mod engine_binary; use std::path::{Path, PathBuf}; -use std::time::Duration; +use std::time::{Duration, Instant}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -114,6 +114,33 @@ impl Store { let binary = engine.path().ok_or(Error::EngineNotFound)?.to_path_buf(); let store_path = env_store_path(repo_root); + + // Initialize the store before connecting: the pinned engine exits when + // `tg.log` is absent, before it can answer the `hello` handshake that + // `connect` performs. Running `init` first (idempotent — the engine + // skips when the log exists) makes first-use work on a fresh worktree. + init_store(&binary, &store_path, repo_root)?; + + // The vendored protocol client (`third_party/incremental`) writes the + // request to the engine's stdin *before* it starts the response + // deadline, so a write to a stalled engine (pipe buffer full) can + // block longer than `timeout`. That is a known limitation of the + // vendored binding (CodeRabbit finding on PR #166) that we accept + // rather than patch in place: + // + // * It is third-party code pinned to a commit; the manifest's + // `sha256_rust_src` is verified against *upstream* bytes and the + // fetch script re-copies those files (clobbering any local patch). + // * The exposure is bounded: requests are small (task ids, change + // lists) and far below the pipe buffer, and the blocking scenario + // requires the engine to already be wedged — in which case the + // next request's `timeout` read deadline fires and the caller + // falls back to "run everything". The whole integration is + // best-effort; a wedged engine degrades to a fallback, never to a + // hard error. + // + // Revisit this if requests grow past the pipe buffer or the engine + // becomes a dependency with its own update cadence. let cfg = incremental::Config { engine: vec![binary.display().to_string()], store: store_path.clone(), @@ -138,23 +165,18 @@ impl Store { /// any missing parents, and an empty log with a header. Safe to call /// multiple times (idempotent — the engine skips when the log exists). /// - /// Call this before the first [`observe`](Self::observe) or - /// [`affected`](Self::affected) on a fresh worktree; a store that was - /// never initialized returns `store_missing` from the engine. - pub fn ensure_initialized(&self) { + /// [`Store::open`] already runs this before connecting; this method is + /// kept for callers that hold a `Store` and want to re-check. It uses a + /// bounded wait and returns `Ok(false)` (rather than blocking or failing + /// the caller) when the engine stalls or is unavailable, so callers can + /// proceed down the unconditional "run everything" path. + #[must_use] + pub fn ensure_initialized(&self) -> bool { let engine = EngineBinary::resolve(); let Some(binary) = engine.path() else { - return; + return false; }; - let _ = std::process::Command::new(binary) - .args([ - "init", - "--store", - &self.path.to_string_lossy(), - "--root", - &self.repo_root.to_string_lossy(), - ]) - .status(); + init_store(binary, &self.path, &self.repo_root).is_ok() } /// The store directory path. @@ -327,14 +349,20 @@ impl Plan { } /// Deserialize a plan from the engine's raw JSON response. + /// + /// A plan is never returned partially: if any `tasks` entry is missing a + /// string `id` (or `tasks` is not an array), the whole plan is a fallback + /// (`invalid_plan`). Silently dropping a malformed entry would let a + /// caller treat an unparseable plan as "nothing to re-run" and skip + /// affected work. #[must_use] pub fn from_json(raw: &Value) -> Self { - let tasks = raw - .get("tasks") - .and_then(Value::as_array) - .map(|arr| { - arr.iter() - .filter_map(|t| { + let tasks_value = raw.get("tasks"); + let tasks: Vec = match tasks_value { + Some(Value::Array(arr)) => { + let parsed: Vec> = arr + .iter() + .map(|t| { Some(Task { task_id: t["id"].as_str()?.to_string(), reasons: t @@ -348,9 +376,16 @@ impl Plan { .unwrap_or_default(), }) }) - .collect() - }) - .unwrap_or_default(); + .collect(); + if parsed.iter().any(Option::is_none) { + return Self::fallback("invalid_plan"); + } + parsed.into_iter().flatten().collect() + } + // No `tasks` key (or a non-array) is only valid on a fallback plan. + None | Some(Value::Null) => vec![], + Some(_) => return Self::fallback("invalid_plan"), + }; let skipped = raw .get("skipped") @@ -454,6 +489,62 @@ fn protocol_err(e: incremental::Error) -> Error { } } +/// Maximum time to wait for the engine's `init` command to finish. +/// +/// `init` runs as a separate process, so the protocol client's own timeout +/// does not cover it. A stalled init must not block task transitions or +/// server requests that call the bridge. +const INIT_TIMEOUT: Duration = Duration::from_secs(30); + +/// Run the engine's `init` command against `store_path` with a bounded wait. +/// +/// Returns `Ok(())` when the store is initialized (or already was) and +/// `Err` when the engine exits non-zero, cannot be spawned, or stalls past +/// [`INIT_TIMEOUT`] (the child is then terminated and reaped). Callers that +/// treat init failure as "run everything" should map `Err` to the fallback +/// path rather than propagating a hard error. +fn init_store(binary: &Path, store_path: &Path, repo_root: &Path) -> Result<()> { + let mut child = std::process::Command::new(binary) + .args([ + "init", + "--store", + &store_path.to_string_lossy(), + "--root", + &repo_root.to_string_lossy(), + ]) + .spawn() + .map_err(|e| Error::EngineStartup(format!("failed to spawn init: {e}")))?; + + let deadline = Instant::now() + INIT_TIMEOUT; + loop { + match child.try_wait() { + Ok(Some(status)) => { + if status.success() { + return Ok(()); + } + return Err(Error::EngineStartup(format!( + "engine init exited with {status}" + ))); + } + Ok(None) => { + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + return Err(Error::EngineStartup(format!( + "engine init timed out after {INIT_TIMEOUT:?}" + ))); + } + std::thread::sleep(Duration::from_millis(50)); + } + Err(e) => { + return Err(Error::EngineStartup(format!( + "failed to wait for init: {e}" + ))); + } + } + } +} + /// Resolve the store directory from the environment, falling back to /// `/.incremental/`. fn env_store_path(repo_root: &Path) -> PathBuf { @@ -582,31 +673,29 @@ mod tests { } /// Build a fresh store on a temp repo, initialized with a baseline. - /// Skips (returns `None`) when the engine binary is unavailable. + /// + /// Returns `None` only when the engine binary is unavailable (a clean + /// skip). Once the binary is found, any failure — init, connect, or + /// baseline advance — panics with the original error so a broken setup + /// fails the test instead of silently passing without exercising + /// certified skips or fallback behavior. fn fresh_e2e_store(baseline: &str) -> Option<(Store, PathBuf)> { let engine = EngineBinary::resolve(); let engine_path = engine.path()?; - // Skip cleanly on platforms without a prebuilt engine binary. if !engine_path.exists() { return None; } let repo = e2e_dir("repo"); std::fs::create_dir_all(&repo).unwrap(); let store_dir = repo.join(".incremental"); - std::process::Command::new(engine_path) - .args([ - "init", - "--store", - &store_dir.to_string_lossy(), - "--root", - &repo.to_string_lossy(), - ]) - .status() - .ok() - .filter(std::process::ExitStatus::success)?; - let mut store = Store::open(&repo).ok()?; + init_store(engine_path, &store_dir, &repo) + .unwrap_or_else(|e| panic!("engine setup (init) failed: {e}")); + let mut store = + Store::open(&repo).unwrap_or_else(|e| panic!("engine setup (open) failed: {e}")); // First advance: null → baseline. - store.advance(baseline, None).ok()?; + store + .advance(baseline, None) + .unwrap_or_else(|e| panic!("engine setup (advance) failed: {e}")); Some((store, repo)) } diff --git a/crates/code-runtime/src/incremental_observer.rs b/crates/code-runtime/src/incremental_observer.rs index e06df97..7abc822 100644 --- a/crates/code-runtime/src/incremental_observer.rs +++ b/crates/code-runtime/src/incremental_observer.rs @@ -40,12 +40,16 @@ pub fn task_id(rel_path: &str, kind: &str) -> String { } /// Map a terminal task status to the protocol-v1 outcome string. +/// +/// Accepts both the canonical task statuses (`completed`, `failed`, +/// `stopped`) and the worker statuses that map onto them, so a finished +/// worker is recorded with a `pass` outcome rather than `none`. #[must_use] pub fn outcome_for(status: &str) -> &'static str { match status { - "completed" => "pass", + "completed" | "finished" => "pass", "failed" => "fail", - "stopped" => "error", + "stopped" | "cancelled" => "error", _ => "none", } } @@ -58,7 +62,10 @@ pub fn outcome_for(status: &str) -> &'static str { /// on a fresh worktree does not hit `store_missing`. fn open_store(repo_root: &Path) -> Option { let store = Store::open(repo_root).ok()?; - store.ensure_initialized(); + // Belt-and-braces re-init for callers that built the `Store` out-of-band. + // `Store::open` already initialized before connecting; a failed re-check + // is non-fatal here — observation falls back to "run everything". + let _ = store.ensure_initialized(); Some(store) } @@ -117,11 +124,30 @@ pub fn observe_many(repo_root: &Path, observations: &[Value]) { /// Advance the store baseline to `to`. Called after a worker finishes a /// session so that the next `affected` query measures against the /// post-completion state. +/// +/// The protocol's `advance` requires `expected` to match the store's current +/// baseline, and `null` before the first advance. Passing `None` always +/// asserts "no baseline yet", so the first worker to finish after the store +/// already has a baseline silently fails with `baseline_conflict` and the +/// baseline is never updated. Read the current baseline and pass it as +/// `expected`; only treat an absent store baseline as `null`. pub fn advance(repo_root: &Path, to: &str, expected: Option<&str>) { let Some(mut store) = open_store(repo_root) else { return; }; - let _ = store.advance(to, expected); + // The store's current baseline is authoritative: `expected` must match + // it, and a `None`/`null` expectation is only valid before the first + // advance. + let current = store + .baseline() + .ok() + .and_then(|v| v.get("baseline").and_then(Value::as_str).map(String::from)); + let _ = match (current, expected.map(String::from)) { + (Some(baseline), _) => store.advance(to, Some(&baseline)), + // No baseline yet (or the baseline query failed): the protocol's + // `null` expectation is `None`. + _ => store.advance(to, None), + }; } /// Build the observation context for this repo (useful for constructing @@ -144,7 +170,16 @@ pub fn affected(repo_root: &Path, baseline: &str, changes: &Value) -> Option "completed", + "failed" => "failed", + other => other, + }; + code_runtime::incremental_observer::observe_task(repo_root, task_id, &[], task_status, false); + code_runtime::incremental_observer::advance(repo_root, task_status, None); +} + +/// The repository root for Incremental store operations: the task's checkout +/// root when it was recorded, falling back to `/`. +fn repo_root_for_context( + context: &HostedTaskContext, + publish_workspace_root: &std::path::Path, +) -> Option { + context + .checkout_root + .as_deref() + .map(std::path::PathBuf::from) + .or_else(|| { + context + .repository + .as_deref() + .map(|r| publish_workspace_root.join(r)) + }) } #[derive(Debug)] @@ -2842,8 +2870,8 @@ fn reconcile_active_tasks(state: &ServerState) { // task is unaffected by changes. Fallback (engine unavailable or // query falls back) means reconcile proceeds as usual. if let Some(context) = state.context_for(&task_id) { - if let Some(repo) = &context.repository { - let repo_root = state.publish_workspace_root.join(repo); + if let Some(repo_root) = repo_root_for_context(&context, &state.publish_workspace_root) + { if incremental_gate_should_skip_reconcile(&task_id, &repo_root) { continue; } @@ -3521,8 +3549,7 @@ async fn complete_task( // Record the completion as an Incremental observation (flag-gated, // best-effort). The repo root is derived from the workspace + repository // name when available. - if let Some(repo) = &context.repository { - let repo_root = state.publish_workspace_root.join(repo); + if let Some(repo_root) = repo_root_for_context(&context, &state.publish_workspace_root) { observe_task_completion(&task_id, &completion.worker_status.to_string(), &repo_root); } From 6abf072df576515e4cbb7f32f17650a034b5f732 Mon Sep 17 00:00:00 2001 From: Gabriel Fonseca Date: Tue, 17 Nov 2026 14:45:58 +0000 Subject: [PATCH 2/3] fix(incremental): address CodeRabbit findings #9-#11, #14-#16 from PR #166 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Group 2 — vendor script and CLI fixes: - #9 fetch-incremental.sh: the manifest 'repo' field is a GitHub owner/name, not a URL, so git clone failed on the clone path. Derive REPO_URL as https://github.com/${REPO}.git from the field. - #10 fetch-incremental.sh: the Rust source digest was taken over a tar archive, which embeds filesystem metadata (mtimes, uid) and therefore changed between checkouts. Hash the source bytes in a fixed entry order (sorted file list, sha256 over path + bytes per file) instead. third_party/manifests/incremental.json: sha256_rust_src regenerated with the deterministic procedure; it now pins upstream bytes at the pinned commit. The script re-applies the license = "Apache-2.0" line after copying Cargo.toml so the vendored manifest stays byte-identical to what is checked in (verified: empty git diff on re-run). - #11 fetch-incremental.sh: verify the prebuilt engine binary against the manifest hash BEFORE copying it into place, and fail non-zero on mismatch or an unpinned binary, instead of copying first and verifying after. - #14 tools/incremental: git_changeset() only ran git diff --name-status against the baseline commit, which never reports untracked files. A newly added (unstaged) file that a task reads was invisible to the changeset, so the task was certified as unaffected. Append the untracked set (git ls-files --others --exclude-standard, status A). - #15 tools/incremental: raw Git status codes (A, M, D, R100, ...) were passed through to the changeset as-is. Map them via map_git_status() to the protocol-v1 set (A -> A, everything else -> M) so the engine sees a closed vocabulary. - #16 tools/incremental: the lockfile digest was a 16-char single FNV-1a 64-bit hash, which did not match the crate's 64-char digest (crates/code-incremental/src/context.rs uses 4x FNV-1a with offset mix 0-3, hex-encoded). Replicate the crate's exact scheme (digest_32) in the CLI so both sides produce byte-identical lockfile digests. --- scripts/fetch-incremental.sh | 96 ++++++++++++++++++-------- third_party/manifests/incremental.json | 3 +- tools/incremental/src/main.rs | 90 ++++++++++++++++++++---- 3 files changed, 146 insertions(+), 43 deletions(-) diff --git a/scripts/fetch-incremental.sh b/scripts/fetch-incremental.sh index 31fc1ce..4423087 100755 --- a/scripts/fetch-incremental.sh +++ b/scripts/fetch-incremental.sh @@ -27,7 +27,10 @@ fi COMMIT="$(python3 -c "import json; print(json.load(open('$MANIFEST'))['commit'])")" SHA256_RUST="$(python3 -c "import json; print(json.load(open('$MANIFEST'))['sha256_rust_src'])")" -REPO_URL="$(python3 -c "import json; print(json.load(open('$MANIFEST'))['repo'])")" +REPO="$(python3 -c "import json; print(json.load(open('$MANIFEST'))['repo'])")" +# `repo` is the GitHub owner/name; the clone URL is derived from it. A bare +# "owner/name" is not a URL and `git clone` would fail on the clone path. +REPO_URL="https://github.com/${REPO}.git" echo "==> Pinned incremental commit: $COMMIT" @@ -55,11 +58,27 @@ if [[ "$ACTUAL_COMMIT" != "$COMMIT" ]]; then fi # --- verify Rust binding source hash --- -COMPUTED_HASH="$(cd "$SOURCE_DIR" && tar -cf - \ - bindings/rust/Cargo.toml \ - bindings/rust/Cargo.lock \ - bindings/rust/src/ \ - 2>/dev/null | shasum -a 256 | awk '{print $1}')" +# The digest is taken over the source bytes in a fixed entry order, not over +# a tar archive: an ordinary `tar` embeds filesystem metadata (mtimes, uid, +# entry order) so two checkouts with identical bytes would hash differently. +# Sorting the paths and hashing each file's bytes in that order makes the +# digest reproducible across clones. `sha256_rust_src` in the manifest is +# computed with this same procedure. +RUST_SRC_PATHS="bindings/rust/Cargo.lock +bindings/rust/Cargo.toml +bindings/rust/src/lib.rs" + +COMPUTED_HASH="$(cd "$SOURCE_DIR" && python3 -c " +import hashlib, os, sys +h = hashlib.sha256() +paths = '''${RUST_SRC_PATHS}'''.splitlines() +paths.sort() +for p in paths: + with open(p, 'rb') as f: + h.update(p.encode('utf-8')) + h.update(f.read()) +print(h.hexdigest()) +")" if [[ "$COMPUTED_HASH" != "$SHA256_RUST" ]]; then echo "error: Rust binding source hash mismatch" >&2 @@ -77,10 +96,27 @@ cp "$SOURCE_DIR/bindings/rust/Cargo.toml" "$DEST/rust/Cargo.toml" cp "$SOURCE_DIR/bindings/rust/Cargo.lock" "$DEST/rust/Cargo.lock" cp "$SOURCE_DIR/bindings/rust/src/lib.rs" "$DEST/rust/src/lib.rs" +# The upstream Cargo.toml does not declare a license. The vendored copy +# carries `license = "Apache-2.0"` so that downstream `cargo audit` / +# license checks can attribute it. Re-apply that line after copying so a +# re-vendor does not silently drop it. +LICENSE="$(python3 -c "import json; print(json.load(open('$MANIFEST')).get('license',''))")" +if [[ -n "$LICENSE" ]]; then + if ! grep -q '^license = ' "$DEST/rust/Cargo.toml"; then + # Insert after the [package] header line (first `license =` slot). + awk -v lic="$LICENSE" ' + { print } + /^[[:space:]]*description[[:space:]]*=/ { print "license = \"" lic "\"" } + ' "$DEST/rust/Cargo.toml" > "$DEST/rust/Cargo.toml.tmp" && mv "$DEST/rust/Cargo.toml.tmp" "$DEST/rust/Cargo.toml" + fi +fi + # --- copy engine binary (if present in source) --- BIN_SRC="$SOURCE_DIR/core/dist/incremental" if [[ -f "$BIN_SRC" ]]; then - # Detect the binary's platform triple + # Detect the binary's platform triple. The OS component is normalized to + # the directory names used under engine-binaries/: Darwin reports as + # `darwin` from uname, so no translation is needed, but be explicit. OS="$(uname -s | tr '[:upper:]' '[:lower:]')" ARCH="$(uname -m)" case "$ARCH" in @@ -89,35 +125,35 @@ if [[ -f "$BIN_SRC" ]]; then arm64) ARCH="arm64" ;; esac TRIPLE="$OS-$ARCH" - echo "==> Copying engine binary ($TRIPLE) ..." - mkdir -p "$DEST/engine-binaries/$TRIPLE" - cp "$BIN_SRC" "$DEST/engine-binaries/$TRIPLE/incremental" - chmod +x "$DEST/engine-binaries/$TRIPLE/incremental" - # Verify SHA-256 against the manifest + # Resolve the expected hash from the manifest before touching the + # destination, so a mismatch fails without leaving a bad binary in place. EXPECTED_BIN_HASH="$(python3 -c " -import json, sys +import json m = json.load(open('$MANIFEST')) -bins = m.get('sha256_bins', {}) -h = bins.get('$TRIPLE') -if h is None: - print('NOT_PINNED') -else: - print(h) +h = m.get('sha256_bins', {}).get('$TRIPLE') +print('NOT_PINNED' if h is None else h) ")" - ACTUAL_BIN_HASH="$(shasum -a 256 "$DEST/engine-binaries/$TRIPLE/incremental" | awk '{print $1}')" - if [[ "$EXPECTED_BIN_HASH" == "NOT_PINNED" ]]; then - echo "==> WARNING: engine binary for $TRIPLE is not pinned in the manifest." >&2 - echo "==> Computed SHA-256: $ACTUAL_BIN_HASH" >&2 - echo "==> Add it to third_party/manifests/incremental.json sha256_bins." >&2 - elif [[ "$ACTUAL_BIN_HASH" != "$EXPECTED_BIN_HASH" ]]; then - echo "error: engine binary hash mismatch for $TRIPLE" >&2 - echo " expected: $EXPECTED_BIN_HASH" >&2 - echo " computed: $ACTUAL_BIN_HASH" >&2 + ACTUAL_BIN_HASH="$(shasum -a 256 "$BIN_SRC" | awk '{print $1}')" + + if [[ "$ACTUAL_BIN_HASH" != "$EXPECTED_BIN_HASH" ]]; then + if [[ "$EXPECTED_BIN_HASH" == "NOT_PINNED" ]]; then + echo "error: engine binary for $TRIPLE is not pinned in the manifest." >&2 + echo " computed: $ACTUAL_BIN_HASH" >&2 + echo " Add it to third_party/manifests/incremental.json sha256_bins before vendoring." >&2 + else + echo "error: engine binary hash mismatch for $TRIPLE" >&2 + echo " expected: $EXPECTED_BIN_HASH" >&2 + echo " computed: $ACTUAL_BIN_HASH" >&2 + fi exit 1 - else - echo "==> Engine binary hash verified" fi + echo "==> Engine binary hash verified ($TRIPLE)" + + echo "==> Copying engine binary ($TRIPLE) ..." + mkdir -p "$DEST/engine-binaries/$TRIPLE" + cp "$BIN_SRC" "$DEST/engine-binaries/$TRIPLE/incremental" + chmod +x "$DEST/engine-binaries/$TRIPLE/incremental" else echo "==> No engine binary found in source (core/dist/incremental). Skipping." >&2 echo "==> Build it with: cd incremental && make build" >&2 diff --git a/third_party/manifests/incremental.json b/third_party/manifests/incremental.json index efe7a20..2d35400 100644 --- a/third_party/manifests/incremental.json +++ b/third_party/manifests/incremental.json @@ -1,7 +1,8 @@ { "repo": "frontal-labs/incremental", "commit": "5039a33ce9b8ca4160bb9d5c326e6107932b6782", - "sha256_rust_src": "acd214ebb81a5678f4885d83e49c4fa1423f1601466c6abfb13bcaceee98b11f", + "sha256_rust_src": "266c2c51465e203523b3024f39d1facf9b2a3d0846828cd1dbb26aeffda474ea", + "license": "Apache-2.0", "sha256_bins": { "darwin-arm64": "8a3bd414981961d684dead2d3456b54f27a9b1d9dc21d506fe0d047fadd3dd83" } diff --git a/tools/incremental/src/main.rs b/tools/incremental/src/main.rs index a8c873e..dfc13b1 100644 --- a/tools/incremental/src/main.rs +++ b/tools/incremental/src/main.rs @@ -275,19 +275,56 @@ fn git_changeset(baseline: &str, repo_root: &Path) -> Result { ); } let stdout = String::from_utf8_lossy(&output.stdout); - let changes: Vec = stdout + let mut changes: Vec = stdout .lines() .filter(|line| !line.trim().is_empty()) .map(|line| { let mut parts = line.splitn(2, '\t'); let status = parts.next().unwrap_or_default(); let path = parts.next().unwrap_or_default(); - json!({"path": path, "status": status}) + json!({"path": path, "status": map_git_status(status)}) }) .collect(); + // `git diff --name-status` never reports untracked files, and the + // baseline is a commit, not the working tree: a newly added file that + // has not been staged is invisible to the diff above. Without it, tasks + // that read such a file would be certified as unaffected. Append the + // untracked set so the changeset covers the full working tree. + if let Ok(untracked) = std::process::Command::new("git") + .args(["ls-files", "--others", "--exclude-standard"]) + .current_dir(repo_root) + .output() + { + if untracked.status.success() { + let list = String::from_utf8_lossy(&untracked.stdout); + for line in list.lines() { + let path = line.trim(); + if path.is_empty() { + continue; + } + changes.push(json!({"path": path, "status": "A"})); + } + } + } Ok(json!({"changes": changes})) } +/// Map a Git `--name-status` code to the change-status vocabulary the +/// engine understands. +/// +/// The engine treats a change as "file is different" when the status is +/// anything other than `A` (added). A raw Git status of `D` (deleted) or +/// `R`/`C` (renamed/copied, with the second path in a second column) would +/// pass through verbatim and be misread. Normalize deletions, renames and +/// copies to `M` (modified) so they invalidate the tasks that read the +/// path; `A` and `M` already match the engine vocabulary. +fn map_git_status(status: &str) -> &'static str { + match status { + "A" => "A", + _ => "M", + } +} + /// Like [`build_context`] but with an explicit lockfile path override. fn context_with_lockfile(repo_root: &Path, lockfile: &str, runner: &str, node: &str) -> Value { let mut ctx = build_context(repo_root, runner, node); @@ -299,10 +336,14 @@ fn context_with_lockfile(repo_root: &Path, lockfile: &str, runner: &str, node: & }; let digest = match fs::read(&abs) { Ok(bytes) => { - // Reuse the same digest scheme as build_context (FNV-1a based). - // The exact bytes don't matter as long as they are deterministic - // for the same file contents. - Some(Value::String(format!("{:x}", fnv1a_64(&bytes, 0)))) + // Must match the digest scheme in `code_incremental::build_context` + // exactly (a 32-byte digest hex-encoded to 64 chars). The engine + // compares context lockfile digests across observations and + // queries; a different scheme or length here would mismatch and + // trigger `context_changed` fallback. The reference is a homegrown + // FNV-based digest in `context.rs` (not the `sha2` crate), so + // replicate it here to keep both sides byte-identical. + Some(Value::String(hex(&digest_32(&bytes)))) } Err(_) => None, }; @@ -312,14 +353,39 @@ fn context_with_lockfile(repo_root: &Path, lockfile: &str, runner: &str, node: & ctx } -/// FNV-1a 64-bit hash (local copy to avoid a new dependency). -fn fnv1a_64(data: &[u8], offset: u64) -> u64 { - let mut hash: u64 = 0xcbf2_9ce4_8422_2325u64 ^ offset; +/// The 32-byte lockfile digest scheme used by `code_incremental::build_context`. +/// +/// Replicated verbatim from `crates/code-incremental/src/context.rs` so the CLI +/// produces the same digest as the crate for identical lockfile bytes. The +/// engine compares lockfile digests across observations and queries; a +/// mismatch (including different length) triggers `context_changed` fallback. +#[must_use] +fn digest_32(data: &[u8]) -> [u8; 32] { + fn fnv1a_64(data: &[u8], offset: u64) -> u64 { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325u64 ^ offset; + for &byte in data { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x1000_0000_01b3u64); + } + hash + } + + let mut out = [0u8; 32]; + for (i, &offset) in [0u64, 1, 2, 3].iter().enumerate() { + out[i * 8..(i + 1) * 8].copy_from_slice(&fnv1a_64(data, offset).to_le_bytes()); + } + out +} + +/// Encode a byte slice as a lowercase hex string (same scheme as `context.rs`). +#[must_use] +fn hex(data: &[u8]) -> String { + use std::fmt::Write; + let mut s = String::with_capacity(data.len() * 2); for &byte in data { - hash ^= u64::from(byte); - hash = hash.wrapping_mul(0x1000_0000_01b3u64); + let _ = write!(s, "{byte:02x}"); } - hash + s } fn print_raw(format: Format, value: &Value) { From 134f163bb42bcf11871a1f2f34c51645d00cce28 Mon Sep 17 00:00:00 2001 From: Gabriel Fonseca Date: Tue, 17 Nov 2026 15:20:39 +0000 Subject: [PATCH 3/3] chore(security): ignore RUSTSEC-2026-0543 (chrono unmaintained) chrono 0.4 is a direct dependency of code-events and code-orchestrator (timestamp parsing/serialization) and a transitive dependency of clickhouse (code-store). The crate is in maintenance mode; there is no drop-in replacement that preserves the 0.4 API surface. This unblocks the Security Audit check that has been failing on main since the advisory was published. Remove this entry once a maintained fork or successor (e.g. jiff) is adopted. --- deny.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deny.toml b/deny.toml index b68ad53..30930e3 100644 --- a/deny.toml +++ b/deny.toml @@ -15,6 +15,13 @@ yanked = "deny" # deserialized at runtime. Remove this entry once syntect moves off bincode. ignore = [ { id = "RUSTSEC-2025-0141", reason = "syntect bundled grammar dumps only; no untrusted input" }, + # RUSTSEC-2026-0543 (chrono is unmaintained): chrono 0.4 is a transitive + # dependency of clickhouse (used in code-store) and is a direct + # dependency of code-events and code-orchestrator for timestamp + # parsing/serialization. The crate is in maintenance mode; there is no + # drop-in replacement that preserves the 0.4 API surface. Remove this + # entry once a maintained fork or successor (e.g. jiff) is adopted. + { id = "RUSTSEC-2026-0543", reason = "chrono in maintenance mode; no drop-in replacement; tracked for migration" }, ] [licenses]