From ce79cbe05af262d9851c46d73580626b7cc576b3 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:58:30 -0700 Subject: [PATCH 1/7] fix(ci): bound pinned Pi tool installation Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .../actions/ci-cli-coverage-shard/action.yaml | 5 +- .../install-pinned-ubuntu-packages.sh | 76 ++++++++ .github/workflows/pr-review-advisor.yaml | 6 +- .../pr-workflow-contract.test.ts | 167 +++++++++++++++--- 4 files changed, 219 insertions(+), 35 deletions(-) create mode 100755 .github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh diff --git a/.github/actions/ci-cli-coverage-shard/action.yaml b/.github/actions/ci-cli-coverage-shard/action.yaml index 9bb0f1a1cfb..eee7373816d 100644 --- a/.github/actions/ci-cli-coverage-shard/action.yaml +++ b/.github/actions/ci-cli-coverage-shard/action.yaml @@ -60,10 +60,7 @@ runs: RIPGREP_VERSION: "14.1.0-1" run: | set -euo pipefail - sudo apt-get update -qq \ - -o Dir::Etc::sourcelist="sources.list.d/ubuntu.sources" \ - -o Dir::Etc::sourceparts="-" - sudo apt-get install -y --no-install-recommends \ + bash "$GITHUB_ACTION_PATH/install-pinned-ubuntu-packages.sh" \ "fd-find=${FD_FIND_VERSION}" \ "ripgrep=${RIPGREP_VERSION}" diff --git a/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh b/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh new file mode 100755 index 00000000000..a876fa0625a --- /dev/null +++ b/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [ "$#" -eq 0 ]; then + echo "::error title=Missing APT packages::Provide pinned package=version specifications." >&2 + exit 2 +fi +for package_spec in "$@"; do + if [[ ! "$package_spec" =~ ^[A-Za-z0-9][A-Za-z0-9.+-]*(:[A-Za-z0-9][A-Za-z0-9.+-]*)?=[^[:space:]=]+$ ]]; then + echo "::error title=Unpinned APT package::Expected package=version." >&2 + exit 2 + fi +done + +ubuntu_sources=/etc/apt/sources.list.d/ubuntu.sources +if ! sudo test -f "$ubuntu_sources" || ! sudo test -r "$ubuntu_sources"; then + echo "::error title=Missing Ubuntu APT source::Expected readable $ubuntu_sources." >&2 + exit 1 +fi +if [ -z "${RUNNER_TEMP:-}" ] || [ ! -d "$RUNNER_TEMP" ]; then + echo "::error title=Missing runner temporary directory::RUNNER_TEMP must be a directory." >&2 + exit 1 +fi + +runner_temp_mode="$(stat -c '%a' "$RUNNER_TEMP")" +apt_lists="$(mktemp -d "$RUNNER_TEMP/nemoclaw-apt-lists.XXXXXXXX")" +cleanup() { + local status=$? + trap - EXIT + sudo rm -rf -- "$apt_lists" || status=1 + sudo chmod "$runner_temp_mode" "$RUNNER_TEMP" || status=1 + exit "$status" +} +trap cleanup EXIT + +# APT's _apt user needs traversal into the isolated package-list directory. +sudo chmod o+x "$RUNNER_TEMP" +sudo chmod 0755 "$apt_lists" +sudo install -d -o _apt -g root -m 0700 "$apt_lists/partial" + +apt_options=( + -o "Dir::Etc::sourcelist=$ubuntu_sources" + -o "Dir::Etc::sourceparts=-" + -o "Dir::State::lists=$apt_lists" + -o "Acquire::http::Timeout=30" + -o "Acquire::https::Timeout=30" +) + +run_apt() { + local operation="$1" + shift + local log status + log="$(mktemp "$RUNNER_TEMP/nemoclaw-apt-${operation}.XXXXXXXX")" + echo "Installing pinned Pi tools: APT $operation started." + if timeout -k 10s 300s sudo apt-get "${apt_options[@]}" "$@" >"$log" 2>&1; then + rm -f -- "$log" + echo "Installing pinned Pi tools: APT $operation completed." + return 0 + else + status=$? + fi + if [ "$status" -eq 124 ]; then + echo "::error title=APT $operation timed out::Ubuntu package $operation exceeded 300 seconds." >&2 + else + echo "::error title=APT $operation failed::Ubuntu package $operation exited with status $status." >&2 + fi + tail -c 8192 "$log" | tail -n 60 >&2 + rm -f -- "$log" + return "$status" +} + +run_apt update update +run_apt install install -y --no-install-recommends "$@" diff --git a/.github/workflows/pr-review-advisor.yaml b/.github/workflows/pr-review-advisor.yaml index d7bbfae8fd5..b58adb13c19 100644 --- a/.github/workflows/pr-review-advisor.yaml +++ b/.github/workflows/pr-review-advisor.yaml @@ -217,10 +217,8 @@ jobs: - name: Install locked runtime run: | set -euo pipefail - sudo apt-get update -qq \ - -o Dir::Etc::sourcelist="sources.list.d/ubuntu.sources" \ - -o Dir::Etc::sourceparts="-" - sudo apt-get install -y --no-install-recommends "fd-find=${FD_FIND_VERSION}" "ripgrep=${RIPGREP_VERSION}" + bash "$ADVISOR_DIR/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh" \ + "fd-find=${FD_FIND_VERSION}" "ripgrep=${RIPGREP_VERSION}" test "$(dpkg-query -W -f='${Version}' fd-find)" = "$FD_FIND_VERSION" test "$(dpkg-query -W -f='${Version}' ripgrep)" = "$RIPGREP_VERSION" (cd "$ADVISOR_DIR" && npm ci --ignore-scripts --no-audit --no-fund) diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index 888cbeb16b3..db5b8c3bac9 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -85,6 +85,78 @@ function runWorkflowShellStep( }; } +function runPinnedAptFixture( + mode: string, + step?: WorkflowStep, + packages = ["fd-find=9.0.0-1", "ripgrep=14.1.0-1"], +) { + const temp = mkdtempSync(join(tmpdir(), "nemoclaw-pinned-apt-")); + const fakeBin = join(temp, "bin"); + const runnerTemp = join(temp, "runner-temp"); + const aptCalls = join(temp, "apt-calls"); + const timeoutCalls = join(temp, "timeout-calls"); + mkdirSync(fakeBin); + mkdirSync(runnerTemp); + writeFileSync(aptCalls, ""); + writeFileSync(timeoutCalls, ""); + writeFileSync( + join(fakeBin, "sudo"), + `#!/usr/bin/env bash +printf '%s\\n' "$*" >> "$APT_CALLS" +if [[ "$1" == test && "$FAKE_APT_MODE" == missing-source ]]; then exit 1; fi +if [[ "$1" == apt-get && "$FAKE_APT_MODE" == update-error ]]; then + for ((line=1; line<=100; line++)); do printf 'apt diagnostic %s\\n' "$line" >&2; done + exit 86 +fi +if [[ "$1" == apt-get && "$FAKE_APT_MODE" == install-error && "$*" == *' install '* ]]; then exit 100; fi +exit 0 +`, + { mode: 0o755 }, + ); + writeFileSync( + join(fakeBin, "timeout"), + `#!/usr/bin/env bash +printf '%s\\n' "$*" >> "$TIMEOUT_CALLS" +shift 3 +if [[ "$FAKE_APT_MODE" == timeout || ( "$FAKE_APT_MODE" == install-timeout && "$*" == *' install '* ) ]]; then exit 124; fi +"$@" +`, + { mode: 0o755 }, + ); + writeFileSync(join(fakeBin, "stat"), "#!/usr/bin/env bash\nprintf '700\\n'\n", { + mode: 0o755, + }); + const env = { + ...process.env, + ADVISOR_DIR: process.cwd(), + APT_CALLS: aptCalls, + FAKE_APT_MODE: mode, + FD_FIND_VERSION: "9.0.0-1", + GITHUB_ACTION_PATH: join(process.cwd(), ".github/actions/ci-cli-coverage-shard"), + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + RIPGREP_VERSION: "14.1.0-1", + RUNNER_TEMP: runnerTemp, + TIMEOUT_CALLS: timeoutCalls, + }; + try { + const result = step + ? runWorkflowShellStep(step, env) + : spawnSync( + "bash", + [join(env.GITHUB_ACTION_PATH, "install-pinned-ubuntu-packages.sh"), ...packages], + { encoding: "utf8", env, timeout: 5_000 }, + ); + return { + status: result.status, + stderr: String(result.stderr), + calls: readFileSync(aptCalls, "utf8").trim().split("\n"), + timeoutCalls: readFileSync(timeoutCalls, "utf8").trim().split("\n"), + }; + } finally { + rmSync(temp, { recursive: true, force: true }); + } +} + const reviewedSdk = { artifactName: "reviewed-sdk.tgz", integrity: "sha512-reviewed", @@ -518,35 +590,76 @@ printf '%s %s\\n' '6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc "Advisor runtime", requiredWorkflowStep(advisorWorkflow.jobs["build-advisor-runtime"], "Install locked runtime"), ], - ])("refreshes only Ubuntu package metadata for %s", (_name, installStep) => { - const temp = mkdtempSync(join(tmpdir(), "nemoclaw-ubuntu-apt-sources-")); - const fakeBin = join(temp, "bin"); - const aptArgs = join(temp, "apt-args"); - mkdirSync(fakeBin); - writeFileSync( - join(fakeBin, "sudo"), - '#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "$APT_ARGS"\nexit 86\n', - { mode: 0o755 }, + ])( + "reports an Ubuntu APT update failure before installing packages in %s (#11320)", + (_name, installStep) => { + const result = runPinnedAptFixture("update-error", installStep); + expect(result.status).toBe(86); + expect(result.stderr).toContain("APT update failed"); + expect(result.stderr).toContain("apt diagnostic 100"); + expect(result.stderr).not.toContain("apt diagnostic 1\n"); + expect(result.stderr.split("\n")).toHaveLength(62); + expect(result.calls.some((call) => /^chmod 700 .*\/runner-temp$/u.test(call))).toBe(true); + const aptCalls = result.calls.filter((call) => call.startsWith("apt-get ")); + expect(aptCalls).toHaveLength(1); + expect(aptCalls[0]).toContain("Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources"); + expect(aptCalls[0]).toContain("Dir::Etc::sourceparts=-"); + expect(aptCalls[0]).toMatch(/Dir::State::lists=\S+\/nemoclaw-apt-lists\.\S+/u); + expect(aptCalls[0]).toMatch(/ update$/u); + }, + ); + + it("uses one isolated Ubuntu package list for APT update and install (#11320)", () => { + const result = runPinnedAptFixture("success"); + expect(result.status).toBe(0); + const aptCalls = result.calls.filter((call) => call.startsWith("apt-get ")); + expect(aptCalls).toHaveLength(2); + const lists = aptCalls.map((call) => call.match(/Dir::State::lists=(\S+)/u)?.[1]); + expect(lists[0]).toMatch(/\/nemoclaw-apt-lists\.\S+$/u); + expect(lists[1]).toBe(lists[0]); + expect(aptCalls[1]).toContain("Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources"); + expect(aptCalls[1]).toContain("Dir::Etc::sourceparts=-"); + expect(aptCalls[1]).toContain( + "install -y --no-install-recommends fd-find=9.0.0-1 ripgrep=14.1.0-1", ); + }); - try { - const result = runWorkflowShellStep(installStep, { - APT_ARGS: aptArgs, - PATH: `${fakeBin}:${process.env.PATH ?? ""}`, - }); - expect(result.status).toBe(86); - expect(readFileSync(aptArgs, "utf8").trim().split("\n")).toEqual([ - "apt-get", - "update", - "-qq", - "-o", - "Dir::Etc::sourcelist=sources.list.d/ubuntu.sources", - "-o", - "Dir::Etc::sourceparts=-", - ]); - } finally { - rmSync(temp, { force: true, recursive: true }); - } + it("stops a stalled APT update within the command budget (#11320)", () => { + const result = runPinnedAptFixture("timeout"); + expect(result.status).toBe(124); + expect(result.stderr).toContain("APT update timed out"); + expect(result.timeoutCalls[0]).toMatch(/^-k 10s 300s sudo apt-get /u); + expect(result.calls.filter((call) => call.startsWith("apt-get "))).toHaveLength(0); + }); + + it("stops a stalled APT install after a successful update (#11320)", () => { + const result = runPinnedAptFixture("install-timeout"); + expect(result.status).toBe(124); + expect(result.stderr).toContain("APT install timed out"); + expect(result.timeoutCalls).toHaveLength(2); + expect(result.timeoutCalls[1]).toMatch(/^-k 10s 300s sudo apt-get /u); + expect(result.calls.filter((call) => call.startsWith("apt-get "))).toHaveLength(1); + }); + + it("fails when the pinned package install fails (#11320)", () => { + const result = runPinnedAptFixture("install-error"); + expect(result.status).toBe(100); + expect(result.stderr).toContain("APT install failed"); + expect(result.calls.filter((call) => call.startsWith("apt-get "))).toHaveLength(2); + }); + + it("rejects a missing Ubuntu source before APT runs (#11320)", () => { + const result = runPinnedAptFixture("missing-source"); + expect(result.status).toBe(1); + expect(result.stderr).toContain("Missing Ubuntu APT source"); + expect(result.calls.filter((call) => call.startsWith("apt-get "))).toHaveLength(0); + }); + + it("rejects an unpinned package before using sudo (#11320)", () => { + const result = runPinnedAptFixture("success", undefined, ["fd-find"]); + expect(result.status).toBe(2); + expect(result.stderr).toContain("Unpinned APT package"); + expect(result.calls).toEqual([""]); }); // source-shape-contract: security -- PR dependency jobs receive only the base-approved SDK archive after credential-free integrity verification From ddfaf299eb7f3bc97b763e019ddb4cbdb2e11f62 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:06:21 -0700 Subject: [PATCH 2/7] fix(ci): cover pinned tool caller success and forced kills Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .../install-pinned-ubuntu-packages.sh | 2 + .../pr-workflow-contract.test.ts | 56 ++++++++++++++++++- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh b/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh index a876fa0625a..d57c9588b25 100755 --- a/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh +++ b/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh @@ -64,6 +64,8 @@ run_apt() { fi if [ "$status" -eq 124 ]; then echo "::error title=APT $operation timed out::Ubuntu package $operation exceeded 300 seconds." >&2 + elif [ "$status" -eq 137 ]; then + echo "::error title=APT $operation was force-killed::Ubuntu package $operation exited with status 137; it may have exceeded 300 seconds and been killed by timeout." >&2 else echo "::error title=APT $operation failed::Ubuntu package $operation exited with status $status." >&2 fi diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index db5b8c3bac9..a824609b8f2 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -3,7 +3,15 @@ import { createHash } from "node:crypto"; import { spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -96,18 +104,22 @@ function runPinnedAptFixture( const aptCalls = join(temp, "apt-calls"); const timeoutCalls = join(temp, "timeout-calls"); mkdirSync(fakeBin); - mkdirSync(runnerTemp); + mkdirSync(runnerTemp, { mode: 0o700 }); writeFileSync(aptCalls, ""); writeFileSync(timeoutCalls, ""); writeFileSync( join(fakeBin, "sudo"), `#!/usr/bin/env bash printf '%s\\n' "$*" >> "$APT_CALLS" +if [[ "$1" == chmod && "$3" == "$RUNNER_TEMP" ]]; then + command chmod "$2" "$3" || exit $? +fi if [[ "$1" == test && "$FAKE_APT_MODE" == missing-source ]]; then exit 1; fi if [[ "$1" == apt-get && "$FAKE_APT_MODE" == update-error ]]; then for ((line=1; line<=100; line++)); do printf 'apt diagnostic %s\\n' "$line" >&2; done exit 86 fi +if [[ "$1" == apt-get && "$FAKE_APT_MODE" == force-killed ]]; then exit 137; fi if [[ "$1" == apt-get && "$FAKE_APT_MODE" == install-error && "$*" == *' install '* ]]; then exit 100; fi exit 0 `, @@ -126,6 +138,18 @@ if [[ "$FAKE_APT_MODE" == timeout || ( "$FAKE_APT_MODE" == install-timeout && "$ writeFileSync(join(fakeBin, "stat"), "#!/usr/bin/env bash\nprintf '700\\n'\n", { mode: 0o755, }); + writeFileSync( + join(fakeBin, "dpkg-query"), + "#!/usr/bin/env bash\ncase \"${*: -1}\" in fd-find) printf '9.0.0-1';; ripgrep) printf '14.1.0-1';; *) exit 1;; esac\n", + { mode: 0o755 }, + ); + writeFileSync(join(fakeBin, "fdfind"), "#!/usr/bin/env bash\nprintf 'fdfind 9.0.0\\n'\n", { + mode: 0o755, + }); + writeFileSync(join(fakeBin, "rg"), "#!/usr/bin/env bash\nprintf 'ripgrep 14.1.0\\n'\n", { + mode: 0o755, + }); + writeFileSync(join(fakeBin, "npm"), "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755 }); const env = { ...process.env, ADVISOR_DIR: process.cwd(), @@ -149,6 +173,7 @@ if [[ "$FAKE_APT_MODE" == timeout || ( "$FAKE_APT_MODE" == install-timeout && "$ return { status: result.status, stderr: String(result.stderr), + runnerTempMode: statSync(runnerTemp).mode & 0o777, calls: readFileSync(aptCalls, "utf8").trim().split("\n"), timeoutCalls: readFileSync(timeoutCalls, "utf8").trim().split("\n"), }; @@ -599,7 +624,7 @@ printf '%s %s\\n' '6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc expect(result.stderr).toContain("apt diagnostic 100"); expect(result.stderr).not.toContain("apt diagnostic 1\n"); expect(result.stderr.split("\n")).toHaveLength(62); - expect(result.calls.some((call) => /^chmod 700 .*\/runner-temp$/u.test(call))).toBe(true); + expect(result.runnerTempMode).toBe(0o700); const aptCalls = result.calls.filter((call) => call.startsWith("apt-get ")); expect(aptCalls).toHaveLength(1); expect(aptCalls[0]).toContain("Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources"); @@ -624,6 +649,23 @@ printf '%s %s\\n' '6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc ); }); + it.each([ + ["CLI shards", requiredStep(sharedActions.cliCoverageShard, "Install pinned Pi search tools")], + [ + "Advisor runtime", + requiredWorkflowStep(advisorWorkflow.jobs["build-advisor-runtime"], "Install locked runtime"), + ], + ])("installs the pinned packages through the %s workflow step (#11320)", (_name, installStep) => { + const result = runPinnedAptFixture("success", installStep); + expect(result.status, result.stderr).toBe(0); + expect(result.runnerTempMode).toBe(0o700); + const aptCalls = result.calls.filter((call) => call.startsWith("apt-get ")); + expect(aptCalls).toHaveLength(2); + expect(aptCalls[1]).toContain( + "install -y --no-install-recommends fd-find=9.0.0-1 ripgrep=14.1.0-1", + ); + }); + it("stops a stalled APT update within the command budget (#11320)", () => { const result = runPinnedAptFixture("timeout"); expect(result.status).toBe(124); @@ -648,6 +690,14 @@ printf '%s %s\\n' '6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc expect(result.calls.filter((call) => call.startsWith("apt-get "))).toHaveLength(2); }); + it("reports a possible force-killed timeout without claiming its cause (#11320)", () => { + const result = runPinnedAptFixture("force-killed"); + expect(result.status).toBe(137); + expect(result.stderr).toContain("APT update was force-killed"); + expect(result.stderr).toContain("may have exceeded 300 seconds"); + expect(result.calls.filter((call) => call.startsWith("apt-get "))).toHaveLength(1); + }); + it("rejects a missing Ubuntu source before APT runs (#11320)", () => { const result = runPinnedAptFixture("missing-source"); expect(result.status).toBe(1); From 445444dee7b53a42d26708a7c549acf027eebe2e Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:55:44 -0700 Subject: [PATCH 3/7] ci: preserve bounded CLI APT retry in isolated installer Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .../install-pinned-ubuntu-packages.sh | 51 +++++++++++++++---- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh b/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh index d57c9588b25..76a4dfe3583 100755 --- a/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh +++ b/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh @@ -4,6 +4,18 @@ set -euo pipefail +update_attempts=1 +update_timeout=300s +install_timeout=300s +acquire_timeout=30 +if [ "${1:-}" = --retry-transient-update ]; then + update_attempts=3 + update_timeout=120s + install_timeout=180s + acquire_timeout=20 + shift +fi + if [ "$#" -eq 0 ]; then echo "::error title=Missing APT packages::Provide pinned package=version specifications." >&2 exit 2 @@ -45,17 +57,23 @@ apt_options=( -o "Dir::Etc::sourcelist=$ubuntu_sources" -o "Dir::Etc::sourceparts=-" -o "Dir::State::lists=$apt_lists" - -o "Acquire::http::Timeout=30" - -o "Acquire::https::Timeout=30" + -o "Acquire::http::Timeout=$acquire_timeout" + -o "Acquire::https::Timeout=$acquire_timeout" ) +if [ "$update_attempts" -gt 1 ]; then + apt_options+=(-o "Acquire::Retries=2") +fi run_apt() { local operation="$1" - shift + local duration="$2" + local attempt="$3" + local total="$4" + shift 4 local log status log="$(mktemp "$RUNNER_TEMP/nemoclaw-apt-${operation}.XXXXXXXX")" - echo "Installing pinned Pi tools: APT $operation started." - if timeout -k 10s 300s sudo apt-get "${apt_options[@]}" "$@" >"$log" 2>&1; then + echo "Installing pinned Pi tools: APT $operation started (attempt $attempt/$total)." + if timeout -k 10s "$duration" sudo apt-get "${apt_options[@]}" "$@" >"$log" 2>&1; then rm -f -- "$log" echo "Installing pinned Pi tools: APT $operation completed." return 0 @@ -63,16 +81,29 @@ run_apt() { status=$? fi if [ "$status" -eq 124 ]; then - echo "::error title=APT $operation timed out::Ubuntu package $operation exceeded 300 seconds." >&2 + if [ "$operation" = update ] && [ "$attempt" -lt "$total" ]; then + echo "::warning title=APT update timed out::Ubuntu package update exceeded $duration on attempt $attempt/$total; retrying the idempotent metadata update." >&2 + else + echo "::error title=APT $operation timed out::Ubuntu package $operation exceeded $duration on attempt $attempt/$total." >&2 + fi elif [ "$status" -eq 137 ]; then - echo "::error title=APT $operation was force-killed::Ubuntu package $operation exited with status 137; it may have exceeded 300 seconds and been killed by timeout." >&2 + echo "::error title=APT $operation was force-killed::Ubuntu package $operation exited with status 137 on attempt $attempt/$total; it may have exceeded $duration and been killed by timeout." >&2 else - echo "::error title=APT $operation failed::Ubuntu package $operation exited with status $status." >&2 + echo "::error title=APT $operation failed::Ubuntu package $operation exited with status $status on attempt $attempt/$total." >&2 fi tail -c 8192 "$log" | tail -n 60 >&2 rm -f -- "$log" return "$status" } -run_apt update update -run_apt install install -y --no-install-recommends "$@" +for ((attempt = 1; attempt <= update_attempts; attempt++)); do + if run_apt update "$update_timeout" "$attempt" "$update_attempts" update; then + break + else + status=$? + fi + if [ "$status" -ne 124 ] || [ "$attempt" -eq "$update_attempts" ]; then + exit "$status" + fi +done +run_apt install "$install_timeout" 1 1 install -y --no-install-recommends "$@" From 8a08e628dfec9e081e07a56055db5701a21670b5 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:39:12 -0700 Subject: [PATCH 4/7] ci: keep pinned APT installer shared and supervised Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .../actions/ci-cli-coverage-shard/action.yaml | 2 +- ...h => ci-install-pinned-ubuntu-packages.sh} | 2 +- .github/workflows/pr-review-advisor.yaml | 2 +- .github/workflows/pr.yaml | 1 + .../pr-workflow-contract.test.ts | 41 +++++++++++++------ 5 files changed, 33 insertions(+), 15 deletions(-) rename .github/actions/{ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh => ci-install-pinned-ubuntu-packages.sh} (97%) diff --git a/.github/actions/ci-cli-coverage-shard/action.yaml b/.github/actions/ci-cli-coverage-shard/action.yaml index b20a379cd33..68738ae1c73 100644 --- a/.github/actions/ci-cli-coverage-shard/action.yaml +++ b/.github/actions/ci-cli-coverage-shard/action.yaml @@ -60,7 +60,7 @@ runs: RIPGREP_VERSION: "14.1.0-1" run: | set -euo pipefail - bash "$GITHUB_ACTION_PATH/install-pinned-ubuntu-packages.sh" --retry-transient-update \ + bash "$GITHUB_ACTION_PATH/../ci-install-pinned-ubuntu-packages.sh" --retry-transient-update \ "fd-find=${FD_FIND_VERSION}" \ "ripgrep=${RIPGREP_VERSION}" diff --git a/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh b/.github/actions/ci-install-pinned-ubuntu-packages.sh similarity index 97% rename from .github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh rename to .github/actions/ci-install-pinned-ubuntu-packages.sh index 76a4dfe3583..ced9cd5a273 100755 --- a/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh +++ b/.github/actions/ci-install-pinned-ubuntu-packages.sh @@ -73,7 +73,7 @@ run_apt() { local log status log="$(mktemp "$RUNNER_TEMP/nemoclaw-apt-${operation}.XXXXXXXX")" echo "Installing pinned Pi tools: APT $operation started (attempt $attempt/$total)." - if timeout -k 10s "$duration" sudo apt-get "${apt_options[@]}" "$@" >"$log" 2>&1; then + if { sudo timeout -k 10s "$duration" apt-get "${apt_options[@]}" "$@"; } >"$log" 2>&1; then rm -f -- "$log" echo "Installing pinned Pi tools: APT $operation completed." return 0 diff --git a/.github/workflows/pr-review-advisor.yaml b/.github/workflows/pr-review-advisor.yaml index 66127509437..d42a2d84a77 100644 --- a/.github/workflows/pr-review-advisor.yaml +++ b/.github/workflows/pr-review-advisor.yaml @@ -217,7 +217,7 @@ jobs: - name: Install locked runtime run: | set -euo pipefail - bash "$ADVISOR_DIR/.github/actions/ci-cli-coverage-shard/install-pinned-ubuntu-packages.sh" \ + bash "$ADVISOR_DIR/.github/actions/ci-install-pinned-ubuntu-packages.sh" \ "fd-find=${FD_FIND_VERSION}" "ripgrep=${RIPGREP_VERSION}" test "$(dpkg-query -W -f='${Version}' fd-find)" = "$FD_FIND_VERSION" test "$(dpkg-query -W -f='${Version}' ripgrep)" = "$RIPGREP_VERSION" diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c6cf4549de6..90621c45fb7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -590,6 +590,7 @@ jobs: .github/actions/ci-build-typecheck .github/actions/ci-compile-artifacts .github/actions/ci-cli-coverage-shard + .github/actions/ci-install-pinned-ubuntu-packages.sh .github/actions/ci-cli-coverage-merge .github/actions/ci-plugin-coverage .github/actions/ci-installer-integration diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index 194095836dc..cd71c6e1ca8 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -115,12 +115,21 @@ if [[ "$1" == chmod && "$3" == "$RUNNER_TEMP" ]]; then command chmod "$2" "$3" || exit $? fi if [[ "$1" == test && "$FAKE_APT_MODE" == missing-source ]]; then exit 1; fi -if [[ "$1" == apt-get && "$FAKE_APT_MODE" == update-error ]]; then +if [[ "$1" == timeout ]]; then shift; timeout "$@"; exit $?; fi +if [[ "$1" == apt-get ]]; then echo 'apt-get must run under sudo timeout' >&2; exit 99; fi +exit 0 +`, + { mode: 0o755 }, + ); + writeFileSync( + join(fakeBin, "apt-get"), + `#!/usr/bin/env bash +printf 'apt-get %s\\n' "$*" >> "$APT_CALLS" +if [[ "$FAKE_APT_MODE" == update-error ]]; then for ((line=1; line<=100; line++)); do printf 'apt diagnostic %s\\n' "$line" >&2; done exit 86 fi -if [[ "$1" == apt-get && "$FAKE_APT_MODE" == force-killed ]]; then exit 137; fi -if [[ "$1" == apt-get && "$FAKE_APT_MODE" == install-error && "$*" == *' install '* ]]; then exit 100; fi +if [[ "$FAKE_APT_MODE" == install-error && "$*" == *' install '* ]]; then exit 100; fi exit 0 `, { mode: 0o755 }, @@ -131,6 +140,7 @@ exit 0 printf '%s\\n' "$*" >> "$TIMEOUT_CALLS" shift 3 if [[ "$FAKE_APT_MODE" == timeout || ( "$FAKE_APT_MODE" == install-timeout && "$*" == *' install '* ) ]]; then exit 124; fi +if [[ "$FAKE_APT_MODE" == force-killed ]]; then exit 137; fi if [[ "$FAKE_APT_MODE" == recover-update && "$*" == *' update' && $(wc -l < "$TIMEOUT_CALLS") -lt 3 ]]; then exit 124; fi "$@" `, @@ -168,7 +178,10 @@ if [[ "$FAKE_APT_MODE" == recover-update && "$*" == *' update' && $(wc -l < "$TI ? runWorkflowShellStep(step, env) : spawnSync( "bash", - [join(env.GITHUB_ACTION_PATH, "install-pinned-ubuntu-packages.sh"), ...packages], + [ + join(process.cwd(), ".github/actions/ci-install-pinned-ubuntu-packages.sh"), + ...packages, + ], { encoding: "utf8", env, timeout: 5_000 }, ); return { @@ -672,7 +685,7 @@ printf '%s %s\\n' '6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc const result = runPinnedAptFixture("timeout"); expect(result.status).toBe(124); expect(result.stderr).toContain("APT update timed out"); - expect(result.timeoutCalls[0]).toMatch(/^-k 10s 300s sudo apt-get /u); + expect(result.timeoutCalls[0]).toMatch(/^-k 10s 300s apt-get /u); expect(result.calls.filter((call) => call.startsWith("apt-get "))).toHaveLength(0); }); @@ -685,9 +698,9 @@ printf '%s %s\\n' '6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc expect(result.status, result.stderr).toBe(0); expect(result.timeoutCalls).toHaveLength(4); expect(result.timeoutCalls.slice(0, 3)).toEqual( - Array.from({ length: 3 }, () => expect.stringMatching(/^-k 10s 120s sudo apt-get /u)), + Array.from({ length: 3 }, () => expect.stringMatching(/^-k 10s 120s apt-get /u)), ); - expect(result.timeoutCalls[3]).toMatch(/^-k 10s 180s sudo apt-get /u); + expect(result.timeoutCalls[3]).toMatch(/^-k 10s 180s apt-get /u); const aptCalls = result.calls.filter((call) => call.startsWith("apt-get ")); expect(aptCalls).toHaveLength(2); expect(aptCalls[0]).toContain("Acquire::Retries=2"); @@ -708,7 +721,7 @@ printf '%s %s\\n' '6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc expect(result.status).toBe(124); expect(result.timeoutCalls).toHaveLength(3); expect(result.timeoutCalls).toEqual( - Array.from({ length: 3 }, () => expect.stringMatching(/^-k 10s 120s sudo apt-get /u)), + Array.from({ length: 3 }, () => expect.stringMatching(/^-k 10s 120s apt-get /u)), ); expect(result.stderr).toContain("APT update timed out"); expect(result.stderr).toContain("attempt 3/3"); @@ -720,7 +733,7 @@ printf '%s %s\\n' '6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc expect(result.status).toBe(124); expect(result.stderr).toContain("APT install timed out"); expect(result.timeoutCalls).toHaveLength(2); - expect(result.timeoutCalls[1]).toMatch(/^-k 10s 300s sudo apt-get /u); + expect(result.timeoutCalls[1]).toMatch(/^-k 10s 300s apt-get /u); expect(result.calls.filter((call) => call.startsWith("apt-get "))).toHaveLength(1); }); @@ -733,8 +746,8 @@ printf '%s %s\\n' '6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc expect(result.status).toBe(124); expect(result.stderr).toContain("APT install timed out"); expect(result.timeoutCalls).toHaveLength(2); - expect(result.timeoutCalls[0]).toMatch(/^-k 10s 120s sudo apt-get /u); - expect(result.timeoutCalls[1]).toMatch(/^-k 10s 180s sudo apt-get /u); + expect(result.timeoutCalls[0]).toMatch(/^-k 10s 120s apt-get /u); + expect(result.timeoutCalls[1]).toMatch(/^-k 10s 180s apt-get /u); expect(result.calls.filter((call) => call.startsWith("apt-get "))).toHaveLength(1); }); @@ -750,7 +763,11 @@ printf '%s %s\\n' '6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc expect(result.status).toBe(137); expect(result.stderr).toContain("APT update was force-killed"); expect(result.stderr).toContain("may have exceeded 300s"); - expect(result.calls.filter((call) => call.startsWith("apt-get "))).toHaveLength(1); + expect(result.calls.filter((call) => call.startsWith("timeout "))).toHaveLength(1); + expect(result.calls.find((call) => call.startsWith("timeout "))).toMatch( + /^timeout -k 10s 300s apt-get /u, + ); + expect(result.calls.filter((call) => call.startsWith("apt-get "))).toHaveLength(0); }); it("rejects a missing Ubuntu source before APT runs (#11320)", () => { From ffb4e2279826c355d0a8233d7bc83092fcd72d7e Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:09:01 -0700 Subject: [PATCH 5/7] test(ci): guard base-pinned CLI installer checkout Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .github/workflows/pr.yaml | 2 ++ ci/source-shape-test-budget.json | 5 +++++ .../pr-workflow-contract.test.ts | 21 +++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 90621c45fb7..6d8e05082f7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -582,6 +582,8 @@ jobs: - name: Checkout trusted CI actions uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + # PR shards run the base action. The revised action and sibling installer + # first run together after both land on main. ref: ${{ github.event.pull_request.base.sha }} path: .trusted-ci-actions persist-credentials: false diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 03ac70a1f6b..531423d130e 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -346,6 +346,11 @@ "test": "keeps %s plugin test typechecking after the trusted production build", "category": "security" }, + { + "file": "test/automation/pull-requests/pr-workflow-contract.test.ts", + "test": "keeps the CLI shard installer with its base-pinned trusted action (#11320)", + "category": "security" + }, { "file": "test/automation/pull-requests/pr-workflow-contract.test.ts", "test": "passes only the base-packaged SDK archive to pull request dependency jobs", diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index cd71c6e1ca8..b56b8d17c74 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -623,6 +623,27 @@ printf '%s %s\\n' '6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc ); }, ); + + // source-shape-contract: security -- The CLI shard action and installer must be selected from one base-pinned trusted checkout + it("keeps the CLI shard installer with its base-pinned trusted action (#11320)", () => { + const job = prWorkflow.jobs["cli-test-shards"]; + const checkout = requiredWorkflowStep(job, "Checkout trusted CI actions"); + expect(checkout.with?.ref).toBe("${{ github.event.pull_request.base.sha }}"); + expect(checkout.with?.path).toBe(".trusted-ci-actions"); + const sparsePaths = String(checkout.with?.["sparse-checkout"] ?? "") + .trim() + .split(/\s+/u); + expect(sparsePaths).toContain(".github/actions/ci-cli-coverage-shard"); + expect(sparsePaths).toContain(".github/actions/ci-install-pinned-ubuntu-packages.sh"); + expect(requiredWorkflowStep(job, "Run CLI coverage shard").uses).toBe( + "./.trusted-ci-actions/.github/actions/ci-cli-coverage-shard", + ); + expect( + requiredStep(sharedActions.cliCoverageShard, "Install pinned Pi search tools").run, + ).toContain('bash "$GITHUB_ACTION_PATH/../ci-install-pinned-ubuntu-packages.sh"'); + expect(existsSync(".github/actions/ci-install-pinned-ubuntu-packages.sh")).toBe(true); + }); + it.each([ ["CLI shards", requiredStep(sharedActions.cliCoverageShard, "Install pinned Pi search tools")], [ From 2bc20c61f47a79e16d10ceb3208bbb82f73d1f05 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:11:14 -0700 Subject: [PATCH 6/7] chore(ci): retain trusted source-shape policy Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- ci/source-shape-test-budget.json | 5 ----- .../pr-workflow-contract.test.ts | 20 ------------------- 2 files changed, 25 deletions(-) diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 531423d130e..03ac70a1f6b 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -346,11 +346,6 @@ "test": "keeps %s plugin test typechecking after the trusted production build", "category": "security" }, - { - "file": "test/automation/pull-requests/pr-workflow-contract.test.ts", - "test": "keeps the CLI shard installer with its base-pinned trusted action (#11320)", - "category": "security" - }, { "file": "test/automation/pull-requests/pr-workflow-contract.test.ts", "test": "passes only the base-packaged SDK archive to pull request dependency jobs", diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index b56b8d17c74..6189f22b3ac 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -624,26 +624,6 @@ printf '%s %s\\n' '6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc }, ); - // source-shape-contract: security -- The CLI shard action and installer must be selected from one base-pinned trusted checkout - it("keeps the CLI shard installer with its base-pinned trusted action (#11320)", () => { - const job = prWorkflow.jobs["cli-test-shards"]; - const checkout = requiredWorkflowStep(job, "Checkout trusted CI actions"); - expect(checkout.with?.ref).toBe("${{ github.event.pull_request.base.sha }}"); - expect(checkout.with?.path).toBe(".trusted-ci-actions"); - const sparsePaths = String(checkout.with?.["sparse-checkout"] ?? "") - .trim() - .split(/\s+/u); - expect(sparsePaths).toContain(".github/actions/ci-cli-coverage-shard"); - expect(sparsePaths).toContain(".github/actions/ci-install-pinned-ubuntu-packages.sh"); - expect(requiredWorkflowStep(job, "Run CLI coverage shard").uses).toBe( - "./.trusted-ci-actions/.github/actions/ci-cli-coverage-shard", - ); - expect( - requiredStep(sharedActions.cliCoverageShard, "Install pinned Pi search tools").run, - ).toContain('bash "$GITHUB_ACTION_PATH/../ci-install-pinned-ubuntu-packages.sh"'); - expect(existsSync(".github/actions/ci-install-pinned-ubuntu-packages.sh")).toBe(true); - }); - it.each([ ["CLI shards", requiredStep(sharedActions.cliCoverageShard, "Install pinned Pi search tools")], [ From f2ca3059c251e7238612eddcc89be1dd14625982 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:12:47 -0700 Subject: [PATCH 7/7] chore(ci): remove review-only test spacing Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- test/automation/pull-requests/pr-workflow-contract.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index 6189f22b3ac..cd71c6e1ca8 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -623,7 +623,6 @@ printf '%s %s\\n' '6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc ); }, ); - it.each([ ["CLI shards", requiredStep(sharedActions.cliCoverageShard, "Install pinned Pi search tools")], [