From 751ae861e0a09d31c1e9bdb281a947571ab48755 Mon Sep 17 00:00:00 2001 From: El-Fitz <8971906+El-Fitz@users.noreply.github.com> Date: Mon, 1 Jun 2026 23:17:57 +0200 Subject: [PATCH 1/4] ci: add Buildkite Tart pipeline --- .buildkite/pipeline.yml | 89 +++++++++++++++++++++++++++++++++++++++++ docs/HANDOFF.md | 8 +++- 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 .buildkite/pipeline.yml diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000..5143045 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,89 @@ +env: + CARGO_TERM_COLOR: always + +steps: + - label: ":linux: Linux ARM64" + key: linux-arm64 + concurrency: 2 + concurrency_group: "big-cabbage/tart-ci" + agents: + queue: "ci-linux-arm64" + plugins: + - github.com/Bande-a-Bonnot/tart-ci#v0.1.1: + image: "ci-linux-arm64-rust-bazel" + os: "linux" + always_pull: false + headless: true + command: | + set -euo pipefail + + if [[ -z "$${HOME:-}" || "$$HOME" == "/" ]]; then + export HOME=/home/admin + fi + export CARGO_HOME="$${CARGO_HOME:-$$HOME/.cargo}" + export RUSTUP_HOME="$${RUSTUP_HOME:-/opt/rustup}" + export PATH="/usr/local/bin:/opt/cargo/bin:$$CARGO_HOME/bin:$$PATH" + + rustc --version + cargo --version + python3 --version + + cd middens + cargo fmt --check + cargo test --locked + cargo build --release --locked + + smoke_output="$$(mktemp -d)" + smoke_xdg="$$(mktemp -d)" + XDG_DATA_HOME="$$smoke_xdg" ./target/release/middens analyze tests/fixtures --split --no-python --output "$$smoke_output" + test -f "$$smoke_output/interactive/markov.json" + test -f "$$smoke_output/subagent/markov.json" + test -f "$$smoke_output/autonomous/markov.json" + + - label: ":mac: macOS Apple Silicon" + key: macos-apple-silicon + concurrency: 2 + concurrency_group: "big-cabbage/tart-ci" + agents: + queue: "ci-macos-apple-silicon" + plugins: + - github.com/Bande-a-Bonnot/tart-ci#v0.1.1: + image: "ghcr.io/cirruslabs/macos-sequoia-base:latest" + os: "macos" + always_pull: false + headless: true + command: | + set -euo pipefail + + if [[ -z "$${HOME:-}" ]]; then + export HOME=/Users/admin + fi + export CARGO_HOME="$${CARGO_HOME:-$$HOME/.cargo}" + export RUSTUP_HOME="$${RUSTUP_HOME:-$$HOME/.rustup}" + export PATH="$$CARGO_HOME/bin:/opt/homebrew/bin:/usr/local/bin:$$PATH" + + if ! command -v cargo >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | HOME="$$HOME" CARGO_HOME="$$CARGO_HOME" RUSTUP_HOME="$$RUSTUP_HOME" sh -s -- -y --profile minimal --no-modify-path + . "$$CARGO_HOME/env" + fi + + if command -v rustup >/dev/null 2>&1; then + rustup component add rustfmt + fi + + rustc --version + cargo --version + python3 --version + + cd middens + cargo fmt --check + cargo test --locked + cargo build --release --locked + + smoke_output="$$(mktemp -d)" + smoke_xdg="$$(mktemp -d)" + XDG_DATA_HOME="$$smoke_xdg" ./target/release/middens analyze tests/fixtures --split --no-python --output "$$smoke_output" + test -f "$$smoke_output/interactive/markov.json" + test -f "$$smoke_output/subagent/markov.json" + test -f "$$smoke_output/autonomous/markov.json" diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 305da4a..d29fde9 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -1,11 +1,17 @@ # Session Handoff -**Last updated:** 2026-05-26 (SWE-chat metadata per-user analysis completed) +**Last updated:** 2026-06-01 (Buildkite/Tart CI PR prepared) Read this at the start of every session. Update before compaction or at natural milestones. --- +## Current session update — 2026-06-01 + +Started Buildkite/Tart CI migration on branch `chore/buildkite-tart-ci`. Added `.buildkite/pipeline.yml` with two non-credentialed PR jobs using `github.com/Bande-a-Bonnot/tart-ci#v0.1.1`: Linux ARM64 on local image `ci-linux-arm64-rust-bazel`, and macOS Apple Silicon on `ghcr.io/cirruslabs/macos-sequoia-base:latest` with Rust bootstrapped by rustup. The first slice intentionally covers only lightweight Middens checks: `cargo fmt --check`, `cargo test --locked`, `cargo build --release --locked`, and `middens analyze tests/fixtures --split --no-python` with checks for all three stratum outputs. HF corpus analysis, scheduled workflows, release artifacts, x86_64 targets, Hugging Face registry/secrets, and large network downloads remain on GitHub Actions/deferred. + +Local validation before PR: `cd middens && cargo test` passed (376 scenarios / 2082 steps plus 1 doctest), `cargo build --release --locked` passed, fixture split/no-python smoke passed, Buildkite YAML parsed with Ruby, `bash -n` passed over both embedded commands after Buildkite dollar escaping, and `git diff --check` passed. `cargo clippy --all-targets --locked -- -D warnings` was tried but is not included in the first CI slice because current `main` has many pre-existing clippy warnings under the local/newer toolchain. Remote `buildkite-agent pipeline upload --dry-run` on `big-cabbage` was attempted but blocked by intermittent SSH publickey auth from this session; no host/config/image changes were made. + ## >>> Read this first <<< **All 23 techniques are working. Zero errors, zero timeouts on 13,423 sessions (commit `867f57b`).** From 7f483c98637dda7ef7c0eef740495884097a4b2c Mon Sep 17 00:00:00 2001 From: El-Fitz <8971906+El-Fitz@users.noreply.github.com> Date: Mon, 1 Jun 2026 23:23:07 +0200 Subject: [PATCH 2/4] test: apply rustfmt after main merge --- docs/HANDOFF.md | 2 +- middens/tests/steps/split.rs | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 598ae7d..2953eb1 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -10,7 +10,7 @@ Read this at the start of every session. Update before compaction or at natural Started Buildkite/Tart CI migration on branch `chore/buildkite-tart-ci`. Added `.buildkite/pipeline.yml` with two non-credentialed PR jobs using `github.com/Bande-a-Bonnot/tart-ci#v0.1.1`: Linux ARM64 on local image `ci-linux-arm64-rust-bazel`, and macOS Apple Silicon on `ghcr.io/cirruslabs/macos-sequoia-base:latest` with Rust bootstrapped by rustup. The first slice intentionally covers only lightweight Middens checks: `cargo fmt --check`, `cargo test --locked`, `cargo build --release --locked`, and `middens analyze tests/fixtures --split --no-python` with checks for all three stratum outputs. HF corpus analysis, scheduled workflows, release artifacts, x86_64 targets, Hugging Face registry/secrets, and large network downloads remain on GitHub Actions/deferred. -Local validation before PR: `cd middens && cargo test` passed (376 scenarios / 2082 steps plus 1 doctest), `cargo build --release --locked` passed, fixture split/no-python smoke passed, Buildkite YAML parsed with Ruby, `bash -n` passed over both embedded commands after Buildkite dollar escaping, and `git diff --check` passed. `cargo clippy --all-targets --locked -- -D warnings` was tried but is not included in the first CI slice because current `main` has many pre-existing clippy warnings under the local/newer toolchain. Remote `buildkite-agent pipeline upload --dry-run` on `big-cabbage` was attempted but blocked by intermittent SSH publickey auth from this session; no host/config/image changes were made. +Local validation before PR, after merging latest `origin/main`: `cd middens && cargo fmt --check`, `cargo test --locked` (377 scenarios / 2087 steps plus 1 doctest), `cargo build --release --locked`, fixture split/no-python smoke, Buildkite YAML parse with Ruby, `bash -n` over both embedded commands after Buildkite dollar escaping, and `git diff --check` all passed. A small rustfmt-only fix was included for `middens/tests/steps/split.rs` because latest `main` had one formatting drift that would fail the new CI slice. `cargo clippy --all-targets --locked -- -D warnings` was tried before merging latest `main` but is not included in the first CI slice because current code has many pre-existing clippy warnings under the local/newer toolchain. Remote `buildkite-agent pipeline upload --dry-run` on `big-cabbage` was attempted but blocked by intermittent SSH publickey auth from this session; no host/config/image changes were made. ## >>> Read this first <<< diff --git a/middens/tests/steps/split.rs b/middens/tests/steps/split.rs index bb3ac08..4eb3c62 100644 --- a/middens/tests/steps/split.rs +++ b/middens/tests/steps/split.rs @@ -206,7 +206,13 @@ fn when_run_middens_analyze_with_split_on_the_mixed_corpus_using_technique( run_analyze_with_args( world, true, - &["--techniques", technique.as_str(), "--timeout", "1800", "--force"], + &[ + "--techniques", + technique.as_str(), + "--timeout", + "1800", + "--force", + ], ); } From 7aae929f70d8d073413a3e7a36793c50bc021825 Mon Sep 17 00:00:00 2001 From: El-Fitz <8971906+El-Fitz@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:24:39 +0200 Subject: [PATCH 3/4] ci: accept downloaded site-data artifact paths --- .github/workflows/hf-corpus-analysis.yml | 2 +- docs/HANDOFF.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/hf-corpus-analysis.yml b/.github/workflows/hf-corpus-analysis.yml index bec5cf3..6101ceb 100644 --- a/.github/workflows/hf-corpus-analysis.yml +++ b/.github/workflows/hf-corpus-analysis.yml @@ -391,7 +391,7 @@ jobs: cp "$corpus_dir/interpretation.md" "site-data/corpora/$corpus_id/" fi found=$((found + 1)) - done < <(find .tmp/public-results-artifacts -path '*/.tmp/site-data/corpora/*/metrics.json' | sort) + done < <(find .tmp/public-results-artifacts \( -path '*/site-data/corpora/*/metrics.json' -o -path '*/.tmp/site-data/corpora/*/metrics.json' \) | sort) if [ "$found" -eq 0 ]; then echo "No public-safe site-data bundles found in downloaded artifacts" >&2 exit 1 diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 2953eb1..7b574ff 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -10,7 +10,7 @@ Read this at the start of every session. Update before compaction or at natural Started Buildkite/Tart CI migration on branch `chore/buildkite-tart-ci`. Added `.buildkite/pipeline.yml` with two non-credentialed PR jobs using `github.com/Bande-a-Bonnot/tart-ci#v0.1.1`: Linux ARM64 on local image `ci-linux-arm64-rust-bazel`, and macOS Apple Silicon on `ghcr.io/cirruslabs/macos-sequoia-base:latest` with Rust bootstrapped by rustup. The first slice intentionally covers only lightweight Middens checks: `cargo fmt --check`, `cargo test --locked`, `cargo build --release --locked`, and `middens analyze tests/fixtures --split --no-python` with checks for all three stratum outputs. HF corpus analysis, scheduled workflows, release artifacts, x86_64 targets, Hugging Face registry/secrets, and large network downloads remain on GitHub Actions/deferred. -Local validation before PR, after merging latest `origin/main`: `cd middens && cargo fmt --check`, `cargo test --locked` (377 scenarios / 2087 steps plus 1 doctest), `cargo build --release --locked`, fixture split/no-python smoke, Buildkite YAML parse with Ruby, `bash -n` over both embedded commands after Buildkite dollar escaping, and `git diff --check` all passed. A small rustfmt-only fix was included for `middens/tests/steps/split.rs` because latest `main` had one formatting drift that would fail the new CI slice. `cargo clippy --all-targets --locked -- -D warnings` was tried before merging latest `main` but is not included in the first CI slice because current code has many pre-existing clippy warnings under the local/newer toolchain. Remote `buildkite-agent pipeline upload --dry-run` on `big-cabbage` was attempted but blocked by intermittent SSH publickey auth from this session; no host/config/image changes were made. +Local validation before PR, after merging latest `origin/main`: `cd middens && cargo fmt --check`, `cargo test --locked` (377 scenarios / 2087 steps plus 1 doctest), `cargo build --release --locked`, fixture split/no-python smoke, Buildkite YAML parse with Ruby, `bash -n` over both embedded commands after Buildkite dollar escaping, and `git diff --check` all passed. A small rustfmt-only fix was included for `middens/tests/steps/split.rs` because latest `main` had one formatting drift that would fail the new CI slice. `cargo clippy --all-targets --locked -- -D warnings` was tried before merging latest `main` but is not included in the first CI slice because current code has many pre-existing clippy warnings under the local/newer toolchain. Remote `buildkite-agent pipeline upload --dry-run` on `big-cabbage` was attempted but blocked by intermittent SSH publickey auth from this session; no host/config/image changes were made. Existing GitHub Actions `Build public results site` initially failed on PR #3; Azure-backed log/artifact downloads were unreachable from this workstation, but workflow inspection found a likely artifact path mismatch where downloaded artifacts may contain `site-data/corpora/...` instead of `.tmp/site-data/corpora/...`. The PR branch now accepts both path shapes in the collector; workflow YAML and a local `find` fixture validated. ## >>> Read this first <<< From c74118e8cbf43619e776d9b6b804078f1825a17e Mon Sep 17 00:00:00 2001 From: El-Fitz <8971906+El-Fitz@users.noreply.github.com> Date: Tue, 9 Jun 2026 21:49:44 +0200 Subject: [PATCH 4/4] ci: keep Tart smoke lightweight --- .buildkite/pipeline.yml | 6 ++++-- docs/HANDOFF.md | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 5143045..c491a16 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -30,7 +30,8 @@ steps: cd middens cargo fmt --check - cargo test --locked + cargo test --locked --lib + cargo test --locked --test cucumber -- --input tests/features/smoke.feature cargo build --release --locked smoke_output="$$(mktemp -d)" @@ -78,7 +79,8 @@ steps: cd middens cargo fmt --check - cargo test --locked + cargo test --locked --lib + cargo test --locked --test cucumber -- --input tests/features/smoke.feature cargo build --release --locked smoke_output="$$(mktemp -d)" diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 54da164..c1ae4eb 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -8,9 +8,9 @@ Read this at the start of every session. Update before compaction or at natural ## Current session update — 2026-06-01 -Started Buildkite/Tart CI migration on branch `chore/buildkite-tart-ci`. Added `.buildkite/pipeline.yml` with two non-credentialed PR jobs using `github.com/Bande-a-Bonnot/tart-ci#v0.1.1`: Linux ARM64 on local image `ci-linux-arm64-rust-bazel`, and macOS Apple Silicon on `ghcr.io/cirruslabs/macos-sequoia-base:latest` with Rust bootstrapped by rustup. The first slice intentionally covers only lightweight Middens checks: `cargo fmt --check`, `cargo test --locked`, `cargo build --release --locked`, and `middens analyze tests/fixtures --split --no-python` with checks for all three stratum outputs. HF corpus analysis, scheduled workflows, release artifacts, x86_64 targets, Hugging Face registry/secrets, and large network downloads remain on GitHub Actions/deferred. +Started Buildkite/Tart CI migration on branch `chore/buildkite-tart-ci`. Added `.buildkite/pipeline.yml` with two non-credentialed PR jobs using `github.com/Bande-a-Bonnot/tart-ci#v0.1.1`: Linux ARM64 on local image `ci-linux-arm64-rust-bazel`, and macOS Apple Silicon on `ghcr.io/cirruslabs/macos-sequoia-base:latest` with Rust bootstrapped by rustup. The first slice intentionally covers lightweight Middens cross-platform checks: `cargo fmt --check`, `cargo test --locked --lib`, the Cucumber harness smoke feature, `cargo build --release --locked`, and `middens analyze tests/fixtures --split --no-python` with checks for all three stratum outputs. The full Cucumber/HF corpus batteries, scheduled workflows, release artifacts, x86_64 targets, Hugging Face registry/secrets, and large network downloads remain on GitHub Actions/deferred. -Local validation before PR, after merging latest `origin/main`: `cd middens && cargo fmt --check`, `cargo test --locked` (377 scenarios / 2087 steps plus 1 doctest), `cargo build --release --locked`, fixture split/no-python smoke, Buildkite YAML parse with Ruby, `bash -n` over both embedded commands after Buildkite dollar escaping, and `git diff --check` all passed. A small rustfmt-only fix was included for `middens/tests/steps/split.rs` because latest `main` had one formatting drift that would fail the new CI slice. `cargo clippy --all-targets --locked -- -D warnings` was tried before merging latest `main` but is not included in the first CI slice because current code has many pre-existing clippy warnings under the local/newer toolchain. Remote `buildkite-agent pipeline upload --dry-run` on `big-cabbage` was attempted but blocked by intermittent SSH publickey auth from this session; no host/config/image changes were made. Existing GitHub Actions `Build public results site` initially failed on PR #3; Azure-backed log/artifact downloads were unreachable from this workstation, but workflow inspection found a likely artifact path mismatch where downloaded artifacts may contain `site-data/corpora/...` instead of `.tmp/site-data/corpora/...`. The PR branch now accepts both path shapes in the collector; workflow YAML and a local `find` fixture validated. +Local validation before PR, after merging latest `origin/main`: `cd middens && cargo fmt --check`, `cargo test --locked --lib` (31 unit tests), Cucumber smoke feature, `cargo build --release --locked`, fixture split/no-python smoke, Buildkite YAML parse with Ruby, `bash -n` over both embedded commands after Buildkite dollar escaping, and `git diff --check` all passed. Full `cargo test --locked` was attempted after the latest main merge but exceeded the local harness timeout twice (>20 minutes), so it is intentionally left to the existing GitHub HF/full test workflow rather than the first Tart smoke slice. A small rustfmt-only fix was included for `middens/tests/steps/split.rs` because latest `main` had one formatting drift that would fail the new CI slice. `cargo clippy --all-targets --locked -- -D warnings` was tried before merging latest `main` but is not included in the first CI slice because current code has many pre-existing clippy warnings under the local/newer toolchain. Remote `buildkite-agent pipeline upload --dry-run` on `big-cabbage` was attempted but blocked by intermittent SSH publickey auth from this session; no host/config/image changes were made. Existing GitHub Actions `Build public results site` initially failed on PR #3; Azure-backed log/artifact downloads were unreachable from this workstation, but workflow inspection found a likely artifact path mismatch where downloaded artifacts may contain `site-data/corpora/...` instead of `.tmp/site-data/corpora/...`. The PR branch now accepts both path shapes in the collector; workflow YAML and a local `find` fixture validated. ## >>> Read this first <<<