diff --git a/.github/repo-settings.json b/.github/repo-settings.json
index d26c38da4..1f254f0bb 100644
--- a/.github/repo-settings.json
+++ b/.github/repo-settings.json
@@ -86,6 +86,9 @@
{
"context": "source-shape"
},
+ {
+ "context": "ci/measurements-report"
+ },
{
"context": "test-integration-notion"
},
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3a5a458e8..efb13ed95 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2963,6 +2963,38 @@ jobs:
run: |
__genie_ci_retry_script='${{ runner.temp }}/genie-ci-scripts/run-with-nix-gc-race-retry.sh'
bash "$__genie_ci_retry_script" 'devenv tasks run buck2:check' 'if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" PNPM_HOME="${PNPM_HOME:-${{ github.workspace }}/.pnpm-home}" PNPM_STORE_DIR="${PNPM_STORE_DIR:-${{ github.workspace }}/.pnpm-store}" PNPM_CONFIG_STORE_DIR="${PNPM_CONFIG_STORE_DIR:-${{ github.workspace }}/.pnpm-store}" DEVENV_TASK_PASSTHROUGH=1 DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run buck2:check'
+ - name: 'Adapt and admit Buck measurements: Megarepo CLI'
+ if: always()
+ shell: bash
+ env:
+ CI_MEASUREMENT_BUCK_TARGET: '{"id":"megarepo-mr","label":"Megarepo CLI","workContract":"megarepo-cli-product/no-equivalent-devenv-lane/v1","benchmarkSchema":"effect-utils-buck2-benchmark/v0","buckTarget":"//packages/@overeng/megarepo:mr","rawPath":"tmp/buck2-benchmark/megarepo-mr.jsonl","runs":7,"assertions":[{"id":"warm-actions","label":"Warm actions","phase":"warm-noop","metric":"actionCount","expectation":{"_tag":"exact","value":0}},{"id":"warm-materializations","label":"Warm materializations","phase":"warm-noop","metric":"materializationCount","expectation":{"_tag":"exact","value":0}},{"id":"mtime-actions","label":"Mtime-only actions","phase":"mtime-only","metric":"actionCount","expectation":{"_tag":"exact","value":0}},{"id":"irrelevant-actions","label":"Role-excluded actions","phase":"irrelevant-edit","metric":"actionCount","expectation":{"_tag":"exact","value":0}},{"id":"irrelevant-materializations","label":"Role-excluded materializations","phase":"irrelevant-edit","metric":"materializationCount","expectation":{"_tag":"exact","value":0}},{"id":"relevant-actions","label":"Relevant-edit actions","phase":"relevant-edit","metric":"actionCount","expectation":{"_tag":"at-least","value":1}},{"id":"declared-unreachable-actions","label":"Declared-unreachable actions","phase":"declared-unreachable-edit","metric":"actionCount","expectation":{"_tag":"at-least","value":1}}]}'
+ CI_MEASUREMENT_SUBJECT_SHA: ${{ inputs.measurement_baseline_ref || github.event.pull_request.head.sha || github.sha }}
+ CI_MEASUREMENT_CHECKOUT_SHA: ${{ github.sha }}
+ run: |
+ set -euo pipefail
+ output="tmp/buck2-ci-measurements/megarepo-mr/measurements.json"
+ DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" shell --no-reload -- bun scripts/buck2-benchmark/assert-invalidation.mjs --contract-json "$CI_MEASUREMENT_BUCK_TARGET" --output "$output" "tmp/buck2-benchmark/megarepo-mr.jsonl"
+ - name: 'Adapt and admit Buck measurements: OTel scrape native product'
+ if: always()
+ shell: bash
+ env:
+ CI_MEASUREMENT_BUCK_TARGET: '{"id":"otel-scrape-product","label":"OTel scrape native product","workContract":"effect-utils/otel-scrape-native-product-v1","benchmarkSchema":"effect-utils-buck2-benchmark/v0","buckTarget":"//packages/@overeng/otel-scrape:product","rawPath":"tmp/buck2-benchmark/otel-scrape-product.jsonl","runs":7,"assertions":[{"id":"warm-actions","label":"Warm actions","phase":"warm-noop","metric":"actionCount","expectation":{"_tag":"exact","value":0}},{"id":"warm-materializations","label":"Warm materializations","phase":"warm-noop","metric":"materializationCount","expectation":{"_tag":"exact","value":0}},{"id":"irrelevant-actions","label":"Irrelevant-edit actions","phase":"irrelevant-edit","metric":"actionCount","expectation":{"_tag":"exact","value":0}},{"id":"irrelevant-materializations","label":"Irrelevant-edit materializations","phase":"irrelevant-edit","metric":"materializationCount","expectation":{"_tag":"exact","value":0}},{"id":"relevant-actions","label":"Relevant-edit actions","phase":"relevant-edit","metric":"actionCount","expectation":{"_tag":"exact","value":2}}]}'
+ CI_MEASUREMENT_SUBJECT_SHA: ${{ inputs.measurement_baseline_ref || github.event.pull_request.head.sha || github.sha }}
+ CI_MEASUREMENT_CHECKOUT_SHA: ${{ github.sha }}
+ run: |
+ set -euo pipefail
+ output="tmp/buck2-ci-measurements/otel-scrape-product/measurements.json"
+ DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" shell --no-reload -- bun scripts/buck2-benchmark/assert-invalidation.mjs --contract-json "$CI_MEASUREMENT_BUCK_TARGET" --output "$output" "tmp/buck2-benchmark/otel-scrape-product.jsonl"
+ - name: 'Upload CI measurements: buck2-measurements'
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: buck2-measurements
+ path: |
+ tmp/buck2-ci-measurements
+ !tmp/buck2-ci-measurements/baseline/**
+ if-no-files-found: error
+ retention-days: 14
- name: Save pnpm state
if: ${{ success() && steps.restore-pnpm-state.outputs.cache-hit != 'true' }}
uses: actions/cache/save@v4
@@ -5668,8 +5700,8 @@ jobs:
cancel-in-progress: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') && (github.event_name != 'pull_request' || (github.event.action != 'labeled' && github.event.action != 'unlabeled')) }}
ci-measurements-report:
name: ci/measurements-report
- if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') }}
- needs: [devenv-perf, nix-closure-sizes, source-shape]
+ if: ${{ always() && !(github.event_name == 'workflow_dispatch' && inputs.measurement_baseline_ref != '') }}
+ needs: [buck2, devenv-perf, nix-closure-sizes, source-shape]
runs-on:
[namespace-profile-linux-x86-64, 'namespace-features:github.run-id=${{ github.run_id }}']
timeout-minutes: 30
@@ -5698,28 +5730,37 @@ jobs:
extra-trusted-public-keys = devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw=
access-tokens = github.com=${{ github.token }}
summarize: true
- - name: Provide CI measurement report tools
+ - name: Provide CI measurement comparison tool
shell: bash
run: |
set -euo pipefail
- for out in $(nix build --no-link --print-out-paths nixpkgs#jq nixpkgs#nodejs nixpkgs#gh nixpkgs#resvg); do
+ for out in $(nix build --no-link --print-out-paths nixpkgs#jq); do
echo "$out/bin" >> "$GITHUB_PATH"
done
+ - name: 'Download current measurement artifact: buck2-measurements'
+ uses: actions/download-artifact@v4
+ with:
+ name: buck2-measurements
+ path: tmp/ci-measurement-report/current/buck2-measurements
+ continue-on-error: true
- name: 'Download current measurement artifact: devenv-perf'
uses: actions/download-artifact@v4
with:
name: devenv-perf
path: tmp/ci-measurement-report/current/devenv-perf
+ continue-on-error: true
- name: 'Download current measurement artifact: nix-closure-measurements'
uses: actions/download-artifact@v4
with:
name: nix-closure-measurements
path: tmp/ci-measurement-report/current/nix-closure-measurements
+ continue-on-error: true
- name: 'Download current measurement artifact: source-shape'
uses: actions/download-artifact@v4
with:
name: source-shape
path: tmp/ci-measurement-report/current/source-shape
+ continue-on-error: true
- name: 'Download previous artifact: devenv-perf'
shell: bash
env:
@@ -6510,6 +6551,7 @@ jobs:
CI_MEASUREMENT_BASELINE_DIR: tmp/ci-measurement-report/baseline
CI_MEASUREMENT_COMPARISON_FILE: tmp/ci-measurement-report/measurement-comparison.json
CI_MEASUREMENT_REGRESSION_MODE: warn
+ CI_MEASUREMENT_ASSERTION_TARGETS: '[{"id":"megarepo-mr","label":"Megarepo CLI","workContract":"megarepo-cli-product/no-equivalent-devenv-lane/v1","benchmarkSchema":"effect-utils-buck2-benchmark/v0","buckTarget":"//packages/@overeng/megarepo:mr","rawPath":"tmp/buck2-benchmark/megarepo-mr.jsonl","runs":7,"assertions":[{"id":"warm-actions","label":"Warm actions","phase":"warm-noop","metric":"actionCount","expectation":{"_tag":"exact","value":0}},{"id":"warm-materializations","label":"Warm materializations","phase":"warm-noop","metric":"materializationCount","expectation":{"_tag":"exact","value":0}},{"id":"mtime-actions","label":"Mtime-only actions","phase":"mtime-only","metric":"actionCount","expectation":{"_tag":"exact","value":0}},{"id":"irrelevant-actions","label":"Role-excluded actions","phase":"irrelevant-edit","metric":"actionCount","expectation":{"_tag":"exact","value":0}},{"id":"irrelevant-materializations","label":"Role-excluded materializations","phase":"irrelevant-edit","metric":"materializationCount","expectation":{"_tag":"exact","value":0}},{"id":"relevant-actions","label":"Relevant-edit actions","phase":"relevant-edit","metric":"actionCount","expectation":{"_tag":"at-least","value":1}},{"id":"declared-unreachable-actions","label":"Declared-unreachable actions","phase":"declared-unreachable-edit","metric":"actionCount","expectation":{"_tag":"at-least","value":1}}],"fingerprint":"9c3ebecfad98051362f3ae2e17f9d5344ffba3e17dba028df5aff079cfe3c5c5"},{"id":"otel-scrape-product","label":"OTel scrape native product","workContract":"effect-utils/otel-scrape-native-product-v1","benchmarkSchema":"effect-utils-buck2-benchmark/v0","buckTarget":"//packages/@overeng/otel-scrape:product","rawPath":"tmp/buck2-benchmark/otel-scrape-product.jsonl","runs":7,"assertions":[{"id":"warm-actions","label":"Warm actions","phase":"warm-noop","metric":"actionCount","expectation":{"_tag":"exact","value":0}},{"id":"warm-materializations","label":"Warm materializations","phase":"warm-noop","metric":"materializationCount","expectation":{"_tag":"exact","value":0}},{"id":"irrelevant-actions","label":"Irrelevant-edit actions","phase":"irrelevant-edit","metric":"actionCount","expectation":{"_tag":"exact","value":0}},{"id":"irrelevant-materializations","label":"Irrelevant-edit materializations","phase":"irrelevant-edit","metric":"materializationCount","expectation":{"_tag":"exact","value":0}},{"id":"relevant-actions","label":"Relevant-edit actions","phase":"relevant-edit","metric":"actionCount","expectation":{"_tag":"exact","value":2}}],"fingerprint":"b0035374449996dfe6adcea2e6b132ce2aef8e8a5dfe9b9a5aba443e5cd3ba48"}]'
CI_MEASUREMENT_PR_COMMENT_ENABLED: 'true'
CI_MEASUREMENT_PR_COMMENT_TITLE: CI Measurements
CI_MEASUREMENT_PR_COMMENT_MAX_ROWS: '16'
@@ -6526,6 +6568,7 @@ jobs:
baseline_dir="${CI_MEASUREMENT_BASELINE_DIR:?CI_MEASUREMENT_BASELINE_DIR not set}"
comparison_file="${CI_MEASUREMENT_COMPARISON_FILE:?CI_MEASUREMENT_COMPARISON_FILE not set}"
mode="${CI_MEASUREMENT_REGRESSION_MODE:-warn}"
+ assertion_targets="${CI_MEASUREMENT_ASSERTION_TARGETS:-[]}"
mkdir -p "$(dirname "$comparison_file")"
if [ "$mode" = "off" ]; then
@@ -6543,13 +6586,46 @@ jobs:
} | sort -u >"$baseline_index" || true
if [ ! -s "$current_index" ]; then
- echo "::error::no current measurements.json files found under $current_dir"
- exit 1
+ if [ "$assertion_targets" = "[]" ]; then
+ echo "::error::no current measurements.json files found under $current_dir"
+ exit 1
+ fi
+ echo "::warning::no current measurements.json files found; reconstructing required assertion rows"
fi
current_json="$comparison_file.current.json"
baseline_json="$comparison_file.baseline.json"
- xargs -r jq -s '.' <"$current_index" >"$current_json"
+ if [ -s "$current_index" ]; then
+ verified_current_dir="$(mktemp -d)"
+ verified_index="$(mktemp)"
+ verified_count=0
+ while IFS= read -r measurement_file; do
+ verified_count=$((verified_count + 1))
+ raw_file="$(dirname "$measurement_file")/raw.jsonl"
+ raw_sha256=""
+ raw_parse_status="missing"
+ raw_records_file="$(mktemp)"
+ printf '[]\n' >"$raw_records_file"
+ if [ -f "$raw_file" ]; then
+ raw_sha256="$(sha256sum "$raw_file" | awk '{print $1}')"
+ raw_parse_status="valid"
+ if ! jq -s '.' "$raw_file" >"$raw_records_file" 2>/dev/null; then
+ echo "::warning::raw CI measurement evidence is malformed: $raw_file"
+ raw_parse_status="invalid"
+ printf '[]\n' >"$raw_records_file"
+ fi
+ fi
+ verified_file="$verified_current_dir/$verified_count.json"
+ jq --arg rawSha256 "$raw_sha256" --arg rawParseStatus "$raw_parse_status" --slurpfile rawRecords "$raw_records_file" \
+ '. + {_consumerVerification:{rawSha256:(if $rawSha256 == "" then null else $rawSha256 end),rawParseStatus:$rawParseStatus,rawRecords:($rawRecords[0] // [])}}' \
+ "$measurement_file" >"$verified_file"
+ rm -f "$raw_records_file"
+ printf '%s\n' "$verified_file" >>"$verified_index"
+ done <"$current_index"
+ xargs -r jq -s '.' <"$verified_index" >"$current_json"
+ else
+ printf '[]\n' >"$current_json"
+ fi
if [ -s "$baseline_index" ]; then
xargs -r jq -s '.' <"$baseline_index" >"$baseline_json"
else
@@ -6561,8 +6637,11 @@ jobs:
--slurpfile baseline "$baseline_json" \
--argjson schemaVersion 1 \
--arg mode "$mode" \
+ --argjson assertionTargets "$assertion_targets" \
--arg currentDir "$current_dir" \
--arg baselineDir "$baseline_dir" \
+ --arg expectedSubjectSha "${CI_MEASUREMENT_SUBJECT_SHA:-${GITHUB_SHA:-unknown}}" \
+ --arg expectedCheckoutSha "${CI_MEASUREMENT_CHECKOUT_SHA:-${GITHUB_SHA:-unknown}}" \
'
def identity_dimensions:
(.dimensions // {})
@@ -6602,7 +6681,7 @@ jobs:
def observations_by_key($docs):
reduce $docs[]? as $doc
({};
- reduce (($doc.observations // [])[]? | select(.value | type == "number")) as $obs
+ reduce (($doc.observations // [])[]? | select((.value | type == "number") and (.policy.comparisonMode // "") != "assertion")) as $obs
(.;
($obs | observation_key($doc)) as $key
| .[$key] = ((.[$key] // []) + [{
@@ -6681,6 +6760,113 @@ jobs:
def policy_enabled($policy):
if ($policy | has("enabled")) then $policy.enabled else true end;
+ def expectation_matches($expectation; $value):
+ if $expectation._tag == "exact" then $value == $expectation.value
+ elif $expectation._tag == "at-least" then $value >= $expectation.value
+ elif $expectation._tag == "at-most" then $value <= $expectation.value
+ elif $expectation._tag == "range" then $value >= $expectation.min and $value <= $expectation.max
+ else false
+ end;
+
+ def assertion_comparisons($docs; $targets):
+ reduce $targets[]? as $target
+ ({};
+ ([ $docs[]? | select(.target.kind == "buck2" and .target.id == $target.id) ]) as $matchingDocs
+ | reduce $target.assertions[] as $expected
+ (.;
+ ($matchingDocs[0] // null) as $doc
+ | ([ $doc.observations[]? | select(.id == $expected.id) ]) as $matchingObservations
+ | ($matchingObservations[0] // null) as $observed
+ | ($observed.statistics.samples // []) as $samples
+ | ($observed.evidence.sampleIndexes // []) as $indexes
+ | ($doc._consumerVerification.rawRecords // []) as $rawRecords
+ | ([ $rawRecords[]? | select(.kind == "metadata") ]) as $rawMetadata
+ | ([ $rawRecords[]?
+ | select(
+ .kind == "sample"
+ and .engine == "buck2"
+ and .warmup == false
+ and .workContract == $target.workContract
+ and .phase == $expected.phase
+ )
+ ]) as $rawSamples
+ | ($rawSamples | map(.sampleIndex)) as $rawIndexes
+ | ($rawSamples | map(.[$expected.metric])) as $rawValues
+ | (
+ ($matchingDocs | length) == 1
+ and $doc.schemaVersion == 1
+ and $doc.producer.measurementProtocol == "buck2-invalidation-v1"
+ and $doc.contract.fingerprint == $target.fingerprint
+ and $doc.contract.snapshot == ($target | del(.fingerprint))
+ and $doc.completeness.status == "complete"
+ and $doc.subject.sha == $expectedSubjectSha
+ and $doc.subject.evidenceSha == $expectedCheckoutSha
+ and $doc._consumerVerification.rawParseStatus == "valid"
+ and ($rawMetadata | length) == 1
+ and $rawMetadata[0].sha == $expectedCheckoutSha
+ and $rawMetadata[0].schema == $target.benchmarkSchema
+ and $rawMetadata[0].target == $target.buckTarget
+ and $rawMetadata[0].workContract == $target.workContract
+ and $rawMetadata[0].samplePolicy.runs == $target.runs
+ and ($matchingObservations | length) == 1
+ and $observed.policy.comparisonMode == "assertion"
+ and $observed.policy.expectation == $expected.expectation
+ and $observed.policy.onNoVerdict == "fail"
+ and $observed.name == ("buck2." + (if $expected.metric == "actionCount" then "action_count" else "materialization_count" end))
+ and $observed.unit == "count"
+ and $observed.dimensions.phase == $expected.phase
+ and $observed.dimensions.measurementProtocol == "buck2-invalidation-v1"
+ and $observed.evidence.status == "complete"
+ and $observed.statistics.sampleCount == $target.runs
+ and $observed.statistics.measuredSampleCount == $target.runs
+ and ($samples | length) == $target.runs
+ and ($indexes | length) == $target.runs
+ and (($indexes | unique | sort) == ([range(0; $target.runs)]))
+ and $rawIndexes == $indexes
+ and $rawValues == $samples
+ and ($rawSamples | all(
+ .schema == $target.benchmarkSchema
+ and .target == $target.buckTarget
+ and .sha == $expectedCheckoutSha
+ and .runId == $rawMetadata[0].runId
+ and .status == "ok"
+ and .buckLogStatus == "ok"
+ ))
+ and ($samples | all(type == "number" and isfinite and floor == . and . >= 0))
+ and (($observed.evidence.rawSha256 // "") | test("^[0-9a-f]{64}$"))
+ and $observed.evidence.rawSha256 == $doc._consumerVerification.rawSha256
+ and $observed.assertion.status == (if ($samples | all(expectation_matches($expected.expectation; .))) then "pass" else "fail" end)
+ ) as $complete
+ | ($complete and ($samples | all(expectation_matches($expected.expectation; .)))) as $passing
+ | ("assertion|" + $target.id + "|" + $expected.id) as $key
+ | .[$key] = {
+ status: (if $complete then (if $passing then "pass" else "fail" end) else "no_verdict" end),
+ target: {kind:"buck2", id:$target.id, label:$target.label},
+ observation: ($observed // {
+ id:$expected.id,
+ label:$expected.label,
+ name:("buck2." + (if $expected.metric == "actionCount" then "action_count" else "materialization_count" end)),
+ unit:"count",
+ measurementKind:"deterministic",
+ dimensions:{phase:$expected.phase, measurementProtocol:"buck2-invalidation-v1"},
+ policy:{enabled:true, comparisonMode:"assertion", expectation:$expected.expectation, onNoVerdict:"fail"}
+ }),
+ current: (if ($samples | length) == 0 then null else ($samples | median) end),
+ currentSamples: ($samples | length),
+ baselineSources: 0,
+ gatePolicy: {enabled:true, comparisonMode:"assertion", expectation:$expected.expectation, onNoVerdict:"fail"},
+ comparisonMode:"assertion",
+ gateable:$complete,
+ gateReason:(if $complete then "eligible" else "incomplete_assertion_evidence" end),
+ confidence:(if $complete then (if $passing then "assertion_satisfied" else "assertion_violated" end) else "no_verdict" end),
+ direction:(if $complete and $passing then "unchanged" else "regressed" end),
+ semanticImpactScore:(if $complete and $passing then 0 else 1 end),
+ semanticImpactKind:(if $complete and $passing then "neutral" else "fail_boundary" end),
+ assertion:{expected:$expected.expectation, samples:$samples}
+ }
+ )
+ );
+
def classify($metric; $unit; $measurementKind; $policy; $current; $currentP25; $currentP75; $currentMad; $baseline; $baselineMin; $baselineMax; $baselineP25; $baselineP75; $baselineP95; $baselineMad; $currentSamples; $baselineSources; $pairedSamples; $pairedDeltaMedian; $pairedDeltaP25; $pairedDeltaP75; $pairedDeltaMad; $pairedDeltaValues):
$policy as $b
| ($policy.comparisonMode // (if $measurementKind == "deterministic" or $unit != "seconds" then "budget" elif $measurementKind == "diagnostic" then "diagnostic" else "historical" end)) as $comparisonMode
@@ -6902,10 +7088,13 @@ jobs:
}
)
| from_entries
- ) as $comparisons
+ ) as $legacyComparisons
+ | assertion_comparisons($current[0]; $assertionTargets) as $assertionComparisons
+ | ($legacyComparisons + $assertionComparisons) as $comparisons
| (
if any($comparisons[]?; .status == "fail") then "fail"
elif any($comparisons[]?; .status == "warn") then "warn"
+ elif any($comparisons[]?; .status == "no_verdict") then "partial"
elif any($comparisons[]?;
(if (.gatePolicy | has("enabled")) then .gatePolicy.enabled else true end)
and (.gateReason == "missing_baseline"
@@ -6919,6 +7108,7 @@ jobs:
) as $status
| (
[$comparisons[]?]
+ | . as $rows
| {
enabledCount: (map(select((if (.gatePolicy | has("enabled")) then .gatePolicy.enabled else true end))) | length),
gateableCount: (map(select(.gateable == true)) | length),
@@ -6927,10 +7117,26 @@ jobs:
lowCurrentSampleCount: (map(select(.gateReason == "low_current_sample_count")) | length),
lowPairedSampleCount: (map(select(.gateReason == "low_paired_sample_count")) | length),
missingPairedDeltaCount: (map(select(.gateReason == "missing_paired_delta")) | length)
+ ,noVerdictCount: (map(select(.status == "no_verdict")) | length),
+ assertions: {
+ requiredCount: ($rows | map(select(.comparisonMode == "assertion")) | length),
+ passingCount: ($rows | map(select(.comparisonMode == "assertion" and .status == "pass")) | length),
+ blockingCount: ($rows | map(select(.comparisonMode == "assertion" and (.status == "fail" or .status == "no_verdict"))) | length)
+ },
+ advisory: {
+ enabledCount: ($rows | map(select(.comparisonMode != "assertion" and (if (.gatePolicy | has("enabled")) then .gatePolicy.enabled else true end))) | length),
+ gateableCount: ($rows | map(select(.comparisonMode != "assertion" and .gateable == true)) | length)
+ }
}
| . + {
nonGateableCount: (.enabledCount - .gateableCount),
- enforceable: (.enabledCount == .gateableCount)
+ enforceable: (.enabledCount == .gateableCount),
+ assertions: (.assertions + {
+ status: (if .assertions.requiredCount == 0 then "not_applicable" elif .assertions.blockingCount == 0 and .assertions.passingCount == .assertions.requiredCount then "pass" else "blocking" end)
+ }),
+ advisory: (.advisory + {
+ status: (if .advisory.enabledCount == .advisory.gateableCount then "ready" else "partial" end)
+ })
}
) as $readiness
| {
@@ -6968,11 +7174,16 @@ jobs:
;;
esac
+ if jq -e 'any(.comparisons[]?; .comparisonMode == "assertion" and (.status == "fail" or .status == "no_verdict"))' "$comparison_file" >/dev/null; then
+ echo "::error::required CI measurement assertion failed or has no verdict"
+ exit_code=1
+ fi
+
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
{
echo "### ${CI_MEASUREMENT_PR_COMMENT_TITLE:-CI Measurements}"
echo ""
- jq -r '"- Status: " + .status + "\n- Gate: " + (if .mode == "fail" then "enforced" elif .mode == "warn" then "advisory" elif .mode == "off" then "off" else (.mode // "unknown") end) + "\n- Baseline: " + .baselineDir' "$comparison_file"
+ jq -r '"- Status: " + .status + "\n- Required assertions: " + .readiness.assertions.status + " (" + (.readiness.assertions.passingCount | tostring) + "/" + (.readiness.assertions.requiredCount | tostring) + " passing)\n- Baseline regressions: " + (if .mode == "fail" then "enforced" elif .mode == "warn" then "advisory" elif .mode == "off" then "off" else (.mode // "unknown") end) + "; readiness " + .readiness.advisory.status + " (" + (.readiness.advisory.gateableCount | tostring) + "/" + (.readiness.advisory.enabledCount | tostring) + " gateable)\n- Baseline: " + .baselineDir' "$comparison_file"
echo ""
echo "| Status | Gate | Target | Observation | Current | Baseline | Delta | Ratio |"
echo "| --- | --- | --- | --- | ---: | ---: | ---: | ---: |"
@@ -7007,9 +7218,11 @@ jobs:
if [ "${CI_MEASUREMENT_PR_COMMENT_ENABLED:-false}" = "true" ]; then
if [ "${GITHUB_EVENT_NAME:-}" != "pull_request" ]; then
echo "::notice::CI measurement PR comments are produced only by pull_request workflows; skipping comment for event ${GITHUB_EVENT_NAME:-unknown}"
- exit 0
+ exit "$exit_code"
fi
+ set +e
+
can_render_pr_comment=true
is_fork_pr=false
if [ "${CI_MEASUREMENT_IS_FORK_PR:-}" = "1" ]; then
@@ -7024,33 +7237,27 @@ jobs:
if command -v "$tool_name" >/dev/null 2>&1; then
return 0
fi
- if ! command -v nix >/dev/null 2>&1; then
- return 1
- fi
- if tool_out="$(nix build --no-link --print-out-paths "nixpkgs#$nix_attr" 2>/dev/null)"; then
- export PATH="$tool_out/bin:$PATH"
- fi
- command -v "$tool_name" >/dev/null 2>&1
+ return 1
}
if ! ensure_ci_measurement_tool gh gh; then
- echo "::error::gh is not available; unable to publish required CI measurement PR comment"
+ echo "::notice::gh is not available; skipping optional CI measurement PR comment"
can_render_pr_comment=false
fi
if ! ensure_ci_measurement_tool node nodejs; then
- echo "::error::node is not available; unable to publish required CI measurement PR comment"
+ echo "::notice::node is not available; skipping optional CI measurement PR comment"
can_render_pr_comment=false
fi
if ! command -v jq >/dev/null 2>&1; then
if ensure_ci_measurement_tool jq jq; then
:
else
- echo "::error::jq is not available; unable to publish required CI measurement PR comment"
+ echo "::notice::jq is not available for the optional CI measurement PR comment"
can_render_pr_comment=false
fi
fi
if [ -z "${GH_TOKEN:-${GITHUB_TOKEN:-}}" ]; then
- echo "::error::GH_TOKEN/GITHUB_TOKEN is not set; unable to publish required CI measurement PR comment"
+ echo "::notice::GH_TOKEN/GITHUB_TOKEN is not set; skipping optional CI measurement PR comment"
can_render_pr_comment=false
fi
@@ -7060,16 +7267,10 @@ jobs:
pr_number="$(jq -r '.pull_request.number // empty' "$event_path")"
fi
if [ "$can_render_pr_comment" = "true" ] && [ -z "$pr_number" ]; then
- echo "::error::pull request number is unavailable; unable to publish required CI measurement PR comment"
+ echo "::notice::pull request number is unavailable; skipping optional CI measurement PR comment"
can_render_pr_comment=false
fi
- if [ "$can_render_pr_comment" != "true" ]; then
- if [ "$is_fork_pr" != "true" ]; then
- exit 1
- fi
- fi
-
if [ "$can_render_pr_comment" = "true" ]; then
repo="${GITHUB_REPOSITORY:?GITHUB_REPOSITORY not set}"
comment_tmp_dir="$(mktemp -d)"
@@ -7219,6 +7420,12 @@ jobs:
}
const formatRowImpact = (row) => {
+ if (row.comparisonMode === 'assertion') {
+ if (row.status === 'pass') return 'pass'
+ if (row.status === 'fail') return 'failed'
+ if (row.status === 'no_verdict') return 'blocking'
+ return 'n/a'
+ }
if (row.confidence === 'diagnostic' || row.gateReason === 'disabled' || row.semanticImpactKind === 'diagnostic') {
return 'diagnostic'
}
@@ -7284,6 +7491,12 @@ jobs:
tone: 'bad',
color: '#ef4444',
}
+ if (row.status === 'no_verdict') return {
+ label: 'No verdict - blocks merge',
+ detail: 'Required evidence is missing, malformed, or contradicts the configured assertion.',
+ tone: 'bad',
+ color: '#ef4444',
+ }
if (row.status === 'warn') return {
label: 'Regression - review',
detail: 'Worse than the configured warning threshold.',
@@ -7464,9 +7677,10 @@ jobs:
const rank = (row) => {
if (row.status === 'fail') return 0
- if (row.status === 'warn') return 1
- if (row.status === 'missing_baseline') return 3
- return 2
+ if (row.status === 'no_verdict') return 1
+ if (row.status === 'warn') return 2
+ if (row.status === 'missing_baseline') return 4
+ return 3
}
const allRows = Object.values(comparison.comparisons || {}).sort((left, right) => {
@@ -7490,6 +7704,7 @@ jobs:
})()
const visibleLimit = Number.isFinite(maxRows) && maxRows > 0 ? maxRows : 10
const comparableRows = allRows.filter((row) => typeof row.baseline === 'number')
+ const assertionRows = allRows.filter((row) => row.comparisonMode === 'assertion')
const hasComparableBaseline = comparableRows.length > 0
const isDiagnosticRow = (row) =>
row.status === 'missing_baseline' ||
@@ -7502,10 +7717,14 @@ jobs:
!Number.isNaN(row.semanticImpactScore) &&
Math.abs(row.semanticImpactScore) < 0.005
const actionableComparableRows = comparableRows.filter((row) => !isDiagnosticRow(row))
- const visibleRows = (hasComparableBaseline
- ? actionableComparableRows
- : allRows.filter((row) => !isDiagnosticRow(row)).sort((left, right) => (right.current || 0) - (left.current || 0))
- ).slice(0, visibleLimit)
+ const visibleRows = [
+ ...assertionRows,
+ ...(hasComparableBaseline
+ ? actionableComparableRows
+ : allRows
+ .filter((row) => !isDiagnosticRow(row) && row.comparisonMode !== 'assertion')
+ .sort((left, right) => (right.current || 0) - (left.current || 0))),
+ ].slice(0, visibleLimit)
const nonZeroImpactRows = actionableComparableRows.filter((row) => !isZeroImpactRow(row))
const zeroImpactRows = actionableComparableRows.filter(isZeroImpactRow)
const visibleNonZeroImpactRows = nonZeroImpactRows.slice(0, visibleLimit)
@@ -7537,6 +7756,7 @@ jobs:
const scanDecision = (row) => {
if (row.status === 'fail') return 'regression blocks'
+ if (row.status === 'no_verdict') return 'evidence missing - blocks'
if (row.status === 'warn') return 'regression review'
if (row.status === 'missing_baseline') return 'needs baseline'
if (row.direction === 'improved') return 'faster'
@@ -7838,6 +8058,12 @@ jobs:
return svg.join('\n')
}
+ const gateModeLabel = (mode) => {
+ if (mode === 'fail') return 'enforced'
+ if (mode === 'warn') return 'advisory'
+ if (mode === 'off') return 'off'
+ return mode || 'unknown'
+ }
const statusWord = comparison.status || 'unknown'
const readiness = comparison.readiness || {}
const readinessLabel = readiness.enforceable
@@ -7845,6 +8071,12 @@ jobs:
: 'partial (' + (readiness.gateableCount ?? 0) + '/' + (readiness.enabledCount ?? 0) + ' enabled observations gateable)'
const runUrl = runId ? serverUrl + '/' + repo + '/actions/runs/' + runId : undefined
const shortSha = (headSha || sha || 'unknown').slice(0, 7)
+ const assertionRows = allRows.filter((row) => row.comparisonMode === 'assertion')
+ const passingAssertionCount = assertionRows.filter((row) => row.status === 'pass').length
+ const assertionReadinessLabel = assertionRows.length > 0
+ ? String(passingAssertionCount) + '/' + String(assertionRows.length) + ' required assertions passing'
+ : 'no required assertions configured'
+ const baselineGateLabel = gateModeLabel(comparison.mode) + ' baseline regressions'
const existingState = extractState(existing?.body)
const currentRun = {
commitSha: headSha || sha || 'unknown',
@@ -7866,7 +8098,7 @@ jobs:
current: formatValue(row.current, row.observation?.unit),
delta: formatDelta(row.delta, row.observation?.unit),
ratio: formatRatio(row.ratio),
- impact: formatSemanticImpact(row.semanticImpactScore),
+ impact: formatRowImpact(row),
})),
}
const hasComparableHistory = (run) => Array.isArray(run.visibleRows) && run.visibleRows.some((row) =>
@@ -7877,12 +8109,6 @@ jobs:
const previousRuns = (existingState?.runs || []).filter((run) => run.commitSha !== currentRun.commitSha && hasComparableHistory(run))
const historyLimit = Number.isFinite(maxHistory) && maxHistory > 0 ? maxHistory : 20
const state = { _tag: stateTag, schemaVersion, title, runs: [currentRun, ...previousRuns].slice(0, historyLimit) }
- const gateModeLabel = (mode) => {
- if (mode === 'fail') return 'enforced'
- if (mode === 'warn') return 'advisory'
- if (mode === 'off') return 'off'
- return mode || 'unknown'
- }
const historyRows = state.runs.slice(1).map((run) => {
const link = run.runUrl ? '[' + run.shortSha + '](' + run.runUrl + ')' : run.shortSha
const top = Array.isArray(run.visibleRows) && run.visibleRows.length > 0
@@ -7901,7 +8127,8 @@ jobs:
schemaVersion,
title,
status: statusWord,
- gate: gateModeLabel(comparison.mode),
+ gate: baselineGateLabel,
+ assertionReadiness: assertionReadinessLabel,
readiness: readinessLabel,
commit: {
shortSha,
@@ -7942,9 +8169,12 @@ jobs:
: ''
const regressionCount = allRows.filter((row) => row.status === 'fail' || row.status === 'warn').length
+ const blockingAssertionCount = assertionRows.filter((row) => row.status === 'fail' || row.status === 'no_verdict').length
const improvementCount = comparableRows.filter((row) => row.direction === 'improved' && !isZeroImpactRow(row)).length
const neutralCount = zeroImpactRows.length + diagnosticRows.length
- const humanSummary = hasComparableBaseline
+ const humanSummary = blockingAssertionCount > 0
+ ? String(blockingAssertionCount) + ' required assertion' + (blockingAssertionCount === 1 ? '' : 's') + ' block merge because evidence failed or has no verdict.'
+ : hasComparableBaseline
? regressionCount > 0
? String(regressionCount) + ' regression' + (regressionCount === 1 ? '' : 's') + ' need review.'
: improvementCount > 0
@@ -7955,7 +8185,7 @@ jobs:
const summaryLines = [
'## ' + title,
'',
- '**' + statusWord + '** - ' + gateModeLabel(comparison.mode) + ' gate - readiness ' + readinessLabel + ' - commit ' + shortSha + ' - protocol ' + protocolLabel + '',
+ '**' + statusWord + '** - ' + assertionReadinessLabel + ' - ' + baselineGateLabel + ' - baseline readiness ' + readinessLabel + ' - commit ' + shortSha + ' - protocol ' + protocolLabel + '',
'',
'> ' + humanSummary,
'',
@@ -8054,7 +8284,7 @@ jobs:
if [ -s "$chart_file" ]; then
if [ "$require_public_asset" = "true" ] && [ -z "$public_asset_command" ]; then
echo "::error::CI measurement chart was rendered for a private repository, but CI_MEASUREMENT_PR_COMMENT_PUBLIC_ASSET_COMMAND is not configured. Private raw GitHub URLs cannot be embedded in PR comments."
- exit 1
+ can_render_pr_comment=false
fi
if ensure_ci_measurement_tool resvg resvg; then
@@ -8138,11 +8368,11 @@ jobs:
fi
if [ "$require_public_asset" = "true" ] && [ -z "$chart_url" ]; then
echo "::error::unable to publish CI measurement chart PNG to a public asset host for private repository $repo"
- exit 1
+ can_render_pr_comment=false
fi
if [ "$require_public_asset" = "true" ] && [ -s "$chart_dark_png_file" ] && [ -z "$chart_dark_url" ]; then
echo "::error::unable to publish dark CI measurement chart PNG to a public asset host for private repository $repo"
- exit 1
+ can_render_pr_comment=false
fi
node "$renderer_script" "$comparison_file" "$comments_json" "$comment_body" "$comment_id_file" "$chart_file" "$chart_dark_file"
fi
diff --git a/.github/workflows/ci.yml.genie.ts b/.github/workflows/ci.yml.genie.ts
index 98456181e..7b0661fc7 100644
--- a/.github/workflows/ci.yml.genie.ts
+++ b/.github/workflows/ci.yml.genie.ts
@@ -40,7 +40,7 @@ import {
validateNixStoreStep,
defaultRefPolicyCheckJob,
} from '../../genie/ci-workflow.ts'
-import { type CoreCIJobName } from '../../genie/ci.ts'
+import { buck2MeasurementTargets, type CoreCIJobName } from '../../genie/ci.ts'
import {
githubWorkflowEvent,
type GitHubWorkflowArgs,
@@ -302,9 +302,11 @@ const normalCiIf = `\${{ ${ciMeasurementNotBaselineBackfillPredicate} }}`
const job = ({
step,
extraSteps = [],
+ postSteps = [],
}: {
step: { name: string; run: string }
extraSteps?: readonly any[]
+ postSteps?: readonly any[]
}) => ({
if: normalCiIf,
'runs-on': namespaceRunner({
@@ -318,6 +320,7 @@ const job = ({
...baseSteps,
...extraSteps,
step,
+ ...postSteps,
savePnpmStateStep(),
nixDiagnosticsSummaryStep,
nixDiagnosticsArtifactStep(),
@@ -482,6 +485,29 @@ const jobs: Record | ReturnType ({
+ name: `Adapt and admit Buck measurements: ${target.label}`,
+ if: 'always()',
+ shell: 'bash',
+ env: {
+ CI_MEASUREMENT_BUCK_TARGET: JSON.stringify(target),
+ CI_MEASUREMENT_SUBJECT_SHA: ciMeasurementSubjectEnv.CI_MEASUREMENT_SUBJECT_SHA,
+ CI_MEASUREMENT_CHECKOUT_SHA: '${{ github.sha }}',
+ },
+ run: [
+ 'set -euo pipefail',
+ `output="tmp/buck2-ci-measurements/${target.id}/measurements.json"`,
+ 'DEVENV_TUI=false "${DEVENV_BIN:?DEVENV_BIN not set}" shell --no-reload -- bun scripts/buck2-benchmark/assert-invalidation.mjs --contract-json "$CI_MEASUREMENT_BUCK_TARGET" --output "$output" "' +
+ target.rawPath +
+ '"',
+ ].join('\n'),
+ })),
+ ciMeasurementsArtifactStep({
+ artifactName: 'buck2-measurements',
+ path: 'tmp/buck2-ci-measurements',
+ }),
+ ],
}),
cargo: job({
step: {
@@ -539,11 +565,11 @@ const downloadCurrentMeasurementArtifactStep = ({
}) as const
const ciMeasurementReportToolStep = {
- name: 'Provide CI measurement report tools',
+ name: 'Provide CI measurement comparison tool',
shell: 'bash',
run: [
'set -euo pipefail',
- 'for out in $(nix build --no-link --print-out-paths nixpkgs#jq nixpkgs#nodejs nixpkgs#gh nixpkgs#resvg); do',
+ 'for out in $(nix build --no-link --print-out-paths nixpkgs#jq); do',
' echo "$out/bin" >> "$GITHUB_PATH"',
'done',
].join('\n'),
@@ -801,8 +827,8 @@ const extraJobs: Record = {
},
'ci-measurements-report': {
name: 'ci/measurements-report',
- if: normalCiIf,
- needs: ['devenv-perf', 'nix-closure-sizes', 'source-shape'],
+ if: `\${{ always() && ${ciMeasurementNotBaselineBackfillPredicate} }}`,
+ needs: ['buck2', 'devenv-perf', 'nix-closure-sizes', 'source-shape'],
'runs-on': namespaceRunner({
profile: 'namespace-profile-linux-x86-64',
runId: '${{ github.run_id }}',
@@ -815,18 +841,34 @@ const extraJobs: Record = {
checkoutStep(),
installNixStep(),
ciMeasurementReportToolStep,
- downloadCurrentMeasurementArtifactStep({
- artifactName: 'devenv-perf',
- outputDir: `${ciMeasurementReportDir}/current/devenv-perf`,
- }),
- downloadCurrentMeasurementArtifactStep({
- artifactName: 'nix-closure-measurements',
- outputDir: `${ciMeasurementReportDir}/current/nix-closure-measurements`,
- }),
- downloadCurrentMeasurementArtifactStep({
- artifactName: 'source-shape',
- outputDir: `${ciMeasurementReportDir}/current/source-shape`,
- }),
+ {
+ ...downloadCurrentMeasurementArtifactStep({
+ artifactName: 'buck2-measurements',
+ outputDir: `${ciMeasurementReportDir}/current/buck2-measurements`,
+ }),
+ 'continue-on-error': true,
+ },
+ {
+ ...downloadCurrentMeasurementArtifactStep({
+ artifactName: 'devenv-perf',
+ outputDir: `${ciMeasurementReportDir}/current/devenv-perf`,
+ }),
+ 'continue-on-error': true,
+ },
+ {
+ ...downloadCurrentMeasurementArtifactStep({
+ artifactName: 'nix-closure-measurements',
+ outputDir: `${ciMeasurementReportDir}/current/nix-closure-measurements`,
+ }),
+ 'continue-on-error': true,
+ },
+ {
+ ...downloadCurrentMeasurementArtifactStep({
+ artifactName: 'source-shape',
+ outputDir: `${ciMeasurementReportDir}/current/source-shape`,
+ }),
+ 'continue-on-error': true,
+ },
downloadPreviousGitHubArtifactStep({
artifactName: 'devenv-perf',
outputDir: `${ciMeasurementReportDir}/baseline/devenv-perf`,
@@ -858,6 +900,7 @@ const extraJobs: Record = {
baselineDir: `${ciMeasurementReportDir}/baseline`,
outputFile: `${ciMeasurementReportDir}/measurement-comparison.json`,
regressionMode: 'warn',
+ assertionTargets: buck2MeasurementTargets,
prComment: {
enabled: true,
title: 'CI Measurements',
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9057444c7..c57d401ac 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file.
## [Unreleased]
+- **CI measurements**: Unify Buck2 invalidation evidence with the existing schema-v1 measurement pipeline. Consumer-owned typed assertions now drive both native Buck admission and the required shared report, while complete per-sample evidence gates semantic invalidation and timing profiles remain advisory.
+
- Fix lazy Buck stage-0 recovery to snapshot only its fingerprinted source inputs, keep unrelated Cargo product manifests out of tool derivations, and retain the observability gate when shell-entry setup is disabled.
- **Buck2 megarepo product**: add Python-free, separately cacheable `mr`,
diff --git a/context/ci-measurements.md b/context/ci-measurements.md
index 193e275aa..3175eb3c3 100644
--- a/context/ci-measurements.md
+++ b/context/ci-measurements.md
@@ -13,6 +13,7 @@ Active.
| `deterministic` | Nix closure size, source lines, file counts | Did a structural quantity exceed its budget? | Budget/diff against a comparable baseline. |
| `wall-clock` | Devenv shell eval, task runtime, CLI command latency | Did this PR make this operation slower on the same runner conditions? | Paired same-run base/head samples before merge blocking. |
| `diagnostic` | OTEL-traced shell eval, host context, trace breakdowns | Where did time go? | Never a regression gate; may be produced by required CI. |
+| `deterministic` | Buck invalidation action/materialization assertions | Did every measured sample preserve the expected semantic boundary? | Consumer-owned assertion over complete raw samples. |
The class is part of the observation contract through `measurementKind`.
The comparison policy is part of the gate contract through `comparisonMode`.
@@ -27,6 +28,7 @@ semantics match:
| `wall-clock` | `paired` | Same PR run, same runner, base/head pairs | Per-pair delta evidence interval. |
| `wall-clock` | `historical` | Previous comparable successful artifacts | Advisory robust bands only. |
| `diagnostic` | none | Optional context artifact or trace attachment | Not gateable. |
+| `deterministic` | `assertion` | No baseline; a typed expected value or range | Every configured raw sample must satisfy. |
Historical comparison is not a substitute for paired wall-clock evidence.
Budget comparison is not a substitute for owner-approved semantic budgets.
@@ -76,6 +78,17 @@ This keeps probe-specific collection code separate from the reusable regression
system. A new probe should not fork comparison, markdown rendering, or asset
publication logic.
+Buck invalidation measurements use the same path. `genie/ci.ts` owns the typed
+product expectations consumed independently by the native Buck admission step
+and the shared report. The additive schema-v1 artifact owns measured samples,
+raw-evidence hashes, and an audit snapshot of that policy; it cannot redefine
+the policy. The comparator rejects a mismatched snapshot or fingerprint.
+
+Required assertions evaluate every sample, require exact configured sample
+indexes and complete Buck logs, and fail closed on missing or malformed
+evidence. Their median numeric value exists only for display. Phase timings and
+non-asserted action/materialization profiles remain diagnostic observations.
+
The reusable engine boundary is specified in
[ci-measurement-engine.md](./ci-measurement-engine.md). The long-term direction
is to keep this artifact and comment contract as the source of truth while
diff --git a/devenv.nix b/devenv.nix
index bd811f315..893ec5a58 100644
--- a/devenv.nix
+++ b/devenv.nix
@@ -945,7 +945,6 @@ in
--irrelevant-path packages/@overeng/megarepo/src/lib/ref.unit.test.ts \
--runs 7 --warmups 2 --isolation-dir megarepo-mr-benchmark \
--output "$output"
- exec ${pkgs.nodejs}/bin/node scripts/buck2-benchmark/assert-invalidation.mjs "$output"
'';
};
@@ -1308,8 +1307,6 @@ in
--execute \
--in-place \
--buck-incremental-only \
- --assert-buck-invalidation \
- --expected-relevant-actions 2 \
--buck-bin ${buck2Machine}/bin/buck2 \
--buck-config-file "$buck2_stage0_config" \
--buck-config-file "$toolchain_config" \
@@ -1317,7 +1314,8 @@ in
--buck-target-platform //buck2/platforms:target_x86_64_linux_musl_static \
--work-contract effect-utils/otel-scrape-native-product-v1 \
--relevant-path packages/@overeng/otel-scrape/src/lib.rs \
- --irrelevant-path context/dependency-materialization/intuition.md
+ --irrelevant-path context/dependency-materialization/intuition.md \
+ --output "$root/tmp/buck2-benchmark/otel-scrape-product.jsonl"
'';
};
diff --git a/genie/ci-scripts/ci-measurement-comparison.test.sh b/genie/ci-scripts/ci-measurement-comparison.test.sh
index 8b2104157..94f86d80f 100755
--- a/genie/ci-scripts/ci-measurement-comparison.test.sh
+++ b/genie/ci-scripts/ci-measurement-comparison.test.sh
@@ -28,12 +28,31 @@ emit_compare_script() {
cat >"$emitter" <"$emitter" <"$(dirname "$file")/raw.jsonl"
+ jq -cn \
+ --arg evidenceSha "$evidence_sha" \
+ --argjson samples "$samples" \
+ '$samples | to_entries[] | {kind:"sample",schema:"effect-utils-buck2-benchmark/v0",engine:"buck2",target:"//fixture:product",runId:"run-1",sha:$evidenceSha,workContract:"fixture/v1",warmup:false,phase:"warm-noop",sampleIndex:.key,status:"ok",buckLogStatus:"ok",actionCount:.value,materializationCount:0,durationMs:1}' \
+ >>"$(dirname "$file")/raw.jsonl"
+ local raw_sha256
+ raw_sha256="$(sha256sum "$(dirname "$file")/raw.jsonl" | awk '{print $1}')"
+ jq -n \
+ --arg fingerprint "$assertion_fingerprint" \
+ --argjson snapshot "$snapshot" \
+ --argjson samples "$samples" \
+ --arg assertedStatus "$asserted_status" \
+ --arg subjectSha "$subject_sha" \
+ --arg evidenceSha "$evidence_sha" \
+ --arg rawSha256 "$raw_sha256" \
+ '{
+ schemaVersion:1,
+ generatedAt:"2026-08-13T00:00:00Z",
+ producer:{name:"effect-utils-ci-measurement",version:1,measurementProtocol:"buck2-invalidation-v1"},
+ target:{kind:"buck2",id:"fixture",label:"Fixture"},
+ subject:{sha:$subjectSha,evidenceSha:$evidenceSha},
+ contract:{fingerprint:$fingerprint,snapshot:$snapshot},
+ completeness:{status:"complete",missing:[]},
+ observations:[{
+ id:"warm-actions",label:"Warm actions",name:"buck2.action_count",unit:"count",value:0,
+ measurementKind:"deterministic",
+ dimensions:{phase:"warm-noop",measurementProtocol:"buck2-invalidation-v1"},
+ policy:{enabled:true,comparisonMode:"assertion",expectation:{_tag:"exact",value:0},onNoVerdict:"fail"},
+ statistics:{sampleCount:2,measuredSampleCount:2,samples:$samples},
+ assertion:{status:$assertedStatus},
+ evidence:{status:"complete",sampleIndexes:[0,1],rawSha256:$rawSha256}
+ }]
+ }' >"$file"
+}
+
+rm -rf "$tmp_dir/current" "$tmp_dir/baseline"
+write_assertion_measurement "$tmp_dir/current/measurements.json" '[0,0]'
+rm "$tmp_dir/current/measurements.json"
+CI_MEASUREMENT_SUBJECT_SHA=abc123 \
+CI_MEASUREMENT_CHECKOUT_SHA=merge123 \
+ bun scripts/buck2-benchmark/assert-invalidation.mjs \
+ --contract-json "$assertion_target_json" \
+ --output "$tmp_dir/current/measurements.json" \
+ "$tmp_dir/current/raw.jsonl"
+emit_assertion_compare_script
+summary_file="$tmp_dir/summary.md"
+GITHUB_STEP_SUMMARY="$summary_file" run_compare
+[ "$(jq -r '.status' "$tmp_dir/comparison.json")" = pass ] || { echo "expected healthy assertion pass" >&2; exit 1; }
+[ "$(jq -r '.readiness.assertions.status + ":" + (.readiness.assertions.passingCount | tostring) + "/" + (.readiness.assertions.requiredCount | tostring)' "$tmp_dir/comparison.json")" = 'pass:1/1' ] || { jq . "$tmp_dir/comparison.json" >&2; echo "expected required assertion readiness" >&2; exit 1; }
+rg -Fx -- '- Required assertions: pass (1/1 passing)' "$summary_file" >/dev/null || { sed -n '1,12p' "$summary_file" >&2; echo "expected separate required assertion summary" >&2; exit 1; }
+rg -F -- '- Baseline regressions: advisory; readiness ready' "$summary_file" >/dev/null || { sed -n '1,12p' "$summary_file" >&2; echo "expected separate advisory baseline summary" >&2; exit 1; }
+
+mkdir -p "$tmp_dir/current/legacy" "$tmp_dir/baseline/run-1"
+write_measurement "$tmp_dir/current/legacy/measurements.json" 10.05 current "$policy"
+write_measurement "$tmp_dir/baseline/run-1/measurements.json" 10 current "$policy"
+run_compare
+[ "$(jq -r '[.comparisons[] | select(.gatePolicy.enabled == true) | .comparisonMode] | sort | join(",")' "$tmp_dir/comparison.json")" = 'assertion,historical' ] || { jq . "$tmp_dir/comparison.json" >&2; echo "expected mixed legacy and Buck comparison" >&2; exit 1; }
+[ "$(jq -r '.readiness.assertions.status + ":" + .readiness.advisory.status' "$tmp_dir/comparison.json")" = 'pass:ready' ] || { jq . "$tmp_dir/comparison.json" >&2; echo "expected separate ready assertion and advisory channels" >&2; exit 1; }
+
+CI_MEASUREMENT_CURRENT_DIR="$tmp_dir/current" \
+CI_MEASUREMENT_BASELINE_DIR="$tmp_dir/baseline" \
+CI_MEASUREMENT_COMPARISON_FILE="$tmp_dir/comparison.json" \
+CI_MEASUREMENT_REGRESSION_MODE=warn \
+CI_MEASUREMENT_SUBJECT_SHA=abc123 \
+CI_MEASUREMENT_CHECKOUT_SHA=merge123 \
+CI_MEASUREMENT_PR_COMMENT_ENABLED=true \
+GITHUB_SHA=merge123 \
+GITHUB_EVENT_NAME=pull_request \
+GITHUB_EVENT_PATH="$tmp_dir/missing-event.json" \
+ bash "$tmp_dir/compare.sh" || { echo "optional comment tooling must not fail a valid semantic verdict" >&2; exit 1; }
+
+rm -rf "$tmp_dir/current" "$tmp_dir/baseline"
+write_assertion_measurement "$tmp_dir/current/measurements.json" '[0,1]'
+set +e
+run_compare
+assertion_rc=$?
+set -e
+[ "$assertion_rc" -ne 0 ] && [ "$(jq -r '.comparisons[].status' "$tmp_dir/comparison.json")" = fail ] || { jq . "$tmp_dir/comparison.json" >&2; echo "expected one bad sample to fail required assertion" >&2; exit 1; }
+
+set +e
+CI_MEASUREMENT_CURRENT_DIR="$tmp_dir/current" \
+CI_MEASUREMENT_BASELINE_DIR="$tmp_dir/baseline" \
+CI_MEASUREMENT_COMPARISON_FILE="$tmp_dir/comparison.json" \
+CI_MEASUREMENT_REGRESSION_MODE=warn \
+CI_MEASUREMENT_SUBJECT_SHA=abc123 \
+CI_MEASUREMENT_CHECKOUT_SHA=merge123 \
+CI_MEASUREMENT_PR_COMMENT_ENABLED=true \
+GITHUB_SHA=merge123 \
+GITHUB_EVENT_NAME=push \
+ bash "$tmp_dir/compare.sh"
+non_pr_assertion_rc=$?
+set -e
+[ "$non_pr_assertion_rc" -ne 0 ] || { echo "expected non-PR comment skip to preserve assertion failure" >&2; exit 1; }
+
+rm -rf "$tmp_dir/current" "$tmp_dir/baseline"
+write_assertion_measurement "$tmp_dir/current/measurements.json" '[0,1]' "$assertion_target_json" pass
+set +e
+run_compare
+assertion_rc=$?
+set -e
+[ "$assertion_rc" -ne 0 ] && [ "$(jq -r '.comparisons[].status' "$tmp_dir/comparison.json")" = no_verdict ] || { echo "expected contradictory pass receipt to fail closed" >&2; exit 1; }
+
+rm -rf "$tmp_dir/current" "$tmp_dir/baseline"
+weakened_target_json="$(printf '%s' "$assertion_target_json" | jq -c '.assertions[0].expectation={"_tag":"at-most","value":1}')"
+write_assertion_measurement "$tmp_dir/current/measurements.json" '[0,0]' "$weakened_target_json"
+set +e
+run_compare
+assertion_rc=$?
+set -e
+[ "$assertion_rc" -ne 0 ] && [ "$(jq -r '.comparisons[].status' "$tmp_dir/comparison.json")" = no_verdict ] || { echo "expected producer-adjusted expectation to fail closed" >&2; exit 1; }
+
+rm -rf "$tmp_dir/current" "$tmp_dir/baseline"
+write_assertion_measurement "$tmp_dir/current/measurements.json" '[0,0]' "$assertion_target_json" pass stale-sha
+set +e
+run_compare
+assertion_rc=$?
+set -e
+[ "$assertion_rc" -ne 0 ] && [ "$(jq -r '.comparisons[].status' "$tmp_dir/comparison.json")" = no_verdict ] || { echo "expected stale artifact subject SHA to fail closed" >&2; exit 1; }
+
+rm -rf "$tmp_dir/current" "$tmp_dir/baseline"
+write_assertion_measurement "$tmp_dir/current/measurements.json" '[0,0]'
+raw_tmp="$tmp_dir/current/raw-tampered.jsonl"
+jq -c 'if .kind == "sample" and .sampleIndex == 0 then .actionCount = 1 else . end' \
+ "$tmp_dir/current/raw.jsonl" >"$raw_tmp"
+mv "$raw_tmp" "$tmp_dir/current/raw.jsonl"
+tampered_raw_sha256="$(sha256sum "$tmp_dir/current/raw.jsonl" | awk '{print $1}')"
+jq --arg rawSha256 "$tampered_raw_sha256" \
+ '.observations[].evidence.rawSha256 = $rawSha256' \
+ "$tmp_dir/current/measurements.json" >"$tmp_dir/current/measurements-tampered.json"
+mv "$tmp_dir/current/measurements-tampered.json" "$tmp_dir/current/measurements.json"
+set +e
+run_compare
+assertion_rc=$?
+set -e
+[ "$assertion_rc" -ne 0 ] && [ "$(jq -r '.comparisons[].status' "$tmp_dir/comparison.json")" = no_verdict ] || { echo "expected raw evidence tampering to fail closed" >&2; exit 1; }
+
+rm -rf "$tmp_dir/current" "$tmp_dir/baseline"
+write_assertion_measurement "$tmp_dir/current/measurements.json" '[0,0]'
+printf 'malformed raw evidence\n' >"$tmp_dir/current/raw.jsonl"
+set +e
+run_compare
+assertion_rc=$?
+set -e
+[ "$assertion_rc" -ne 0 ] && [ "$(jq -r '.comparisons[].status' "$tmp_dir/comparison.json")" = no_verdict ] || { echo "expected malformed raw evidence to produce a blocking no-verdict" >&2; exit 1; }
+
+rm -rf "$tmp_dir/current" "$tmp_dir/baseline"
+set +e
+run_compare
+assertion_rc=$?
+set -e
+[ "$assertion_rc" -ne 0 ] && [ "$(jq -r '.comparisons[].status' "$tmp_dir/comparison.json")" = no_verdict ] || { echo "expected missing artifact to reconstruct no-verdict row" >&2; exit 1; }
+
+echo "ci-measurement assertion tests passed"
diff --git a/genie/ci-workflow/measurements.ts b/genie/ci-workflow/measurements.ts
index 2cc5a11d9..734919124 100644
--- a/genie/ci-workflow/measurements.ts
+++ b/genie/ci-workflow/measurements.ts
@@ -1,4 +1,7 @@
+import { createHash } from 'node:crypto'
+
import type { GitHubWorkflowArgs } from '../../packages/@overeng/genie/src/runtime/mod.ts'
+import type { Buck2MeasurementTarget, CiMeasurementExpectation } from '../ci.ts'
import {
checkoutStep,
installNixStep,
@@ -24,7 +27,9 @@ export type CiMeasurementDescriptor = {
export type CiMeasurementGatePolicy = {
readonly enabled?: boolean
- readonly comparisonMode?: 'budget' | 'historical' | 'paired'
+ readonly comparisonMode?: 'budget' | 'historical' | 'paired' | 'assertion'
+ readonly expectation?: CiMeasurementExpectation
+ readonly onNoVerdict?: 'advisory' | 'fail'
readonly minBaselineSources?: number
readonly minCurrentSamples?: number
readonly minPairedSamples?: number
@@ -82,6 +87,13 @@ export type CiMeasurementObservation = {
readonly pairedDeltaP75?: number
readonly pairedDeltaMad?: number
readonly pairedDeltaSamples?: readonly number[]
+ readonly samples?: readonly number[]
+ }
+ readonly assertion?: { readonly status: 'pass' | 'fail' | 'no-verdict' }
+ readonly evidence?: {
+ readonly status: 'complete' | 'partial'
+ readonly sampleIndexes?: readonly number[]
+ readonly rawSha256?: string
}
}
@@ -117,10 +129,16 @@ export type CiMeasurementArtifact = {
readonly subject?: {
readonly ref?: string
readonly sha?: string
+ readonly evidenceSha?: string
readonly label?: string
}
readonly target: CiMeasurementTarget
readonly observations: readonly CiMeasurementObservation[]
+ readonly contract?: { readonly fingerprint: string; readonly snapshot: unknown }
+ readonly completeness?: {
+ readonly status: 'complete' | 'partial'
+ readonly missing: readonly { readonly observationId?: string; readonly reason: string }[]
+ }
readonly attachments?: readonly {
readonly name: string
readonly path: string
@@ -254,6 +272,7 @@ export type CiMeasurementsComparisonStepOptions = {
readonly baselineDir?: string
readonly outputFile?: string
readonly regressionMode?: 'off' | 'warn' | 'fail'
+ readonly assertionTargets?: readonly Buck2MeasurementTarget[]
readonly prComment?: {
readonly enabled?: boolean
readonly title?: string
@@ -1955,8 +1974,14 @@ cat "$artifact_file"
} as const
}
-export const compareCiMeasurementsStep = (opts?: CiMeasurementsComparisonStepOptions) =>
- ({
+export const compareCiMeasurementsStep = (opts?: CiMeasurementsComparisonStepOptions) => {
+ const assertionTargets = (opts?.assertionTargets ?? []).map((target) =>
+ Object.assign({}, target, {
+ fingerprint: createHash('sha256').update(JSON.stringify(target)).digest('hex'),
+ }),
+ )
+
+ return {
name: 'Compare CI measurements with baseline',
shell: 'bash',
env: {
@@ -1965,6 +1990,7 @@ export const compareCiMeasurementsStep = (opts?: CiMeasurementsComparisonStepOpt
CI_MEASUREMENT_COMPARISON_FILE:
opts?.outputFile ?? 'tmp/ci-measurements/measurement-comparison.json',
CI_MEASUREMENT_REGRESSION_MODE: opts?.regressionMode ?? 'warn',
+ CI_MEASUREMENT_ASSERTION_TARGETS: JSON.stringify(assertionTargets),
CI_MEASUREMENT_PR_COMMENT_ENABLED: opts?.prComment?.enabled === true ? 'true' : 'false',
CI_MEASUREMENT_PR_COMMENT_TITLE: opts?.prComment?.title ?? 'CI Measurements',
CI_MEASUREMENT_PR_COMMENT_MAX_ROWS: String(opts?.prComment?.maxRows ?? 10),
@@ -1991,6 +2017,7 @@ current_dir="${dollar}{CI_MEASUREMENT_CURRENT_DIR:?CI_MEASUREMENT_CURRENT_DIR no
baseline_dir="${dollar}{CI_MEASUREMENT_BASELINE_DIR:?CI_MEASUREMENT_BASELINE_DIR not set}"
comparison_file="${dollar}{CI_MEASUREMENT_COMPARISON_FILE:?CI_MEASUREMENT_COMPARISON_FILE not set}"
mode="${dollar}{CI_MEASUREMENT_REGRESSION_MODE:-warn}"
+assertion_targets="${dollar}{CI_MEASUREMENT_ASSERTION_TARGETS:-[]}"
mkdir -p "$(dirname "$comparison_file")"
if [ "$mode" = "off" ]; then
@@ -2008,13 +2035,46 @@ find "$current_dir" -name baseline -type d -prune -o -name measurements.json -ty
} | sort -u >"$baseline_index" || true
if [ ! -s "$current_index" ]; then
- echo "::error::no current measurements.json files found under $current_dir"
- exit 1
+ if [ "$assertion_targets" = "[]" ]; then
+ echo "::error::no current measurements.json files found under $current_dir"
+ exit 1
+ fi
+ echo "::warning::no current measurements.json files found; reconstructing required assertion rows"
fi
current_json="$comparison_file.current.json"
baseline_json="$comparison_file.baseline.json"
-xargs -r jq -s '.' <"$current_index" >"$current_json"
+if [ -s "$current_index" ]; then
+ verified_current_dir="$(mktemp -d)"
+ verified_index="$(mktemp)"
+ verified_count=0
+ while IFS= read -r measurement_file; do
+ verified_count=$((verified_count + 1))
+ raw_file="$(dirname "$measurement_file")/raw.jsonl"
+ raw_sha256=""
+ raw_parse_status="missing"
+ raw_records_file="$(mktemp)"
+ printf '[]\n' >"$raw_records_file"
+ if [ -f "$raw_file" ]; then
+ raw_sha256="$(sha256sum "$raw_file" | awk '{print $1}')"
+ raw_parse_status="valid"
+ if ! jq -s '.' "$raw_file" >"$raw_records_file" 2>/dev/null; then
+ echo "::warning::raw CI measurement evidence is malformed: $raw_file"
+ raw_parse_status="invalid"
+ printf '[]\n' >"$raw_records_file"
+ fi
+ fi
+ verified_file="$verified_current_dir/$verified_count.json"
+ jq --arg rawSha256 "$raw_sha256" --arg rawParseStatus "$raw_parse_status" --slurpfile rawRecords "$raw_records_file" \
+ '. + {_consumerVerification:{rawSha256:(if $rawSha256 == "" then null else $rawSha256 end),rawParseStatus:$rawParseStatus,rawRecords:($rawRecords[0] // [])}}' \
+ "$measurement_file" >"$verified_file"
+ rm -f "$raw_records_file"
+ printf '%s\n' "$verified_file" >>"$verified_index"
+ done <"$current_index"
+ xargs -r jq -s '.' <"$verified_index" >"$current_json"
+else
+ printf '[]\n' >"$current_json"
+fi
if [ -s "$baseline_index" ]; then
xargs -r jq -s '.' <"$baseline_index" >"$baseline_json"
else
@@ -2026,8 +2086,11 @@ jq -n \
--slurpfile baseline "$baseline_json" \
--argjson schemaVersion 1 \
--arg mode "$mode" \
+ --argjson assertionTargets "$assertion_targets" \
--arg currentDir "$current_dir" \
--arg baselineDir "$baseline_dir" \
+ --arg expectedSubjectSha "${dollar}{CI_MEASUREMENT_SUBJECT_SHA:-${dollar}{GITHUB_SHA:-unknown}}" \
+ --arg expectedCheckoutSha "${dollar}{CI_MEASUREMENT_CHECKOUT_SHA:-${dollar}{GITHUB_SHA:-unknown}}" \
'
def identity_dimensions:
(.dimensions // {})
@@ -2067,7 +2130,7 @@ jq -n \
def observations_by_key($docs):
reduce $docs[]? as $doc
({};
- reduce (($doc.observations // [])[]? | select(.value | type == "number")) as $obs
+ reduce (($doc.observations // [])[]? | select((.value | type == "number") and (.policy.comparisonMode // "") != "assertion")) as $obs
(.;
($obs | observation_key($doc)) as $key
| .[$key] = ((.[$key] // []) + [{
@@ -2146,6 +2209,113 @@ jq -n \
def policy_enabled($policy):
if ($policy | has("enabled")) then $policy.enabled else true end;
+ def expectation_matches($expectation; $value):
+ if $expectation._tag == "exact" then $value == $expectation.value
+ elif $expectation._tag == "at-least" then $value >= $expectation.value
+ elif $expectation._tag == "at-most" then $value <= $expectation.value
+ elif $expectation._tag == "range" then $value >= $expectation.min and $value <= $expectation.max
+ else false
+ end;
+
+ def assertion_comparisons($docs; $targets):
+ reduce $targets[]? as $target
+ ({};
+ ([ $docs[]? | select(.target.kind == "buck2" and .target.id == $target.id) ]) as $matchingDocs
+ | reduce $target.assertions[] as $expected
+ (.;
+ ($matchingDocs[0] // null) as $doc
+ | ([ $doc.observations[]? | select(.id == $expected.id) ]) as $matchingObservations
+ | ($matchingObservations[0] // null) as $observed
+ | ($observed.statistics.samples // []) as $samples
+ | ($observed.evidence.sampleIndexes // []) as $indexes
+ | ($doc._consumerVerification.rawRecords // []) as $rawRecords
+ | ([ $rawRecords[]? | select(.kind == "metadata") ]) as $rawMetadata
+ | ([ $rawRecords[]?
+ | select(
+ .kind == "sample"
+ and .engine == "buck2"
+ and .warmup == false
+ and .workContract == $target.workContract
+ and .phase == $expected.phase
+ )
+ ]) as $rawSamples
+ | ($rawSamples | map(.sampleIndex)) as $rawIndexes
+ | ($rawSamples | map(.[$expected.metric])) as $rawValues
+ | (
+ ($matchingDocs | length) == 1
+ and $doc.schemaVersion == 1
+ and $doc.producer.measurementProtocol == "buck2-invalidation-v1"
+ and $doc.contract.fingerprint == $target.fingerprint
+ and $doc.contract.snapshot == ($target | del(.fingerprint))
+ and $doc.completeness.status == "complete"
+ and $doc.subject.sha == $expectedSubjectSha
+ and $doc.subject.evidenceSha == $expectedCheckoutSha
+ and $doc._consumerVerification.rawParseStatus == "valid"
+ and ($rawMetadata | length) == 1
+ and $rawMetadata[0].sha == $expectedCheckoutSha
+ and $rawMetadata[0].schema == $target.benchmarkSchema
+ and $rawMetadata[0].target == $target.buckTarget
+ and $rawMetadata[0].workContract == $target.workContract
+ and $rawMetadata[0].samplePolicy.runs == $target.runs
+ and ($matchingObservations | length) == 1
+ and $observed.policy.comparisonMode == "assertion"
+ and $observed.policy.expectation == $expected.expectation
+ and $observed.policy.onNoVerdict == "fail"
+ and $observed.name == ("buck2." + (if $expected.metric == "actionCount" then "action_count" else "materialization_count" end))
+ and $observed.unit == "count"
+ and $observed.dimensions.phase == $expected.phase
+ and $observed.dimensions.measurementProtocol == "buck2-invalidation-v1"
+ and $observed.evidence.status == "complete"
+ and $observed.statistics.sampleCount == $target.runs
+ and $observed.statistics.measuredSampleCount == $target.runs
+ and ($samples | length) == $target.runs
+ and ($indexes | length) == $target.runs
+ and (($indexes | unique | sort) == ([range(0; $target.runs)]))
+ and $rawIndexes == $indexes
+ and $rawValues == $samples
+ and ($rawSamples | all(
+ .schema == $target.benchmarkSchema
+ and .target == $target.buckTarget
+ and .sha == $expectedCheckoutSha
+ and .runId == $rawMetadata[0].runId
+ and .status == "ok"
+ and .buckLogStatus == "ok"
+ ))
+ and ($samples | all(type == "number" and isfinite and floor == . and . >= 0))
+ and (($observed.evidence.rawSha256 // "") | test("^[0-9a-f]{64}$"))
+ and $observed.evidence.rawSha256 == $doc._consumerVerification.rawSha256
+ and $observed.assertion.status == (if ($samples | all(expectation_matches($expected.expectation; .))) then "pass" else "fail" end)
+ ) as $complete
+ | ($complete and ($samples | all(expectation_matches($expected.expectation; .)))) as $passing
+ | ("assertion|" + $target.id + "|" + $expected.id) as $key
+ | .[$key] = {
+ status: (if $complete then (if $passing then "pass" else "fail" end) else "no_verdict" end),
+ target: {kind:"buck2", id:$target.id, label:$target.label},
+ observation: ($observed // {
+ id:$expected.id,
+ label:$expected.label,
+ name:("buck2." + (if $expected.metric == "actionCount" then "action_count" else "materialization_count" end)),
+ unit:"count",
+ measurementKind:"deterministic",
+ dimensions:{phase:$expected.phase, measurementProtocol:"buck2-invalidation-v1"},
+ policy:{enabled:true, comparisonMode:"assertion", expectation:$expected.expectation, onNoVerdict:"fail"}
+ }),
+ current: (if ($samples | length) == 0 then null else ($samples | median) end),
+ currentSamples: ($samples | length),
+ baselineSources: 0,
+ gatePolicy: {enabled:true, comparisonMode:"assertion", expectation:$expected.expectation, onNoVerdict:"fail"},
+ comparisonMode:"assertion",
+ gateable:$complete,
+ gateReason:(if $complete then "eligible" else "incomplete_assertion_evidence" end),
+ confidence:(if $complete then (if $passing then "assertion_satisfied" else "assertion_violated" end) else "no_verdict" end),
+ direction:(if $complete and $passing then "unchanged" else "regressed" end),
+ semanticImpactScore:(if $complete and $passing then 0 else 1 end),
+ semanticImpactKind:(if $complete and $passing then "neutral" else "fail_boundary" end),
+ assertion:{expected:$expected.expectation, samples:$samples}
+ }
+ )
+ );
+
def classify($metric; $unit; $measurementKind; $policy; $current; $currentP25; $currentP75; $currentMad; $baseline; $baselineMin; $baselineMax; $baselineP25; $baselineP75; $baselineP95; $baselineMad; $currentSamples; $baselineSources; $pairedSamples; $pairedDeltaMedian; $pairedDeltaP25; $pairedDeltaP75; $pairedDeltaMad; $pairedDeltaValues):
$policy as $b
| ($policy.comparisonMode // (if $measurementKind == "deterministic" or $unit != "seconds" then "budget" elif $measurementKind == "diagnostic" then "diagnostic" else "historical" end)) as $comparisonMode
@@ -2367,10 +2537,13 @@ jq -n \
}
)
| from_entries
- ) as $comparisons
+ ) as $legacyComparisons
+ | assertion_comparisons($current[0]; $assertionTargets) as $assertionComparisons
+ | ($legacyComparisons + $assertionComparisons) as $comparisons
| (
if any($comparisons[]?; .status == "fail") then "fail"
elif any($comparisons[]?; .status == "warn") then "warn"
+ elif any($comparisons[]?; .status == "no_verdict") then "partial"
elif any($comparisons[]?;
(if (.gatePolicy | has("enabled")) then .gatePolicy.enabled else true end)
and (.gateReason == "missing_baseline"
@@ -2384,6 +2557,7 @@ jq -n \
) as $status
| (
[$comparisons[]?]
+ | . as $rows
| {
enabledCount: (map(select((if (.gatePolicy | has("enabled")) then .gatePolicy.enabled else true end))) | length),
gateableCount: (map(select(.gateable == true)) | length),
@@ -2392,10 +2566,26 @@ jq -n \
lowCurrentSampleCount: (map(select(.gateReason == "low_current_sample_count")) | length),
lowPairedSampleCount: (map(select(.gateReason == "low_paired_sample_count")) | length),
missingPairedDeltaCount: (map(select(.gateReason == "missing_paired_delta")) | length)
+ ,noVerdictCount: (map(select(.status == "no_verdict")) | length),
+ assertions: {
+ requiredCount: ($rows | map(select(.comparisonMode == "assertion")) | length),
+ passingCount: ($rows | map(select(.comparisonMode == "assertion" and .status == "pass")) | length),
+ blockingCount: ($rows | map(select(.comparisonMode == "assertion" and (.status == "fail" or .status == "no_verdict"))) | length)
+ },
+ advisory: {
+ enabledCount: ($rows | map(select(.comparisonMode != "assertion" and (if (.gatePolicy | has("enabled")) then .gatePolicy.enabled else true end))) | length),
+ gateableCount: ($rows | map(select(.comparisonMode != "assertion" and .gateable == true)) | length)
+ }
}
| . + {
nonGateableCount: (.enabledCount - .gateableCount),
- enforceable: (.enabledCount == .gateableCount)
+ enforceable: (.enabledCount == .gateableCount),
+ assertions: (.assertions + {
+ status: (if .assertions.requiredCount == 0 then "not_applicable" elif .assertions.blockingCount == 0 and .assertions.passingCount == .assertions.requiredCount then "pass" else "blocking" end)
+ }),
+ advisory: (.advisory + {
+ status: (if .advisory.enabledCount == .advisory.gateableCount then "ready" else "partial" end)
+ })
}
) as $readiness
| {
@@ -2433,11 +2623,16 @@ case "$status:$mode" in
;;
esac
+if jq -e 'any(.comparisons[]?; .comparisonMode == "assertion" and (.status == "fail" or .status == "no_verdict"))' "$comparison_file" >/dev/null; then
+ echo "::error::required CI measurement assertion failed or has no verdict"
+ exit_code=1
+fi
+
if [ -n "${dollar}{GITHUB_STEP_SUMMARY:-}" ]; then
{
echo "### ${dollar}{CI_MEASUREMENT_PR_COMMENT_TITLE:-CI Measurements}"
echo ""
- jq -r '"- Status: " + .status + "\n- Gate: " + (if .mode == "fail" then "enforced" elif .mode == "warn" then "advisory" elif .mode == "off" then "off" else (.mode // "unknown") end) + "\n- Baseline: " + .baselineDir' "$comparison_file"
+ jq -r '"- Status: " + .status + "\n- Required assertions: " + .readiness.assertions.status + " (" + (.readiness.assertions.passingCount | tostring) + "/" + (.readiness.assertions.requiredCount | tostring) + " passing)\n- Baseline regressions: " + (if .mode == "fail" then "enforced" elif .mode == "warn" then "advisory" elif .mode == "off" then "off" else (.mode // "unknown") end) + "; readiness " + .readiness.advisory.status + " (" + (.readiness.advisory.gateableCount | tostring) + "/" + (.readiness.advisory.enabledCount | tostring) + " gateable)\n- Baseline: " + .baselineDir' "$comparison_file"
echo ""
echo "| Status | Gate | Target | Observation | Current | Baseline | Delta | Ratio |"
echo "| --- | --- | --- | --- | ---: | ---: | ---: | ---: |"
@@ -2474,9 +2669,11 @@ ${
? String.raw`if [ "${dollar}{CI_MEASUREMENT_PR_COMMENT_ENABLED:-false}" = "true" ]; then
if [ "${dollar}{GITHUB_EVENT_NAME:-}" != "pull_request" ]; then
echo "::notice::CI measurement PR comments are produced only by pull_request workflows; skipping comment for event ${dollar}{GITHUB_EVENT_NAME:-unknown}"
- exit 0
+ exit "$exit_code"
fi
+ set +e
+
can_render_pr_comment=true
is_fork_pr=false
if [ "${dollar}{CI_MEASUREMENT_IS_FORK_PR:-}" = "1" ]; then
@@ -2491,33 +2688,27 @@ ${
if command -v "$tool_name" >/dev/null 2>&1; then
return 0
fi
- if ! command -v nix >/dev/null 2>&1; then
- return 1
- fi
- if tool_out="$(nix build --no-link --print-out-paths "nixpkgs#$nix_attr" 2>/dev/null)"; then
- export PATH="$tool_out/bin:$PATH"
- fi
- command -v "$tool_name" >/dev/null 2>&1
+ return 1
}
if ! ensure_ci_measurement_tool gh gh; then
- echo "::error::gh is not available; unable to publish required CI measurement PR comment"
+ echo "::notice::gh is not available; skipping optional CI measurement PR comment"
can_render_pr_comment=false
fi
if ! ensure_ci_measurement_tool node nodejs; then
- echo "::error::node is not available; unable to publish required CI measurement PR comment"
+ echo "::notice::node is not available; skipping optional CI measurement PR comment"
can_render_pr_comment=false
fi
if ! command -v jq >/dev/null 2>&1; then
if ensure_ci_measurement_tool jq jq; then
:
else
- echo "::error::jq is not available; unable to publish required CI measurement PR comment"
+ echo "::notice::jq is not available for the optional CI measurement PR comment"
can_render_pr_comment=false
fi
fi
if [ -z "${dollar}{GH_TOKEN:-${dollar}{GITHUB_TOKEN:-}}" ]; then
- echo "::error::GH_TOKEN/GITHUB_TOKEN is not set; unable to publish required CI measurement PR comment"
+ echo "::notice::GH_TOKEN/GITHUB_TOKEN is not set; skipping optional CI measurement PR comment"
can_render_pr_comment=false
fi
@@ -2527,16 +2718,10 @@ ${
pr_number="$(jq -r '.pull_request.number // empty' "$event_path")"
fi
if [ "$can_render_pr_comment" = "true" ] && [ -z "$pr_number" ]; then
- echo "::error::pull request number is unavailable; unable to publish required CI measurement PR comment"
+ echo "::notice::pull request number is unavailable; skipping optional CI measurement PR comment"
can_render_pr_comment=false
fi
- if [ "$can_render_pr_comment" != "true" ]; then
- if [ "$is_fork_pr" != "true" ]; then
- exit 1
- fi
- fi
-
if [ "$can_render_pr_comment" = "true" ]; then
repo="${dollar}{GITHUB_REPOSITORY:?GITHUB_REPOSITORY not set}"
comment_tmp_dir="$(mktemp -d)"
@@ -2686,6 +2871,12 @@ const formatSemanticImpact = (value) => {
}
const formatRowImpact = (row) => {
+ if (row.comparisonMode === 'assertion') {
+ if (row.status === 'pass') return 'pass'
+ if (row.status === 'fail') return 'failed'
+ if (row.status === 'no_verdict') return 'blocking'
+ return 'n/a'
+ }
if (row.confidence === 'diagnostic' || row.gateReason === 'disabled' || row.semanticImpactKind === 'diagnostic') {
return 'diagnostic'
}
@@ -2751,6 +2942,12 @@ const interpretation = (row) => {
tone: 'bad',
color: '#ef4444',
}
+ if (row.status === 'no_verdict') return {
+ label: 'No verdict - blocks merge',
+ detail: 'Required evidence is missing, malformed, or contradicts the configured assertion.',
+ tone: 'bad',
+ color: '#ef4444',
+ }
if (row.status === 'warn') return {
label: 'Regression - review',
detail: 'Worse than the configured warning threshold.',
@@ -2931,9 +3128,10 @@ const dimensions = (row) => {
const rank = (row) => {
if (row.status === 'fail') return 0
- if (row.status === 'warn') return 1
- if (row.status === 'missing_baseline') return 3
- return 2
+ if (row.status === 'no_verdict') return 1
+ if (row.status === 'warn') return 2
+ if (row.status === 'missing_baseline') return 4
+ return 3
}
const allRows = Object.values(comparison.comparisons || {}).sort((left, right) => {
@@ -2957,6 +3155,7 @@ const protocolLabel = (() => {
})()
const visibleLimit = Number.isFinite(maxRows) && maxRows > 0 ? maxRows : 10
const comparableRows = allRows.filter((row) => typeof row.baseline === 'number')
+const assertionRows = allRows.filter((row) => row.comparisonMode === 'assertion')
const hasComparableBaseline = comparableRows.length > 0
const isDiagnosticRow = (row) =>
row.status === 'missing_baseline' ||
@@ -2969,10 +3168,14 @@ const isZeroImpactRow = (row) =>
!Number.isNaN(row.semanticImpactScore) &&
Math.abs(row.semanticImpactScore) < 0.005
const actionableComparableRows = comparableRows.filter((row) => !isDiagnosticRow(row))
-const visibleRows = (hasComparableBaseline
- ? actionableComparableRows
- : allRows.filter((row) => !isDiagnosticRow(row)).sort((left, right) => (right.current || 0) - (left.current || 0))
-).slice(0, visibleLimit)
+const visibleRows = [
+ ...assertionRows,
+ ...(hasComparableBaseline
+ ? actionableComparableRows
+ : allRows
+ .filter((row) => !isDiagnosticRow(row) && row.comparisonMode !== 'assertion')
+ .sort((left, right) => (right.current || 0) - (left.current || 0))),
+].slice(0, visibleLimit)
const nonZeroImpactRows = actionableComparableRows.filter((row) => !isZeroImpactRow(row))
const zeroImpactRows = actionableComparableRows.filter(isZeroImpactRow)
const visibleNonZeroImpactRows = nonZeroImpactRows.slice(0, visibleLimit)
@@ -3004,6 +3207,7 @@ const confidenceSummary = (row) => {
const scanDecision = (row) => {
if (row.status === 'fail') return 'regression blocks'
+ if (row.status === 'no_verdict') return 'evidence missing - blocks'
if (row.status === 'warn') return 'regression review'
if (row.status === 'missing_baseline') return 'needs baseline'
if (row.direction === 'improved') return 'faster'
@@ -3305,6 +3509,12 @@ const renderPerfChangeSvg = (rows, theme = 'adaptive') => {
return svg.join('\n')
}
+const gateModeLabel = (mode) => {
+ if (mode === 'fail') return 'enforced'
+ if (mode === 'warn') return 'advisory'
+ if (mode === 'off') return 'off'
+ return mode || 'unknown'
+}
const statusWord = comparison.status || 'unknown'
const readiness = comparison.readiness || {}
const readinessLabel = readiness.enforceable
@@ -3312,6 +3522,12 @@ const readinessLabel = readiness.enforceable
: 'partial (' + (readiness.gateableCount ?? 0) + '/' + (readiness.enabledCount ?? 0) + ' enabled observations gateable)'
const runUrl = runId ? serverUrl + '/' + repo + '/actions/runs/' + runId : undefined
const shortSha = (headSha || sha || 'unknown').slice(0, 7)
+const assertionRows = allRows.filter((row) => row.comparisonMode === 'assertion')
+const passingAssertionCount = assertionRows.filter((row) => row.status === 'pass').length
+const assertionReadinessLabel = assertionRows.length > 0
+ ? String(passingAssertionCount) + '/' + String(assertionRows.length) + ' required assertions passing'
+ : 'no required assertions configured'
+const baselineGateLabel = gateModeLabel(comparison.mode) + ' baseline regressions'
const existingState = extractState(existing?.body)
const currentRun = {
commitSha: headSha || sha || 'unknown',
@@ -3333,7 +3549,7 @@ const currentRun = {
current: formatValue(row.current, row.observation?.unit),
delta: formatDelta(row.delta, row.observation?.unit),
ratio: formatRatio(row.ratio),
- impact: formatSemanticImpact(row.semanticImpactScore),
+ impact: formatRowImpact(row),
})),
}
const hasComparableHistory = (run) => Array.isArray(run.visibleRows) && run.visibleRows.some((row) =>
@@ -3344,12 +3560,6 @@ const hasComparableHistory = (run) => Array.isArray(run.visibleRows) && run.visi
const previousRuns = (existingState?.runs || []).filter((run) => run.commitSha !== currentRun.commitSha && hasComparableHistory(run))
const historyLimit = Number.isFinite(maxHistory) && maxHistory > 0 ? maxHistory : 20
const state = { _tag: stateTag, schemaVersion, title, runs: [currentRun, ...previousRuns].slice(0, historyLimit) }
-const gateModeLabel = (mode) => {
- if (mode === 'fail') return 'enforced'
- if (mode === 'warn') return 'advisory'
- if (mode === 'off') return 'off'
- return mode || 'unknown'
-}
const historyRows = state.runs.slice(1).map((run) => {
const link = run.runUrl ? '[' + run.shortSha + '](' + run.runUrl + ')' : run.shortSha
const top = Array.isArray(run.visibleRows) && run.visibleRows.length > 0
@@ -3368,7 +3578,8 @@ const sourceOfTruth = {
schemaVersion,
title,
status: statusWord,
- gate: gateModeLabel(comparison.mode),
+ gate: baselineGateLabel,
+ assertionReadiness: assertionReadinessLabel,
readiness: readinessLabel,
commit: {
shortSha,
@@ -3409,9 +3620,12 @@ const chartMarkdown = chartImageMarkdown
: ''
const regressionCount = allRows.filter((row) => row.status === 'fail' || row.status === 'warn').length
+const blockingAssertionCount = assertionRows.filter((row) => row.status === 'fail' || row.status === 'no_verdict').length
const improvementCount = comparableRows.filter((row) => row.direction === 'improved' && !isZeroImpactRow(row)).length
const neutralCount = zeroImpactRows.length + diagnosticRows.length
-const humanSummary = hasComparableBaseline
+const humanSummary = blockingAssertionCount > 0
+ ? String(blockingAssertionCount) + ' required assertion' + (blockingAssertionCount === 1 ? '' : 's') + ' block merge because evidence failed or has no verdict.'
+ : hasComparableBaseline
? regressionCount > 0
? String(regressionCount) + ' regression' + (regressionCount === 1 ? '' : 's') + ' need review.'
: improvementCount > 0
@@ -3422,7 +3636,7 @@ const humanSummary = hasComparableBaseline
const summaryLines = [
'## ' + title,
'',
- '**' + statusWord + '** - ' + gateModeLabel(comparison.mode) + ' gate - readiness ' + readinessLabel + ' - commit ' + shortSha + ' - protocol ' + protocolLabel + '',
+ '**' + statusWord + '** - ' + assertionReadinessLabel + ' - ' + baselineGateLabel + ' - baseline readiness ' + readinessLabel + ' - commit ' + shortSha + ' - protocol ' + protocolLabel + '',
'',
'> ' + humanSummary,
'',
@@ -3521,7 +3735,7 @@ EOF
if [ -s "$chart_file" ]; then
if [ "$require_public_asset" = "true" ] && [ -z "$public_asset_command" ]; then
echo "::error::CI measurement chart was rendered for a private repository, but CI_MEASUREMENT_PR_COMMENT_PUBLIC_ASSET_COMMAND is not configured. Private raw GitHub URLs cannot be embedded in PR comments."
- exit 1
+ can_render_pr_comment=false
fi
if ensure_ci_measurement_tool resvg resvg; then
@@ -3605,11 +3819,11 @@ EOF
fi
if [ "$require_public_asset" = "true" ] && [ -z "$chart_url" ]; then
echo "::error::unable to publish CI measurement chart PNG to a public asset host for private repository $repo"
- exit 1
+ can_render_pr_comment=false
fi
if [ "$require_public_asset" = "true" ] && [ -s "$chart_dark_png_file" ] && [ -z "$chart_dark_url" ]; then
echo "::error::unable to publish dark CI measurement chart PNG to a public asset host for private repository $repo"
- exit 1
+ can_render_pr_comment=false
fi
node "$renderer_script" "$comparison_file" "$comments_json" "$comment_body" "$comment_id_file" "$chart_file" "$chart_dark_file"
fi
@@ -3638,7 +3852,8 @@ if [ "$exit_code" -ne 0 ]; then
exit "$exit_code"
fi
`,
- }) as const
+ } as const
+}
export const devenvPerfJob = (opts?: DevenvPerfJobOptions) => {
const artifactDir = opts?.artifactDir ?? 'tmp/devenv-perf-ci'
diff --git a/genie/ci.ts b/genie/ci.ts
index 450ac489f..3238d6c04 100644
--- a/genie/ci.ts
+++ b/genie/ci.ts
@@ -14,6 +14,141 @@ export const RUNNER_PROFILES = [
/** Union of supported GitHub Actions runner profile labels. */
export type RunnerProfile = (typeof RUNNER_PROFILES)[number]
+export type CiMeasurementExpectation =
+ | { readonly _tag: 'exact'; readonly value: number }
+ | { readonly _tag: 'at-least'; readonly value: number }
+ | { readonly _tag: 'at-most'; readonly value: number }
+ | { readonly _tag: 'range'; readonly min: number; readonly max: number }
+
+export type Buck2MeasurementAssertion = {
+ readonly id: string
+ readonly label: string
+ readonly phase: string
+ readonly metric: 'actionCount' | 'materializationCount'
+ readonly expectation: CiMeasurementExpectation
+}
+
+export type Buck2MeasurementTarget = {
+ readonly id: string
+ readonly label: string
+ readonly workContract: string
+ readonly benchmarkSchema: 'effect-utils-buck2-benchmark/v0'
+ readonly buckTarget: string
+ readonly rawPath: string
+ readonly runs: number
+ readonly assertions: readonly Buck2MeasurementAssertion[]
+}
+
+/** Consumer-owned Buck measurement policy used by native admission and the canonical report. */
+export const buck2MeasurementTargets = [
+ {
+ id: 'megarepo-mr',
+ label: 'Megarepo CLI',
+ workContract: 'megarepo-cli-product/no-equivalent-devenv-lane/v1',
+ benchmarkSchema: 'effect-utils-buck2-benchmark/v0',
+ buckTarget: '//packages/@overeng/megarepo:mr',
+ rawPath: 'tmp/buck2-benchmark/megarepo-mr.jsonl',
+ runs: 7,
+ assertions: [
+ {
+ id: 'warm-actions',
+ label: 'Warm actions',
+ phase: 'warm-noop',
+ metric: 'actionCount',
+ expectation: { _tag: 'exact', value: 0 },
+ },
+ {
+ id: 'warm-materializations',
+ label: 'Warm materializations',
+ phase: 'warm-noop',
+ metric: 'materializationCount',
+ expectation: { _tag: 'exact', value: 0 },
+ },
+ {
+ id: 'mtime-actions',
+ label: 'Mtime-only actions',
+ phase: 'mtime-only',
+ metric: 'actionCount',
+ expectation: { _tag: 'exact', value: 0 },
+ },
+ {
+ id: 'irrelevant-actions',
+ label: 'Role-excluded actions',
+ phase: 'irrelevant-edit',
+ metric: 'actionCount',
+ expectation: { _tag: 'exact', value: 0 },
+ },
+ {
+ id: 'irrelevant-materializations',
+ label: 'Role-excluded materializations',
+ phase: 'irrelevant-edit',
+ metric: 'materializationCount',
+ expectation: { _tag: 'exact', value: 0 },
+ },
+ {
+ id: 'relevant-actions',
+ label: 'Relevant-edit actions',
+ phase: 'relevant-edit',
+ metric: 'actionCount',
+ expectation: { _tag: 'at-least', value: 1 },
+ },
+ {
+ id: 'declared-unreachable-actions',
+ label: 'Declared-unreachable actions',
+ phase: 'declared-unreachable-edit',
+ metric: 'actionCount',
+ expectation: { _tag: 'at-least', value: 1 },
+ },
+ ],
+ },
+ {
+ id: 'otel-scrape-product',
+ label: 'OTel scrape native product',
+ workContract: 'effect-utils/otel-scrape-native-product-v1',
+ benchmarkSchema: 'effect-utils-buck2-benchmark/v0',
+ buckTarget: '//packages/@overeng/otel-scrape:product',
+ rawPath: 'tmp/buck2-benchmark/otel-scrape-product.jsonl',
+ runs: 7,
+ assertions: [
+ {
+ id: 'warm-actions',
+ label: 'Warm actions',
+ phase: 'warm-noop',
+ metric: 'actionCount',
+ expectation: { _tag: 'exact', value: 0 },
+ },
+ {
+ id: 'warm-materializations',
+ label: 'Warm materializations',
+ phase: 'warm-noop',
+ metric: 'materializationCount',
+ expectation: { _tag: 'exact', value: 0 },
+ },
+ {
+ id: 'irrelevant-actions',
+ label: 'Irrelevant-edit actions',
+ phase: 'irrelevant-edit',
+ metric: 'actionCount',
+ expectation: { _tag: 'exact', value: 0 },
+ },
+ {
+ id: 'irrelevant-materializations',
+ label: 'Irrelevant-edit materializations',
+ phase: 'irrelevant-edit',
+ metric: 'materializationCount',
+ expectation: { _tag: 'exact', value: 0 },
+ },
+ {
+ id: 'relevant-actions',
+ label: 'Relevant-edit actions',
+ phase: 'relevant-edit',
+ metric: 'actionCount',
+ expectation: { _tag: 'exact', value: 2 },
+ },
+ ],
+ },
+] as const satisfies readonly Buck2MeasurementTarget[]
+
/** Core CI job keys used for the typed product-job block in the workflow generator. */
export const CORE_CI_JOB_NAMES = [
'typecheck',
@@ -47,6 +182,7 @@ export const EXTRA_CI_JOB_NAMES = [
'devenv-perf',
'nix-closure-sizes',
'source-shape',
+ 'ci-measurements-report',
'test-integration-notion',
'test-integration-restate',
'test-live-deploy-ci-tools',
@@ -56,7 +192,7 @@ export const EXTRA_CI_JOB_NAMES = [
export const REQUIRED_DEPLOY_CI_JOB_NAMES = ['deploy-storybooks'] as const
/** Workflow jobs that intentionally do not block merging. */
-export const advisoryCIJobNames = ['ci-measurements-report', 'notify-alignment'] as const
+export const advisoryCIJobNames = ['notify-alignment'] as const
/** CI job keys emitted by the generated workflow. */
export const CI_JOB_NAMES = [
diff --git a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts
index d6d0c301e..d9b6c2992 100644
--- a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts
+++ b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts
@@ -98,7 +98,7 @@ const generatedCiJobKeys = Array.from(
([, jobKey]) => jobKey,
).filter((jobKey): jobKey is string => jobKey !== undefined)
-const advisoryCheckContexts = new Set(['ci/measurements-report', 'notify-alignment'])
+const advisoryCheckContexts = new Set(['notify-alignment'])
const matrixCheckJobs = new Set(['test', 'nix-check', 'nix-fod-check'])
const matrixRunners = ['namespace-profile-linux-x86-64', 'namespace-profile-macos-arm64'] as const
@@ -871,6 +871,8 @@ describe('ci workflow devenv perf helpers', () => {
expect(generatedCiWorkflowYamlSource).not.toContain('DEVENV_PERF_REGRESSION_MODE')
expect(generatedCiWorkflowYamlSource).toContain('devenv-perf-warm-median-v2')
expect(generatedCiWorkflowYamlSource).toContain("CI_MEASUREMENT_PR_COMMENT_ENABLED: 'true'")
+ expect(generatedCiWorkflowYamlSource).toContain('CI_MEASUREMENT_SUBJECT_SHA:')
+ expect(generatedCiWorkflowYamlSource).toContain('CI_MEASUREMENT_CHECKOUT_SHA:')
expect(generatedCiWorkflowYamlSource).toContain(
'CI_MEASUREMENT_PR_COMMENT_TITLE: CI Measurements',
)
@@ -901,7 +903,8 @@ describe('ci workflow devenv perf helpers', () => {
expect(ciWorkflowSource).toContain(
'CI measurement PR comments are produced only by pull_request workflows',
)
- expect(ciWorkflowSource).toContain('unable to publish required CI measurement PR comment')
+ expect(ciWorkflowSource).toContain('skipping optional CI measurement PR comment')
+ expect(ciWorkflowSource).not.toContain('unable to publish required CI measurement PR comment')
expect(ciWorkflowSource).toContain('seedRuns: ($seedRuns[0] // [])')
expect(ciWorkflowSource).toContain('baselineProvenance: ($baselineProvenance[0] // null)')
expect(ciWorkflowSource).toContain(
diff --git a/scripts/buck2-benchmark/assert-invalidation.mjs b/scripts/buck2-benchmark/assert-invalidation.mjs
index 03e99a902..13679ba22 100755
--- a/scripts/buck2-benchmark/assert-invalidation.mjs
+++ b/scripts/buck2-benchmark/assert-invalidation.mjs
@@ -1,13 +1,61 @@
#!/usr/bin/env node
+/* oxlint-disable overeng/named-args, overeng/explicit-boolean-compare -- This internal CLI validates parsed JSON and boolean evidence predicates directly. */
-import { readFileSync } from 'node:fs'
+import { createHash } from 'node:crypto'
+import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { dirname, join, resolve } from 'node:path'
-const [rawPath] = process.argv.slice(2)
+const argv = process.argv.slice(2)
+const optionValue = (name) => {
+ const index = argv.indexOf(name)
+ return index === -1 ? undefined : argv[index + 1]
+}
+const contractJson = optionValue('--contract-json')
+const outputPath = optionValue('--output')
+const subjectSha = process.env.CI_MEASUREMENT_SUBJECT_SHA
+const checkoutSha = process.env.CI_MEASUREMENT_CHECKOUT_SHA
+const rawPath = argv.find(
+ (arg, index) =>
+ !arg.startsWith('--') &&
+ argv[index - 1] !== '--contract-json' &&
+ argv[index - 1] !== '--output',
+)
if (rawPath === undefined) {
- console.error('usage: assert-invalidation.mjs RAW_BENCHMARK_JSONL')
+ console.error(
+ 'usage: assert-invalidation.mjs [--contract-json JSON --output FILE] RAW_BENCHMARK_JSONL',
+ )
process.exit(2)
}
+if (!existsSync(rawPath)) {
+ if (contractJson === undefined || outputPath === undefined)
+ throw new Error(`benchmark evidence not found: ${rawPath}`)
+ const contract = JSON.parse(contractJson)
+ const fingerprint = createHash('sha256').update(JSON.stringify(contract)).digest('hex')
+ mkdirSync(dirname(outputPath), { recursive: true })
+ writeFileSync(
+ outputPath,
+ `${JSON.stringify(
+ {
+ schemaVersion: 1,
+ generatedAt: new Date().toISOString(),
+ producer: {
+ name: 'effect-utils-ci-measurement',
+ version: 1,
+ measurementProtocol: 'buck2-invalidation-v1',
+ },
+ target: { kind: 'buck2', id: contract.id, label: contract.label },
+ contract: { fingerprint, snapshot: contract },
+ completeness: { status: 'partial', missing: [{ reason: 'raw-benchmark-missing' }] },
+ observations: [],
+ },
+ null,
+ 2,
+ )}\n`,
+ )
+ process.exit(1)
+}
+
const records = readFileSync(rawPath, 'utf8')
.split(/\r?\n/u)
.filter((line) => line.trim() !== '')
@@ -16,24 +64,217 @@ const samples = records.filter(
(record) => record.kind === 'sample' && record.engine === 'buck2' && record.warmup === false,
)
-const assertPhase = ({ phase, expected }) => {
- const phaseSamples = samples.filter((sample) => sample.phase === phase)
- if (phaseSamples.length === 0) throw new Error(`${phase}: no measured samples`)
- for (const sample of phaseSamples) {
- if (sample.status !== 'ok' || sample.buckLogStatus !== 'ok')
- throw new Error(`${phase}: incomplete action evidence`)
- if (expected === 'zero' && sample.actionCount !== 0)
- throw new Error(`${phase}: expected zero actions, observed ${sample.actionCount}`)
- if (expected === 'positive' && !(sample.actionCount > 0))
- throw new Error(`${phase}: expected at least one action, observed ${sample.actionCount}`)
+const expectationMatches = (expectation, value) => {
+ if (expectation._tag === 'exact') return value === expectation.value
+ if (expectation._tag === 'at-least') return value >= expectation.value
+ if (expectation._tag === 'at-most') return value <= expectation.value
+ if (expectation._tag === 'range') return value >= expectation.min && value <= expectation.max
+ return false
+}
+
+const validExpectation = (expectation) => {
+ if (expectation === null || typeof expectation !== 'object') return false
+ if (expectation._tag === 'range')
+ return (
+ Number.isSafeInteger(expectation.min) &&
+ expectation.min >= 0 &&
+ Number.isSafeInteger(expectation.max) &&
+ expectation.max >= expectation.min
+ )
+ return (
+ ['exact', 'at-least', 'at-most'].includes(expectation._tag) &&
+ Number.isSafeInteger(expectation.value) &&
+ expectation.value >= 0
+ )
+}
+
+const median = (values) => {
+ const sorted = values.toSorted((left, right) => left - right)
+ if (sorted.length === 0) return 0
+ const middle = Math.floor(sorted.length / 2)
+ return sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2
+}
+
+const emitCiMeasurementArtifact = (contract) => {
+ if (!Number.isSafeInteger(contract.runs) || contract.runs < 1)
+ throw new Error('contract runs must be a positive integer')
+ if (!Array.isArray(contract.assertions) || contract.assertions.length === 0)
+ throw new Error('contract must contain assertions')
+ if (
+ new Set(contract.assertions.map((assertion) => assertion.id)).size !==
+ contract.assertions.length
+ )
+ throw new Error('contract assertion IDs must be unique')
+ for (const assertion of contract.assertions) {
+ if (!['actionCount', 'materializationCount'].includes(assertion.metric))
+ throw new Error(`${assertion.id}: unknown metric`)
+ if (!validExpectation(assertion.expectation))
+ throw new Error(`${assertion.id}: invalid expectation`)
}
+
+ const metadata = records.filter((record) => record.kind === 'metadata')
+ const soleMetadata = metadata.length === 1 ? metadata[0] : null
+ const rawSha256 = createHash('sha256').update(readFileSync(rawPath)).digest('hex')
+ const contractFingerprint = createHash('sha256').update(JSON.stringify(contract)).digest('hex')
+ const missing = []
+ const observations = []
+ const assertedKeys = new Set(
+ contract.assertions.map((assertion) => `${assertion.phase}:${assertion.metric}`),
+ )
+ const matchingSamples = samples.filter((sample) => sample.workContract === contract.workContract)
+
+ for (const assertion of contract.assertions) {
+ const phaseSamples = matchingSamples.filter((sample) => sample.phase === assertion.phase)
+ const indexes = phaseSamples.map((sample) => sample.sampleIndex)
+ const values = phaseSamples.map((sample) => sample[assertion.metric])
+ const complete =
+ soleMetadata !== null &&
+ soleMetadata.schema === contract.benchmarkSchema &&
+ soleMetadata.target === contract.buckTarget &&
+ soleMetadata.workContract === contract.workContract &&
+ (checkoutSha === undefined || soleMetadata.sha === checkoutSha) &&
+ soleMetadata.samplePolicy?.runs === contract.runs &&
+ phaseSamples.length === contract.runs &&
+ new Set(indexes).size === contract.runs &&
+ indexes.every(
+ (index) => Number.isSafeInteger(index) && index >= 0 && index < contract.runs,
+ ) &&
+ phaseSamples.every(
+ (sample) =>
+ sample.schema === contract.benchmarkSchema &&
+ sample.runId === soleMetadata?.runId &&
+ sample.sha === soleMetadata?.sha &&
+ sample.workContract === soleMetadata?.workContract &&
+ sample.status === 'ok' &&
+ sample.buckLogStatus === 'ok',
+ ) &&
+ values.every((value) => Number.isSafeInteger(value) && value >= 0)
+ const passing =
+ complete && values.every((value) => expectationMatches(assertion.expectation, value))
+ if (!complete)
+ missing.push({ observationId: assertion.id, reason: 'incomplete-sample-evidence' })
+ observations.push({
+ id: assertion.id,
+ label: assertion.label,
+ group: 'buck2 / invalidation',
+ name: `buck2.${assertion.metric === 'actionCount' ? 'action_count' : 'materialization_count'}`,
+ unit: 'count',
+ value: complete ? median(values) : 0,
+ measurementKind: 'deterministic',
+ dimensions: { phase: assertion.phase, measurementProtocol: 'buck2-invalidation-v1' },
+ policy: {
+ enabled: true,
+ comparisonMode: 'assertion',
+ expectation: assertion.expectation,
+ onNoVerdict: 'fail',
+ },
+ statistics: {
+ sampleCount: phaseSamples.length,
+ measuredSampleCount: phaseSamples.length,
+ samples: values,
+ },
+ assertion: { status: complete ? (passing ? 'pass' : 'fail') : 'no-verdict' },
+ evidence: { status: complete ? 'complete' : 'partial', sampleIndexes: indexes, rawSha256 },
+ })
+ }
+
+ const phases = [...new Set(matchingSamples.map((sample) => sample.phase))].toSorted(
+ (left, right) => left.localeCompare(right),
+ )
+ for (const phase of phases) {
+ const phaseSamples = matchingSamples.filter(
+ (sample) => sample.phase === phase && sample.status === 'ok',
+ )
+ for (const [metric, name, unit] of [
+ ['durationMs', 'buck2.duration', 'milliseconds'],
+ ['actionCount', 'buck2.action_count', 'count'],
+ ['materializationCount', 'buck2.materialization_count', 'count'],
+ ['materializationBytes', 'buck2.materialization_bytes', 'bytes'],
+ ]) {
+ if (assertedKeys.has(`${phase}:${metric}`)) continue
+ const values = phaseSamples
+ .map((sample) => sample[metric])
+ .filter((value) => typeof value === 'number' && Number.isFinite(value) && value >= 0)
+ if (values.length === 0) continue
+ observations.push({
+ id: `${phase}-${name.slice('buck2.'.length).replaceAll('_', '-')}`,
+ label: `${phase} ${name.slice('buck2.'.length).replaceAll('_', ' ')}`,
+ group: 'buck2 / profile',
+ name,
+ unit,
+ value: median(values),
+ measurementKind: 'diagnostic',
+ dimensions: { phase, measurementProtocol: 'buck2-invalidation-v1' },
+ policy: { enabled: false, comparisonMode: 'historical' },
+ statistics: {
+ sampleCount: values.length,
+ measuredSampleCount: values.length,
+ samples: values,
+ },
+ evidence: {
+ status: 'complete',
+ sampleIndexes: phaseSamples.map((sample) => sample.sampleIndex),
+ rawSha256,
+ },
+ })
+ }
+ }
+
+ const artifact = {
+ schemaVersion: 1,
+ generatedAt: new Date().toISOString(),
+ producer: {
+ name: 'effect-utils-ci-measurement',
+ version: 1,
+ measurementProtocol: 'buck2-invalidation-v1',
+ },
+ subject: {
+ sha: subjectSha ?? metadata[0]?.sha ?? null,
+ evidenceSha: checkoutSha ?? metadata[0]?.sha ?? null,
+ },
+ target: { kind: 'buck2', id: contract.id, label: contract.label },
+ contract: { fingerprint: contractFingerprint, snapshot: contract },
+ completeness: { status: missing.length === 0 ? 'complete' : 'partial', missing },
+ observations,
+ attachments: [
+ { name: 'raw-benchmark-jsonl', path: 'raw.jsonl', contentType: 'application/x-ndjson' },
+ ],
+ }
+ mkdirSync(dirname(outputPath), { recursive: true })
+ const artifactRawPath = join(dirname(outputPath), 'raw.jsonl')
+ if (resolve(rawPath) !== resolve(artifactRawPath)) copyFileSync(rawPath, artifactRawPath)
+ writeFileSync(outputPath, `${JSON.stringify(artifact, null, 2)}\n`)
+ if (
+ missing.length > 0 ||
+ observations.some((observation) => observation.assertion?.status === 'fail')
+ )
+ process.exitCode = 1
}
-assertPhase({ phase: 'warm-noop', expected: 'zero' })
-assertPhase({ phase: 'mtime-only', expected: 'zero' })
-assertPhase({ phase: 'irrelevant-edit', expected: 'zero' })
-assertPhase({ phase: 'relevant-edit', expected: 'positive' })
-assertPhase({ phase: 'declared-unreachable-edit', expected: 'positive' })
-console.log(
- 'buck2 benchmark invalidation assertions: PASS (declared-unreachable production boundary remains coarse)',
-)
+if (contractJson !== undefined || outputPath !== undefined) {
+ if (contractJson === undefined || outputPath === undefined)
+ throw new Error('--contract-json and --output must be provided together')
+ emitCiMeasurementArtifact(JSON.parse(contractJson))
+} else {
+ const assertPhase = ({ phase, expected }) => {
+ const phaseSamples = samples.filter((sample) => sample.phase === phase)
+ if (phaseSamples.length === 0) throw new Error(`${phase}: no measured samples`)
+ for (const sample of phaseSamples) {
+ if (sample.status !== 'ok' || sample.buckLogStatus !== 'ok')
+ throw new Error(`${phase}: incomplete action evidence`)
+ if (expected === 'zero' && sample.actionCount !== 0)
+ throw new Error(`${phase}: expected zero actions, observed ${sample.actionCount}`)
+ if (expected === 'positive' && !(sample.actionCount > 0))
+ throw new Error(`${phase}: expected at least one action, observed ${sample.actionCount}`)
+ }
+ }
+
+ assertPhase({ phase: 'warm-noop', expected: 'zero' })
+ assertPhase({ phase: 'mtime-only', expected: 'zero' })
+ assertPhase({ phase: 'irrelevant-edit', expected: 'zero' })
+ assertPhase({ phase: 'relevant-edit', expected: 'positive' })
+ assertPhase({ phase: 'declared-unreachable-edit', expected: 'positive' })
+ console.log(
+ 'buck2 benchmark invalidation assertions: PASS (declared-unreachable production boundary remains coarse)',
+ )
+}
diff --git a/scripts/buck2-benchmark/assert-invalidation.unit.test.mjs b/scripts/buck2-benchmark/assert-invalidation.unit.test.mjs
index 5eaf7a266..2cedaf7d0 100644
--- a/scripts/buck2-benchmark/assert-invalidation.unit.test.mjs
+++ b/scripts/buck2-benchmark/assert-invalidation.unit.test.mjs
@@ -1,6 +1,7 @@
import assert from 'node:assert/strict'
+/* oxlint-disable overeng/named-args, overeng/explicit-boolean-compare -- Test fixtures mirror the internal CLI's compact JSON inputs. */
import { spawnSync } from 'node:child_process'
-import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { describe, it } from 'node:test'
@@ -38,6 +39,96 @@ const runAssertion = (records) => {
}
}
+const measurementContract = {
+ id: 'fixture-product',
+ label: 'Fixture product',
+ workContract: 'fixture/product-v1',
+ benchmarkSchema: 'effect-utils-buck2-benchmark/v0',
+ buckTarget: '//fixture:product',
+ runs: 2,
+ assertions: [
+ {
+ id: 'warm-actions',
+ label: 'Warm actions',
+ phase: 'warm-noop',
+ metric: 'actionCount',
+ expectation: { _tag: 'exact', value: 0 },
+ },
+ {
+ id: 'relevant-actions',
+ label: 'Relevant actions',
+ phase: 'relevant-edit',
+ metric: 'actionCount',
+ expectation: { _tag: 'at-least', value: 1 },
+ },
+ ],
+}
+
+const measurementRecords = ({ relevant = [1, 1], omitWarm = false } = {}) => [
+ {
+ kind: 'metadata',
+ schema: measurementContract.benchmarkSchema,
+ target: measurementContract.buckTarget,
+ runId: 'run-1',
+ sha: 'abc123',
+ workContract: measurementContract.workContract,
+ samplePolicy: { runs: 2, warmups: 0 },
+ },
+ ...(!omitWarm
+ ? [0, 0].map((actionCount, sampleIndex) =>
+ Object.assign(sample({ phase: 'warm-noop', actionCount }), {
+ schema: measurementContract.benchmarkSchema,
+ sampleIndex,
+ runId: 'run-1',
+ sha: 'abc123',
+ workContract: measurementContract.workContract,
+ materializationCount: 0,
+ durationMs: 1,
+ }),
+ )
+ : []),
+ ...relevant.map((actionCount, sampleIndex) => ({
+ ...sample({ phase: 'relevant-edit', actionCount }),
+ schema: measurementContract.benchmarkSchema,
+ sampleIndex,
+ runId: 'run-1',
+ sha: 'abc123',
+ workContract: measurementContract.workContract,
+ materializationCount: 1,
+ durationMs: 2,
+ })),
+]
+
+const runMeasurementAdmission = (records, contract = measurementContract) => {
+ const directory = mkdtempSync(join(tmpdir(), 'buck2-measurement-admission-test-'))
+ const input = join(directory, 'raw.jsonl')
+ const output = join(directory, 'measurements.json')
+ writeFileSync(input, records.map((record) => JSON.stringify(record)).join('\n') + '\n')
+ const result = spawnSync(
+ process.execPath,
+ [
+ join(import.meta.dirname, 'assert-invalidation.mjs'),
+ '--contract-json',
+ JSON.stringify(contract),
+ '--output',
+ output,
+ input,
+ ],
+ {
+ encoding: 'utf8',
+ env: {
+ ...process.env,
+ CI_MEASUREMENT_SUBJECT_SHA: 'pr-head-123',
+ CI_MEASUREMENT_CHECKOUT_SHA: 'abc123',
+ },
+ },
+ )
+ const rawEvidence = readFileSync(input, 'utf8')
+ const artifact = existsSync(output) ? JSON.parse(readFileSync(output, 'utf8')) : null
+ rmSync(directory, { recursive: true, force: true })
+ return { result, artifact, rawEvidence }
+}
+
describe('Buck invalidation assertions', () => {
it('accepts role exclusion while explicitly recording the coarse declared-input boundary', () => {
const result = runAssertion(validRecords)
@@ -67,3 +158,69 @@ describe('Buck invalidation assertions', () => {
assert.match(result.stderr, /irrelevant-edit: expected zero actions/u)
})
})
+
+describe('Buck CI measurement admission', () => {
+ it('emits additive schema-v1 assertion and advisory sample evidence', () => {
+ const { result, artifact, rawEvidence } = runMeasurementAdmission(measurementRecords())
+ assert.equal(result.status, 0, result.stderr)
+ assert.equal(artifact.subject.sha, 'pr-head-123')
+ assert.equal(artifact.subject.evidenceSha, 'abc123')
+ assert.equal(
+ rawEvidence,
+ measurementRecords()
+ .map((record) => JSON.stringify(record))
+ .join('\n') + '\n',
+ )
+ assert.equal(artifact.schemaVersion, 1)
+ assert.equal(artifact.completeness.status, 'complete')
+ assert.deepEqual(
+ artifact.observations.find((row) => row.id === 'warm-actions').statistics.samples,
+ [0, 0],
+ )
+ assert.equal(
+ artifact.observations.find((row) => row.id === 'relevant-actions').policy.comparisonMode,
+ 'assertion',
+ )
+ assert.ok(artifact.observations.some((row) => row.id === 'warm-noop-duration'))
+ })
+
+ it('rejects one bad sample hidden by a healthy median', () => {
+ const { result, artifact } = runMeasurementAdmission(measurementRecords({ relevant: [0, 2] }))
+ assert.notEqual(result.status, 0)
+ assert.equal(artifact.completeness.status, 'complete')
+ assert.equal(
+ artifact.observations.find((row) => row.id === 'relevant-actions').assertion.status,
+ 'fail',
+ )
+ })
+
+ it('emits no-verdict evidence and fails when configured samples are missing', () => {
+ const { result, artifact } = runMeasurementAdmission(measurementRecords({ omitWarm: true }))
+ assert.notEqual(result.status, 0)
+ assert.equal(artifact.completeness.status, 'partial')
+ assert.ok(artifact.completeness.missing.some((row) => row.observationId === 'warm-actions'))
+ })
+
+ it('rejects malformed contracts', () => {
+ const malformed = structuredClone(measurementContract)
+ malformed.assertions[0].expectation = { _tag: 'exact', value: -1 }
+ const { result } = runMeasurementAdmission(measurementRecords(), malformed)
+ assert.notEqual(result.status, 0)
+ })
+
+ it('rejects evidence produced from a different checkout', () => {
+ const records = structuredClone(measurementRecords())
+ for (const record of records) {
+ if (record.kind === 'metadata' || record.kind === 'sample') record.sha = 'unexpected-checkout'
+ }
+ const { result, artifact } = runMeasurementAdmission(records)
+ assert.equal(result.status, 1)
+ assert.equal(artifact.completeness.status, 'partial')
+ assert.equal(
+ artifact.observations
+ .filter((observation) => observation.policy.comparisonMode === 'assertion')
+ .every((observation) => observation.assertion?.status === 'no-verdict'),
+ true,
+ )
+ })
+})