From 3d0bc7c61613c8d1826fc68160751ab89c8ae02f Mon Sep 17 00:00:00 2001 From: Jonathan Haas Date: Sun, 26 Jul 2026 12:07:10 -0700 Subject: [PATCH] fix(ci): provision Ruby for authorship labels --- .github/scripts/classify-agent-authorship.rb | 83 +++++ .github/workflows/agent-authorship-label.yml | 323 +++++++++++++++++++ test/classify_agent_authorship_test.rb | 83 +++++ test/workflow_pr_ref_guard_test.rb | 93 ++++++ 4 files changed, 582 insertions(+) create mode 100644 .github/scripts/classify-agent-authorship.rb create mode 100644 .github/workflows/agent-authorship-label.yml create mode 100644 test/classify_agent_authorship_test.rb create mode 100644 test/workflow_pr_ref_guard_test.rb diff --git a/.github/scripts/classify-agent-authorship.rb b/.github/scripts/classify-agent-authorship.rb new file mode 100644 index 0000000..7061f46 --- /dev/null +++ b/.github/scripts/classify-agent-authorship.rb @@ -0,0 +1,83 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "json" +require "optparse" + +options = { + github_output: nil, +} + +OptionParser.new do |parser| + parser.on("--github-output PATH", "Append key=value outputs for GitHub Actions") do |path| + options[:github_output] = path + end +end.parse! + +input = ARGF.read + +messages = input.each_line.map do |line| + next if line.strip.empty? + + parsed = JSON.parse(line) + if parsed.is_a?(Hash) + parsed.dig("commit", "message") || parsed["message"] + end +end.compact + +required_patterns = { + "co_author" => /^Co-Authored-By:\s*Maestro\s+\s*$/i, + "version" => /^Maestro-Version:\s*\S.*$/i, + "prompt_id" => /^Maestro-Prompt-Id:\s*\S.*$/i, + "approvals_id" => /^Maestro-Approvals-Id:\s*\S.*$/i, +} + +marker_pattern = / + ^Co-Authored-By:\s*Maestro\s+\s*$ | + ^Maestro-(?:Version|Prompt-Id|Approvals-Id): +/ix + +agent_commits = 0 +untrailered_commits = 0 +incomplete_commits = 0 + +messages.each do |message| + has_marker = message.lines.any? { |line| line.match?(marker_pattern) } + + unless has_marker + untrailered_commits += 1 + next + end + + agent_commits += 1 + missing_required = required_patterns.values.any? do |pattern| + message.lines.none? { |line| line.match?(pattern) } + end + incomplete_commits += 1 if missing_required +end + +label = + if agent_commits.positive? && untrailered_commits.positive? + "mixed-authorship" + elsif agent_commits.positive? + "agent-authored" + else + "agent-assisted" + end + +outputs = { + "label" => label, + "total_commits" => messages.length, + "agent_commits" => agent_commits, + "untrailered_commits" => untrailered_commits, + "human_commits" => untrailered_commits, + "incomplete_agent_commits" => incomplete_commits, +} + +outputs.each { |key, value| puts "#{key}=#{value}" } + +if options[:github_output] + File.open(options[:github_output], "a") do |file| + outputs.each { |key, value| file.puts("#{key}=#{value}") } + end +end diff --git a/.github/workflows/agent-authorship-label.yml b/.github/workflows/agent-authorship-label.yml new file mode 100644 index 0000000..0bc0c0f --- /dev/null +++ b/.github/workflows/agent-authorship-label.yml @@ -0,0 +1,323 @@ +name: agent-authorship-label + +on: + workflow_call: + inputs: + fail_on_incomplete: + description: "Fail when a Maestro-marked commit is missing one or more required Maestro trailers" + required: false + type: boolean + default: false + runner_label: + description: "Runner label used for the label job" + required: false + type: string + default: ubuntu-latest + helper_ref: + description: "evalops/.github ref used to checkout helper scripts" + required: false + type: string + default: main + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + label: + runs-on: ${{ inputs.runner_label }} + timeout-minutes: 10 + steps: + - name: Checkout org workflow helpers + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + repository: evalops/.github + ref: ${{ inputs.helper_ref }} + path: org-defaults + + - name: Set up Ruby + uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 + with: + ruby-version: "3.3.9" + + - name: Resolve pull request + id: pr + shell: bash + run: | + set -euo pipefail + + number="$(jq -r '.pull_request.number // empty' "${GITHUB_EVENT_PATH}")" + if [ -z "${number}" ]; then + echo "::error::agent-authorship-label must run from a pull_request or pull_request_target event." + exit 1 + fi + + echo "number=${number}" >> "${GITHUB_OUTPUT}" + + - name: Fetch pull request commits + shell: bash + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ steps.pr.outputs.number }} + run: | + set -euo pipefail + gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/commits" \ + --jq '.[] | {sha: .sha, message: .commit.message}' > commits.jsonl + + - name: Classify authorship + id: classify + shell: bash + run: | + set -euo pipefail + ruby org-defaults/.github/scripts/classify-agent-authorship.rb \ + --github-output "${GITHUB_OUTPUT}" \ + commits.jsonl + + - name: Ensure authorship labels exist + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + + label_api_denied() { + local output_file="$1" + grep -qiE '(Bad credentials|HTTP 401|Resource not accessible|HTTP 403)' "${output_file}" + } + + warn_label_api_denied() { + local action="$1" + local output_file="$2" + echo "::warning::Skipping authorship label maintenance while ${action}; GitHub API denied label access." + cat "${output_file}" >&2 + } + + ensure_label() { + local name="$1" + local color="$2" + local description="$3" + local label_json + local lookup_stderr + local lookup_status=0 + lookup_stderr="$(mktemp)" + label_json="$(gh api "repos/${GITHUB_REPOSITORY}/labels/${name}" 2>"${lookup_stderr}")" || lookup_status=$? + + if [ "${lookup_status}" -eq 0 ]; then + rm -f "${lookup_stderr}" + local existing_color + local existing_description + existing_color="$(jq -r '.color // ""' <<<"${label_json}")" + existing_description="$(jq -r '.description // ""' <<<"${label_json}")" + + if [ "${existing_color}" != "${color}" ] || [ "${existing_description}" != "${description}" ]; then + local update_stderr + local update_status=0 + update_stderr="$(mktemp)" + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/labels/${name}" \ + -f color="${color}" \ + -f description="${description}" >/dev/null 2>"${update_stderr}" || update_status=$? + if [ "${update_status}" -ne 0 ]; then + if label_api_denied "${update_stderr}"; then + warn_label_api_denied "updating ${name}" "${update_stderr}" + rm -f "${update_stderr}" + return 0 + fi + cat "${update_stderr}" >&2 + rm -f "${update_stderr}" + return "${update_status}" + fi + rm -f "${update_stderr}" + fi + return 0 + fi + + if label_api_denied "${lookup_stderr}"; then + warn_label_api_denied "looking up ${name}" "${lookup_stderr}" + rm -f "${lookup_stderr}" + return 0 + fi + + if ! grep -qiE '(not found|404)' "${lookup_stderr}" && ! jq -e '(.status | tostring) == "404" or .message == "Not Found"' <<<"${label_json}" >/dev/null 2>&1; then + cat "${lookup_stderr}" >&2 + if [ -n "${label_json}" ]; then + printf '%s\n' "${label_json}" >&2 + fi + rm -f "${lookup_stderr}" + return "${lookup_status}" + fi + rm -f "${lookup_stderr}" + + local output_file + local create_status=0 + output_file="$(mktemp)" + + gh api --method POST "repos/${GITHUB_REPOSITORY}/labels" \ + -f name="${name}" \ + -f color="${color}" \ + -f description="${description}" >"${output_file}" 2>&1 || create_status=$? + + if [ "${create_status}" -eq 0 ]; then + rm -f "${output_file}" + return 0 + fi + + if grep -qiE '(already.?exists|already_exists|Validation Failed)' "${output_file}"; then + echo "Label ${name} was created concurrently; updating metadata if needed." + rm -f "${output_file}" + create_status=0 + output_file="$(mktemp)" + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/labels/${name}" \ + -f color="${color}" \ + -f description="${description}" >"${output_file}" 2>&1 || create_status=$? + if [ "${create_status}" -ne 0 ]; then + if label_api_denied "${output_file}"; then + warn_label_api_denied "updating concurrently-created ${name}" "${output_file}" + rm -f "${output_file}" + return 0 + fi + cat "${output_file}" >&2 + rm -f "${output_file}" + return "${create_status}" + fi + rm -f "${output_file}" + return 0 + fi + + if label_api_denied "${output_file}"; then + warn_label_api_denied "creating ${name}" "${output_file}" + rm -f "${output_file}" + return 0 + fi + + cat "${output_file}" >&2 + rm -f "${output_file}" + return "${create_status}" + } + + ensure_label "agent-authored" "6f42c1" "All PR commits carry explicit Maestro authorship trailers" + ensure_label "agent-assisted" "1d76db" "PR commits are assumed LLM-assisted but do not carry explicit Maestro trailers" + ensure_label "mixed-authorship" "fbca04" "Some PR commits carry explicit Maestro trailers and some are untrailered" + + - name: Apply authorship label + shell: bash + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ steps.pr.outputs.number }} + AUTHORSHIP_LABEL: ${{ steps.classify.outputs.label }} + run: | + set -euo pipefail + + label_api_denied() { + local output_file="$1" + grep -qiE '(Bad credentials|HTTP 401|Resource not accessible|HTTP 403)' "${output_file}" + } + + skip_label_apply() { + local action="$1" + local output_file="$2" + echo "::warning::Skipping authorship label apply while ${action}; GitHub API denied label access." + cat "${output_file}" >&2 + exit 0 + } + + current_labels_file="$(mktemp)" + current_labels_stderr="$(mktemp)" + current_labels_status=0 + gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels" \ + --jq '.[].name' >"${current_labels_file}" 2>"${current_labels_stderr}" || current_labels_status=$? + if [ "${current_labels_status}" -ne 0 ]; then + if label_api_denied "${current_labels_stderr}"; then + skip_label_apply "listing labels on #${PR_NUMBER}" "${current_labels_stderr}" + fi + cat "${current_labels_stderr}" >&2 + rm -f "${current_labels_file}" "${current_labels_stderr}" + exit "${current_labels_status}" + fi + current_labels="$(cat "${current_labels_file}")" + rm -f "${current_labels_file}" "${current_labels_stderr}" + + has_label() { + local label="$1" + grep -Fxq "${label}" <<<"${current_labels}" + } + + authorship_labels=(agent-authored agent-assisted mixed-authorship human-authored) + conflicting_labels=() + has_desired_label=false + + for label in "${authorship_labels[@]}"; do + if ! has_label "${label}"; then + continue + fi + if [ "${label}" = "${AUTHORSHIP_LABEL}" ]; then + has_desired_label=true + else + conflicting_labels+=("${label}") + fi + done + + if [ "${has_desired_label}" = true ] && [ "${#conflicting_labels[@]}" -eq 0 ]; then + echo "Authorship label ${AUTHORSHIP_LABEL} is already current on #${PR_NUMBER}; no label changes needed." + exit 0 + fi + + delete_label_if_present() { + local label="$1" + local stderr_file + local status=0 + stderr_file="$(mktemp)" + + gh api --method DELETE \ + "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${label}" >/dev/null 2>"${stderr_file}" || status=$? + + if [ "${status}" -eq 0 ]; then + rm -f "${stderr_file}" + return 0 + fi + + if grep -qiE '(not found|404)' "${stderr_file}"; then + echo "Label ${label} was already absent; continuing." + rm -f "${stderr_file}" + return 0 + fi + + if label_api_denied "${stderr_file}"; then + skip_label_apply "removing ${label} from #${PR_NUMBER}" "${stderr_file}" + fi + + cat "${stderr_file}" >&2 + rm -f "${stderr_file}" + return "${status}" + } + + for label in "${conflicting_labels[@]}"; do + delete_label_if_present "${label}" + done + + if [ "${has_desired_label}" != true ]; then + add_stderr="$(mktemp)" + add_status=0 + gh api --method POST "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels" \ + -f "labels[]=${AUTHORSHIP_LABEL}" >/dev/null 2>"${add_stderr}" || add_status=$? + if [ "${add_status}" -ne 0 ]; then + if label_api_denied "${add_stderr}"; then + skip_label_apply "adding ${AUTHORSHIP_LABEL} to #${PR_NUMBER}" "${add_stderr}" + fi + cat "${add_stderr}" >&2 + rm -f "${add_stderr}" + exit "${add_status}" + fi + rm -f "${add_stderr}" + fi + + echo "Applied ${AUTHORSHIP_LABEL} to #${PR_NUMBER}." + + - name: Check required Maestro trailers + if: ${{ inputs.fail_on_incomplete && steps.classify.outputs.incomplete_agent_commits != '0' }} + shell: bash + env: + INCOMPLETE_AGENT_COMMITS: ${{ steps.classify.outputs.incomplete_agent_commits }} + run: | + echo "::error::${INCOMPLETE_AGENT_COMMITS} Maestro-marked commit(s) are missing required authorship trailers." + exit 1 diff --git a/test/classify_agent_authorship_test.rb b/test/classify_agent_authorship_test.rb new file mode 100644 index 0000000..eec8a3d --- /dev/null +++ b/test/classify_agent_authorship_test.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +require "json" +require "minitest/autorun" +require "open3" +require "tempfile" + +class ClassifyAgentAuthorshipTest < Minitest::Test + ROOT = File.expand_path("..", __dir__) + SCRIPT = File.join(ROOT, ".github/scripts/classify-agent-authorship.rb") + + def test_untrailered_commits_are_agent_assisted + outputs = classify([{ "sha" => "abc", "message" => "fix: regular change" }]) + + assert_equal "agent-assisted", outputs.fetch("label") + assert_equal "1", outputs.fetch("total_commits") + assert_equal "0", outputs.fetch("agent_commits") + assert_equal "1", outputs.fetch("untrailered_commits") + assert_equal "0", outputs.fetch("incomplete_agent_commits") + end + + def test_complete_maestro_trailers_are_agent_authored + outputs = classify([{ "sha" => "abc", "message" => <<~MSG }]) + feat: ship change + + Co-Authored-By: Maestro + Maestro-Version: 2026.04.28 / gpt-5 + Maestro-Prompt-Id: prompt-123 + Maestro-Approvals-Id: approval-456 + MSG + + assert_equal "agent-authored", outputs.fetch("label") + assert_equal "1", outputs.fetch("agent_commits") + assert_equal "0", outputs.fetch("untrailered_commits") + assert_equal "0", outputs.fetch("incomplete_agent_commits") + end + + def test_mixed_authorship_and_incomplete_trailers_are_reported + outputs = classify( + [ + { "sha" => "abc", "message" => <<~MSG }, + feat: partial agent change + + Co-Authored-By: Maestro + Maestro-Version: 2026.04.28 / gpt-5 + MSG + { "sha" => "def", "message" => "docs: human follow-up" }, + ], + ) + + assert_equal "mixed-authorship", outputs.fetch("label") + assert_equal "1", outputs.fetch("agent_commits") + assert_equal "1", outputs.fetch("untrailered_commits") + assert_equal "1", outputs.fetch("incomplete_agent_commits") + end + + def test_github_output_file_gets_same_outputs + Tempfile.create("github-output") do |file| + outputs = classify( + [{ "sha" => "abc", "message" => "fix: regular change" }], + github_output: file.path, + ) + file_outputs = parse_outputs(File.read(file.path)) + + assert_equal outputs, file_outputs + end + end + + private + + def classify(commits, github_output: nil) + input = commits.map(&:to_json).join("\n") + args = ["ruby", SCRIPT] + args += ["--github-output", github_output] if github_output + stdout, stderr, status = Open3.capture3(*args, stdin_data: input) + assert status.success?, stderr + parse_outputs(stdout) + end + + def parse_outputs(text) + text.each_line(chomp: true).to_h { |line| line.split("=", 2) } + end +end diff --git a/test/workflow_pr_ref_guard_test.rb b/test/workflow_pr_ref_guard_test.rb new file mode 100644 index 0000000..721da1a --- /dev/null +++ b/test/workflow_pr_ref_guard_test.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "yaml" + +class WorkflowPrRefGuardTest < Minitest::Test + def test_review_workflows_do_not_depend_on_synthetic_pull_request_merge_refs + offenders = [] + workflow_paths.each do |path| + File.readlines(path, chomp: true).each_with_index do |line, index| + next unless line.match?(%r{refs/pull/.*/merge}) + + offenders << "#{relative_path(path)}:#{index + 1}: #{line.strip}" + end + end + + assert_empty( + offenders, + "Synthetic PR merge refs disappear for open conflicting PRs. " \ + "Review automation should check out refs/pull//head, then fetch base/head SHAs for diffs.\n" \ + "#{offenders.join("\n")}" + ) + end + + def test_upload_artifact_steps_set_retention_days + offenders = [] + workflow_paths.each do |path| + data = YAML.safe_load(File.read(path), aliases: true) || {} + jobs = data.fetch("jobs", {}) || {} + jobs.each do |job_name, job| + Array(job && job["steps"]).each_with_index do |step, index| + next unless step.is_a?(Hash) && step["uses"].to_s.include?("actions/upload-artifact") + + with = step["with"].is_a?(Hash) ? step["with"] : {} + next if with.key?("retention-days") + + offenders << "#{relative_path(path)} #{job_name} step #{index + 1}" + end + end + end + + assert_empty( + offenders, + "Every upload-artifact step must set retention-days so diagnostic artifacts do not silently keep the repo default.\n" \ + "#{offenders.join("\n")}" + ) + end + + def test_agent_authorship_label_apply_is_best_effort_on_token_denial + workflow = File.read(File.join(root, ".github", "workflows", "agent-authorship-label.yml")) + + assert_includes workflow, "Skipping authorship label apply" + assert_match(/Bad credentials\|HTTP 401\|Resource not accessible\|HTTP 403/, workflow) + assert_operator( + workflow.index("Apply authorship label"), + :<, + workflow.index("Check required Maestro trailers"), + "The required trailer gate should still run after best-effort label application.", + ) + end + + def test_agent_authorship_sets_up_pinned_ruby_before_classification + workflow = YAML.safe_load( + File.read(File.join(root, ".github", "workflows", "agent-authorship-label.yml")), + aliases: true, + ) + steps = workflow.fetch("jobs").fetch("label").fetch("steps") + setup_index = steps.index { |step| step["name"] == "Set up Ruby" } + classify_index = steps.index { |step| step["name"] == "Classify authorship" } + + refute_nil setup_index, "The reusable workflow must provision Ruby on minimal ARC runners." + refute_nil classify_index + assert_operator setup_index, :<, classify_index + + setup = steps.fetch(setup_index) + assert_match(%r{\Aruby/setup-ruby@[0-9a-f]{40}\z}, setup.fetch("uses")) + assert_match(/\A\d+\.\d+\.\d+\z/, setup.fetch("with").fetch("ruby-version")) + end + + private + + def root + File.expand_path("..", __dir__) + end + + def workflow_paths + Dir.glob(File.join(root, ".github", "{workflows,workflow-templates}", "*.{yml,yaml}")).sort + end + + def relative_path(path) + path.delete_prefix("#{root}/") + end +end