diff --git a/.github/scripts/classify-agent-authorship.rb b/.github/scripts/classify-agent-authorship.rb deleted file mode 100644 index 7061f46..0000000 --- a/.github/scripts/classify-agent-authorship.rb +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require "json" -require "optparse" - -options = { - github_output: nil, -} - -OptionParser.new do |parser| - parser.on("--github-output PATH", "Append key=value outputs for GitHub Actions") do |path| - options[:github_output] = path - end -end.parse! - -input = ARGF.read - -messages = input.each_line.map do |line| - next if line.strip.empty? - - parsed = JSON.parse(line) - if parsed.is_a?(Hash) - parsed.dig("commit", "message") || parsed["message"] - end -end.compact - -required_patterns = { - "co_author" => /^Co-Authored-By:\s*Maestro\s+\s*$/i, - "version" => /^Maestro-Version:\s*\S.*$/i, - "prompt_id" => /^Maestro-Prompt-Id:\s*\S.*$/i, - "approvals_id" => /^Maestro-Approvals-Id:\s*\S.*$/i, -} - -marker_pattern = / - ^Co-Authored-By:\s*Maestro\s+\s*$ | - ^Maestro-(?:Version|Prompt-Id|Approvals-Id): -/ix - -agent_commits = 0 -untrailered_commits = 0 -incomplete_commits = 0 - -messages.each do |message| - has_marker = message.lines.any? { |line| line.match?(marker_pattern) } - - unless has_marker - untrailered_commits += 1 - next - end - - agent_commits += 1 - missing_required = required_patterns.values.any? do |pattern| - message.lines.none? { |line| line.match?(pattern) } - end - incomplete_commits += 1 if missing_required -end - -label = - if agent_commits.positive? && untrailered_commits.positive? - "mixed-authorship" - elsif agent_commits.positive? - "agent-authored" - else - "agent-assisted" - end - -outputs = { - "label" => label, - "total_commits" => messages.length, - "agent_commits" => agent_commits, - "untrailered_commits" => untrailered_commits, - "human_commits" => untrailered_commits, - "incomplete_agent_commits" => incomplete_commits, -} - -outputs.each { |key, value| puts "#{key}=#{value}" } - -if options[:github_output] - File.open(options[:github_output], "a") do |file| - outputs.each { |key, value| file.puts("#{key}=#{value}") } - end -end diff --git a/.github/scripts/classify_agent_authorship.py b/.github/scripts/classify_agent_authorship.py new file mode 100644 index 0000000..655b2e0 --- /dev/null +++ b/.github/scripts/classify_agent_authorship.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Classify PR commit authorship from a JSONL stream of commits. + +Port of classify-agent-authorship.rb. Ruby is not present on every runner +image this reusable workflow runs on, so the classifier uses python3, which +is proven to exist on all of them. Standard library only. +""" + +import argparse +import json +import re +import sys + +# Ruby \s / \S semantics are ASCII-only; keep re.ASCII so unicode whitespace +# does not change trailer matching relative to the Ruby original. +FLAGS = re.IGNORECASE | re.ASCII + +REQUIRED_PATTERNS = { + "co_author": re.compile(r"^Co-Authored-By:\s*Maestro\s+\s*$", FLAGS), + "version": re.compile(r"^Maestro-Version:\s*\S.*$", FLAGS), + "prompt_id": re.compile(r"^Maestro-Prompt-Id:\s*\S.*$", FLAGS), + "approvals_id": re.compile(r"^Maestro-Approvals-Id:\s*\S.*$", FLAGS), +} + +MARKER_PATTERN = re.compile( + r"^Co-Authored-By:\s*Maestro\s+\s*$" + r"|^Maestro-(?:Version|Prompt-Id|Approvals-Id):", + FLAGS, +) + + +def read_input(paths): + if paths: + chunks = [] + for path in paths: + with open(path, encoding="utf-8") as handle: + chunks.append(handle.read()) + return "".join(chunks) + return sys.stdin.read() + + +def extract_messages(text): + messages = [] + # Split only on "\n" to match Ruby's each_line: jq emits characters like + # U+2028/U+2029 literally inside JSON strings, and str.splitlines() would + # tear one JSON record into unparseable fragments. + for line in text.split("\n"): + if not line.strip(): + continue + parsed = json.loads(line) + if isinstance(parsed, dict): + commit = parsed.get("commit") + message = commit.get("message") if isinstance(commit, dict) else None + if message is None: + message = parsed.get("message") + if message is not None: + messages.append(message) + return messages + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--github-output", + metavar="PATH", + help="Append key=value outputs for GitHub Actions", + ) + parser.add_argument("files", nargs="*") + args = parser.parse_args() + + messages = extract_messages(read_input(args.files)) + + agent_commits = 0 + untrailered_commits = 0 + incomplete_commits = 0 + + for message in messages: + # Ruby each_line splits on "\n" only; keep that exact segmentation. + parts = message.split("\n") + lines = [part + "\n" for part in parts[:-1]] + if parts[-1]: + lines.append(parts[-1]) + has_marker = any(MARKER_PATTERN.search(line) for line in lines) + + if not has_marker: + untrailered_commits += 1 + continue + + agent_commits += 1 + missing_required = any( + not any(pattern.search(line) for line in lines) + for pattern in REQUIRED_PATTERNS.values() + ) + if missing_required: + incomplete_commits += 1 + + if agent_commits > 0 and untrailered_commits > 0: + label = "mixed-authorship" + elif agent_commits > 0: + label = "agent-authored" + else: + label = "agent-assisted" + + outputs = { + "label": label, + "total_commits": len(messages), + "agent_commits": agent_commits, + "untrailered_commits": untrailered_commits, + "human_commits": untrailered_commits, + "incomplete_agent_commits": incomplete_commits, + } + + for key, value in outputs.items(): + print(f"{key}={value}") + + if args.github_output: + with open(args.github_output, "a", encoding="utf-8") as handle: + for key, value in outputs.items(): + handle.write(f"{key}={value}\n") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/agent-authorship-label.yml b/.github/workflows/agent-authorship-label.yml index 0bc0c0f..d9c6112 100644 --- a/.github/workflows/agent-authorship-label.yml +++ b/.github/workflows/agent-authorship-label.yml @@ -1,5 +1,12 @@ name: agent-authorship-label +# Reusable org authorship labeler, called by repositories across the +# organisation. It runs on whatever runner the caller names, so it may only +# depend on tools proven to exist on every runner image we operate. Ruby is +# absent from newer images (fleet-wide `ruby: command not found`, see +# evalops/platform#5205), so the classifier runs on python3, asserted up +# front by the preflight. + on: workflow_call: inputs: @@ -36,10 +43,16 @@ jobs: ref: ${{ inputs.helper_ref }} path: org-defaults - - name: Set up Ruby - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 - with: - ruby-version: "3.3.9" + - name: Preflight the classification toolchain + shell: bash + run: | + set -euo pipefail + + if ! command -v python3 >/dev/null 2>&1; then + echo "::error::python3 is not on this runner (${RUNNER_NAME:-unknown}). agent-authorship-label cannot classify commits without it." + exit 1 + fi + python3 -V - name: Resolve pull request id: pr @@ -70,7 +83,7 @@ jobs: shell: bash run: | set -euo pipefail - ruby org-defaults/.github/scripts/classify-agent-authorship.rb \ + python3 org-defaults/.github/scripts/classify_agent_authorship.py \ --github-output "${GITHUB_OUTPUT}" \ commits.jsonl diff --git a/test/classify_agent_authorship_test.py b/test/classify_agent_authorship_test.py new file mode 100644 index 0000000..e29da7f --- /dev/null +++ b/test/classify_agent_authorship_test.py @@ -0,0 +1,93 @@ +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / ".github" / "scripts" / "classify_agent_authorship.py" + + +def parse_outputs(text): + return dict(line.split("=", 1) for line in text.splitlines()) + + +def classify(commits, github_output=None): + input_data = "\n".join(json.dumps(commit) for commit in commits) + args = [sys.executable, str(SCRIPT)] + if github_output: + args += ["--github-output", github_output] + result = subprocess.run( + args, input=input_data, capture_output=True, text=True + ) + assert result.returncode == 0, result.stderr + return parse_outputs(result.stdout) + + +class ClassifyAgentAuthorshipTest(unittest.TestCase): + def test_untrailered_commits_are_agent_assisted(self): + outputs = classify([{"sha": "abc", "message": "fix: regular change"}]) + + self.assertEqual("agent-assisted", outputs["label"]) + self.assertEqual("1", outputs["total_commits"]) + self.assertEqual("0", outputs["agent_commits"]) + self.assertEqual("1", outputs["untrailered_commits"]) + self.assertEqual("0", outputs["incomplete_agent_commits"]) + + def test_complete_maestro_trailers_are_agent_authored(self): + outputs = classify( + [ + { + "sha": "abc", + "message": ( + "feat: ship change\n" + "\n" + "Co-Authored-By: Maestro \n" + "Maestro-Version: 2026.04.28 / gpt-5\n" + "Maestro-Prompt-Id: prompt-123\n" + "Maestro-Approvals-Id: approval-456\n" + ), + } + ] + ) + + self.assertEqual("agent-authored", outputs["label"]) + self.assertEqual("1", outputs["agent_commits"]) + self.assertEqual("0", outputs["untrailered_commits"]) + self.assertEqual("0", outputs["incomplete_agent_commits"]) + + def test_mixed_authorship_and_incomplete_trailers_are_reported(self): + outputs = classify( + [ + { + "sha": "abc", + "message": ( + "feat: partial agent change\n" + "\n" + "Co-Authored-By: Maestro \n" + "Maestro-Version: 2026.04.28 / gpt-5\n" + ), + }, + {"sha": "def", "message": "docs: human follow-up"}, + ] + ) + + self.assertEqual("mixed-authorship", outputs["label"]) + self.assertEqual("1", outputs["agent_commits"]) + self.assertEqual("1", outputs["untrailered_commits"]) + self.assertEqual("1", outputs["incomplete_agent_commits"]) + + def test_github_output_file_gets_same_outputs(self): + with tempfile.NamedTemporaryFile(mode="r", suffix="github-output") as handle: + outputs = classify( + [{"sha": "abc", "message": "fix: regular change"}], + github_output=handle.name, + ) + file_outputs = parse_outputs(handle.read()) + + self.assertEqual(outputs, file_outputs) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/classify_agent_authorship_test.rb b/test/classify_agent_authorship_test.rb deleted file mode 100644 index eec8a3d..0000000 --- a/test/classify_agent_authorship_test.rb +++ /dev/null @@ -1,83 +0,0 @@ -# frozen_string_literal: true - -require "json" -require "minitest/autorun" -require "open3" -require "tempfile" - -class ClassifyAgentAuthorshipTest < Minitest::Test - ROOT = File.expand_path("..", __dir__) - SCRIPT = File.join(ROOT, ".github/scripts/classify-agent-authorship.rb") - - def test_untrailered_commits_are_agent_assisted - outputs = classify([{ "sha" => "abc", "message" => "fix: regular change" }]) - - assert_equal "agent-assisted", outputs.fetch("label") - assert_equal "1", outputs.fetch("total_commits") - assert_equal "0", outputs.fetch("agent_commits") - assert_equal "1", outputs.fetch("untrailered_commits") - assert_equal "0", outputs.fetch("incomplete_agent_commits") - end - - def test_complete_maestro_trailers_are_agent_authored - outputs = classify([{ "sha" => "abc", "message" => <<~MSG }]) - feat: ship change - - Co-Authored-By: Maestro - Maestro-Version: 2026.04.28 / gpt-5 - Maestro-Prompt-Id: prompt-123 - Maestro-Approvals-Id: approval-456 - MSG - - assert_equal "agent-authored", outputs.fetch("label") - assert_equal "1", outputs.fetch("agent_commits") - assert_equal "0", outputs.fetch("untrailered_commits") - assert_equal "0", outputs.fetch("incomplete_agent_commits") - end - - def test_mixed_authorship_and_incomplete_trailers_are_reported - outputs = classify( - [ - { "sha" => "abc", "message" => <<~MSG }, - feat: partial agent change - - Co-Authored-By: Maestro - Maestro-Version: 2026.04.28 / gpt-5 - MSG - { "sha" => "def", "message" => "docs: human follow-up" }, - ], - ) - - assert_equal "mixed-authorship", outputs.fetch("label") - assert_equal "1", outputs.fetch("agent_commits") - assert_equal "1", outputs.fetch("untrailered_commits") - assert_equal "1", outputs.fetch("incomplete_agent_commits") - end - - def test_github_output_file_gets_same_outputs - Tempfile.create("github-output") do |file| - outputs = classify( - [{ "sha" => "abc", "message" => "fix: regular change" }], - github_output: file.path, - ) - file_outputs = parse_outputs(File.read(file.path)) - - assert_equal outputs, file_outputs - end - end - - private - - def classify(commits, github_output: nil) - input = commits.map(&:to_json).join("\n") - args = ["ruby", SCRIPT] - args += ["--github-output", github_output] if github_output - stdout, stderr, status = Open3.capture3(*args, stdin_data: input) - assert status.success?, stderr - parse_outputs(stdout) - end - - def parse_outputs(text) - text.each_line(chomp: true).to_h { |line| line.split("=", 2) } - end -end diff --git a/test/workflow_pr_ref_guard_test.rb b/test/workflow_pr_ref_guard_test.rb index 721da1a..10770af 100644 --- a/test/workflow_pr_ref_guard_test.rb +++ b/test/workflow_pr_ref_guard_test.rb @@ -59,22 +59,33 @@ def test_agent_authorship_label_apply_is_best_effort_on_token_denial ) end - def test_agent_authorship_sets_up_pinned_ruby_before_classification + def test_agent_authorship_classification_runs_on_python3_without_ruby workflow = YAML.safe_load( File.read(File.join(root, ".github", "workflows", "agent-authorship-label.yml")), aliases: true, ) steps = workflow.fetch("jobs").fetch("label").fetch("steps") - setup_index = steps.index { |step| step["name"] == "Set up Ruby" } + + setup_ruby = steps.select { |step| step["uses"].to_s.include?("ruby/setup-ruby") } + assert_empty( + setup_ruby, + "Ruby is absent from newer runner images; the classifier must run on python3, not a live Ruby install.", + ) + + preflight_index = steps.index { |step| step["name"] == "Preflight the classification toolchain" } classify_index = steps.index { |step| step["name"] == "Classify authorship" } - refute_nil setup_index, "The reusable workflow must provision Ruby on minimal ARC runners." + refute_nil preflight_index, "The reusable workflow must assert python3 exists before relying on it." refute_nil classify_index - assert_operator setup_index, :<, classify_index + assert_operator preflight_index, :<, classify_index + + preflight_run = steps.fetch(preflight_index).fetch("run") + assert_includes preflight_run, "command -v python3" - setup = steps.fetch(setup_index) - assert_match(%r{\Aruby/setup-ruby@[0-9a-f]{40}\z}, setup.fetch("uses")) - assert_match(/\A\d+\.\d+\.\d+\z/, setup.fetch("with").fetch("ruby-version")) + classify_run = steps.fetch(classify_index).fetch("run") + assert_includes classify_run, "python3" + assert_includes classify_run, "classify_agent_authorship.py" + refute_includes classify_run, "ruby " end private