Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 83 additions & 0 deletions .github/scripts/classify-agent-authorship.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#!/usr/bin/env ruby
# frozen_string_literal: true

require "json"
require "optparse"

options = {
github_output: nil,
}

OptionParser.new do |parser|
parser.on("--github-output PATH", "Append key=value outputs for GitHub Actions") do |path|
options[:github_output] = path
end
end.parse!

input = ARGF.read

messages = input.each_line.map do |line|
next if line.strip.empty?

parsed = JSON.parse(line)
if parsed.is_a?(Hash)
parsed.dig("commit", "message") || parsed["message"]
end
end.compact

required_patterns = {
"co_author" => /^Co-Authored-By:\s*Maestro\s+<maestro@evalops\.dev>\s*$/i,
"version" => /^Maestro-Version:\s*\S.*$/i,
"prompt_id" => /^Maestro-Prompt-Id:\s*\S.*$/i,
"approvals_id" => /^Maestro-Approvals-Id:\s*\S.*$/i,
}

marker_pattern = /
^Co-Authored-By:\s*Maestro\s+<maestro@evalops\.dev>\s*$ |
^Maestro-(?:Version|Prompt-Id|Approvals-Id):
/ix
Comment thread
haasonsaas marked this conversation as resolved.

agent_commits = 0
untrailered_commits = 0
incomplete_commits = 0

messages.each do |message|
has_marker = message.lines.any? { |line| line.match?(marker_pattern) }

unless has_marker
untrailered_commits += 1
next
end

agent_commits += 1
missing_required = required_patterns.values.any? do |pattern|
message.lines.none? { |line| line.match?(pattern) }
end
incomplete_commits += 1 if missing_required
end

label =
if agent_commits.positive? && untrailered_commits.positive?
"mixed-authorship"
elsif agent_commits.positive?
"agent-authored"
else
"agent-assisted"
end

outputs = {
"label" => label,
"total_commits" => messages.length,
"agent_commits" => agent_commits,
"untrailered_commits" => untrailered_commits,
"human_commits" => untrailered_commits,
Comment thread
haasonsaas marked this conversation as resolved.
"incomplete_agent_commits" => incomplete_commits,
}

outputs.each { |key, value| puts "#{key}=#{value}" }

if options[:github_output]
File.open(options[:github_output], "a") do |file|
outputs.each { |key, value| file.puts("#{key}=#{value}") }
end
end
Loading
Loading