git_stress_eval/chatdsl_opencode_eval.chatdsl at main · jon2allen/git_stress_eval · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
# chatdsl_opencode_eval.chatdsl
# Usage: /script chatdsl_opencode_eval.chatdsl x="/Users/jon2allen/projects/git_stress" y="cohere_north" z="/Users/jon2allen/projects/git_stress_eval/eval_output.txt"

# Parameter mapping and defaults
if ${x} != "" then set target_repo = ${x}
if ${target_repo} == "" then set target_repo = "/Users/jon2allen/projects/git_stress"

if ${y} != "" then set target_model = ${y}
if ${target_model} == "" then set target_model = "cohere_north"

set eval_dir = "/Users/jon2allen/projects/git_stress_eval"

set output_name = "eval_output.txt"
if ${z} != "" then set output_name = ${z}

# If the user passed the full path of the default log, use it, otherwise prefix relative names with eval_dir
if ${output_name} == "/Users/jon2allen/projects/git_stress_eval/eval_output.txt" then set output_log = "/Users/jon2allen/projects/git_stress_eval/eval_output.txt"
if ${output_log} == "" then set output_log = "${eval_dir}/${output_name}"

# Map Chatybot model name to opencode model name
set opencode_model = ""
if ${target_model} == "cohere_north" then set opencode_model = "openrouter/cohere/north-mini-code:free"
if ${target_model} == "gemma_4_27" then set opencode_model = "openrouter/google/gemma-4-26b-a4b-it"
# Fallback default
if ${opencode_model} == "" then set opencode_model = "openrouter/cohere/north-mini-code:free"

# Setup output log files for each step
set step1_log = "${output_log}.step1"
set step2_log = "${output_log}.step2"
set step3_log = "${output_log}.step3"
set step4_log = "${output_log}.step4"

# Setup opencode commands
set step1_cmd = "opencode run --dangerously-skip-permissions --model ${opencode_model} 'Please list the files in the repository located at ${target_repo}, report the current git branch, and read the contents of the main implementation file blockchain_non_oo.pas.'"
set step2_cmd = "opencode run --dangerously-skip-permissions --model ${opencode_model} 'Please run the commands to check out the branch blockchain-oo inside the repository at ${target_repo}, list its files, check its git log history, and read the contents of the main implementation file blockchain_oo.pas.'"
set step3_cmd = "opencode run --dangerously-skip-permissions --model ${opencode_model} 'Please run the commands to check out the branch blockchain-non-oo-generic inside the repository at ${target_repo}, list its files, explain the difference in file structure compared to the other branches, and read the contents of the main implementation file blockchain_non_oo_generic.pas.'"
set step4_cmd = "opencode run --dangerously-skip-permissions --model ${opencode_model} 'Compare all the branches you analyzed in ${target_repo} (blockchain-non-oo, blockchain-oo, and blockchain-non-oo-generic). Summarize: 1. What architectural and programming styles each branch implements. 2. Where the intentional validation flaws are located in the files.'"

/echo "Starting Evaluation of ${target_repo} using model ${target_model} via opencode..."

/logging start

# 1. Analyze target repository on active branch (blockchain-non-oo)
/echo "Step 1: Listing files and checking git status..."
/run "${step1_cmd}"
/savevar RUN_COMPLETION ${step1_log}

# 2. Analyze repository on blockchain-oo branch
/echo "Step 2: Checking out blockchain-oo and listing files..."
/run "${step2_cmd}"
/savevar RUN_COMPLETION ${step2_log}

# 3. Analyze repository on blockchain-non-oo-generic branch
/echo "Step 3: Checking out blockchain-non-oo-generic and checking history..."
/run "${step3_cmd}"
/savevar RUN_COMPLETION ${step3_log}

# 4. Final summary and verification
/echo "Step 4: Creating a consolidated summary report..."
/run "${step4_cmd}"
/savevar RUN_COMPLETION ${step4_log}

/logging end

# 5. Summarize using mistral_1 model
/echo "Running final summary using mistral_1 model..."

# Configure Chatybot to use mistral_1 model
/model mistral_1

# Load each step's output log into a Chatybot file bank
/filebank1 ${step1_log}
/filebank2 ${step2_log}
/filebank3 ${step3_log}
/filebank4 ${step4_log}

# Send the consolidated prompt to mistral_1
/multiline
Please analyze and summarize the following execution outputs for the branches of the git_stress repository:

=== Step 1: blockchain-non-oo Analysis ===
${filebank1}

=== Step 2: blockchain-oo Analysis ===
${filebank2}

=== Step 3: blockchain-non-oo-generic Analysis ===
${filebank3}

=== Step 4: Comparison Summary ===
${filebank4}
;;

# Save the final summary from mistral_1 to the output log file
/save ${output_log}

/echo "Evaluation script completed. Final summary written to ${output_log}."