Skip to content

Commit a5b89b5

Browse files
cieplypolarcieplypolar
authored
feat: Resolution time benchmark github action (#2239)
1 parent a86350a commit a5b89b5

3 files changed

Lines changed: 190 additions & 28 deletions

File tree

‎.github/workflows/resolution-time.yml‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
name: Resolution time benchmark
2+
3+
on:
4+
pull_request:
5+
6+
jobs:
7+
resolution-time:
8+
if: github.event.pull_request.head.repo.full_name == github.repository
9+
permissions:
10+
contents: read
11+
pull-requests: write
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Install pnpm
15+
uses: pnpm/action-setup@v4
16+
with:
17+
version: 10.27.0
18+
run_install: false
19+
20+
- name: Checkout PR branch
21+
uses: actions/checkout@v4
22+
with:
23+
path: pr-branch
24+
25+
- name: Checkout target branch
26+
uses: actions/checkout@v4
27+
with:
28+
path: target-branch
29+
ref: ${{ github.base_ref }}
30+
31+
- name: Checkout latest release
32+
uses: actions/checkout@v4
33+
with:
34+
path: release-branch
35+
ref: release
36+
37+
- name: Copy resolution-time app to release checkout
38+
run: |
39+
if [ ! -d release-branch/apps/resolution-time ]; then
40+
cp -r target-branch/apps/resolution-time release-branch/apps/resolution-time
41+
echo "Copied resolution-time app from target branch to release checkout"
42+
else
43+
echo "resolution-time app already exists in release checkout"
44+
fi
45+
46+
- name: Set up Node.js
47+
uses: actions/setup-node@v4
48+
with:
49+
node-version: 24.x
50+
cache: 'pnpm'
51+
cache-dependency-path: |
52+
pr-branch/pnpm-lock.yaml
53+
target-branch/pnpm-lock.yaml
54+
release-branch/pnpm-lock.yaml
55+
56+
- name: Install dependencies (PR branch)
57+
working-directory: pr-branch
58+
run: pnpm install --frozen-lockfile
59+
60+
- name: Install dependencies (target branch)
61+
working-directory: target-branch
62+
run: pnpm install --frozen-lockfile
63+
64+
- name: Install dependencies (release branch)
65+
working-directory: release-branch
66+
run: pnpm install --no-frozen-lockfile
67+
68+
- name: Run benchmark (PR branch)
69+
working-directory: pr-branch
70+
run: pnpm --filter resolution-time resolution-time
71+
72+
- name: Run benchmark (target branch)
73+
working-directory: target-branch
74+
run: pnpm --filter resolution-time resolution-time
75+
76+
- name: Run benchmark (release branch)
77+
working-directory: release-branch
78+
run: pnpm --filter resolution-time resolution-time
79+
80+
- name: Generate chart (random)
81+
run: |
82+
node pr-branch/apps/resolution-time/generateChart.ts \
83+
--input "PR:pr-branch/apps/resolution-time/results-random.json" \
84+
--input "main:target-branch/apps/resolution-time/results-random.json" \
85+
--input "release:release-branch/apps/resolution-time/results-random.json" \
86+
--title "Random Branching" \
87+
--xAxisTitle "max depth" \
88+
--yAxisTitle "time (ms)" \
89+
--output chart-random.md
90+
91+
- name: Generate chart (linear)
92+
run: |
93+
node pr-branch/apps/resolution-time/generateChart.ts \
94+
--input "PR:pr-branch/apps/resolution-time/results-linear-recursion.json" \
95+
--input "main:target-branch/apps/resolution-time/results-linear-recursion.json" \
96+
--input "release:release-branch/apps/resolution-time/results-linear-recursion.json" \
97+
--title "Linear Recursion" \
98+
--xAxisTitle "max depth" \
99+
--yAxisTitle "time (ms)" \
100+
--output chart-linear.md
101+
102+
- name: Generate chart (max depth)
103+
run: |
104+
node pr-branch/apps/resolution-time/generateChart.ts \
105+
--input "PR:pr-branch/apps/resolution-time/results-max-depth.json" \
106+
--input "main:target-branch/apps/resolution-time/results-max-depth.json" \
107+
--input "release:release-branch/apps/resolution-time/results-max-depth.json" \
108+
--title "Full Tree" \
109+
--xAxisTitle "max depth" \
110+
--yAxisTitle "time (ms)" \
111+
--output chart-max-depth.md
112+
113+
- name: Prepare comment
114+
run: |
115+
{
116+
echo "## Resolution Time Benchmark"
117+
echo ""
118+
cat chart-random.md
119+
echo ""
120+
cat chart-linear.md
121+
echo ""
122+
cat chart-max-depth.md
123+
} > comparison.md
124+
125+
- name: Comment PR with results
126+
uses: actions/github-script@v7
127+
with:
128+
script: |
129+
const fs = require('fs');
130+
const comparison = fs.readFileSync('comparison.md', 'utf8');
131+
132+
const botCommentIdentifier = '<!-- resolution-time-bot-comment -->';
133+
134+
async function findBotComment(issueNumber) {
135+
if (!issueNumber) return null;
136+
const comments = await github.rest.issues.listComments({
137+
owner: context.repo.owner,
138+
repo: context.repo.repo,
139+
issue_number: issueNumber,
140+
});
141+
return comments.data.find((comment) =>
142+
comment.body.includes(botCommentIdentifier)
143+
);
144+
}
145+
146+
async function createOrUpdateComment(issueNumber) {
147+
if (!issueNumber) {
148+
console.log('No issue number provided. Cannot post or update comment.');
149+
return;
150+
}
151+
152+
const existingComment = await findBotComment(issueNumber);
153+
if (existingComment) {
154+
await github.rest.issues.updateComment({
155+
...context.repo,
156+
comment_id: existingComment.id,
157+
body: botCommentIdentifier + '\n' + comparison,
158+
});
159+
} else {
160+
await github.rest.issues.createComment({
161+
...context.repo,
162+
issue_number: issueNumber,
163+
body: botCommentIdentifier + '\n' + comparison,
164+
});
165+
}
166+
}
167+
168+
const issueNumber = context.issue.number;
169+
if (!issueNumber) {
170+
console.log('No issue number found in context. Skipping comment.');
171+
} else {
172+
await createOrUpdateComment(issueNumber);
173+
}
174+
await core.summary
175+
.addRaw(comparison)
176+
.write();

‎apps/resolution-time/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,14 @@ const myLeafFn = tgpu.comptime(() => {
1818

1919
### Recursive instruction
2020

21-
Use `tgpu.unroll` over `arrayForUnroll(BRANCHING)` and call `instructions[choice()]()()` to branch into other instructions. The `choice()` function handles depth tracking and picks a leaf when at max depth.
21+
Use `tgpu.unroll` over `std.range(n)` and call `instructions[choice()]()()` to branch into other instructions. The `choice()` function handles depth tracking and picks a leaf when at max depth.
2222

2323
```ts
2424
const myRecursiveFn = tgpu.comptime(() => {
2525
return tgpu.fn(() => {
2626
'use gpu';
2727
// ...
28-
for (const _i of tgpu.unroll(arrayForUnroll(BRANCHING))) {
28+
for (const _i of tgpu.unroll(std.range(n))) {
2929
instructions[choice()]()();
3030
}
3131
popDepth(); // REQUIRED — always call at the end, after the unroll

‎apps/resolution-time/procedural.ts‎

Lines changed: 12 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ interface ProcGenConfig {
1212
}
1313

1414
// default config
15-
const SAMPLES = 10;
15+
const SAMPLES = 50;
1616
const config: ProcGenConfig & { samples: number } = {
1717
mainBranching: 2,
1818
branching: 2,
@@ -45,10 +45,6 @@ const state = tgpu.lazy(() => ({
4545
const instructions: TgpuComptime<() => () => void>[] = [];
4646
const LEAF_COUNT = 4;
4747

48-
// TODO: replace it with number, when unroll supports that
49-
const getArrayForUnroll = tgpu.comptime((n: number) => Array.from({ length: n }));
50-
let branchingUnrollArray = getArrayForUnroll(config.branching);
51-
5248
const choice = tgpu.comptime((): number => {
5349
if (state.$.stackDepth == config.maxDepth - 1 || rand() > config.recurseProb) {
5450
state.$.stackDepth++;
@@ -156,7 +152,7 @@ const waveFn = tgpu.comptime(() => {
156152
v = d.vec2f(std.sin(v.x * Math.PI), std.cos(v.y * Math.PI));
157153
const _energy = std.dot(v, v);
158154

159-
for (const _i of tgpu.unroll(branchingUnrollArray)) {
155+
for (const _i of tgpu.unroll(std.range(config.branching))) {
160156
// @ts-expect-error trust me
161157
instructions[choice()]()();
162158
}
@@ -176,7 +172,7 @@ const accFn = tgpu.comptime(() => {
176172
let acc = d.vec2f();
177173
acc = d.vec2f(acc.x + offset.x * scale, acc.y + offset.y * scale);
178174

179-
for (const _i of tgpu.unroll(branchingUnrollArray)) {
175+
for (const _i of tgpu.unroll(std.range(config.branching))) {
180176
// @ts-expect-error trust me
181177
instructions[choice()]()();
182178
}
@@ -198,7 +194,7 @@ const rotateFn = tgpu.comptime(() => {
198194
const s = std.sin(angle);
199195
v = d.vec2f(v.x * c - v.y * s, v.x * s + v.y * c);
200196

201-
for (const _i of tgpu.unroll(branchingUnrollArray)) {
197+
for (const _i of tgpu.unroll(std.range(config.branching))) {
202198
// @ts-expect-error trust me
203199
instructions[choice()]()();
204200
}
@@ -220,7 +216,7 @@ const spiralFn = tgpu.comptime(() => {
220216
const pos = d.vec2f(center.x + radius * std.cos(angle), center.y + radius * std.sin(angle));
221217
const _dist = std.length(pos);
222218

223-
for (const _i of tgpu.unroll(branchingUnrollArray)) {
219+
for (const _i of tgpu.unroll(std.range(config.branching))) {
224220
// @ts-expect-error trust me
225221
instructions[choice()]()();
226222
}
@@ -236,7 +232,7 @@ instructions.push(baseFn, blendFn, thresholdFn, filterFn, waveFn, accFn, rotateF
236232
const main = () => {
237233
'use gpu';
238234

239-
for (const _i of tgpu.unroll(getArrayForUnroll(config.mainBranching))) {
235+
for (const _i of tgpu.unroll(std.range(config.mainBranching))) {
240236
// @ts-expect-error trust me
241237
instructions[choice()]()();
242238
}
@@ -259,10 +255,15 @@ const outDir = resolve(import.meta.dirname ?? '.', '.');
259255

260256
function runBenchmark(input: ProcGenConfig, output: BenchmarkResult[]) {
261257
Object.assign(config, { samples: input.samples ?? SAMPLES }, input);
262-
branchingUnrollArray = getArrayForUnroll(config.branching);
263258

259+
// warmup
264260
for (let i = 0; i < config.samples; i++) {
265261
rand = splitmix32(config.seed);
262+
benchmarkResolve();
263+
}
264+
265+
for (let i = 0; i < config.samples; i++) {
266+
rand = splitmix32((config.seed << i) | (config.seed >> (32 - i)));
266267
const result = benchmarkResolve();
267268
output.push(result);
268269
console.log(
@@ -273,21 +274,6 @@ function runBenchmark(input: ProcGenConfig, output: BenchmarkResult[]) {
273274
}
274275
}
275276

276-
function warmupJIT() {
277-
runBenchmark(
278-
{
279-
mainBranching: 1,
280-
branching: 1,
281-
maxDepth: 1,
282-
recurseProb: 0,
283-
seed: 0.1882 * 2 ** 32,
284-
},
285-
[],
286-
);
287-
}
288-
289-
warmupJIT();
290-
291277
const results: BenchmarkResult[] = [];
292278
const DEPTHS = Array.from({ length: 8 }, (_, i) => i + 1);
293279

0 commit comments

Comments
 (0)