diff --git a/docs/nav.json b/docs/nav.json index f3d3f07fc..08fd82622 100644 --- a/docs/nav.json +++ b/docs/nav.json @@ -936,6 +936,10 @@ "label": "Benchmarks", "path": "project/benchmark-results.md" }, + { + "label": "Performance Scorecards", + "path": "project/performance-scorecards.md" + }, { "label": "Compile Profile", "path": "project/compile-profile.md" diff --git a/docs/project/performance-scorecards.md b/docs/project/performance-scorecards.md new file mode 100644 index 000000000..76fd9c77f --- /dev/null +++ b/docs/project/performance-scorecards.md @@ -0,0 +1,46 @@ +# Local performance scorecards (#727) + +The performance epic is a collection of separately audited comparisons, not +one aggregate speedup. The existing `bench_harness` measures comparable Flow, +CPython and native C subjects on the same host. The `perf_scorecard` tool +classifies **one** such result file without running any benchmark or comparing +old runs from a different CPU or operating system. + +Run the tools locally from the repository root: + +```sh +./flow tool bench_harness --smoke --out build/perf-measurements.json +./flow tool perf_scorecard build/perf-measurements.json --out build/perf-scorecard.json +./flow tool perf_scorecard build/perf-measurements.json --strict +./flow tool tests/scripts/perf_scorecard.flow +``` + +For stable results, use the normal (non-`--smoke`) benchmark harness and +repeat on a quiet system. These commands are **not** a hosted CI gate. + +The output schema `flow-perf-scorecard/1` preserves `source_sha`, `os` and +`cpu`, then emits an individual classification for every workload, its timing +scope, a measured Flow/Python median ratio and any available Flow/native +ratio. Summary counts do not average unrelated workloads. + +A `win` means that Flow takes at most Python's time divided by 1.05. A +`loss` means Flow takes more than 1.05 times Python's time. Between these +bounds, the performance is a `tie`, or a separate `memory_win` if observed +peak RSS is at least 5% lower for Flow. Speed comparisons require verified +semantic parity, known timing scope, at least three repetitions, positive +medians and recorded per-subject dispersion at most 10% of the median. +Missing Python subjects, unresolved parity and noisy observations remain +**unscored**; they are never quietly converted to wins. + +Native tax is a second, independent dimension: a Flow/native ratio above +1.05 is marked `tax`. It is not hidden by a good Python comparison. +`--strict` fails on losses, native taxes or unscored rows. Empty input and +missing source or machine metadata are input errors. These are deterministic +classification rules; they do **not** establish statistical significance. +Threshold-adjacent observations should be re-measured. + +The scorecard does not implement compiler caching, MLIR passes, memory +safety or the RT callable contracts. The respective epics remain open. +Before making any headline performance claim, inspect correctness, +measurement settings, hardware identity, and the raw per-workload result +file. Run and validate all performance data locally as required by #727. diff --git a/tests/scripts/perf_scorecard.flow b/tests/scripts/perf_scorecard.flow new file mode 100644 index 000000000..41def573d --- /dev/null +++ b/tests/scripts/perf_scorecard.flow @@ -0,0 +1,121 @@ +# Local-only scorecard contract tests (#727). Synthetic numbers test +# classification, not any assertion about Flow's measured performance. + +import "tools/flow_cli/sys.flow" +import "tests/scripts/harness.flow" + +function row(name: string, parity: string, fm: string, pm: string, + fd: string, pd: string, frss: string, prss: string, + native: string) -> string { + return "{\"workload_id\":\"" + name + "\"," + + "\"semantic_parity_status\":\"" + parity + "\"," + + "\"timing_scope\":\"precompiled_workload\"," + + "\"repeats\":5," + + "\"flow_metrics\":{\"median\":" + fm + ",\"dispersion\":" + fd + + ",\"median_rss_kb\":" + frss + "}," + + "\"python_metrics\":{\"median\":" + pm + ",\"dispersion\":" + pd + + ",\"median_rss_kb\":" + prss + "}," + + "\"attribution\":{\"flow_vs_python\":1.0,\"flow_vs_native\":" + native + "}}" +} + +function fixture() -> string { + let mut a: string = "{\"system\":{\"git_sha\":\"abc123\",\"os\":\"Linux\",\"cpu\":\"test-cpu\"},\"benchmarks\":[" + a = a + row("runtime_win", "verified", "8", "10", "0.1", "0.1", "100", "100", "1.02") + "," + a = a + row("runtime_tie", "verified", "10", "10", "0.1", "0.1", "100", "100", "null") + "," + a = a + row("runtime_loss", "verified", "15", "10", "0.1", "0.1", "100", "100", "1.1") + "," + a = a + row("memory_win", "verified", "10", "10", "0.1", "0.1", "50", "100", "1.0") + "," + a = a + row("missing_python", "verified", "8", "null", "0.1", "null", "50", "null", "null") + "," + a = a + row("bad_parity", "unresolved", "5", "10", "0.1", "0.1", "100", "100", "1.0") + "," + a = a + row("noisy_run", "verified", "8", "10", "3", "0.1", "100", "100", "1.0") + "," + a = a + s_replace(row("not_enough_repeats", "verified", "10", "10", "0.1", "0.1", "100", "100", "1.0"), "\"repeats\":5", "\"repeats\":2") + return a + "]}" +} + +function command(path: string, strict: bool, dest: string) -> i32 { + let argv: ptr = sl_of3("./flow", "tool", "perf_scorecard") + sl_push(argv, path) + if strict { + sl_push(argv, "--strict") + } + if s_len(dest) > 0 { + sl_push2(argv, "--out", dest) + } + return t_run_argv(argv) +} + +function check_verdicts() -> i32 { + if t_need_cc() != T_PASS { return T_SKIP } + let input: string = t_tmp() + "/valid.json" + if not write_text(input, fixture()) { return T_FAIL } + if command(input, false, "") != 0 { + t_cat(t_err()) + return T_FAIL + } + let s: string = read_text(t_out()) + if not a_contains(s, "\"schema\": \"flow-perf-scorecard/1\"") { return T_FAIL } + if not a_contains(s, "\"source_sha\": \"abc123\"") { return T_FAIL } + if not a_contains(s, "\"workload_id\": \"runtime_win\", \"timing_scope\": \"precompiled_workload\", \"verdict\": \"win\"") { return T_FAIL } + if not a_contains(s, "\"workload_id\": \"runtime_tie\", \"timing_scope\": \"precompiled_workload\", \"verdict\": \"tie\"") { return T_FAIL } + if not a_contains(s, "\"workload_id\": \"runtime_loss\", \"timing_scope\": \"precompiled_workload\", \"verdict\": \"loss\"") { return T_FAIL } + if not a_contains(s, "\"workload_id\": \"memory_win\", \"timing_scope\": \"precompiled_workload\", \"verdict\": \"memory_win\"") { return T_FAIL } + if not a_contains(s, "\"verdict\": \"no_python_data\"") { return T_FAIL } + if not a_contains(s, "\"verdict\": \"unresolved_parity\"") { return T_FAIL } + if not a_contains(s, "\"verdict\": \"noisy\"") { return T_FAIL } + if not a_contains(s, "\"verdict\": \"insufficient_repetitions\"") { return T_FAIL } + if not a_contains(s, "\"flow_vs_python\": 0.8") { return T_FAIL } + if not a_contains(s, "\"native_tax\": \"tax\"") { return T_FAIL } + if not a_contains(s, "\"wins\": 1, \"memory_wins\": 1, \"ties\": 1, \"losses\": 1, \"unscored\": 4, \"native_taxes\": 1") { return T_FAIL } + return T_PASS +} + +function check_strict_is_a_gate() -> i32 { + if t_need_cc() != T_PASS { return T_SKIP } + let input: string = t_tmp() + "/strict.json" + let _w: bool = write_text(input, fixture()) + if command(input, true, "") != 1 { return T_FAIL } + return T_PASS +} + +function check_output_file() -> i32 { + if t_need_cc() != T_PASS { return T_SKIP } + let input: string = t_tmp() + "/input.json" + let dest: string = t_tmp() + "/scorecard.json" + let _w: bool = write_text(input, fixture()) + if command(input, false, dest) != 0 { return T_FAIL } + if not a_file_contains(dest, "\"source_sha\": \"abc123\"") { return T_FAIL } + return T_PASS +} + +function check_rejects_malformed_json() -> i32 { + if t_need_cc() != T_PASS { return T_SKIP } + let input: string = t_tmp() + "/broken.json" + let _w: bool = write_text(input, "{") + if command(input, false, "") != 2 { return T_FAIL } + if not a_contains(read_text(t_err()), "invalid JSON") { return T_FAIL } + return T_PASS +} + +function check_rejects_missing_metadata() -> i32 { + if t_need_cc() != T_PASS { return T_SKIP } + let input: string = t_tmp() + "/missing.json" + let _w: bool = write_text(input, "{\"system\":{\"os\":\"Linux\",\"cpu\":\"test\"},\"benchmarks\":[]}") + if command(input, false, "") != 2 { return T_FAIL } + if not a_contains(read_text(t_err()), "missing system git_sha/os/cpu") { return T_FAIL } + return T_PASS +} + +function main() -> i32 { + t_begin("perf_scorecard") + t_start() + t_end("classifies_each_row", check_verdicts()) + t_start() + t_end("strict_gate", check_strict_is_a_gate()) + t_start() + t_end("json_output", check_output_file()) + t_start() + t_end("malformed_json", check_rejects_malformed_json()) + t_start() + t_end("missing_metadata", check_rejects_missing_metadata()) + t_done() + return 0 +} diff --git a/tools/perf_scorecard/main.flow b/tools/perf_scorecard/main.flow index 694c98fd1..77b43401e 100644 --- a/tools/perf_scorecard/main.flow +++ b/tools/perf_scorecard/main.flow @@ -1,12 +1,252 @@ -# perf_scorecard tool -# Orchestrates performance benchmarks for Epic #727. +# @module tools.perf_scorecard +# @means Local-only per-workload performance verdicts for #727. # -# Usage: -# flow tool perf_scorecard +# Input is the JSON written by ./flow tool bench_harness --out RESULTS.json. +# No benchmark is run here; no old results or different hardware are compared. +# A row is scored only with verified parity, positive measured medians, +# >=3 repetitions, and sufficiently stable samples. Unknown is not a win. +# +# ./flow tool perf_scorecard RESULTS.json [--strict] [--out SCORECARD.json] +# --strict returns 1 if there are any losses, native taxes or unscored rows. + +import "tools/flow_cli/sys.flow" +import "scripts/tools/lib/json.flow" +import std.process { proc_argc, proc_argv } + +extern { + function atof(s: string) -> f64 + function snprintf(buf: ptr, size: i64, fmt: string, ...) -> i32 + function malloc(size: i64) -> ptr +} + +function field_text(j: ptr, node: i32, name: string) -> string { + let at: i32 = json_get(j, node, name) + if at < 0 or json_kind(j, at) != json_kind_string() { + return "" + } + return json_str(j, at) +} -import std.process { proc_cmd, proc_arg, proc_run } +# -1 is the missing/bad sentinel, while zero is valid for dispersion. +function number(j: ptr, node: i32, name: string) -> f64 { + let at: i32 = json_get(j, node, name) + if at < 0 or json_kind(j, at) != json_kind_number() { + return 0.0 - 1.0 + } + let v: f64 = atof(json_str(j, at)) + # Reject unbounded/overflow values; NaN also fails both comparisons. + if not (v >= 0.0 and v <= 1000000000000.0) { + return 0.0 - 1.0 + } + return v +} + +function metric(j: ptr, row: i32, group: string, key: string) -> f64 { + return number(j, json_get(j, row, group), key) +} + +# Always compute the Flow/Python ratio from the medians that were graded. +# A stale or inconsistent attribution field cannot misreport the verdict. +function measured_ratio(j: ptr, row: i32) -> string { + let fm: f64 = metric(j, row, "flow_metrics", "median") + let pm: f64 = metric(j, row, "python_metrics", "median") + if fm <= 0.0 or pm <= 0.0 { + return "null" + } + let result: f64 = fm / pm + let buf: ptr = malloc(64) as ptr + let _n: i32 = snprintf(buf, 64, "%.9g", result) + return buf as string +} + +function native_ratio(j: ptr, row: i32) -> string { + let at: i32 = json_get(j, json_get(j, row, "attribution"), "flow_vs_native") + if at < 0 or json_kind(j, at) != json_kind_number() { + return "null" + } + let v: f64 = atof(json_str(j, at)) + if not (v > 0.0 and v <= 1000000000000.0) { + return "null" + } + return json_str(j, at) +} + +# A 5% speed edge is required; 5% less RSS at performance parity is +# recorded separately as memory_win. 10% relative sample dispersion is +# a conservative local noise gate, not a claim of statistical significance. +function verdict(j: ptr, row: i32) -> string { + if s_len(field_text(j, row, "workload_id")) == 0 { + return "unresolved_workload" + } + if not s_eq(field_text(j, row, "semantic_parity_status"), "verified") { + return "unresolved_parity" + } + let scope: string = field_text(j, row, "timing_scope") + if not s_eq(scope, "compile_and_execute") and + not s_eq(scope, "precompiled_exec_to_exit") and + not s_eq(scope, "precompiled_workload") { + return "unresolved_scope" + } + if number(j, row, "repeats") < 3.0 { + return "insufficient_repetitions" + } + let fm: f64 = metric(j, row, "flow_metrics", "median") + let pm: f64 = metric(j, row, "python_metrics", "median") + if fm <= 0.0 { + return "no_flow_data" + } + if pm <= 0.0 { + return "no_python_data" + } + let fd: f64 = metric(j, row, "flow_metrics", "dispersion") + let pd: f64 = metric(j, row, "python_metrics", "dispersion") + if fd < 0.0 or pd < 0.0 { + return "no_dispersion_data" + } + if fd > fm * 0.10 or pd > pm * 0.10 { + return "noisy" + } + if fm * 1.05 <= pm { + return "win" + } + if fm > pm * 1.05 { + return "loss" + } + let frss: f64 = metric(j, row, "flow_metrics", "median_rss_kb") + let prss: f64 = metric(j, row, "python_metrics", "median_rss_kb") + if frss > 0.0 and prss > 0.0 and frss * 1.05 <= prss { + return "memory_win" + } + return "tie" +} + +function native_tax(j: ptr, row: i32) -> string { + let att: i32 = json_get(j, row, "attribution") + let v: f64 = number(j, att, "flow_vs_native") + if v <= 0.0 { + return "no_native" + } + if v > 1.05 { + return "tax" + } + return "ok" +} function main() -> i32 { - let rc: i32 = 0 - return rc + let mut input: string = "" + let mut dest: string = "" + let mut strict: bool = false + let mut i: i32 = 1 + while i < proc_argc() { + let arg: string = proc_argv(i) + if s_eq(arg, "--strict") { + strict = true + } elif s_eq(arg, "--out") { + i = i + 1 + if i >= proc_argc() { + errln("perf_scorecard: --out needs a path") + return 2 + } + dest = proc_argv(i) + } elif s_eq(arg, "--help") or s_eq(arg, "-h") { + outln("usage: ./flow tool perf_scorecard RESULTS.json [--strict] [--out SCORECARD.json]") + return 0 + } elif s_starts(arg, "-") or s_len(input) > 0 { + errln("perf_scorecard: unexpected argument " + arg) + return 2 + } else { + input = arg + } + i = i + 1 + } + if s_len(input) == 0 or not is_file(input) { + errln("perf_scorecard: expected an existing bench_harness JSON file") + return 2 + } + let source: string = read_text(input) + let j: ptr = json_parse(source) + if s_len(json_error(j)) > 0 { + errln("perf_scorecard: invalid JSON: " + json_error(j)) + json_free(j) + return 2 + } + let system: i32 = json_get(j, 0, "system") + let sha: string = field_text(j, system, "git_sha") + let os: string = field_text(j, system, "os") + let cpu: string = field_text(j, system, "cpu") + let rows: i32 = json_get(j, 0, "benchmarks") + if s_len(sha) == 0 or s_len(os) == 0 or s_len(cpu) == 0 or + rows < 0 or json_kind(j, rows) != json_kind_array() { + errln("perf_scorecard: missing system git_sha/os/cpu or benchmarks array") + json_free(j) + return 2 + } + + if json_first(j, rows) < 0 { + errln("perf_scorecard: no measured benchmark rows") + json_free(j) + return 2 + } + + let mut report: string = "{\n \"schema\": \"flow-perf-scorecard/1\",\n" + report = report + " \"source_sha\": " + json_quote(sha) + ",\n" + report = report + " \"os\": " + json_quote(os) + ",\n" + report = report + " \"cpu\": " + json_quote(cpu) + ",\n \"rows\": [\n" + let mut wins: i32 = 0 + let mut memory_wins: i32 = 0 + let mut ties: i32 = 0 + let mut losses: i32 = 0 + let mut unscored: i32 = 0 + let mut taxes: i32 = 0 + let mut row: i32 = json_first(j, rows) + let mut first: bool = true + while row >= 0 { + let name: string = field_text(j, row, "workload_id") + let v: string = verdict(j, row) + let tax: string = native_tax(j, row) + if s_eq(v, "win") { + wins = wins + 1 + } elif s_eq(v, "memory_win") { + memory_wins = memory_wins + 1 + } elif s_eq(v, "tie") { + ties = ties + 1 + } elif s_eq(v, "loss") { + losses = losses + 1 + } else { + unscored = unscored + 1 + } + if s_eq(tax, "tax") { + taxes = taxes + 1 + } + if not first { + report = report + ",\n" + } + first = false + report = report + " {\"workload_id\": " + json_quote(name) + report = report + ", \"timing_scope\": " + json_quote(field_text(j, row, "timing_scope")) + report = report + ", \"verdict\": " + json_quote(v) + report = report + ", \"flow_vs_python\": " + measured_ratio(j, row) + report = report + ", \"flow_vs_native\": " + native_ratio(j, row) + report = report + ", \"native_tax\": " + json_quote(tax) + "}" + row = json_next(j, row) + } + report = report + "\n ],\n \"summary\": {\"wins\": " + s_itoa(wins) + report = report + ", \"memory_wins\": " + s_itoa(memory_wins) + report = report + ", \"ties\": " + s_itoa(ties) + report = report + ", \"losses\": " + s_itoa(losses) + report = report + ", \"unscored\": " + s_itoa(unscored) + report = report + ", \"native_taxes\": " + s_itoa(taxes) + "}\n}\n" + json_free(j) + if s_len(dest) > 0 { + if not write_text(dest, report) { + errln("perf_scorecard: cannot write " + dest) + return 2 + } + } else { + out(report) + } + if strict and (losses > 0 or unscored > 0 or taxes > 0) { + return 1 + } + return 0 }