[Performance #727] Add local-only, parity-gated per-workload scorecards

docs/nav.json

@@ -936,6 +936,10 @@
           "label": "Benchmarks",
           "path": "project/benchmark-results.md"
         },
+        {
+          "label": "Performance Scorecards",
+          "path": "project/performance-scorecards.md"
+        },
         {
           "label": "Compile Profile",
           "path": "project/compile-profile.md"

docs/project/performance-scorecards.md

@@ -0,0 +1,46 @@
+# Local performance scorecards (#727)
+
+The performance epic is a collection of separately audited comparisons, not
+one aggregate speedup. The existing `bench_harness` measures comparable Flow,
+CPython and native C subjects on the same host. The `perf_scorecard` tool
+classifies **one** such result file without running any benchmark or comparing
+old runs from a different CPU or operating system.
+
+Run the tools locally from the repository root:
+
+```sh
+./flow tool bench_harness --smoke --out build/perf-measurements.json
+./flow tool perf_scorecard build/perf-measurements.json --out build/perf-scorecard.json
+./flow tool perf_scorecard build/perf-measurements.json --strict
+./flow tool tests/scripts/perf_scorecard.flow
+```
+
+For stable results, use the normal (non-`--smoke`) benchmark harness and
+repeat on a quiet system. These commands are **not** a hosted CI gate.
+
+The output schema `flow-perf-scorecard/1` preserves `source_sha`, `os` and
+`cpu`, then emits an individual classification for every workload, its timing
+scope, a measured Flow/Python median ratio and any available Flow/native
+ratio. Summary counts do not average unrelated workloads.
+
+A `win` means that Flow takes at most Python's time divided by 1.05. A
+`loss` means Flow takes more than 1.05 times Python's time. Between these
+bounds, the performance is a `tie`, or a separate `memory_win` if observed
+peak RSS is at least 5% lower for Flow. Speed comparisons require verified
+semantic parity, known timing scope, at least three repetitions, positive
+medians and recorded per-subject dispersion at most 10% of the median.
+Missing Python subjects, unresolved parity and noisy observations remain
+**unscored**; they are never quietly converted to wins.
+
+Native tax is a second, independent dimension: a Flow/native ratio above
+1.05 is marked `tax`. It is not hidden by a good Python comparison.
+`--strict` fails on losses, native taxes or unscored rows. Empty input and
+missing source or machine metadata are input errors. These are deterministic
+classification rules; they do **not** establish statistical significance.
+Threshold-adjacent observations should be re-measured.
+
+The scorecard does not implement compiler caching, MLIR passes, memory
+safety or the RT callable contracts. The respective epics remain open.
+Before making any headline performance claim, inspect correctness,
+measurement settings, hardware identity, and the raw per-workload result
+file. Run and validate all performance data locally as required by #727.

tests/scripts/perf_scorecard.flow

@@ -0,0 +1,121 @@
+# Local-only scorecard contract tests (#727). Synthetic numbers test
+# classification, not any assertion about Flow's measured performance.
+
+import "tools/flow_cli/sys.flow"
+import "tests/scripts/harness.flow"
+
+function row(name: string, parity: string, fm: string, pm: string,
+             fd: string, pd: string, frss: string, prss: string,
+             native: string) -> string {
+    return "{\"workload_id\":\"" + name + "\","
+        + "\"semantic_parity_status\":\"" + parity + "\","
+        + "\"timing_scope\":\"precompiled_workload\","
+        + "\"repeats\":5,"
+        + "\"flow_metrics\":{\"median\":" + fm + ",\"dispersion\":" + fd
+        + ",\"median_rss_kb\":" + frss + "},"
+        + "\"python_metrics\":{\"median\":" + pm + ",\"dispersion\":" + pd
+        + ",\"median_rss_kb\":" + prss + "},"
+        + "\"attribution\":{\"flow_vs_python\":1.0,\"flow_vs_native\":" + native + "}}"
+}
+
+function fixture() -> string {
+    let mut a: string = "{\"system\":{\"git_sha\":\"abc123\",\"os\":\"Linux\",\"cpu\":\"test-cpu\"},\"benchmarks\":["
+    a = a + row("runtime_win", "verified", "8", "10", "0.1", "0.1", "100", "100", "1.02") + ","
+    a = a + row("runtime_tie", "verified", "10", "10", "0.1", "0.1", "100", "100", "null") + ","
+    a = a + row("runtime_loss", "verified", "15", "10", "0.1", "0.1", "100", "100", "1.1") + ","
+    a = a + row("memory_win", "verified", "10", "10", "0.1", "0.1", "50", "100", "1.0") + ","
+    a = a + row("missing_python", "verified", "8", "null", "0.1", "null", "50", "null", "null") + ","
+    a = a + row("bad_parity", "unresolved", "5", "10", "0.1", "0.1", "100", "100", "1.0") + ","
+    a = a + row("noisy_run", "verified", "8", "10", "3", "0.1", "100", "100", "1.0") + ","
+    a = a + s_replace(row("not_enough_repeats", "verified", "10", "10", "0.1", "0.1", "100", "100", "1.0"), "\"repeats\":5", "\"repeats\":2")
+    return a + "]}"
+}
+
+function command(path: string, strict: bool, dest: string) -> i32 {
+    let argv: ptr<ProcStrs> = sl_of3("./flow", "tool", "perf_scorecard")
+    sl_push(argv, path)
+    if strict {
+        sl_push(argv, "--strict")
+    }
+    if s_len(dest) > 0 {
+        sl_push2(argv, "--out", dest)
+    }
+    return t_run_argv(argv)
+}
+
+function check_verdicts() -> i32 {
+    if t_need_cc() != T_PASS { return T_SKIP }
+    let input: string = t_tmp() + "/valid.json"
+    if not write_text(input, fixture()) { return T_FAIL }
+    if command(input, false, "") != 0 {
+        t_cat(t_err())
+        return T_FAIL
+    }
+    let s: string = read_text(t_out())
+    if not a_contains(s, "\"schema\": \"flow-perf-scorecard/1\"") { return T_FAIL }
+    if not a_contains(s, "\"source_sha\": \"abc123\"") { return T_FAIL }
+    if not a_contains(s, "\"workload_id\": \"runtime_win\", \"timing_scope\": \"precompiled_workload\", \"verdict\": \"win\"") { return T_FAIL }
+    if not a_contains(s, "\"workload_id\": \"runtime_tie\", \"timing_scope\": \"precompiled_workload\", \"verdict\": \"tie\"") { return T_FAIL }
+    if not a_contains(s, "\"workload_id\": \"runtime_loss\", \"timing_scope\": \"precompiled_workload\", \"verdict\": \"loss\"") { return T_FAIL }
+    if not a_contains(s, "\"workload_id\": \"memory_win\", \"timing_scope\": \"precompiled_workload\", \"verdict\": \"memory_win\"") { return T_FAIL }
+    if not a_contains(s, "\"verdict\": \"no_python_data\"") { return T_FAIL }
+    if not a_contains(s, "\"verdict\": \"unresolved_parity\"") { return T_FAIL }
+    if not a_contains(s, "\"verdict\": \"noisy\"") { return T_FAIL }
+    if not a_contains(s, "\"verdict\": \"insufficient_repetitions\"") { return T_FAIL }
+    if not a_contains(s, "\"flow_vs_python\": 0.8") { return T_FAIL }
+    if not a_contains(s, "\"native_tax\": \"tax\"") { return T_FAIL }
+    if not a_contains(s, "\"wins\": 1, \"memory_wins\": 1, \"ties\": 1, \"losses\": 1, \"unscored\": 4, \"native_taxes\": 1") { return T_FAIL }
+    return T_PASS
+}
+
+function check_strict_is_a_gate() -> i32 {
+    if t_need_cc() != T_PASS { return T_SKIP }
+    let input: string = t_tmp() + "/strict.json"
+    let _w: bool = write_text(input, fixture())
+    if command(input, true, "") != 1 { return T_FAIL }
+    return T_PASS
+}
+
+function check_output_file() -> i32 {
+    if t_need_cc() != T_PASS { return T_SKIP }
+    let input: string = t_tmp() + "/input.json"
+    let dest: string = t_tmp() + "/scorecard.json"
+    let _w: bool = write_text(input, fixture())
+    if command(input, false, dest) != 0 { return T_FAIL }
+    if not a_file_contains(dest, "\"source_sha\": \"abc123\"") { return T_FAIL }
+    return T_PASS
+}
+
+function check_rejects_malformed_json() -> i32 {
+    if t_need_cc() != T_PASS { return T_SKIP }
+    let input: string = t_tmp() + "/broken.json"
+    let _w: bool = write_text(input, "{")
+    if command(input, false, "") != 2 { return T_FAIL }
+    if not a_contains(read_text(t_err()), "invalid JSON") { return T_FAIL }
+    return T_PASS
+}
+
+function check_rejects_missing_metadata() -> i32 {
+    if t_need_cc() != T_PASS { return T_SKIP }
+    let input: string = t_tmp() + "/missing.json"
+    let _w: bool = write_text(input, "{\"system\":{\"os\":\"Linux\",\"cpu\":\"test\"},\"benchmarks\":[]}")
+    if command(input, false, "") != 2 { return T_FAIL }
+    if not a_contains(read_text(t_err()), "missing system git_sha/os/cpu") { return T_FAIL }
+    return T_PASS
+}
+
+function main() -> i32 {
+    t_begin("perf_scorecard")
+    t_start()
+    t_end("classifies_each_row", check_verdicts())
+    t_start()
+    t_end("strict_gate", check_strict_is_a_gate())
+    t_start()
+    t_end("json_output", check_output_file())
+    t_start()
+    t_end("malformed_json", check_rejects_malformed_json())
+    t_start()
+    t_end("missing_metadata", check_rejects_missing_metadata())
+    t_done()
+    return 0
+}