diff --git a/docs/cx/BENCHKIT_SPEC.md b/docs/cx/BENCHKIT_SPEC.md index 7bff685a..152bd883 100644 --- a/docs/cx/BENCHKIT_SPEC.md +++ b/docs/cx/BENCHKIT_SPEC.md @@ -484,6 +484,7 @@ Benchkit は、pre-staged input、restart、学習済みモデル、公開 archi - manifest digest または content digest - 生成 recipe または取得 recipe - 公開 URL または archive reference +- self-contained な runtime parameters - 検証状態 入力が top-level app repository 内に含まれ、その repository の `source_info.resolved_commit` で固定される場合は、別の manifest や content digest を必須にしない。 @@ -493,6 +494,9 @@ Benchkit は、pre-staged input、restart、学習済みモデル、公開 archi 入力が別の public input repository や public archive から来る場合は、`input_info` 側に公開URL、ref、resolved commit または digest、取得/生成 recipe を置く。 この場合も、入力が記録済み source commit で固定されるなら `verification_status: "public_source_commit"` として `Covered` に分類できる。 +入力が実ファイルではなく、公開可能な command と arguments だけで完全に表せる場合は、`kind: "runtime-parameters"`、`source: "inline"`、`arguments`、`verification_status: "self_contained"` を使える。 +同じ job から複数 result が出る場合は、`result_exp` などの result scope を添えて、各 result と対応する入力指定を明示してよい。 + site-local path は所在情報であり、長期的な input identity ではない。 巨大データや共同研究由来データを site-local shared storage に置くことは許容されるが、Result provenance では path より dataset identity、recipe、manifest、digest を優先する。 public surface では、必要がない限り detailed local path を表示しない。 @@ -511,6 +515,7 @@ When an application passes input metadata through the Benchkit input metadata he - manifest digest or content digest - generation or acquisition recipe - public URL or archive reference +- self-contained runtime parameters - verification status When the input is already stored in the top-level application repository and is fixed by `source_info.resolved_commit`, a separate manifest or content digest is not mandatory. @@ -520,6 +525,9 @@ In that case, `source_info` is the fixed point for both the application source a When the input comes from a separate public input repository or public archive, `input_info` should carry the public URL, ref, resolved commit or digest, and acquisition or generation recipe. If the input is fixed by a recorded source commit, it may use `verification_status: "public_source_commit"` and be classified as `Covered`. +When the input is not a file and can be fully represented by public command arguments, `input_info` may use `kind: "runtime-parameters"`, `source: "inline"`, `arguments`, and `verification_status: "self_contained"`. +When one job emits multiple results, a result scope such as `result_exp` may be attached so each Result JSON is evaluated against the matching input declaration. + A site-local path is location information, not a durable input identity. Large datasets and collaboration-derived inputs may still be staged on site-local shared storage, but Result provenance should prefer dataset identity, recipe, manifest, and digest over paths. Detailed local paths should not be exposed on the public surface unless they are necessary. diff --git a/programs/qws/run.sh b/programs/qws/run.sh index 0c9b5c38..d757172e 100644 --- a/programs/qws/run.sh +++ b/programs/qws/run.sh @@ -6,6 +6,14 @@ numproc_node="$3" nthreads="$4" export OMP_NUM_THREADS=$nthreads +REPO_URL="https://github.com/RIKEN-LQCD/qws.git" +REPO_DIR="qws" +BRANCH="${QWS_BRANCH:-master}" +SOURCE_COMMIT="${QWS_SOURCE_COMMIT:-}" +qws_case0_args=(32 6 4 3 1 1 1 1 -1 -1 6 50) +qws_case1_args=(32 6 4 3 1 1 1 2 -1 -1 6 50) +qws_case7_args=(32 6 4 3 1 2 2 2 -1 -1 6 50) + source "${PWD}/scripts/bk_functions.sh" qws_profiler_tool=$(bk_resolve_profiler_tool fapp QWS_PROFILER_TOOL) qws_profiler_level=$(bk_resolve_profiler_level detailed QWS_PROFILER_LEVEL) @@ -14,7 +22,32 @@ qws_profiler_level=$(bk_resolve_profiler_level detailed QWS_PROFILER_LEVEL) # app-side section timings and artifacts. # source "${PWD}/programs/qws/estimate.sh" -mkdir -p results && > results/result +mkdir -p results && : > results/result + +record_qws_runtime_parameter_inputs() { + rm -f results/.input_info_items.jsonl + bk_record_runtime_parameter_input \ + --dataset-id qws-case0-parameters \ + --dataset-version "${BK_SOURCE_REF_NAME:-$BRANCH}" \ + --parameter-set-id CASE0 \ + --result-exp CASE0 \ + --command ./main \ + -- "${qws_case0_args[@]}" + bk_record_runtime_parameter_input \ + --dataset-id qws-case1-parameters \ + --dataset-version "${BK_SOURCE_REF_NAME:-$BRANCH}" \ + --parameter-set-id CASE1 \ + --result-exp CASE1 \ + --command ./main \ + -- "${qws_case1_args[@]}" + bk_record_runtime_parameter_input \ + --dataset-id qws-case7-parameters \ + --dataset-version "${BK_SOURCE_REF_NAME:-$BRANCH}" \ + --parameter-set-id CASE7 \ + --result-exp CASE7 \ + --command ./main \ + -- "${qws_case7_args[@]}" +} # print_results: extract FOM from the benchmark output and append a result line. print_results() { @@ -41,36 +74,37 @@ print_results() { # tar -czf "$1" ./pa # } -[[ -d qws ]] || git clone https://github.com/RIKEN-LQCD/qws.git +bk_fetch_recorded_source "${REPO_URL}" "${REPO_DIR}" "${BRANCH}" "${SOURCE_COMMIT}" +record_qws_runtime_parameter_inputs if [[ -f artifacts/main ]]; then - cp artifacts/main qws + cp artifacts/main "${REPO_DIR}" else echo "ERROR: artifacts/main not found" exit 1 fi -cd qws +cd "${REPO_DIR}" case "$system" in Fugaku|FugakuCN) case "$nodes" in 1) - mpiexec -n 1 ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpiexec -n 1 ./main "${qws_case0_args[@]}" > CASE0 print_results output.${PJM_JOBID}/0/1/stdout.1.0 CASE0 1 >> ../results/result - mpiexec -n 2 ./main 32 6 4 3 1 1 1 2 -1 -1 6 50 > CASE1 + mpiexec -n 2 ./main "${qws_case1_args[@]}" > CASE1 print_results output.${PJM_JOBID}/0/2/stdout.2.0 CASE1 2 >> ../results/result if bk_profiler_enabled "$qws_profiler_tool"; then - bk_profiler "$qws_profiler_tool" --level "$qws_profiler_level" --archive ../results/padata0.tgz --raw-dir pa -- mpiexec -n 1 ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0.profile + bk_profiler "$qws_profiler_tool" --level "$qws_profiler_level" --archive ../results/padata0.tgz --raw-dir pa -- mpiexec -n 1 ./main "${qws_case0_args[@]}" > CASE0.profile # else # emit_qws_dummy_padata ../results/padata0.tgz fi ;; 2) - mpiexec -n 8 ./main 32 6 4 3 1 2 2 2 -1 -1 6 50 > CASE7 + mpiexec -n 8 ./main "${qws_case7_args[@]}" > CASE7 print_results output.${PJM_JOBID}/0/1/stdout.1.0 CASE7 4 >> ../results/result if bk_profiler_enabled "$qws_profiler_tool"; then - bk_profiler "$qws_profiler_tool" --level "$qws_profiler_level" --archive ../results/padata0.tgz --raw-dir pa -- mpiexec -n 8 ./main 32 6 4 3 1 2 2 2 -1 -1 6 50 > CASE7.profile + bk_profiler "$qws_profiler_tool" --level "$qws_profiler_level" --archive ../results/padata0.tgz --raw-dir pa -- mpiexec -n 8 ./main "${qws_case7_args[@]}" > CASE7.profile # else # emit_qws_dummy_padata ../results/padata0.tgz fi @@ -92,38 +126,38 @@ case "$system" in export OMP_NUM_THREADS="$nthreads" export OMP_PLACES=cores export OMP_PROC_BIND=close - mpirun --bind-to none -n 1 ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun --bind-to none -n 1 ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 1 >> ../results/result ;; RC_GH200) module load system/qc-gh200 nvhpc-hpcx/25.9 - mpirun -n 1 --bind-to core --map-by ppr:1:node:PE=72 ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun -n 1 --bind-to core --map-by ppr:1:node:PE=72 ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 1 >> ../results/result ;; RC_GENOA) module load system/genoa mpi/openmpi-x86_64 - mpirun -n 1 --bind-to core --map-by ppr:1:node:PE=96 ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun -n 1 --bind-to core --map-by ppr:1:node:PE=96 ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 1 >> ../results/result ;; RC_DGXSP) source /etc/profile.d/modules.sh module load system/ng-dgx nvhpc-hpcx/26.3 - mpirun -n 1 --bind-to core --map-by ppr:1:node:PE=20 ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun -n 1 --bind-to core --map-by ppr:1:node:PE=20 ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 1 >> ../results/result ;; RC_FX700) module load system/fx700 FJSVstclanga - mpirun -n 1 --bind-to core --map-by ppr:1:node:PE=12 ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun -n 1 --bind-to core --map-by ppr:1:node:PE=12 ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 1 >> ../results/result ;; MiyabiG|MiyabiC) - mpirun -n 1 ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun -n 1 ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 1 >> ../results/result ;; GenkaiA|GenkaiB|GenkaiC) qws_numproc=$((nodes * numproc_node)) module load intel/2023.2 mvapich/3.0-intel2023.2 - mpirun -n ${qws_numproc} ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun -n ${qws_numproc} ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 ${numproc_node} >> ../results/result ;; Grand_C|Grand_G) @@ -144,12 +178,12 @@ case "$system" in env | sort >&2 exit 1 fi - "$qws_mpi_launcher" -n ${qws_numproc} ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + "$qws_mpi_launcher" -n ${qws_numproc} ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 ${numproc_node} >> ../results/result ;; AOBA_A|AOBA_B|AOBA_S) qws_numproc=$((nodes * numproc_node)) - mpirun -np ${qws_numproc} ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun -np ${qws_numproc} ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 ${numproc_node} >> ../results/result ;; SQUID_CPU) @@ -160,7 +194,7 @@ case "$system" in fi module load BaseCPU export OMP_NUM_THREADS="${nthreads}" - mpirun "${qws_mpi_opts[@]}" -np ${qws_numproc} ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun "${qws_mpi_opts[@]}" -np ${qws_numproc} ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 ${numproc_node} >> ../results/result ;; SQUID_GPU) @@ -171,7 +205,7 @@ case "$system" in fi module load BaseGPU export OMP_NUM_THREADS="${nthreads}" - mpirun "${qws_mpi_opts[@]}" -np ${qws_numproc} --bind-to none ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun "${qws_mpi_opts[@]}" -np ${qws_numproc} --bind-to none ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 ${numproc_node} >> ../results/result ;; SQUID_VECTOR) @@ -182,7 +216,7 @@ case "$system" in fi module load BaseVEC export OMP_NUM_THREADS="${nthreads}" - mpirun "${qws_mpi_opts[@]}" -np ${qws_numproc} ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun "${qws_mpi_opts[@]}" -np ${qws_numproc} ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 ${numproc_node} >> ../results/result ;; Odyssey) @@ -195,7 +229,7 @@ case "$system" in module load odyssey fj fjmpi export OMP_NUM_THREADS=12 export PLE_MPI_STD_EMPTYFILE=off - mpiexec -n 1 -ofout CASE0 ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 + mpiexec -n 1 -ofout CASE0 ./main "${qws_case0_args[@]}" print_results CASE0 CASE0 1 >> ../results/result ;; Aquarius) @@ -204,26 +238,26 @@ case "$system" in source /work/opt/local/x86_64/cores/intel/2023.0.0/mpi/latest/env/vars.sh export OMP_NUM_THREADS=8 export I_MPI_PIN=1 - mpiexec -n 1 ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpiexec -n 1 ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 1 >> ../results/result ;; Pegasus) qws_numproc=$((nodes * numproc_node)) module load intel/2025.3.1 intmpi/2025.3.1 - mpirun -n ${qws_numproc} ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun -n ${qws_numproc} ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 ${numproc_node} >> ../results/result ;; Sirius) qws_numproc=$((nodes * numproc_node)) module load aocc/5.0.0 openmpi/5.0.10/aocc5.0.0 - mpirun -n ${qws_numproc} ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun -n ${qws_numproc} ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 ${numproc_node} >> ../results/result ;; TSUBAME4) qws_numproc=$((nodes * numproc_node)) module load openmpi/5.0.10-gcc aocc/4.1.0 export OMPI_CC=clang OMPI_CXX=clang++ OMPI_FC=flang - mpirun -n ${qws_numproc} ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun -n ${qws_numproc} ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 ${numproc_node} >> ../results/result ;; OCTOPUS) @@ -232,7 +266,7 @@ case "$system" in export OMP_NUM_THREADS="${nthreads}" export OMP_PROC_BIND=close export OMP_PLACES=cores - mpirun -n ${qws_numproc} ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + mpirun -n ${qws_numproc} ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 ${numproc_node} >> ../results/result ;; Camphor3) @@ -257,7 +291,7 @@ case "$system" in if [[ "${SLURM_CONF:-}" == /etc/slurm/sysA/* ]]; then unset SLURM_CONF fi - srun -n 1 -c "${nthreads}" ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + srun -n 1 -c "${nthreads}" ./main "${qws_case0_args[@]}" > CASE0 print_results CASE0 CASE0 1 >> ../results/result ;; *) diff --git a/result_server/tests/test_public_result_routes.py b/result_server/tests/test_public_result_routes.py index 72ab2d9e..4835eca0 100644 --- a/result_server/tests/test_public_result_routes.py +++ b/result_server/tests/test_public_result_routes.py @@ -255,7 +255,7 @@ def test_public_reuse_packet_exports_only_public_projection(tmp_path): text = response.get_data(as_text=True) assert response.status_code == 200 assert response.content_type == "text/markdown; charset=utf-8" - assert "Benchkit Public Reuse Packet" in text + assert "CX Public Reuse Packet" in text assert "demoapp / DemoSystem / CASE1" in text assert "https://example.test/repo.git" in text assert "abcdef1234567890" in text @@ -285,7 +285,7 @@ def test_public_reuse_manifest_exports_machine_readable_projection(tmp_path): assert response.content_type == "application/json; charset=utf-8" manifest = response.get_json() assert manifest["schema_version"] == 1 - assert manifest["kind"] == "benchkit_public_reuse_packet" + assert manifest["kind"] == "cx_public_reuse_packet" assert manifest["eligibility"]["status"] == "eligible" assert manifest["result"]["experiment"] == "CASE1" assert manifest["source"]["repository_url"] == "https://example.test/repo.git" @@ -297,6 +297,73 @@ def test_public_reuse_manifest_exports_machine_readable_projection(tmp_path): assert "local-input-placeholder" not in json.dumps(manifest) +def test_public_reuse_manifest_accepts_scoped_runtime_parameters(tmp_path): + app, received_dir = _build_public_app(tmp_path) + filename = "result_20260824_090000_11111111-2222-3333-4444-555555555555.json" + payload = _eligible_public_result_payload() + payload["Exp"] = "CASE1" + payload["input_info"] = { + "schema_version": 1, + "inputs": [ + { + "dataset_id": "qws-case0-parameters", + "kind": "runtime-parameters", + "source": "inline", + "parameter_set_id": "CASE0", + "result_exp": "CASE0", + "command": "./main", + "arguments": [ + "32", "6", "4", "3", "1", "1", + "1", "1", "-1", "-1", "6", "50", + ], + "verification_status": "self_contained", + }, + { + "dataset_id": "qws-case1-parameters", + "kind": "runtime-parameters", + "source": "inline", + "parameter_set_id": "CASE1", + "result_exp": "CASE1", + "command": "./main", + "arguments": [ + "32", "6", "4", "3", "1", "1", + "1", "2", "-1", "-1", "6", "50", + ], + "verification_status": "self_contained", + }, + ], + } + _write_result(received_dir, filename, payload) + + with app.test_client() as client: + manifest_response = client.get(f"/results/detail/{filename}/reuse-manifest.json") + packet_response = client.get(f"/results/detail/{filename}/reuse-packet.md") + + assert manifest_response.status_code == 200 + manifest = manifest_response.get_json() + assert manifest["eligibility"]["status"] == "eligible" + assert manifest["input"]["items"] == [ + { + "dataset_id": "qws-case1-parameters", + "kind": "runtime-parameters", + "source": "inline", + "parameter_set_id": "CASE1", + "result_exp": "CASE1", + "command": "./main", + "arguments": [ + "32", "6", "4", "3", "1", "1", + "1", "2", "-1", "-1", "6", "50", + ], + "verification_status": "self_contained", + } + ] + + packet_text = packet_response.get_data(as_text=True) + assert packet_response.status_code == 200 + assert "dataset_id: qws-case1-parameters" in packet_text + assert "dataset_id: qws-case0-parameters" not in packet_text + + def test_public_reuse_packet_requires_public_input_binding(tmp_path): app, received_dir = _build_public_app(tmp_path) filename = "result_20260824_090000_11111111-2222-3333-4444-555555555555.json" @@ -318,6 +385,38 @@ def test_public_reuse_packet_requires_public_input_binding(tmp_path): assert "Download Reuse Packet" not in text +def test_public_reuse_packet_requires_matching_scoped_input(tmp_path): + app, received_dir = _build_public_app(tmp_path) + filename = "result_20260824_090000_11111111-2222-3333-4444-555555555555.json" + payload = _eligible_public_result_payload() + payload["Exp"] = "CASE1" + payload["input_info"] = { + "schema_version": 1, + "inputs": [ + { + "dataset_id": "qws-case0-parameters", + "kind": "runtime-parameters", + "source": "inline", + "parameter_set_id": "CASE0", + "result_exp": "CASE0", + "command": "./main", + "arguments": ["32"], + "verification_status": "self_contained", + } + ], + } + _write_result(received_dir, filename, payload) + + with app.test_client() as client: + response = client.get(f"/results/detail/{filename}/reuse-packet.md") + detail_response = client.get(f"/results/detail/{filename}") + + assert response.status_code == 404 + text = detail_response.get_data(as_text=True) + assert detail_response.status_code == 200 + assert "needs public input" in text + + def test_public_portal_evidence_packet_route_is_blocked_until_release_review(tmp_path): app, received_dir = _build_public_app(tmp_path) filename = "result_20260824_090000_11111111-2222-3333-4444-555555555555.json" @@ -393,8 +492,8 @@ def test_console_evidence_packet_uses_result_permissions(tmp_path): assert response.status_code == 200 text = response.get_data(as_text=True) - assert "portable review note for one Benchkit benchmark result" in text - assert "readers who may not know the surrounding Benchkit operation" in text + assert "portable review note for one benchmark result" in text + assert "readers who may not know the surrounding benchmark operation" in text assert "does not guarantee independent reproduction" in text assert "Pipeline ID" not in text assert "Raw Result JSON" in text diff --git a/result_server/utils/evidence_packet.py b/result_server/utils/evidence_packet.py index 4ce3e975..071dbacd 100644 --- a/result_server/utils/evidence_packet.py +++ b/result_server/utils/evidence_packet.py @@ -11,7 +11,11 @@ build_cache_digest_help, build_cache_host_environment_help, ) -from utils.result_records import format_numeric_value, summarize_input_info +from utils.result_records import ( + format_numeric_value, + input_info_items_for_result, + summarize_input_info, +) from utils.trigger_display import summarize_execution_trigger @@ -28,15 +32,15 @@ def build_result_evidence_packet( ) -> str: """Build a Markdown evidence packet for one Result JSON.""" packet = _MarkdownBuilder() - packet.heading(1, "Benchkit Result Evidence Packet") + packet.heading(1, "CX Result Evidence Packet") packet.paragraph( - "This Evidence Packet is a portable review note for one Benchkit " - "benchmark result. Use it as the starting point for investigating " - "reproducibility, unexpected performance changes, source and input " - "provenance, build-cache reuse, and profiling or estimation evidence." + "This Evidence Packet is a portable review note for one benchmark " + "result. Use it as the starting point for investigating reproducibility, " + "unexpected performance changes, source and input provenance, " + "build-cache reuse, and profiling or estimation evidence." ) packet.paragraph( - "It is written for readers who may not know the surrounding Benchkit " + "It is written for readers who may not know the surrounding benchmark " "operation. Start from the Result, Source, Input, Quality, Profiling, " "and Build Cache sections below. Treat missing, declared-only, or " "unverifiable evidence as follow-up questions for the application or " @@ -87,7 +91,7 @@ def build_result_evidence_packet( ("Input status", input_summary["label"]), ("Summary", input_summary["summary"]), ]) - input_rows = _input_rows(result.get("input_info")) + input_rows = _input_rows(result) if input_rows: packet.bullets(input_rows) @@ -201,12 +205,8 @@ def _safe_url_or_placeholder(value: str) -> str: return value -def _input_rows(input_info: Any) -> list[str]: - if not isinstance(input_info, dict) or not input_info: - return [] - inputs = input_info.get("inputs") - input_items = inputs if isinstance(inputs, list) and inputs else [input_info] - +def _input_rows(result: dict[str, Any]) -> list[str]: + input_items = input_info_items_for_result(result) rows = [] for index, item in enumerate(input_items, start=1): if not isinstance(item, dict): @@ -217,6 +217,11 @@ def _input_rows(input_info: Any) -> list[str]: "dataset_version", "kind", "source", + "parameter_set_id", + "result_exp", + "command", + "arguments", + "parameters", "repo_relative_path", "verification_status", "manifest_digest", diff --git a/result_server/utils/evidence_snapshot.py b/result_server/utils/evidence_snapshot.py index 1d90a757..facc035b 100644 --- a/result_server/utils/evidence_snapshot.py +++ b/result_server/utils/evidence_snapshot.py @@ -146,6 +146,7 @@ def _merge_latest_results( public_input_available = has_public_input_info( data.get("input_info"), public_source_available, + result=data, ) public_result_available = not get_file_confidential_tags(record["filename"], received_dir) if public_result_available and public_source_available and public_input_available: diff --git a/result_server/utils/public_reuse.py b/result_server/utils/public_reuse.py index 21fa0635..05284b72 100644 --- a/result_server/utils/public_reuse.py +++ b/result_server/utils/public_reuse.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os import re from ipaddress import ip_address @@ -11,13 +12,14 @@ from utils.result_records import ( format_numeric_value, format_result_timestamp, + input_info_items_for_result, summarize_input_info, summarize_result_quality, ) PUBLIC_REUSE_MANIFEST_SCHEMA_VERSION = 1 -PUBLIC_REUSE_MANIFEST_KIND = "benchkit_public_reuse_packet" +PUBLIC_REUSE_MANIFEST_KIND = "cx_public_reuse_packet" def evaluate_public_reuse_packet( @@ -27,7 +29,12 @@ def evaluate_public_reuse_packet( ) -> dict[str, Any]: """Return the public reuse packet status for one Result JSON.""" public_source = has_public_source_info(result.get("source_info")) - public_input = has_public_input_info(result.get("input_info"), public_source) + input_items = input_info_items_for_result(result) + public_input = has_public_input_info( + result.get("input_info"), + public_source, + result=result, + ) if not public_result: status = "not exportable" @@ -35,7 +42,7 @@ def evaluate_public_reuse_packet( elif not public_source: status = "needs public source" next_action = "Record public source provenance" - elif not result.get("input_info"): + elif not input_items: status = "needs public input" next_action = "Declare public input binding" elif not public_input: @@ -113,7 +120,7 @@ def build_public_reuse_manifest( ), "result": _result_summary(result, filename), "source": _source_summary(result.get("source_info")), - "input": _input_summary(result.get("input_info"), input_summary), + "input": _input_summary(result, input_summary), "build": _build_summary(result.get("build_cache")), "profile": _profile_summary( result, @@ -143,12 +150,12 @@ def build_public_reuse_markdown_packet(manifest: dict[str, Any]) -> str: _clean(result.get("experiment")), ] title_suffix = " / ".join(part for part in title_parts if part) - packet.heading(1, "Benchkit Public Reuse Packet") + packet.heading(1, "CX Public Reuse Packet") if title_suffix: packet.paragraph(f"Target: {title_suffix}") packet.paragraph( - "This packet summarizes a public Benchkit benchmark result for reuse. " - "It includes public result metadata, public source provenance, public " + "This packet summarizes a public benchmark result for reuse. It " + "includes public result metadata, public source provenance, public " "input binding, and reusable build, profile, and estimation evidence." ) @@ -278,8 +285,17 @@ def has_public_source_info(source_info: Any) -> bool: ) -def has_public_input_info(input_info: Any, public_source_available: bool) -> bool: - input_items = _input_info_items(input_info) +def has_public_input_info( + input_info: Any, + public_source_available: bool, + *, + result: dict[str, Any] | None = None, +) -> bool: + input_items = ( + input_info_items_for_result(result) + if result is not None + else _input_info_items(input_info) + ) if not input_items: return False return all(_has_public_input_item(item, public_source_available) for item in input_items) @@ -341,9 +357,9 @@ def _source_summary(source_info: Any) -> dict[str, Any]: ) -def _input_summary(input_info: Any, input_summary: dict[str, Any]) -> dict[str, Any]: +def _input_summary(result: dict[str, Any], input_summary: dict[str, Any]) -> dict[str, Any]: items = [] - for item in _input_info_items(input_info): + for item in input_info_items_for_result(result): public_item = _public_input_item_summary(item) if public_item: items.append(public_item) @@ -364,6 +380,11 @@ def _public_input_item_summary(item: Any) -> dict[str, Any]: "dataset_version": item.get("dataset_version"), "kind": item.get("kind"), "source": item.get("source"), + "parameter_set_id": item.get("parameter_set_id"), + "result_exp": item.get("result_exp"), + "command": item.get("command"), + "arguments": item.get("arguments"), + "parameters": item.get("parameters"), "source_ref": item.get("source_ref"), "resolved_commit": item.get("resolved_commit"), "commit_hash": item.get("commit_hash"), @@ -553,6 +574,7 @@ def _has_public_input_item(item: Any, public_source_available: bool) -> bool: if not isinstance(item, dict): return False + kind = _clean(item.get("kind")).lower() source = _clean(item.get("source")).lower() verification_status = _clean(item.get("verification_status")).lower() repo_relative_path = _clean(item.get("repo_relative_path")) @@ -560,6 +582,15 @@ def _has_public_input_item(item: Any, public_source_available: bool) -> bool: if source == "source_info" or verification_status == "covered_by_source_commit": return True + if ( + kind in {"runtime-parameters", "inline-parameters"} + and source in {"inline", "self-contained", "self_contained"} + and verification_status in {"self_contained", "self-contained"} + and _clean(item.get("command")) + and isinstance(item.get("arguments"), list) + ): + return True + if _clean(item.get("doi")): return True @@ -655,6 +686,11 @@ def _format_input_item(item: dict[str, Any]) -> str: "dataset_version", "kind", "source", + "parameter_set_id", + "result_exp", + "command", + "arguments", + "parameters", "public_url", "source_url", "archive_url", @@ -674,10 +710,20 @@ def _format_input_item(item: dict[str, Any]) -> str: "recipe", "doi", ) - parts = [f"{key}: {item[key]}" for key in keys if item.get(key) not in (None, "", [], {})] + parts = [ + f"{key}: {_format_input_value(item[key])}" + for key in keys + if item.get(key) not in (None, "", [], {}) + ] return "; ".join(parts) +def _format_input_value(value: Any) -> str: + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False, sort_keys=True) + return str(value) + + def _format_profile_artifact(artifact: dict[str, Any]) -> str: archive = artifact.get("archive") url = artifact.get("url") diff --git a/result_server/utils/result_compare_view.py b/result_server/utils/result_compare_view.py index 858bd0b7..bb6b6e8d 100644 --- a/result_server/utils/result_compare_view.py +++ b/result_server/utils/result_compare_view.py @@ -6,6 +6,7 @@ build_compare_headline, format_numeric_value, format_result_timestamp, + input_info_items_for_result, load_result_json_batch, short_identifier, summarize_result_quality, @@ -250,19 +251,15 @@ def _source_summary(source_info): def _input_summary(data, stats): status = stats.get("input_info_status") or "none" label = stats.get("input_info_label") or "None" - input_info = data.get("input_info") - descriptors = _input_descriptors(input_info) + descriptors = _input_descriptors(data) display = label if not descriptors else f"{label}: {'; '.join(descriptors[:2])}" if len(descriptors) > 2: display = f"{display}; +{len(descriptors) - 2} more" return {"display": display, "key": (status, tuple(descriptors))} -def _input_descriptors(input_info): - if not isinstance(input_info, dict) or not input_info: - return [] - inputs = input_info.get("inputs") - items = inputs if isinstance(inputs, list) and inputs else [input_info] +def _input_descriptors(data): + items = input_info_items_for_result(data) descriptors = [] for item in items: if not isinstance(item, dict): diff --git a/result_server/utils/result_records.py b/result_server/utils/result_records.py index b3ed65ec..ce728f41 100644 --- a/result_server/utils/result_records.py +++ b/result_server/utils/result_records.py @@ -228,8 +228,15 @@ def summarize_input_info(data): "summary": "No input_info object is stored for this result.", } - inputs = input_info.get("inputs") - input_items = inputs if isinstance(inputs, list) and inputs else [input_info] + input_items = input_info_items_for_result(data) + if not input_items: + return { + "present": False, + "status": "none", + "label": "None", + "summary": "No matching input_info item is stored for this result.", + } + ranks = { "declared": 1, "covered": 2, @@ -263,12 +270,62 @@ def summarize_input_info(data): } +def input_info_items_for_result(data): + if not isinstance(data, dict): + return [] + input_info = data.get("input_info") + if not isinstance(input_info, dict) or not input_info: + return [] + + inputs = input_info.get("inputs") + input_items = inputs if isinstance(inputs, list) and inputs else [input_info] + result_exp = str(data.get("Exp") or data.get("exp") or "").strip() + + scoped_items = [] + matching_or_unscoped_items = [] + for item in input_items: + scope_values = _input_item_result_scope_values(item) + if scope_values: + scoped_items.append(item) + if result_exp and result_exp in scope_values: + matching_or_unscoped_items.append(item) + else: + matching_or_unscoped_items.append(item) + + if scoped_items: + return matching_or_unscoped_items + return input_items + + +def _input_item_result_scope_values(item): + if not isinstance(item, dict): + return [] + + values = [] + for key in ("result_exp", "Exp", "exp"): + value = item.get(key) + if value not in (None, ""): + values.append(str(value).strip()) + + for key in ("result_scope", "result"): + scope = item.get(key) + if not isinstance(scope, dict): + continue + for scope_key in ("Exp", "exp", "experiment"): + value = scope.get(scope_key) + if value not in (None, ""): + values.append(str(value).strip()) + + return [value for value in values if value] + + def _classify_input_info_item(item, has_source_commit): if not isinstance(item, dict): return "declared" verification_status = str(item.get("verification_status") or "").strip().lower() source = str(item.get("source") or "").strip().lower() + kind = str(item.get("kind") or "").strip().lower() digest_fields = ( "manifest_digest", "content_digest", @@ -297,6 +354,16 @@ def _classify_input_info_item(item, has_source_commit): if repo_local_covered: return "covered" + runtime_parameters_covered = ( + kind in {"runtime-parameters", "inline-parameters"} + and source in {"inline", "self-contained", "self_contained"} + and verification_status in {"self_contained", "self-contained"} + and bool(item.get("command")) + and isinstance(item.get("arguments"), list) + ) + if runtime_parameters_covered: + return "covered" + public_source_covered = ( has_input_revision and verification_status in {"public_source_commit", "covered_by_public_source_commit"} diff --git a/scripts/bk_functions.sh b/scripts/bk_functions.sh index a0f84800..a2e76ad6 100644 --- a/scripts/bk_functions.sh +++ b/scripts/bk_functions.sh @@ -978,6 +978,54 @@ bk_json_string_array() { printf ']' } +bk_decode_base64_value() { + if command -v base64 >/dev/null 2>&1; then + base64 --decode 2>/dev/null || base64 -d 2>/dev/null + return $? + fi + if command -v openssl >/dev/null 2>&1; then + openssl base64 -d -A + return $? + fi + return 1 +} + +bk_env_file_value() { + _bk_env_file="$1" + _bk_env_key="$2" + _bk_env_line="" + + [ -f "$_bk_env_file" ] || return 0 + + _bk_env_line=$(awk -F= -v k="${_bk_env_key}_B64" '$1 == k {print substr($0, length(k) + 2); exit}' "$_bk_env_file") + if [ -n "$_bk_env_line" ]; then + printf '%s' "$_bk_env_line" | bk_decode_base64_value 2>/dev/null || true + return 0 + fi + + _bk_env_line=$(awk -F= -v k="$_bk_env_key" ' + $1 == k { + print substr($0, length(k) + 2) + exit + } + ' "$_bk_env_file") + if [ -n "$_bk_env_line" ]; then + printf '%s' "$_bk_env_line" + return 0 + fi + + awk -v key="$_bk_env_key" ' + index($0, "export " key "=\"") == 1 && substr($0, length($0), 1) == "\"" { + prefix = "export " key "=\"" + value = substr($0, length(prefix) + 1, length($0) - length(prefix) - 1) + if (value !~ /[`$\\]/) { + print value + } + exit + } + ' "$_bk_env_file" +} + # bk_record_input_info - Pass benchmark input metadata to Benchkit. # # Usage: @@ -1035,6 +1083,137 @@ bk_record_input_info() { fi } +bk_record_runtime_parameter_input() { + _bk_rt_dataset_id="" + _bk_rt_dataset_version="" + _bk_rt_parameter_set_id="" + _bk_rt_result_exp="" + _bk_rt_command="" + _bk_rt_recipe="" + + while [ $# -gt 0 ]; do + case "$1" in + --dataset-id) + if [ $# -lt 2 ]; then + echo "bk_record_runtime_parameter_input: --dataset-id requires a value" >&2 + return 1 + fi + shift + _bk_rt_dataset_id="$1" + ;; + --dataset-version) + if [ $# -lt 2 ]; then + echo "bk_record_runtime_parameter_input: --dataset-version requires a value" >&2 + return 1 + fi + shift + _bk_rt_dataset_version="$1" + ;; + --parameter-set-id) + if [ $# -lt 2 ]; then + echo "bk_record_runtime_parameter_input: --parameter-set-id requires a value" >&2 + return 1 + fi + shift + _bk_rt_parameter_set_id="$1" + ;; + --result-exp) + if [ $# -lt 2 ]; then + echo "bk_record_runtime_parameter_input: --result-exp requires a value" >&2 + return 1 + fi + shift + _bk_rt_result_exp="$1" + ;; + --command) + if [ $# -lt 2 ]; then + echo "bk_record_runtime_parameter_input: --command requires a value" >&2 + return 1 + fi + shift + _bk_rt_command="$1" + ;; + --recipe) + if [ $# -lt 2 ]; then + echo "bk_record_runtime_parameter_input: --recipe requires a value" >&2 + return 1 + fi + shift + _bk_rt_recipe="$1" + ;; + --) + shift + break + ;; + *) + echo "bk_record_runtime_parameter_input: unknown option: $1" >&2 + return 1 + ;; + esac + shift + done + + if [ -z "$_bk_rt_command" ]; then + echo "bk_record_runtime_parameter_input: --command is required" >&2 + return 1 + fi + if [ -z "$_bk_rt_parameter_set_id" ]; then + _bk_rt_parameter_set_id="runtime-parameters" + fi + if [ -z "$_bk_rt_dataset_id" ]; then + _bk_rt_dataset_id="runtime-parameters-${_bk_rt_parameter_set_id}" + fi + + _bk_rt_info_file="${BK_INPUT_INFO_FILE:-results/input_info.json}" + _bk_rt_item_file="${BK_INPUT_INFO_ITEMS_FILE:-results/.input_info_items.jsonl}" + mkdir -p "$(dirname "$_bk_rt_item_file")" || return 1 + + { + printf '{' + printf '"dataset_id":' + bk_json_string "$_bk_rt_dataset_id" + if [ -n "$_bk_rt_dataset_version" ]; then + printf ',"dataset_version":' + bk_json_string "$_bk_rt_dataset_version" + fi + printf ',"kind":"runtime-parameters"' + printf ',"source":"inline"' + printf ',"parameter_set_id":' + bk_json_string "$_bk_rt_parameter_set_id" + if [ -n "$_bk_rt_result_exp" ]; then + printf ',"result_exp":' + bk_json_string "$_bk_rt_result_exp" + fi + printf ',"command":' + bk_json_string "$_bk_rt_command" + printf ',"arguments":' + bk_json_string_array "$@" + if [ -n "$_bk_rt_recipe" ]; then + printf ',"recipe":' + bk_json_string "$_bk_rt_recipe" + fi + printf ',"verification_status":"self_contained"}\n' + } >> "$_bk_rt_item_file" + + { + printf '{\n' + printf ' "schema_version": 1,\n' + printf ' "inputs": [\n' + _bk_rt_first=1 + while IFS= read -r _bk_rt_item; do + [ -n "$_bk_rt_item" ] || continue + if [ "$_bk_rt_first" -eq 0 ]; then + printf ',\n' + fi + printf ' %s' "$_bk_rt_item" + _bk_rt_first=0 + done < "$_bk_rt_item_file" + printf '\n' + printf ' ]\n' + printf '}\n' + } | BK_INPUT_INFO_FILE="$_bk_rt_info_file" bk_record_input_info +} + # Write a compact, tool-neutral manifest for the profiler archive. Result JSON # generation reads this manifest to expose summary fields without opening every # raw profiler artifact. For fapp, run_events contains counter names; for ncu it @@ -2102,14 +2281,16 @@ bk_fetch_source() { fi if [ -n "$_bk_expected_commit" ]; then - if ! git -C "$_bk_dest" cat-file -e "${_bk_expected_commit}^{commit}" 2>/dev/null; then - git -C "$_bk_dest" fetch origin "$_bk_expected_commit" 2>/dev/null || true - fi - if ! git -C "$_bk_dest" checkout --detach "$_bk_expected_commit" 2>&1; then - echo "bk_fetch_source: expected commit not available: $_bk_expected_commit" >&2 - return 1 + if [ "$BK_COMMIT_HASH" != "$_bk_expected_commit" ]; then + if ! git -C "$_bk_dest" cat-file -e "${_bk_expected_commit}^{commit}" 2>/dev/null; then + git -C "$_bk_dest" fetch origin "$_bk_expected_commit" 2>/dev/null || true + fi + if ! git -C "$_bk_dest" checkout --detach "$_bk_expected_commit" 2>&1; then + echo "bk_fetch_source: expected commit not available: $_bk_expected_commit" >&2 + return 1 + fi + BK_COMMIT_HASH=$(git -C "$_bk_dest" rev-parse HEAD 2>/dev/null || echo "") fi - BK_COMMIT_HASH=$(git -C "$_bk_dest" rev-parse HEAD 2>/dev/null || echo "") if [ "$BK_COMMIT_HASH" != "$_bk_expected_commit" ]; then echo "bk_fetch_source: commit mismatch for '$_bk_src'" >&2 echo " expected: $_bk_expected_commit" >&2 @@ -2154,3 +2335,43 @@ bk_fetch_source() { return 0 } + +bk_fetch_recorded_source() { + if [ $# -lt 2 ]; then + echo "bk_fetch_recorded_source: requires and arguments" >&2 + return 1 + fi + + _bk_recorded_src="$1" + _bk_recorded_dest="$2" + _bk_recorded_ref="${3:-}" + _bk_recorded_expected="${4:-}" + _bk_recorded_info_file="${5:-results/source_info.env}" + + if [ -f "$_bk_recorded_info_file" ]; then + _bk_recorded_type=$(bk_env_file_value "$_bk_recorded_info_file" BK_SOURCE_TYPE) + if [ "$_bk_recorded_type" = "git" ]; then + _bk_recorded_repo_url=$(bk_env_file_value "$_bk_recorded_info_file" BK_REPO_URL) + _bk_recorded_source_ref=$(bk_env_file_value "$_bk_recorded_info_file" BK_SOURCE_REF_NAME) + if [ -z "$_bk_recorded_source_ref" ]; then + _bk_recorded_source_ref=$(bk_env_file_value "$_bk_recorded_info_file" BK_BRANCH) + fi + _bk_recorded_commit=$(bk_env_file_value "$_bk_recorded_info_file" BK_SOURCE_RESOLVED_COMMIT) + if [ -z "$_bk_recorded_commit" ]; then + _bk_recorded_commit=$(bk_env_file_value "$_bk_recorded_info_file" BK_COMMIT_HASH) + fi + + if [ -n "$_bk_recorded_repo_url" ]; then + _bk_recorded_src="$_bk_recorded_repo_url" + fi + if [ -n "$_bk_recorded_source_ref" ]; then + _bk_recorded_ref="$_bk_recorded_source_ref" + fi + if [ -n "$_bk_recorded_commit" ]; then + _bk_recorded_expected="$_bk_recorded_commit" + fi + fi + fi + + bk_fetch_source "$_bk_recorded_src" "$_bk_recorded_dest" "$_bk_recorded_ref" "$_bk_recorded_expected" +} diff --git a/scripts/result.sh b/scripts/result.sh index 009b12bf..73ff6c3e 100644 --- a/scripts/result.sh +++ b/scripts/result.sh @@ -533,6 +533,53 @@ if ! input_info_block=$(build_input_info_block); then exit 1 fi +filter_input_info_block_for_result() { + local result_exp="$1" + + if [ -z "$input_info_block" ]; then + printf '%s' "" + return 0 + fi + if [ -z "$result_exp" ] || [ "$result_exp" = "null" ]; then + printf '%s' "$input_info_block" + return 0 + fi + + printf '%s' "$input_info_block" | jq -cS --arg exp "$result_exp" ' + def scope_values: + [ + .result_exp?, + .Exp?, + .exp?, + .result_scope?.Exp?, + .result_scope?.exp?, + .result_scope?.experiment?, + .result?.Exp?, + .result?.exp?, + .result?.experiment? + ] + | map(select(. != null and . != "") | tostring); + + if (.inputs | type) == "array" then + (.inputs | map(select((scope_values | length) == 0 or (scope_values | index($exp))))) as $items + | (.inputs | map(select((scope_values | length) > 0)) | length) as $scoped_count + | if ($items | length) > 0 then + .inputs = $items + elif $scoped_count > 0 then + empty + else + . + end + else + if (scope_values | length) == 0 or (scope_values | index($exp)) then + . + else + empty + end + end + ' 2>/dev/null || true +} + profiled_run_included=false if has_profiler_archive; then profiled_run_included=true @@ -635,9 +682,11 @@ write_result_json() { fi local input_info_json_block="" - if [ -n "$input_info_block" ]; then + local result_input_info_block="" + result_input_info_block=$(filter_input_info_block_for_result "$exp") + if [ -n "$result_input_info_block" ]; then input_info_json_block=", - \"input_info\": ${input_info_block}" + \"input_info\": ${result_input_info_block}" fi # Attach the profiler summary that matches this FOM index. fapp exposes diff --git a/scripts/tests/test_bk_fetch_source.sh b/scripts/tests/test_bk_fetch_source.sh index 70fe81c1..702825f5 100644 --- a/scripts/tests/test_bk_fetch_source.sh +++ b/scripts/tests/test_bk_fetch_source.sh @@ -75,6 +75,28 @@ jq -e --arg commit "$commit_one" ' ' results/result0.json >/dev/null popd >/dev/null +mkdir -p "${TMP_DIR}/git-recorded-work" +pushd "${TMP_DIR}/git-recorded-work" >/dev/null +write_minimal_result +bk_write_source_info_env \ + git \ + "${TMP_DIR}/origin.git" \ + main \ + "$commit_one" \ + "" "" "" "" "" \ + main \ + branch \ + "$commit_one" +bk_fetch_recorded_source "${TMP_DIR}/origin.git" checkout main +test "$(git -C checkout rev-parse HEAD)" = "$commit_one" +bash "${REPO_DIR}/scripts/result.sh" app TestSystem native build run 123 >/dev/null +jq -e --arg commit "$commit_one" ' + .source_info.source_type == "git" and + .source_info.branch == "main" and + .source_info.resolved_commit == $commit +' results/result0.json >/dev/null +popd >/dev/null + mkdir -p "${TMP_DIR}/git-mismatch" pushd "${TMP_DIR}/git-mismatch" >/dev/null write_minimal_result diff --git a/scripts/tests/test_bk_input_info.sh b/scripts/tests/test_bk_input_info.sh index 86428be3..d6da44bb 100755 --- a/scripts/tests/test_bk_input_info.sh +++ b/scripts/tests/test_bk_input_info.sh @@ -47,6 +47,51 @@ if command -v jq >/dev/null 2>&1; then jq -e '.inputs[0].dataset_id == "demo-case1"' custom/input_info.json >/dev/null fi +rm -f results/input_info.json results/.input_info_items.jsonl +bk_record_runtime_parameter_input \ + --dataset-id demo-case0-parameters \ + --dataset-version v1 \ + --parameter-set-id CASE0 \ + --result-exp CASE0 \ + --command ./main \ + --recipe "run ./main with recorded arguments" \ + -- 32 6 4 3 1 1 1 1 -1 -1 6 50 +bk_record_runtime_parameter_input \ + --dataset-id demo-case1-parameters \ + --parameter-set-id CASE1 \ + --result-exp CASE1 \ + --command ./main \ + -- 32 6 4 3 1 1 1 2 -1 -1 6 50 +test -s results/input_info.json +if command -v jq >/dev/null 2>&1; then + jq -e ' + .schema_version == 1 and + (.inputs | length) == 2 and + .inputs[0].dataset_id == "demo-case0-parameters" and + .inputs[0].dataset_version == "v1" and + .inputs[0].kind == "runtime-parameters" and + .inputs[0].source == "inline" and + .inputs[0].parameter_set_id == "CASE0" and + .inputs[0].result_exp == "CASE0" and + .inputs[0].command == "./main" and + .inputs[0].arguments == ["32", "6", "4", "3", "1", "1", "1", "1", "-1", "-1", "6", "50"] and + .inputs[0].verification_status == "self_contained" and + .inputs[1].parameter_set_id == "CASE1" + ' results/input_info.json >/dev/null +fi + +bk_write_source_info_env \ + git \ + "https://example.test/demo.git" \ + main \ + 1234567890abcdef1234567890abcdef12345678 \ + "" "" "" "" "" \ + main \ + branch \ + 1234567890abcdef1234567890abcdef12345678 +test "$(bk_env_file_value results/source_info.env BK_REPO_URL)" = "https://example.test/demo.git" +test "$(bk_env_file_value results/source_info.env BK_SOURCE_RESOLVED_COMMIT)" = "1234567890abcdef1234567890abcdef12345678" + if bk_record_input_info missing.json >/dev/null 2>&1; then echo "bk_record_input_info accepted a missing file" >&2 exit 1 diff --git a/scripts/tests/test_result_common_json_contract.sh b/scripts/tests/test_result_common_json_contract.sh index a47f3551..38f4772a 100644 --- a/scripts/tests/test_result_common_json_contract.sh +++ b/scripts/tests/test_result_common_json_contract.sh @@ -19,6 +19,7 @@ FOM:1.25 FOM_unit:s FOM_version:contract-v1 Exp:CASE0 node_count:2 numproc_node: SECTION:solve time:1.0 SECTION:io time:0.25 OVERLAP:solve,io time:0.10 +FOM:2.50 FOM_unit:s FOM_version:contract-v1 Exp:CASE1 node_count:2 numproc_node:4 nthreads:8 description:smoke confidential:false EOF cat > "${TMP_DIR}/results/source_info.env" <<'EOF' @@ -39,8 +40,20 @@ cat > "${TMP_DIR}/results/input_info.json" <<'EOF' "dataset_id": "demo-case0", "dataset_version": "2026-09", "kind": "repo-local-input", + "result_exp": "CASE0", "verification_status": "covered_by_source_commit", "repo_relative_path": "inputs/demo-case0" + }, + { + "dataset_id": "demo-case1", + "dataset_version": "2026-09", + "kind": "runtime-parameters", + "source": "inline", + "parameter_set_id": "CASE1", + "result_exp": "CASE1", + "command": "./demo", + "arguments": ["--case", "1"], + "verification_status": "self_contained" } ] } @@ -132,6 +145,7 @@ bash "${REPO_DIR}/scripts/result.sh" demoapp DemoSystem cross demoapp_DemoSystem popd >/dev/null RESULT_JSON="${TMP_DIR}/results/result0.json" +RESULT_JSON1="${TMP_DIR}/results/result1.json" test -f "${RESULT_JSON}" jq -e ' @@ -162,7 +176,9 @@ jq -e ' .source_info.ref_kind == "branch" and .source_info.resolved_commit == "abcdef1234567890abcdef1234567890abcdef12" and .input_info.schema_version == 1 and + (.input_info.inputs | length) == 1 and .input_info.inputs[0].dataset_id == "demo-case0" and + .input_info.inputs[0].result_exp == "CASE0" and .input_info.inputs[0].verification_status == "covered_by_source_commit" and .pipeline_timing.build_time == 12 and .pipeline_timing.queue_time == 0 and @@ -174,6 +190,15 @@ jq -e ' (.pipeline_timing | has("profiled_run_included") | not) ' "${RESULT_JSON}" >/dev/null +jq -e ' + .Exp == "CASE1" and + .input_info.schema_version == 1 and + (.input_info.inputs | length) == 1 and + .input_info.inputs[0].dataset_id == "demo-case1" and + .input_info.inputs[0].kind == "runtime-parameters" and + .input_info.inputs[0].arguments == ["--case", "1"] +' "${RESULT_JSON1}" >/dev/null + jq -e ' .fom_breakdown.sections[0].name == "solve" and .fom_breakdown.sections[0].time == 1 and