From 1cb7266f4f6ed3e43fc67125940b642aca492e46 Mon Sep 17 00:00:00 2001 From: yoshifuminakamura Date: Thu, 10 Sep 2026 23:33:37 +0900 Subject: [PATCH] Record public input source binding Signed-off-by: yoshifuminakamura --- docs/cx/BENCHKIT_SPEC.md | 6 ++ docs/guides/add-app.md | 2 +- programs/genesis/run.sh | 50 +++++++++++++++-- ...sage_report_evidence_snapshot_section.html | 2 +- result_server/tests/test_evidence_snapshot.py | 56 +++++++++++++++++++ .../tests/test_portal_list_templates.py | 2 +- result_server/tests/test_results_loader.py | 22 ++++++++ result_server/utils/result_records.py | 23 +++++++- 8 files changed, 154 insertions(+), 9 deletions(-) diff --git a/docs/cx/BENCHKIT_SPEC.md b/docs/cx/BENCHKIT_SPEC.md index f13d1263..1d03709c 100644 --- a/docs/cx/BENCHKIT_SPEC.md +++ b/docs/cx/BENCHKIT_SPEC.md @@ -490,6 +490,9 @@ Benchkit は、pre-staged input、restart、学習済みモデル、公開 archi この場合、`source_info` が実際に使った app source と repo 内 input の両方の固定点になる。 `input_info` は省略してもよく、Portal や review で dataset 名を見せたい場合だけ、`kind: "repo-local-input"`、`source: "source_info"`、`repo_relative_path`、`verification_status: "covered_by_source_commit"` などの補助情報を持たせてよい。 +入力が別の public input repository や public archive から来る場合は、`input_info` 側に公開URL、ref、resolved commit または digest、取得/生成 recipe を置く。 +この場合も、入力が記録済み source commit で固定されるなら `verification_status: "public_source_commit"` として `Covered` に分類できる。 + site-local path は所在情報であり、長期的な input identity ではない。 巨大データや共同研究由来データを site-local shared storage に置くことは許容されるが、Result provenance では path より dataset identity、recipe、manifest、digest を優先する。 public surface では、必要がない限り detailed local path を表示しない。 @@ -514,6 +517,9 @@ When the input is already stored in the top-level application repository and is In that case, `source_info` is the fixed point for both the application source and the repository-local input actually used. `input_info` may be omitted, or it may carry lightweight helper fields such as `kind: "repo-local-input"`, `source: "source_info"`, `repo_relative_path`, and `verification_status: "covered_by_source_commit"` when the dataset name should be visible in the Portal or during review. +When the input comes from a separate public input repository or public archive, `input_info` should carry the public URL, ref, resolved commit or digest, and acquisition or generation recipe. +If the input is fixed by a recorded source commit, it may use `verification_status: "public_source_commit"` and be classified as `Covered`. + A site-local path is location information, not a durable input identity. Large datasets and collaboration-derived inputs may still be staged on site-local shared storage, but Result provenance should prefer dataset identity, recipe, manifest, and digest over paths. Detailed local paths should not be exposed on the public surface unless they are necessary. diff --git a/docs/guides/add-app.md b/docs/guides/add-app.md index 3e360750..4efdd053 100644 --- a/docs/guides/add-app.md +++ b/docs/guides/add-app.md @@ -193,7 +193,7 @@ Portal の `/results/usage` では、通常の benchmark result に対する入 - `None`: `input_info` がない - `Declared`: `input_info` はあるが、digest 検証や source commit coverage までは示していない -- `Covered`: repo-local input が `source_info.resolved_commit` で固定されることを示している +- `Covered`: repo-local input または public input source が記録済み source commit で固定されることを示している - `Verified`: manifest / content digest などの証跡と `verification_status: "verified"` がある `None` や `Declared` はただちに CI failure ではありません。 diff --git a/programs/genesis/run.sh b/programs/genesis/run.sh index cc340a2c..b0488fc4 100644 --- a/programs/genesis/run.sh +++ b/programs/genesis/run.sh @@ -35,6 +35,43 @@ stderr="${resultsdir}/log_${header}_err.txt" binary="spdyn" inputdir="../../../inputs/apoa1/" +record_public_input_info() { + local input_source_commit="$1" + { + printf '{\n' + printf ' "schema_version": 1,\n' + printf ' "inputs": [\n' + printf ' {\n' + printf ' "dataset_id": ' + bk_json_string "apoa1-p8" + printf ',\n' + printf ' "dataset_version": ' + bk_json_string "$BRANCH" + printf ',\n' + printf ' "kind": "public-git",\n' + printf ' "source": "public_url",\n' + printf ' "public_url": ' + bk_json_string "$REPO_URL" + printf ',\n' + printf ' "source_ref": ' + bk_json_string "$BRANCH" + printf ',\n' + printf ' "resolved_commit": ' + bk_json_string "$input_source_commit" + printf ',\n' + printf ' "repo_relative_path": ' + bk_json_string "$dir_path" + printf ',\n' + printf ' "recipe": ' + bk_json_string "${input}.sub generated from ${input}" + printf ',\n' + printf ' "verification_status": "public_source_commit"\n' + printf ' }\n' + printf ' ]\n' + printf '}\n' + } | bk_record_input_info +} + echo "[${REPO_DIR}] Running on system: $system" if [[ -d "${REPO_DIR}" ]]; then @@ -51,7 +88,6 @@ echo "System=$system" echo "Nodes=$nodes" echo "numproc=$numproc" echo "nthreads=$nthreads" -totalcores=$(( numproc * nthreads )) if [[ ! -d ${REPO_DIR} ]]; then git clone --branch "${BRANCH}" "${REPO_URL}" "${REPO_DIR}" @@ -59,6 +95,8 @@ else echo "Reposiotry already exists and looks valid. Skipping clone." fi +input_source_commit=$(git -C "${REPO_DIR}" rev-parse HEAD) +record_public_input_info "$input_source_commit" if [[ ! -f "${artifactsdir}/spdyn" ]]; then echo "Error: spdyn does not exist." @@ -128,6 +166,8 @@ run_genesis_nvidia_gpu() { local cuda_visible_devices_var="${env_prefix}_CUDA_VISIBLE_DEVICES" local profiler_tool_var="${env_prefix}_PROFILER_TOOL" local profiler_level_var="${env_prefix}_PROFILER_LEVEL" + local -a nvidia_mpi_cmd + local -a nvidia_mpi_args local module_name="${!module_var:-$default_module}" if [ "$module_name" != "none" ] && command -v module >/dev/null 2>&1; then @@ -135,10 +175,10 @@ run_genesis_nvidia_gpu() { module load "${module_names[@]}" fi - read -r -a mpi_cmd <<< "${!mpi_cmd_var:-mpirun -np ${numproc}}" + read -r -a nvidia_mpi_cmd <<< "${!mpi_cmd_var:-mpirun -np ${numproc}}" if [ -n "${!mpi_args_var:-}" ]; then read -r -a nvidia_mpi_args <<< "${!mpi_args_var}" - mpi_cmd+=("${nvidia_mpi_args[@]}") + nvidia_mpi_cmd+=("${nvidia_mpi_args[@]}") fi export OMP_NUM_THREADS=${nthreads} @@ -150,8 +190,8 @@ run_genesis_nvidia_gpu() { genesis_configure_ncu_profile "$system_name" "$profiler_tool_var" "$profiler_level_var" "$module_var" || return 1 echo "Running ${system_name} as NVIDIA GPU benchmark run without profiler" - "${mpi_cmd[@]}" ./${binary} ${input}.sub 2>&1 | tee ${output} - genesis_run_configured_ncu_profiles "$system_name" "${mpi_cmd[@]}" ./${binary} ${input}.sub || return 1 + "${nvidia_mpi_cmd[@]}" ./${binary} ${input}.sub 2>&1 | tee ${output} + genesis_run_configured_ncu_profiles "$system_name" "${nvidia_mpi_cmd[@]}" ./${binary} ${input}.sub || return 1 } genesis_rikyu_apptainer_run_prefix() { diff --git a/result_server/templates/_usage_report_evidence_snapshot_section.html b/result_server/templates/_usage_report_evidence_snapshot_section.html index 7631e7eb..5350faae 100644 --- a/result_server/templates/_usage_report_evidence_snapshot_section.html +++ b/result_server/templates/_usage_report_evidence_snapshot_section.html @@ -17,7 +17,7 @@

Evidence Snapshot

Evidence Snapshot: the roll-up and CSV export source for configured, executed, profiled, estimated, source, input, and build-cache evidence. Configured: yes = enabled and implemented; partial = enabled but script support incomplete; off = configured off; no = not listed. Result Quality: missing = no result; basic = core result only; ready = estimation bindings present; rich = source provenance and artifacts present. - Input Status: None = no input_info; Declared = input_info only; Covered = repo-local input fixed by source_info; Verified = digest-backed input verification. + Input Status: None = no input_info; Declared = input_info only; Covered = input fixed by a recorded source commit; Verified = digest-backed input verification. Reuse Package: complete = public packet eligible with profile and estimate evidence; public packet eligible = source material is ready for a public Markdown reuse packet. Public Packet: eligible = public result with public source provenance and public input binding. Next Action: the first practical follow-up suggested by the current evidence state. diff --git a/result_server/tests/test_evidence_snapshot.py b/result_server/tests/test_evidence_snapshot.py index fda62ca8..285331c6 100644 --- a/result_server/tests/test_evidence_snapshot.py +++ b/result_server/tests/test_evidence_snapshot.py @@ -162,6 +162,62 @@ def test_evidence_snapshot_requires_public_source_for_public_packet(tmp_path): assert row["reuse_package_status"] == "needs public evidence" +def test_evidence_snapshot_accepts_public_input_commit_for_public_packet(tmp_path): + received_dir = tmp_path / "received" + estimated_dir = tmp_path / "estimated" + received_dir.mkdir() + estimated_dir.mkdir() + + _write_json( + received_dir / "result_20260901_010101_aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee.json", + { + "code": "demoapp", + "system": "DemoSystem", + "Exp": "CASE0", + "FOM": 1.0, + "source_info": { + "source_type": "git", + "repo_url": "https://example.com/demoapp.git", + "ref_name": "main", + "resolved_commit": "abcdef1234567890", + }, + "input_info": { + "inputs": [ + { + "dataset_id": "apoa1-p8", + "kind": "public-git", + "source": "public_url", + "public_url": "https://example.com/input.git", + "source_ref": "main", + "resolved_commit": "1234567890abcdef", + "repo_relative_path": "npt/apoa1", + "verification_status": "public_source_commit", + } + ], + }, + }, + ) + + snapshot = build_evidence_snapshot( + str(received_dir), + str(estimated_dir), + generated_at="2026-09-07T00:00:00Z", + app_support_rows=[ + { + "app": "demoapp", + "systems": { + "DemoSystem": {"status": "enabled"}, + }, + } + ], + ) + + row = snapshot["rows"][0] + assert row["input_status"] == "Covered" + assert row["public_packet_status"] == "eligible" + assert row["reuse_package_status"] == "public packet eligible" + + def test_evidence_snapshot_uses_estimate_benchmark_systems_without_future_target_rows(tmp_path): received_dir = tmp_path / "received" estimated_dir = tmp_path / "estimated" diff --git a/result_server/tests/test_portal_list_templates.py b/result_server/tests/test_portal_list_templates.py index 052578bd..aa08b77b 100644 --- a/result_server/tests/test_portal_list_templates.py +++ b/result_server/tests/test_portal_list_templates.py @@ -823,5 +823,5 @@ def test_usage_report_evidence_snapshot_consolidates_coverage_and_quality(): assert "Maturity Gaps" in html assert "Input Status" in html assert "None = no input_info" in html - assert "Covered = repo-local input fixed by source_info" in html + assert "Covered = input fixed by a recorded source commit" in html assert "no profile; no estimate; source incomplete; input not declared" in html diff --git a/result_server/tests/test_results_loader.py b/result_server/tests/test_results_loader.py index 3ef0f9f9..3c808a29 100644 --- a/result_server/tests/test_results_loader.py +++ b/result_server/tests/test_results_loader.py @@ -504,6 +504,28 @@ def test_input_info_status_classification(self): }) assert covered["stats"]["input_info_status"] == "covered" + public_source_covered = summarize_result_quality({ + "code": "test", + "system": "sys", + "FOM": 1.0, + "input_info": { + "schema_version": 1, + "inputs": [ + { + "dataset_id": "case0", + "kind": "public-git", + "source": "public_url", + "public_url": "https://example.com/input.git", + "source_ref": "main", + "resolved_commit": "abcdef1234567890", + "repo_relative_path": "benchmarks/case0", + "verification_status": "public_source_commit", + } + ], + }, + }) + assert public_source_covered["stats"]["input_info_status"] == "covered" + missing_source_commit = summarize_result_quality({ "code": "test", "system": "sys", diff --git a/result_server/utils/result_records.py b/result_server/utils/result_records.py index 9495f13a..b3ed65ec 100644 --- a/result_server/utils/result_records.py +++ b/result_server/utils/result_records.py @@ -252,7 +252,7 @@ def summarize_input_info(data): } summaries = { "declared": "input_info is present, but digest or source-commit coverage is not declared as verified.", - "covered": "input_info declares repository-local input covered by source_info.resolved_commit.", + "covered": "input_info declares input covered by a recorded source commit.", "verified": "input_info declares verified input with digest evidence.", } return { @@ -277,6 +277,14 @@ def _classify_input_info_item(item, has_source_commit): "digest", ) has_digest = any(item.get(field) for field in digest_fields) + revision_fields = ( + "resolved_commit", + "commit_hash", + "source_commit", + "revision", + "dataset_revision", + ) + has_input_revision = any(item.get(field) for field in revision_fields) if verification_status == "verified" and has_digest: return "verified" @@ -289,6 +297,19 @@ def _classify_input_info_item(item, has_source_commit): if repo_local_covered: return "covered" + public_source_covered = ( + has_input_revision + and verification_status in {"public_source_commit", "covered_by_public_source_commit"} + and ( + source in {"public_url", "public_git", "public-git"} + or item.get("public_url") + or item.get("source_url") + or item.get("archive_url") + ) + ) + if public_source_covered: + return "covered" + return "declared"