diff --git a/docs/reference.md b/docs/reference.md index 869d08d..753fe09 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -1557,7 +1557,8 @@ Recommended layout: - `bounded_download_smoke_summary.json`: read-only JSON summary for `download-smoke prepare`. Stable fields include `schema_version`, `command`, `source_download_plan_path`, `limit`, `requested_quality_tier`, `resolved_quality_tier`, `quality_tier`, `selected_row_count`, `selected_high_quality_row_count`, `selected_quality_tier_counts`, `selected_assembly_level_counts`, `selected_refseq_category_counts`, `selected_accession_quality_preview`, `selected_accession_quality_preview_truncated`, `selected_datasets_command_preview`, `selected_datasets_command_preview_truncated`, `selected_datasets_command_preview_only`, `inspection_min_fasta_n50_bases`, `inspection_max_fasta_record_count`, `inspection_max_fasta_ambiguous_bases`, `inspection_min_fasta_total_bases`, `inspection_min_fasta_longest_record_bases`, `inspection_quality_profile`, `inspection_block_fragmented_fasta`, `inspection_block_fasta_header_keywords`, `recommended_inspection_request_target`, `recommended_inspection_request`, `recommended_inspection_next_command`, `recommended_inspection_command`, `recommended_execution_validation_request_target`, `recommended_execution_validation_request`, `recommended_execution_validation_next_command`, `recommended_execution_validation_command`, `handoff_checklist`, `source_planned_row_count`, `source_high_quality_planned_row_count`, `source_draft_or_fragmented_planned_row_count`, `source_unknown_assembly_level_planned_row_count`, `source_refseq_category_counts`, `source_total_rows`, `ready`, `blockers`, `execution_boundary`, `safe_for_unattended_download`, `downloads_triggered`, `providers_contacted`, `network_access`, `external_tools`, `manifest_mutated`, and `strict_scientific_deliverable`. `selected_high_quality_row_count`, `selected_quality_tier_counts`, and `selected_refseq_category_counts` count only selected rows; source-level high-quality and RefSeq-category availability are reported separately. `selected_accession_quality_preview` includes selected accessions, normalized assembly levels, normalized RefSeq category labels, and quality tiers. `selected_datasets_command_preview` lists at most five bounded `datasets download genome accession ... --include genome --filename ...` command arrays for operator inspection; it is a preview only and does not run `datasets`. `handoff_checklist` marks bounded datasets execution and final genome acceptance as separate approval/review steps. The `recommended_execution_validation_*` fields are empty until `prepare --write` has produced an isolated command manifest, then point to validation-only `download-smoke execute` without `--execute`; that handoff validates the pinned command manifest and does not download. The `recommended_inspection_*` fields are empty until `prepare --write` has produced an isolated bounded plan, then point to `download-smoke inspect` with that plan path and the deterministic sibling `inspection/` output directory. Controllers can pass `recommended_inspection_request` or `recommended_execution_validation_request` to `commands render`; the `recommended_*_command` fields remain compatibility argv lists for operator inspection. The default `inspection_quality_profile=fragmentation` carries `--quality-profile fragmentation` into that recommended handoff, enabling local blockers for fragmented FASTA signals and WGS/scaffold/contig header keywords during the later inspection; `--inspection-quality-profile none` leaves those profile gates disabled unless explicit gate flags are supplied. The `inspection_*` fields only carry optional local inspection quality gates into that recommended handoff; prepare does not inspect ZIPs, run downloads, contact providers, or decide final genome usability. `quality_tier` is the resolved effective tier. `--quality-tier recommended` is the default and resolves the readiness recommendation to `high`, `all`, or `none`; `--quality-tier high` selects only planned Complete Genome or Chromosome rows; Scaffold and Contig remain valid planned rows when `all` is selected explicitly or when `recommended` falls back to `all`. - `bounded_download_smoke_execution.tsv`: bounded execution audit rows for `download-smoke execute`. Fields are `record_id`, `assembly_accession`, `assembly_level`, `refseq_category`, `quality_tier`, `datasets_zip_path`, `command_json`, `command_valid`, `executed`, `returncode`, `status`, and `notes`. A row is executed only when `command_json` exactly matches the bounded datasets command contract and `--execute` is supplied; otherwise the command is validation-only. Execution success means a ZIP is ready for `download-smoke inspect`, not that a genome is final-use accepted. - `bounded_download_smoke_execution_summary.json`: read-only JSON summary for `download-smoke execute`. Stable fields include `schema_version`, `command`, `source_commands_manifest_path`, `limit`, `selected_row_count`, `command_valid_count`, `command_invalid_count`, `execute_requested`, `executed_command_count`, `datasets_zip_ready_for_inspection_count`, `status_counts`, `ready`, `blockers`, `execution_boundary`, `safe_for_unattended_download`, `recommended_inspection_request_target`, `recommended_inspection_request`, `recommended_inspection_request_blockers`, `recommended_inspection_next_command`, `recommended_inspection_command`, `downloads_triggered`, `providers_contacted`, `network_access`, `external_tools`, `manifest_mutated`, `strict_scientific_deliverable`, and legacy `recommended_next_command`. When execution audit outputs are written and the sibling `bounded_download_smoke_plan.tsv` is present, the structured `recommended_inspection_*` fields point to the matching local `download-smoke inspect` handoff. If the sibling plan is absent, the request fields stay empty and `recommended_inspection_request_blockers` includes `bounded_download_smoke_plan_missing`. Dry-run validation leaves `downloads_triggered=0`, `network_access=false`, and `external_tools=false`. Explicit execution may set those fields when commands run, but it still writes only isolated audit outputs and requires later ZIP/FASTA inspection before any bounded-smoke quality decision. - `--download-smoke-execution-dir ` is an explicit read-only report/package surface for these two files. Report-only mode may show `## Bounded Download Smoke Execution`; `package-results --include reports` and `--include all` may copy validated members under `download_smoke/` with `evidence_policy=download_smoke_execution_audit` and `strict_scientific_deliverable=false`. Missing input is omitted; partial or malformed input copies only valid members and emits a compact warning. Failed-handoff packages exclude the execution files. + If the `datasets` executable is missing, execution records `datasets_executable_missing`, leaves the row unexecuted, and keeps `downloads_triggered=0`, `network_access=false`, and `external_tools=false`. + `--download-smoke-execution-dir ` is an explicit read-only report/package surface for these two files. Report-only mode may show `## Bounded Download Smoke Execution`; `package-results --include reports` and `--include all` may copy validated members under `download_smoke/` with `evidence_policy=download_smoke_execution_audit` and `strict_scientific_deliverable=false`. Missing input is omitted; partial or malformed input copies only valid members and emits a compact warning. Failed-handoff packages exclude the execution files. - `bounded_download_smoke_inspection.tsv`: local inspection rows for a bounded smoke plan after a separately authorized download attempt. Fields are `record_id`, `assembly_accession`, `assembly_level`, `refseq_category`, `quality_tier`, `zip_path`, `zip_exists`, `zip_valid`, `unsafe_zip_member_count`, `genome_fasta_present`, `genome_fasta_member_count`, `genomic_named_fasta_member_count`, `genome_fasta_install_selection_status`, `fasta_record_count`, `fasta_total_bases`, `fasta_longest_record_bases`, `fasta_n50_bases`, `fasta_ambiguous_bases`, `fasta_header_wgs_keyword_count`, `fasta_header_scaffold_keyword_count`, `fasta_header_contig_keyword_count`, `empty_genome_fasta_count`, `multiple_genome_fasta_members_count`, `fasta_fragmentation_signal`, `fasta_quality_gate_blockers`, `installable_genome_fasta_ready`, `installable_genome_fasta_not_ready_reasons`, and `status`. `assembly_level`, `refseq_category`, and `quality_tier` are copied from annotated bounded smoke plans when available and otherwise use `unknown`. `unsafe_zip_member_count` is a count-only fail-closed signal for absolute, path-traversing, Windows-drive-prefixed, or symlink-like ZIP members; row status becomes `zip_unsafe_members` and blocker `unsafe_zip_member_paths` is reported before FASTA inspection. `genome_fasta_install_selection_status` is `selected`, `ambiguous`, or `not_evaluated` and mirrors the later reference-genome installer selection rule: one FASTA member is selectable, more than one member is selectable only when exactly one member is named `genomic.fna` or `*_genomic.fna`. When the source member is uniquely selectable, FASTA quality statistics describe that installable member; member-count fields still describe all FASTA members in the ZIP. `fasta_quality_gate_blockers` is empty unless explicit local quality gates are supplied, then contains semicolon-separated controlled blocker codes for that row. `installable_genome_fasta_ready` is a row-level local smoke signal derived from ZIP validity, unsafe-member checks, genome FASTA presence, install-selection status, empty-FASTA checks, and enabled quality-gate blockers; `installable_genome_fasta_not_ready_reasons` contains semicolon-separated controlled reason codes when the row is not ready. The FASTA statistics, N50 value, header keyword counts, unsafe-member count, empty-FASTA count, multiple-member count, install-selection status, fragmentation signal, row-level readiness, and row-level blocker/reason codes are counts-only audit visibility; raw sequence text and raw FASTA headers are not copied into the inspection files. Readers accept earlier inspection TSVs that lack `assembly_level`, `refseq_category`, `quality_tier`, `unsafe_zip_member_count`, `genome_fasta_install_selection_status`, `genomic_named_fasta_member_count`, `multiple_genome_fasta_members_count`, `empty_genome_fasta_count`, `fasta_quality_gate_blockers`, `installable_genome_fasta_ready`, or `installable_genome_fasta_not_ready_reasons`. - `bounded_download_smoke_inspection_summary.json`: read-only JSON summary for `download-smoke inspect`. Stable fields include `schema_version`, `command`, `source_download_plan_path`, `selected_row_count`, `zip_exists_count`, `zip_valid_count`, `unsafe_zip_member_count`, `genome_fasta_present_count`, `genome_fasta_member_count`, `genomic_named_fasta_member_count`, `genome_fasta_install_selection_status_counts`, `genome_fasta_install_selection_ambiguous_count`, `installable_genome_fasta_ready_count`, `installable_genome_fasta_not_ready_count`, `installable_genome_fasta_not_ready_reason_counts`, `installable_genome_fasta_ready_preview`, `installable_genome_fasta_ready_preview_truncated`, `installable_genome_fasta_not_ready_preview`, `installable_genome_fasta_not_ready_preview_truncated`, `assembly_metadata_high_quality_row_count`, `assembly_metadata_high_quality_installable_ready_count`, `assembly_metadata_high_quality_fasta_quality_blocked_count`, `assembly_metadata_high_quality_fasta_quality_blocker_counts`, `assembly_metadata_high_quality_fasta_quality_blocked_preview`, `assembly_metadata_high_quality_fasta_quality_blocked_preview_truncated`, `fasta_record_count`, `fasta_total_bases`, `fasta_longest_record_bases`, `fasta_max_n50_bases`, `fasta_ambiguous_bases`, `fasta_header_wgs_keyword_count`, `fasta_header_scaffold_keyword_count`, `fasta_header_contig_keyword_count`, `empty_genome_fasta_count`, `multiple_genome_fasta_members_count`, `fasta_fragmentation_signal_counts`, `installable_genome_fasta_fragmentation_signal_counts`, `installable_genome_fasta_header_fragment_keyword_row_count`, `min_fasta_n50_bases`, `max_fasta_record_count`, `max_fasta_ambiguous_bases`, `min_fasta_total_bases`, `min_fasta_longest_record_bases`, `quality_profile`, `block_fragmented_fasta`, `block_fasta_header_keywords`, `fasta_n50_below_minimum_count`, `fasta_record_count_above_maximum_count`, `fasta_ambiguous_bases_above_maximum_count`, `fasta_total_bases_below_minimum_count`, `fasta_longest_record_below_minimum_count`, `fragmented_fasta_signal_count`, `fasta_header_fragment_keyword_row_count`, `fasta_quality_gate_passed_row_count`, `fasta_quality_gate_blocked_row_count`, `fasta_quality_gate_blocker_counts`, `quality_gate_recommendation`, `quality_gate_recommendation_reasons`, `recommended_quality_gate_request_target`, `recommended_quality_gate_request`, `recommended_quality_gate_next_command`, `recommended_quality_gate_command`, `recommended_review_queue_request_target`, `recommended_review_queue_request`, `recommended_review_queue_next_command`, `recommended_execution_validation_request_target`, `recommended_execution_validation_request`, `recommended_execution_validation_next_command`, `recommended_execution_validation_command`, `bounded_smoke_next_action`, `bounded_smoke_next_action_reasons`, `status_counts`, `ready`, `blockers`, `execution_boundary`, `safe_for_unattended_download`, `downloads_triggered`, `providers_contacted`, `network_access`, `external_tools`, `manifest_mutated`, and `strict_scientific_deliverable`. `installable_genome_fasta_ready_count` counts rows with a uniquely selectable non-empty genome FASTA source and no enabled FASTA quality-gate blockers; `installable_genome_fasta_not_ready_count` counts the remaining selected rows, including missing, invalid, unsafe, empty, ambiguous, missing-FASTA, or quality-gate-blocked rows. `installable_genome_fasta_not_ready_reason_counts` maps those non-ready rows to controlled blocker-style reason counts such as missing ZIPs, invalid ZIPs, unsafe ZIP members, missing or empty genome FASTA, ambiguous install selection, and enabled FASTA quality-gate blockers. The ready and not-ready previews list at most five rows each using only record IDs, accessions, assembly metadata, row status, controlled not-ready reasons, controlled quality blocker codes, and fragmentation signal; they do not include ZIP paths, raw FASTA headers, or sequence text. `assembly_metadata_high_quality_*` fields separately summarize rows selected from high-quality assembly metadata, including how many remain locally FASTA-installable and how many are blocked by local FASTA quality gates; this prevents Complete Genome or Chromosome metadata from being mistaken for automatic local FASTA acceptance. The high-quality blocked preview lists at most five affected rows using only record IDs, accessions, assembly metadata, row status, and controlled blocker codes; it does not include raw FASTA headers or sequence text. The `recommended_quality_gate_*` fields are populated when the inspection observed fragmentation/header signals without the matching active gates; controllers can pass `recommended_quality_gate_request` to `commands render`, while `recommended_quality_gate_command` remains a compatibility argv list. The `recommended_review_queue_*` fields are populated only when `inspect --write` has written an isolated inspection directory with bounded rows, and point to the local `coverage-pipeline server-validation-result review-queue` handoff. The `recommended_execution_validation_*` fields are populated when ZIP outputs are missing and the sibling `bounded_download_smoke_commands.tsv` exists, and point to validation-only `download-smoke execute` without `--execute`; that handoff validates the pinned command manifest and does not download. `bounded_smoke_next_action` is a controlled AI/operator routing hint such as `rerun_with_fragmentation_quality_gates`, `review_high_quality_metadata_fasta_quality_blockers`, `review_or_rerun_bounded_download_outputs`, `prepare_bounded_download_smoke_plan`, or `review_bounded_smoke_outputs`; it does not authorize downloads, provider access, manifest mutation, installation, or strict deliverable changes. The default `quality_profile` is `fragmentation`, which enables fragmented FASTA and WGS/scaffold/contig header keyword blockers so obvious scaffold/contig/WGS-like outputs fail bounded-smoke readiness by default. `--quality-profile none` preserves compatibility/visibility-only behavior and may return `quality_gate_recommendation=rerun_with_fragmentation_quality_gates` when those signals are observed. `installable_genome_fasta_fragmentation_signal_counts` and `installable_genome_fasta_header_fragment_keyword_row_count` summarize fragmentation/header keyword signals only among rows that remain locally installable under the active quality gates. `fasta_n50_bases`, header keyword counts, and `fasta_fragmentation_signal` are descriptive fields (`single_record`, `multi_record_single_dominant`, `multi_record_fragmented`, or `not_evaluated`). Unsafe ZIP member paths are blocked by default as `unsafe_zip_member_paths`, with row status `zip_unsafe_members`; no FASTA member content is inspected for that row. A genome FASTA member with zero records or zero bases is blocked by default as `empty_genome_fasta_outputs`, with row status `genome_fasta_empty`. A bounded row with more than one genome FASTA member remains visible through `multiple_genome_fasta_members_count`; it is blocked by default only when the installer selection rule cannot uniquely choose the source member, reported as `genome_fasta_install_selection_ambiguous` with row status `genome_fasta_multiple_members`. Optional flags `--min-fasta-n50-bases`, `--max-fasta-record-count`, `--max-fasta-ambiguous-bases`, `--min-fasta-total-bases`, `--min-fasta-longest-record-bases`, `--block-fragmented-fasta`, and `--block-fasta-header-keywords` can add or repeat local bounded smoke blockers for obvious short, ambiguous-base-heavy, scaffold/contig, or WGS-style outputs, and the quality-gate passed/blocked row counts plus blocker-count map summarize those controlled row blockers. `--quality-profile fragmentation` is shorthand for enabling the fragmented FASTA and WGS/scaffold/contig header keyword blockers. These readiness blockers are local smoke acceptance criteria only; they do not change strict status, completion, or final genome usability. The inspection only checks local ZIP existence, ZIP validity, unsafe member paths, whether genome FASTA members are present, install-selection ambiguity, and bounded FASTA count statistics; it does not run `datasets`, extract ZIPs, write raw sequences, access the network, contact providers, mutate manifests, or create strict deliverables. `--download-smoke-inspection-dir ` is an explicit read-only report/package surface for these two files. Report-only mode may show `## Bounded Download Smoke Inspection`; `package-results --include reports` and `--include all` may copy validated members under `download_smoke/` with `evidence_policy=download_smoke_inspection_audit` and `strict_scientific_deliverable=false`. Missing input is omitted; partial or malformed input copies only valid members and emits a compact warning. Failed-handoff packages exclude the inspection files. diff --git a/tests/test_cli_download_smoke.py b/tests/test_cli_download_smoke.py index d8ab02d..6f76c67 100644 --- a/tests/test_cli_download_smoke.py +++ b/tests/test_cli_download_smoke.py @@ -655,6 +655,35 @@ def run(self, command, cwd=None): assert summary["strict_scientific_deliverable"] is False +def test_download_smoke_execute_reports_missing_datasets_executable(tmp_path): + zip_path = tmp_path / "rec-1.zip" + commands = tmp_path / "bounded_download_smoke_commands.tsv" + _write_commands_manifest(commands, [_command_row(zip_path=zip_path)]) + + class MissingDatasetsRunner: + def run(self, command, cwd=None): + raise FileNotFoundError("datasets") + + result = execute_bounded_download_smoke_commands( + commands, + limit=1, + execute=True, + runner=MissingDatasetsRunner(), + ) + + summary = result["summary"] + assert summary["ready"] is False + assert summary["blockers"] == ["datasets_executable_missing"] + assert summary["downloads_triggered"] == 0 + assert summary["network_access"] is False + assert summary["external_tools"] is False + assert summary["status_counts"] == {"datasets_executable_missing": 1} + assert result["rows"][0]["status"] == "datasets_executable_missing" + assert result["rows"][0]["executed"] is False + assert result["rows"][0]["returncode"] == "" + assert "datasets executable was not found" in result["rows"][0]["notes"] + + def test_download_smoke_inspect_accepts_annotated_prepare_plan(capsys, tmp_path): plan = tmp_path / "cache" / "ncbi" / "download_plan.tsv" outdir = tmp_path / "smoke-input" diff --git a/typetreeflow/download_smoke_cli.py b/typetreeflow/download_smoke_cli.py index 8208a92..22aac63 100644 --- a/typetreeflow/download_smoke_cli.py +++ b/typetreeflow/download_smoke_cli.py @@ -1420,7 +1420,14 @@ def _execution_row_for_manifest_row( "notes": "dry-run only; datasets was not executed", } - command_result = runner.run(command) + try: + command_result = runner.run(command) + except FileNotFoundError: + return { + **base, + "status": "datasets_executable_missing", + "notes": "datasets executable was not found; install or expose NCBI Datasets before executing bounded downloads", + } zip_path = Path(row.get("datasets_zip_path", "").strip()) executed_base = { **base,