diff --git a/CHANGELOG.md b/CHANGELOG.md index d165f3f..d7938a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Initial release of nf-core/datasync, created with the [nf-core](https://nf-co.re ### `Added` +- [[#74](https://github.com/nf-core/datasync/pull/74)] - Implement warning and `--download` parameter when working with remote directories and SHA checksums ([@delfiterradas](https://github.com/delfiterradas), review by [@atrigila](https://github.com/atrigila), [@apeltzer](https://github.com/apeltzer) and [@antoniasaracco](https://github.com/antoniasaracco)). - [[#67](https://github.com/nf-core/datasync/pull/67)] - Copy only files which are successfully validated ([@delfiterradas](https://github.com/delfiterradas), review by [@atrigila](https://github.com/atrigila) and [@antoniasaracco](https://github.com/antoniasaracco)). - [[#57](https://github.com/nf-core/datasync/pull/57)] - Add nf-tests with edge cases([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)). - [[#55](https://github.com/nf-core/datasync/pull/55)] - Add subdirectories with sample id to results. Improvements to MultiQC report (width of columns, sections and sub-section headers)colors to multiqc results and display first errors in tables ([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)). diff --git a/assets/schema_input.json b/assets/schema_input.json index 5985ddb..0eff487 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -26,13 +26,13 @@ "checksum_md5": { "type": "string", "format": "file-path", - "pattern": "^\\S+\\.(csv|tsv)$", + "pattern": "^\\S+\\.(tsv|txt|md5)$", "errorMessage": "Checksum_md5 cannot contain spaces" }, "checksum_sha": { "type": "string", "format": "file-path", - "pattern": "^\\S+\\.(csv|tsv)$", + "pattern": "^\\S+\\.(tsv|txt|sha256)$", "errorMessage": "Checksum_sha cannot contain spaces" } }, diff --git a/conf/modules.config b/conf/modules.config index 794c4d9..701c4a7 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -18,6 +18,25 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] + withName: 'RCLONE_CHECKSUM' { + tag = { "${meta.id}_${hash}" } + ext.args = { + def base_args = [ + '--no-check-certificate', + "--one-way" + ] + if (params.download && meta.check_format == 'sha') { + base_args.add("--download") + } + base_args.join(' ') + } + publishDir = [ + path: { "${params.outdir}/rclone/checksum/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + withName: 'RCLONE_COPY' { ext.args = { def base_args = [ @@ -58,22 +77,6 @@ process { ] } - withName: 'RCLONE_CHECKSUM' { - tag = { "${meta.id}_${hash}" } - ext.args = { - def base_args = [ - '--no-check-certificate', - "--one-way" - ] - base_args.join(' ') - } - publishDir = [ - path: { "${params.outdir}/rclone/checksum/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - withName: 'MULTIQC' { ext.args = { def args = ['--custom-css-file */multiqc_custom.css'] diff --git a/conf/test_full.config b/conf/test_full.config index af92cc7..58cf51e 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -20,6 +20,7 @@ params { input = params.pipelines_testdata_base_path + "test-data/samplesheet_full.csv" //Rclone options - rclone_config = params.pipelines_testdata_base_path + "test-data/rclone.conf" + rclone_config = params.pipelines_testdata_base_path + "test-data/rclone.conf" rclone_dry_run = true + download = true } diff --git a/docs/usage.md b/docs/usage.md index 2e7c6fb..418adef 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -32,7 +32,7 @@ At least one checksum manifest is required on every row. If both are supplied, b For a directory input, the source root is the directory named in the samplesheet. For example, if the samplesheet `input` is `/data/run_001` and one file in that directory is `/data/run_001/reads/sample_R1.fastq.gz`, the checksum manifest path must be `reads/sample_R1.fastq.gz`. Do not write `/data/run_001/reads/sample_R1.fastq.gz` in the manifest. For a single-file input, use the input file name as the manifest path. -Checksum manifests may use a `.tsv` or `.csv` filename extension, but their contents are not tab-separated or comma-separated tables and must not include a header. Each record is plain text with the hash and path separated by exactly two spaces. The required fields are: +Checksum manifests may use a `.tsv`, `.txt`, `.md5` or `.sha256` filename extension, but their contents are not tab-separated or comma-separated tables and must not include a header. Each record is plain text with the hash and path separated by exactly **two spaces**. The required fields are: | Field | Required | Description | | ----- | -------- | ----------------------------------------------------------------------------------------------- | @@ -116,6 +116,30 @@ The corresponding input values could be `source_s3:incoming/run_001` and `instit The pipeline also accepts an `s3://bucket/path` source or destination. In that form, ensure credentials and provider settings are available to both Nextflow and `rclone` in the execution environment. A named remote such as `source_s3:bucket/path` makes the selected configuration section explicit and is preferable when a config file contains multiple S3 providers. +## SHA256 checksum verification for remote inputs + +When validating files stored on cloud storage providers (e.g. S3, azure, google cloud), only MD5 hashes are typically available through the storage provider (see [Overview of cloud storage systems](https://rclone.org/overview/)). SHA256 checksums are not exposed by the remote API, so they cannot be verified directly. + +In order to validate SHA256 checksums for remote inputs, `rclone checksum` must download each file and compute its SHA256 checksum locally. If a `checksum_sha` file is provided for remote inputs, the `--download` parameter must be enabled. Otherwise, SHA checksum verification cannot be performed and the pipeline will terminate with an error. + +> [!NOTE] +> Providing `--download` does not force all files to be downloaded in all modules. It is only used when verifying SHA256 checksum files for remote source directories in `RCLONE_CHECKSUM`. + +> [!WARNING] +> Enabling `--download` may incur substantial cloud data transfer and egress costs, particularly when validating large datasets. Make sure this is the intended behaviour before running the pipeline. + +## Copying only successfully validated files + +By default, the pipeline will copy all files in the source directory, regardless of whether they were successfully validated against the provided checksum or not. + +However, it is possible to restrict copying of files to only the ones that successfully pass checksum validation by enabling the `--copy_matching_only` parameter: + +- If only an MD5 checksum file is provided, only files that successfully match their MD5 checksum will be copied. +- If only a SHA256 checksum file is provided, only files that successfully match their SHA256 checksum will be copied. +- If both MD5 and SHA256 checksum files are provided, the pipeline will copy only files that successfully pass **both** checksum validations. + +Files that fail checksum validation, are missing, or cannot be verified are excluded from the copy operation when this parameter is enabled. + ## Destination layout The pipeline preserves the source basename: diff --git a/nextflow.config b/nextflow.config index 2423a8a..75c2071 100644 --- a/nextflow.config +++ b/nextflow.config @@ -24,6 +24,7 @@ params { rclone_config = null rclone_dry_run = false copy_matching_only = false + download = false // Boilerplate options outdir = null diff --git a/nextflow_schema.json b/nextflow_schema.json index a8dc091..738538e 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -60,6 +60,12 @@ "description": "Only copy files that matched their provided input checksums.", "fa_icon": "fas fa-eye", "help_text": "If set, the pipeline will only copy files that were correctly validated and will skip any file that did not match their input checksum." + }, + "download": { + "type": "boolean", + "description": "Download remote files for sha256 checksum verification in `RCLONE_CHECKSUM`.", + "fa_icon": "fas fa-eye", + "help_text": "If set, `RCLONE_CHECKSUM` will download remote files for any sample for which a SHA256 file was provided." } } }, diff --git a/tests/edge.nf.test b/tests/edge.nf.test index 5fce38e..f5c5188 100644 --- a/tests/edge.nf.test +++ b/tests/edge.nf.test @@ -12,6 +12,7 @@ nextflow_pipeline { params { input = "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/datasync/test-data/samplesheet_edge.csv" outdir = "$outputDir" + download = true } } @@ -41,6 +42,7 @@ nextflow_pipeline { input = "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/datasync/test-data/samplesheet_edge.csv" outdir = "$outputDir" copy_matching_only = true + download = true } } @@ -62,4 +64,18 @@ nextflow_pipeline { ) } } + + test("-profile test remote source with SHA256 checksums - fail") { + + when { + params { + input = "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/datasync/test-data/samplesheet_edge.csv" + outdir = "$outputDir" + } + } + + then { + assert workflow.failed + } + } } diff --git a/tests/edge.nf.test.snap b/tests/edge.nf.test.snap index ea3559b..52620b3 100644 --- a/tests/edge.nf.test.snap +++ b/tests/edge.nf.test.snap @@ -25,6 +25,7 @@ "multiqc/multiqc_data/multiqc_data.json", "multiqc/multiqc_data/multiqc_rclone_check.txt", "multiqc/multiqc_data/multiqc_rclone_checksum_md5.txt", + "multiqc/multiqc_data/multiqc_rclone_checksum_sha.txt", "multiqc/multiqc_data/multiqc_samplesheet.txt", "multiqc/multiqc_data/multiqc_software_versions.txt", "multiqc/multiqc_data/multiqc_sources.txt", @@ -51,6 +52,8 @@ "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.match.txt", "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.missing_on_dst.txt", "rclone/checksum/Illumina_annotation_sha_only", + "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.combined.txt", + "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.differ.txt", "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.exit_code.txt", "rclone/copy", "rclone/copy/Illumina_annotation_incorrect-rclone-copy.log", @@ -60,10 +63,11 @@ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", "multiqc_rclone_check.txt:md5,4b2a6990836593f69e8e81a82c3daf42", "multiqc_rclone_checksum_md5.txt:md5,cf92203df21f04ddf8315fc606812e0b", + "multiqc_rclone_checksum_sha.txt:md5,8d38305a4ad03c7ea18aa9827d34a396", "multiqc_samplesheet.txt:md5,bbfc817ff43c49cde14a2b33f46b5ae5" ] ], - "timestamp": "2026-07-28T21:45:08.268780076", + "timestamp": "2026-07-30T20:44:41.242808343", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.1" @@ -95,6 +99,7 @@ "multiqc/multiqc_data/multiqc_data.json", "multiqc/multiqc_data/multiqc_rclone_check.txt", "multiqc/multiqc_data/multiqc_rclone_checksum_md5.txt", + "multiqc/multiqc_data/multiqc_rclone_checksum_sha.txt", "multiqc/multiqc_data/multiqc_samplesheet.txt", "multiqc/multiqc_data/multiqc_software_versions.txt", "multiqc/multiqc_data/multiqc_sources.txt", @@ -124,6 +129,8 @@ "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.match.txt", "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.missing_on_dst.txt", "rclone/checksum/Illumina_annotation_sha_only", + "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.combined.txt", + "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.differ.txt", "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.exit_code.txt", "rclone/copy", "rclone/copy/Illumina_annotation_incorrect-rclone-copy.log", @@ -134,10 +141,11 @@ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", "multiqc_rclone_check.txt:md5,c8f13c10aa4f0e4b356c5c08a9ae7ee9", "multiqc_rclone_checksum_md5.txt:md5,cf92203df21f04ddf8315fc606812e0b", + "multiqc_rclone_checksum_sha.txt:md5,8d38305a4ad03c7ea18aa9827d34a396", "multiqc_samplesheet.txt:md5,bbfc817ff43c49cde14a2b33f46b5ae5" ] ], - "timestamp": "2026-07-22T22:01:06.286788462", + "timestamp": "2026-07-30T20:44:11.000298059", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.1" diff --git a/tests/main_full.nf.test.snap b/tests/main_full.nf.test.snap index 7bc1eeb..96249fd 100644 --- a/tests/main_full.nf.test.snap +++ b/tests/main_full.nf.test.snap @@ -25,6 +25,7 @@ "multiqc/multiqc_data/multiqc_data.json", "multiqc/multiqc_data/multiqc_rclone_check.txt", "multiqc/multiqc_data/multiqc_rclone_checksum_md5.txt", + "multiqc/multiqc_data/multiqc_rclone_checksum_sha.txt", "multiqc/multiqc_data/multiqc_samplesheet.txt", "multiqc/multiqc_data/multiqc_software_versions.txt", "multiqc/multiqc_data/multiqc_sources.txt", @@ -39,7 +40,6 @@ "rclone/checksum", "rclone/checksum/demultiplex", "rclone/checksum/demultiplex/demultiplex.combined.txt", - "rclone/checksum/demultiplex/demultiplex.exit_code.txt", "rclone/checksum/demultiplex/demultiplex.match.txt", "rclone/copy", "rclone/copy/demultiplex-rclone-copy.log" @@ -48,10 +48,11 @@ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", "multiqc_rclone_check.txt:md5,03a73f087a286ec6580aada2a5db2e7c", "multiqc_rclone_checksum_md5.txt:md5,d7c7dd71a9ff75955dd0e508d1ad9968", + "multiqc_rclone_checksum_sha.txt:md5,03a73f087a286ec6580aada2a5db2e7c", "multiqc_samplesheet.txt:md5,00788a52d1a014c12ac4ed554b0b44ca" ] ], - "timestamp": "2026-07-22T22:01:43.699965019", + "timestamp": "2026-07-30T20:46:20.306063412", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.1" diff --git a/workflows/datasync.nf b/workflows/datasync.nf index a601e0d..966fc23 100644 --- a/workflows/datasync.nf +++ b/workflows/datasync.nf @@ -38,6 +38,15 @@ workflow DATASYNC { ch_samplesheet = ch_samplesheet.multiMap { meta, input_path, output_path, md5, sha -> + if (sha && input_path.contains('://')) { + if (params.download) { + log.warn("The `--download` parameter is enabled. `RCLONE_CHECKSUM` will download remote files. Make sure this is what you want, as it may incur substantial cloud costs !") + } else { + log.error("A SHA checksum file was provided, but `--download` is not enabled. `RCLONE_CHECKSUM` cannot verify SHA256 checksums for remote files without downloading them. Enable `--download` to proceed.") + System.exit(1) + } + } + def source = file(input_path) def source_uri = source.toUriString()