From 4a705aa420364fc30ef043d62096b49d234f96a0 Mon Sep 17 00:00:00 2001 From: delfiterradas Date: Thu, 30 Jul 2026 20:38:47 +0200 Subject: [PATCH 01/15] Add download parameter when using shasums --- conf/modules.config | 35 +++++++++++++++++++---------------- nextflow.config | 1 + nextflow_schema.json | 6 ++++++ 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 794c4d9..701c4a7 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -18,6 +18,25 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] + withName: 'RCLONE_CHECKSUM' { + tag = { "${meta.id}_${hash}" } + ext.args = { + def base_args = [ + '--no-check-certificate', + "--one-way" + ] + if (params.download && meta.check_format == 'sha') { + base_args.add("--download") + } + base_args.join(' ') + } + publishDir = [ + path: { "${params.outdir}/rclone/checksum/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + withName: 'RCLONE_COPY' { ext.args = { def base_args = [ @@ -58,22 +77,6 @@ process { ] } - withName: 'RCLONE_CHECKSUM' { - tag = { "${meta.id}_${hash}" } - ext.args = { - def base_args = [ - '--no-check-certificate', - "--one-way" - ] - base_args.join(' ') - } - publishDir = [ - path: { "${params.outdir}/rclone/checksum/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - withName: 'MULTIQC' { ext.args = { def args = ['--custom-css-file */multiqc_custom.css'] diff --git a/nextflow.config b/nextflow.config index 2423a8a..75c2071 100644 --- a/nextflow.config +++ b/nextflow.config @@ -24,6 +24,7 @@ params { rclone_config = null rclone_dry_run = false copy_matching_only = false + download = false // Boilerplate options outdir = null diff --git a/nextflow_schema.json b/nextflow_schema.json index a8dc091..7e8a2a4 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -60,6 +60,12 @@ "description": "Only copy files that matched their provided input checksums.", "fa_icon": "fas fa-eye", "help_text": "If set, the pipeline will only copy files that were correctly validated and will skip any file that did not match their input checksum." + }, + "download": { + "type": "boolean", + "description": "Download remote files for sha256 checksum verification.", + "fa_icon": "fas fa-eye", + "help_text": "If set, rclone checksum will download remote files for any sample for which a shasum file was provided." } } }, From fa6dd326683af1195eea065148159738ad55f18d Mon Sep 17 00:00:00 2001 From: delfiterradas Date: Thu, 30 Jul 2026 20:39:49 +0200 Subject: [PATCH 02/15] Add validation when using shasums --- workflows/datasync.nf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/workflows/datasync.nf b/workflows/datasync.nf index a601e0d..2262557 100644 --- a/workflows/datasync.nf +++ b/workflows/datasync.nf @@ -38,6 +38,15 @@ workflow DATASYNC { ch_samplesheet = ch_samplesheet.multiMap { meta, input_path, output_path, md5, sha -> + if (sha && input_path.contains('://')) { + if (params.download) { + log.warn("The `--download` parameter is enabled. RCLONE_CHECKSUM will download remote files. Make sure this is what you want, as it may incur substantial cloud costs !") + } else { + log.error("A SHA checksum file was provided, but `--download` is not enabled. Rclone cannot verify SHA checksums for remote files without downloading them. Enable `--download` to proceed.") + System.exit(1) + } + } + def source = file(input_path) def source_uri = source.toUriString() From 0ed68fc4bdbb726e98578fc60a4b42989799f764 Mon Sep 17 00:00:00 2001 From: delfiterradas Date: Thu, 30 Jul 2026 20:42:08 +0200 Subject: [PATCH 03/15] Checksums cannot be csvs and accept additional formats --- assets/schema_input.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/schema_input.json b/assets/schema_input.json index 5985ddb..0eff487 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -26,13 +26,13 @@ "checksum_md5": { "type": "string", "format": "file-path", - "pattern": "^\\S+\\.(csv|tsv)$", + "pattern": "^\\S+\\.(tsv|txt|md5)$", "errorMessage": "Checksum_md5 cannot contain spaces" }, "checksum_sha": { "type": "string", "format": "file-path", - "pattern": "^\\S+\\.(csv|tsv)$", + "pattern": "^\\S+\\.(tsv|txt|sha256)$", "errorMessage": "Checksum_sha cannot contain spaces" } }, From 447fc7beac921bff979b38eaf832ef2a62edf94f Mon Sep 17 00:00:00 2001 From: delfiterradas Date: Thu, 30 Jul 2026 20:46:38 +0200 Subject: [PATCH 04/15] Update tests and snapshots --- conf/test_full.config | 3 ++- tests/edge.nf.test | 2 ++ tests/edge.nf.test.snap | 12 ++++++++++-- tests/main_full.nf.test.snap | 5 +++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/conf/test_full.config b/conf/test_full.config index af92cc7..58cf51e 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -20,6 +20,7 @@ params { input = params.pipelines_testdata_base_path + "test-data/samplesheet_full.csv" //Rclone options - rclone_config = params.pipelines_testdata_base_path + "test-data/rclone.conf" + rclone_config = params.pipelines_testdata_base_path + "test-data/rclone.conf" rclone_dry_run = true + download = true } diff --git a/tests/edge.nf.test b/tests/edge.nf.test index 5fce38e..4e626ae 100644 --- a/tests/edge.nf.test +++ b/tests/edge.nf.test @@ -12,6 +12,7 @@ nextflow_pipeline { params { input = "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/datasync/test-data/samplesheet_edge.csv" outdir = "$outputDir" + download = true } } @@ -41,6 +42,7 @@ nextflow_pipeline { input = "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/datasync/test-data/samplesheet_edge.csv" outdir = "$outputDir" copy_matching_only = true + download = true } } diff --git a/tests/edge.nf.test.snap b/tests/edge.nf.test.snap index ea3559b..52620b3 100644 --- a/tests/edge.nf.test.snap +++ b/tests/edge.nf.test.snap @@ -25,6 +25,7 @@ "multiqc/multiqc_data/multiqc_data.json", "multiqc/multiqc_data/multiqc_rclone_check.txt", "multiqc/multiqc_data/multiqc_rclone_checksum_md5.txt", + "multiqc/multiqc_data/multiqc_rclone_checksum_sha.txt", "multiqc/multiqc_data/multiqc_samplesheet.txt", "multiqc/multiqc_data/multiqc_software_versions.txt", "multiqc/multiqc_data/multiqc_sources.txt", @@ -51,6 +52,8 @@ "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.match.txt", "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.missing_on_dst.txt", "rclone/checksum/Illumina_annotation_sha_only", + "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.combined.txt", + "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.differ.txt", "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.exit_code.txt", "rclone/copy", "rclone/copy/Illumina_annotation_incorrect-rclone-copy.log", @@ -60,10 +63,11 @@ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", "multiqc_rclone_check.txt:md5,4b2a6990836593f69e8e81a82c3daf42", "multiqc_rclone_checksum_md5.txt:md5,cf92203df21f04ddf8315fc606812e0b", + "multiqc_rclone_checksum_sha.txt:md5,8d38305a4ad03c7ea18aa9827d34a396", "multiqc_samplesheet.txt:md5,bbfc817ff43c49cde14a2b33f46b5ae5" ] ], - "timestamp": "2026-07-28T21:45:08.268780076", + "timestamp": "2026-07-30T20:44:41.242808343", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.1" @@ -95,6 +99,7 @@ "multiqc/multiqc_data/multiqc_data.json", "multiqc/multiqc_data/multiqc_rclone_check.txt", "multiqc/multiqc_data/multiqc_rclone_checksum_md5.txt", + "multiqc/multiqc_data/multiqc_rclone_checksum_sha.txt", "multiqc/multiqc_data/multiqc_samplesheet.txt", "multiqc/multiqc_data/multiqc_software_versions.txt", "multiqc/multiqc_data/multiqc_sources.txt", @@ -124,6 +129,8 @@ "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.match.txt", "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.missing_on_dst.txt", "rclone/checksum/Illumina_annotation_sha_only", + "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.combined.txt", + "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.differ.txt", "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.exit_code.txt", "rclone/copy", "rclone/copy/Illumina_annotation_incorrect-rclone-copy.log", @@ -134,10 +141,11 @@ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", "multiqc_rclone_check.txt:md5,c8f13c10aa4f0e4b356c5c08a9ae7ee9", "multiqc_rclone_checksum_md5.txt:md5,cf92203df21f04ddf8315fc606812e0b", + "multiqc_rclone_checksum_sha.txt:md5,8d38305a4ad03c7ea18aa9827d34a396", "multiqc_samplesheet.txt:md5,bbfc817ff43c49cde14a2b33f46b5ae5" ] ], - "timestamp": "2026-07-22T22:01:06.286788462", + "timestamp": "2026-07-30T20:44:11.000298059", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.1" diff --git a/tests/main_full.nf.test.snap b/tests/main_full.nf.test.snap index 7bc1eeb..96249fd 100644 --- a/tests/main_full.nf.test.snap +++ b/tests/main_full.nf.test.snap @@ -25,6 +25,7 @@ "multiqc/multiqc_data/multiqc_data.json", "multiqc/multiqc_data/multiqc_rclone_check.txt", "multiqc/multiqc_data/multiqc_rclone_checksum_md5.txt", + "multiqc/multiqc_data/multiqc_rclone_checksum_sha.txt", "multiqc/multiqc_data/multiqc_samplesheet.txt", "multiqc/multiqc_data/multiqc_software_versions.txt", "multiqc/multiqc_data/multiqc_sources.txt", @@ -39,7 +40,6 @@ "rclone/checksum", "rclone/checksum/demultiplex", "rclone/checksum/demultiplex/demultiplex.combined.txt", - "rclone/checksum/demultiplex/demultiplex.exit_code.txt", "rclone/checksum/demultiplex/demultiplex.match.txt", "rclone/copy", "rclone/copy/demultiplex-rclone-copy.log" @@ -48,10 +48,11 @@ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", "multiqc_rclone_check.txt:md5,03a73f087a286ec6580aada2a5db2e7c", "multiqc_rclone_checksum_md5.txt:md5,d7c7dd71a9ff75955dd0e508d1ad9968", + "multiqc_rclone_checksum_sha.txt:md5,03a73f087a286ec6580aada2a5db2e7c", "multiqc_samplesheet.txt:md5,00788a52d1a014c12ac4ed554b0b44ca" ] ], - "timestamp": "2026-07-22T22:01:43.699965019", + "timestamp": "2026-07-30T20:46:20.306063412", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.1" From 726112672d6466f0fe95837bb6f4d0b100edcfee Mon Sep 17 00:00:00 2001 From: delfiterradas Date: Thu, 30 Jul 2026 21:45:08 +0200 Subject: [PATCH 05/15] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 852306a..de791fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Initial release of nf-core/datasync, created with the [nf-core](https://nf-co.re ### `Added` +- [[#74](https://github.com/nf-core/datasync/pull/74)] - Implement warning and `--download` parameter when working with remote directories and SHA checksums ([@delfiterradas](https://github.com/delfiterradas), review by [@atrigila](https://github.com/atrigila) and [@antoniasaracco](https://github.com/antoniasaracco)). - [[#67](https://github.com/nf-core/datasync/pull/67)] - Copy only files which are successfully validated ([@delfiterradas](https://github.com/delfiterradas), review by [@atrigila](https://github.com/atrigila) and [@antoniasaracco](https://github.com/antoniasaracco)). - [[#57](https://github.com/nf-core/datasync/pull/57)] - Add nf-tests with edge cases([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)). - [[#55](https://github.com/nf-core/datasync/pull/55)] - Add subdirectories with sample id to results. Improvements to MultiQC report (width of columns, sections and sub-section headers)colors to multiqc results and display first errors in tables ([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)). From e3e47d2e753578b4e67bc45e19e6e8bf91ce8c9c Mon Sep 17 00:00:00 2001 From: delfiterradas Date: Fri, 31 Jul 2026 18:20:34 +0200 Subject: [PATCH 06/15] Update usage documentation --- docs/usage.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 2e7c6fb..21f97e7 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -32,7 +32,7 @@ At least one checksum manifest is required on every row. If both are supplied, b For a directory input, the source root is the directory named in the samplesheet. For example, if the samplesheet `input` is `/data/run_001` and one file in that directory is `/data/run_001/reads/sample_R1.fastq.gz`, the checksum manifest path must be `reads/sample_R1.fastq.gz`. Do not write `/data/run_001/reads/sample_R1.fastq.gz` in the manifest. For a single-file input, use the input file name as the manifest path. -Checksum manifests may use a `.tsv` or `.csv` filename extension, but their contents are not tab-separated or comma-separated tables and must not include a header. Each record is plain text with the hash and path separated by exactly two spaces. The required fields are: +Checksum manifests may use a `.tsv`, `.txt`, `.md5` or `.sha256` filename extension, but their contents are not tab-separated or comma-separated tables and must not include a header. Each record is plain text with the hash and path separated by exactly **two spaces**. The required fields are: | Field | Required | Description | | ----- | -------- | ----------------------------------------------------------------------------------------------- | @@ -116,6 +116,30 @@ The corresponding input values could be `source_s3:incoming/run_001` and `instit The pipeline also accepts an `s3://bucket/path` source or destination. In that form, ensure credentials and provider settings are available to both Nextflow and `rclone` in the execution environment. A named remote such as `source_s3:bucket/path` makes the selected configuration section explicit and is preferable when a config file contains multiple S3 providers. +## SHA checksum verification for remote inputs + +When validating files stored on cloud storage providers (e.g. S3, azure, google cloud), only MD5 hashes are typically available through the storage provider. SHA checksums are not exposed by the remote API, so they cannot be verified directly. + +In order to validate SHA checksums for remote inputs, `rclone checksum` must download each file and compute its SHA checksum locally. If a `checksum_sha` file is provided for remote inputs, the `--download` parameter must be enabled. Otherwise, SHA checksum verification cannot be performed and the pipeline will terminate with an error. + +> [!NOTE] +> Providing `--download` does not force all files to be downloaded. It is only used when verifying SHA checksum files for remote source directories. + +> [!WARNING] +> Enabling `--download` may incur substantial cloud data transfer and egress costs, particularly when validating large datasets. Make sure this is the intended behaviour before running the pipeline. + +## Copying only successfully validated files + +By default, the pipeline will copy all files in the source directory, regardless of whether they were successfully validated against the provided checksum or not. + +However, it is possible to restrict copying of files to only the ones that successfully pass checksum validation by enabling the `--copy_matching_only` parameter: + +- If only an MD5 checksum file is provided, only files that successfully match their MD5 checksum will be copied. +- If only a SHA checksum file is provided, only files that successfully match their SHA checksum will be copied. +- If both MD5 and SHA checksum files are provided, the pipeline will copy only files that successfully pass **both** checksum validations. + +Files that fail checksum validation, are missing, or cannot be verified are excluded from the copy operation when this parameter is enabled. + ## Destination layout The pipeline preserves the source basename: From 97e6f57159878d1a57c6b3a6fab34d6e97a684a0 Mon Sep 17 00:00:00 2001 From: Delfina Terradas <155591053+delfiterradas@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:12:58 -0300 Subject: [PATCH 07/15] Update checksum verification documentation to SHA256 --- docs/usage.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 21f97e7..a28bb96 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -116,11 +116,11 @@ The corresponding input values could be `source_s3:incoming/run_001` and `instit The pipeline also accepts an `s3://bucket/path` source or destination. In that form, ensure credentials and provider settings are available to both Nextflow and `rclone` in the execution environment. A named remote such as `source_s3:bucket/path` makes the selected configuration section explicit and is preferable when a config file contains multiple S3 providers. -## SHA checksum verification for remote inputs +## SHA256 checksum verification for remote inputs -When validating files stored on cloud storage providers (e.g. S3, azure, google cloud), only MD5 hashes are typically available through the storage provider. SHA checksums are not exposed by the remote API, so they cannot be verified directly. +When validating files stored on cloud storage providers (e.g. S3, azure, google cloud), only MD5 hashes are typically available through the storage provider (see [Overview of cloud storage systems](https://rclone.org/overview/)). SHA256 checksums are not exposed by the remote API, so they cannot be verified directly. -In order to validate SHA checksums for remote inputs, `rclone checksum` must download each file and compute its SHA checksum locally. If a `checksum_sha` file is provided for remote inputs, the `--download` parameter must be enabled. Otherwise, SHA checksum verification cannot be performed and the pipeline will terminate with an error. +In order to validate SHA256 checksums for remote inputs, `rclone checksum` must download each file and compute its SHA256 checksum locally. If a `checksum_sha` file is provided for remote inputs, the `--download` parameter must be enabled. Otherwise, SHA checksum verification cannot be performed and the pipeline will terminate with an error. > [!NOTE] > Providing `--download` does not force all files to be downloaded. It is only used when verifying SHA checksum files for remote source directories. @@ -135,8 +135,8 @@ By default, the pipeline will copy all files in the source directory, regardless However, it is possible to restrict copying of files to only the ones that successfully pass checksum validation by enabling the `--copy_matching_only` parameter: - If only an MD5 checksum file is provided, only files that successfully match their MD5 checksum will be copied. -- If only a SHA checksum file is provided, only files that successfully match their SHA checksum will be copied. -- If both MD5 and SHA checksum files are provided, the pipeline will copy only files that successfully pass **both** checksum validations. +- If only a SHA256 checksum file is provided, only files that successfully match their SHA256 checksum will be copied. +- If both MD5 and SHA256 checksum files are provided, the pipeline will copy only files that successfully pass **both** checksum validations. Files that fail checksum validation, are missing, or cannot be verified are excluded from the copy operation when this parameter is enabled. From 3621357c836dbefa32370bccba75ad5b0e776965 Mon Sep 17 00:00:00 2001 From: Delfina Terradas <155591053+delfiterradas@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:26:33 -0300 Subject: [PATCH 08/15] Update docs/usage.md Co-authored-by: Anabella Trigila <18577080+atrigila@users.noreply.github.com> --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index a28bb96..418adef 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -123,7 +123,7 @@ When validating files stored on cloud storage providers (e.g. S3, azure, google In order to validate SHA256 checksums for remote inputs, `rclone checksum` must download each file and compute its SHA256 checksum locally. If a `checksum_sha` file is provided for remote inputs, the `--download` parameter must be enabled. Otherwise, SHA checksum verification cannot be performed and the pipeline will terminate with an error. > [!NOTE] -> Providing `--download` does not force all files to be downloaded. It is only used when verifying SHA checksum files for remote source directories. +> Providing `--download` does not force all files to be downloaded in all modules. It is only used when verifying SHA256 checksum files for remote source directories in `RCLONE_CHECKSUM`. > [!WARNING] > Enabling `--download` may incur substantial cloud data transfer and egress costs, particularly when validating large datasets. Make sure this is the intended behaviour before running the pipeline. From b91df99e5f7b7ccbeda32bc52920a1f30e8040d4 Mon Sep 17 00:00:00 2001 From: Delfina Terradas <155591053+delfiterradas@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:27:00 -0300 Subject: [PATCH 09/15] Update workflows/datasync.nf Co-authored-by: Anabella Trigila <18577080+atrigila@users.noreply.github.com> --- workflows/datasync.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/datasync.nf b/workflows/datasync.nf index 2262557..201706c 100644 --- a/workflows/datasync.nf +++ b/workflows/datasync.nf @@ -40,7 +40,7 @@ workflow DATASYNC { if (sha && input_path.contains('://')) { if (params.download) { - log.warn("The `--download` parameter is enabled. RCLONE_CHECKSUM will download remote files. Make sure this is what you want, as it may incur substantial cloud costs !") + log.warn("The `--download` parameter is enabled. `RCLONE_CHECKSUM` will download remote files. Make sure this is what you want, as it may incur substantial cloud costs !") } else { log.error("A SHA checksum file was provided, but `--download` is not enabled. Rclone cannot verify SHA checksums for remote files without downloading them. Enable `--download` to proceed.") System.exit(1) From 0bb34e3b66d3fc67a49a026ba24a43bc17053353 Mon Sep 17 00:00:00 2001 From: Delfina Terradas <155591053+delfiterradas@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:27:11 -0300 Subject: [PATCH 10/15] Update CHANGELOG.md Co-authored-by: Anabella Trigila <18577080+atrigila@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a581188..d7938a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ Initial release of nf-core/datasync, created with the [nf-core](https://nf-co.re ### `Added` -- [[#74](https://github.com/nf-core/datasync/pull/74)] - Implement warning and `--download` parameter when working with remote directories and SHA checksums ([@delfiterradas](https://github.com/delfiterradas), review by [@atrigila](https://github.com/atrigila) and [@antoniasaracco](https://github.com/antoniasaracco)). +- [[#74](https://github.com/nf-core/datasync/pull/74)] - Implement warning and `--download` parameter when working with remote directories and SHA checksums ([@delfiterradas](https://github.com/delfiterradas), review by [@atrigila](https://github.com/atrigila), [@apeltzer](https://github.com/apeltzer) and [@antoniasaracco](https://github.com/antoniasaracco)). - [[#67](https://github.com/nf-core/datasync/pull/67)] - Copy only files which are successfully validated ([@delfiterradas](https://github.com/delfiterradas), review by [@atrigila](https://github.com/atrigila) and [@antoniasaracco](https://github.com/antoniasaracco)). - [[#57](https://github.com/nf-core/datasync/pull/57)] - Add nf-tests with edge cases([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)). - [[#55](https://github.com/nf-core/datasync/pull/55)] - Add subdirectories with sample id to results. Improvements to MultiQC report (width of columns, sections and sub-section headers)colors to multiqc results and display first errors in tables ([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)). From fff74fc2b222e7fcb4338f9e40856337d600a2d0 Mon Sep 17 00:00:00 2001 From: Delfina Terradas <155591053+delfiterradas@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:27:27 -0300 Subject: [PATCH 11/15] Update workflows/datasync.nf Co-authored-by: Anabella Trigila <18577080+atrigila@users.noreply.github.com> --- workflows/datasync.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/datasync.nf b/workflows/datasync.nf index 201706c..966fc23 100644 --- a/workflows/datasync.nf +++ b/workflows/datasync.nf @@ -42,7 +42,7 @@ workflow DATASYNC { if (params.download) { log.warn("The `--download` parameter is enabled. `RCLONE_CHECKSUM` will download remote files. Make sure this is what you want, as it may incur substantial cloud costs !") } else { - log.error("A SHA checksum file was provided, but `--download` is not enabled. Rclone cannot verify SHA checksums for remote files without downloading them. Enable `--download` to proceed.") + log.error("A SHA checksum file was provided, but `--download` is not enabled. `RCLONE_CHECKSUM` cannot verify SHA256 checksums for remote files without downloading them. Enable `--download` to proceed.") System.exit(1) } } From a7c7f0887a87db105245540c3e8a9327e979610e Mon Sep 17 00:00:00 2001 From: Delfina Terradas <155591053+delfiterradas@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:27:39 -0300 Subject: [PATCH 12/15] Update nextflow_schema.json Co-authored-by: Anabella Trigila <18577080+atrigila@users.noreply.github.com> --- nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 7e8a2a4..807710c 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -63,7 +63,7 @@ }, "download": { "type": "boolean", - "description": "Download remote files for sha256 checksum verification.", + "description": "Download remote files for sha256 checksum verification in `RCLONE_CHECKSUM`.", "fa_icon": "fas fa-eye", "help_text": "If set, rclone checksum will download remote files for any sample for which a shasum file was provided." } From d5b06b4739a4ec621be5d0b9f8dc206e1f08a3f7 Mon Sep 17 00:00:00 2001 From: Delfina Terradas <155591053+delfiterradas@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:52:57 -0300 Subject: [PATCH 13/15] Update nextflow_schema.json Co-authored-by: Anabella Trigila <18577080+atrigila@users.noreply.github.com> --- nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 807710c..738538e 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -65,7 +65,7 @@ "type": "boolean", "description": "Download remote files for sha256 checksum verification in `RCLONE_CHECKSUM`.", "fa_icon": "fas fa-eye", - "help_text": "If set, rclone checksum will download remote files for any sample for which a shasum file was provided." + "help_text": "If set, `RCLONE_CHECKSUM` will download remote files for any sample for which a SHA256 file was provided." } } }, From 8cbd838c0db14524a1473c91957bb67b9769729a Mon Sep 17 00:00:00 2001 From: delfiterradas Date: Fri, 31 Jul 2026 20:25:13 +0200 Subject: [PATCH 14/15] Add failing test scenario --- tests/edge.nf.test | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/edge.nf.test b/tests/edge.nf.test index 4e626ae..b30aa79 100644 --- a/tests/edge.nf.test +++ b/tests/edge.nf.test @@ -64,4 +64,22 @@ nextflow_pipeline { ) } } + + test("-profile test remote source with SHA256 checksums - fail") { + + when { + params { + input = "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/datasync/test-data/samplesheet_edge.csv" + outdir = "$outputDir" + } + } + + then { + // stable_path: All files + folders in ${params.outdir}/ with a stable path (including file name) + def stable_path = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_content: All files in ${params.outdir}/ with stable content + def stable_content = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + assert workflow.failed + } + } } From c80269523f3d6f865adfbe5936fbbcdc78dfe45a Mon Sep 17 00:00:00 2001 From: Delfina Terradas <155591053+delfiterradas@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:26:33 -0300 Subject: [PATCH 15/15] Clean up edge.nf.test by removing unused variables --- tests/edge.nf.test | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/edge.nf.test b/tests/edge.nf.test index b30aa79..f5c5188 100644 --- a/tests/edge.nf.test +++ b/tests/edge.nf.test @@ -75,10 +75,6 @@ nextflow_pipeline { } then { - // stable_path: All files + folders in ${params.outdir}/ with a stable path (including file name) - def stable_path = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) - // stable_content: All files in ${params.outdir}/ with stable content - def stable_content = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') assert workflow.failed } }