From d881783eeb285f3dbfcfd838ec43c8447c72f5f4 Mon Sep 17 00:00:00 2001 From: delfiterradas Date: Mon, 27 Jul 2026 16:13:36 +0200 Subject: [PATCH 1/8] Implement only copying files that matched their checksum --- conf/modules.config | 24 +++++++++++ modules/nf-core/rclone/copy/main.nf | 2 +- nextflow.config | 1 + nextflow_schema.json | 6 +++ workflows/datasync.nf | 67 +++++++++++++++++++++-------- 5 files changed, 81 insertions(+), 19 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 3a376b6..847d32d 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -18,6 +18,29 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] + withName: 'RCLONE_COPY_MATCHING_ONLY' { + ext.args = { + def base_args = [ + '--log-level INFO', + '--stats 30s', + '--stats-one-line', + '--stats-log-level INFO', + '--s3-chunk-size 64M', + '--no-check-certificate', + "--include-from ${filter_file}" + ] + if (params.rclone_dry_run) { + base_args.add('--dry-run') + } + base_args.join(' ') + } + publishDir = [ + path: { "${params.outdir}/rclone/copy" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + withName: 'RCLONE_COPY' { ext.args = { def base_args = [ @@ -56,6 +79,7 @@ process { } withName: 'RCLONE_CHECKSUM' { + tag = { "${meta.id}_${hash}" } ext.args = { def base_args = [ '--no-check-certificate', diff --git a/modules/nf-core/rclone/copy/main.nf b/modules/nf-core/rclone/copy/main.nf index 2c97b43..a063abf 100644 --- a/modules/nf-core/rclone/copy/main.nf +++ b/modules/nf-core/rclone/copy/main.nf @@ -8,7 +8,7 @@ process RCLONE_COPY { : 'community.wave.seqera.io/library/rclone:1.65.0--ff88b2e0040147be'}" input: - tuple val(meta), val(source_path), val(destination_path) + tuple val(meta), val(source_path), val(destination_path), path(filter_file) path rclone_config output: diff --git a/nextflow.config b/nextflow.config index e87a679..216b3fc 100644 --- a/nextflow.config +++ b/nextflow.config @@ -23,6 +23,7 @@ params { // Rclone options rclone_config = null rclone_dry_run = false + copy_matching_only = false // Boilerplate options outdir = null diff --git a/nextflow_schema.json b/nextflow_schema.json index e17f7cb..a8dc091 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -54,6 +54,12 @@ "description": "Perform a dry run of the rclone copy command.", "fa_icon": "fas fa-eye", "help_text": "If set, the pipeline will not actually copy any files to the destination. Instead, it will print out what would have been copied. This is useful for testing and debugging." + }, + "copy_matching_only": { + "type": "boolean", + "description": "Only copy files that matched their provided input checksums.", + "fa_icon": "fas fa-eye", + "help_text": "If set, the pipeline will only copy files that were correctly validated and will skip any file that did not match their input checksum." } } }, diff --git a/workflows/datasync.nf b/workflows/datasync.nf index 23eb696..b35058a 100644 --- a/workflows/datasync.nf +++ b/workflows/datasync.nf @@ -3,15 +3,16 @@ IMPORT MODULES / SUBWORKFLOWS / FUNCTIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { MULTIQC } from '../modules/nf-core/multiqc/main' -include { RCLONE_COPY } from '../modules/nf-core/rclone/copy/main' -include { RCLONE_CHECK } from '../modules/nf-core/rclone/check/main' -include { RCLONE_CHECKSUM } from '../modules/nf-core/rclone/checksum/main' -include { paramsSummaryMap } from 'plugin/nf-schema' -include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' -include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' -include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_datasync_pipeline' -include { parseRcloneCheck } from '../subworkflows/local/utils_nfcore_datasync_pipeline' +include { MULTIQC } from '../modules/nf-core/multiqc/main' +include { RCLONE_COPY } from '../modules/nf-core/rclone/copy/main' +include { RCLONE_COPY as RCLONE_COPY_MATCHING_ONLY } from '../modules/nf-core/rclone/copy/main' +include { RCLONE_CHECK } from '../modules/nf-core/rclone/check/main' +include { RCLONE_CHECKSUM } from '../modules/nf-core/rclone/checksum/main' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_datasync_pipeline' +include { parseRcloneCheck } from '../subworkflows/local/utils_nfcore_datasync_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -88,19 +89,49 @@ workflow DATASYNC { // // MODULE: Rclone data copying // - RCLONE_COPY( - ch_samplesheet.rclone, - ch_rclone_config - ) + if(params.copy_matching_only) { + files_to_copy = RCLONE_CHECKSUM.out.match.map { + meta, match -> [ meta.subMap(meta.keySet() - 'check_format'), match ] + } + .groupTuple() + .map{ meta, files -> + def common = files + .collect { it.readLines() } + .inject { a, b -> a.intersect(b) } + + def copy_files = file("${outdir}/${meta.id}_files_to_copy.txt") + copy_files.text = common.join('\n') + '\n' + + tuple(meta, copy_files) + } + + ch_rclone_copy = ch_samplesheet.rclone + .join(files_to_copy) + + RCLONE_COPY_MATCHING_ONLY( + ch_rclone_copy, + ch_rclone_config, + ) + + // Wait for file copy to finish before running RCLONE_CHECK + ch_rclone_check = ch_samplesheet.rclone + .join(RCLONE_COPY_MATCHING_ONLY.out.log) + .map { meta, input, output, log -> [ meta, input, output ]} + } else { + RCLONE_COPY( + ch_samplesheet.rclone.map { meta, source, destination -> [ meta, source, destination, [] ]}, + ch_rclone_config, + ) + + // Wait for file copy to finish before running RCLONE_CHECK + ch_rclone_check = ch_samplesheet.rclone + .join(RCLONE_COPY.out.log) + .map { meta, input, output, log -> [ meta, input, output ]} + } // // File transfer validation // - // Wait for file copy to finish before running RCLONE_CHECK - ch_rclone_check = ch_samplesheet.rclone - .join(RCLONE_COPY.out.log) - .map { meta, input, output, log -> [ meta, input, output ]} - RCLONE_CHECK( ch_rclone_check, ch_rclone_config From 999dca5225ed51460925a21ee8a34e2d56e5bcb2 Mon Sep 17 00:00:00 2001 From: delfiterradas Date: Mon, 27 Jul 2026 19:45:48 +0200 Subject: [PATCH 2/8] Compute siz to avoid silent data loss --- workflows/datasync.nf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/workflows/datasync.nf b/workflows/datasync.nf index b35058a..e201b82 100644 --- a/workflows/datasync.nf +++ b/workflows/datasync.nf @@ -90,9 +90,18 @@ workflow DATASYNC { // MODULE: Rclone data copying // if(params.copy_matching_only) { + // Compute expected group size per meta.id from the input + ch_with_size = ch_checksum + .map { meta, checksum, hash, source -> [ meta.subMap(meta.keySet() - 'check_format'), 1 ] } + .groupTuple() + .map { meta, ones -> tuple(meta, ones.size()) } + .view() + files_to_copy = RCLONE_CHECKSUM.out.match.map { meta, match -> [ meta.subMap(meta.keySet() - 'check_format'), match ] } + .combine(ch_with_size, by: 0) + .map { meta, match, size -> tuple(groupKey(meta, size), match) } .groupTuple() .map{ meta, files -> def common = files From 32dcb749eb7d0fa159ce4b1f6933ac1ad41213e0 Mon Sep 17 00:00:00 2001 From: delfiterradas Date: Mon, 27 Jul 2026 20:37:53 +0200 Subject: [PATCH 3/8] Refactor to only use rclone_copy once --- conf/modules.config | 26 +++----------------------- workflows/datasync.nf | 34 ++++++++++++---------------------- 2 files changed, 15 insertions(+), 45 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 847d32d..794c4d9 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -18,29 +18,6 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] - withName: 'RCLONE_COPY_MATCHING_ONLY' { - ext.args = { - def base_args = [ - '--log-level INFO', - '--stats 30s', - '--stats-one-line', - '--stats-log-level INFO', - '--s3-chunk-size 64M', - '--no-check-certificate', - "--include-from ${filter_file}" - ] - if (params.rclone_dry_run) { - base_args.add('--dry-run') - } - base_args.join(' ') - } - publishDir = [ - path: { "${params.outdir}/rclone/copy" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - withName: 'RCLONE_COPY' { ext.args = { def base_args = [ @@ -54,6 +31,9 @@ process { if (params.rclone_dry_run) { base_args.add('--dry-run') } + if (params.copy_matching_only) { + base_args.add("--include-from ${filter_file}") + } base_args.join(' ') } publishDir = [ diff --git a/workflows/datasync.nf b/workflows/datasync.nf index e201b82..7f64a97 100644 --- a/workflows/datasync.nf +++ b/workflows/datasync.nf @@ -5,7 +5,6 @@ */ include { MULTIQC } from '../modules/nf-core/multiqc/main' include { RCLONE_COPY } from '../modules/nf-core/rclone/copy/main' -include { RCLONE_COPY as RCLONE_COPY_MATCHING_ONLY } from '../modules/nf-core/rclone/copy/main' include { RCLONE_CHECK } from '../modules/nf-core/rclone/check/main' include { RCLONE_CHECKSUM } from '../modules/nf-core/rclone/checksum/main' include { paramsSummaryMap } from 'plugin/nf-schema' @@ -95,7 +94,6 @@ workflow DATASYNC { .map { meta, checksum, hash, source -> [ meta.subMap(meta.keySet() - 'check_format'), 1 ] } .groupTuple() .map { meta, ones -> tuple(meta, ones.size()) } - .view() files_to_copy = RCLONE_CHECKSUM.out.match.map { meta, match -> [ meta.subMap(meta.keySet() - 'check_format'), match ] @@ -108,7 +106,7 @@ workflow DATASYNC { .collect { it.readLines() } .inject { a, b -> a.intersect(b) } - def copy_files = file("${outdir}/${meta.id}_files_to_copy.txt") + def copy_files = file("${workDir}/${meta.id}_files_to_copy.txt") copy_files.text = common.join('\n') + '\n' tuple(meta, copy_files) @@ -116,28 +114,20 @@ workflow DATASYNC { ch_rclone_copy = ch_samplesheet.rclone .join(files_to_copy) - - RCLONE_COPY_MATCHING_ONLY( - ch_rclone_copy, - ch_rclone_config, - ) - - // Wait for file copy to finish before running RCLONE_CHECK - ch_rclone_check = ch_samplesheet.rclone - .join(RCLONE_COPY_MATCHING_ONLY.out.log) - .map { meta, input, output, log -> [ meta, input, output ]} } else { - RCLONE_COPY( - ch_samplesheet.rclone.map { meta, source, destination -> [ meta, source, destination, [] ]}, - ch_rclone_config, - ) - - // Wait for file copy to finish before running RCLONE_CHECK - ch_rclone_check = ch_samplesheet.rclone - .join(RCLONE_COPY.out.log) - .map { meta, input, output, log -> [ meta, input, output ]} + ch_rclone_copy = ch_samplesheet.rclone.map { meta, source, destination -> [ meta, source, destination, [] ] } } + RCLONE_COPY( + ch_rclone_copy, + ch_rclone_config, + ) + + // Wait for file copy to finish before running RCLONE_CHECK + ch_rclone_check = ch_samplesheet.rclone + .join(RCLONE_COPY.out.log) + .map { meta, input, output, log -> [ meta, input, output ] } + // // File transfer validation // From e0d1df971a7320f7f4d3f8305a35d7d24e779320 Mon Sep 17 00:00:00 2001 From: delfiterradas Date: Mon, 27 Jul 2026 22:02:00 +0200 Subject: [PATCH 4/8] Bump nf-schema to 2.7.2 to fix errors in validation of boolean parameters in CLI --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 216b3fc..2423a8a 100644 --- a/nextflow.config +++ b/nextflow.config @@ -260,7 +260,7 @@ manifest { // Nextflow plugins plugins { - id 'nf-schema@2.5.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-schema@2.7.2' // Validation of pipeline parameters and creation of an input channel from a sample sheet } validation { From f892c202c9fd24004d518b7c6727a804799beebf Mon Sep 17 00:00:00 2001 From: delfiterradas Date: Tue, 28 Jul 2026 21:51:33 +0200 Subject: [PATCH 5/8] Add test scenario copying only matching files --- tests/edge.nf.test | 29 +++++++++++++++++ tests/edge.nf.test.snap | 70 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/tests/edge.nf.test b/tests/edge.nf.test index d69a8f8..5fce38e 100644 --- a/tests/edge.nf.test +++ b/tests/edge.nf.test @@ -33,4 +33,33 @@ nextflow_pipeline { ) } } + + test("-profile test copy matching files only") { + + when { + params { + input = "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/datasync/test-data/samplesheet_edge.csv" + outdir = "$outputDir" + copy_matching_only = true + } + } + + then { + // stable_path: All files + folders in ${params.outdir}/ with a stable path (including file name) + def stable_path = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_content: All files in ${params.outdir}/ with stable content + def stable_content = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + assert workflow.success + assertAll( + { assert snapshot( + // pipeline versions.yml file for multiqc from which Nextflow version is removed because we test pipelines on multiple Nextflow versions + removeNextflowVersion("$outputDir/pipeline_info/nf_core_datasync_software_mqc_versions.yml"), + // All stable path name, with a relative path + stable_path, + // All files with stable contents + stable_content + ).match() } + ) + } + } } diff --git a/tests/edge.nf.test.snap b/tests/edge.nf.test.snap index 517b27b..ea3559b 100644 --- a/tests/edge.nf.test.snap +++ b/tests/edge.nf.test.snap @@ -1,4 +1,74 @@ { + "-profile test copy matching files only": { + "content": [ + { + "RCLONE_CHECK": { + "rclone": "1.74.3-DEV" + }, + "RCLONE_CHECKSUM": { + "rclone": "1.74.3-DEV" + }, + "RCLONE_COPY": { + "rclone": "1.65.0-DEV" + }, + "Workflow": { + "nf-core/datasync": "v1.0dev" + } + }, + [ + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/llms-full.txt", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_rclone_check.txt", + "multiqc/multiqc_data/multiqc_rclone_checksum_md5.txt", + "multiqc/multiqc_data/multiqc_samplesheet.txt", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/nf_core_datasync_software_mqc_versions.yml", + "rclone", + "rclone/check", + "rclone/check/Illumina_annotation_incorrect", + "rclone/check/Illumina_annotation_incorrect/Illumina_annotation_incorrect.combined.txt", + "rclone/check/Illumina_annotation_incorrect/Illumina_annotation_incorrect.match.txt", + "rclone/check/Illumina_annotation_missing", + "rclone/check/Illumina_annotation_missing/Illumina_annotation_missing.combined.txt", + "rclone/check/Illumina_annotation_missing/Illumina_annotation_missing.match.txt", + "rclone/checksum", + "rclone/checksum/Illumina_annotation_incorrect", + "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.combined.txt", + "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.differ.txt", + "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.exit_code.txt", + "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.match.txt", + "rclone/checksum/Illumina_annotation_missing", + "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.combined.txt", + "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.exit_code.txt", + "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.match.txt", + "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.missing_on_dst.txt", + "rclone/checksum/Illumina_annotation_sha_only", + "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.exit_code.txt", + "rclone/copy", + "rclone/copy/Illumina_annotation_incorrect-rclone-copy.log", + "rclone/copy/Illumina_annotation_missing-rclone-copy.log" + ], + [ + "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", + "multiqc_rclone_check.txt:md5,4b2a6990836593f69e8e81a82c3daf42", + "multiqc_rclone_checksum_md5.txt:md5,cf92203df21f04ddf8315fc606812e0b", + "multiqc_samplesheet.txt:md5,bbfc817ff43c49cde14a2b33f46b5ae5" + ] + ], + "timestamp": "2026-07-28T21:45:08.268780076", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.1" + } + }, "-profile test edge cases": { "content": [ { From fe97a5c2a459e3c6d9c936a5eb8015637ef00be9 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 28 Jul 2026 14:17:39 +0200 Subject: [PATCH 6/8] Remove vulnerable PR-comment artifact pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This pipeline's TEMPLATE branch never received the nf-core/tools 4.0.3 sync, so the patch was taken from nf-core/abotyper's TEMPLATE branch with the repository identity in branch.yml rewritten for this pipeline. That identity is the only pipeline-specific content the template renders into any of the patched workflows, so this is the same change this pipeline's own sync would have produced — but the substituted source is worth a look. --- .github/workflows/branch.yml | 60 +++++++++----- .github/workflows/linting.yml | 18 ++++ .github/workflows/linting_comment.yml | 28 ------- .github/workflows/nf-test.yml | 57 +++++++++++++ .github/workflows/pr-comment.yml | 82 +++++++++++++++++++ .../workflows/template-version-comment.yml | 56 ++++++++----- .nf-core.yml | 8 +- 7 files changed, 236 insertions(+), 73 deletions(-) delete mode 100644 .github/workflows/linting_comment.yml create mode 100644 .github/workflows/pr-comment.yml diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index 7716a7b..720e16c 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -2,11 +2,13 @@ name: nf-core branch protection # This workflow is triggered on PRs to `main`/`master` branch on the repository # It fails when someone tries to make a PR against the nf-core `main`/`master` branch instead of `dev` on: - pull_request_target: + pull_request: branches: - main - master +permissions: {} + jobs: test: runs-on: ubuntu-latest @@ -14,33 +16,47 @@ jobs: # PRs to the nf-core repo main/master branch are only ok if coming from the nf-core repo `dev` or any `patch` branches - name: Check PRs if: github.repository == 'nf-core/datasync' + env: + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} run: | - { [[ ${{github.event.pull_request.head.repo.full_name }} == nf-core/datasync ]] && [[ $GITHUB_HEAD_REF == "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] + { [[ "$HEAD_REPO" == nf-core/datasync ]] && [[ $GITHUB_HEAD_REF == "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] - # If the above check failed, post a comment on the PR explaining the failure - # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets - - name: Post PR comment + # If the above check failed, build a comment to be posted by the shared poster workflow + - name: Build PR comment if: failure() - uses: mshick/add-pr-comment@8e4927817251f1ff60c001f04568532b38e0b4a0 # v3 - with: - message: | - ## This PR is against the `${{github.event.pull_request.base.ref}}` branch :x: + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + PR_USER: ${{ github.event.pull_request.user.login }} + run: | + mkdir -p pr-comment + echo "$PR_NUMBER" > pr-comment/pr_number.txt + echo "branch" > pr-comment/header.txt + cat > pr-comment/comment.md < pr-comment/pr_number.txt + echo "lint" > pr-comment/header.txt + [ -f lint_results.md ] && cp lint_results.md pr-comment/comment.md || true + + - name: Upload PR comment artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: pr-comment + path: pr-comment/ diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml deleted file mode 100644 index 5b0c24f..0000000 --- a/.github/workflows/linting_comment.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: nf-core linting comment -# This workflow is triggered after the linting action is complete -# It posts an automated comment to the PR, even if the PR is coming from a fork - -on: - workflow_run: - workflows: ["nf-core linting"] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Download lint results - uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21 - with: - workflow: linting.yml - workflow_conclusion: completed - - - name: Get PR number - id: pr_number - run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - - - name: Post PR comment - uses: marocchino/sticky-pull-request-comment@70d2764d1a7d5d9560b100cbea0077fc8f633987 # v3 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - number: ${{ steps.pr_number.outputs.pr_number }} - path: linting-logs/lint_results.md diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index efd72d6..4de681a 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -116,6 +116,22 @@ jobs: fi fi + # continue-on-error keeps latest-everything from failing the job, so it never shows up in + # `needs.nf-test.result` downstream and CI stays green. Surface it via a PR comment instead; + # other NXF_VER failures already fail the job/CI directly, so no comment is needed for those. + - name: Prepare PR comment fragment + if: ${{ always() && steps.run_nf_test.outcome == 'failure' && matrix.NXF_VER == 'latest-everything' }} + run: | + mkdir -p pr-comment-fragment + echo "* ❌ \`${{ matrix.profile }}\` | \`${{ matrix.NXF_VER }}\` | Shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }}" > pr-comment-fragment/fragment.md + + - name: Upload PR comment fragment + if: ${{ always() && steps.run_nf_test.outcome == 'failure' && matrix.NXF_VER == 'latest-everything' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: pr-comment-fragment-${{ strategy.job-index }} + path: pr-comment-fragment/ + confirm-pass: needs: [nf-test] if: always() @@ -142,3 +158,44 @@ jobs: echo "DEBUG: toJSON(needs) = ${{ toJSON(needs) }}" echo "DEBUG: toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" echo "::endgroup::" + + - name: Download PR comment fragments + if: ${{ always() }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + pattern: pr-comment-fragment-* + path: pr-comment-fragments + merge-multiple: true + + # Build a comment for the shared pr-comment.yml poster to publish on the PR. + # Based on the fragments above (not needs.*.result) so non-blocking failures are still reported. + - name: Prepare PR comment + if: ${{ always() }} + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + mkdir -p pr-comment + echo "$PR_NUMBER" > pr-comment/pr_number.txt + echo "nf-test" > pr-comment/header.txt + if [ -d pr-comment-fragments ] && [ -n "$(ls -A pr-comment-fragments)" ]; then + { + echo "## ❌ nf-test failed with latest Nextflow version" + echo "" + echo "> [!NOTE]" + echo "> Tests with Nextflow's latest version failed but it will not cause a CI workflow failure." + echo "> Please check if the failure is expected with newer (edge-)releases of Nextflow or if it needs fixing." + echo "" + cat pr-comment-fragments/*.md + echo "" + echo "See the [full run](${RUN_URL}) for details." + } > pr-comment/comment.md + fi + + - name: Upload PR comment artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: pr-comment + path: pr-comment/ diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.yml new file mode 100644 index 0000000..ab7b59d --- /dev/null +++ b/.github/workflows/pr-comment.yml @@ -0,0 +1,82 @@ +name: Post PR comment +# Shared, privileged comment poster. +# +# This is the single workflow that runs with a write token. It is triggered +# after any of the listed "producer" workflows complete on a pull request. +# Each producer runs untrusted PR code (if any) with a read-only token and +# uploads a `pr-comment` artifact describing the comment to post; this workflow +# only ever reads that plain-text artifact, so no PR code is executed here. +# +# Artifact contract (uploaded by producers under the name `pr-comment`): +# pr_number.txt - the pull request number +# header.txt - sticky-comment identifier (keeps comment types separate) +# comment.md - the Markdown body (omit the file to post nothing) + +on: + workflow_run: + workflows: + - "nf-core linting" + - "nf-core template version comment" + - "nf-core branch protection" + - "Run nf-test" + +permissions: + actions: read + contents: read + pull-requests: write + +jobs: + post-comment: + runs-on: ubuntu-latest + if: github.event.workflow_run.event == 'pull_request' + steps: + - name: Download PR comment artifact + uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21 + with: + run_id: ${{ github.event.workflow_run.id }} + name: pr-comment + path: pr-comment + if_no_artifact_found: ignore + + - name: Read comment metadata + id: meta + run: | + echo "::group::Downloaded pr-comment contents" + ls -la pr-comment 2>/dev/null || echo "No pr-comment/ directory was downloaded." + echo "::endgroup::" + + if [ ! -d pr-comment ]; then + echo "No pr-comment artifact found; nothing to post." + exit 0 + fi + + if [ ! -f pr-comment/comment.md ]; then + echo "Artifact present but no comment.md; nothing to post." + exit 0 + fi + + pr_number=$(cat pr-comment/pr_number.txt) + header=$(cat pr-comment/header.txt) + echo "Found comment.md (header='$header', pr_number='$pr_number')." + + # Guard against anything unexpected ending up in the PR number. + case "$pr_number" in + ''|*[!0-9]*) + echo "Invalid PR number: '$pr_number'" + exit 1 + ;; + esac + + echo "pr_number=$pr_number" >> "$GITHUB_OUTPUT" + echo "header=$header" >> "$GITHUB_OUTPUT" + echo "post=true" >> "$GITHUB_OUTPUT" + echo "Will post comment to PR #${pr_number}." + + - name: Post PR comment + if: steps.meta.outputs.post == 'true' + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + number: ${{ steps.meta.outputs.pr_number }} + header: ${{ steps.meta.outputs.header }} + path: pr-comment/comment.md diff --git a/.github/workflows/template-version-comment.yml b/.github/workflows/template-version-comment.yml index ea30827..ee102f7 100644 --- a/.github/workflows/template-version-comment.yml +++ b/.github/workflows/template-version-comment.yml @@ -2,10 +2,13 @@ name: nf-core template version comment # This workflow is triggered on PRs to check if the pipeline template version matches the latest nf-core version. # It posts a comment to the PR, even if it comes from a fork. -on: pull_request_target +on: + pull_request: + +permissions: {} jobs: - template_version: + check_template_version: runs-on: ubuntu-latest steps: - name: Check out pipeline code @@ -22,25 +25,36 @@ jobs: - name: Install nf-core run: | python -m pip install --upgrade pip - pip install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }} + pip install nf-core + + - name: Build PR comment if template is outdated + # The fork-controlled version is passed via the environment and only ever + # used as quoted shell data (never interpolated into a command), so it + # cannot be used for script injection. + env: + PR_VERSION: ${{ steps.read_yml.outputs['nf_core_version'] }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + mkdir -p pr-comment + echo "$PR_NUMBER" > pr-comment/pr_number.txt + echo "template-version" > pr-comment/header.txt + + latest_version=$(nf-core --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n1) - - name: Check nf-core outdated - id: nf_core_outdated - run: echo "OUTPUT=$(pip list --outdated | grep nf-core)" >> ${GITHUB_ENV} + if [ -n "$PR_VERSION" ] && [ -n "$latest_version" ] && [ "$PR_VERSION" != "$latest_version" ]; then + cat > pr-comment/comment.md < [!WARNING] + > Newer version of the nf-core template is available. + > + > Your pipeline is using an old version of the nf-core template: ${PR_VERSION}. + > Please update your pipeline to the latest version. + > + > For more documentation on how to update your pipeline, please see the [Synchronisation documentation](https://nf-co.re/docs/developing/template-syncs/overview). + EOF + fi - - name: Post nf-core template version comment - uses: mshick/add-pr-comment@8e4927817251f1ff60c001f04568532b38e0b4a0 # v3 - if: | - contains(env.OUTPUT, 'nf-core') + - name: Upload PR comment artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - repo-token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} - allow-repeats: false - message: | - > [!WARNING] - > Newer version of the nf-core template is available. - > - > Your pipeline is using an old version of the nf-core template: ${{ steps.read_yml.outputs['nf_core_version'] }}. - > Please update your pipeline to the latest version. - > - > For more documentation on how to update your pipeline, please see the [Synchronisation documentation](https://nf-co.re/docs/developing/template-syncs/overview). - # + name: pr-comment + path: pr-comment/ diff --git a/.nf-core.yml b/.nf-core.yml index cf24bd7..c75165a 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -2,8 +2,7 @@ nf_core_version: 4.0.2 repository_type: pipeline template: author: Alexander Peltzer - description: A simple sysops pipeline that can be used to synchronize, integrity - check and permanently archive data. + description: A simple sysops pipeline that can be used to synchronize, integrity check and permanently archive data. force: false is_nfcore: true name: datasync @@ -12,3 +11,8 @@ template: version: 1.0dev lint: multiqc_config: false + files_exist: + - .github/workflows/linting_comment.yml + files_unchanged: + - .github/workflows/branch.yml + - .github/workflows/linting.yml From cac46ddc7192eb8899842a597fced988e9cf0793 Mon Sep 17 00:00:00 2001 From: delfiterradas Date: Tue, 28 Jul 2026 21:56:30 +0200 Subject: [PATCH 7/8] Update changelog.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01faa82..739d9d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Initial release of nf-core/datasync, created with the [nf-core](https://nf-co.re ### `Added` +- [[#67](https://github.com/nf-core/datasync/pull/67)] - Copy only files which are successfully validated ([@delfiterradas](https://github.com/delfiterradas), review by [@atrigila](https://github.com/atrigila) and [@antoniasaracco](https://github.com/antoniasaracco)). - [[#57](https://github.com/nf-core/datasync/pull/57)] - Add nf-tests with edge cases([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)). - [[#55](https://github.com/nf-core/datasync/pull/55)] - Add subdirectories with sample id to results. Improvements to MultiQC report (width of columns, sections and sub-section headers)colors to multiqc results and display first errors in tables ([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)). - [[#54](https://github.com/nf-core/datasync/pull/54)] - Add colors to multiqc results and display first errors in tables ([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)). From 6cb7cac6c58643c12a074bfddf41f15bf8f1af83 Mon Sep 17 00:00:00 2001 From: delfiterradas Date: Wed, 29 Jul 2026 15:19:00 +0200 Subject: [PATCH 8/8] Update rclone_copy --- modules.json | 2 +- modules/nf-core/rclone/copy/meta.yml | 7 ++++ .../nf-core/rclone/copy/tests/main.nf.test | 42 +++++++++++++++++-- .../rclone/copy/tests/main.nf.test.snap | 26 ++++++++++++ .../nf-core/rclone/copy/tests/nextflow.config | 9 +++- 5 files changed, 81 insertions(+), 5 deletions(-) diff --git a/modules.json b/modules.json index 66e3022..df30276 100644 --- a/modules.json +++ b/modules.json @@ -22,7 +22,7 @@ }, "rclone/copy": { "branch": "master", - "git_sha": "a554e54c63c3f03f91c88a18ece7b89eca5a5212", + "git_sha": "b34acd9d361bb226be8f0cf446b8b3a20ad0e3da", "installed_by": ["modules"] } } diff --git a/modules/nf-core/rclone/copy/meta.yml b/modules/nf-core/rclone/copy/meta.yml index 93ed4f9..18c5bf1 100644 --- a/modules/nf-core/rclone/copy/meta.yml +++ b/modules/nf-core/rclone/copy/meta.yml @@ -36,6 +36,13 @@ input: path internally. Examples: `results/output`, `s3:bucket/output`, `gs:bucket/output`, `remote:path/to/output`. + - filter_file: + type: file + description: | + Optional plain text file containing one file path or pattern per line to be + passed to Rclone copy as a filter list (e.g. via `--files-from` or + `--include-from`). This input is only used when the corresponding rclone + filtering option is enabled through the module configuration. - rclone_config: type: file description: | diff --git a/modules/nf-core/rclone/copy/tests/main.nf.test b/modules/nf-core/rclone/copy/tests/main.nf.test index b9f0a77..85d92a0 100644 --- a/modules/nf-core/rclone/copy/tests/main.nf.test +++ b/modules/nf-core/rclone/copy/tests/main.nf.test @@ -3,6 +3,7 @@ nextflow_process { name "Test RCLONE_COPY" script "../main.nf" process "RCLONE_COPY" + config "./nextflow.config" tag "modules" tag "modules_nfcore" @@ -11,15 +12,46 @@ nextflow_process { test("homo_sapiens - gvcf - copy from https - dry-run") { - config "./nextflow.config" + when { + params { + exclude_from = false + } + process { + """ + input[0] = [ + [ id:'test' ], + params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz', + '/tmp/', + [] + ] + input[1] = [] + """ + } + } + + then { + assert process.success + assertAll( + { assert snapshot(sanitizeOutput(process.out, unstableKeys: ["log"])).match() } + ) + } + } + + test("homo_sapiens - gvcf - filter - dry-run") { when { + params { + exclude_from = true + } process { """ + file('exclude.txt').text = 'test.genome.vcf.gz' + input[0] = [ [ id:'test' ], params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz', - '/tmp/' + '/tmp/', + file('exclude.txt') ] input[1] = [] """ @@ -39,12 +71,16 @@ nextflow_process { options "-stub" when { + params { + exclude_from = true + } process { """ input[0] = [ [ id:'test' ], params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz', - '/tmp/' + '/tmp/', + [] ] input[1] = [] """ diff --git a/modules/nf-core/rclone/copy/tests/main.nf.test.snap b/modules/nf-core/rclone/copy/tests/main.nf.test.snap index eeaef63..a430eb4 100644 --- a/modules/nf-core/rclone/copy/tests/main.nf.test.snap +++ b/modules/nf-core/rclone/copy/tests/main.nf.test.snap @@ -50,5 +50,31 @@ "nf-test": "0.9.5", "nextflow": "25.10.4" } + }, + "homo_sapiens - gvcf - filter - dry-run": { + "content": [ + { + "log": [ + [ + { + "id": "test" + }, + "test-rclone-copy.log" + ] + ], + "versions_rclone": [ + [ + "RCLONE_COPY", + "rclone", + "1.65.0-DEV" + ] + ] + } + ], + "timestamp": "2026-07-28T15:53:02.864977147", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.1" + } } } \ No newline at end of file diff --git a/modules/nf-core/rclone/copy/tests/nextflow.config b/modules/nf-core/rclone/copy/tests/nextflow.config index 0e911db..dd7038e 100644 --- a/modules/nf-core/rclone/copy/tests/nextflow.config +++ b/modules/nf-core/rclone/copy/tests/nextflow.config @@ -1,7 +1,14 @@ process { withName: 'RCLONE_COPY' { ext.args = { - "--dry-run --no-check-certificate" + def base_args = [ + '--dry-run', + '--no-check-certificate' + ] + if (params.exclude_from) { + base_args.add("--exclude-from ${filter_file}") + } + base_args.join(' ') } } }