diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index 7716a7b..720e16c 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -2,11 +2,13 @@ name: nf-core branch protection # This workflow is triggered on PRs to `main`/`master` branch on the repository # It fails when someone tries to make a PR against the nf-core `main`/`master` branch instead of `dev` on: - pull_request_target: + pull_request: branches: - main - master +permissions: {} + jobs: test: runs-on: ubuntu-latest @@ -14,33 +16,47 @@ jobs: # PRs to the nf-core repo main/master branch are only ok if coming from the nf-core repo `dev` or any `patch` branches - name: Check PRs if: github.repository == 'nf-core/datasync' + env: + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} run: | - { [[ ${{github.event.pull_request.head.repo.full_name }} == nf-core/datasync ]] && [[ $GITHUB_HEAD_REF == "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] + { [[ "$HEAD_REPO" == nf-core/datasync ]] && [[ $GITHUB_HEAD_REF == "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] - # If the above check failed, post a comment on the PR explaining the failure - # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets - - name: Post PR comment + # If the above check failed, build a comment to be posted by the shared poster workflow + - name: Build PR comment if: failure() - uses: mshick/add-pr-comment@8e4927817251f1ff60c001f04568532b38e0b4a0 # v3 - with: - message: | - ## This PR is against the `${{github.event.pull_request.base.ref}}` branch :x: + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + PR_USER: ${{ github.event.pull_request.user.login }} + run: | + mkdir -p pr-comment + echo "$PR_NUMBER" > pr-comment/pr_number.txt + echo "branch" > pr-comment/header.txt + cat > pr-comment/comment.md < pr-comment/pr_number.txt + echo "lint" > pr-comment/header.txt + [ -f lint_results.md ] && cp lint_results.md pr-comment/comment.md || true + + - name: Upload PR comment artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: pr-comment + path: pr-comment/ diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml deleted file mode 100644 index 5b0c24f..0000000 --- a/.github/workflows/linting_comment.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: nf-core linting comment -# This workflow is triggered after the linting action is complete -# It posts an automated comment to the PR, even if the PR is coming from a fork - -on: - workflow_run: - workflows: ["nf-core linting"] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Download lint results - uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21 - with: - workflow: linting.yml - workflow_conclusion: completed - - - name: Get PR number - id: pr_number - run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - - - name: Post PR comment - uses: marocchino/sticky-pull-request-comment@70d2764d1a7d5d9560b100cbea0077fc8f633987 # v3 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - number: ${{ steps.pr_number.outputs.pr_number }} - path: linting-logs/lint_results.md diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index efd72d6..4de681a 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -116,6 +116,22 @@ jobs: fi fi + # continue-on-error keeps latest-everything from failing the job, so it never shows up in + # `needs.nf-test.result` downstream and CI stays green. Surface it via a PR comment instead; + # other NXF_VER failures already fail the job/CI directly, so no comment is needed for those. + - name: Prepare PR comment fragment + if: ${{ always() && steps.run_nf_test.outcome == 'failure' && matrix.NXF_VER == 'latest-everything' }} + run: | + mkdir -p pr-comment-fragment + echo "* ❌ \`${{ matrix.profile }}\` | \`${{ matrix.NXF_VER }}\` | Shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }}" > pr-comment-fragment/fragment.md + + - name: Upload PR comment fragment + if: ${{ always() && steps.run_nf_test.outcome == 'failure' && matrix.NXF_VER == 'latest-everything' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: pr-comment-fragment-${{ strategy.job-index }} + path: pr-comment-fragment/ + confirm-pass: needs: [nf-test] if: always() @@ -142,3 +158,44 @@ jobs: echo "DEBUG: toJSON(needs) = ${{ toJSON(needs) }}" echo "DEBUG: toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" echo "::endgroup::" + + - name: Download PR comment fragments + if: ${{ always() }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + pattern: pr-comment-fragment-* + path: pr-comment-fragments + merge-multiple: true + + # Build a comment for the shared pr-comment.yml poster to publish on the PR. + # Based on the fragments above (not needs.*.result) so non-blocking failures are still reported. + - name: Prepare PR comment + if: ${{ always() }} + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + mkdir -p pr-comment + echo "$PR_NUMBER" > pr-comment/pr_number.txt + echo "nf-test" > pr-comment/header.txt + if [ -d pr-comment-fragments ] && [ -n "$(ls -A pr-comment-fragments)" ]; then + { + echo "## ❌ nf-test failed with latest Nextflow version" + echo "" + echo "> [!NOTE]" + echo "> Tests with Nextflow's latest version failed but it will not cause a CI workflow failure." + echo "> Please check if the failure is expected with newer (edge-)releases of Nextflow or if it needs fixing." + echo "" + cat pr-comment-fragments/*.md + echo "" + echo "See the [full run](${RUN_URL}) for details." + } > pr-comment/comment.md + fi + + - name: Upload PR comment artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: pr-comment + path: pr-comment/ diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.yml new file mode 100644 index 0000000..ab7b59d --- /dev/null +++ b/.github/workflows/pr-comment.yml @@ -0,0 +1,82 @@ +name: Post PR comment +# Shared, privileged comment poster. +# +# This is the single workflow that runs with a write token. It is triggered +# after any of the listed "producer" workflows complete on a pull request. +# Each producer runs untrusted PR code (if any) with a read-only token and +# uploads a `pr-comment` artifact describing the comment to post; this workflow +# only ever reads that plain-text artifact, so no PR code is executed here. +# +# Artifact contract (uploaded by producers under the name `pr-comment`): +# pr_number.txt - the pull request number +# header.txt - sticky-comment identifier (keeps comment types separate) +# comment.md - the Markdown body (omit the file to post nothing) + +on: + workflow_run: + workflows: + - "nf-core linting" + - "nf-core template version comment" + - "nf-core branch protection" + - "Run nf-test" + +permissions: + actions: read + contents: read + pull-requests: write + +jobs: + post-comment: + runs-on: ubuntu-latest + if: github.event.workflow_run.event == 'pull_request' + steps: + - name: Download PR comment artifact + uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21 + with: + run_id: ${{ github.event.workflow_run.id }} + name: pr-comment + path: pr-comment + if_no_artifact_found: ignore + + - name: Read comment metadata + id: meta + run: | + echo "::group::Downloaded pr-comment contents" + ls -la pr-comment 2>/dev/null || echo "No pr-comment/ directory was downloaded." + echo "::endgroup::" + + if [ ! -d pr-comment ]; then + echo "No pr-comment artifact found; nothing to post." + exit 0 + fi + + if [ ! -f pr-comment/comment.md ]; then + echo "Artifact present but no comment.md; nothing to post." + exit 0 + fi + + pr_number=$(cat pr-comment/pr_number.txt) + header=$(cat pr-comment/header.txt) + echo "Found comment.md (header='$header', pr_number='$pr_number')." + + # Guard against anything unexpected ending up in the PR number. + case "$pr_number" in + ''|*[!0-9]*) + echo "Invalid PR number: '$pr_number'" + exit 1 + ;; + esac + + echo "pr_number=$pr_number" >> "$GITHUB_OUTPUT" + echo "header=$header" >> "$GITHUB_OUTPUT" + echo "post=true" >> "$GITHUB_OUTPUT" + echo "Will post comment to PR #${pr_number}." + + - name: Post PR comment + if: steps.meta.outputs.post == 'true' + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + number: ${{ steps.meta.outputs.pr_number }} + header: ${{ steps.meta.outputs.header }} + path: pr-comment/comment.md diff --git a/.github/workflows/template-version-comment.yml b/.github/workflows/template-version-comment.yml index ea30827..ee102f7 100644 --- a/.github/workflows/template-version-comment.yml +++ b/.github/workflows/template-version-comment.yml @@ -2,10 +2,13 @@ name: nf-core template version comment # This workflow is triggered on PRs to check if the pipeline template version matches the latest nf-core version. # It posts a comment to the PR, even if it comes from a fork. -on: pull_request_target +on: + pull_request: + +permissions: {} jobs: - template_version: + check_template_version: runs-on: ubuntu-latest steps: - name: Check out pipeline code @@ -22,25 +25,36 @@ jobs: - name: Install nf-core run: | python -m pip install --upgrade pip - pip install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }} + pip install nf-core + + - name: Build PR comment if template is outdated + # The fork-controlled version is passed via the environment and only ever + # used as quoted shell data (never interpolated into a command), so it + # cannot be used for script injection. + env: + PR_VERSION: ${{ steps.read_yml.outputs['nf_core_version'] }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + mkdir -p pr-comment + echo "$PR_NUMBER" > pr-comment/pr_number.txt + echo "template-version" > pr-comment/header.txt + + latest_version=$(nf-core --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n1) - - name: Check nf-core outdated - id: nf_core_outdated - run: echo "OUTPUT=$(pip list --outdated | grep nf-core)" >> ${GITHUB_ENV} + if [ -n "$PR_VERSION" ] && [ -n "$latest_version" ] && [ "$PR_VERSION" != "$latest_version" ]; then + cat > pr-comment/comment.md < [!WARNING] + > Newer version of the nf-core template is available. + > + > Your pipeline is using an old version of the nf-core template: ${PR_VERSION}. + > Please update your pipeline to the latest version. + > + > For more documentation on how to update your pipeline, please see the [Synchronisation documentation](https://nf-co.re/docs/developing/template-syncs/overview). + EOF + fi - - name: Post nf-core template version comment - uses: mshick/add-pr-comment@8e4927817251f1ff60c001f04568532b38e0b4a0 # v3 - if: | - contains(env.OUTPUT, 'nf-core') + - name: Upload PR comment artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - repo-token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} - allow-repeats: false - message: | - > [!WARNING] - > Newer version of the nf-core template is available. - > - > Your pipeline is using an old version of the nf-core template: ${{ steps.read_yml.outputs['nf_core_version'] }}. - > Please update your pipeline to the latest version. - > - > For more documentation on how to update your pipeline, please see the [Synchronisation documentation](https://nf-co.re/docs/developing/template-syncs/overview). - # + name: pr-comment + path: pr-comment/ diff --git a/.nf-core.yml b/.nf-core.yml index cf24bd7..c75165a 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -2,8 +2,7 @@ nf_core_version: 4.0.2 repository_type: pipeline template: author: Alexander Peltzer - description: A simple sysops pipeline that can be used to synchronize, integrity - check and permanently archive data. + description: A simple sysops pipeline that can be used to synchronize, integrity check and permanently archive data. force: false is_nfcore: true name: datasync @@ -12,3 +11,8 @@ template: version: 1.0dev lint: multiqc_config: false + files_exist: + - .github/workflows/linting_comment.yml + files_unchanged: + - .github/workflows/branch.yml + - .github/workflows/linting.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 01faa82..8438f5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Initial release of nf-core/datasync, created with the [nf-core](https://nf-co.re ### `Added` +- [[#67](https://github.com/nf-core/datasync/pull/67)] - Copy only files which are successfully validated ([@delfiterradas](https://github.com/delfiterradas), review by [@atrigila](https://github.com/atrigila) and [@antoniasaracco](https://github.com/antoniasaracco)). - [[#57](https://github.com/nf-core/datasync/pull/57)] - Add nf-tests with edge cases([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)). - [[#55](https://github.com/nf-core/datasync/pull/55)] - Add subdirectories with sample id to results. Improvements to MultiQC report (width of columns, sections and sub-section headers)colors to multiqc results and display first errors in tables ([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)). - [[#54](https://github.com/nf-core/datasync/pull/54)] - Add colors to multiqc results and display first errors in tables ([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)). @@ -17,6 +18,7 @@ Initial release of nf-core/datasync, created with the [nf-core](https://nf-co.re - [[#46](https://github.com/nf-core/datasync/pull/46)] - Import Rclone module from nf-core([@antoniasaracco](https://github.com/antoniasaracco), review by [@delfiterradas](https://github.com/delfiterradas)). - [[#41](https://github.com/nf-core/datasync/pull/41)] - Generate MultiQC Report with comparechecksum tables and input samplesheet ([@delfiterradas](https://github.com/delfiterradas), review by [@atrigila](https://github.com/atrigila)). - [[#49](https://github.com/nf-core/datasync/pull/49)] - Install `RCLONE_CHECK` and `RCLONE_CHECKSUM` modules from nf-core ([@delfiterradas](https://github.com/delfiterradas), review by [@atrigila](https://github.com/atrigila) and [@antoniasaracco](https://github.com/antoniasaracco)). +- [[#59](https://github.com/nf-core/datasync/pull/59)] - Create pipeline documentation ([@antoniasaracco](https://github.com/antoniasaracco), review by [@atrigila](https://github.com/atrigila) and [@delfiterradas](https://github.com/delfiterradas)). ### `Fixed` diff --git a/README.md b/README.md index 1c76aa4..ce077c1 100644 --- a/README.md +++ b/README.md @@ -21,52 +21,65 @@ ## Introduction -[WIP] WORK IN PROGRESS AND NOT YET STABLE - DO NOT USE FOR PRODUCTIVE SETTINGS YET +> [!WARNING] +> nf-core/datasync is under active development and is not yet recommended for production data transfers. Validate its behaviour with representative data before relying on it. + +**nf-core/datasync** is a Nextflow pipeline for copying files and directories between storage locations and documenting their integrity. For every row in an input samplesheet, the pipeline: + +1. validates the source against a supplied MD5 and/or SHA-256 checksum manifest using [`rclone checksum`](https://rclone.org/commands/rclone_checksum/); +2. copies the source to the requested destination with [`rclone copy`](https://rclone.org/); +3. compares the copied data with the source using [`rclone check`](https://rclone.org/commands/rclone_check/); and +4. produces detailed `rclone` status files and a consolidated MultiQC report. + +Sources and destinations may be local paths, HTTP(S) URLs, or rclone-supported remote storage such as Amazon S3, S3-compatible object storage, or Azure Blob Storage. -**nf-core/datasync** is a system operation pipeline that provides several workflows for handling system operation / automation tasks that are commonly helpful for various tasks in large data processing / analysis facilities. This includes: +The current tested use case for this pipeline is transfer between S3 buckets. -- Data Synchronization & Checksum generation - - Configurable: Can provide YAML file which files to include or exclude from sync - - Checksum backend: Can configure which backend to use for checksum generation (e.g. sha256sum, md5, ...) - - Configurable whether to include (sub-) folders in the sync or not (search for checkpoint files, e.g. has to have DEMUX_DONE that signals a demultiplexing run was finished & successfully copied) -- Data Integrity validation - - Provided with a directory to check, can validate that file(s) found are matching checksums from Synchronization subworkflow -- Data Archival & Deletion - - Can check source and target location for existence of file(s) and decide based on user configurable rules whether files can be considered archived - - Timestamp older than X days - - Checksums match Integrity validation report - - Create empty files to make it obvious that archival was performed - - Optionally: Delete files or create list of files to be deleted for manual deletion process +Pass an `rclone` configuration with `--rclone_config` whenever a source or destination needs a configured remote, endpoint, or credentials. A single configuration file can contain separate named remotes for multiple providers; for non-S3 layouts, design and validate the provider-specific configuration using the upstream [rclone documentation](https://rclone.org/docs/). -The pipeline can be configured by users to execute any of the aforementioned subworkflows and then produces a report using MultiQC custom content that also serves as a report of _what_ was done by the pipeline for documentation purposes. +![nf-core/datasync metro map](docs/images/datasync-metromap.png) -## Usage +## Quick start > [!NOTE] -> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/get_started/environment_setup/overview) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/get_started/run-your-first-pipeline) with `-profile test` before running the workflow on actual data. +> If you are new to Nextflow and nf-core, see the [nf-core environment setup guide](https://nf-co.re/docs/get_started/environment_setup/overview). Nextflow 25.10.4 or later is required. -Now, you can run the pipeline using: +To explore the pipeline outputs before preparing your own data, run the bundled `test` profile with a container profile: ```bash nextflow run nf-core/datasync \ - -profile \ - --input samplesheet.csv \ - --outdir - --sync - --sync_backend 'sha256' - --sync_done true #Creates SYNC_DONE file when done in each folder + -profile test,docker \ + --outdir results ``` -> [!WARNING] -> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; see [docs](https://nf-co.re/docs/running/run-pipelines#using-parameter-files). +The `test` profile supplies a small samplesheet and `rclone` configuration automatically. It also enables `--rclone_dry_run`, so no files are actually transferred. This makes it useful for exploring the `rclone/` output folders and `multiqc/multiqc_report.html`; remember that post-copy comparison reports describe whatever is already present at the destination because the dry run does not write transfer data. + +To run the pipeline on your own data, create a samplesheet containing one transfer per row: + +```csv +sample,input,output_path,checksum_md5,checksum_sha +run_001,/data/run_001,s3://archive/runs,/data/manifests/run_001_md5.tsv +reference,https://example.org/reference.fa,/data/references,,/data/manifests/reference_sha256.tsv +``` + +Then launch the pipeline using: + +```bash +nextflow run nf-core/datasync \ + -r \ + -profile docker \ + --input samplesheet.csv \ + --outdir results \ + --rclone_config /path/to/rclone.conf +``` + +`--rclone_config` is optional only when every source and destination is accessible without a configured rclone remote. See the [`rclone` configuration section](docs/usage.md#configuring-rclone-remotes) for the tested S3-to-S3 use case and guidance on adapting rclone configuration files for other providers. To preview copy operations without transferring data, add `--rclone_dry_run`; note that subsequent comparison reports will then describe the unchanged destination. -For more details and further functionality, please refer to the [usage documentation](https://nf-co.re/datasync/usage) and the [parameter documentation](https://nf-co.re/datasync/parameters). +See the [usage documentation](docs/usage.md) for samplesheet rules, destination semantics, remote configuration, and reproducible execution. The complete generated parameter reference is available on the [nf-core pipeline page](https://nf-co.re/datasync/parameters). ## Pipeline output -To see the results of an example test run with a full size dataset refer to the [results](https://nf-co.re/datasync/results) tab on the nf-core website pipeline page. -For more details about the output files and reports, please refer to the -[output documentation](https://nf-co.re/datasync/output). +Results are written below `--outdir`. See the [output documentation](docs/output.md) for file names and status-code interpretation. ## Credits diff --git a/conf/modules.config b/conf/modules.config index 32a6697..794c4d9 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -31,6 +31,9 @@ process { if (params.rclone_dry_run) { base_args.add('--dry-run') } + if (params.copy_matching_only) { + base_args.add("--include-from ${filter_file}") + } base_args.join(' ') } publishDir = [ @@ -56,6 +59,14 @@ process { } withName: 'RCLONE_CHECKSUM' { + tag = { "${meta.id}_${hash}" } + ext.args = { + def base_args = [ + '--no-check-certificate', + "--one-way" + ] + base_args.join(' ') + } publishDir = [ path: { "${params.outdir}/rclone/checksum/${meta.id}" }, mode: params.publish_dir_mode, diff --git a/docs/images/datasync-metromap.png b/docs/images/datasync-metromap.png new file mode 100644 index 0000000..108bbc0 Binary files /dev/null and b/docs/images/datasync-metromap.png differ diff --git a/docs/images/datasync-multiqc-checksum-md5.png b/docs/images/datasync-multiqc-checksum-md5.png new file mode 100644 index 0000000..42968b7 Binary files /dev/null and b/docs/images/datasync-multiqc-checksum-md5.png differ diff --git a/docs/images/datasync-multiqc-post-transfer-check.png b/docs/images/datasync-multiqc-post-transfer-check.png new file mode 100644 index 0000000..22014f3 Binary files /dev/null and b/docs/images/datasync-multiqc-post-transfer-check.png differ diff --git a/docs/output.md b/docs/output.md index 1b7eb8b..548f2e0 100644 --- a/docs/output.md +++ b/docs/output.md @@ -2,60 +2,136 @@ ## Introduction -This document describes the output produced by the pipeline. Most of the plots are taken from the MultiQC report, which summarises results at the end of the pipeline. +This document describes the reports produced by nf-core/datasync. Paths below are relative to the directory supplied with `--outdir`. + +> [!IMPORTANT] +> The copied payload is written to each samplesheet row's `output_path`. It is not placed in `--outdir` unless `output_path` explicitly points there. + +## Output overview + +```text +/ +├── rclone/ +│ ├── copy/ +│ │ └── -rclone-copy.log +│ ├── checksum/ +│ │ └── / +│ │ ├── .combined.txt +│ │ ├── .match.txt +│ │ ├── .differ.txt +│ │ ├── .missing_on_dst.txt +│ │ ├── .missing_on_src.txt +│ │ └── .error.txt +│ └── check/ +│ └── / +│ ├── .combined.txt +│ ├── .match.txt +│ ├── .differ.txt +│ ├── .missing_on_dst.txt +│ ├── .missing_on_src.txt +│ └── .error.txt +├── multiqc/ +│ ├── multiqc_report.html +│ └── multiqc_data/ +└── pipeline_info/ + ├── nf_core_datasync_software_mqc_versions.yml + └── execution_* / pipeline_dag_* +``` + +The `rclone/` directory is split by module stage. Copy logs are published to `rclone/copy/`, pre-copy checksum validation reports are published to `rclone/checksum//`, and post-copy source-to-destination comparison reports are published to `rclone/check//`. The `` directory name is taken from the `sample` value in the samplesheet row. + +## `rclone` directory -The directories listed below will be created in the results directory after the pipeline has finished. All paths are relative to the top-level results directory. - - +
+Output files -## Pipeline overview +- `rclone/copy/` + - `-rclone-copy.log`: informational log from the copy operation. +- `rclone/checksum//` + - `.combined.txt`: combined pre-copy checksum-validation status, one path per line. + - `.match.txt`: paths whose content matched the supplied checksum manifest (`=`). + - `.differ.txt`: paths present in the source and manifest but with different content (`*`). + - `.missing_on_dst.txt`: paths present in the checksum manifest but absent from the checked source (`-`). + - `.missing_on_src.txt`: paths present in the checked source but absent from the checksum manifest (`+`). + - `.error.txt`: paths that could not be read or hashed (`!`). +- `rclone/check//` + - `.combined.txt`: combined post-copy source-to-destination comparison status, one path per line. + - `.match.txt`: paths whose content matched between source and destination (`=`). + - `.differ.txt`: paths present on both sides but with different content (`*`). + - `.missing_on_dst.txt`: paths found in the source but absent from the destination (`-`). + - `.missing_on_src.txt`: paths found at the destination but absent from the source (`+`). + - `.error.txt`: paths that could not be read or hashed (`!`). -The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes data using the following steps: +
-- [FastQC](#fastqc) - Raw read QC -- [MultiQC](#multiqc) - Aggregate report describing results and QC from the whole pipeline -- [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution +Two integrity stages create reports: -### FastQC +1. **Pre-copy checksum validation** uses each supplied MD5 and/or SHA-256 manifest to check the source and publishes reports under `rclone/checksum//`. +2. **Post-copy validation** compares the source with the destination after the copy task finishes and publishes reports under `rclone/check//`. -
-Output files +Both stages use the same `.*.txt` naming convention and publish to `rclone/`. When a row supplies a checksum manifest, similarly named pre-copy and post-copy files may target the same published path; use the consolidated MultiQC sections for the stage-specific summary and retain the Nextflow work directory if both raw report sets must be audited independently. -- `fastqc/` - - `*_fastqc.html`: FastQC report containing quality metrics. - - `*_fastqc.zip`: Zip archive containing the FastQC report, tab-delimited data file and plot images. +The combined files use `rclone`'s one-character status prefixes: -
+| Prefix | Meaning | Action | +| ------ | ------------------------ | --------------------------------------------------------------------------------------------------------------------------- | +| `=` | File matches | No action required. | +| `-` | Missing from destination | Investigate an incomplete source checksum set or transfer. | +| `+` | Missing from source | Review unexpected destination content. The post-copy check uses `--one-way`, so destination-only files are tolerated there. | +| `*` | Content differs | Re-copy or investigate source/destination mutation. | +| `!` | Read/hash error | Inspect permissions, credentials, connectivity, and the copy log. | -[FastQC](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/) gives general quality metrics about your sequenced reads. It provides information about the quality score distribution across your reads, per base sequence content (%A/T/G/C), adapter contamination and overrepresented sequences. For further reading and documentation see the [FastQC help pages](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/). +Empty category files mean that `rclone` reported no entries in that category. The commands are designed to preserve these reports rather than terminate the whole workflow on comparison differences. Always inspect the reports; workflow success alone is not an integrity guarantee. -### MultiQC +## MultiQC
Output files - `multiqc/` - - `multiqc_report.html`: a standalone HTML file that can be viewed in your web browser. - - `multiqc_data/`: directory containing parsed statistics from the different tools used in the pipeline. - - `multiqc_plots/`: directory containing static images from the report in various formats. + - `multiqc_report.html`: standalone report viewable in a browser. + - `multiqc_data/`: machine-readable data, logs, source inventory, software versions, and parsed rclone tables.
-[MultiQC](http://multiqc.info) is a visualization tool that generates a single HTML report summarising all samples in your project. Most of the pipeline QC results are visualised in the report and further statistics are available in the report data directory. +The MultiQC report consolidates: + +- checksum validation status for MD5 and/or SHA-256 manifests generated from `rclone checksum`; +- post-copy source-to-destination validation status generated from `rclone check`; +- the validated samplesheet and workflow parameter summary; and +- pipeline and tool versions. -Results generated by MultiQC collate pipeline QC from supported tools e.g. FastQC. The pipeline has special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see . +### `rclone checksum` section (source integrity checks) -### Pipeline information +The MD5 and SHA-256 input-validation sections show the results from [`rclone checksum`](https://rclone.org/commands/rclone_checksum/). Use these sections to confirm that each source file matched the checksum manifest supplied in `checksum_md5` and/or `checksum_sha` before copying. + +When a samplesheet `input` points to S3, `rclone checksum` can normally validate MD5 manifests from S3 object hashes, but S3 does not provide SHA-256 object hashes for rclone to read remotely. SHA-256 validation for S3 inputs therefore requires `rclone checksum --download`, which downloads object data and calculates the hash locally during validation. Configure the checksum process to pass `--download` when SHA-256 validation is required for S3 inputs; see the [`rclone checksum` --download documentation](https://rclone.org/commands/rclone_checksum/). + +![nf-core/multiqc checksum md5](images/datasync-multiqc-checksum-md5.png) + +### `rclone check` section (post-transfer checks) + +The source-destination validation section shows the results from `rclone check` after the copy step. Use this section to confirm that copied files at `output_path` match the corresponding source files. + +![nf-core/multiqc checksum md5](images/datasync-multiqc-post-transfer-check.png) + +Open `multiqc_report.html` after every run and investigate any non-matching, missing, or error entries. In the `rclone` result sections, entries are prioritised so rows needing attention are shown before successful matches: errors, mismatches, and missing-file statuses appear ahead of matching files. Data under `multiqc_data/` can be retained for automated auditing or downstream reporting; exact filenames may vary with the MultiQC version and the checksum types present in the samplesheet. + +## Pipeline information
Output files - `pipeline_info/` - - Reports generated by Nextflow: `execution_report.html`, `execution_timeline.html`, `execution_trace.txt` and `pipeline_dag.dot`/`pipeline_dag.svg`. - - Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. The `pipeline_report*` files will only be present if the `--email` / `--email_on_fail` parameter's are used when running the pipeline. - - Reformatted samplesheet files used as input to the pipeline: `samplesheet.valid.csv`. - - Parameters used by the pipeline run: `params.json`. + - `nf_core_datasync_software_mqc_versions.yml`: versions of the pipeline and tools collected for MultiQC. + - `execution_timeline_.html`: chronological task execution view. + - `execution_report_.html`: task runtime and resource report. + - `execution_trace_.txt`: tabular task-level execution trace. + - `pipeline_dag_.html`: workflow dependency graph. + - Completion reports generated when `--email` or `--email_on_fail` is configured may also be present.
-[Nextflow](https://www.nextflow.io/docs/latest/tracing.html) provides excellent functionality for generating various reports relevant to the running and execution of the pipeline. This will allow you to troubleshoot errors with the running of the pipeline, and also provide you with other information such as launch commands, run times and resource usage. +These files provide operational provenance and help diagnose performance or failures. Archive them with the MultiQC and rclone reports. The Nextflow `work/` directory and `.nextflow.log` remain in the launch directory rather than `--outdir`; keep them until transfer verification is complete if detailed troubleshooting or `-resume` may be needed. + +The supplied rclone configuration is an input credential file and is not intentionally copied to `--outdir`. Nevertheless, execution logs may contain remote names and object paths. Review logs before sharing them, and manage `rclone.conf` separately as a secret. diff --git a/docs/usage.md b/docs/usage.md index 1c85a3e..2e7c6fb 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -2,65 +2,157 @@ ## :warning: Please read this documentation on the nf-core website: [https://nf-co.re/datasync/usage](https://nf-co.re/datasync/usage) -> _Documentation of pipeline parameters is generated automatically from the pipeline schema and can no longer be found in markdown files._ +> Pipeline parameter documentation is generated automatically from [`nextflow_schema.json`](../nextflow_schema.json). This page explains how to prepare a transfer and operate the pipeline. -## Introduction +## Prerequisites - +Install Nextflow 25.10.4 or later and use a supported software profile. Docker or Singularity/Apptainer is recommended for reproducibility. Ensure that the account running Nextflow can read each source and checksum manifest and can write to every destination. + +For cloud or other authenticated rclone remotes, create an [rclone configuration](https://rclone.org/docs/) and pass it with `--rclone_config`. The configuration applies to remote **sources and destinations**. The pipeline has currently been tested for transfers between S3 buckets. Other rclone-supported layouts, such as Azure Blob Storage to S3 or transfers between S3-compatible providers, should be configured and validated against the upstream `rclone` documentation for each provider before use. ## Samplesheet input -You will need to create a samplesheet with information about the samples you would like to analyse before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row as shown in the examples below. +Supply a comma-separated samplesheet with `--input`: ```bash ---input '[path to samplesheet file]' +--input /path/to/samplesheet.csv ``` -### Multiple runs of the same sample +Each row describes an independent transfer. The header names are fixed; columns may be in any order. + +| Column | Required | Description | +| -------------- | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `sample` | Yes | Unique identifier used in task labels and output report names. It must not contain whitespace. Use a unique value for each row to prevent published report files from colliding. | +| `input` | Yes | Source file or directory. This can be a local path, HTTP(S) URL, object-storage URL such as `s3://bucket/prefix`, or configured remote such as `source_s3:bucket/prefix` or `source_azure:container/prefix`. Whitespace is not allowed. | +| `output_path` | Yes | Destination directory understood by rclone, such as `/archive/runs`, `s3://bucket/prefix`, or a configured `remote:path`. Whitespace is not allowed. | +| `checksum_md5` | One checksum column | Path or URL to an MD5 checksum manifest used to validate `input` before copying. The manifest format is described below. Leave empty when using SHA-256 only. | +| `checksum_sha` | One checksum column | Path or URL to a SHA-256 checksum manifest used to validate `input` before copying. The manifest format is described below. Leave empty when using MD5 only. | + +At least one checksum manifest is required on every row. If both are supplied, both validations run. Checksum files must use the format accepted by [`rclone checksum`](https://rclone.org/commands/rclone_checksum/): one checksum record per line with the hash value followed by two spaces and then the file path. Paths must be relative to the source root from the `input` column, not absolute paths. + +For a directory input, the source root is the directory named in the samplesheet. For example, if the samplesheet `input` is `/data/run_001` and one file in that directory is `/data/run_001/reads/sample_R1.fastq.gz`, the checksum manifest path must be `reads/sample_R1.fastq.gz`. Do not write `/data/run_001/reads/sample_R1.fastq.gz` in the manifest. For a single-file input, use the input file name as the manifest path. -The `sample` identifiers have to be the same when you have re-sequenced the same sample more than once e.g. to increase sequencing depth. The pipeline will concatenate the raw reads before performing any downstream analysis. Below is an example for the same sample sequenced across 3 lanes: +Checksum manifests may use a `.tsv` or `.csv` filename extension, but their contents are not tab-separated or comma-separated tables and must not include a header. Each record is plain text with the hash and path separated by exactly two spaces. The required fields are: + +| Field | Required | Description | +| ----- | -------- | ----------------------------------------------------------------------------------------------- | +| Hash | Yes | MD5 hash for `checksum_md5` files or SHA-256 hash for `checksum_sha` files. | +| Path | Yes | Relative path to the file being validated, resolved from the corresponding `input` source root. | + +Example samplesheet: ```csv title="samplesheet.csv" -sample,fastq_1,fastq_2 -CONTROL_REP1,AEG588A1_S1_L002_R1_001.fastq.gz,AEG588A1_S1_L002_R2_001.fastq.gz -CONTROL_REP1,AEG588A1_S1_L003_R1_001.fastq.gz,AEG588A1_S1_L003_R2_001.fastq.gz -CONTROL_REP1,AEG588A1_S1_L004_R1_001.fastq.gz,AEG588A1_S1_L004_R2_001.fastq.gz +sample,input,output_path,checksum_md5,checksum_sha +run_001,/data/run_001,s3://archive/runs,/data/checksums/run_001_md5.tsv, +reference,https://example.org/reference.fa,/data/references,,/data/checksums/reference_sha256.tsv +run_002,/data/run_002,archive:runs,/data/checksums/run_002_md5.tsv,/data/checksums/run_002_sha256.tsv ``` -### Full samplesheet +For the `run_001` directory example, `/data/checksums/run_001_md5.tsv` could contain: -The pipeline will auto-detect whether a sample is single- or paired-end using the information provided in the samplesheet. The samplesheet can have as many columns as you desire, however, there is a strict requirement for the first 3 columns to match those defined in the table below. +```text title="run_001_md5.tsv" +d41d8cd98f00b204e9800998ecf8427e reads/sample_R1.fastq.gz +0cc175b9c0f1b6a831c399e269772661 reads/sample_R2.fastq.gz +900150983cd24fb0d6963f7d28e17f72 reports/qc_summary.txt +``` -A final samplesheet file consisting of both single- and paired-end data may look something like the one below. This is for 6 samples, where `TREATMENT_REP3` has been sequenced twice. +For a SHA-256 manifest, the same relative paths are used with SHA-256 hashes: -```csv title="samplesheet.csv" -sample,fastq_1,fastq_2 -CONTROL_REP1,AEG588A1_S1_L002_R1_001.fastq.gz,AEG588A1_S1_L002_R2_001.fastq.gz -CONTROL_REP2,AEG588A2_S2_L002_R1_001.fastq.gz,AEG588A2_S2_L002_R2_001.fastq.gz -CONTROL_REP3,AEG588A3_S3_L002_R1_001.fastq.gz,AEG588A3_S3_L002_R2_001.fastq.gz -TREATMENT_REP1,AEG588A4_S4_L003_R1_001.fastq.gz, -TREATMENT_REP2,AEG588A5_S5_L003_R1_001.fastq.gz, -TREATMENT_REP3,AEG588A6_S6_L003_R1_001.fastq.gz, -TREATMENT_REP3,AEG588A6_S6_L004_R1_001.fastq.gz, +```text title="run_001_sha256.tsv" +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 reads/sample_R1.fastq.gz +ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb reads/sample_R2.fastq.gz +ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad reports/qc_summary.txt +``` + +An [example samplesheet](../assets/samplesheet.csv) is included in the repository. + +## Configuring `rclone` remotes + +The file supplied with `--rclone_config` uses `rclone`'s INI-style format. + +Each `[name]` section defines a remote, and samplesheet paths refer to it as `name:path`. The remote name is an arbitrary local label; it does not need to match the provider or bucket name. + +> [!NOTE] +> The pipeline's documented and tested configuration pattern is S3-to-S3 transfer. + +One file may contain several sections, so other cloud-to-cloud transfers can define both providers in the same file, but provider-specific options should be taken from the relevant `rclone` documentation. This is an example using S3: + +```text +source_s3:incoming/run_001 +archive_azure:research-archive/run_001 +``` + +Create the file interactively where possible: + +```bash +rclone config --config /secure/rclone.conf +rclone listremotes --config /secure/rclone.conf +``` + +Then provide that exact file to the pipeline: + +```bash +nextflow run nf-core/datasync \ + -profile docker \ + --input samplesheet.csv \ + --outdir results \ + --rclone_config /secure/rclone.conf +``` + +### S3 and S3-compatible storage + +The main use case tested for nf-core/datasync is transferring files between S3 buckets. An S3 remote specifies the provider, region, and credentials. For example: + +```ini title="rclone.conf" +[s3] +type = s3 +provider = AWS +access_key_id = YOUR_ACCESS_KEY_ID +secret_access_key = YOUR_SECRET_ACCESS_KEY +region = eu-central-1 ``` -| Column | Description | -| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `sample` | Custom sample name. This entry will be identical for multiple sequencing libraries/runs from the same sample. Spaces in sample names are automatically converted to underscores (`_`). | -| `fastq_1` | Full path to FastQ file for Illumina short reads 1. File has to be gzipped and have the extension ".fastq.gz" or ".fq.gz". | -| `fastq_2` | Full path to FastQ file for Illumina short reads 2. File has to be gzipped and have the extension ".fastq.gz" or ".fq.gz". | +The corresponding input values could be `source_s3:incoming/run_001` and `institutional_s3:project/run_002`. Provider-specific settings vary: consult the [`rclone` S3 documentation](https://rclone.org/s3/) and your storage provider's endpoint, region, addressing-style, and credential documentation rather than copying example values unchanged. -An [example samplesheet](../assets/samplesheet.csv) has been provided with the pipeline. +The pipeline also accepts an `s3://bucket/path` source or destination. In that form, ensure credentials and provider settings are available to both Nextflow and `rclone` in the execution environment. A named remote such as `source_s3:bucket/path` makes the selected configuration section explicit and is preferable when a config file contains multiple S3 providers. + +## Destination layout + +The pipeline preserves the source basename: + +- for a file source, `rclone` copies the file into `output_path`, and validation expects `output_path/`; +- for a directory source, the pipeline appends the source directory name, so `/data/run_001` with `output_path=/archive/runs` is copied and checked at `/archive/runs/run_001`. + +A trailing slash on `output_path` is removed before these paths are constructed. Ensure that a destination does not already contain unrelated files: post-copy validation uses `rclone check --one-way`, which checks that source content exists and matches at the destination while tolerating destination-only files. You can override this behavior by providing your own config file with external arguments for `rclone check`. ## Running the pipeline -The typical command for running the pipeline is as follows: +A typical local-to-cloud run is: + +```bash +nextflow run nf-core/datasync \ + -r \ + -profile docker \ + --input /data/samplesheet.csv \ + --outdir /data/datasync-results \ + --rclone_config /secure/rclone.conf +``` + +`--outdir` stores logs, integrity reports, MultiQC, and execution metadata. It does **not** override the transfer destinations in the samplesheet. + +To inspect the proposed copy without writing destination data: ```bash -nextflow run nf-core/datasync --input ./samplesheet.csv --outdir ./results --genome GRCh37 -profile docker +nextflow run nf-core/datasync \ + -r \ + -profile docker \ + --input /data/samplesheet.csv \ + --outdir /data/datasync-dry-run \ + --rclone_config /secure/rclone.conf \ + --rclone_dry_run ``` -This will launch the pipeline with the `docker` configuration profile. See below for more information about profiles. +The checksum and post-copy check stages still run during a dry run. Consequently, post-copy results reflect whatever was already present at the destination rather than a simulated final state. Note that the pipeline will create the following files in your working directory: @@ -95,14 +187,86 @@ genome: 'GRCh37' You can also generate such `YAML`/`JSON` files via [nf-core/launch](https://nf-co.re/launch). -### Updating the pipeline +### Parameter files -When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: +Frequently reused settings can be stored in YAML or JSON and loaded with `-params-file`: + +```yaml title="params.yaml" +input: /data/samplesheet.csv +outdir: /data/datasync-results +rclone_config: /secure/rclone.conf +multiqc_title: July archive transfer +``` ```bash -nextflow pull nf-core/datasync +nextflow run nf-core/datasync -r -profile docker -params-file params.yaml +``` + +Do not use `-c` for pipeline parameters. Use it only for Nextflow executor, resources, and other infrastructure configuration. + +### Common integrity outcomes + +The workflow is designed to collect `rclone` reports even when `rclone` detects differences. The table below summarises common edge cases and how to interpret them in the published reports and MultiQC. + +| Situation | Where it is detected | Report status | Pipeline behaviour and action | +| ------------------------------------------------------------------ | ---------------------------------------------------------------- | --------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| File exists and checksum/content matches | `rclone checksum` before copy and `rclone check` after copy | `=` / `Match` | Expected result; no action needed. | +| File is listed in the checksum manifest but absent from the source | Pre-copy `rclone checksum` | `-` / missing from checked source | The report is retained for review. Fix the manifest or restore the missing source file before relying on the transfer. | +| Source file exists but is absent from the checksum manifest | Pre-copy `rclone checksum` | `+` / missing from manifest | Review whether the manifest is incomplete or whether the extra source file should be excluded from the transfer. | +| Source file hash differs from the supplied manifest | Pre-copy `rclone checksum` | `*` / mismatch | Investigate source mutation, stale manifests, or incorrect checksum files before accepting the copy. | +| Source file cannot be read or hashed | Pre-copy `rclone checksum` | `!` / error | Inspect credentials, permissions, connectivity, and source path spelling. | +| Destination is missing a copied file | Post-copy `rclone check` | `-` / missing from destination | Treat as an incomplete transfer unless the file was intentionally excluded; re-run or inspect the rclone copy log. | +| Destination file exists but content differs from source | Post-copy `rclone check` | `*` / mismatch | Re-copy or investigate concurrent source/destination changes. | +| Destination contains files absent from the source | Post-copy `rclone check` | `+` / missing from source | The post-copy check uses `--one-way`, so destination-only files are tolerated, but should still be reviewed for unexpected stale or unrelated data. | +| Dry-run execution | Copy step uses `--dry-run`; checksum and check reports still run | Depends on existing destination | No transfer data is written. Post-copy reports describe whatever was already present at the destination. | + +### Including or excluding files + +Filter files by passing additional `rclone` filter flags to the relevant rclone module through a Nextflow configuration file. `rclone` supports flags such as `--include`, `--exclude`, `--filter`, `--files-from`, and related rule files; see the [`rclone` filtering documentation](https://rclone.org/filtering/) for rule syntax and ordering. + +For example, to copy and check only FASTQ files while excluding temporary files, create a small infrastructure config: + +```groovy title="rclone_filters.config" +process { + withName: 'RCLONE_COPY' { + ext.args = { + [ + '--log-level INFO', + '--stats 30s', + '--stats-one-line', + '--stats-log-level INFO', + '--s3-chunk-size 64M', + '--no-check-certificate', + params.rclone_dry_run ? '--dry-run' : '', + '--include "*.fastq.gz"', + '--include "*.fq.gz"', + '--exclude "*.tmp"', + '--exclude "*"' + ].findAll { it }.join(' ') + } + } + + withName: 'RCLONE_CHECK' { + ext.args = { + [ + '--no-check-certificate', + '--one-way', + '--include "*.fastq.gz"', + '--include "*.fq.gz"', + '--exclude "*.tmp"', + '--exclude "*"' + ].join(' ') + } + } +} ``` +Run it with `-c rclone_filters.config` in addition to your normal profile and parameters. Because `ext.args` overrides module defaults, include the default `rclone` flags you still need when adding filters. Keep checksum manifests consistent with the same filtering rules: if a file is intentionally excluded from copy/check, remove it from the checksum manifest or generate a manifest for only the included files. + +## Understanding completion and integrity + +For each row, the pipeline first validates supplied checksum manifests, performs the copy, and then compares source and destination. `rclone` comparison commands write status reports even when differences are found, allowing all results to be collected in MultiQC. Therefore, a successful Nextflow run means the workflow completed; it does **not by itself** prove every object matched. Review `multiqc/multiqc_report.html` and the reports under `rclone/`, especially lines marked `-`, `+`, `*`, or `!` (see [output documentation](output.md)). + ### Reproducibility It is a good idea to specify the pipeline version when running the pipeline on your data. This ensures that a specific version of the pipeline code and software are used when you run your pipeline. If you keep using the same tag, you'll be running the same version of the pipeline, even if there have been changes to the code since. @@ -171,8 +335,19 @@ Specify the path to a specific config file (this is a core Nextflow command). Se ### Resource requests -Whilst the default requirements set within the pipeline will hopefully work for most people and with most input data, you may find that you want to customise the compute resources that the pipeline requests. Each step in the pipeline has a default set of requirements for number of CPUs, memory and time. For most of the pipeline steps, if the job exits with any of the error codes specified [here](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L18) it will automatically be resubmitted with higher resources request (2 x original, then 3 x original). If it still fails after the third attempt then the pipeline execution is stopped. +The `rclone` processes use the `process_low` label. Configure executors and override CPU, memory, or time in a Nextflow config, for example: +```groovy title="resources.config" +process { + withLabel: process_low { + cpus = 8 + memory = '16 GB' + time = '12h' + } +} +``` + +Run with `-c resources.config`. `rclone` derives its checker count from allocated CPUs, and the copy step uses roughly half that count (minimum one) for parallel transfers. To change the resource requests, please see the [max resources](https://nf-co.re/docs/running/configuration/nextflow-for-your-system#set-max-resources) and [customise process resources](https://nf-co.re/docs/running/configuration/nextflow-for-your-system#customize-process-resources) section of the nf-core website. ### Custom Containers @@ -194,21 +369,3 @@ In most cases, you will only need to create a custom config as a one-off but if See the main [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for more information about creating your own configuration files. If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack) on the [`#configs` channel](https://nfcore.slack.com/channels/configs). - -## Running in the background - -Nextflow handles job submissions and supervises the running jobs. The Nextflow process must run until the pipeline is finished. - -The Nextflow `-bg` flag launches Nextflow in the background, detached from your terminal so that the workflow does not stop if you log out of your session. The logs are saved to a file. - -Alternatively, you can use `screen` / `tmux` or similar tool to create a detached session which you can log back into at a later time. -Some HPC setups also allow you to run nextflow within a cluster job submitted your job scheduler (from where it submits more jobs). - -## Nextflow memory requirements - -In some cases, the Nextflow Java virtual machines can start to request a large amount of memory. -We recommend adding the following line to your environment to limit this (typically in `~/.bashrc` or `~./bash_profile`): - -```bash -NXF_OPTS='-Xms1g -Xmx4g' -``` diff --git a/modules.json b/modules.json index 36c8de3..df30276 100644 --- a/modules.json +++ b/modules.json @@ -12,17 +12,17 @@ }, "rclone/check": { "branch": "master", - "git_sha": "0983b1ef5f0729946953d921100af925f86fafb0", + "git_sha": "a554e54c63c3f03f91c88a18ece7b89eca5a5212", "installed_by": ["modules"] }, "rclone/checksum": { "branch": "master", - "git_sha": "0983b1ef5f0729946953d921100af925f86fafb0", + "git_sha": "a554e54c63c3f03f91c88a18ece7b89eca5a5212", "installed_by": ["modules"] }, "rclone/copy": { "branch": "master", - "git_sha": "969013a2a91dc8152ca537bbd4dd4feaa15c13a7", + "git_sha": "b34acd9d361bb226be8f0cf446b8b3a20ad0e3da", "installed_by": ["modules"] } } diff --git a/modules/nf-core/rclone/check/main.nf b/modules/nf-core/rclone/check/main.nf index 56b2a67..ee1d36b 100644 --- a/modules/nf-core/rclone/check/main.nf +++ b/modules/nf-core/rclone/check/main.nf @@ -8,7 +8,8 @@ process RCLONE_CHECK { : 'community.wave.seqera.io/library/rclone:1.74.3--2ef33c5b9132aa97' }" input: - tuple val(meta), path(source, stageAs: 'source/*'), path(destination, stageAs: 'destination/*') + tuple val(meta), val(source), val(destination) + path rclone_config output: tuple val(meta), path("${prefix}.combined.txt") , emit: combined , optional: true @@ -17,6 +18,7 @@ process RCLONE_CHECK { tuple val(meta), path("${prefix}.missing_on_src.txt") , emit: missing_on_src, optional: true tuple val(meta), path("${prefix}.match.txt") , emit: match , optional: true tuple val(meta), path("${prefix}.error.txt") , emit: error , optional: true + tuple val(meta), path("${prefix}.exit_code.txt") , emit: exit_code , optional: true tuple val("${task.process}"), val('rclone'), eval("rclone --version | sed -n '1s/^rclone v//p'"), topic: versions, emit: versions_rclone when: @@ -25,10 +27,10 @@ process RCLONE_CHECK { script: def args = task.ext.args ?: '' prefix = task.ext.prefix ?: "${meta.id}" + def configArg = rclone_config ? "--config ${rclone_config}" : '' """ - rclone check \\ - --copy-links \\ + rclone check ${configArg} \\ $args \\ --combined ${prefix}.combined.txt \\ --differ ${prefix}.differ.txt \\ @@ -37,8 +39,13 @@ process RCLONE_CHECK { --match ${prefix}.match.txt \\ --error ${prefix}.error.txt \\ --checkers $task.cpus \\ - source \\ - destination || true + ${source} \\ + ${destination} || echo \$? > ${prefix}.exit_code.txt + + # Do not emit empty output files + for f in *.txt; do + [ -s "\$f" ] || rm -f "\$f" + done """ stub: diff --git a/modules/nf-core/rclone/check/meta.yml b/modules/nf-core/rclone/check/meta.yml index 05766b3..2ae5d92 100644 --- a/modules/nf-core/rclone/check/meta.yml +++ b/modules/nf-core/rclone/check/meta.yml @@ -21,17 +21,29 @@ input: Groovy Map containing source information e.g. [ id:'test' ] - source: - type: file - description: File or directory containing source files to compare. - pattern: "" - ontologies: - - edam: "http://edamontology.org/data_0006" + type: string + description: | + File or directory containing source files to compare. This should be a path understood by Rclone, + such as a local path or a configured remote path. + Examples: `data/input`, `s3:bucket/path`, `gs:bucket/path`, + `remote:path/to/data`. - destination: - type: file - description: File or directory containing destination files to compare. - pattern: "" - ontologies: - - edam: "http://edamontology.org/data_0006" + type: string + description: | + Directory containing destination files to compare. This should be the fully resolved target + path understood by Rclone, such as a local path or a configured remote + path. The module does not append sample IDs, basenames, or modify this + path internally. + Examples: `results/output`, `s3:bucket/output`, `gs:bucket/output`, + `remote:path/to/output`. + - rclone_config: + type: file + description: | + Rclone configuration file defining the remotes used by source_path and/or + destination_path. Authentication and remote-specific options should be + configured in this file where possible. + pattern: "*.conf" + ontologies: [] output: combined: - - meta: @@ -105,6 +117,18 @@ output: pattern: "*.error.txt" ontologies: - edam: "http://edamontology.org/data_3671" + exit_code: + - - meta: + type: map + description: | + Groovy Map containing checksum source information + e.g. [ id:'test' ] + - ${prefix}.exit_code.txt: + type: file + description: File with exit status. + pattern: "*.exit_code.txt" + ontologies: + - edam: "http://edamontology.org/data_3671" versions_rclone: - - ${task.process}: type: string diff --git a/modules/nf-core/rclone/check/tests/main.nf.test b/modules/nf-core/rclone/check/tests/main.nf.test index 093265e..312d631 100644 --- a/modules/nf-core/rclone/check/tests/main.nf.test +++ b/modules/nf-core/rclone/check/tests/main.nf.test @@ -3,17 +3,21 @@ nextflow_process { name "Test Process RCLONE_CHECK" script "../main.nf" process "RCLONE_CHECK" + config "./nextflow.config" tag "modules" tag "modules_nfcore" tag "rclone" tag "rclone/check" - test("sarscov2 - fastq") { + test("match - malt") { when { process { """ - input[0] = [[id:'test'], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true)] + file('rclone.config').text = '[s3]\\ntype = s3\\nprovider = AWS\\nregion = eu-west-1\\nenv_auth = false\\nno_check_bucket = true' + + input[0] = [[id:'test'], "s3://nf-core-test-datasets/modules/data/delete_me/malt/", "s3://nf-core-test-datasets/modules/data/delete_me/malt/"] + input[1] = file('rclone.config') """ } } @@ -21,7 +25,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(sanitizeOutput(process.out, unstableKeys: ["combined", "differ", "missing_on_dst", "missing_on_src", "error"])).match() } + { assert snapshot(sanitizeOutput(process.out, unstableKeys: ["combined", "differ", "match", "missing_on_dst", "missing_on_src", "error"])).match() } ) } } @@ -32,7 +36,8 @@ nextflow_process { when { process { """ - input[0] = [[id:'test'], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true)] + input[0] = [[id:'test'], params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/', params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/'] + input[1] = [] """ } } @@ -45,11 +50,14 @@ nextflow_process { } } - test("different files - fails") { + test("different files") { when { process { """ - input[0] = [[id:'test'], file(params.modules_testdata_base_path + 'delete_me/aracne3/README.md', checkIfExists: true), file(params.modules_testdata_base_path + 'delete_me/binette_checkm2/README.md', checkIfExists: true)] + file('rclone.config').text = '[https]\\ntype = http\\n[s3]\\ntype = s3\\nprovider = AWS\\nregion = eu-west-1\\nenv_auth = false\\nno_check_bucket = true' + + input[0] = [[id:'test'], "s3://nf-core-test-datasets/modules/data/delete_me/malt/", "s3://annotation-cache/snpeff_cache/WBcel235.105/"] + input[1] = file('rclone.config') """ } } @@ -62,11 +70,14 @@ nextflow_process { } } - test("destination subset - partial match and missing") { + test("differ and missing") { when { process { """ - input[0] = [[id:'test'], [file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true)], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true)] + file('rclone.config').text = '[https]\\ntype = http\\n[s3]\\ntype = s3\\nprovider = AWS\\nregion = eu-west-1\\nenv_auth = false\\nno_check_bucket = true' + + input[0] = [[id:'test'], "s3://nf-core-test-datasets/modules/data/delete_me/mirtop/", "s3://nf-core-test-datasets/modules/data/delete_me/metabuli"] + input[1] = file('rclone.config') """ } } @@ -74,7 +85,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(sanitizeOutput(process.out, unstableKeys: ["combined", "differ", "missing_on_src", "error"])).match() } + { assert snapshot(sanitizeOutput(process.out, unstableKeys: ["combined", "differ", "missing_on_dst", "missing_on_src", "error"])).match() } ) } } diff --git a/modules/nf-core/rclone/check/tests/main.nf.test.snap b/modules/nf-core/rclone/check/tests/main.nf.test.snap index 5d14e75..031a7c7 100644 --- a/modules/nf-core/rclone/check/tests/main.nf.test.snap +++ b/modules/nf-core/rclone/check/tests/main.nf.test.snap @@ -1,5 +1,5 @@ { - "sarscov2 - fastq - stub": { + "match - malt": { "content": [ { "combined": [ @@ -11,20 +11,13 @@ ] ], "differ": [ - [ - { - "id": "test" - }, - "test.differ.txt" - ] + ], "error": [ - [ - { - "id": "test" - }, - "test.error.txt" - ] + + ], + "exit_code": [ + ], "match": [ [ @@ -35,20 +28,10 @@ ] ], "missing_on_dst": [ - [ - { - "id": "test" - }, - "test.missing_on_dst.txt" - ] + ], "missing_on_src": [ - [ - { - "id": "test" - }, - "test.missing_on_src.txt" - ] + ], "versions_rclone": [ [ @@ -59,13 +42,13 @@ ] } ], - "timestamp": "2026-07-10T21:39:36.202209966", + "timestamp": "2026-07-22T14:46:30.042575576", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.6" + "nextflow": "26.04.1" } }, - "different files - fails": { + "sarscov2 - fastq - stub": { "content": [ { "combined": [ @@ -81,7 +64,7 @@ { "id": "test" }, - "test.differ.txt:md5,ac036608c5c402f92926505d0f0ab0e4" + "test.differ.txt" ] ], "error": [ @@ -91,6 +74,9 @@ }, "test.error.txt" ] + ], + "exit_code": [ + ], "match": [ [ @@ -125,13 +111,13 @@ ] } ], - "timestamp": "2026-07-10T21:39:43.336064968", + "timestamp": "2026-07-21T21:03:37.410543015", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.6" + "nextflow": "26.04.1" } }, - "sarscov2 - fastq": { + "different files": { "content": [ { "combined": [ @@ -143,28 +129,21 @@ ] ], "differ": [ - [ - { - "id": "test" - }, - "test.differ.txt" - ] + ], "error": [ + + ], + "exit_code": [ [ { "id": "test" }, - "test.error.txt" + "test.exit_code.txt:md5,b026324c6904b2a9cb4b88d6d61c81d1" ] ], "match": [ - [ - { - "id": "test" - }, - "test.match.txt:md5,6f2d815bc8fe93026183cc59946ffc46" - ] + ], "missing_on_dst": [ [ @@ -191,13 +170,13 @@ ] } ], - "timestamp": "2026-07-10T21:39:29.273416483", + "timestamp": "2026-07-21T21:03:45.979925514", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.6" + "nextflow": "26.04.1" } }, - "destination subset - partial match and missing": { + "differ and missing": { "content": [ { "combined": [ @@ -217,27 +196,25 @@ ] ], "error": [ + + ], + "exit_code": [ [ { "id": "test" }, - "test.error.txt" + "test.exit_code.txt:md5,b026324c6904b2a9cb4b88d6d61c81d1" ] ], "match": [ - [ - { - "id": "test" - }, - "test.match.txt:md5,6f2d815bc8fe93026183cc59946ffc46" - ] + ], "missing_on_dst": [ [ { "id": "test" }, - "test.missing_on_dst.txt:md5,827cbdd053152d88295a331ab8bae87c" + "test.missing_on_dst.txt" ] ], "missing_on_src": [ @@ -257,10 +234,10 @@ ] } ], - "timestamp": "2026-07-13T17:42:54.497150157", + "timestamp": "2026-07-22T14:46:52.993591106", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.6" + "nextflow": "26.04.1" } } } \ No newline at end of file diff --git a/modules/nf-core/rclone/check/tests/nextflow.config b/modules/nf-core/rclone/check/tests/nextflow.config new file mode 100644 index 0000000..b208cef --- /dev/null +++ b/modules/nf-core/rclone/check/tests/nextflow.config @@ -0,0 +1,7 @@ +process { + withName: 'RCLONE_CHECK' { + ext.args = { + "--no-check-certificate" + } + } +} diff --git a/modules/nf-core/rclone/checksum/main.nf b/modules/nf-core/rclone/checksum/main.nf index 71a21f1..bef4c5e 100644 --- a/modules/nf-core/rclone/checksum/main.nf +++ b/modules/nf-core/rclone/checksum/main.nf @@ -8,7 +8,8 @@ process RCLONE_CHECKSUM { : 'community.wave.seqera.io/library/rclone:1.74.3--2ef33c5b9132aa97' }" input: - tuple val(meta), path(sumfile), val(hash), path(destination, stageAs: 'destination/*') + tuple val(meta), path(sumfile), val(hash), val(destination) + path rclone_config output: tuple val(meta), path("${prefix}.combined.txt") , emit: combined , optional: true @@ -17,6 +18,7 @@ process RCLONE_CHECKSUM { tuple val(meta), path("${prefix}.missing_on_src.txt") , emit: missing_on_src, optional: true tuple val(meta), path("${prefix}.match.txt") , emit: match , optional: true tuple val(meta), path("${prefix}.error.txt") , emit: error , optional: true + tuple val(meta), path("${prefix}.exit_code.txt") , emit: exit_code , optional: true tuple val("${task.process}"), val('rclone'), eval("rclone --version | sed -n '1s/^rclone v//p'"), topic: versions, emit: versions_rclone when: @@ -25,9 +27,10 @@ process RCLONE_CHECKSUM { script: def args = task.ext.args ?: '' prefix = task.ext.prefix ?: "${meta.id}" + def configArg = rclone_config ? "--config ${rclone_config}" : '' """ - rclone checksum \\ + rclone checksum ${configArg} \\ --copy-links \\ $args \\ --combined ${prefix}.combined.txt \\ @@ -39,7 +42,12 @@ process RCLONE_CHECKSUM { --checkers $task.cpus \\ $hash \\ $sumfile \\ - destination || true + ${destination} || echo \$? > ${prefix}.exit_code.txt + + # Do not emit empty output files + for f in *.txt; do + [ -s "\$f" ] || rm -f "\$f" + done """ stub: diff --git a/modules/nf-core/rclone/checksum/meta.yml b/modules/nf-core/rclone/checksum/meta.yml index 8239375..40a8f87 100644 --- a/modules/nf-core/rclone/checksum/meta.yml +++ b/modules/nf-core/rclone/checksum/meta.yml @@ -15,7 +15,6 @@ tools: licence: - "MIT" identifier: "" - input: - - meta: type: map @@ -24,7 +23,8 @@ input: e.g. [ id:'test' ] - sumfile: type: file - description: Checksum SUM file containing hashes and destination paths to verify. + description: Checksum SUM file containing hashes and destination paths to + verify. pattern: "*.{md5,sha1,sha256,sum,txt}" ontologies: - edam: "http://edamontology.org/data_3671" @@ -32,11 +32,22 @@ input: type: string description: Hash type to check, for example MD5, SHA1, or SHA256. - destination: - type: file - description: File or directory containing destination files to check against the SUM file. - pattern: "" - ontologies: - - edam: "http://edamontology.org/data_0006" + type: string + description: | + Directory containing destination files to check against the SUM file. This should be the fully resolved target + path understood by Rclone, such as a local path or a configured remote + path. The module does not append sample IDs, basenames, or modify this + path internally. + Examples: `results/output`, `s3:bucket/output`, `gs:bucket/output`, + `remote:path/to/output`. + - rclone_config: + type: file + description: | + Rclone configuration file defining the remotes used by source_path and/or + destination_path. Authentication and remote-specific options should be + configured in this file where possible. + pattern: "*.conf" + ontologies: [] output: combined: - - meta: @@ -46,7 +57,8 @@ output: e.g. [ id:'test' ] - ${prefix}.combined.txt: type: file - description: Combined report of matching, missing, differing, and error paths. + description: Combined report of matching, missing, differing, and error + paths. pattern: "*.combined.txt" ontologies: - edam: "http://edamontology.org/data_3671" @@ -58,7 +70,8 @@ output: e.g. [ id:'test' ] - ${prefix}.differ.txt: type: file - description: Report of paths present in both source SUM file and destination but different. + description: Report of paths present in both source SUM file and + destination but different. pattern: "*.differ.txt" ontologies: - edam: "http://edamontology.org/data_3671" @@ -70,7 +83,8 @@ output: e.g. [ id:'test' ] - ${prefix}.missing_on_dst.txt: type: file - description: Report of paths present in the SUM file but missing from the destination. + description: Report of paths present in the SUM file but missing from + the destination. pattern: "*.missing_on_dst.txt" ontologies: - edam: "http://edamontology.org/data_3671" @@ -82,7 +96,8 @@ output: e.g. [ id:'test' ] - ${prefix}.missing_on_src.txt: type: file - description: Report of paths present in the destination but missing from the SUM file. + description: Report of paths present in the destination but missing from + the SUM file. pattern: "*.missing_on_src.txt" ontologies: - edam: "http://edamontology.org/data_3671" @@ -120,6 +135,18 @@ output: - rclone --version | sed -n '1s/^rclone v//p': type: eval description: The expression to obtain the version of the tool + exit_code: + - - meta: + type: map + description: | + Groovy Map containing checksum source information + e.g. [ id:'test' ] + - ${prefix}.exit_code.txt: + type: file + description: File with exit status. + pattern: "*.exit_code.txt" + ontologies: + - edam: "http://edamontology.org/data_3671" topics: versions: - - ${task.process}: diff --git a/modules/nf-core/rclone/checksum/tests/main.nf.test b/modules/nf-core/rclone/checksum/tests/main.nf.test index 2d833de..bbfab84 100644 --- a/modules/nf-core/rclone/checksum/tests/main.nf.test +++ b/modules/nf-core/rclone/checksum/tests/main.nf.test @@ -3,19 +3,22 @@ nextflow_process { name "Test Process RCLONE_CHECKSUM" script "../main.nf" process "RCLONE_CHECKSUM" + config "./nextflow.config" tag "modules" tag "modules_nfcore" tag "rclone" tag "rclone/checksum" - test("hello - md5") { + test("test - md5 - match") { + when { process { """ - file('hello.md5').text = 'e59ff97941044f85df5297e1c302d260 hello.txt\\n' + file('rclone.config').text = '[s3]\\ntype = s3\\nprovider = AWS\\nregion = eu-west-1\\nenv_auth = false\\nno_check_bucket = true' - input[0] = [[id:'test'], file('hello.md5'), 'MD5', file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + input[0] = [[id:'test'], file(params.modules_testdata_base_path + 'generic/txt/snpeff_cache_md5.txt', checkIfExists: true), 'MD5', "s3://annotation-cache/snpeff_cache/WBcel235.105/"] + input[1] = file('rclone.config') """ } } @@ -23,20 +26,21 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(sanitizeOutput(process.out, unstableKeys: ["combined", "differ", "missing_on_dst", "missing_on_src", "error"]) ).match() } + { assert snapshot(sanitizeOutput(process.out, unstableKeys: ["combined", "match", "differ", "missing_on_dst", "missing_on_src", "error"]) ).match() } ) } } - test("hello - md5 - stub") { + test("test - md5 - stub") { options "-stub" when { process { """ - file('hello.md5').text = 'e59ff97941044f85df5297e1c302d260 hello.txt\\n' + file('test.md5').text = 'e59ff97941044f85df5297e1c302d260 hello.txt\\n' - input[0] = [[id:'test'], file('hello.md5'), 'MD5', file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + input[0] = [[id:'test'], file('test.md5'), 'MD5', "s3://nf-core-test-datasets/modules/data/delete_me/malt/"] + input[1] = [] """ } } @@ -49,13 +53,15 @@ nextflow_process { } } - test("hello - differs") { + test("test - md5 - differs") { when { process { """ - file('hello_bad.md5').text = '00000000000000000000000000000000 hello.txt\\n' + file('test_bad.md5').text = '00000000000000000000000000000000 hello.txt' + file('rclone.config').text = '[s3]\\ntype = s3\\nprovider = AWS\\nregion = eu-west-1\\nenv_auth = false\\nno_check_bucket = true' - input[0] = [[id:'test'], file('hello_bad.md5'), 'MD5', file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + input[0] = [[id:'test'], file('test_bad.md5'), 'MD5', "s3://annotation-cache/snpeff_cache/WBcel235.105/"] + input[1] = file('rclone.config') """ } } @@ -67,23 +73,4 @@ nextflow_process { ) } } - - test("missing destination") { - when { - process { - """ - file('missing.md5').text = 'e59ff97941044f85df5297e1c302d260 missing.txt\\n' - - input[0] = [[id:'test'], file('missing.md5'), 'MD5', file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert snapshot(sanitizeOutput(process.out, unstableKeys: ["combined", "differ", "match", "missing_on_src", "error"])).match() } - ) - } - } } diff --git a/modules/nf-core/rclone/checksum/tests/main.nf.test.snap b/modules/nf-core/rclone/checksum/tests/main.nf.test.snap index 57ba70b..a100d10 100644 --- a/modules/nf-core/rclone/checksum/tests/main.nf.test.snap +++ b/modules/nf-core/rclone/checksum/tests/main.nf.test.snap @@ -1,5 +1,5 @@ { - "missing destination": { + "test - md5 - match": { "content": [ { "combined": [ @@ -11,20 +11,13 @@ ] ], "differ": [ - [ - { - "id": "test" - }, - "test.differ.txt" - ] + ], "error": [ - [ - { - "id": "test" - }, - "test.error.txt" - ] + + ], + "exit_code": [ + ], "match": [ [ @@ -35,20 +28,10 @@ ] ], "missing_on_dst": [ - [ - { - "id": "test" - }, - "test.missing_on_dst.txt:md5,9ddf8bbca47985a064c1d7163d0235da" - ] + ], "missing_on_src": [ - [ - { - "id": "test" - }, - "test.missing_on_src.txt" - ] + ], "versions_rclone": [ [ @@ -59,13 +42,13 @@ ] } ], - "timestamp": "2026-07-10T21:34:34.881758203", + "timestamp": "2026-07-21T23:18:32.247989694", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.6" + "nextflow": "26.04.1" } }, - "hello - md5 - stub": { + "test - md5 - differs": { "content": [ { "combined": [ @@ -77,28 +60,21 @@ ] ], "differ": [ - [ - { - "id": "test" - }, - "test.differ.txt" - ] + ], "error": [ + + ], + "exit_code": [ [ { "id": "test" }, - "test.error.txt" + "test.exit_code.txt:md5,b026324c6904b2a9cb4b88d6d61c81d1" ] ], "match": [ - [ - { - "id": "test" - }, - "test.match.txt" - ] + ], "missing_on_dst": [ [ @@ -109,12 +85,7 @@ ] ], "missing_on_src": [ - [ - { - "id": "test" - }, - "test.missing_on_src.txt" - ] + ], "versions_rclone": [ [ @@ -125,13 +96,13 @@ ] } ], - "timestamp": "2026-07-10T21:34:20.471038214", + "timestamp": "2026-07-21T23:20:27.477738495", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.6" + "nextflow": "26.04.1" } }, - "hello - md5": { + "test - md5 - stub": { "content": [ { "combined": [ @@ -158,71 +129,8 @@ "test.error.txt" ] ], - "match": [ - [ - { - "id": "test" - }, - "test.match.txt:md5,a438474115db37daf9d8e1307c06eb4a" - ] - ], - "missing_on_dst": [ - [ - { - "id": "test" - }, - "test.missing_on_dst.txt" - ] - ], - "missing_on_src": [ - [ - { - "id": "test" - }, - "test.missing_on_src.txt" - ] - ], - "versions_rclone": [ - [ - "RCLONE_CHECKSUM", - "rclone", - "1.74.3-DEV" - ] - ] - } - ], - "timestamp": "2026-07-10T21:34:13.264088794", - "meta": { - "nf-test": "0.9.5", - "nextflow": "26.04.6" - } - }, - "hello - differs": { - "content": [ - { - "combined": [ - [ - { - "id": "test" - }, - "test.combined.txt" - ] - ], - "differ": [ - [ - { - "id": "test" - }, - "test.differ.txt:md5,a438474115db37daf9d8e1307c06eb4a" - ] - ], - "error": [ - [ - { - "id": "test" - }, - "test.error.txt" - ] + "exit_code": [ + ], "match": [ [ @@ -257,10 +165,10 @@ ] } ], - "timestamp": "2026-07-10T21:34:27.556428476", + "timestamp": "2026-07-21T23:19:49.064384002", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.6" + "nextflow": "26.04.1" } } } \ No newline at end of file diff --git a/modules/nf-core/rclone/checksum/tests/nextflow.config b/modules/nf-core/rclone/checksum/tests/nextflow.config new file mode 100644 index 0000000..a78067e --- /dev/null +++ b/modules/nf-core/rclone/checksum/tests/nextflow.config @@ -0,0 +1,7 @@ +process { + withName: 'RCLONE_CHECKSUM' { + ext.args = { + "--no-check-certificate --one-way" + } + } +} diff --git a/modules/nf-core/rclone/copy/main.nf b/modules/nf-core/rclone/copy/main.nf index 2c97b43..a063abf 100644 --- a/modules/nf-core/rclone/copy/main.nf +++ b/modules/nf-core/rclone/copy/main.nf @@ -8,7 +8,7 @@ process RCLONE_COPY { : 'community.wave.seqera.io/library/rclone:1.65.0--ff88b2e0040147be'}" input: - tuple val(meta), val(source_path), val(destination_path) + tuple val(meta), val(source_path), val(destination_path), path(filter_file) path rclone_config output: diff --git a/modules/nf-core/rclone/copy/meta.yml b/modules/nf-core/rclone/copy/meta.yml index 93ed4f9..18c5bf1 100644 --- a/modules/nf-core/rclone/copy/meta.yml +++ b/modules/nf-core/rclone/copy/meta.yml @@ -36,6 +36,13 @@ input: path internally. Examples: `results/output`, `s3:bucket/output`, `gs:bucket/output`, `remote:path/to/output`. + - filter_file: + type: file + description: | + Optional plain text file containing one file path or pattern per line to be + passed to Rclone copy as a filter list (e.g. via `--files-from` or + `--include-from`). This input is only used when the corresponding rclone + filtering option is enabled through the module configuration. - rclone_config: type: file description: | diff --git a/modules/nf-core/rclone/copy/tests/main.nf.test b/modules/nf-core/rclone/copy/tests/main.nf.test index 14d79cb..85d92a0 100644 --- a/modules/nf-core/rclone/copy/tests/main.nf.test +++ b/modules/nf-core/rclone/copy/tests/main.nf.test @@ -3,6 +3,7 @@ nextflow_process { name "Test RCLONE_COPY" script "../main.nf" process "RCLONE_COPY" + config "./nextflow.config" tag "modules" tag "modules_nfcore" @@ -11,15 +12,46 @@ nextflow_process { test("homo_sapiens - gvcf - copy from https - dry-run") { - config "./nextflow.config" + when { + params { + exclude_from = false + } + process { + """ + input[0] = [ + [ id:'test' ], + params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz', + '/tmp/', + [] + ] + input[1] = [] + """ + } + } + + then { + assert process.success + assertAll( + { assert snapshot(sanitizeOutput(process.out, unstableKeys: ["log"])).match() } + ) + } + } + + test("homo_sapiens - gvcf - filter - dry-run") { when { + params { + exclude_from = true + } process { """ + file('exclude.txt').text = 'test.genome.vcf.gz' + input[0] = [ [ id:'test' ], params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz', - '/tmp/test.genome.vcf.gz' + '/tmp/', + file('exclude.txt') ] input[1] = [] """ @@ -39,12 +71,16 @@ nextflow_process { options "-stub" when { + params { + exclude_from = true + } process { """ input[0] = [ [ id:'test' ], params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz', - '/tmp/test.genome.vcf.gz' + '/tmp/', + [] ] input[1] = [] """ diff --git a/modules/nf-core/rclone/copy/tests/main.nf.test.snap b/modules/nf-core/rclone/copy/tests/main.nf.test.snap index eeaef63..a430eb4 100644 --- a/modules/nf-core/rclone/copy/tests/main.nf.test.snap +++ b/modules/nf-core/rclone/copy/tests/main.nf.test.snap @@ -50,5 +50,31 @@ "nf-test": "0.9.5", "nextflow": "25.10.4" } + }, + "homo_sapiens - gvcf - filter - dry-run": { + "content": [ + { + "log": [ + [ + { + "id": "test" + }, + "test-rclone-copy.log" + ] + ], + "versions_rclone": [ + [ + "RCLONE_COPY", + "rclone", + "1.65.0-DEV" + ] + ] + } + ], + "timestamp": "2026-07-28T15:53:02.864977147", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.1" + } } } \ No newline at end of file diff --git a/modules/nf-core/rclone/copy/tests/nextflow.config b/modules/nf-core/rclone/copy/tests/nextflow.config index 0e911db..dd7038e 100644 --- a/modules/nf-core/rclone/copy/tests/nextflow.config +++ b/modules/nf-core/rclone/copy/tests/nextflow.config @@ -1,7 +1,14 @@ process { withName: 'RCLONE_COPY' { ext.args = { - "--dry-run --no-check-certificate" + def base_args = [ + '--dry-run', + '--no-check-certificate' + ] + if (params.exclude_from) { + base_args.add("--exclude-from ${filter_file}") + } + base_args.join(' ') } } } diff --git a/nextflow.config b/nextflow.config index e87a679..2423a8a 100644 --- a/nextflow.config +++ b/nextflow.config @@ -23,6 +23,7 @@ params { // Rclone options rclone_config = null rclone_dry_run = false + copy_matching_only = false // Boilerplate options outdir = null @@ -259,7 +260,7 @@ manifest { // Nextflow plugins plugins { - id 'nf-schema@2.5.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-schema@2.7.2' // Validation of pipeline parameters and creation of an input channel from a sample sheet } validation { diff --git a/nextflow_schema.json b/nextflow_schema.json index e17f7cb..a8dc091 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -54,6 +54,12 @@ "description": "Perform a dry run of the rclone copy command.", "fa_icon": "fas fa-eye", "help_text": "If set, the pipeline will not actually copy any files to the destination. Instead, it will print out what would have been copied. This is useful for testing and debugging." + }, + "copy_matching_only": { + "type": "boolean", + "description": "Only copy files that matched their provided input checksums.", + "fa_icon": "fas fa-eye", + "help_text": "If set, the pipeline will only copy files that were correctly validated and will skip any file that did not match their input checksum." } } }, diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index c71d4e5..2b27518 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -35,63 +35,32 @@ "rclone/check", "rclone/check/Illumina_annotation", "rclone/check/Illumina_annotation/Illumina_annotation.combined.txt", - "rclone/check/Illumina_annotation/Illumina_annotation.differ.txt", - "rclone/check/Illumina_annotation/Illumina_annotation.error.txt", "rclone/check/Illumina_annotation/Illumina_annotation.match.txt", - "rclone/check/Illumina_annotation/Illumina_annotation.missing_on_dst.txt", - "rclone/check/Illumina_annotation/Illumina_annotation.missing_on_src.txt", "rclone/check/benchmark_bed", "rclone/check/benchmark_bed/benchmark_bed.combined.txt", - "rclone/check/benchmark_bed/benchmark_bed.differ.txt", - "rclone/check/benchmark_bed/benchmark_bed.error.txt", "rclone/check/benchmark_bed/benchmark_bed.match.txt", - "rclone/check/benchmark_bed/benchmark_bed.missing_on_dst.txt", - "rclone/check/benchmark_bed/benchmark_bed.missing_on_src.txt", - "rclone/check/test_fastq", - "rclone/check/test_fastq/test_fastq.combined.txt", - "rclone/check/test_fastq/test_fastq.differ.txt", - "rclone/check/test_fastq/test_fastq.error.txt", - "rclone/check/test_fastq/test_fastq.match.txt", - "rclone/check/test_fastq/test_fastq.missing_on_dst.txt", - "rclone/check/test_fastq/test_fastq.missing_on_src.txt", "rclone/checksum", "rclone/checksum/Illumina_annotation", "rclone/checksum/Illumina_annotation/Illumina_annotation.combined.txt", - "rclone/checksum/Illumina_annotation/Illumina_annotation.differ.txt", - "rclone/checksum/Illumina_annotation/Illumina_annotation.error.txt", "rclone/checksum/Illumina_annotation/Illumina_annotation.match.txt", - "rclone/checksum/Illumina_annotation/Illumina_annotation.missing_on_dst.txt", - "rclone/checksum/Illumina_annotation/Illumina_annotation.missing_on_src.txt", "rclone/checksum/benchmark_bed", "rclone/checksum/benchmark_bed/benchmark_bed.combined.txt", - "rclone/checksum/benchmark_bed/benchmark_bed.differ.txt", - "rclone/checksum/benchmark_bed/benchmark_bed.error.txt", "rclone/checksum/benchmark_bed/benchmark_bed.match.txt", - "rclone/checksum/benchmark_bed/benchmark_bed.missing_on_dst.txt", - "rclone/checksum/benchmark_bed/benchmark_bed.missing_on_src.txt", - "rclone/checksum/test_fastq", - "rclone/checksum/test_fastq/test_fastq.combined.txt", - "rclone/checksum/test_fastq/test_fastq.differ.txt", - "rclone/checksum/test_fastq/test_fastq.error.txt", - "rclone/checksum/test_fastq/test_fastq.match.txt", - "rclone/checksum/test_fastq/test_fastq.missing_on_dst.txt", - "rclone/checksum/test_fastq/test_fastq.missing_on_src.txt", "rclone/copy", "rclone/copy/Illumina_annotation-rclone-copy.log", - "rclone/copy/benchmark_bed-rclone-copy.log", - "rclone/copy/test_fastq-rclone-copy.log" + "rclone/copy/benchmark_bed-rclone-copy.log" ], [ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", - "multiqc_rclone_check.txt:md5,14b1ff73415db0533e1467cdf3dfccf8", - "multiqc_rclone_checksum_md5.txt:md5,14b1ff73415db0533e1467cdf3dfccf8", - "multiqc_samplesheet.txt:md5,608c07d7a768f3d66b44f8daddcec95e" + "multiqc_rclone_check.txt:md5,f9dc48d3c4d35cd7297b0a16227c3bcd", + "multiqc_rclone_checksum_md5.txt:md5,f9dc48d3c4d35cd7297b0a16227c3bcd", + "multiqc_samplesheet.txt:md5,373d903a41ab29bd26db35a3041626e4" ] ], - "timestamp": "2026-07-17T19:26:17.292861031", + "timestamp": "2026-07-22T22:00:27.445840436", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.6" + "nextflow": "26.04.1" } } } \ No newline at end of file diff --git a/tests/edge.nf.test b/tests/edge.nf.test index d69a8f8..5fce38e 100644 --- a/tests/edge.nf.test +++ b/tests/edge.nf.test @@ -33,4 +33,33 @@ nextflow_pipeline { ) } } + + test("-profile test copy matching files only") { + + when { + params { + input = "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/datasync/test-data/samplesheet_edge.csv" + outdir = "$outputDir" + copy_matching_only = true + } + } + + then { + // stable_path: All files + folders in ${params.outdir}/ with a stable path (including file name) + def stable_path = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_content: All files in ${params.outdir}/ with stable content + def stable_content = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + assert workflow.success + assertAll( + { assert snapshot( + // pipeline versions.yml file for multiqc from which Nextflow version is removed because we test pipelines on multiple Nextflow versions + removeNextflowVersion("$outputDir/pipeline_info/nf_core_datasync_software_mqc_versions.yml"), + // All stable path name, with a relative path + stable_path, + // All files with stable contents + stable_content + ).match() } + ) + } + } } diff --git a/tests/edge.nf.test.snap b/tests/edge.nf.test.snap index e43ee05..ea3559b 100644 --- a/tests/edge.nf.test.snap +++ b/tests/edge.nf.test.snap @@ -1,4 +1,74 @@ { + "-profile test copy matching files only": { + "content": [ + { + "RCLONE_CHECK": { + "rclone": "1.74.3-DEV" + }, + "RCLONE_CHECKSUM": { + "rclone": "1.74.3-DEV" + }, + "RCLONE_COPY": { + "rclone": "1.65.0-DEV" + }, + "Workflow": { + "nf-core/datasync": "v1.0dev" + } + }, + [ + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/llms-full.txt", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_rclone_check.txt", + "multiqc/multiqc_data/multiqc_rclone_checksum_md5.txt", + "multiqc/multiqc_data/multiqc_samplesheet.txt", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/nf_core_datasync_software_mqc_versions.yml", + "rclone", + "rclone/check", + "rclone/check/Illumina_annotation_incorrect", + "rclone/check/Illumina_annotation_incorrect/Illumina_annotation_incorrect.combined.txt", + "rclone/check/Illumina_annotation_incorrect/Illumina_annotation_incorrect.match.txt", + "rclone/check/Illumina_annotation_missing", + "rclone/check/Illumina_annotation_missing/Illumina_annotation_missing.combined.txt", + "rclone/check/Illumina_annotation_missing/Illumina_annotation_missing.match.txt", + "rclone/checksum", + "rclone/checksum/Illumina_annotation_incorrect", + "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.combined.txt", + "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.differ.txt", + "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.exit_code.txt", + "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.match.txt", + "rclone/checksum/Illumina_annotation_missing", + "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.combined.txt", + "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.exit_code.txt", + "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.match.txt", + "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.missing_on_dst.txt", + "rclone/checksum/Illumina_annotation_sha_only", + "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.exit_code.txt", + "rclone/copy", + "rclone/copy/Illumina_annotation_incorrect-rclone-copy.log", + "rclone/copy/Illumina_annotation_missing-rclone-copy.log" + ], + [ + "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", + "multiqc_rclone_check.txt:md5,4b2a6990836593f69e8e81a82c3daf42", + "multiqc_rclone_checksum_md5.txt:md5,cf92203df21f04ddf8315fc606812e0b", + "multiqc_samplesheet.txt:md5,bbfc817ff43c49cde14a2b33f46b5ae5" + ] + ], + "timestamp": "2026-07-28T21:45:08.268780076", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.1" + } + }, "-profile test edge cases": { "content": [ { @@ -25,7 +95,6 @@ "multiqc/multiqc_data/multiqc_data.json", "multiqc/multiqc_data/multiqc_rclone_check.txt", "multiqc/multiqc_data/multiqc_rclone_checksum_md5.txt", - "multiqc/multiqc_data/multiqc_rclone_checksum_sha.txt", "multiqc/multiqc_data/multiqc_samplesheet.txt", "multiqc/multiqc_data/multiqc_software_versions.txt", "multiqc/multiqc_data/multiqc_sources.txt", @@ -36,47 +105,26 @@ "rclone/check", "rclone/check/Illumina_annotation_incorrect", "rclone/check/Illumina_annotation_incorrect/Illumina_annotation_incorrect.combined.txt", - "rclone/check/Illumina_annotation_incorrect/Illumina_annotation_incorrect.differ.txt", - "rclone/check/Illumina_annotation_incorrect/Illumina_annotation_incorrect.error.txt", "rclone/check/Illumina_annotation_incorrect/Illumina_annotation_incorrect.match.txt", - "rclone/check/Illumina_annotation_incorrect/Illumina_annotation_incorrect.missing_on_dst.txt", - "rclone/check/Illumina_annotation_incorrect/Illumina_annotation_incorrect.missing_on_src.txt", "rclone/check/Illumina_annotation_missing", "rclone/check/Illumina_annotation_missing/Illumina_annotation_missing.combined.txt", - "rclone/check/Illumina_annotation_missing/Illumina_annotation_missing.differ.txt", - "rclone/check/Illumina_annotation_missing/Illumina_annotation_missing.error.txt", "rclone/check/Illumina_annotation_missing/Illumina_annotation_missing.match.txt", - "rclone/check/Illumina_annotation_missing/Illumina_annotation_missing.missing_on_dst.txt", - "rclone/check/Illumina_annotation_missing/Illumina_annotation_missing.missing_on_src.txt", "rclone/check/Illumina_annotation_sha_only", "rclone/check/Illumina_annotation_sha_only/Illumina_annotation_sha_only.combined.txt", - "rclone/check/Illumina_annotation_sha_only/Illumina_annotation_sha_only.differ.txt", - "rclone/check/Illumina_annotation_sha_only/Illumina_annotation_sha_only.error.txt", "rclone/check/Illumina_annotation_sha_only/Illumina_annotation_sha_only.match.txt", - "rclone/check/Illumina_annotation_sha_only/Illumina_annotation_sha_only.missing_on_dst.txt", - "rclone/check/Illumina_annotation_sha_only/Illumina_annotation_sha_only.missing_on_src.txt", "rclone/checksum", "rclone/checksum/Illumina_annotation_incorrect", "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.combined.txt", "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.differ.txt", - "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.error.txt", + "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.exit_code.txt", "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.match.txt", - "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.missing_on_dst.txt", - "rclone/checksum/Illumina_annotation_incorrect/Illumina_annotation_incorrect.missing_on_src.txt", "rclone/checksum/Illumina_annotation_missing", "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.combined.txt", - "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.differ.txt", - "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.error.txt", + "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.exit_code.txt", "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.match.txt", "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.missing_on_dst.txt", - "rclone/checksum/Illumina_annotation_missing/Illumina_annotation_missing.missing_on_src.txt", "rclone/checksum/Illumina_annotation_sha_only", - "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.combined.txt", - "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.differ.txt", - "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.error.txt", - "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.match.txt", - "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.missing_on_dst.txt", - "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.missing_on_src.txt", + "rclone/checksum/Illumina_annotation_sha_only/Illumina_annotation_sha_only.exit_code.txt", "rclone/copy", "rclone/copy/Illumina_annotation_incorrect-rclone-copy.log", "rclone/copy/Illumina_annotation_missing-rclone-copy.log", @@ -84,16 +132,15 @@ ], [ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", - "multiqc_rclone_check.txt:md5,df42d80150a8a1f2da93c5eed053dc3f", - "multiqc_rclone_checksum_md5.txt:md5,679b4cda086bfacf50fe3ec5b22dd386", - "multiqc_rclone_checksum_sha.txt:md5,9ba162249102c1b0af4464411b3a7eb5", + "multiqc_rclone_check.txt:md5,c8f13c10aa4f0e4b356c5c08a9ae7ee9", + "multiqc_rclone_checksum_md5.txt:md5,cf92203df21f04ddf8315fc606812e0b", "multiqc_samplesheet.txt:md5,bbfc817ff43c49cde14a2b33f46b5ae5" ] ], - "timestamp": "2026-07-20T14:40:38.905083297", + "timestamp": "2026-07-22T22:01:06.286788462", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.6" + "nextflow": "26.04.1" } } } \ No newline at end of file diff --git a/tests/main_full.nf.test.snap b/tests/main_full.nf.test.snap index dcf61c3..7bc1eeb 100644 --- a/tests/main_full.nf.test.snap +++ b/tests/main_full.nf.test.snap @@ -25,7 +25,6 @@ "multiqc/multiqc_data/multiqc_data.json", "multiqc/multiqc_data/multiqc_rclone_check.txt", "multiqc/multiqc_data/multiqc_rclone_checksum_md5.txt", - "multiqc/multiqc_data/multiqc_rclone_checksum_sha.txt", "multiqc/multiqc_data/multiqc_samplesheet.txt", "multiqc/multiqc_data/multiqc_software_versions.txt", "multiqc/multiqc_data/multiqc_sources.txt", @@ -36,34 +35,26 @@ "rclone/check", "rclone/check/demultiplex", "rclone/check/demultiplex/demultiplex.combined.txt", - "rclone/check/demultiplex/demultiplex.differ.txt", - "rclone/check/demultiplex/demultiplex.error.txt", "rclone/check/demultiplex/demultiplex.match.txt", - "rclone/check/demultiplex/demultiplex.missing_on_dst.txt", - "rclone/check/demultiplex/demultiplex.missing_on_src.txt", "rclone/checksum", "rclone/checksum/demultiplex", "rclone/checksum/demultiplex/demultiplex.combined.txt", - "rclone/checksum/demultiplex/demultiplex.differ.txt", - "rclone/checksum/demultiplex/demultiplex.error.txt", + "rclone/checksum/demultiplex/demultiplex.exit_code.txt", "rclone/checksum/demultiplex/demultiplex.match.txt", - "rclone/checksum/demultiplex/demultiplex.missing_on_dst.txt", - "rclone/checksum/demultiplex/demultiplex.missing_on_src.txt", "rclone/copy", "rclone/copy/demultiplex-rclone-copy.log" ], [ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", - "multiqc_rclone_check.txt:md5,204410f4dab8ab5db201f4833baa2f81", - "multiqc_rclone_checksum_md5.txt:md5,f59d50960061aee8afac0d9389b90a2e", - "multiqc_rclone_checksum_sha.txt:md5,204410f4dab8ab5db201f4833baa2f81", + "multiqc_rclone_check.txt:md5,03a73f087a286ec6580aada2a5db2e7c", + "multiqc_rclone_checksum_md5.txt:md5,d7c7dd71a9ff75955dd0e508d1ad9968", "multiqc_samplesheet.txt:md5,00788a52d1a014c12ac4ed554b0b44ca" ] ], - "timestamp": "2026-07-17T19:31:26.493917757", + "timestamp": "2026-07-22T22:01:43.699965019", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.6" + "nextflow": "26.04.1" } } } \ No newline at end of file diff --git a/workflows/datasync.nf b/workflows/datasync.nf index c47edd8..7f64a97 100644 --- a/workflows/datasync.nf +++ b/workflows/datasync.nf @@ -3,15 +3,15 @@ IMPORT MODULES / SUBWORKFLOWS / FUNCTIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { MULTIQC } from '../modules/nf-core/multiqc/main' -include { RCLONE_COPY } from '../modules/nf-core/rclone/copy/main' -include { RCLONE_CHECK } from '../modules/nf-core/rclone/check/main' -include { RCLONE_CHECKSUM } from '../modules/nf-core/rclone/checksum/main' -include { paramsSummaryMap } from 'plugin/nf-schema' -include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' -include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' -include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_datasync_pipeline' -include { parseRcloneCheck } from '../subworkflows/local/utils_nfcore_datasync_pipeline' +include { MULTIQC } from '../modules/nf-core/multiqc/main' +include { RCLONE_COPY } from '../modules/nf-core/rclone/copy/main' +include { RCLONE_CHECK } from '../modules/nf-core/rclone/check/main' +include { RCLONE_CHECKSUM } from '../modules/nf-core/rclone/checksum/main' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_datasync_pipeline' +include { parseRcloneCheck } from '../subworkflows/local/utils_nfcore_datasync_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -33,6 +33,7 @@ workflow DATASYNC { ch_versions = channel.empty() ch_multiqc_files = channel.empty() + ch_rclone_config = rclone_config ? file(rclone_config, checkIfExists: true) : [] ch_samplesheet = ch_samplesheet.multiMap { meta, input_path, output_path, md5, sha -> @@ -44,11 +45,13 @@ workflow DATASYNC { def rclone_destination = source.isFile() ? output_path.toString().replaceAll('/+$', '') : "${output_path.toString().replaceAll('/+$', '')}/${source.name}" - def rclone_check = "${output_path.toString().replaceAll('/+$', '')}/${source.name}" + + def rclone_check = source.isFile() + ? input_path.replaceFirst('/[^/]+$', '') + : input_path.toString().replaceAll('/+$', '') rclone: [ meta, source_uri, rclone_destination ] - checksum: [ meta, md5, sha, source ] - check : [ meta, source, file(rclone_check) ] + checksum: [ meta, md5, sha, rclone_check ] } // Group input md5sum/shasum with their respective generated checksum @@ -66,7 +69,8 @@ workflow DATASYNC { } RCLONE_CHECKSUM( - ch_checksum + ch_checksum, + ch_rclone_config ) ch_multiqc_files = ch_multiqc_files.mix(RCLONE_CHECKSUM.out.combined @@ -84,21 +88,52 @@ workflow DATASYNC { // // MODULE: Rclone data copying // + if(params.copy_matching_only) { + // Compute expected group size per meta.id from the input + ch_with_size = ch_checksum + .map { meta, checksum, hash, source -> [ meta.subMap(meta.keySet() - 'check_format'), 1 ] } + .groupTuple() + .map { meta, ones -> tuple(meta, ones.size()) } + + files_to_copy = RCLONE_CHECKSUM.out.match.map { + meta, match -> [ meta.subMap(meta.keySet() - 'check_format'), match ] + } + .combine(ch_with_size, by: 0) + .map { meta, match, size -> tuple(groupKey(meta, size), match) } + .groupTuple() + .map{ meta, files -> + def common = files + .collect { it.readLines() } + .inject { a, b -> a.intersect(b) } + + def copy_files = file("${workDir}/${meta.id}_files_to_copy.txt") + copy_files.text = common.join('\n') + '\n' + + tuple(meta, copy_files) + } + + ch_rclone_copy = ch_samplesheet.rclone + .join(files_to_copy) + } else { + ch_rclone_copy = ch_samplesheet.rclone.map { meta, source, destination -> [ meta, source, destination, [] ] } + } + RCLONE_COPY( - ch_samplesheet.rclone, - rclone_config ? file(rclone_config, checkIfExists: true) : [] + ch_rclone_copy, + ch_rclone_config, ) - // - // File transfer validation - // // Wait for file copy to finish before running RCLONE_CHECK - ch_rclone_check = ch_samplesheet.check + ch_rclone_check = ch_samplesheet.rclone .join(RCLONE_COPY.out.log) - .map { meta, input, output, log -> [ meta, input, output ]} + .map { meta, input, output, log -> [ meta, input, output ] } + // + // File transfer validation + // RCLONE_CHECK( - ch_rclone_check + ch_rclone_check, + ch_rclone_config ) ch_multiqc_files = ch_multiqc_files.mix(RCLONE_CHECK.out.combined