merge dev into rclone to test in seqera

.github/workflows/branch.yml

@@ -2,45 +2,61 @@ name: nf-core branch protection
 # This workflow is triggered on PRs to `main`/`master` branch on the repository
 # It fails when someone tries to make a PR against the nf-core `main`/`master` branch instead of `dev`
 on:
-  pull_request_target:
+  pull_request:
     branches:
       - main
       - master
 
+permissions: {}
+
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
       # PRs to the nf-core repo main/master branch are only ok if coming from the nf-core repo `dev` or any `patch` branches
       - name: Check PRs
         if: github.repository == 'nf-core/datasync'
+        env:
+          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
         run: |
-          { [[ ${{github.event.pull_request.head.repo.full_name }} == nf-core/datasync ]] && [[ $GITHUB_HEAD_REF == "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]]
+          { [[ "$HEAD_REPO" == nf-core/datasync ]] && [[ $GITHUB_HEAD_REF == "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]]
 
-      # If the above check failed, post a comment on the PR explaining the failure
-      # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets
-      - name: Post PR comment
+      # If the above check failed, build a comment to be posted by the shared poster workflow
+      - name: Build PR comment
         if: failure()
-        uses: mshick/add-pr-comment@8e4927817251f1ff60c001f04568532b38e0b4a0 # v3
-        with:
-          message: |
-            ## This PR is against the `${{github.event.pull_request.base.ref}}` branch :x:
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          BASE_REF: ${{ github.event.pull_request.base.ref }}
+          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
+          PR_USER: ${{ github.event.pull_request.user.login }}
+        run: |
+          mkdir -p pr-comment
+          echo "$PR_NUMBER" > pr-comment/pr_number.txt
+          echo "branch" > pr-comment/header.txt
+          cat > pr-comment/comment.md <<EOF
+          ## This PR is against the \`${BASE_REF}\` branch :x:
 
-            * Do not close this PR
-            * Click _Edit_ and change the `base` to `dev`
-            * This CI test will remain failed until you push a new commit
+          * Do not close this PR
+          * Click _Edit_ and change the \`base\` to \`dev\`
+          * This CI test will remain failed until you push a new commit
 
-            ---
+          ---
 
-            Hi @${{ github.event.pull_request.user.login }},
+          Hi @${PR_USER},
 
-            It looks like this pull-request is has been made against the [${{github.event.pull_request.head.repo.full_name }}](https://github.com/${{github.event.pull_request.head.repo.full_name }}) ${{github.event.pull_request.base.ref}} branch.
-            The ${{github.event.pull_request.base.ref}} branch on nf-core repositories should always contain code from the latest release.
-            Because of this, PRs to ${{github.event.pull_request.base.ref}} are only allowed if they come from the [${{github.event.pull_request.head.repo.full_name }}](https://github.com/${{github.event.pull_request.head.repo.full_name }}) `dev` branch.
+          It looks like this pull-request is has been made against the [${HEAD_REPO}](https://github.com/${HEAD_REPO}) ${BASE_REF} branch.
+          The ${BASE_REF} branch on nf-core repositories should always contain code from the latest release.
+          Because of this, PRs to ${BASE_REF} are only allowed if they come from the [${HEAD_REPO}](https://github.com/${HEAD_REPO}) \`dev\` branch.
 
-            You do not need to close this PR, you can change the target branch to `dev` by clicking the _"Edit"_ button at the top of this page.
-            Note that even after this, the test will continue to show as failing until you push a new commit.
+          You do not need to close this PR, you can change the target branch to \`dev\` by clicking the _"Edit"_ button at the top of this page.
+          Note that even after this, the test will continue to show as failing until you push a new commit.
 
-            Thanks again for your contribution!
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          allow-repeats: false
+          Thanks again for your contribution!
+          EOF
+
+      - name: Upload PR comment artifact
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: pr-comment
+          path: pr-comment/

.github/workflows/linting.yml

@@ -74,3 +74,21 @@ jobs:
             lint_log.txt
             lint_results.md
             PR_number.txt
+
+      # Build a comment for the shared pr-comment.yml poster to publish on the PR
+      - name: Prepare PR comment
+        if: ${{ always() }}
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          mkdir -p pr-comment
+          echo "$PR_NUMBER" > pr-comment/pr_number.txt
+          echo "lint" > pr-comment/header.txt
+          [ -f lint_results.md ] && cp lint_results.md pr-comment/comment.md || true
+
+      - name: Upload PR comment artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: pr-comment
+          path: pr-comment/

.github/workflows/nf-test.yml

@@ -116,6 +116,22 @@ jobs:
             fi
           fi
 
+      # continue-on-error keeps latest-everything from failing the job, so it never shows up in
+      # `needs.nf-test.result` downstream and CI stays green. Surface it via a PR comment instead;
+      # other NXF_VER failures already fail the job/CI directly, so no comment is needed for those.
+      - name: Prepare PR comment fragment
+        if: ${{ always() && steps.run_nf_test.outcome == 'failure' && matrix.NXF_VER == 'latest-everything' }}
+        run: |
+          mkdir -p pr-comment-fragment
+          echo "* ❌ \`${{ matrix.profile }}\` | \`${{ matrix.NXF_VER }}\` | Shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }}" > pr-comment-fragment/fragment.md
+
+      - name: Upload PR comment fragment
+        if: ${{ always() && steps.run_nf_test.outcome == 'failure' && matrix.NXF_VER == 'latest-everything' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: pr-comment-fragment-${{ strategy.job-index }}
+          path: pr-comment-fragment/
+
   confirm-pass:
     needs: [nf-test]
     if: always()
@@ -142,3 +158,44 @@ jobs:
           echo "DEBUG: toJSON(needs) = ${{ toJSON(needs) }}"
           echo "DEBUG: toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"
           echo "::endgroup::"
+
+      - name: Download PR comment fragments
+        if: ${{ always() }}
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        continue-on-error: true
+        with:
+          pattern: pr-comment-fragment-*
+          path: pr-comment-fragments
+          merge-multiple: true
+
+      # Build a comment for the shared pr-comment.yml poster to publish on the PR.
+      # Based on the fragments above (not needs.*.result) so non-blocking failures are still reported.
+      - name: Prepare PR comment
+        if: ${{ always() }}
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          mkdir -p pr-comment
+          echo "$PR_NUMBER" > pr-comment/pr_number.txt
+          echo "nf-test" > pr-comment/header.txt
+          if [ -d pr-comment-fragments ] && [ -n "$(ls -A pr-comment-fragments)" ]; then
+            {
+              echo "## ❌ nf-test failed with latest Nextflow version"
+              echo ""
+              echo "> [!NOTE]"
+              echo "> Tests with Nextflow's latest version failed but it will not cause a CI workflow failure."
+              echo "> Please check if the failure is expected with newer (edge-)releases of Nextflow or if it needs fixing."
+              echo ""
+              cat pr-comment-fragments/*.md
+              echo ""
+              echo "See the [full run](${RUN_URL}) for details."
+            } > pr-comment/comment.md
+          fi
+
+      - name: Upload PR comment artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: pr-comment
+          path: pr-comment/

.github/workflows/pr-comment.yml

@@ -0,0 +1,82 @@
+name: Post PR comment
+# Shared, privileged comment poster.
+#
+# This is the single workflow that runs with a write token. It is triggered
+# after any of the listed "producer" workflows complete on a pull request.
+# Each producer runs untrusted PR code (if any) with a read-only token and
+# uploads a `pr-comment` artifact describing the comment to post; this workflow
+# only ever reads that plain-text artifact, so no PR code is executed here.
+#
+# Artifact contract (uploaded by producers under the name `pr-comment`):
+#   pr_number.txt  - the pull request number
+#   header.txt     - sticky-comment identifier (keeps comment types separate)
+#   comment.md     - the Markdown body (omit the file to post nothing)
+
+on:
+  workflow_run:
+    workflows:
+      - "nf-core linting"
+      - "nf-core template version comment"
+      - "nf-core branch protection"
+      - "Run nf-test"
+
+permissions:
+  actions: read
+  contents: read
+  pull-requests: write
+
+jobs:
+  post-comment:
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.event == 'pull_request'
+    steps:
+      - name: Download PR comment artifact
+        uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21
+        with:
+          run_id: ${{ github.event.workflow_run.id }}
+          name: pr-comment
+          path: pr-comment
+          if_no_artifact_found: ignore
+
+      - name: Read comment metadata
+        id: meta
+        run: |
+          echo "::group::Downloaded pr-comment contents"
+          ls -la pr-comment 2>/dev/null || echo "No pr-comment/ directory was downloaded."
+          echo "::endgroup::"
+
+          if [ ! -d pr-comment ]; then
+            echo "No pr-comment artifact found; nothing to post."
+            exit 0
+          fi
+
+          if [ ! -f pr-comment/comment.md ]; then
+            echo "Artifact present but no comment.md; nothing to post."
+            exit 0
+          fi
+
+          pr_number=$(cat pr-comment/pr_number.txt)
+          header=$(cat pr-comment/header.txt)
+          echo "Found comment.md (header='$header', pr_number='$pr_number')."
+
+          # Guard against anything unexpected ending up in the PR number.
+          case "$pr_number" in
+            ''|*[!0-9]*)
+              echo "Invalid PR number: '$pr_number'"
+              exit 1
+              ;;
+          esac
+
+          echo "pr_number=$pr_number" >> "$GITHUB_OUTPUT"
+          echo "header=$header" >> "$GITHUB_OUTPUT"
+          echo "post=true" >> "$GITHUB_OUTPUT"
+          echo "Will post comment to PR #${pr_number}."
+
+      - name: Post PR comment
+        if: steps.meta.outputs.post == 'true'
+        uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          number: ${{ steps.meta.outputs.pr_number }}
+          header: ${{ steps.meta.outputs.header }}
+          path: pr-comment/comment.md

.github/workflows/template-version-comment.yml

@@ -2,10 +2,13 @@ name: nf-core template version comment
 # This workflow is triggered on PRs to check if the pipeline template version matches the latest nf-core version.
 # It posts a comment to the PR, even if it comes from a fork.
 
-on: pull_request_target
+on:
+  pull_request:
+
+permissions: {}
 
 jobs:
-  template_version:
+  check_template_version:
     runs-on: ubuntu-latest
     steps:
       - name: Check out pipeline code
@@ -22,25 +25,36 @@ jobs:
       - name: Install nf-core
         run: |
           python -m pip install --upgrade pip
-          pip install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }}
+          pip install nf-core
+
+      - name: Build PR comment if template is outdated
+        # The fork-controlled version is passed via the environment and only ever
+        # used as quoted shell data (never interpolated into a command), so it
+        # cannot be used for script injection.
+        env:
+          PR_VERSION: ${{ steps.read_yml.outputs['nf_core_version'] }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          mkdir -p pr-comment
+          echo "$PR_NUMBER" > pr-comment/pr_number.txt
+          echo "template-version" > pr-comment/header.txt
+
+          latest_version=$(nf-core --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n1)
 
-      - name: Check nf-core outdated
-        id: nf_core_outdated
-        run: echo "OUTPUT=$(pip list --outdated | grep nf-core)" >> ${GITHUB_ENV}
+          if [ -n "$PR_VERSION" ] && [ -n "$latest_version" ] && [ "$PR_VERSION" != "$latest_version" ]; then
+            cat > pr-comment/comment.md <<EOF
+          > [!WARNING]
+          > Newer version of the nf-core template is available.
+          >
+          > Your pipeline is using an old version of the nf-core template: ${PR_VERSION}.
+          > Please update your pipeline to the latest version.
+          >
+          > For more documentation on how to update your pipeline, please see the [Synchronisation documentation](https://nf-co.re/docs/developing/template-syncs/overview).
+          EOF
+          fi
 
-      - name: Post nf-core template version comment
-        uses: mshick/add-pr-comment@8e4927817251f1ff60c001f04568532b38e0b4a0 # v3
-        if: |
-          contains(env.OUTPUT, 'nf-core')
+      - name: Upload PR comment artifact
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          repo-token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }}
-          allow-repeats: false
-          message: |
-            > [!WARNING]
-            > Newer version of the nf-core template is available.
-            >
-            > Your pipeline is using an old version of the nf-core template: ${{ steps.read_yml.outputs['nf_core_version'] }}.
-            > Please update your pipeline to the latest version.
-            >
-            > For more documentation on how to update your pipeline, please see the [Synchronisation documentation](https://nf-co.re/docs/developing/template-syncs/overview).
-          #
+          name: pr-comment
+          path: pr-comment/

.nf-core.yml

@@ -2,8 +2,7 @@ nf_core_version: 4.0.2
 repository_type: pipeline
 template:
   author: Alexander Peltzer
-  description: A simple sysops pipeline that can be used to synchronize, integrity
-    check and permanently archive data.
+  description: A simple sysops pipeline that can be used to synchronize, integrity check and permanently archive data.
   force: false
   is_nfcore: true
   name: datasync
@@ -12,3 +11,8 @@ template:
   version: 1.0dev
 lint:
   multiqc_config: false
+  files_exist:
+    - .github/workflows/linting_comment.yml
+  files_unchanged:
+    - .github/workflows/branch.yml
+    - .github/workflows/linting.yml

CHANGELOG.md

@@ -9,6 +9,7 @@ Initial release of nf-core/datasync, created with the [nf-core](https://nf-co.re
 
 ### `Added`
 
+- [[#67](https://github.com/nf-core/datasync/pull/67)] - Copy only files which are successfully validated ([@delfiterradas](https://github.com/delfiterradas), review by [@atrigila](https://github.com/atrigila) and [@antoniasaracco](https://github.com/antoniasaracco)).
 - [[#57](https://github.com/nf-core/datasync/pull/57)] - Add nf-tests with edge cases([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)).
 - [[#55](https://github.com/nf-core/datasync/pull/55)] - Add subdirectories with sample id to results. Improvements to MultiQC report (width of columns, sections and sub-section headers)colors to multiqc results and display first errors in tables ([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)).
 - [[#54](https://github.com/nf-core/datasync/pull/54)] - Add colors to multiqc results and display first errors in tables ([@atrigila](https://github.com/atrigila), review by [@delfiterradas](https://github.com/delfiterradas)).
@@ -17,6 +18,7 @@ Initial release of nf-core/datasync, created with the [nf-core](https://nf-co.re
 - [[#46](https://github.com/nf-core/datasync/pull/46)] - Import Rclone module from nf-core([@antoniasaracco](https://github.com/antoniasaracco), review by [@delfiterradas](https://github.com/delfiterradas)).
 - [[#41](https://github.com/nf-core/datasync/pull/41)] - Generate MultiQC Report with comparechecksum tables and input samplesheet ([@delfiterradas](https://github.com/delfiterradas), review by [@atrigila](https://github.com/atrigila)).
 - [[#49](https://github.com/nf-core/datasync/pull/49)] - Install `RCLONE_CHECK` and `RCLONE_CHECKSUM` modules from nf-core ([@delfiterradas](https://github.com/delfiterradas), review by [@atrigila](https://github.com/atrigila) and [@antoniasaracco](https://github.com/antoniasaracco)).
+- [[#59](https://github.com/nf-core/datasync/pull/59)] - Create pipeline documentation ([@antoniasaracco](https://github.com/antoniasaracco), review by [@atrigila](https://github.com/atrigila) and [@delfiterradas](https://github.com/delfiterradas)).
 
 ### `Fixed`