rust_test.yml

on:
  workflow_call:
    inputs:
      required_packages:
        type: string
        default: ""
        description: Package that needs to be installed before Rust compilation can happens
      toolchain:
        type: string
        default: "1.75"
        description: Rust toolchain to install. Do not set this to moving targets like "stable", instead leave it empty and regularly bump the default in this file.
      nightly_toolchain:
        type: string
        default: "nightly-2023-08-02"
        description: Rust toolchain to install. Do not set this to moving targets like "nightly", instead leave it empty and regularly bump the default in this file.
      additional_args:
        type: string
        default: ""
        description: Additional arguments to pass to the cargo command
      additional_cache_path:
        type: string
        default: ""
        description: Path of additional cache to get
      additional_cache_key:
        type: string
        default: ""
        description: Key of additional cache to get
      additional_cache_miss:
        type: string
        default: ""
        description: Script to run if additional cache miss
      additional_script:
        type: string
        default: ""
        description: Additional script to run before the additional packages
      working_directory:
        type: string
        default: ""
        description: Subdirectory to treat as repo root
      custom_cargo_commands:
        type: string
        default: ""
        description: Custom cargo commands that will be run after login
      fail_fast:
        type: string
        default: "true"
        description: Should all the test ran or fail early
      skip_miri_test:
        type: string
        default: "false"
        description: Should we skip miri test (useful when tests are incompatible)
      test_publish_required:
        type: string
        default: "false"
        description: "Should the publish dry-run test be marked as required"
      test_doc_required:
        type: string
        default: "false"
        description: "Should the doc test be required"
      service_database:
        type: string
        default: "false"
        description: "Should Postgres be deployed alongside for testing"
      service_azurite:
        type: string
        default: "false"
        description: "Should Azurite emulator be deployed alongside for testing"

env:
  CARGO_TERM_COLOR: never
  CARGO_NET_GIT_FETCH_WITH_CLI: true
  CARGO_HTTP_USER_AGENT: "shipyard ${{ secrets.CARGO_PRIVATE_REGISTRY_TOKEN }}"
  CARGO_PROFILE_TEST_DEBUG: false
  LAUNCHER_APP_NAME: ""
  LAUNCHER_APP_FALLBACK: ""
  SQLX_OFFLINE: true

jobs:
  derive_runner:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          sparse-checkout: "rust-toolchain.toml"
      - name: Get rust channel
        id: rust-channel
        shell: bash
        run: |
          rust_version="$(yq -p toml '.toolchain.channel' < rust-toolchain.toml || echo '${{ inputs.toolchain }}')"
          rust_version="${rust_version/./-}"
          echo "rust_version=${rust_version}" >> $GITHUB_OUTPUT
          sanitized_working_directory="${{ inputs.working_directory }}"
          sanitized_working_directory="${sanitized_working_directory//\//_}"
          echo "sanitized_working_directory=${sanitized_working_directory}" >> "$GITHUB_OUTPUT"
    outputs:
      rust-channel: ${{ steps.rust-channel.outputs.rust_version }}
      sanitized_working_directory: ${{ steps.rust-channel.outputs.sanitized_working_directory }}

  check:
    name: Rust ${{ matrix.type }} ${{ inputs.working_directory }}
    runs-on: "rust-${{ needs.derive_runner.outputs.rust-channel }}-scale-set"
    needs: derive_runner
    env:
      CARGO_HTTP_USER_AGENT: "shipyard ${{ secrets.CARGO_PRIVATE_REGISTRY_TOKEN }}"
      RUSTC_WRAPPER: "sccache"
    strategy:
      matrix:
        type: [check, test, miri]
        include:
          - toolchain: ${{ inputs.toolchain }}
          - type: miri
            toolchain: ${{ inputs.nightly_toolchain }}
      fail-fast: ${{ inputs.fail_fast == 'true' }}
    services:
      azurite:
        # Hack for conditionally enabling the service:
        # https://github.com/actions/runner/issues/822
        image: ${{ (inputs.service_azurite == 'true') && 'mcr.microsoft.com/azure-storage/azurite' || '' }}
        ports:
          - 10000:10000 # BLOB
          - 10001:10001 # Queue
          - 10002:10002 # Table
        options: --health-cmd "nc -z 127.0.0.1 10000"
        credentials:
          username: ${{ secrets.FSLABS_DOCKERHUB_USERNAME }}
          password: ${{ secrets.FSLABS_DOCKERHUB_PASSWORD }}
      postgres:
        # Hack for conditionally enabling the service:
        # https://github.com/actions/runner/issues/822
        image: ${{ (inputs.service_database == 'true') && 'postgres' || '' }}
        env:
          POSTGRES_PASSWORD: postgres
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        credentials:
          username: ${{ secrets.FSLABS_DOCKERHUB_USERNAME }}
          password: ${{ secrets.FSLABS_DOCKERHUB_PASSWORD }}
        ports:
          - 5432:5432
    steps:
      - name: Set DATABASE_URL
        if: inputs.service_database == 'true'
        shell: bash
        run: |
          echo "DATABASE_URL=postgresql://postgres:postgres@localhost/postgres" >> "$GITHUB_ENV"
      - name: Globally configure Git
        shell: bash
        run: |
          git config --global core.excludesfile ~/.gitignore_global
          git config --global core.compression 0
      - name: "Ensure we don't track the new resources in git"
        shell: bash
        run: |
          echo "output.txt" >> ~/.gitignore_global
          echo "deny.toml" >> ~/.gitignore_global
      - name: "Update rustup only if version is different thant toolchain"
        shell: bash
        run: |
          rustup set auto-self-update disable
      - name: Checkout
        uses: actions/checkout@v4
      - uses: ForesightMiningSoftwareCorporation/github/.github/actions/login-private-registry@v3.2.0
        with:
          private_key: ${{ secrets.CARGO_PRIVATE_REGISTRY_SSH_PRIVATE_KEY }}
          host: ${{ secrets.CARGO_PRIVATE_REGISTRY_HOST }}
          name: ${{ secrets.CARGO_PRIVATE_REGISTRY_NAME }}
          token: ${{ secrets.CARGO_PRIVATE_REGISTRY_TOKEN }}
          cos_username: ${{ secrets.CARGO_COS_PRIVATE_REGISTRY_USERNAME }}
          cos_password: ${{ secrets.CARGO_COS_PRIVATE_REGISTRY_PASSWORD }}
          cos_tenant: ${{ secrets.CARGO_COS_PRIVATE_REGISTRY_TENANT }}
          cos_subscription: ${{ secrets.CARGO_COS_PRIVATE_REGISTRY_SUBSCRIPTION }}
          additional_private_keys: |
            ${{ secrets.FSE_SSH_PRIVATE_KEY }}
            ${{ secrets.BEVY_CLIPMAP_SSH_PRIVATE_KEY }}
            ${{ secrets.DAG_TABLES_SSH_PRIVATE_KEY }}
            ${{ secrets.VOLUMESIGHT_SSH_PRIVATE_KEY }}
            ${{ secrets.SPARKL_BLAST_SSH_PRIVATE_KEY }}
      - name: Additional Cache
        id: additional-cache
        uses: actions/cache@v4
        if: inputs.additional_cache_key != '' && inputs.additional_cache_path != ''
        with:
          path: ${{ inputs.additional_cache_path }}
          key: ${{ inputs.additional_cache_key }}-${{ github.ref_name }}
          restore-keys: |
            ${{ inputs.additional_cache_key }}-${{ github.base_ref }}
            ${{ inputs.additional_cache_key }}
      - name: Additional Cache Miss Command
        if: steps.additional-cache.outputs.cache-hit != 'true'
        shell: bash
        run: ${{ inputs.additional_cache_miss }}
      - name: Additional Script
        if: ${{ inputs.additional_script != '' }}
        shell: bash
        working-directory: ${{ inputs.working_directory }}
        run: ${{ inputs.additional_script }}
      - name: Cache Dependencies
        uses: Swatinem/rust-cache@v2
        with:
          shared-key: "${{ github.job }}-${{ matrix.type }}"
          workspaces: ${{ inputs.working_directory }}
          env-vars: ""
      - name: Get crate name
        id: name
        if: always()
        shell: bash
        working-directory: ${{ inputs.working_directory }}
        run: |
          echo "value=$(yq -p toml '.package.name' < Cargo.toml)" >> $GITHUB_OUTPUT
      - name: Set start time as env variable
        id: start_time
        run: echo "value=$(date +'%s')000" >> $GITHUB_OUTPUT

      - name: custom
        uses: ForesightMiningSoftwareCorporation/github/.github/actions/shell-with-output@v2
        id: custom
        if: inputs.custom_cargo_commands != ''
        continue-on-error: true
        with:
          command: ${{ inputs.custom_cargo_commands }}
          working_directory: ${{ inputs.working_directory }}
      - name: fmt
        id: fmt
        uses: ForesightMiningSoftwareCorporation/github/.github/actions/shell-with-output@v2
        continue-on-error: true
        if: matrix.type == 'check' && (inputs.fail_fast == 'false' || inputs.custom_cargo_commands == '' || steps.custom.outcome == 'success' && !cancelled())
        with:
          command: "cargo fmt --verbose -- --check"
          working_directory: ${{ inputs.working_directory }}
      - name: check
        id: check
        uses: ForesightMiningSoftwareCorporation/github/.github/actions/shell-with-output@v2
        continue-on-error: true
        if: matrix.type == 'check' && (inputs.fail_fast == 'false' || steps.fmt.outcome == 'success' && !cancelled())
        with:
          command: "cargo check ${{ inputs.additional_args }}" # todo: feed back the `workspace
          working_directory: ${{ inputs.working_directory }}
      - name: clippy
        id: clippy
        uses: ForesightMiningSoftwareCorporation/github/.github/actions/shell-with-output@v2
        continue-on-error: true
        if: matrix.type == 'check' && (inputs.fail_fast == 'false' || steps.check.outcome == 'success' && !cancelled())
        with:
          command: "cargo clippy ${{ inputs.additional_args }} -- -D warnings"
          working_directory: ${{ inputs.working_directory }}
      - name: doc
        id: doc
        uses: ForesightMiningSoftwareCorporation/github/.github/actions/shell-with-output@v2
        continue-on-error: true
        if: matrix.type == 'check' && (inputs.fail_fast == 'false' || steps.clippy.outcome == 'success' && !cancelled())
        with:
          command: "RUSTDOCFLAGS='-D warnings' cargo doc --no-deps"
          working_directory: ${{ inputs.working_directory }}
      - name: Copy Default deny if not exists
        shell: bash
        working-directory: ${{ inputs.working_directory }}
        if: matrix.type == 'check' && (inputs.fail_fast == 'false' || steps.clippy.outcome == 'success' && !cancelled())
        run: |
          if [ ! -f deny.toml ]; then
            curl -o deny.toml https://raw.githubusercontent.com/ForesightMiningSoftwareCorporation/github/main/deny.toml
          fi
      - name: deny-license
        uses: ForesightMiningSoftwareCorporation/github/.github/actions/shell-with-output@v2
        id: deny-license
        continue-on-error: true
        if: matrix.type == 'check' && (inputs.fail_fast == 'false' || steps.clippy.outcome == 'success' && !cancelled())
        with:
          command: "cargo deny check licenses"
          working_directory: ${{ inputs.working_directory }}
      - name: deny-bans
        uses: ForesightMiningSoftwareCorporation/github/.github/actions/shell-with-output@v2
        id: deny-bans
        continue-on-error: true
        if: matrix.type == 'check' && (inputs.fail_fast == 'false' || steps.clippy.outcome == 'success' && !cancelled())
        with:
          command: "cargo deny check bans"
          working_directory: ${{ inputs.working_directory }}
      - name: deny-advisories
        uses: ForesightMiningSoftwareCorporation/github/.github/actions/shell-with-output@v2
        id: deny-advisories
        continue-on-error: true
        if: matrix.type == 'check' && (inputs.fail_fast == 'false' || steps.clippy.outcome == 'success' && !cancelled())
        with:
          command: "cargo deny check advisories"
          working_directory: ${{ inputs.working_directory }}
      - name: deny-sources
        uses: ForesightMiningSoftwareCorporation/github/.github/actions/shell-with-output@v2
        id: deny-sources
        continue-on-error: true
        if: matrix.type == 'check' && (inputs.fail_fast == 'false' || steps.clippy.outcome == 'success' && !cancelled())
        with:
          command: "cargo deny check sources"
          working_directory: ${{ inputs.working_directory }}
      - name: dependencies
        uses: ForesightMiningSoftwareCorporation/github/.github/actions/shell-with-output@v2
        id: dependencies
        continue-on-error: true
        if: matrix.type == 'check' && (inputs.fail_fast == 'false' || steps.clippy.outcome == 'success' && !cancelled())
        with:
          command: "cargo machete"
          working_directory: ${{ inputs.working_directory }}
      - name: publish-dryrun
        uses: ForesightMiningSoftwareCorporation/github/.github/actions/shell-with-output@v2
        id: publish-dryrun
        continue-on-error: true
        if: matrix.type == 'check' && (inputs.fail_fast == 'false' || steps.clippy.outcome == 'success' && !cancelled())
        with:
          command: "cargo package"
          working_directory: ${{ inputs.working_directory }}
      - name: tests
        uses: ForesightMiningSoftwareCorporation/github/.github/actions/shell-with-output@v2
        id: tests
        if: matrix.type == 'test' && (inputs.fail_fast == 'false' || !cancelled())
        with:
          command: "cargo test ${{ inputs.fail_fast == 'false' && '--no-fail_fast' || '' }} ${{ inputs.additional_args }}"
          working_directory: ${{ inputs.working_directory }}
      - name: miri
        uses: ForesightMiningSoftwareCorporation/github/.github/actions/shell-with-output@v2
        id: miri
        continue-on-error: true
        if: matrix.type == 'miri' && inputs.skip_miri_test == 'false' && (inputs.fail_fast == 'false' || !cancelled())
        with:
          command: "cargo miri test ${{ inputs.fail_fast == 'false' && '--keep-going' || '' }} ${{ inputs.additional_args }}"
          working_directory: ${{ inputs.working_directory }}
      - name: Write output to file
        shell: bash
        if: always()
        working-directory: ${{ runner.temp }}
        run: |
          tee -a ${{ steps.name.outputs.value }}-${{ matrix.type }}.json <<EOF
          {
            "name": "${{ steps.name.outputs.value }}",
            "start_time": "${{ steps.start_time.outputs.value }}",
            "end_time": "$(date +'%s')000",
            "working_directory": "${{ inputs.working_directory }}",
            "type": "${{ matrix.type }}",
            "server_url": "${{ github.server_url }}",
            "repository": "${{ github.repository }}",
            "run_id": "${{ github.run_id }}",
            "run_attempt": "${{ github.run_attempt }}",
            "actor": "${{ github.actor }}",
            "event_name": "${{ github.event_name }}",
            "outputs": {
          EOF

          case "${{ matrix.type }}" in
            "check")
              tee -a ${{ steps.name.outputs.value }}-${{ matrix.type }}.json <<EOF
                "custom": {
                  "outcome": "${{ steps.custom.outcome }}",
                  "required": false
                },
                "fmt": {
                  "outcome": "${{ steps.fmt.outcome }}",
                  "required": true
                },
                "check": {
                  "outcome": "${{ steps.check.outcome }}",
                  "required": true
                },
                "clippy": {
                  "outcome": "${{ steps.clippy.outcome }}",
                  "required": true
                },
                "doc": {
                  "outcome": "${{ steps.doc.outcome }}",
                  "required": ${{ inputs.test_doc_required }}
                },
                "dependencies": {
                  "outcome": "${{ steps.dependencies.outcome }}",
                  "required": false
                },
                "deny-advisories": {
                  "outcome": "${{ steps.deny-advisories.outcome }}",
                  "required": false
                },
                "deny-bans": {
                  "outcome": "${{ steps.deny-bans.outcome }}",
                  "required": false
                },
                "deny-license": {
                  "outcome": "${{ steps.deny-license.outcome }}",
                  "required": false
                },
                "deny-sources": {
                  "outcome": "${{ steps.deny-sources.outcome }}",
                  "required": false
                },
                "publish-dryrun": {
                  "outcome": "${{ steps.publish-dryrun.outcome }}",
                  "required": ${{ inputs.test_publish_required }}
                }
              }
          EOF
              ;;
            "miri")
              tee -a ${{ steps.name.outputs.value }}-${{ matrix.type }}.json <<EOF
                "miri": {
                  "outcome": "${{ steps.miri.outcome }}",
                  "required": false
                }
              }
          EOF
              ;;
            "test")
              tee -a ${{ steps.name.outputs.value }}-${{ matrix.type }}.json <<EOF
                "tests": {
                  "outcome": "${{ steps.tests.outcome }}",
                  "required": true
                }
              }
          EOF
              ;;
            *)
          esac

          tee -a ${{ steps.name.outputs.value }}-${{ matrix.type }}.json <<EOF
          }
          EOF
      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: fslabs__checks__${{ github.run_id }}__${{ github.run_attempt }}__${{ steps.name.outputs.value }}-${{ matrix.type }}
          path: ${{ runner.temp }}/${{ steps.name.outputs.value }}-${{ matrix.type }}.json