diff --git a/.github/gh-config-template/README.md b/.github/gh-config-template/README.md new file mode 100644 index 000000000..3e50acc22 --- /dev/null +++ b/.github/gh-config-template/README.md @@ -0,0 +1,12 @@ +# Generate github actions from template + +ytt -f ./gh_template.yml -f [ytt-helpers.star](https://github.com/cloudfoundry/wg-app-platform-runtime-ci/blob/main/shared/helpers/ytt-helpers.star) -f [index.yml](https://github.com/cloudfoundry/wg-app-platform-runtime-ci/blob/main/ci-networking-release/index.yml) > ./workflows/tests-workflow.yml + +## Supported jobs +- Template tests +- Basic Verifications +- Unit and Integration tests + +### How to run + +Request the repo owner to add a label as `ready-to-run` to validate PR. \ No newline at end of file diff --git a/.github/gh-config-template/gh_template.yml b/.github/gh-config-template/gh_template.yml new file mode 100644 index 000000000..008206e4c --- /dev/null +++ b/.github/gh-config-template/gh_template.yml @@ -0,0 +1,216 @@ +#@ load("@ytt:data", "data") +#@ load("ytt-helpers.star", "helpers") +name: unit-integration-tests + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + branches: + - develop + paths: + - 'src/**' + - 'packages/**' + - 'jobs/**' + - 'config/**' + - 'scripts/**' + - '.github/workflows/**' + - '.github/helpers/**' + +env: + MAPPING: | + build_nats_server=src/code.cloudfoundry.org/vendor/github.com/nats-io/nats-server/v2 + FLAGS: | + --keep-going + --trace + -r + --fail-on-pending + --randomize-all + --nodes=7 + --race + --timeout 30m + --flake-attempts 2 + RUN_AS: root + VERIFICATIONS: | + verify_go repo/$DIR + verify_go_version_match_bosh_release repo + verify_gofmt repo/$DIR + verify_govet repo/$DIR + verify_staticcheck repo/$DIR + FUNCTIONS: "" + DB: "" + +jobs: + repo-clone: + runs-on: ubuntu-latest + steps: + - name: cf-networking-release-repo + uses: actions/checkout@v4.3.1 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + submodules: recursive + path: repo + - name: Check out wg-appruntime code + uses: actions/checkout@v4.3.1 + with: + repository: cloudfoundry/wg-app-platform-runtime-ci + path: ci + - name: zip repo artifacts + run: | + tar -czf repo-artifact.tar.gz repo + tar -czf ci-artifact.tar.gz ci + - name: upload artifact + uses: actions/upload-artifact@v4 + with: + name: repo + path: | + repo-artifact.tar.gz + ci-artifact.tar.gz + determine-image-tag: + if: contains(github.event.pull_request.labels.*.name, 'ready-to-run') + runs-on: ubuntu-latest + outputs: + go_version: ${{ steps.get-version.outputs.go_version }} + steps: + - name: checkout ci repo + uses: actions/checkout@v4.3.1 + with: + repository: cloudfoundry/wg-app-platform-runtime-ci + sparse-checkout: go-version.json + sparse-checkout-cone-mode: false + - name: get-version + id: get-version + run: | + version=$(jq -r '.releases["cf-networking"] // .default' go-version.json) + echo "go_version=${version}" >> "$GITHUB_OUTPUT" + template-tests: + runs-on: ubuntu-latest + needs: [repo-clone] + container: cloudfoundry/tas-runtime-build:latest + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: template-tests + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-tests-templates/task.bash + lint-repo: + runs-on: ubuntu-latest + needs: [repo-clone] + container: cloudfoundry/tas-runtime-build:latest + env: + LINTERS: | + sync-package-specs.bash + sync-submodule-config.bash + match-golang-os-package-versions.bash + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: lint-repo + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/lint-repo/task.bash + test-on-mysql-5-7: + if: contains(github.event.pull_request.labels.*.name, 'ready-to-run') + runs-on: ubuntu-latest + needs: [repo-clone, determine-image-tag] + env: + BUILD_IMAGE: cloudfoundry/tas-runtime-mysql-5.7:${{ needs.determine-image-tag.outputs.go_version }} + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: pull image + run: docker pull "$BUILD_IMAGE" +#@ for package in helpers.packages_with_configure_db(data.values.internal_repos): + - name: #@ "{}-mysql".format(package.name) + env: + DIR: #@ "src/code.cloudfoundry.org/{}".format(package.name) + DB: mysql + run: | + ./repo/.github/helpers/test.bash ${{ github.workspace }} "$BUILD_IMAGE" +#@ end + test-repos-withoutdb: + if: contains(github.event.pull_request.labels.*.name, 'ready-to-run') + runs-on: ubuntu-latest + needs: [repo-clone, determine-image-tag] + env: + BUILD_IMAGE: cloudfoundry/tas-runtime-build:${{ needs.determine-image-tag.outputs.go_version }} + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: pull image + run: docker pull "$BUILD_IMAGE" +#@ for package in helpers.packages_without_configure_db(data.values.internal_repos): + - name: #@ package.name + env: + DIR: #@ "src/code.cloudfoundry.org/{}".format(package.name) + run: | + export DIR=$DIR + ./repo/.github/helpers/test.bash ${{ github.workspace }} "$BUILD_IMAGE" +#@ end + test-on-postgres: + if: contains(github.event.pull_request.labels.*.name, 'ready-to-run') + runs-on: ubuntu-latest + needs: [repo-clone, determine-image-tag] + env: + BUILD_IMAGE: cloudfoundry/tas-runtime-postgres:${{ needs.determine-image-tag.outputs.go_version }} + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: pull image + run: docker pull "$BUILD_IMAGE" +#@ for package in helpers.packages_with_configure_db(data.values.internal_repos): + - name: #@ "{}-postgres".format(package.name) + env: + DIR: #@ "src/code.cloudfoundry.org/{}".format(package.name) + DB: postgres + run: | + ./repo/.github/helpers/test.bash ${{ github.workspace }} "$BUILD_IMAGE" +#@ end + test-on-mysql-8-0: + if: contains(github.event.pull_request.labels.*.name, 'ready-to-run') + runs-on: ubuntu-latest + needs: [repo-clone, determine-image-tag] + env: + BUILD_IMAGE: cloudfoundry/tas-runtime-mysql-8.0:${{ needs.determine-image-tag.outputs.go_version }} + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: pull image + run: docker pull "$BUILD_IMAGE" +#@ for package in helpers.packages_with_configure_db(data.values.internal_repos): + - name: #@ "{}-mysql".format(package.name) + env: + DIR: #@ "src/code.cloudfoundry.org/{}".format(package.name) + DB: mysql + run: | + ./repo/.github/helpers/test.bash ${{ github.workspace }} "$BUILD_IMAGE" +#@ end diff --git a/.github/helpers/test.bash b/.github/helpers/test.bash new file mode 100755 index 000000000..8106d2abb --- /dev/null +++ b/.github/helpers/test.bash @@ -0,0 +1,12 @@ +#!/bin/bash + +set -eu +set -o pipefail +GH_WORKSPACE=$1 +BUILD_IMAGE=$2 +docker run --cap-add=SYS_ADMIN --rm --privileged \ + -v "$GH_WORKSPACE:/workspace" \ + -w /workspace \ + -e MAPPING="$MAPPING" -e DB="$DB" -e DIR="$DIR" -e RUN_AS="$RUN_AS" -e VERIFICATIONS="$VERIFICATIONS" -e FUNCTIONS="$FUNCTIONS" \ + "$BUILD_IMAGE" \ + bash ./ci/shared/tasks/run-bin-test/task.bash --keep-going --trace -r --fail-on-pending --randomize-all --nodes=7 --race --timeout 30m --flake-attempts 2 diff --git a/.github/workflows/tests-workflow.yml b/.github/workflows/tests-workflow.yml new file mode 100644 index 000000000..34b45ad25 --- /dev/null +++ b/.github/workflows/tests-workflow.yml @@ -0,0 +1,228 @@ +name: unit-integration-tests + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + branches: + - develop + paths: + - 'src/**' + - 'packages/**' + - 'jobs/**' + - 'config/**' + - 'scripts/**' + - '.github/workflows/**' + - '.github/helpers/**' +env: + MAPPING: | + build_nats_server=src/code.cloudfoundry.org/vendor/github.com/nats-io/nats-server/v2 + FLAGS: | + --keep-going + --trace + -r + --fail-on-pending + --randomize-all + --nodes=7 + --race + --timeout 30m + --flake-attempts 2 + RUN_AS: root + VERIFICATIONS: | + verify_go repo/$DIR + verify_go_version_match_bosh_release repo + verify_gofmt repo/$DIR + verify_govet repo/$DIR + verify_staticcheck repo/$DIR + FUNCTIONS: "" + DB: "" +jobs: + repo-clone: + runs-on: ubuntu-latest + steps: + - name: cf-networking-release-repo + uses: actions/checkout@v4.3.1 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + submodules: recursive + path: repo + - name: Check out wg-appruntime code + uses: actions/checkout@v4.3.1 + with: + repository: cloudfoundry/wg-app-platform-runtime-ci + path: ci + - name: zip repo artifacts + run: | + tar -czf repo-artifact.tar.gz repo + tar -czf ci-artifact.tar.gz ci + - name: upload artifact + uses: actions/upload-artifact@v4 + with: + name: repo + path: | + repo-artifact.tar.gz + ci-artifact.tar.gz + determine-image-tag: + if: contains(github.event.pull_request.labels.*.name, 'ready-to-run') + runs-on: ubuntu-latest + outputs: + go_version: ${{ steps.get-version.outputs.go_version }} + steps: + - name: checkout ci repo + uses: actions/checkout@v4.3.1 + with: + repository: cloudfoundry/wg-app-platform-runtime-ci + sparse-checkout: go-version.json + sparse-checkout-cone-mode: false + - name: get-version + id: get-version + run: | + version=$(jq -r '.releases["cf-networking"] // .default' go-version.json) + echo "go_version=${version}" >> "$GITHUB_OUTPUT" + template-tests: + runs-on: ubuntu-latest + needs: [repo-clone] + container: cloudfoundry/tas-runtime-build:latest + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: template-tests + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/run-tests-templates/task.bash + lint-repo: + runs-on: ubuntu-latest + needs: [repo-clone] + container: cloudfoundry/tas-runtime-build:latest + env: + LINTERS: | + sync-package-specs.bash + sync-submodule-config.bash + match-golang-os-package-versions.bash + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: lint-repo + run: | + "${GITHUB_WORKSPACE}"/ci/shared/tasks/lint-repo/task.bash + test-on-mysql-5-7: + if: contains(github.event.pull_request.labels.*.name, 'ready-to-run') + runs-on: ubuntu-latest + needs: [repo-clone, determine-image-tag] + env: + BUILD_IMAGE: cloudfoundry/tas-runtime-mysql-5.7:${{ needs.determine-image-tag.outputs.go_version }} + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: pull image + run: docker pull "$BUILD_IMAGE" + - name: policy-server-mysql + env: + DIR: src/code.cloudfoundry.org/policy-server + DB: mysql + run: | + ./repo/.github/helpers/test.bash ${{ github.workspace }} "$BUILD_IMAGE" + test-repos-withoutdb: + if: contains(github.event.pull_request.labels.*.name, 'ready-to-run') + runs-on: ubuntu-latest + needs: [repo-clone, determine-image-tag] + env: + BUILD_IMAGE: cloudfoundry/tas-runtime-build:${{ needs.determine-image-tag.outputs.go_version }} + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: pull image + run: docker pull "$BUILD_IMAGE" + - name: bosh-dns-adapter + env: + DIR: src/code.cloudfoundry.org/bosh-dns-adapter + run: | + export DIR=$DIR + ./repo/.github/helpers/test.bash ${{ github.workspace }} "$BUILD_IMAGE" + - name: cf-pusher + env: + DIR: src/code.cloudfoundry.org/cf-pusher + run: | + export DIR=$DIR + ./repo/.github/helpers/test.bash ${{ github.workspace }} "$BUILD_IMAGE" + - name: garden-external-networker + env: + DIR: src/code.cloudfoundry.org/garden-external-networker + run: | + export DIR=$DIR + ./repo/.github/helpers/test.bash ${{ github.workspace }} "$BUILD_IMAGE" + - name: lib + env: + DIR: src/code.cloudfoundry.org/lib + run: | + export DIR=$DIR + ./repo/.github/helpers/test.bash ${{ github.workspace }} "$BUILD_IMAGE" + - name: service-discovery-controller + env: + DIR: src/code.cloudfoundry.org/service-discovery-controller + run: | + export DIR=$DIR + ./repo/.github/helpers/test.bash ${{ github.workspace }} "$BUILD_IMAGE" + test-on-postgres: + if: contains(github.event.pull_request.labels.*.name, 'ready-to-run') + runs-on: ubuntu-latest + needs: [repo-clone, determine-image-tag] + env: + BUILD_IMAGE: cloudfoundry/tas-runtime-postgres:${{ needs.determine-image-tag.outputs.go_version }} + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: pull image + run: docker pull "$BUILD_IMAGE" + - name: policy-server-postgres + env: + DIR: src/code.cloudfoundry.org/policy-server + DB: postgres + run: | + ./repo/.github/helpers/test.bash ${{ github.workspace }} "$BUILD_IMAGE" + test-on-mysql-8-0: + if: contains(github.event.pull_request.labels.*.name, 'ready-to-run') + runs-on: ubuntu-latest + needs: [repo-clone, determine-image-tag] + env: + BUILD_IMAGE: cloudfoundry/tas-runtime-mysql-8.0:${{ needs.determine-image-tag.outputs.go_version }} + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: repo + - run: | + tar -xzvf repo-artifact.tar.gz + tar -xzvf ci-artifact.tar.gz + - name: pull image + run: docker pull "$BUILD_IMAGE" + - name: policy-server-mysql + env: + DIR: src/code.cloudfoundry.org/policy-server + DB: mysql + run: | + ./repo/.github/helpers/test.bash ${{ github.workspace }} "$BUILD_IMAGE"