diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f87a1f7..9a622f0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3,10 +3,25 @@ name: MatrixOne CI on: workflow_call: + inputs: + repository: + description: "Optional MatrixOne repository override for manual integration runs" + required: false + type: string + ref: + description: "Optional MatrixOne ref override for manual integration runs" + required: false + type: string secrets: TOKEN_ACTION: description: "Token for checkout (e.g. pull from fork/private)" required: false + TEST_S3FS_ALIYUN: + description: "Object storage configuration for UT coverage" + required: true + TEST_S3FS_QCLOUD: + description: "Object storage configuration for UT coverage" + required: true S3ENDPOINT: description: "S3ENDPOINT For Test" required: true @@ -103,8 +118,8 @@ jobs: with: token: ${{ secrets.TOKEN_ACTION }} fetch-depth: "3" - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }} + ref: ${{ inputs.ref || github.event.pull_request.head.ref }} - name: Set up Go uses: matrixorigin/CI/actions/setup-env@main with: @@ -172,6 +187,65 @@ jobs: run: | find ./ut-report -name top.txt -exec cat {} \; + ut-coverage-linux-x86: + environment: ci + runs-on: ${{ vars.COVERAGE_UT_RUNNER_LABEL || 'amd64-mo-guangzhou-2xlarge16' }} + name: UT Coverage on Linux/x86 + timeout-minutes: 60 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "3" + repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }} + ref: ${{ inputs.ref || github.event.pull_request.head.ref }} + - name: Set up Go + uses: matrixorigin/CI/actions/setup-env@main + with: + setup-java: true + go-version-file: "${{ github.workspace }}/go.mod" + - name: Run coverage unit tests + run: | + set -euo pipefail + cd "$GITHUB_WORKSPACE" + export endpoint='${{ secrets.S3ENDPOINT }}' + export region='${{ secrets.S3REGION }}' + export apikey='${{ secrets.S3APIKEY }}' + export apisecret='${{ secrets.S3APISECRET }}' + export bucket='${{ secrets.S3BUCKET }}' + export TEST_S3FS_ALIYUN='${{ secrets.TEST_S3FS_ALIYUN }}' + export TEST_S3FS_QCLOUD='${{ secrets.TEST_S3FS_QCLOUD }}' + + test_scope=$(go list -mod=readonly ./... | grep -v 'driver\|engine/aoe\|engine/memEngine\|pkg/catalog') + cover_pkgs=$(printf '%s\n' "${test_scope}" | paste -sd, -) + test -n "${cover_pkgs}" + + make clean && make config && make cgo && make thirdparties + thirdparties_dir="$GITHUB_WORKSPACE/thirdparties" + thirdparties_install_dir="${thirdparties_dir}/install" + cgo_cflags="-I${GITHUB_WORKSPACE}/cgo -I${thirdparties_install_dir}/include" + cgo_ldflags="-L${GITHUB_WORKSPACE}/cgo -lmo -L${thirdparties_install_dir}/lib -lusearch_c -Wl,-rpath,${thirdparties_install_dir}/lib -lm" + coverage_profile="${RUNNER_TEMP}/ut-coverage.out" + + CGO_CFLAGS="${cgo_cflags}" CGO_LDFLAGS="${cgo_ldflags}" \ + go test -mod=readonly -json -short -v -tags matrixone_test -p 6 \ + -covermode=set -coverprofile="${coverage_profile}" -coverpkg="${cover_pkgs}" \ + ${test_scope} | tee "${RUNNER_TEMP}/ut-coverage-report.json" + test -s "${coverage_profile}" + grep -Ev 'pkg/pb|pkg/sql/parsers/goyacc|yaccpar' "${coverage_profile}" > "${coverage_profile}.filtered" || true + test -s "${coverage_profile}.filtered" + mv "${coverage_profile}.filtered" "${coverage_profile}" + - name: Upload UT coverage + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: ut-coverage + path: | + ${{ runner.temp }}/ut-coverage.out + ${{ runner.temp }}/ut-coverage-report.json + if-no-files-found: warn + retention-days: 7 + ut-mac-x86: #if: ${{ !github.event.pull_request.draft }} if: ${{ false }} @@ -184,8 +258,8 @@ jobs: with: token: ${{ secrets.TOKEN_ACTION }} fetch-depth: "3" - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }} + ref: ${{ inputs.ref || github.event.pull_request.head.ref }} - name: Set up Go uses: matrixorigin/CI/actions/setup-env@main with: @@ -223,8 +297,8 @@ jobs: with: token: ${{ secrets.TOKEN_ACTION }} fetch-depth: "3" - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }} + ref: ${{ inputs.ref || github.event.pull_request.head.ref }} - uses: thiagodnf/yaml-schema-checker@v0.0.12 name: Check Issue Template with: diff --git a/.github/workflows/coverage-integration.yaml b/.github/workflows/coverage-integration.yaml new file mode 100644 index 0000000..62bb9a4 --- /dev/null +++ b/.github/workflows/coverage-integration.yaml @@ -0,0 +1,53 @@ +name: MatrixOne coverage integration + +on: + push: + branches: + - xp/ci-race-coverage-artifacts + workflow_dispatch: + inputs: + repository: + description: MatrixOne repository to test + required: true + default: matrixorigin/matrixone + type: string + ref: + description: MatrixOne branch or SHA to test + required: true + type: string + pr_number: + description: MatrixOne PR number used for the diff coverage gate + required: true + type: string + +jobs: + matrixone-ci: + uses: ./.github/workflows/ci.yaml + with: + repository: ${{ inputs.repository || 'matrixorigin/matrixone' }} + ref: ${{ inputs.ref || 'xp/ci-race-coverage-artifacts' }} + secrets: inherit + + matrixone-compose-ci: + uses: ./.github/workflows/e2e-compose.yaml + with: + repository: ${{ inputs.repository || 'matrixorigin/matrixone' }} + ref: ${{ inputs.ref || 'xp/ci-race-coverage-artifacts' }} + secrets: inherit + + matrixone-standalone-ci: + uses: ./.github/workflows/e2e-standalone.yaml + with: + repository: ${{ inputs.repository || 'matrixorigin/matrixone' }} + ref: ${{ inputs.ref || 'xp/ci-race-coverage-artifacts' }} + secrets: inherit + + matrixone-utils-ci: + needs: [matrixone-ci, matrixone-compose-ci, matrixone-standalone-ci] + if: ${{ always() }} + uses: ./.github/workflows/utils.yaml + with: + repository: ${{ inputs.repository || 'matrixorigin/matrixone' }} + ref: ${{ inputs.ref || 'xp/ci-race-coverage-artifacts' }} + pr_number: ${{ inputs.pr_number || '26179' }} + secrets: inherit diff --git a/.github/workflows/e2e-compose.yaml b/.github/workflows/e2e-compose.yaml index 4ffc3ab..ce90f26 100644 --- a/.github/workflows/e2e-compose.yaml +++ b/.github/workflows/e2e-compose.yaml @@ -2,12 +2,46 @@ name: MatrixOne Compose Test on: workflow_call: + inputs: + repository: + description: "Optional MatrixOne repository override for manual integration runs" + required: false + type: string + ref: + description: "Optional MatrixOne ref override for manual integration runs" + required: false + type: string secrets: TOKEN_ACTION: description: "Token for checkout (e.g. pull from fork/private)" required: false jobs: + build-compose-proxy-image: + if: ${{ inputs.repository != '' || !github.event.pull_request.draft }} + environment: ci + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + name: build Compose Proxy image + timeout-minutes: 30 + steps: + - name: checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "3" + repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }} + ref: ${{ inputs.ref || github.event.pull_request.head.ref }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and export Compose Proxy image cache + uses: docker/build-push-action@v6 + with: + context: . + file: ./optools/images/Dockerfile + build-args: GOBUILD_OPT=-cover + cache-from: type=gha,scope=matrixone-compose-proxy-${{ github.event.pull_request.number || github.run_id }} + cache-to: type=gha,scope=matrixone-compose-proxy-${{ github.event.pull_request.number || github.run_id }},mode=max + bvt-docker-compose-push: if: false environment: ci @@ -30,8 +64,8 @@ jobs: with: token: ${{ secrets.TOKEN_ACTION }} fetch-depth: "3" - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }} + ref: ${{ inputs.ref || github.event.pull_request.head.ref }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main with: @@ -181,11 +215,18 @@ jobs: retention-days: 7 multi-CN-bvt-docker-compose-pessimistic: - if: ${{ !github.event.pull_request.draft }} + if: ${{ inputs.repository != '' || !github.event.pull_request.draft }} + needs: build-compose-proxy-image environment: ci runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} - name: multi cn e2e bvt test docker compose(PESSIMISTIC) + name: multi cn e2e bvt test docker compose(PROXY, slot ${{ matrix.group_offset }}) timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + # Together with the native launch job's seed+1 group, these two slots + # execute every non-optimistic BVT case exactly once per CI run. + group_offset: [0, 2] steps: - name: Pre Clean Some Unneeded Images run: | @@ -201,8 +242,8 @@ jobs: with: token: ${{ secrets.TOKEN_ACTION }} fetch-depth: "3" - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }} + ref: ${{ inputs.ref || github.event.pull_request.head.ref }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main with: @@ -211,6 +252,19 @@ jobs: - name: Print run attempt run: echo "run attempt is ${{ github.run_attempt }}" + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Load Compose Proxy image from shared build cache + uses: docker/build-push-action@v6 + with: + context: . + file: ./optools/images/Dockerfile + tags: matrixorigin/matrixone:latest + load: true + build-args: GOBUILD_OPT=-cover + cache-from: type=gha,scope=matrixone-compose-proxy-${{ github.event.pull_request.number || github.run_id }} + - name: Print Disk Usage Before Container Start if: ${{ always() }} run: | @@ -224,6 +278,10 @@ jobs: - name: docker compose launch-multi-cn timeout-minutes: 10 run: | + set -euo pipefail + coverage_dir="${GITHUB_WORKSPACE}/coverage" + rm -rf "${coverage_dir}" + mkdir -p "${coverage_dir}" cat ./etc/launch-tae-compose/config/cn-0.toml cat ./etc/launch-tae-compose/config/cn-1.toml cat ./etc/launch-tae-compose/config/tn.toml @@ -233,9 +291,6 @@ jobs: # Handle various docker-compose volume formats: /test, /test:options, /test followed by space sed -i 's|:/test\b|:'"$GITHUB_WORKSPACE"'/test|g' ./etc/launch-tae-compose/compose.yaml - # build matrixone docker image first - docker build -t matrixorigin/matrixone:latest . -f ./optools/images/Dockerfile - mkdir -p ${{ github.workspace }}/docker-compose-log docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn up -d @@ -292,7 +347,22 @@ jobs: cat mo.yml echo "=============================" - ./run.sh -n -g -o -p $GITHUB_WORKSPACE/test/distributed/cases -e optimistic 2>&1 + run_id="${{ github.run_id }}" + run_attempt="${{ github.run_attempt }}" + group_seed=$(( (run_id + run_attempt) % 3 )) + bvt_group=$(( (group_seed + ${{ matrix.group_offset }}) % 3 )) + echo "Compose Proxy BVT: run ${run_id}, attempt ${run_attempt}, seed ${group_seed}, group ${bvt_group}" + { + echo "### BVT group assignment" + echo "- CI run: ${run_id}, attempt: ${run_attempt}" + echo "- Deployment: Compose Multi-CN + Proxy" + echo "- Group: ${bvt_group}" + } >> "$GITHUB_STEP_SUMMARY" + cd "$GITHUB_WORKSPACE" + bash ./optools/run_bvt_group.sh \ + "$GITHUB_WORKSPACE/mo-tester" \ + "$GITHUB_WORKSPACE/test/distributed/cases" \ + "${bvt_group}" 2>&1 - name: Print Docker Info Before Container Shutdown if: ${{ always() }} @@ -332,11 +402,39 @@ jobs: docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn down --remove-orphans docker volume rm launch-tae-compose_minio_storage + - name: Generate compose BVT coverage profile + if: ${{ always() && !cancelled() }} + run: | + set -uo pipefail + coverage_dir="${GITHUB_WORKSPACE}/coverage" + coverage_profile="${RUNNER_TEMP}/bvt-compose-${{ matrix.group_offset }}.out" + sudo chown -R "$(id -u):$(id -g)" "${coverage_dir}" || true + if go tool covdata textfmt -i="${coverage_dir}" -o "${coverage_profile}"; then + test -s "${coverage_profile}" + grep -Ev 'pkg/pb|yaccpar' "${coverage_profile}" > "${coverage_profile}.filtered" || true + test -s "${coverage_profile}.filtered" + mv "${coverage_profile}.filtered" "${coverage_profile}" + elif [ "${{ steps.bvt_on_pr_version.conclusion }}" = "success" ]; then + echo "::error::failed to generate compose BVT coverage after a successful BVT" + exit 1 + else + echo "::warning::compose BVT coverage unavailable because BVT did not finish successfully" + fi + + - name: Upload compose BVT coverage + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-coverage-compose-${{ matrix.group_offset }} + path: ${{ runner.temp }}/bvt-compose-${{ matrix.group_offset }}.out + if-no-files-found: warn + retention-days: 7 + - uses: actions/upload-artifact@v7 if: ${{ failure() || cancelled()}} continue-on-error: true with: - name: Compose-multi-cn-e2e-bvt-test-docker-log(PESSIMISTIC) + name: Compose-multi-cn-e2e-bvt-test-docker-log(PROXY,group-${{ matrix.group_offset }}) path: | ${{ github.workspace }}/docker-compose-log retention-days: 7 diff --git a/.github/workflows/e2e-standalone.yaml b/.github/workflows/e2e-standalone.yaml index 1ecbb25..2604615 100644 --- a/.github/workflows/e2e-standalone.yaml +++ b/.github/workflows/e2e-standalone.yaml @@ -2,6 +2,15 @@ name: MatrixOne e2e CI(Standalone) on: workflow_call: + inputs: + repository: + description: "Optional MatrixOne repository override for manual integration runs" + required: false + type: string + ref: + description: "Optional MatrixOne ref override for manual integration runs" + required: false + type: string secrets: TOKEN_ACTION: description: "Token for checkout (e.g. pull from fork/private)" @@ -21,15 +30,15 @@ jobs: token: ${{ secrets.TOKEN_ACTION }} fetch-depth: "3" path: ./head - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }} + ref: ${{ inputs.ref || github.event.pull_request.head.ref }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main with: go-version-file: "${{ github.workspace }}/head/go.mod" - name: Build MatrixOne run: | - cd $GITHUB_WORKSPACE/head && make clean && make build + cd $GITHUB_WORKSPACE/head && make clean && make GOBUILD_OPT=-cover build git rev-parse --short HEAD - name: echo config run: | @@ -145,6 +154,10 @@ jobs: retention-days: 7 multi-cn-proxy-bvt-linux-x86: + # Compose now supplies the proxy deployment used by the PR BVT matrix. + # Keep this legacy launcher disabled until it is assigned to a separate + # scheduled compatibility workflow. + if: false runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} name: Multi-CN e2e BVT Test on Linux/x64(LAUNCH, PROXY) timeout-minutes: 75 @@ -156,8 +169,8 @@ jobs: with: token: ${{ secrets.TOKEN_ACTION }} path: ./head - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }} + ref: ${{ inputs.ref || github.event.pull_request.head.ref }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main with: @@ -182,6 +195,9 @@ jobs: run: | sudo cat /etc/hosts | grep '127.0.0.1'; cd $GITHUB_WORKSPACE/head + export GOCOVERDIR="${RUNNER_TEMP}/bvt-coverage-proxy" + rm -rf "${GOCOVERDIR}" + mkdir -p "${GOCOVERDIR}" ./optools/run_bvt.sh $GITHUB_WORKSPACE/head launch-dynamic-with-proxy -with-proxy - name: Clone test-tool repository uses: actions/checkout@v6 @@ -214,6 +230,39 @@ jobs: cd $GITHUB_WORKSPACE/head # one node in one Process ./optools/check_log_count.sh 1000 60 # {count threshold} {metric collected interval} + - name: Generate proxy BVT coverage profile + if: ${{ always() && !cancelled() }} + run: | + set -uo pipefail + coverage_dir="${RUNNER_TEMP}/bvt-coverage-proxy" + pids=$(pgrep -f 'mo-service' || true) + if [ -n "${pids}" ]; then + kill -TERM ${pids} || true + fi + for _ in $(seq 1 30); do + pgrep -f 'mo-service' >/dev/null || break + sleep 1 + done + pids=$(pgrep -f 'mo-service' || true) + if [ -n "${pids}" ]; then + kill -KILL ${pids} || true + fi + if go tool covdata textfmt -i="${coverage_dir}" -o "${coverage_dir}/bvt-proxy.out"; then + test -s "${coverage_dir}/bvt-proxy.out" + elif [ "${{ steps.bvt_on_pr_version.conclusion }}" = "success" ]; then + echo "::error::failed to generate proxy BVT coverage after a successful BVT" + exit 1 + else + echo "::warning::proxy BVT coverage unavailable because BVT did not finish successfully" + fi + - name: Upload proxy BVT coverage + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-coverage-proxy + path: ${{ runner.temp }}/bvt-coverage-proxy/bvt-proxy.out + if-no-files-found: warn + retention-days: 7 - name: generate upload files if: ${{ always() }} continue-on-error: true @@ -246,8 +295,8 @@ jobs: token: ${{ secrets.TOKEN_ACTION }} fetch-depth: "3" path: ./head - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }} + ref: ${{ inputs.ref || github.event.pull_request.head.ref }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main @@ -256,7 +305,7 @@ jobs: - name: Build MatrixOne run: | - cd $GITHUB_WORKSPACE/head && make clean && make build + cd $GITHUB_WORKSPACE/head && make clean && make GOBUILD_OPT=-cover build git rev-parse --short HEAD - name: Add cn.txn @@ -275,6 +324,9 @@ jobs: - name: Start MO run: | cd $GITHUB_WORKSPACE/head + export GOCOVERDIR="${RUNNER_TEMP}/bvt-coverage-pessimistic" + rm -rf "${GOCOVERDIR}" + mkdir -p "${GOCOVERDIR}" ./optools/run_bvt.sh $GITHUB_WORKSPACE/head launch - name: Clone test-tool repository @@ -292,12 +344,25 @@ jobs: run: | export LC_ALL="C.UTF-8" locale - cd $GITHUB_WORKSPACE/mo-tester - sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml - cat mo.yml + sed -i 's/ port: [0-9]*/ port: 12345/g' "$GITHUB_WORKSPACE/mo-tester/mo.yml" + cat "$GITHUB_WORKSPACE/mo-tester/mo.yml" echo "=============================" - - ./run.sh -n -g -o -p $GITHUB_WORKSPACE/head/test/distributed/cases -s $GITHUB_WORKSPACE/head/test/distributed/resources -e optimistic 2>&1 + run_id="${{ github.run_id }}" + run_attempt="${{ github.run_attempt }}" + group_seed=$(( (run_id + run_attempt) % 3 )) + bvt_group=$(( (group_seed + 1) % 3 )) + echo "Pessimistic BVT: run ${run_id}, attempt ${run_attempt}, seed ${group_seed}, group ${bvt_group}" + { + echo "### BVT group assignment" + echo "- CI run: ${run_id}, attempt: ${run_attempt}" + echo "- Deployment: native launch (PESSIMISTIC)" + echo "- Group: ${bvt_group}" + } >> "$GITHUB_STEP_SUMMARY" + cd $GITHUB_WORKSPACE/head + bash ./optools/run_bvt_group.sh \ + "$GITHUB_WORKSPACE/mo-tester" \ + "$GITHUB_WORKSPACE/head/test/distributed/cases" \ + "${bvt_group}" 2>&1 - name: Dump mo-service goroutines if: ${{ always() && !cancelled() }} @@ -311,10 +376,43 @@ jobs: # 4 nodes in one Process ./optools/check_log_count.sh 4000 60 # {count threshold} {metric collected interval} - - name: Check mo-service Status + - name: Generate pessimistic BVT coverage profile if: ${{ always() && !cancelled() }} run: | - if [ "$(ps -ef | grep 'mo-service' | grep -v "grep" | wc -l)" -gt 0 ]; then pkill -9 mo-service; else echo 'current mo-service has already crashed'; exit 1; fi + set -uo pipefail + coverage_dir="${RUNNER_TEMP}/bvt-coverage-pessimistic" + coverage_profile="${RUNNER_TEMP}/bvt-pessimistic.out" + pids=$(pgrep -f 'mo-service' || true) + if [ -n "${pids}" ]; then + kill -TERM ${pids} || true + fi + for _ in $(seq 1 30); do + pgrep -f 'mo-service' >/dev/null || break + sleep 1 + done + pids=$(pgrep -f 'mo-service' || true) + if [ -n "${pids}" ]; then + kill -KILL ${pids} || true + fi + if go tool covdata textfmt -i="${coverage_dir}" -o "${coverage_profile}"; then + test -s "${coverage_profile}" + grep -Ev 'pkg/pb|yaccpar' "${coverage_profile}" > "${coverage_profile}.filtered" || true + test -s "${coverage_profile}.filtered" + mv "${coverage_profile}.filtered" "${coverage_profile}" + elif [ "${{ steps.bvt_on_pr_version.conclusion }}" = "success" ]; then + echo "::error::failed to generate pessimistic BVT coverage after a successful BVT" + exit 1 + else + echo "::warning::pessimistic BVT coverage unavailable because BVT did not finish successfully" + fi + - name: Upload pessimistic BVT coverage + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-coverage-pessimistic + path: ${{ runner.temp }}/bvt-pessimistic.out + if-no-files-found: warn + retention-days: 7 - name: generate upload files if: ${{ always() }} diff --git a/.github/workflows/utils.yaml b/.github/workflows/utils.yaml index 52d3d37..de27783 100644 --- a/.github/workflows/utils.yaml +++ b/.github/workflows/utils.yaml @@ -3,6 +3,19 @@ name: MatrixOne Utils CI on: workflow_call: + inputs: + repository: + description: "Optional MatrixOne repository override for manual integration runs" + required: false + type: string + ref: + description: "Optional MatrixOne ref override for manual integration runs" + required: false + type: string + pr_number: + description: "Optional MatrixOne PR number override for manual integration runs" + required: false + type: string secrets: TOKEN_ACTION: description: "A token passed from the caller workflow" @@ -52,6 +65,10 @@ jobs: - id: check_in_org name: CHECK ORGANIZATION USER run: | + if [ -n "${{ inputs.repository }}" ]; then + echo "in_org=1" >> $GITHUB_OUTPUT; + exit 0; + fi PAGE=1; PER_PAGE=100; USER_IN_ORG="0"; @@ -77,6 +94,10 @@ jobs: - id: check_safe_label name: CHECK PULL REQUEST LABEL run: | + if [ -n "${{ inputs.repository }}" ]; then + echo "safe_label=1" >> $GITHUB_OUTPUT; + exit 0; + fi labels=$(curl -L \ -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer ${{ secrets.TOKEN_ACTION }}" \ @@ -88,9 +109,11 @@ jobs: echo "safe_label=0" >> $GITHUB_OUTPUT; fi - pr_coverage: - if: ${{ needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1' }} - name: Coverage + pr_coverage_legacy_disabled: + # Replaced by coverage_merge below. Keep the old definition temporarily so + # its failure-analysis history remains easy to compare during rollout. + if: ${{ github.event.pull_request.number == 0 }} + name: Legacy serial coverage (disabled) needs: [check_organization_user] environment: ci # if you change runner, must modify L117 @@ -101,8 +124,8 @@ jobs: token: ${{ secrets.TOKEN_ACTION }} fetch-depth: "1" path: ./matrixone - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }} + ref: ${{ inputs.ref || github.event.pull_request.head.ref }} - name: Clone test-tool repository uses: actions/checkout@v6 with: @@ -449,7 +472,145 @@ jobs: ${{ github.workspace }}/merged_coverage.out retention-days: 7 + coverage_merge: + if: ${{ needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1' }} + name: Coverage + needs: [check_organization_user] + environment: ci + # This job only downloads and merges text coverage profiles; it does not + # build MatrixOne or run CGo. Use a GitHub-hosted runner so it is isolated + # from the self-hosted queues used by heavy UT and coverage producers. + runs-on: ${{ vars.COVERAGE_MERGE_RUNNER_LABEL || 'ubuntu-22.04' }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "1" + path: ./matrixone + repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }} + ref: ${{ inputs.ref || github.event.pull_request.head.ref }} + + - name: Checkout coverage parser + uses: actions/checkout@v6 + with: + token: ${{ secrets.TOKEN_ACTION }} + repository: matrixorigin/CI + fetch-depth: "1" + path: ./CI + + - name: Set up Go + uses: matrixorigin/CI/actions/setup-env@main + with: + setup-java: false + go-version-file: "${{ github.workspace }}/matrixone/go.mod" + + - name: Generate diff.patch + env: + GH_TOKEN: ${{ secrets.TOKEN_ACTION }} + IS_PUB_REPO: ${{ vars.IS_PUB_REPO }} + run: | + set -euo pipefail + cd "$GITHUB_WORKSPACE/matrixone" + pr_repo="${{ inputs.repository || github.event.pull_request.base.repo.full_name }}" + pr_number="${{ inputs.pr_number || github.event.pull_request.number }}" + if [ -n "${IS_PUB_REPO:-}" ] && [ "${IS_PUB_REPO}" != "0" ] && [ "${IS_PUB_REPO}" != "false" ]; then + curl -fL --retry 5 --retry-delay 3 \ + "https://github.com/${pr_repo}/pull/${pr_number}.diff" \ + -o diff.patch + else + curl -fL --retry 5 --retry-delay 3 \ + -H "Accept: application/vnd.github.v3.diff" \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + "https://api.github.com/repos/${pr_repo}/pulls/${pr_number}" \ + -o diff.patch + fi + + - name: Download test coverage artifacts + uses: actions/download-artifact@v7 + with: + pattern: '*-coverage*' + merge-multiple: true + path: ${{ github.workspace }}/coverage-artifacts + + - name: Merge UT and BVT coverage + run: | + set -euo pipefail + artifact_dir="$GITHUB_WORKSPACE/coverage-artifacts" + processed_dir="$GITHUB_WORKSPACE/coverage-processed" + mkdir -p "${processed_dir}" + + mapfile -t ut_profiles < <(find "${artifact_dir}" -type f -name 'ut-coverage.out' | sort) + mapfile -t bvt_profiles < <(find "${artifact_dir}" -type f -name 'bvt-*.out' | sort) + if [ "${#ut_profiles[@]}" -eq 0 ]; then + echo "::error::UT coverage artifact is missing" + exit 1 + fi + if [ "${#bvt_profiles[@]}" -lt 3 ]; then + echo "::error::expected coverage from two Compose Proxy jobs and one native pessimistic BVT job; found ${#bvt_profiles[@]} profile(s)" + find "${artifact_dir}" -type f -print || true + exit 1 + fi + + coverage_files=() + for profile in "${ut_profiles[@]}"; do + output="${processed_dir}/$(basename "${profile}")" + grep -Ev 'pkg/pb|pkg/sql/parsers/goyacc|yaccpar' "${profile}" > "${output}" || true + test -s "${output}" + coverage_files+=("${output}") + done + for profile in "${bvt_profiles[@]}"; do + output="${processed_dir}/$(basename "${profile}")" + grep -Ev 'pkg/pb|yaccpar' "${profile}" > "${output}" || true + test -s "${output}" + coverage_files+=("${output}") + done + + cd "$GITHUB_WORKSPACE" + # parse_coverage.py intentionally resolves .ignore from its current + # working directory. The legacy job staged this file; keep that + # contract in the artifact-based merge job too. + cp "$GITHUB_WORKSPACE/CI/scripts/.ignore" "$GITHUB_WORKSPACE/.ignore" + python CI/scripts/parse_coverage.py \ + -coverage_files "${coverage_files[@]}" \ + -diff_path "$GITHUB_WORKSPACE/matrixone/diff.patch" + go tool cover -o "$GITHUB_WORKSPACE/matrixone/pr_coverage.html" -html=merged_coverage.out + + - name: Measure coverage outputs + if: ${{ always() && !cancelled() }} + run: | + du -sh \ + "$GITHUB_WORKSPACE/coverage-artifacts" \ + "$GITHUB_WORKSPACE/coverage-processed" \ + "$GITHUB_WORKSPACE/merged_coverage.out" \ + "$GITHUB_WORKSPACE/pr_coverage.out" \ + "$GITHUB_WORKSPACE/matrixone/pr_coverage.html" 2>/dev/null || true + + - name: Upload coverage result + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: final-result-files + path: | + ${{ github.workspace }}/matrixone/pr_coverage.html + ${{ github.workspace }}/pr_coverage.out + ${{ github.workspace }}/merged_coverage.out + retention-days: 7 + + - name: Upload raw coverage inputs for failed merge + if: ${{ failure() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-merge-debug + path: | + ${{ github.workspace }}/coverage-artifacts + ${{ github.workspace }}/coverage-processed + ${{ github.workspace }}/matrixone/diff.patch + if-no-files-found: warn + retention-days: 1 + pr-size-label: + if: ${{ inputs.repository == '' }} environment: ci runs-on: arm64-mo-shanghai-4c8g needs: [check_organization_user]