ci: merge race UT and BVT coverage artifacts

.github/workflows/ci.yaml

@@ -3,10 +3,25 @@ name: MatrixOne CI
 
 on:
   workflow_call:
+    inputs:
+      repository:
+        description: "Optional MatrixOne repository override for manual integration runs"
+        required: false
+        type: string
+      ref:
+        description: "Optional MatrixOne ref override for manual integration runs"
+        required: false
+        type: string
     secrets:
       TOKEN_ACTION:
         description: "Token for checkout (e.g. pull from fork/private)"
         required: false
+      TEST_S3FS_ALIYUN:
+        description: "Object storage configuration for UT coverage"
+        required: true
+      TEST_S3FS_QCLOUD:
+        description: "Object storage configuration for UT coverage"
+        required: true
       S3ENDPOINT:
         description: "S3ENDPOINT For Test"
         required: true
@@ -103,8 +118,8 @@ jobs:
         with:
           token: ${{ secrets.TOKEN_ACTION }}
           fetch-depth: "3"
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }}
+          ref: ${{ inputs.ref || github.event.pull_request.head.ref }}
       - name: Set up Go
         uses: matrixorigin/CI/actions/setup-env@main
         with:
@@ -172,6 +187,65 @@ jobs:
         run: |
           find ./ut-report -name top.txt -exec cat {} \;
 
+  ut-coverage-linux-x86:
+    environment: ci
+    runs-on: ${{ vars.COVERAGE_UT_RUNNER_LABEL || 'amd64-mo-guangzhou-2xlarge16' }}
+    name: UT Coverage on Linux/x86
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          token: ${{ secrets.TOKEN_ACTION }}
+          fetch-depth: "3"
+          repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }}
+          ref: ${{ inputs.ref || github.event.pull_request.head.ref }}
+      - name: Set up Go
+        uses: matrixorigin/CI/actions/setup-env@main
+        with:
+          setup-java: true
+          go-version-file: "${{ github.workspace }}/go.mod"
+      - name: Run coverage unit tests
+        run: |
+          set -euo pipefail
+          cd "$GITHUB_WORKSPACE"
+          export endpoint='${{ secrets.S3ENDPOINT }}'
+          export region='${{ secrets.S3REGION }}'
+          export apikey='${{ secrets.S3APIKEY }}'
+          export apisecret='${{ secrets.S3APISECRET }}'
+          export bucket='${{ secrets.S3BUCKET }}'
+          export TEST_S3FS_ALIYUN='${{ secrets.TEST_S3FS_ALIYUN }}'
+          export TEST_S3FS_QCLOUD='${{ secrets.TEST_S3FS_QCLOUD }}'
+
+          test_scope=$(go list -mod=readonly ./... | grep -v 'driver\|engine/aoe\|engine/memEngine\|pkg/catalog')
+          cover_pkgs=$(printf '%s\n' "${test_scope}" | paste -sd, -)
+          test -n "${cover_pkgs}"
+
+          make clean && make config && make cgo && make thirdparties
+          thirdparties_dir="$GITHUB_WORKSPACE/thirdparties"
+          thirdparties_install_dir="${thirdparties_dir}/install"
+          cgo_cflags="-I${GITHUB_WORKSPACE}/cgo -I${thirdparties_install_dir}/include"
+          cgo_ldflags="-L${GITHUB_WORKSPACE}/cgo -lmo -L${thirdparties_install_dir}/lib -lusearch_c -Wl,-rpath,${thirdparties_install_dir}/lib -lm"
+          coverage_profile="${RUNNER_TEMP}/ut-coverage.out"
+
+          CGO_CFLAGS="${cgo_cflags}" CGO_LDFLAGS="${cgo_ldflags}" \
+            go test -mod=readonly -json -short -v -tags matrixone_test -p 6 \
+            -covermode=set -coverprofile="${coverage_profile}" -coverpkg="${cover_pkgs}" \
+            ${test_scope} | tee "${RUNNER_TEMP}/ut-coverage-report.json"
+          test -s "${coverage_profile}"
+          grep -Ev 'pkg/pb|pkg/sql/parsers/goyacc|yaccpar' "${coverage_profile}" > "${coverage_profile}.filtered" || true
+          test -s "${coverage_profile}.filtered"
+          mv "${coverage_profile}.filtered" "${coverage_profile}"
+      - name: Upload UT coverage
+        if: ${{ always() && !cancelled() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: ut-coverage
+          path: |
+            ${{ runner.temp }}/ut-coverage.out
+            ${{ runner.temp }}/ut-coverage-report.json
+          if-no-files-found: warn
+          retention-days: 7
+
   ut-mac-x86:
     #if: ${{ !github.event.pull_request.draft }}
     if: ${{ false }}
@@ -184,8 +258,8 @@ jobs:
         with:
           token: ${{ secrets.TOKEN_ACTION }}
           fetch-depth: "3"
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }}
+          ref: ${{ inputs.ref || github.event.pull_request.head.ref }}
       - name: Set up Go
         uses: matrixorigin/CI/actions/setup-env@main
         with:
@@ -223,8 +297,8 @@ jobs:
         with:
           token: ${{ secrets.TOKEN_ACTION }}
           fetch-depth: "3"
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }}
+          ref: ${{ inputs.ref || github.event.pull_request.head.ref }}
       - uses: thiagodnf/yaml-schema-checker@v0.0.12
         name: Check Issue Template
         with:

.github/workflows/coverage-integration.yaml

@@ -0,0 +1,53 @@
+name: MatrixOne coverage integration
+
+on:
+  push:
+    branches:
+      - xp/ci-race-coverage-artifacts
+  workflow_dispatch:
+    inputs:
+      repository:
+        description: MatrixOne repository to test
+        required: true
+        default: matrixorigin/matrixone
+        type: string
+      ref:
+        description: MatrixOne branch or SHA to test
+        required: true
+        type: string
+      pr_number:
+        description: MatrixOne PR number used for the diff coverage gate
+        required: true
+        type: string
+
+jobs:
+  matrixone-ci:
+    uses: ./.github/workflows/ci.yaml
+    with:
+      repository: ${{ inputs.repository || 'matrixorigin/matrixone' }}
+      ref: ${{ inputs.ref || 'xp/ci-race-coverage-artifacts' }}
+    secrets: inherit
+
+  matrixone-compose-ci:
+    uses: ./.github/workflows/e2e-compose.yaml
+    with:
+      repository: ${{ inputs.repository || 'matrixorigin/matrixone' }}
+      ref: ${{ inputs.ref || 'xp/ci-race-coverage-artifacts' }}
+    secrets: inherit
+
+  matrixone-standalone-ci:
+    uses: ./.github/workflows/e2e-standalone.yaml
+    with:
+      repository: ${{ inputs.repository || 'matrixorigin/matrixone' }}
+      ref: ${{ inputs.ref || 'xp/ci-race-coverage-artifacts' }}
+    secrets: inherit
+
+  matrixone-utils-ci:
+    needs: [matrixone-ci, matrixone-compose-ci, matrixone-standalone-ci]
+    if: ${{ always() }}
+    uses: ./.github/workflows/utils.yaml
+    with:
+      repository: ${{ inputs.repository || 'matrixorigin/matrixone' }}
+      ref: ${{ inputs.ref || 'xp/ci-race-coverage-artifacts' }}
+      pr_number: ${{ inputs.pr_number || '26179' }}
+    secrets: inherit

.github/workflows/e2e-compose.yaml

@@ -2,12 +2,46 @@ name: MatrixOne Compose Test
 
 on:
   workflow_call:
+    inputs:
+      repository:
+        description: "Optional MatrixOne repository override for manual integration runs"
+        required: false
+        type: string
+      ref:
+        description: "Optional MatrixOne ref override for manual integration runs"
+        required: false
+        type: string
     secrets:
       TOKEN_ACTION:
         description: "Token for checkout (e.g. pull from fork/private)"
         required: false
 
 jobs:
+  build-compose-proxy-image:
+    if: ${{ inputs.repository != '' || !github.event.pull_request.draft }}
+    environment: ci
+    runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }}
+    name: build Compose Proxy image
+    timeout-minutes: 30
+    steps:
+      - name: checkout
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          token: ${{ secrets.TOKEN_ACTION }}
+          fetch-depth: "3"
+          repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }}
+          ref: ${{ inputs.ref || github.event.pull_request.head.ref }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and export Compose Proxy image cache
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./optools/images/Dockerfile
+          build-args: GOBUILD_OPT=-cover
+          cache-from: type=gha,scope=matrixone-compose-proxy-${{ github.event.pull_request.number || github.run_id }}
+          cache-to: type=gha,scope=matrixone-compose-proxy-${{ github.event.pull_request.number || github.run_id }},mode=max
+
   bvt-docker-compose-push:
     if: false
     environment: ci
@@ -30,8 +64,8 @@ jobs:
         with:
           token: ${{ secrets.TOKEN_ACTION }}
           fetch-depth: "3"
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }}
+          ref: ${{ inputs.ref || github.event.pull_request.head.ref }}
       - name: Set up Go And Java
         uses: matrixorigin/CI/actions/setup-env@main
         with:
@@ -181,11 +215,18 @@ jobs:
           retention-days: 7
 
   multi-CN-bvt-docker-compose-pessimistic:
-    if: ${{ !github.event.pull_request.draft }}
+    if: ${{ inputs.repository != '' || !github.event.pull_request.draft }}
+    needs: build-compose-proxy-image
     environment: ci
     runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }}
-    name: multi cn e2e bvt test docker compose(PESSIMISTIC)
+    name: multi cn e2e bvt test docker compose(PROXY, slot ${{ matrix.group_offset }})
     timeout-minutes: 75
+    strategy:
+      fail-fast: false
+      matrix:
+        # Together with the native launch job's seed+1 group, these two slots
+        # execute every non-optimistic BVT case exactly once per CI run.
+        group_offset: [0, 2]
     steps:
       - name: Pre Clean Some Unneeded Images
         run: |
@@ -201,8 +242,8 @@ jobs:
         with:
           token: ${{ secrets.TOKEN_ACTION }}
           fetch-depth: "3"
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ inputs.repository || github.event.pull_request.head.repo.full_name }}
+          ref: ${{ inputs.ref || github.event.pull_request.head.ref }}
       - name: Set up Go And Java
         uses: matrixorigin/CI/actions/setup-env@main
         with:
@@ -211,6 +252,19 @@ jobs:
       - name: Print run attempt
         run: echo "run attempt is ${{ github.run_attempt }}"
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Load Compose Proxy image from shared build cache
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./optools/images/Dockerfile
+          tags: matrixorigin/matrixone:latest
+          load: true
+          build-args: GOBUILD_OPT=-cover
+          cache-from: type=gha,scope=matrixone-compose-proxy-${{ github.event.pull_request.number || github.run_id }}
+
       - name: Print Disk Usage Before Container Start
         if: ${{ always() }}
         run: |
@@ -224,6 +278,10 @@ jobs:
       - name: docker compose launch-multi-cn
         timeout-minutes: 10
         run: |
+          set -euo pipefail
+          coverage_dir="${GITHUB_WORKSPACE}/coverage"
+          rm -rf "${coverage_dir}"
+          mkdir -p "${coverage_dir}"
           cat ./etc/launch-tae-compose/config/cn-0.toml
           cat ./etc/launch-tae-compose/config/cn-1.toml
           cat ./etc/launch-tae-compose/config/tn.toml
@@ -233,9 +291,6 @@ jobs:
           # Handle various docker-compose volume formats: /test, /test:options, /test followed by space
           sed -i 's|:/test\b|:'"$GITHUB_WORKSPACE"'/test|g' ./etc/launch-tae-compose/compose.yaml
 
-          # build matrixone docker image first
-          docker build -t matrixorigin/matrixone:latest . -f ./optools/images/Dockerfile
-
           mkdir -p ${{ github.workspace }}/docker-compose-log
           docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn up -d
 
@@ -292,7 +347,22 @@ jobs:
           cat mo.yml
           echo "============================="
 
-          ./run.sh -n -g -o -p $GITHUB_WORKSPACE/test/distributed/cases -e optimistic 2>&1
+          run_id="${{ github.run_id }}"
+          run_attempt="${{ github.run_attempt }}"
+          group_seed=$(( (run_id + run_attempt) % 3 ))
+          bvt_group=$(( (group_seed + ${{ matrix.group_offset }}) % 3 ))
+          echo "Compose Proxy BVT: run ${run_id}, attempt ${run_attempt}, seed ${group_seed}, group ${bvt_group}"
+          {
+            echo "### BVT group assignment"
+            echo "- CI run: ${run_id}, attempt: ${run_attempt}"
+            echo "- Deployment: Compose Multi-CN + Proxy"
+            echo "- Group: ${bvt_group}"
+          } >> "$GITHUB_STEP_SUMMARY"
+          cd "$GITHUB_WORKSPACE"
+          bash ./optools/run_bvt_group.sh \
+            "$GITHUB_WORKSPACE/mo-tester" \
+            "$GITHUB_WORKSPACE/test/distributed/cases" \
+            "${bvt_group}" 2>&1
 
       - name: Print Docker Info Before Container Shutdown
         if: ${{ always() }}
@@ -332,11 +402,39 @@ jobs:
           docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn down --remove-orphans
           docker volume rm launch-tae-compose_minio_storage
 
+      - name: Generate compose BVT coverage profile
+        if: ${{ always() && !cancelled() }}
+        run: |
+          set -uo pipefail
+          coverage_dir="${GITHUB_WORKSPACE}/coverage"
+          coverage_profile="${RUNNER_TEMP}/bvt-compose-${{ matrix.group_offset }}.out"
+          sudo chown -R "$(id -u):$(id -g)" "${coverage_dir}" || true
+          if go tool covdata textfmt -i="${coverage_dir}" -o "${coverage_profile}"; then
+            test -s "${coverage_profile}"
+            grep -Ev 'pkg/pb|yaccpar' "${coverage_profile}" > "${coverage_profile}.filtered" || true
+            test -s "${coverage_profile}.filtered"
+            mv "${coverage_profile}.filtered" "${coverage_profile}"
+          elif [ "${{ steps.bvt_on_pr_version.conclusion }}" = "success" ]; then
+            echo "::error::failed to generate compose BVT coverage after a successful BVT"
+            exit 1
+          else
+            echo "::warning::compose BVT coverage unavailable because BVT did not finish successfully"
+          fi
+
+      - name: Upload compose BVT coverage
+        if: ${{ always() && !cancelled() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: bvt-coverage-compose-${{ matrix.group_offset }}
+          path: ${{ runner.temp }}/bvt-compose-${{ matrix.group_offset }}.out
+          if-no-files-found: warn
+          retention-days: 7
+
       - uses: actions/upload-artifact@v7
         if: ${{ failure() || cancelled()}}
         continue-on-error: true
         with:
-          name: Compose-multi-cn-e2e-bvt-test-docker-log(PESSIMISTIC)
+          name: Compose-multi-cn-e2e-bvt-test-docker-log(PROXY,group-${{ matrix.group_offset }})
           path: |
             ${{ github.workspace }}/docker-compose-log
           retention-days: 7