[PECOBLR-2086] Add comprehensive MST transaction metadata and edge case tests

.github/workflows/code-coverage.yml

@@ -0,0 +1,163 @@
+name: Code Coverage
+
+permissions:
+  contents: read
+
+on: [pull_request, workflow_dispatch]
+
+jobs:
+  test-with-coverage:
+    runs-on: ubuntu-latest
+    environment: azure-prod
+    env:
+      DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
+      DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
+      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
+      DATABRICKS_CATALOG: peco
+      DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
+    steps:
+      #----------------------------------------------
+      #       check-out repo and set-up python
+      #----------------------------------------------
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      #----------------------------------------------
+      #  -----  install system dependencies  -----
+      #----------------------------------------------
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libkrb5-dev
+      #----------------------------------------------
+      #  -----  install & configure poetry  -----
+      #----------------------------------------------
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: "2.2.1"
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      #----------------------------------------------
+      #       load cached venv if cache exists
+      #----------------------------------------------
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
+      #----------------------------------------------
+      # install dependencies if cache does not exist
+      #----------------------------------------------
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+      #----------------------------------------------
+      # install your root project, if required
+      #----------------------------------------------
+      - name: Install Kerberos system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libkrb5-dev
+      - name: Install library
+        run: poetry install --no-interaction --all-extras
+      #----------------------------------------------
+      #    run parallel tests with coverage
+      #----------------------------------------------
+      - name: Run parallel tests with coverage
+        continue-on-error: false
+        run: |
+          poetry run pytest tests/unit tests/e2e \
+            -m "not serial" \
+            -n auto \
+            --cov=src \
+            --cov-report=xml \
+            --cov-report=term \
+            -v
+
+      #----------------------------------------------
+      #    run telemetry tests with coverage (isolated)
+      #----------------------------------------------
+      - name: Run telemetry tests with coverage (isolated)
+        continue-on-error: false
+        run: |
+          # Run test_concurrent_telemetry.py separately for isolation
+          poetry run pytest tests/e2e/test_concurrent_telemetry.py \
+            --cov=src \
+            --cov-append \
+            --cov-report=xml \
+            --cov-report=term \
+            -v
+
+      #----------------------------------------------
+      #         check for coverage override
+      #----------------------------------------------
+      - name: Check for coverage override
+        id: override
+        run: |
+          OVERRIDE_COMMENT=$(echo "${{ github.event.pull_request.body }}" | grep -E "SKIP_COVERAGE_CHECK\s*=" || echo "")
+          if [ -n "$OVERRIDE_COMMENT" ]; then
+            echo "override=true" >> $GITHUB_OUTPUT
+            REASON=$(echo "$OVERRIDE_COMMENT" | sed -E 's/.*SKIP_COVERAGE_CHECK\s*=\s*(.+)/\1/')
+            echo "reason=$REASON" >> $GITHUB_OUTPUT
+            echo "Coverage override found in PR description: $REASON"
+          else
+            echo "override=false" >> $GITHUB_OUTPUT
+            echo "No coverage override found"
+          fi
+      #----------------------------------------------
+      #         check coverage percentage
+      #----------------------------------------------
+      - name: Check coverage percentage
+        if: steps.override.outputs.override == 'false'
+        run: |
+          COVERAGE_FILE="coverage.xml"
+          if [ ! -f "$COVERAGE_FILE" ]; then
+            echo "ERROR: Coverage file not found at $COVERAGE_FILE"
+            exit 1
+          fi
+
+          # Install xmllint if not available
+          if ! command -v xmllint &> /dev/null; then
+            sudo apt-get update && sudo apt-get install -y libxml2-utils
+          fi
+
+          COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "$COVERAGE_FILE")
+          TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "$COVERAGE_FILE")
+          PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))")
+
+          echo "Branch Coverage: $PERCENTAGE%"
+          echo "Required Coverage: 85%"
+
+          # Use Python to compare the coverage with 85
+          python3 -c "import sys; sys.exit(0 if float('$PERCENTAGE') >= 85 else 1)"
+          if [ $? -eq 1 ]; then
+            echo "ERROR: Coverage is $PERCENTAGE%, which is less than the required 85%"
+            exit 1
+          else
+            echo "SUCCESS: Coverage is $PERCENTAGE%, which meets the required 85%"
+          fi
+
+      #----------------------------------------------
+      #         coverage enforcement summary
+      #----------------------------------------------
+      - name: Coverage enforcement summary
+        run: |
+          if [ "${{ steps.override.outputs.override }}" == "true" ]; then
+            echo "⚠️ Coverage checks bypassed: ${{ steps.override.outputs.reason }}"
+            echo "Please ensure this override is justified and temporary"
+          else
+            echo "✅ Coverage checks enforced - minimum 85% required"
+          fi
+

.github/workflows/code-quality-checks.yml

@@ -1,17 +1,23 @@
 name: Code Quality Checks
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
+
+on: [pull_request]
+
 jobs:
   run-unit-tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+        dependency-version: ["default", "min"]
+        # Optimize matrix - test min/max on subset of Python versions
+        exclude:
+          - python-version: "3.12"
+            dependency-version: "min"
+          - python-version: "3.13"
+            dependency-version: "min"
+
+    name: "Unit Tests (Python ${{ matrix.python-version }}, ${{ matrix.dependency-version }} deps)"
+
     steps:
       #----------------------------------------------
       #       check-out repo and set-up python
@@ -29,6 +35,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: "2.2.1"
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
@@ -41,7 +48,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: .venv
-          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ matrix.dependency-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
       #----------------------------------------------
       # install dependencies if cache does not exist
       #----------------------------------------------
@@ -54,15 +61,47 @@ jobs:
       - name: Install library
         run: poetry install --no-interaction
       #----------------------------------------------
+      # override with custom dependency versions
+      #----------------------------------------------
+      - name: Install Python tools for custom versions
+        if: matrix.dependency-version != 'default'
+        run: poetry run pip install toml packaging
+
+      - name: Generate requirements file
+        if: matrix.dependency-version != 'default'
+        run: |
+          poetry run python scripts/dependency_manager.py ${{ matrix.dependency-version }} --output requirements-${{ matrix.dependency-version }}.txt
+          echo "Generated requirements for ${{ matrix.dependency-version }} versions:"
+          cat requirements-${{ matrix.dependency-version }}.txt
+
+      - name: Override with custom dependency versions
+        if: matrix.dependency-version != 'default'
+        run: poetry run pip install -r requirements-${{ matrix.dependency-version }}.txt
+
+      #----------------------------------------------
       #              run test suite
       #----------------------------------------------
+      - name: Show installed versions
+        run: |
+          echo "=== Dependency Version: ${{ matrix.dependency-version }} ==="
+          poetry run pip list
+
       - name: Run tests
         run: poetry run python -m pytest tests/unit
   run-unit-tests-with-arrow:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+        dependency-version: ["default", "min"]
+        exclude:
+          - python-version: "3.12"
+            dependency-version: "min"
+          - python-version: "3.13"
+            dependency-version: "min"
+
+    name: "Unit Tests + PyArrow (Python ${{ matrix.python-version }}, ${{ matrix.dependency-version }} deps)"
+
     steps:
       #----------------------------------------------
       #       check-out repo and set-up python
@@ -80,6 +119,7 @@ jobs:
       -   name: Install Poetry
           uses: snok/install-poetry@v1
           with:
+            version: "2.2.1"
             virtualenvs-create: true
             virtualenvs-in-project: true
             installer-parallel: true
@@ -92,7 +132,7 @@ jobs:
           uses: actions/cache@v4
           with:
             path: .venv-pyarrow
-            key: venv-pyarrow-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
+            key: venv-pyarrow-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ matrix.dependency-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
       #----------------------------------------------
       # install dependencies if cache does not exist
       #----------------------------------------------
@@ -102,18 +142,44 @@ jobs:
       #----------------------------------------------
       # install your root project, if required
       #----------------------------------------------
+      -   name: Install Kerberos system dependencies
+          run: |
+            sudo apt-get update
+            sudo apt-get install -y libkrb5-dev
       -   name: Install library
           run: poetry install --no-interaction --all-extras
       #----------------------------------------------
+      # override with custom dependency versions
+      #----------------------------------------------
+      -   name: Install Python tools for custom versions
+          if: matrix.dependency-version != 'default'
+          run: poetry run pip install toml packaging
+
+      -   name: Generate requirements file with pyarrow
+          if: matrix.dependency-version != 'default'
+          run: |
+            poetry run python scripts/dependency_manager.py ${{ matrix.dependency-version }} --output requirements-${{ matrix.dependency-version }}-arrow.txt
+            echo "Generated requirements for ${{ matrix.dependency-version }} versions with PyArrow:"
+            cat requirements-${{ matrix.dependency-version }}-arrow.txt
+
+      -   name: Override with custom dependency versions
+          if: matrix.dependency-version != 'default'
+          run: poetry run pip install -r requirements-${{ matrix.dependency-version }}-arrow.txt
+      #----------------------------------------------
       #              run test suite
       #----------------------------------------------
+      -   name: Show installed versions
+          run: |
+            echo "=== Dependency Version: ${{ matrix.dependency-version }} with PyArrow ==="
+            poetry run pip list
+
       -   name: Run tests
           run: poetry run python -m pytest tests/unit
   check-linting:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
       #----------------------------------------------
       #       check-out repo and set-up python
@@ -131,6 +197,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: "2.2.1"
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
@@ -165,7 +232,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
       #----------------------------------------------
       #       check-out repo and set-up python
@@ -183,6 +250,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: "2.2.1"
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true