From 53f7482eca3e16cf9a22b253486f0bb9fd4e3336 Mon Sep 17 00:00:00 2001 From: Jeremy Hatfield Date: Tue, 6 Oct 2026 03:42:18 +0000 Subject: [PATCH] chore: revert coverage reporting to Codecov Aikido coverage reporting requires a paid plan, so restore the Codecov workflow, config, guard script, and docs from before the migration. --- .claude/commands/fix-patch-coverage.md | 70 ++++++ .dockerignore | 1 + .github/agents/Backend_Dev.agent.md | 4 +- .../instructions/ARCHITECTURE.instructions.md | 5 +- .github/instructions/copilot-instructions.md | 4 +- .github/instructions/testing.instructions.md | 4 +- .../codecov-patch-coverage-fix.prompt.md | 208 ++++++++++++++++++ .github/renovate.json | 17 +- .../test-e2e-playwright-coverage.SKILL.md | 12 +- ...aikido-coverage.yml => codecov-upload.yml} | 127 +++-------- .github/workflows/nightly-build.yml | 2 +- .github/workflows/quality-checks.yml | 18 +- .../workflows/weekly-nightly-promotion.yml | 2 +- ARCHITECTURE.md | 9 +- CLAUDE.md | 2 +- codecov.yml | 176 +++++++++++++++ docs/ci/toolchain-image.md | 2 +- scripts/agent-test-coverage.sh | 2 +- scripts/ci/check-codecov-trigger-parity.sh | 35 +++ scripts/local-patch-report.sh | 2 +- 20 files changed, 562 insertions(+), 140 deletions(-) create mode 100644 .claude/commands/fix-patch-coverage.md create mode 100644 .github/prompts/codecov-patch-coverage-fix.prompt.md rename .github/workflows/{aikido-coverage.yml => codecov-upload.yml} (65%) create mode 100644 codecov.yml create mode 100644 scripts/ci/check-codecov-trigger-parity.sh diff --git a/.claude/commands/fix-patch-coverage.md b/.claude/commands/fix-patch-coverage.md new file mode 100644 index 000000000..9ee1e72d3 --- /dev/null +++ b/.claude/commands/fix-patch-coverage.md @@ -0,0 +1,70 @@ +# Fix Codecov Patch Coverage + +You are a senior test engineer. Analyze the provided Codecov report and generate the minimum set of high-quality tests required to achieve **100% patch coverage** on all modified lines. + +## Input + +$ARGUMENTS + +(Provide ONE of: a Codecov bot comment from a PR, a Codecov report link, or specific file + line references like `backend/internal/services/mail_service.go lines 45-48`.) + +## Execution Protocol + +### Phase 1: Parse and Identify + +Extract: files with missing patch coverage, specific uncovered line numbers, current patch coverage percentage. + +### Phase 2: Analyze Uncovered Code + +For each file: read the source, understand what the uncovered lines do, identify what inputs or conditions trigger those lines (error paths, edge cases, conditional branches). + +Find corresponding test files: +- Go: `*_test.go` in the same package +- TypeScript: `*.test.ts` or `*.spec.ts` + +### Phase 3: Generate Tests + +Follow project patterns from existing tests. Write targeted tests that: +- Exercise the specific uncovered lines +- Verify behavior (not just coverage) +- Are deterministic and independent +- Use descriptive test names + +**Go pattern** (table-driven tests): +```go +func TestFunctionName_Scenario(t *testing.T) { + tests := []struct { + name string + input InputType + want OutputType + wantErr bool + }{ + {name: "descriptive case", input: ..., want: ...}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { ... }) + } +} +``` + +**TypeScript pattern** (Vitest): +```typescript +describe('Component', () => { + it('should handle edge case at line XX', () => { + // Arrange + // Act + // Assert + }); +}); +``` + +### Phase 4: Validate + +Run the new tests (`go test ./...` or `npm test`), verify they pass, confirm existing tests still pass. + +## Constraints + +- **Do NOT relax coverage thresholds** — always aim for 100% patch coverage +- **Do NOT write tests that only exist for coverage** — tests must verify behavior +- **Do NOT modify production code** unless a bug is discovered during testing +- **Do NOT create flaky tests** — all tests must be deterministic diff --git a/.dockerignore b/.dockerignore index f79cabc14..e7bfde518 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,6 +9,7 @@ .git/ .gitignore .github/ +codecov.yml .sourcery.yml .claude/ diff --git a/.github/agents/Backend_Dev.agent.md b/.github/agents/Backend_Dev.agent.md index 53f1f0bdd..7589fb70b 100644 --- a/.github/agents/Backend_Dev.agent.md +++ b/.github/agents/Backend_Dev.agent.md @@ -63,8 +63,8 @@ Your priority is writing code that is clean, tested, and secure by default. - **Local Patch Coverage Preflight (MANDATORY)**: Run VS Code task `Test: Local Patch Report` or `bash scripts/local-patch-report.sh` before backend coverage runs. - Ensure artifacts exist: `test-results/local-patch-report.md` and `test-results/local-patch-report.json`. - Use the file-level coverage gap list to target tests before final coverage validation. - - **Coverage (MANDATORY)**: Run the coverage task/script explicitly and confirm modified lines are covered (`bash scripts/local-patch-report.sh`). - - **MANDATORY**: Patch coverage must cover 100% of new/modified code. This prevents patch-coverage regressions. + - **Coverage (MANDATORY)**: Run the coverage task/script explicitly and confirm Codecov Patch view is green for modified lines. + - **MANDATORY**: Patch coverage must cover 100% of new/modified code. This prevents CodeCov Report failing CI. - **VS Code Task**: Use "Test: Backend with Coverage" (recommended) - **Manual Script**: Execute `/projects/Charon/scripts/go-test-coverage.sh` from the root directory - **Minimum**: 85% coverage (configured via `CHARON_MIN_COVERAGE` or `CPM_MIN_COVERAGE`) diff --git a/.github/instructions/ARCHITECTURE.instructions.md b/.github/instructions/ARCHITECTURE.instructions.md index c8b072942..d1881e811 100644 --- a/.github/instructions/ARCHITECTURE.instructions.md +++ b/.github/instructions/ARCHITECTURE.instructions.md @@ -276,6 +276,7 @@ graph TB ├── go.work # Go workspace definition ├── package.json # Frontend dependencies ├── playwright.config.js # E2E test configuration +├── codecov.yml # Code coverage settings ├── README.md # Project overview ├── CONTRIBUTING.md # Contribution guidelines ├── CHANGELOG.md # Version history @@ -1117,7 +1118,7 @@ go test ./integration/... 2. **Test:** Go tests, Vitest, Playwright 3. **Security:** Trivy, CodeQL, Grype, Govulncheck 4. **Build:** Docker image build -5. **Coverage:** Upload to Aikido (85% gate enforced by the coverage scripts) +5. **Coverage:** Upload to Codecov (85% gate) 6. **Supply Chain:** SBOM generation, Cosign signing --- @@ -1398,7 +1399,7 @@ All agents (`Planning`, `Backend_Dev`, `Frontend_Dev`, `DevOps`) must reference **Tools:** - SonarQube: Code quality and technical debt -- Aikido: Security findings and code coverage reporting +- Codecov: Coverage tracking and trend analysis - Grafana: Runtime metrics and performance - GitHub Insights: Contributor activity and velocity diff --git a/.github/instructions/copilot-instructions.md b/.github/instructions/copilot-instructions.md index 9828ab8f0..8abe8299b 100644 --- a/.github/instructions/copilot-instructions.md +++ b/.github/instructions/copilot-instructions.md @@ -120,7 +120,7 @@ Before proposing ANY code change or fix, you must build a mental map of the feat - **Sync**: React Query expects the exact JSON produced by GORM tags (snake_case). Keep API and UI field names aligned. - **Migrations**: When adding models, update `internal/models` AND `internal/api/routes/routes.go` (AutoMigrate). - **Testing**: All new code MUST include accompanying unit tests. -- **Ignore Files**: Always check `.gitignore` and `.dockerignore` when adding new file or folders. +- **Ignore Files**: Always check `.gitignore`, `.dockerignore`, and `.codecov.yml` when adding new file or folders. ## Documentation @@ -222,7 +222,7 @@ Before marking an implementation task as complete, perform the following in orde 6. **Coverage Testing** (MANDATORY - Non-negotiable): - **Overall Coverage**: Minimum 85% coverage is MANDATORY and will fail the PR if not met. - - **Patch Coverage**: Changed lines must be covered by tests (check locally with `bash scripts/local-patch-report.sh`); the project gate (`CHARON_MIN_COVERAGE`) is enforced by the coverage scripts, and reports go to Aikido, not a suggestion. If patch coverage is incomplete, add targeted tests covering the uncovered changed lines before the PR can merge. + - **Patch Coverage**: Codecov's `patch` status (Codecov Patch view) is a mandatory, merge-blocking gate (`codecov.yml`, `coverage.status.patch.default`, target aligned with the project coverage target / `CHARON_MIN_COVERAGE`), not a suggestion. If patch coverage is incomplete, add targeted tests covering the uncovered changed lines before the PR can merge. - **Backend Changes**: Run the VS Code task "Test: Backend with Coverage" or execute `scripts/go-test-coverage.sh`. - Minimum coverage: 85% (set via `CHARON_MIN_COVERAGE` or `CPM_MIN_COVERAGE`). - If coverage drops below threshold, write additional tests to restore coverage. diff --git a/.github/instructions/testing.instructions.md b/.github/instructions/testing.instructions.md index a2edf3182..271a86eb8 100644 --- a/.github/instructions/testing.instructions.md +++ b/.github/instructions/testing.instructions.md @@ -156,7 +156,7 @@ Before pushing code, verify E2E coverage: # View HTML report open coverage/e2e/index.html - # Check LCOV file exists for the coverage upload + # Check LCOV file exists for Codecov ls -la coverage/e2e/lcov.info ``` @@ -185,7 +185,7 @@ Before pushing code, verify E2E coverage: ## 3. Coverage & Completion * **Coverage Gate:** A task is not "Complete" until a coverage report is generated. * **Threshold Compliance:** You must compare the final coverage percentage against the project's threshold (Default: 85% unless specified otherwise). If coverage drops, you must identify the "uncovered lines" and add targeted tests. -* **Patch Coverage (Mandatory Gate):** Changed/added lines in a PR's diff must be covered by tests (check locally with `bash scripts/local-patch-report.sh`); the project coverage gate (`CHARON_MIN_COVERAGE`) is enforced in CI by the coverage scripts, and coverage reports are uploaded to Aikido for its PR checks — not a suggestion. If patch coverage is below target, add targeted tests covering the uncovered changed lines before completing the task. +* **Patch Coverage (Mandatory Gate):** Codecov's `patch` status is a mandatory, merge-blocking gate (`codecov.yml`, `coverage.status.patch.default`, target aligned with the project coverage target / `CHARON_MIN_COVERAGE`) — not a suggestion. Every changed/added line in a PR's diff must meet the patch coverage target before the PR can merge, consistent with the rest of the Definition of Done. If patch coverage is below target, add targeted tests covering the uncovered changed lines before completing the task. * **Review Patch Coverage:** When reviewing patch coverage reports, first attempt a real, targeted test for every uncovered line. Only treat a line as an acceptable exception (e.g., a branch proven structurally unreachable via any real caller) if you can document the specific reachability proof in a comment at that line — do not wave off uncovered lines as "acceptable" without that documented justification, and do not add contrived tests that don't reflect real reachable states just to inflate the number. ## 4. GORM Security Validation (Manual Stage) diff --git a/.github/prompts/codecov-patch-coverage-fix.prompt.md b/.github/prompts/codecov-patch-coverage-fix.prompt.md new file mode 100644 index 000000000..7e42bdb22 --- /dev/null +++ b/.github/prompts/codecov-patch-coverage-fix.prompt.md @@ -0,0 +1,208 @@ +--- +mode: 'agent' +description: 'Generate targeted tests to achieve 100% Codecov patch coverage when CI reports uncovered lines' +tools: ['changes', 'search/codebase', 'edit/editFiles', 'fetch', 'findTestFiles', 'problems', 'runCommands', 'runTasks', 'runTests', 'search', 'search/searchResults', 'runCommands/terminalLastCommand', 'runCommands/terminalSelection', 'testFailure', 'usages'] +--- + +# Codecov Patch Coverage Fix + +You are a senior test engineer with deep expertise in test-driven development, code coverage analysis, and writing effective unit and integration tests. You have extensive experience with: + +- Interpreting Codecov reports and understanding patch vs project coverage +- Writing targeted tests that exercise specific code paths and edge cases +- Go testing patterns (`testing` package, table-driven tests, mocks, test helpers) +- JavaScript/TypeScript testing with Vitest, Jest, and React Testing Library +- Achieving 100% patch coverage without writing redundant or brittle tests + +## Primary Objective + +Analyze the provided Codecov comment or report and generate the minimum set of high-quality tests required to achieve **100% patch coverage** on all modified lines. Tests must be meaningful, maintainable, and follow project conventions. + +## Input Requirements + +The user will provide ONE of the following: + +1. **Codecov Comment (Copy/Pasted)**: The full text of a Codecov bot comment from a PR +2. **Codecov Report Link**: A URL to the Codecov coverage report for the PR +3. **Specific File + Lines**: Direct reference to files and uncovered line ranges + +### Example Input Formats + +**Format 1 - Codecov Comment:** +``` +Codecov Report +Attention: Patch coverage is 75.00000% with 4 lines in your changes missing coverage. +Project coverage is 82.45%. Comparing base (abc123) to head (def456). + +Files with missing coverage: +| File | Coverage | Lines | +|------|----------|-------| +| backend/internal/services/mail_service.go | 75.00% | 45-48 | +``` + +**Format 2 - Link:** +`https://app.codecov.io/gh/Owner/Repo/pull/123` + +**Format 3 - Direct Reference:** +`backend/internal/services/mail_service.go lines 45-48, 62, 78-82` + +## Execution Protocol + +### Phase 1: Parse and Identify + +1. **Extract Coverage Data**: Parse the Codecov comment/report to identify: + - Files with missing patch coverage + - Specific line numbers or ranges that are uncovered + - The current patch coverage percentage + - The target coverage (always 100% for patch coverage) + +2. **Document Findings**: Create a structured list: + ``` + UNCOVERED FILES: + - FILE-001: [path/to/file.go] - Lines: [45-48, 62] + - FILE-002: [path/to/other.ts] - Lines: [23, 67-70] + ``` + +### Phase 2: Analyze Uncovered Code + +For each file with missing coverage: + +1. **Read the Source File**: Use the codebase tool to read the file and understand: + - What the uncovered lines do + - What functions/methods contain the uncovered code + - What conditions or branches lead to those lines + - Any dependencies or external calls + +2. **Identify Code Paths**: Determine what inputs, states, or conditions would cause execution of the uncovered lines: + - Error handling paths + - Edge cases (nil, empty, boundary values) + - Conditional branches (if/else, switch cases) + - Loop iterations (zero, one, many) + +3. **Find Existing Tests**: Locate the corresponding test file(s): + - Go: `*_test.go` in the same package + - TypeScript/JavaScript: `*.test.ts`, `*.spec.ts`, or in `__tests__/` directory + +### Phase 3: Generate Tests + +For each uncovered code path: + +1. **Follow Project Patterns**: Analyze existing tests to match: + - Test naming conventions + - Setup/teardown patterns + - Mocking strategies + - Assertion styles + - Table-driven test structures (especially for Go) + +2. **Write Targeted Tests**: Create tests that specifically exercise the uncovered lines: + - One test case per distinct code path + - Use descriptive test names that explain the scenario + - Include appropriate setup and teardown + - Use meaningful assertions that verify behavior, not just coverage + +3. **Test Quality Standards**: + - Tests must be deterministic (no flaky tests) + - Tests must be independent (no shared state between tests) + - Tests must be fast (mock external dependencies) + - Tests must be readable (clear arrange-act-assert structure) + +### Phase 4: Validate + +1. **Run the Tests**: Execute the new tests to ensure they pass +2. **Verify Coverage**: If possible, run coverage locally to confirm the lines are now covered +3. **Check for Regressions**: Ensure existing tests still pass + +## Language-Specific Guidelines + +### Go Testing + +```go +// Table-driven test pattern for multiple cases +func TestFunctionName_Scenario(t *testing.T) { + tests := []struct { + name string + input InputType + want OutputType + wantErr bool + }{ + { + name: "descriptive case name", + input: InputType{...}, + want: OutputType{...}, + }, + // Additional cases for uncovered paths + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := FunctionName(tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("FunctionName() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("FunctionName() = %v, want %v", got, tt.want) + } + }) + } +} +``` + +### TypeScript/JavaScript Testing (Vitest) + +```typescript +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +describe('ComponentOrFunction', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('should handle specific edge case for uncovered line', () => { + // Arrange + const input = createTestInput({ edgeCase: true }); + + // Act + const result = functionUnderTest(input); + + // Assert + expect(result).toMatchObject({ expected: 'value' }); + }); + + it('should handle error condition at line XX', async () => { + // Arrange - setup condition that triggers error path + vi.spyOn(dependency, 'method').mockRejectedValue(new Error('test error')); + + // Act & Assert + await expect(functionUnderTest()).rejects.toThrow('expected error message'); + }); +}); +``` + +## Output Requirements + +1. **Coverage Triage Report**: Document each uncovered file/line and the test strategy +2. **Test Code**: Complete, runnable test code placed in appropriate test files +3. **Execution Results**: Output from running the tests showing they pass +4. **Coverage Verification**: Confirmation that the previously uncovered lines are now exercised + +## Constraints + +- **Do NOT relax coverage thresholds** - always aim for 100% patch coverage +- **Do NOT write tests that only exist for coverage** - tests must verify behavior +- **Do NOT modify production code** unless a bug is discovered during testing +- **Do NOT skip error handling paths** - these often cause coverage gaps +- **Do NOT create flaky tests** - all tests must be deterministic + +## Success Criteria + +- [ ] All files from Codecov report have been addressed +- [ ] All previously uncovered lines now have test coverage +- [ ] All new tests pass consistently +- [ ] All existing tests continue to pass +- [ ] Test code follows project conventions and patterns +- [ ] Tests are meaningful and maintainable, not just coverage padding + +## Begin + +Please provide the Codecov comment, report link, or file/line references that you want me to analyze and fix. diff --git a/.github/renovate.json b/.github/renovate.json index 2a60c1934..4373a0781 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -181,9 +181,9 @@ }, { "customType": "regex", - "description": "Track gotestsum version in coverage workflow", + "description": "Track gotestsum version in codecov workflow", "managerFilePatterns": [ - "/^\\.github/workflows/aikido-coverage\\.yml$/" + "/^\\.github/workflows/codecov-upload\\.yml$/" ], "matchStrings": [ "gotestsum@v(?[^\\s]+)" @@ -192,19 +192,6 @@ "datasourceTemplate": "go", "versioningTemplate": "semver" }, - { - "customType": "regex", - "description": "Track gcov2lcov version in coverage workflow", - "managerFilePatterns": [ - "/^\\.github/workflows/aikido-coverage\\.yml$/" - ], - "matchStrings": [ - "gcov2lcov@v(?[^\\s]+)" - ], - "depNameTemplate": "github.com/jandelgado/gcov2lcov", - "datasourceTemplate": "go", - "versioningTemplate": "semver" - }, { "customType": "regex", "description": "Track gotestsum version in quality checks workflow", diff --git a/.github/skills/test-e2e-playwright-coverage.SKILL.md b/.github/skills/test-e2e-playwright-coverage.SKILL.md index 83fe3084d..ccd3ed6b0 100644 --- a/.github/skills/test-e2e-playwright-coverage.SKILL.md +++ b/.github/skills/test-e2e-playwright-coverage.SKILL.md @@ -70,7 +70,7 @@ metadata: ## Overview -Runs Playwright end-to-end tests with code coverage collection using `@bgotink/playwright-coverage`. This skill collects V8 coverage data during test execution and generates reports in LCOV, HTML, and JSON formats suitable for coverage tooling. +Runs Playwright end-to-end tests with code coverage collection using `@bgotink/playwright-coverage`. This skill collects V8 coverage data during test execution and generates reports in LCOV, HTML, and JSON formats suitable for upload to Codecov. **IMPORTANT**: This skill starts the **Vite dev server** (not Docker) because V8 coverage requires access to source files. Running coverage against the Docker container will result in `0%` coverage. @@ -119,6 +119,12 @@ For use in GitHub Actions or other CI/CD pipelines: env: PLAYWRIGHT_BASE_URL: http://localhost:8080 CI: true + +- name: Upload E2E Coverage to Codecov + uses: codecov/codecov-action@v5 + with: + files: ./coverage/e2e/lcov.info + flags: e2e ``` ## Parameters @@ -146,7 +152,7 @@ For use in GitHub Actions or other CI/CD pipelines: ### Output Directories - **coverage/e2e/**: Coverage reports (LCOV, HTML, JSON) - - `lcov.info` - LCOV format + - `lcov.info` - LCOV format for Codecov upload - `coverage.json` - JSON format for programmatic access - `index.html` - HTML report for visual inspection - **playwright-report/**: HTML test report with results and traces @@ -173,7 +179,7 @@ The skill generates coverage in multiple formats: | Format | File | Purpose | |--------|------|---------| -| LCOV | `coverage/e2e/lcov.info` | Coverage tooling | +| LCOV | `coverage/e2e/lcov.info` | Codecov upload | | HTML | `coverage/e2e/index.html` | Visual inspection | | JSON | `coverage/e2e/coverage.json` | Programmatic access | diff --git a/.github/workflows/aikido-coverage.yml b/.github/workflows/codecov-upload.yml similarity index 65% rename from .github/workflows/aikido-coverage.yml rename to .github/workflows/codecov-upload.yml index a14c5608a..500280e76 100644 --- a/.github/workflows/aikido-coverage.yml +++ b/.github/workflows/codecov-upload.yml @@ -1,4 +1,4 @@ -name: Upload Coverage to Aikido +name: Upload Coverage to Codecov on: pull_request: @@ -34,10 +34,11 @@ env: permissions: contents: read + pull-requests: write jobs: - backend-coverage: - name: Backend Coverage + backend-codecov: + name: Backend Codecov Upload runs-on: ubuntu-latest timeout-minutes: 15 if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_backend }} @@ -160,34 +161,16 @@ jobs: path: backend/test-output.txt retention-days: 7 - - name: Install gcov2lcov - run: | - for attempt in 1 2 3; do - go install github.com/jandelgado/gcov2lcov@v1.1.1 && break - if [ "$attempt" -lt 3 ]; then - echo "Attempt ${attempt}/3 failed; retrying in $((attempt * 15))s..." >&2 - sleep $((attempt * 15)) - else - echo "ERROR: go install gcov2lcov failed after 3 attempts" >&2 - exit 1 - fi - done - - # Aikido accepts LCOV/Cobertura only; convert Go's coverage profile. - # Run from the module dir so SF: paths resolve to repo-relative files. - - name: Convert backend coverage to LCOV - working-directory: backend - run: gcov2lcov -infile coverage.txt -outfile backend.lcov - - - name: Upload backend coverage artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - name: Upload backend coverage to Codecov + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: - name: coverage-backend - path: backend/backend.lcov - retention-days: 7 + token: ${{ secrets.CODECOV_TOKEN }} + files: ./backend/coverage.txt + flags: backend + fail_ci_if_error: true - frontend-coverage: - name: Frontend Coverage + frontend-codecov: + name: Frontend Codecov Upload runs-on: ubuntu-latest timeout-minutes: 15 if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_frontend }} @@ -216,19 +199,16 @@ jobs: bash scripts/frontend-test-coverage.sh 2>&1 | tee frontend/test-output.txt exit "${PIPESTATUS[0]}" - # Aikido detects the report format from the filename. - - name: Rename frontend LCOV report - run: cp frontend/coverage/lcov.info frontend/coverage/frontend.lcov - - - name: Upload frontend coverage artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - name: Upload frontend coverage to Codecov + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: - name: coverage-frontend - path: frontend/coverage/frontend.lcov - retention-days: 7 + token: ${{ secrets.CODECOV_TOKEN }} + directory: ./frontend/coverage + flags: frontend + fail_ci_if_error: true - agent-coverage: - name: Agent Coverage + agent-codecov: + name: Agent Codecov Upload runs-on: ubuntu-latest timeout-minutes: 15 if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_agent }} @@ -266,65 +246,10 @@ jobs: path: agent/test-output.txt retention-days: 7 - - name: Install gcov2lcov - run: | - for attempt in 1 2 3; do - go install github.com/jandelgado/gcov2lcov@v1.1.1 && break - if [ "$attempt" -lt 3 ]; then - echo "Attempt ${attempt}/3 failed; retrying in $((attempt * 15))s..." >&2 - sleep $((attempt * 15)) - else - echo "ERROR: go install gcov2lcov failed after 3 attempts" >&2 - exit 1 - fi - done - - # Aikido accepts LCOV/Cobertura only; convert Go's coverage profile. - # Run from the module dir so SF: paths resolve to repo-relative files. - - name: Convert agent coverage to LCOV - working-directory: agent - run: gcov2lcov -infile coverage.txt -outfile agent.lcov - - - name: Upload agent coverage artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: coverage-agent - path: agent/agent.lcov - retention-days: 7 - - # Single upload per commit: Aikido merges server-side, so per-job uploads - # would send partial coverage and can race. Runs only after every enabled - # coverage job succeeded (jobs disabled via workflow_dispatch are skipped). - upload-aikido: - name: Upload Coverage to Aikido - runs-on: ubuntu-latest - timeout-minutes: 10 - needs: [backend-coverage, frontend-coverage, agent-coverage] - # Aikido authenticates via GitHub OIDC (no secret). Fork and Dependabot - # pull_request runs are never granted id-token, so skip them rather than fail. - if: >- - ${{ !cancelled() && !failure() - && (github.event_name != 'pull_request' - || (github.event.pull_request.head.repo.full_name == github.repository - && github.actor != 'dependabot[bot]')) }} - permissions: - contents: read - id-token: write - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - persist-credentials: false - ref: ${{ github.sha }} - - - name: Download coverage reports - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: coverage-* - path: coverage-reports - merge-multiple: true - - - name: Upload coverage to Aikido - uses: AikidoSec/code-coverage-github-action@af64fbf145a1d80a532f18d7bb75daff398b7d00 # v2.1.0 + - name: Upload agent coverage to Codecov + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: - file-paths: coverage-reports/*.lcov + token: ${{ secrets.CODECOV_TOKEN }} + files: ./agent/coverage.txt + flags: agent + fail_ci_if_error: true diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 5868c0b23..362039470 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -128,7 +128,7 @@ jobs: const workflows = [ { id: 'e2e-tests-split.yml' }, - { id: 'aikido-coverage.yml', inputs: { run_backend: 'true', run_frontend: 'true' } }, + { id: 'codecov-upload.yml', inputs: { run_backend: 'true', run_frontend: 'true' } }, { id: 'supply-chain-verify.yml' }, { id: 'codeql.yml' }, ]; diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 5fc4065d8..eed96965a 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -79,6 +79,18 @@ jobs: set -euo pipefail go run scripts/ci/check_muzzle_allowlist_parity.go + codecov-trigger-parity-guard: + name: Codecov Trigger/Comment Parity Guard + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Enforce Codecov trigger and comment parity + run: | + bash scripts/ci/check-codecov-trigger-parity.sh + toolchain-key-tests: name: Toolchain key / freshness-guard scripts (bats) runs-on: ubuntu-latest @@ -309,7 +321,7 @@ jobs: fi } >> "$GITHUB_STEP_SUMMARY" - # Coverage upload to Aikido lives in `aikido-coverage.yml`. + # Codecov upload moved to `codecov-upload.yml` (pull_request + workflow_dispatch). - name: GORM Security Scanner @@ -436,7 +448,7 @@ jobs: fi } >> "$GITHUB_STEP_SUMMARY" - # Coverage upload to Aikido is handled in aikido-coverage.yml (agent-coverage job). + # Codecov upload handled separately in codecov-upload.yml (agent-codecov job). # Last step so lint never hides test results; !cancelled() so a failing # test step never skips lint. The agent module shares the full config that @@ -545,7 +557,7 @@ jobs: fi } >> "$GITHUB_STEP_SUMMARY" - # Coverage upload to Aikido lives in `aikido-coverage.yml`. + # Codecov upload moved to `codecov-upload.yml` (pull_request + workflow_dispatch). diff --git a/.github/workflows/weekly-nightly-promotion.yml b/.github/workflows/weekly-nightly-promotion.yml index 69e443981..d3b76daf2 100644 --- a/.github/workflows/weekly-nightly-promotion.yml +++ b/.github/workflows/weekly-nightly-promotion.yml @@ -545,7 +545,7 @@ jobs: // here would be redundant. These remaining workflows are supplementary // checks not part of the health gate, dispatched once the PR exists. const requiredWorkflows = [ - { id: 'aikido-coverage.yml', inputs: { run_backend: 'true', run_frontend: 'true' } }, + { id: 'codecov-upload.yml', inputs: { run_backend: 'true', run_frontend: 'true' } }, { id: 'security-pr.yml', inputs: { pr_number: prNumber } }, { id: 'supply-chain-verify.yml' }, ]; diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index aca043cfd..3f75439bb 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -298,6 +298,7 @@ graph TB ├── go.work # Go workspace definition ├── package.json # Frontend dependencies ├── playwright.config.js # E2E test configuration +├── codecov.yml # Code coverage settings ├── README.md # Project overview ├── CONTRIBUTING.md # Contribution guidelines ├── CHANGELOG.md # Version history @@ -1769,9 +1770,9 @@ go test ./integration/... 2. **Test:** Go tests, Vitest, Playwright 3. **Security:** Trivy, CodeQL, Grype, Govulncheck, Semgrep 4. **Build:** Docker image build -5. **Coverage:** Coverage scripts enforce the 85% gate; `backend`, `frontend`, and - `agent` LCOV reports are merged into a single Aikido upload via GitHub OIDC - (`.github/workflows/aikido-coverage.yml`); `quality-checks.yml` runs a +5. **Coverage:** Upload to Codecov (85% gate) — `backend`, `frontend`, and + `agent` each upload under a distinct Codecov flag + (`.github/workflows/codecov-upload.yml`); `quality-checks.yml` runs a matching `agent-quality` job (go vet, lint, coverage gate) unconditionally on every PR, not only PRs that touch `agent/**` 6. **Supply Chain:** SBOM generation, Cosign signing @@ -2134,7 +2135,7 @@ All agents (`Planning`, `Backend_Dev`, `Frontend_Dev`, `DevOps`) must reference **Tools:** - SonarQube: Code quality and technical debt -- Aikido: Security findings and code coverage reporting +- Codecov: Coverage tracking and trend analysis - Grafana: Runtime metrics and performance - GitHub Insights: Contributor activity and velocity diff --git a/CLAUDE.md b/CLAUDE.md index f597fa553..d18fa4c6f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -131,7 +131,7 @@ never affected by this. - **Sync**: React Query expects the exact JSON produced by GORM tags (snake_case). Keep API and UI field names aligned. - **Migrations**: When adding models, update `internal/models` AND `internal/api/routes/routes.go` (AutoMigrate). - **Testing**: All new code MUST include accompanying unit tests. -- **Ignore Files**: Always check `.gitignore` and `.dockerignore` when adding new files or folders. +- **Ignore Files**: Always check `.gitignore`, `.dockerignore`, and `.codecov.yml` when adding new files or folders. ## Documentation diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..cd936761e --- /dev/null +++ b/codecov.yml @@ -0,0 +1,176 @@ +# ============================================================================= +# Codecov Configuration +# Require 75% overall coverage, exclude test files and non-source code +# ============================================================================= + +coverage: + status: + project: + default: + target: 87% + threshold: 1% + patch: + default: + # Patch coverage is a mandatory merge-blocking gate, consistent with + # the rest of the Definition of Done (see + # .github/instructions/testing.instructions.md §3). Target matches + # the project coverage target / CHARON_MIN_COVERAGE default (87%) + # for consistency across all coverage gates. + target: 87% + threshold: 1% + +# Fail CI if Codecov upload/report indicates a problem +require_ci_to_pass: yes + +# ----------------------------------------------------------------------------- +# Notification gating +# ----------------------------------------------------------------------------- +# Coverage is uploaded by three independent, parallel jobs in +# .github/workflows/codecov-upload.yml (flags: backend, frontend, agent). +# Without this, Codecov re-evaluates the project/patch commit statuses after +# EACH upload lands, so the first upload produces a partial report that fails +# the 87% gate — the status shows red, then flips green once the remaining +# uploads arrive. after_n_builds makes Codecov hold the status as pending +# until all 3 uploads are in, then post a single pass/fail. +# +# NOTE: on manual workflow_dispatch runs that disable a job (run_frontend=false +# etc.) fewer than 3 uploads arrive and the status stays pending. Normal +# pull_request runs always execute all 3 jobs, so this only affects ad-hoc +# manual runs. +codecov: + notify: + after_n_builds: 3 + wait_for_ci: yes + +# ----------------------------------------------------------------------------- +# PR Comment Configuration +# ----------------------------------------------------------------------------- +comment: + # Hold the PR comment until all 3 coverage uploads land, matching + # codecov.notify.after_n_builds — avoids posting partial numbers that + # get edited moments later. + after_n_builds: 3 + # Post coverage report as PR comment + require_changes: false + require_base: false + require_head: true + layout: "reach, diff, flags, files" + behavior: default + +# ----------------------------------------------------------------------------- +# Exclude from coverage reporting +# ----------------------------------------------------------------------------- +ignore: + # Test files + - "**/tests/**" + - "**/test/**" + - "**/__tests__/**" + - "**/test_*.go" + - "**/*_test.go" + - "**/*.test.ts" + - "**/*.test.tsx" + - "**/*.spec.ts" + - "**/*.spec.tsx" + - "**/vitest.config.ts" + - "**/vitest.setup.ts" + + # E2E tests + - "**/e2e/**" + - "**/integration/**" + + # Documentation + - "docs/**" + - "*.md" + + # CI/CD & Config + - ".github/**" + - "scripts/**" + - "tools/**" + - "*.yml" + - "*.yaml" + - "*.json" + + # Frontend build artifacts & dependencies + - "frontend/node_modules/**" + - "frontend/dist/**" + - "frontend/coverage/**" + - "frontend/test-results/**" + - "frontend/public/**" + + # Backend non-source files + - "backend/cmd/seed/**" + - "backend/data/**" + - "backend/coverage/**" + - "backend/bin/**" + - "backend/*.cover" + - "backend/*.out" + - "backend/*.html" + - "backend/codeql-db/**" + + # CodeQL artifacts + - "codeql-db/**" + - "codeql-db-*/**" + - "codeql-agent-results/**" + - "codeql-custom-queries-*/**" + - "*.sarif" + + # Config files (no logic) + - "**/tailwind.config.js" + - "**/postcss.config.js" + - "**/eslint.config.js" + - "**/vite.config.ts" + - "**/tsconfig*.json" + + # Type definitions only + - "**/*.d.ts" + + # Import/data directories + - "import/**" + - "data/**" + - ".cache/**" + + # CrowdSec config files (no logic to test) + - "configs/crowdsec/**" + + # ========================================================================== + # Backend packages excluded from coverage (match go-test-coverage.sh) + # These are entrypoints and infrastructure code that don't benefit from + # unit tests - they are tested via integration tests instead. + # ========================================================================== + + # Main entry points (bootstrap code only) + - "backend/cmd/api/**" + - "agent/main.go" + + # Infrastructure packages (logging, metrics, tracing) + # These are thin wrappers around external libraries with no business logic + - "backend/internal/logger/**" + - "backend/internal/metrics/**" + - "backend/internal/trace/**" + + # Backend test utilities (test infrastructure, not application code) + # These files contain testing helpers that take *testing.T and are only + # callable from *_test.go files - they cannot be covered by production code + - "backend/internal/api/handlers/testdb.go" + - "backend/internal/api/handlers/test_helpers.go" + + # DNS provider implementations (tested via integration tests, not unit tests) + # These are plugin implementations that interact with external DNS APIs + # and are validated through service-level integration tests + - "backend/pkg/dnsprovider/builtin/**" + + # ========================================================================== + # Frontend test utilities and helpers + # These are test infrastructure, not application code + # ========================================================================== + + # Test setup and utilities directory + - "frontend/src/test/**" + + # Vitest setup files + - "frontend/vitest.config.ts" + - "frontend/src/setupTests.ts" + + # Playwright E2E config + - "frontend/playwright.config.ts" + - "frontend/e2e/**" diff --git a/docs/ci/toolchain-image.md b/docs/ci/toolchain-image.md index 1ff81fdc6..3d6c728ea 100644 --- a/docs/ci/toolchain-image.md +++ b/docs/ci/toolchain-image.md @@ -87,5 +87,5 @@ identical to before, and the app image content is byte-identical (same recipe). `scripts/lib/dockerfile-stage.sh`, and `scripts/tests/` — are **excluded from the image context via `.dockerignore`** (they run from a plain checkout in `toolchain-image.yml` / `quality-checks.yml`, never from inside a container). - No `.gitignore` change (source files, must be committed); no coverage-config + No `.gitignore` change (source files, must be committed); no `.codecov.yml` change (shell/bats/YAML carry no Go/TS coverage). diff --git a/scripts/agent-test-coverage.sh b/scripts/agent-test-coverage.sh index ba211f208..a452f74a5 100755 --- a/scripts/agent-test-coverage.sh +++ b/scripts/agent-test-coverage.sh @@ -24,7 +24,7 @@ cd "$AGENT_DIR" # Packages to exclude from coverage (main package/entrypoint only). # Mirrors backend/cmd/api's exclusion in scripts/go-test-coverage.sh: # CLI bootstrap/flag-parsing code that doesn't benefit from unit tests, per -# the former codecov.yml "entrypoints and infrastructure code" exclusion precedent. +# codecov.yml's "entrypoints and infrastructure code" exclusion precedent. EXCLUDE_PACKAGES=( "github.com/Wikid82/charon/agent" ) diff --git a/scripts/ci/check-codecov-trigger-parity.sh b/scripts/ci/check-codecov-trigger-parity.sh new file mode 100644 index 000000000..ef59280f7 --- /dev/null +++ b/scripts/ci/check-codecov-trigger-parity.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +QUALITY_WORKFLOW=".github/workflows/quality-checks.yml" +CODECOV_WORKFLOW=".github/workflows/codecov-upload.yml" +EXPECTED_COMMENT='Codecov upload moved to `codecov-upload.yml` (pull_request + workflow_dispatch).' + +fail() { + local message="$1" + echo "::error title=Codecov trigger/comment drift::${message}" + exit 1 +} + +[[ -f "$QUALITY_WORKFLOW" ]] || fail "Missing workflow file: $QUALITY_WORKFLOW" +[[ -f "$CODECOV_WORKFLOW" ]] || fail "Missing workflow file: $CODECOV_WORKFLOW" + +grep -qE '^on:' "$QUALITY_WORKFLOW" || fail "quality-checks workflow is missing an 'on:' block" +grep -qE '^on:' "$CODECOV_WORKFLOW" || fail "codecov-upload workflow is missing an 'on:' block" + +grep -qE '^ pull_request:' "$QUALITY_WORKFLOW" || fail "quality-checks must run on pull_request" +if grep -qE '^ workflow_dispatch:' "$QUALITY_WORKFLOW"; then + fail "quality-checks unexpectedly includes workflow_dispatch; keep Codecov manual trigger scoped to codecov-upload workflow" +fi + +grep -qE '^ pull_request:' "$CODECOV_WORKFLOW" || fail "codecov-upload must run on pull_request" +grep -qE '^ workflow_dispatch:' "$CODECOV_WORKFLOW" || fail "codecov-upload must run on workflow_dispatch" +if grep -qE '^ pull_request_target:' "$CODECOV_WORKFLOW"; then + fail "codecov-upload must not use pull_request_target" +fi + +if ! grep -Fq "$EXPECTED_COMMENT" "$QUALITY_WORKFLOW"; then + fail "quality-checks Codecov handoff comment is missing or changed; expected: $EXPECTED_COMMENT" +fi + +echo "Codecov trigger/comment parity check passed" diff --git a/scripts/local-patch-report.sh b/scripts/local-patch-report.sh index 04d6ee959..211cf7d1a 100755 --- a/scripts/local-patch-report.sh +++ b/scripts/local-patch-report.sh @@ -83,7 +83,7 @@ fi # Three-tier baseline resolution (F.3): # Tier 1 - explicit $CHARON_PATCH_BASELINE override (handled above, already set). # Tier 2 - ask gh what the current branch's actual open PR base is, so the -# local diff matches the PR diff reviewers see. +# local diff matches exactly what Codecov compares against. # Tier 3 - static heuristic fallback, preferring origin/development over # origin/main (per F.2: development is the default integration # branch; main is only ever a target for the scheduled nightly