Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 95 additions & 34 deletions .github/workflows/pr-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,35 +26,96 @@ jobs:
contents: read
pull-requests: read
outputs:
go: ${{ steps.filter.outputs.go }}
go: ${{ github.event_name != 'pull_request' || steps.filter.outputs.full == 'true' || steps.filter.outputs.go == 'true' }}
agent_skill: ${{ github.event_name != 'pull_request' || steps.filter.outputs.full == 'true' || steps.filter.outputs.agent_skill == 'true' }}
licenses: ${{ github.event_name != 'pull_request' || steps.filter.outputs.full == 'true' || steps.filter.outputs.licenses == 'true' }}
gh_credentials: ${{ github.event_name != 'pull_request' || steps.filter.outputs.full == 'true' || steps.filter.outputs.gh_credentials == 'true' }}
provider_update: ${{ github.event_name != 'pull_request' || steps.filter.outputs.full == 'true' || steps.filter.outputs.provider_update == 'true' }}
mode: ${{ github.event_name == 'pull_request' && 'pr' || 'full' }}
changed_files: ${{ steps.filter.outputs.all_files || '[]' }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4
id: filter
if: github.event_name == 'pull_request'
with:
list-files: json
filters: |
go:
- '**/*.go'
- 'pkg/inject/**'
full:
- '.github/workflows/pr-ci.yml'
- '.github/actions/**'
- 'hack/ci/**'
- 'go.mod'
- 'go.sum'
- '.goreleaser.yml'
- 'e2e/framework/**'
- 'e2e/cmd/**'
- 'e2e/*.go'
go:
- '**/*.go'
- 'cmd/**'
- 'pkg/**'
- 'providers/**'
- 'e2e/**'
- 'hack/ci/**'
- 'hack/microsandbox/**'
- 'hack/provider_update_ci/**'
- '.github/workflows/pr-ci.yml'
- 'hack/licenses/**'
- 'hack/podman_ci/**'
- '.golangci.yaml'
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
agent_skill:
- '.agents/skills/devsy/**'
- 'hack/validate_agent_skill/**'
licenses:
- '**/*.go'
- 'hack/licenses/**'
- 'THIRD_PARTY_LICENSES.md'
gh_credentials:
- 'pkg/ghcredentials/**'
- 'pkg/credentials/**'
- 'pkg/tunnel/**'
- 'pkg/agent/tunnel/**'
- 'pkg/agent/tunnelserver/**'
- 'pkg/config/**'
- 'pkg/log/**'
- 'pkg/http/**'
- 'pkg/port/**'
- 'pkg/netstat/**'
- 'pkg/random/**'
- 'pkg/gitsshsigning/**'
- 'pkg/transport/**'
- 'pkg/client/**'
- 'pkg/ssh/**'
- 'pkg/flags/**'
- 'pkg/exitcode/**'
- 'e2e/cmd/ghfixture/**'
provider_update:
- 'pkg/workspace/**'
- 'pkg/provider/**'
- 'pkg/config/**'
- 'pkg/extract/**'
- 'pkg/copy/**'
- 'pkg/download/**'
- 'pkg/http/**'
- 'pkg/gitcredentials/**'
- 'pkg/hash/**'
- 'pkg/log/**'
- 'pkg/id/**'
- 'pkg/types/**'
- 'providers/**'
- 'hack/provider_update_ci/**'
all:
- '**'

plan:
name: Plan
needs: [changes]
if: always()
runs-on: ubuntu-latest
permissions:
contents: read
outputs:
unit: ${{ steps.matrix.outputs.unit }}
build: ${{ steps.matrix.outputs.build }}
unprivileged: ${{ steps.matrix.outputs.unprivileged }}
integration: ${{ steps.matrix.outputs.integration }}
microsandbox: ${{ steps.matrix.outputs.microsandbox }}
Expand All @@ -71,7 +132,11 @@ jobs:
id: matrix
env:
SECRET_AVAILABLE: ${{ secrets.DEVSY_GITHUB_APP_ID != '' }}
run: go run ./hack/ci matrix --secret-available="$SECRET_AVAILABLE" >> "$GITHUB_OUTPUT"
MATRIX_MODE: ${{ needs.changes.outputs.mode }}
CHANGED_FILES_JSON: ${{ needs.changes.outputs.changed_files }}
run: |
printf '%s' "$CHANGED_FILES_JSON" | jq -r '.[]' > "$RUNNER_TEMP/changed-files.txt"
go run ./hack/ci matrix --mode="$MATRIX_MODE" --changed-files="$RUNNER_TEMP/changed-files.txt" --secret-available="$SECRET_AVAILABLE" >> "$GITHUB_OUTPUT"

precommit:
name: Pre-commit
Expand All @@ -81,10 +146,12 @@ jobs:

- uses: jdx/mise-action@b44d12e7d0d8ee7461bb893e1b1993500433185b # v5.2.7

- run: prek run --all-files
- run: SKIP=golangci-lint prek run --all-files

agent-skill:
name: Agent Skill
needs: [changes]
if: needs.changes.outputs.agent_skill == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
Expand All @@ -106,6 +173,8 @@ jobs:

lint:
name: Lint
needs: [changes]
if: needs.changes.outputs.go == 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
Expand All @@ -128,7 +197,7 @@ jobs:
github-credential-host-tests:
name: GitHub Credential Host Tests (${{ matrix.runner }})
needs: [changes]
if: needs.changes.outputs.go == 'true'
if: needs.changes.outputs.gh_credentials == 'true'
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -193,7 +262,7 @@ jobs:
provider-update-windows:
name: Provider Update Tests (windows-latest)
needs: [changes]
if: needs.changes.outputs.go == 'true'
if: needs.changes.outputs.provider_update == 'true'
runs-on: windows-latest
# Allow cold Go compilation and the post-job cache save to complete.
timeout-minutes: 30
Expand Down Expand Up @@ -221,17 +290,11 @@ jobs:

build-cli:
name: Build CLI Binary on ${{ matrix.runner }}
needs: [changes, precommit, lint]
if: needs.changes.outputs.go == 'true'
needs: [changes, plan]
if: needs.changes.outputs.go == 'true' && needs.plan.outputs.build != '{"include":[]}'
strategy:
matrix:
include:
- runner: ubuntu-latest
job_timeout_minutes: 45
- runner: macos-latest
- runner: windows-latest
# Cold builds need room to save the Go cache after artifact uploads.
job_timeout_minutes: 45
fail-fast: false
matrix: ${{ fromJSON(needs.plan.outputs.build) }}
runs-on: ${{ matrix.runner }}
timeout-minutes: ${{ matrix.job_timeout_minutes || 20 }}
steps:
Expand Down Expand Up @@ -318,24 +381,24 @@ jobs:
path: dist/devsy-${{ steps.os.outputs.runner_os }}_*/devsy-${{ steps.os.outputs.runner_os }}-*

- name: build MicroSandbox CI helper
if: runner.os == 'Linux'
if: runner.os == 'Linux' && matrix.build_microsandbox
env:
CGO_ENABLED: 0
run: go build -trimpath -o "${RUNNER_TEMP}/devsy-ci-microsandbox" ./hack/microsandbox

- name: build pinned runtime performance tools
if: runner.os == 'Linux'
if: runner.os == 'Linux' && matrix.build_microsandbox
run: '"${RUNNER_TEMP}/devsy-ci-microsandbox" build-performance-tools --output-dir "${RUNNER_TEMP}/runtime-performance-tools"'

- name: upload runtime performance tools
if: runner.os == 'Linux'
if: runner.os == 'Linux' && matrix.build_microsandbox
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: runtime-performance-tools-linux
path: ${{ runner.temp }}/runtime-performance-tools/

- name: upload MicroSandbox CI helper
if: runner.os == 'Linux'
if: runner.os == 'Linux' && matrix.build_microsandbox
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: devsy-ci-microsandbox-linux
Expand Down Expand Up @@ -380,8 +443,8 @@ jobs:

licenses:
name: Third-party licenses
needs: [changes, precommit, lint]
if: needs.changes.outputs.go == 'true'
needs: [changes]
if: needs.changes.outputs.licenses == 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
Expand All @@ -397,7 +460,7 @@ jobs:
integration-tests-unprivileged:
name: ${{ matrix.title }}
needs: [changes, plan, build-cli]
if: needs.changes.outputs.go == 'true'
if: needs.changes.outputs.go == 'true' && needs.plan.outputs.unprivileged != '{"include":[]}'
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.plan.outputs.unprivileged) }}
Expand Down Expand Up @@ -446,8 +509,8 @@ jobs:

prepare-microsandbox-provider:
name: Prepare MicroSandbox provider assets
needs: [changes, build-cli]
if: needs.changes.outputs.go == 'true'
needs: [changes, plan, build-cli]
if: needs.changes.outputs.go == 'true' && needs.plan.outputs.microsandbox != '{"include":[]}'
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
Expand Down Expand Up @@ -496,7 +559,7 @@ jobs:
integration-tests-microsandbox:
name: ${{ matrix.title }}
needs: [changes, plan, build-cli, prepare-microsandbox-provider]
if: needs.changes.outputs.go == 'true'
if: needs.changes.outputs.go == 'true' && needs.plan.outputs.microsandbox != '{"include":[]}'
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.plan.outputs.microsandbox) }}
Expand Down Expand Up @@ -621,7 +684,7 @@ jobs:
integration-tests:
name: ${{ matrix.title }}
needs: [changes, plan, build-cli]
if: needs.changes.outputs.go == 'true'
if: needs.changes.outputs.go == 'true' && needs.plan.outputs.integration != '{"include":[]}'
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.plan.outputs.integration) }}
Expand Down Expand Up @@ -933,9 +996,7 @@ jobs:
with:
go-version-file: go.mod

# Jobs gated on Go changes may be skipped only when no Go-relevant file
# changed; the jobs listed in --always-run must never be skipped.
- name: Check required jobs
env:
NEEDS_JSON: ${{ toJSON(needs) }}
run: go run ./hack/ci success --always-run changes,plan,precommit,lint,agent-skill
run: go run ./hack/ci success --always-run changes,plan,precommit
36 changes: 34 additions & 2 deletions hack/ci/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,32 @@ and subcommands here decide which tests exist, where they run, and how.
runs it on each runner in `unit.runners`. The workflow builds one job matrix per
e2e group from `e2e.suites`.

## Change selection

Pushes to main and manual workflow runs use `--mode=full`. They run every
suite whose required secret is available and build Linux, macOS, and Windows
binaries.

Pull requests use `--mode=pr --changed-files FILE`. The file contains one
repository-relative path per line. Paths in `selection.full_run_paths` select
every suite. Paths in `selection.core_paths` select the 13 suites marked
`pr_smoke` plus suites with a matching `paths` entry. Other changes select
only suites with a matching entry. Patterns support `*` within one path
segment and `**` across zero or more segments.

Changes under `e2e/cmd/` are shared CI dependencies and select the full matrix.

The build matrix follows selected suites. Every E2E run needs the Linux
binary, including Windows runs that inject it into containers. Windows
suites also need a Windows build. MicroSandbox suites enable the provider
build on Linux. Full-run paths retain all three build runners. Unmatched
changes produce empty E2E and build matrices. Unit test runners are unchanged.

`ci success` accepts conditional skips only when successful change detection
and planning provide valid outputs that explain them. Missing or failed
planning, invalid outputs, and unexpectedly skipped required jobs fail the
gate.

## Adding tests

- **Go unit test in an ordinary package**: no change. The `packages` step finds
Expand All @@ -31,10 +57,16 @@ e2e group from `e2e.suites`.
- **Ginkgo spec**: add its label to `e2e.suites` and choose a `group`. Set only
the fields that differ from the group defaults, such as `runner`,
`test_timeout`, `install_kind`, `install_podman`, `requires_secret`, and
`host_setup`. List labels that PR CI must not select directly in
`host_setup`. Add `paths` for the suite's test files and product dependencies.
Set `pr_smoke` only for the shared core smoke tier. List labels that PR CI
must not select directly in
`e2e.ungated` with a reason. These include sub-labels exercised by gated
suites and labels run manually or by another workflow.

Keep suite paths aligned with implementation packages when code moves.
Include runtime helpers such as `pkg/dockerinstall/**`, not only provider wrappers.
Include direct compile dependencies in native host job filters.

`go test ./hack/ci` fails when a Ginkgo label or Go test package has no tier, or
when the manifest names a label that no longer exists. New tests need no
workflow edits.
Expand All @@ -43,7 +75,7 @@ workflow edits.

| Command | Purpose |
| --- | --- |
| `ci matrix [--group G] [--secret-available]` | Print the job matrices as GitHub outputs. Drops suites that need the GitHub App secret when it is unavailable. |
| `ci matrix [--group G] [--secret-available] [--mode full\|pr] [--changed-files FILE]` | Print unit, build, and E2E job matrices as GitHub outputs. Apply change selection and secret availability. |
| `ci unit [--os GOOS] [--step NAME] [--list] [--coverprofile P]` | Run the unit tier on this machine. |
| `ci e2e prepare --bin-src DIR --e2e-dir DIR` | Put the CLI binaries next to `e2e.test`. |
| `ci e2e run --label L --runner R` | Run a suite, with its focused checks, sudo and environment setup, and post checks. |
Expand Down
32 changes: 32 additions & 0 deletions hack/ci/coverage_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io/fs"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"slices"
Expand Down Expand Up @@ -148,6 +149,37 @@ func (s *coverageSuite) TestUnitEntriesExistOnDisk() {
}
}

func (s *coverageSuite) TestEverySuiteHasSelectionPaths() {
s.NotEmpty(s.manifest.Selection.FullRunPaths)
s.NotEmpty(s.manifest.Selection.CorePaths)
patterns := append([]string{}, s.manifest.Selection.FullRunPaths...)
patterns = append(patterns, s.manifest.Selection.CorePaths...)
for _, selected := range s.manifest.E2E.Suites {
s.NotEmpty(selected.Paths, "%s/%s needs selection paths", selected.Runner, selected.Label)
patterns = append(patterns, selected.Paths...)
}
for _, pattern := range patterns {
normalized := normalizeSelectionPath(pattern)
prefix := normalized
wildcard := strings.IndexAny(normalized, "*?[")
if wildcard >= 0 {
prefix = path.Dir(normalized[:wildcard])
}
info, err := os.Stat(filepath.Join(s.root, filepath.FromSlash(prefix)))
if !s.NoError(err, "selection pattern %q has missing static prefix %q", pattern, prefix) {
continue
}
if wildcard >= 0 {
s.True(
info.IsDir(),
"selection pattern %q has a non-directory static prefix %q",
pattern,
prefix,
)
}
}
}

func (s *coverageSuite) assignedLabels() map[string]bool {
assigned := map[string]bool{}
for _, label := range s.manifestLabels() {
Expand Down
Loading
Loading