From fb88a4891a2e75afa55eee3295dcf0f26e29ca47 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 9 Nov 2026 22:57:21 -0600 Subject: [PATCH 1/7] feat(ci): add suite path mappings and PR smoke selection rules --- hack/ci/coverage_test.go | 32 +++++ hack/ci/manifest.go | 28 +++- hack/ci/manifest_test.go | 35 +++++ hack/ci/select.go | 151 ++++++++++++++++++++ hack/ci/select_test.go | 301 +++++++++++++++++++++++++++++++++++++++ hack/ci/suites.yaml | 208 +++++++++++++++++---------- 6 files changed, 676 insertions(+), 79 deletions(-) create mode 100644 hack/ci/select.go create mode 100644 hack/ci/select_test.go diff --git a/hack/ci/coverage_test.go b/hack/ci/coverage_test.go index 006998233c..a717f3ec4e 100644 --- a/hack/ci/coverage_test.go +++ b/hack/ci/coverage_test.go @@ -7,6 +7,7 @@ import ( "io/fs" "os" "os/exec" + "path" "path/filepath" "regexp" "slices" @@ -148,6 +149,37 @@ func (s *coverageSuite) TestUnitEntriesExistOnDisk() { } } +func (s *coverageSuite) TestEverySuiteHasSelectionPaths() { + s.NotEmpty(s.manifest.Selection.FullRunPaths) + s.NotEmpty(s.manifest.Selection.CorePaths) + patterns := append([]string{}, s.manifest.Selection.FullRunPaths...) + patterns = append(patterns, s.manifest.Selection.CorePaths...) + for _, selected := range s.manifest.E2E.Suites { + s.NotEmpty(selected.Paths, "%s/%s needs selection paths", selected.Runner, selected.Label) + patterns = append(patterns, selected.Paths...) + } + for _, pattern := range patterns { + normalized := normalizeSelectionPath(pattern) + prefix := normalized + wildcard := strings.IndexAny(normalized, "*?[") + if wildcard >= 0 { + prefix = path.Dir(normalized[:wildcard]) + } + info, err := os.Stat(filepath.Join(s.root, filepath.FromSlash(prefix))) + if !s.NoError(err, "selection pattern %q has missing static prefix %q", pattern, prefix) { + continue + } + if wildcard >= 0 { + s.True( + info.IsDir(), + "selection pattern %q has a non-directory static prefix %q", + pattern, + prefix, + ) + } + } +} + func (s *coverageSuite) assignedLabels() map[string]bool { assigned := map[string]bool{} for _, label := range s.manifestLabels() { diff --git a/hack/ci/manifest.go b/hack/ci/manifest.go index 3402b2ab19..ddbb495d17 100644 --- a/hack/ci/manifest.go +++ b/hack/ci/manifest.go @@ -38,12 +38,19 @@ var ( errDuplicateSuite = errors.New("duplicate suite") errInvalidSuite = errors.New("invalid suite") errInvalidStep = errors.New("invalid unit step") + errInvalidGlob = errors.New("invalid selection glob") ) // Manifest is the parsed hack/ci/suites.yaml. type Manifest struct { - Unit UnitTier `yaml:"unit"` - E2E E2ETier `yaml:"e2e"` + Unit UnitTier `yaml:"unit"` + E2E E2ETier `yaml:"e2e"` + Selection SelectionConfig `yaml:"selection"` +} + +type SelectionConfig struct { + FullRunPaths []string `yaml:"full_run_paths"` + CorePaths []string `yaml:"core_paths"` } // UnitTier describes how Go unit tests run. @@ -114,9 +121,11 @@ type FocusedCheck struct { // Suite is one Ginkgo label executed as one CI job. After Resolve, every // inherited field is populated. type Suite struct { - Group string `yaml:"group"` - Label string `yaml:"label"` - Runner string `yaml:"runner"` + Group string `yaml:"group"` + Label string `yaml:"label"` + Runner string `yaml:"runner"` + PRSmoke bool `yaml:"pr_smoke"` + Paths []string `yaml:"paths"` // Privileged selects sudo for the e2e binary on Linux. Nil inherits the group. Privileged *bool `yaml:"privileged"` TestTimeout string `yaml:"test_timeout"` @@ -220,6 +229,9 @@ func (s Suite) validate() error { case s.requiresSpecCount() && s.ExpectedSpecCount <= 0: return fmt.Errorf("%w: %s needs expected_spec_count", errInvalidSuite, s.Label) } + if err := validateGlobs(s.Paths); err != nil { + return fmt.Errorf("%w: %s paths: %w", errInvalidSuite, s.Label, err) + } if err := s.validateFixtures(); err != nil { return err } @@ -312,6 +324,12 @@ func (m *Manifest) resolve() error { } func (m *Manifest) validate() error { + if err := validateGlobs(m.Selection.FullRunPaths); err != nil { + return fmt.Errorf("full_run_paths: %w", err) + } + if err := validateGlobs(m.Selection.CorePaths); err != nil { + return fmt.Errorf("core_paths: %w", err) + } seen := map[string]bool{} for _, s := range m.E2E.Suites { key := s.Runner + "/" + s.Label diff --git a/hack/ci/manifest_test.go b/hack/ci/manifest_test.go index ce9f3f27de..19f2ac8ba0 100644 --- a/hack/ci/manifest_test.go +++ b/hack/ci/manifest_test.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "testing" "github.com/stretchr/testify/suite" @@ -200,6 +201,40 @@ func (s *manifestSuite) TestParseRejectsUnknownKey() { s.ErrorContains(err, "flake_attempt") } +func (s *manifestSuite) TestSelectionGlobValidation() { + const pathsField = "paths" + for _, pattern := range []string{"", " ", "[", "pkg/[", "pkg/[/../file.go", "/pkg/**", "../pkg/**"} { + for _, field := range []string{"full_run_paths", "core_paths", pathsField} { + s.Run(field+"/"+pattern, func() { + data := testGroupsYAML + " - { group: integration, label: x }\n" + if field == pathsField { + data = testGroupsYAML + fmt.Sprintf( + " - { group: integration, label: x, paths: [%q] }\n", pattern, + ) + } else { + data += fmt.Sprintf("selection:\n %s: [%q]\n", field, pattern) + } + _, err := ParseManifest([]byte(data)) + s.ErrorIs(err, errInvalidGlob) + if field == pathsField { + s.ErrorIs(err, errInvalidSuite) + } + }) + } + } +} + +func (s *manifestSuite) TestSyntheticManifestNeedsNoSelectionRules() { + m, err := ParseManifest([]byte(testGroupsYAML + " - { group: integration, label: x }\n")) + s.Require().NoError(err) + s.Equal( + []string{"x"}, + selectionLabels( + selectSuites(m, groupIntegration, selectionOptions{mode: selectionModeFull}), + ), + ) +} + // Footgun: RunsOn disagrees with the os list, so a step runs on a platform it // was meant to skip or is skipped on its own platform. func (s *manifestSuite) TestUnitStepRunsOn() { diff --git a/hack/ci/select.go b/hack/ci/select.go new file mode 100644 index 0000000000..60b104ccd8 --- /dev/null +++ b/hack/ci/select.go @@ -0,0 +1,151 @@ +package main + +import ( + "fmt" + "path" + "strings" +) + +const ( + selectionModeFull = "full" + selectionModePR = "pr" +) + +type selectionOptions struct { + secretAvailable bool + mode string + changedFiles []string +} + +type globMatcher struct { + patterns []string + files []string + patternIndex int + fileIndex int + globstarIndex int + globstarFileIndex int +} + +func normalizeSelectionPath(value string) string { + return path.Clean(strings.ReplaceAll(value, "\\", "/")) +} + +func validateGlobs(patterns []string) error { + for _, pattern := range patterns { + normalized := normalizeSelectionPath(pattern) + if strings.TrimSpace(pattern) == "" || normalized == "." || path.IsAbs(normalized) || + normalized == ".." || strings.HasPrefix(normalized, "../") { + return fmt.Errorf( + "%w: %q must be a non-empty repository-relative pattern", + errInvalidGlob, + pattern, + ) + } + if _, err := path.Match(strings.ReplaceAll(pattern, "\\", "/"), ""); err != nil { + return fmt.Errorf("%w: %q: %w", errInvalidGlob, pattern, err) + } + } + return nil +} + +func matchGlob(pattern, filePath string) bool { + filePath = normalizeSelectionPath(filePath) + if filePath == "." || path.IsAbs(filePath) || filePath == ".." || + strings.HasPrefix(filePath, "../") { + return false + } + matcher := globMatcher{ + patterns: strings.Split(normalizeSelectionPath(pattern), "/"), + files: strings.Split(filePath, "/"), + globstarIndex: -1, + } + return matcher.match() +} + +func (m *globMatcher) match() bool { + for m.fileIndex < len(m.files) { + if m.matchSegment() { + continue + } + if !m.backtrack() { + return false + } + } + for m.patternIndex < len(m.patterns) && m.patterns[m.patternIndex] == "**" { + m.patternIndex++ + } + return m.patternIndex == len(m.patterns) +} + +func (m *globMatcher) matchSegment() bool { + if m.patternIndex == len(m.patterns) { + return false + } + if m.patterns[m.patternIndex] == "**" { + m.globstarIndex = m.patternIndex + m.globstarFileIndex = m.fileIndex + m.patternIndex++ + return true + } + matched, err := path.Match(m.patterns[m.patternIndex], m.files[m.fileIndex]) + if err != nil || !matched { + return false + } + m.patternIndex++ + m.fileIndex++ + return true +} + +func (m *globMatcher) backtrack() bool { + if m.globstarIndex < 0 { + return false + } + m.globstarFileIndex++ + m.fileIndex = m.globstarFileIndex + m.patternIndex = m.globstarIndex + 1 + return true +} + +func matchesAny(changedFiles, patterns []string) bool { + for _, file := range changedFiles { + for _, pattern := range patterns { + if matchGlob(pattern, file) { + return true + } + } + } + return false +} + +func selectSuites(m *Manifest, group string, opts selectionOptions) []Suite { + suites := m.SuitesIn(group) + selected := make([]Suite, 0, len(suites)) + full, coreChanged := opts.scope(m.Selection) + for _, s := range suites { + if s.RequiresSecret && !opts.secretAvailable { + continue + } + if opts.includes(s, full, coreChanged) { + selected = append(selected, s) + } + } + return selected +} + +func (opts selectionOptions) scope(config SelectionConfig) (bool, bool) { + if opts.mode == "" || opts.mode == selectionModeFull { + return true, false + } + if opts.mode != selectionModePR || len(opts.changedFiles) == 0 { + return false, false + } + if matchesAny(opts.changedFiles, config.FullRunPaths) { + return true, false + } + return false, matchesAny(opts.changedFiles, config.CorePaths) +} + +func (opts selectionOptions) includes(s Suite, full, coreChanged bool) bool { + return full || + (opts.mode == selectionModePR && ((coreChanged && s.PRSmoke) || matchesAny(opts.changedFiles, s.Paths))) +} diff --git a/hack/ci/select_test.go b/hack/ci/select_test.go new file mode 100644 index 0000000000..3603ac813f --- /dev/null +++ b/hack/ci/select_test.go @@ -0,0 +1,301 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +const ( + selectionGoMod = "go.mod" + selectionMCPDir = "pkg/mcp" + selectionMCPFile = "pkg/mcp/server.go" + selectionNestedMCPFile = "pkg/mcp/internal/server.go" + selectionSegmentPattern = "pkg/*/server.go" + selectionAgentPattern = "cmd/internal/agent*.go" + selectionMCPPattern = "pkg/mcp/**" + selectionRecursivePattern = "pkg/**/server.go" + selectionMultiplePattern = "**/a/**/b.go" + selectionCommandFile = "cmd/mcp/serve.go" + selectionGlobstar = "**" + selectionSmokeLabel = "smoke" + selectionLeafLabel = "leaf" + selectionSecretLabel = "secret" + selectionLeafFile = "pkg/leaf/file.go" + selectionCoreFile = "pkg/core/file.go" +) + +type selectionSuite struct { + suite.Suite +} + +func TestSelectionSuite(t *testing.T) { + suite.Run(t, new(selectionSuite)) +} + +func (s *selectionSuite) TestGlobBoundaries() { + tests := []struct { + name string + pattern string + file string + want bool + }{ + {"literal", selectionGoMod, selectionGoMod, true}, + {"literal suffix", selectionGoMod, "go.mod.bak", false}, + {"literal directory", selectionMCPDir, selectionMCPFile, false}, + {"segment", selectionSegmentPattern, selectionMCPFile, true}, + {"segment does not descend", selectionSegmentPattern, selectionNestedMCPFile, false}, + {"segment does not disappear", selectionSegmentPattern, "pkg/server.go", false}, + {"filename star empty", selectionAgentPattern, "cmd/internal/agent.go", true}, + {"filename star suffix", selectionAgentPattern, "cmd/internal/agent_daemon.go", true}, + { + "filename star stops at slash", + selectionAgentPattern, + "cmd/internal/agent/daemon.go", + false, + }, + {"recursive immediate child", selectionMCPPattern, selectionMCPFile, true}, + {"recursive deep child", selectionMCPPattern, selectionNestedMCPFile, true}, + {"recursive empty suffix", selectionMCPPattern, selectionMCPDir, true}, + {"recursive sibling prefix", selectionMCPPattern, "pkg/mcp-other/server.go", false}, + {"globstar zero directories", selectionRecursivePattern, "pkg/server.go", true}, + {"globstar one directory", selectionRecursivePattern, selectionMCPFile, true}, + {"globstar many directories", selectionRecursivePattern, selectionNestedMCPFile, true}, + {"globstar requires suffix", selectionRecursivePattern, "pkg/mcp/client.go", false}, + {"globstar backtracks", "**/a/b.go", "a/x/a/b.go", true}, + {"multiple globstars", selectionMultiplePattern, "x/a/y/a/z/b.go", true}, + {"multiple empty globstars", selectionMultiplePattern, "a/b.go", true}, + {"multiple globstars missing suffix", selectionMultiplePattern, "x/a/y/c.go", false}, + {"root globstar", "**/*.go", "main.go", true}, + {"root globstar nested", "**/*.go", selectionCommandFile, true}, + {"single star root only", "*.go", selectionCommandFile, false}, + {"windows changed path", "cmd/mcp/**", `cmd\mcp\serve.go`, true}, + {"windows pattern", `cmd\mcp\**`, selectionCommandFile, true}, + {"relative path prefix", "./cmd/mcp/**", "./cmd/mcp/serve.go", true}, + {"duplicate slash", "cmd//mcp/**", "cmd/mcp//serve.go", true}, + {"character class", "pkg/[ab]/*.go", "pkg/a/file.go", true}, + {"character class boundary", "pkg/[ab]/*.go", "pkg/c/file.go", false}, + {"invalid pattern", "pkg/[", "pkg/file.go", false}, + {"empty changed path", selectionGlobstar, "", false}, + {"absolute changed path", selectionGlobstar, "/cmd/mcp/serve.go", false}, + {"parent changed path", selectionGlobstar, "../cmd/mcp/serve.go", false}, + } + for _, tt := range tests { + s.Run(tt.name, func() { + s.Equal(tt.want, matchGlob(tt.pattern, tt.file), "%q against %q", tt.pattern, tt.file) + }) + } +} + +func (s *selectionSuite) TestSelectionPrecedenceAndSecrets() { + m := &Manifest{ + Selection: SelectionConfig{ + FullRunPaths: []string{"hack/ci/**"}, + CorePaths: []string{"pkg/core/**"}, + }, + E2E: E2ETier{Suites: []Suite{ + { + Group: groupIntegration, + Label: selectionSmokeLabel, + PRSmoke: true, + Paths: []string{"pkg/smoke/**"}, + }, + {Group: groupIntegration, Label: selectionLeafLabel, Paths: []string{"pkg/leaf/**"}}, + { + Group: groupIntegration, Label: selectionSecretLabel, PRSmoke: true, + RequiresSecret: true, Paths: []string{"pkg/private/**"}, + }, + { + Group: groupMicrosandbox, + Label: "other", + PRSmoke: true, + Paths: []string{"pkg/leaf/**"}, + }, + }}, + } + for _, tt := range selectionPrecedenceCases { + s.Run(tt.name, func() { + selected := selectSuites(m, groupIntegration, selectionOptions{ + secretAvailable: tt.secret, + mode: tt.mode, + changedFiles: tt.files, + }) + s.NotNil(selected) + s.Equal(tt.want, selectionLabels(selected)) + }) + } + s.Empty( + selectSuites( + m, + "missing", + selectionOptions{secretAvailable: true, mode: selectionModeFull}, + ), + ) + opts := selectionOptions{ + secretAvailable: true, + mode: selectionModePR, + changedFiles: []string{selectionLeafFile}, + } + s.Equal([]string{"other"}, selectionLabels(selectSuites(m, groupMicrosandbox, opts))) +} + +func (s *selectionSuite) TestEmbeddedLeafAndDocumentationSelection() { + m, err := LoadManifest() + s.Require().NoError(err) + for _, group := range []string{groupUnprivileged, groupIntegration, groupMicrosandbox} { + s.Run(group, func() { + want := []string{} + if group == groupIntegration { + want = []string{"mcp"} + } + leaf := selectionOptions{ + secretAvailable: true, + mode: selectionModePR, + changedFiles: []string{`cmd\mcp\serve.go`}, + } + docs := selectionOptions{ + secretAvailable: true, mode: selectionModePR, + changedFiles: []string{"sites/docs-devsy-sh/content/docs/index.mdx"}, + } + s.Equal(want, selectionLabels(selectSuites(m, group, leaf))) + s.Equal([]string{}, selectionLabels(selectSuites(m, group, docs))) + }) + } +} + +func (s *selectionSuite) TestEmbeddedCoreSmokeTier() { + m, err := LoadManifest() + s.Require().NoError(err) + var smoke []string + opts := selectionOptions{ + secretAvailable: true, + mode: selectionModePR, + changedFiles: []string{"cmd/root.go"}, + } + for _, group := range []string{groupUnprivileged, groupIntegration, groupMicrosandbox} { + for _, selected := range selectSuites(m, group, opts) { + smoke = append(smoke, selected.Runner+"/"+selected.Label) + } + } + s.ElementsMatch([]string{ + "ubuntu-latest/context", + "ubuntu-latest/extends", + "ubuntu-latest/up-workspaces", + "ubuntu-latest/up-provider-docker", + "ubuntu-latest/up-docker-compose", + "ubuntu-latest/up-features", + "ubuntu-latest/ssh", + "ubuntu-latest/build", + "ubuntu-latest/up-provider-kubernetes", + "ubuntu-24.04/up-provider-podman-rootless-basic", + "ubuntu-24.04/up-provider-podman-rootful-basic", + "windows-latest/up-docker-wsl", + "ubuntu-latest/up-provider-microsandbox", + }, smoke) +} + +func (s *selectionSuite) TestInactivityImplementationSelectsLongRunningSuite() { + m, err := LoadManifest() + s.Require().NoError(err) + for _, file := range []string{ + "cmd/internal/agentcontainer/daemon.go", + "cmd/internal/agentcontainer/ssh_server.go", + "pkg/daemon/agent/lifecycle_monitor.go", + } { + opts := selectionOptions{ + secretAvailable: true, + mode: selectionModePR, + changedFiles: []string{file}, + } + selected := selectSuites(m, groupIntegration, opts) + s.Contains(selectionLabels(selected), "up-provider-kubernetes-inactivity", file) + } +} + +func selectionLabels(suites []Suite) []string { + labels := make([]string, 0, len(suites)) + for _, selected := range suites { + labels = append(labels, selected.Label) + } + return labels +} + +var selectionPrecedenceCases = []struct { + name string + mode string + files []string + secret bool + want []string +}{ + {"default full", "", nil, false, []string{selectionSmokeLabel, selectionLeafLabel}}, + { + "explicit full with secret", selectionModeFull, nil, true, + []string{selectionSmokeLabel, selectionLeafLabel, selectionSecretLabel}, + }, + { + "full ignores unrelated files", selectionModeFull, + []string{"docs/readme.md"}, + false, + []string{selectionSmokeLabel, selectionLeafLabel}, + }, + {"pr empty", selectionModePR, nil, true, []string{}}, + {"pr unrelated", selectionModePR, []string{"docs/readme.md"}, true, []string{}}, + { + "leaf excludes unrelated smoke", + selectionModePR, + []string{selectionLeafFile}, + false, + []string{selectionLeafLabel}, + }, + { + "core selects smoke", + selectionModePR, + []string{selectionCoreFile}, + false, + []string{selectionSmokeLabel}, + }, + { + "core includes secret when available", selectionModePR, + []string{selectionCoreFile}, + true, + []string{selectionSmokeLabel, selectionSecretLabel}, + }, + { + "direct secret excluded", + selectionModePR, + []string{"pkg/private/file.go"}, + false, + []string{}, + }, + { + "direct secret included", + selectionModePR, + []string{"pkg/private/file.go"}, + true, + []string{selectionSecretLabel}, + }, + { + "core plus leaf union", selectionModePR, + []string{selectionLeafFile, selectionCoreFile}, + true, + []string{selectionSmokeLabel, selectionLeafLabel, selectionSecretLabel}, + }, + { + "overlapping and repeated changes", selectionModePR, + []string{selectionCoreFile, "pkg/smoke/file.go", selectionCoreFile}, + false, + []string{selectionSmokeLabel}, + }, + { + "full trigger excludes secret", selectionModePR, + []string{"hack/ci/select.go"}, + false, + []string{selectionSmokeLabel, selectionLeafLabel}, + }, + { + "full trigger includes secret", selectionModePR, + []string{"hack/ci/select.go"}, + true, + []string{selectionSmokeLabel, selectionLeafLabel, selectionSecretLabel}, + }, +} diff --git a/hack/ci/suites.yaml b/hack/ci/suites.yaml index 3414e9334c..8faea4a2f0 100644 --- a/hack/ci/suites.yaml +++ b/hack/ci/suites.yaml @@ -1,3 +1,58 @@ +selection: + full_run_paths: + - .github/workflows/pr-ci.yml + - .github/actions/** + - hack/ci/** + - go.mod + - go.sum + - .goreleaser.yml + - e2e/framework/** + - e2e/*.go + core_paths: + - main.go + - cmd/root.go + - cmd/flags/** + - cmd/workspace/up/** + - cmd/internal/agent*.go + - cmd/internal/agentworkspace/** + - cmd/internal/agentcontainer/** + - cmd/internal/runusercommands.go + - pkg/agent/** + - pkg/client/** + - pkg/command/** + - pkg/config/** + - pkg/copy/** + - pkg/devcontainer/*.go + - pkg/devcontainer/build/** + - pkg/devcontainer/config/** + - pkg/devcontainer/setup/** + - pkg/devcontainer/metadata/** + - pkg/dockerfile/** + - pkg/driver/*.go + - pkg/driver/drivercreate/create.go + - pkg/encoding/** + - pkg/envfile/** + - pkg/exitcode/** + - pkg/extract/** + - pkg/flags/** + - pkg/git/** + - pkg/image/** + - pkg/inject/** + - pkg/language/** + - pkg/log/** + - pkg/options/** + - pkg/output/** + - pkg/provider/** + - pkg/secrets/** + - pkg/shell/** + - pkg/ssh/** + - pkg/status/** + - pkg/telemetry/** + - pkg/types/** + - pkg/util/** + - pkg/workspace/** + - providers/providers.go + unit: # One job per runner; every runner executes the steps whose `os` matches. runners: @@ -180,55 +235,58 @@ e2e: suites: # --- unprivileged --- - - { group: unprivileged, label: context } - - { group: unprivileged, label: extends } - - { group: unprivileged, label: outdated } - - { group: unprivileged, label: machine } - - { group: unprivileged, label: self-update } - - { group: unprivileged, label: upgrade } + - { group: unprivileged, label: context, pr_smoke: true, paths: ["e2e/tests/context/**", "cmd/context/**", "pkg/config/**", "pkg/options/**"] } + - { group: unprivileged, label: extends, pr_smoke: true, paths: ["e2e/tests/extends/**", "cmd/workspace/describe.go", "pkg/devcontainer/config/**", "pkg/compose/**"] } + - { group: unprivileged, label: outdated, paths: ["e2e/tests/outdated/**", "cmd/feature/outdated.go", "pkg/devcontainer/feature/**", "pkg/image/**"] } + - { group: unprivileged, label: machine, paths: ["e2e/tests/machine/**", "cmd/machine/**", "pkg/driver/custom/**"] } + - { group: unprivileged, label: self-update, paths: ["e2e/tests/self/**", "cmd/update/**", "pkg/selfupdate/**", "pkg/version/**"] } + - { group: unprivileged, label: upgrade, paths: ["e2e/tests/upgrade/**", "cmd/feature/upgrade.go", "pkg/devcontainer/feature/**", "pkg/image/**"] } # --- integration: Linux --- - - { group: integration, label: docker-context, test_timeout: 300s, job_timeout_minutes: 10 } - - { group: integration, label: ide } - - { group: integration, label: machine-provider } - - { group: integration, label: integration } - - { group: integration, label: tunnel, flake_attempts: 2 } - - { group: integration, label: mcp } - - { group: integration, label: provider, free_disk_space: true, install_kind: true, flake_attempts: 2 } - - { group: integration, label: ssh, fixtures: [ghfixture-linux] } - - { group: integration, label: ssh-agent-forward } - - { group: integration, label: ssh-ports-attributes } - - { group: integration, label: ssh-tunnel-mode } - - { group: integration, label: task-cancel } - - { group: integration, label: ssh-credentials-server-race } - - { group: integration, label: build, install_kind: true, requires_secret: true } + - { group: integration, label: docker-context, test_timeout: 300s, job_timeout_minutes: 10, paths: ["e2e/tests/dockercontext/**", "pkg/docker/**", "pkg/driver/docker/**", "providers/docker/**"] } + - { group: integration, label: ide, paths: ["e2e/tests/ide/**", "cmd/ide/**", "cmd/workspace/set_ide.go", "cmd/internal/browser_tunnel.go", "pkg/ide/**", "pkg/ide/opener/**", "pkg/open/**"] } + - { group: integration, label: machine-provider, paths: ["e2e/tests/machineprovider/**", "cmd/machine/**", "cmd/provider/**", "pkg/driver/custom/**"] } + - { group: integration, label: integration, paths: ["e2e/tests/integration/**", "cmd/workspace/**", "pkg/workspace/**"] } + - { group: integration, label: tunnel, flake_attempts: 2, paths: ["e2e/tests/tunnel/**", "cmd/workspace/**", "pkg/tunnel/**", "pkg/port/**", "pkg/credentials/**"] } + - { group: integration, label: mcp, paths: ["e2e/tests/mcp/**", "cmd/mcp/**"] } + - { group: integration, label: provider, free_disk_space: true, install_kind: true, flake_attempts: 2, paths: ["e2e/tests/provider/**", "cmd/provider/**", "pkg/provider/**", "pkg/options/**", "providers/**"] } + - { group: integration, label: ssh, fixtures: [ghfixture-linux], pr_smoke: true, paths: ["e2e/tests/ssh/**", "cmd/workspace/ssh.go", "cmd/workspace/port_forward.go", "cmd/workspace/gpg_tunnel.go", "cmd/internal/ssh_*.go", "cmd/internal/container_tunnel.go", "cmd/internal/gh.go", "cmd/internal/install_gh_wrapper.go", "cmd/internal/git_*.go", "pkg/ssh/**", "pkg/gpg/**", "pkg/credentials/**", "pkg/ghcredentials/**", "pkg/gitcredentials/**", "pkg/gitsshsigning/**", "pkg/port/**"] } + - { group: integration, label: ssh-agent-forward, paths: ["e2e/tests/ssh/**", "cmd/internal/ssh_*.go", "pkg/ssh/**"] } + - { group: integration, label: ssh-ports-attributes, paths: ["e2e/tests/ssh/**", "cmd/internal/container_tunnel.go", "pkg/port/**", "pkg/netstat/**", "pkg/devcontainer/config/**"] } + - { group: integration, label: ssh-tunnel-mode, paths: ["e2e/tests/ssh/**", "cmd/workspace/up/**", "cmd/workspace/task*.go", "pkg/ssh/**", "pkg/task/**", "pkg/command/**"] } + - { group: integration, label: task-cancel, paths: ["e2e/tests/ssh/**", "cmd/workspace/task*.go", "cmd/workspace/stop.go", "pkg/task/**", "pkg/command/**"] } + - { group: integration, label: ssh-credentials-server-race, paths: ["e2e/tests/ssh/**", "cmd/workspace/ssh.go", "cmd/internal/container_tunnel.go", "pkg/credentials/**"] } + - { group: integration, label: build, install_kind: true, requires_secret: true, pr_smoke: true, paths: ["e2e/tests/build/**", "cmd/workspace/build.go", "pkg/devcontainer/build/**", "pkg/devcontainer/feature/**", "pkg/dockerfile/**", "pkg/devcontainer/metadata/**", "pkg/devcontainer/buildkit/**", "pkg/driver/docker/**"] } - group: integration label: docker-install + paths: ["e2e/tests/dockerinstall/**", "pkg/docker/**", "pkg/driver/docker/**", "providers/docker/**"] host_setup: [remove-docker] post_checks: [docker-installed] - - { group: integration, label: exec } - - { group: integration, label: ci } - - { group: integration, label: feature } - - { group: integration, label: extends-up } - - { group: integration, label: down } - - { group: integration, label: rename } - - { group: integration, label: delivery } - - { group: integration, label: logs } - - { group: integration, label: templates } - - { group: integration, label: config-apply } - - { group: integration, label: run-user-commands } - - { group: integration, label: config-read } - - { group: integration, label: up-workspaces, free_disk_space: true, install_kind: true } - - { group: integration, label: up-behaviors } - - { group: integration, label: up-handle-errors, free_disk_space: true, install_kind: true } - - { group: integration, label: up-private-token, requires_secret: true } - - { group: integration, label: up-provider-kubernetes, free_disk_space: true, install_kind: true } - - { group: integration, label: up-provider-kubernetes-restricted-scc, free_disk_space: true, install_kind: true } - - { group: integration, label: up-provider-kubernetes-inactivity, free_disk_space: true, install_kind: true, test_timeout: 60m, job_timeout_minutes: 75 } - - { group: integration, label: up-provider-docker, install_kind: true } - - { group: integration, label: snapshot, host_setup: [insecure-registry] } + - { group: integration, label: exec, paths: ["e2e/tests/exec/**", "cmd/workspace/exec.go", "pkg/driver/docker/**", "pkg/ssh/**"] } + - { group: integration, label: ci, paths: ["e2e/tests/ci/**", "cmd/workspace/up/up.go", "cmd/workspace/build.go", "cmd/workspace/**", "pkg/git/**", "pkg/workspace/**"] } + - { group: integration, label: feature, paths: ["e2e/tests/feature/**", "cmd/feature/**", "pkg/devcontainer/feature/**", "pkg/image/**"] } + - { group: integration, label: extends-up, paths: ["e2e/tests/extends-up/**", "pkg/devcontainer/config/**"] } + - { group: integration, label: down, paths: ["e2e/tests/down/**", "cmd/workspace/stop.go", "cmd/workspace/delete.go", "pkg/workspace/**"] } + - { group: integration, label: rename, paths: ["e2e/tests/rename/**", "cmd/workspace/rename.go", "pkg/workspace/**", "pkg/ssh/**", "pkg/ide/vscode/**"] } + - { group: integration, label: delivery, paths: ["e2e/tests/delivery/**", "cmd/workspace/up/up.go", "cmd/workspace/up/**", "pkg/devcontainer/**", "pkg/driver/docker/**"] } + - { group: integration, label: logs, paths: ["e2e/tests/logs/**", "cmd/workspace/logs.go", "cmd/workspace/setup_logs.go", "cmd/internal/logs_daemon.go", "cmd/internal/runusercommands.go", "pkg/devcontainer/setup/**"] } + - { group: integration, label: templates, paths: ["e2e/tests/template/**", "cmd/template/**", "pkg/template/**"] } + - { group: integration, label: config-apply, paths: ["e2e/tests/configapply/**", "pkg/devcontainer/config/**", "pkg/driver/docker/**"] } + - { group: integration, label: run-user-commands, paths: ["e2e/tests/runusercommands/**", "cmd/internal/runusercommands.go", "pkg/devcontainer/setup/**"] } + - { group: integration, label: config-read, paths: ["e2e/tests/configread/**", "cmd/workspace/describe.go", "pkg/devcontainer/config/**", "pkg/compose/**"] } + - { group: integration, label: up-workspaces, free_disk_space: true, install_kind: true, pr_smoke: true, paths: ["e2e/tests/up/**", "cmd/workspace/up/up.go", "cmd/workspace/events.go", "cmd/container/**", "pkg/git/**", "pkg/language/**", "pkg/envfile/**"] } + - { group: integration, label: up-behaviors, paths: ["e2e/tests/up/**", "pkg/dotfiles/**", "pkg/ide/openvscode/**", "pkg/devcontainer/**"] } + - { group: integration, label: up-handle-errors, free_disk_space: true, install_kind: true, paths: ["e2e/tests/up/**", "cmd/workspace/up/**", "pkg/exitcode/**", "pkg/devcontainer/**"] } + - { group: integration, label: up-private-token, requires_secret: true, paths: ["e2e/tests/up/up_private_token.go", "cmd/internal/git_credentials.go", "pkg/git/**", "pkg/gitcredentials/**"] } + - { group: integration, label: up-provider-kubernetes, free_disk_space: true, install_kind: true, pr_smoke: true, paths: ["e2e/tests/up/provider_kubernetes*.go", "pkg/driver/kubernetes/**", "providers/kubernetes/**", "cmd/internal/ssh_git_clone.go"] } + - { group: integration, label: up-provider-kubernetes-restricted-scc, free_disk_space: true, install_kind: true, paths: ["e2e/tests/up/provider_kubernetes*.go", "pkg/driver/kubernetes/**", "providers/kubernetes/**", "cmd/internal/ssh_git_clone.go"] } + - { group: integration, label: up-provider-kubernetes-inactivity, free_disk_space: true, install_kind: true, test_timeout: 60m, job_timeout_minutes: 75, paths: ["e2e/tests/up/provider_kubernetes_inactivity.go", "pkg/driver/kubernetes/**", "providers/kubernetes/**", "cmd/internal/agent_daemon.go", "cmd/internal/ssh_server.go", "cmd/internal/agentcontainer/daemon.go", "cmd/internal/agentcontainer/ssh_server.go", "pkg/daemon/agent/**"] } + - { group: integration, label: up-provider-docker, install_kind: true, pr_smoke: true, paths: ["e2e/tests/up/provider_docker.go", "e2e/tests/up/dockerfile_build.go", "e2e/tests/up/interactive_post_start.go", "e2e/tests/up/sops.go", "cmd/secrets/**", "pkg/driver/docker/**", "pkg/docker/**", "pkg/secrets/**", "providers/docker/**"] } + - { group: integration, label: snapshot, host_setup: [insecure-registry], paths: ["e2e/tests/snapshot/**", "cmd/workspace/snapshot*.go", "pkg/snapshot/**", "pkg/driver/docker/**"] } - group: integration label: up-docker-compose + pr_smoke: true + paths: ["e2e/tests/up-docker-compose/**", "pkg/compose/**", "pkg/driver/docker/**", "pkg/secrets/**"] # Runs first, as the existing non-root runner user, before the sudo run. focused_checks: - name: compose-uid-nonroot @@ -236,33 +294,34 @@ e2e: expect_leaf: implements updateRemoteUserUID with root container user require_uid: 1001 require_gid: 1001 - - { group: integration, label: up-features, install_kind: true } + - { group: integration, label: up-features, install_kind: true, pr_smoke: true, paths: ["e2e/tests/up-features/**", "pkg/devcontainer/feature/**", "pkg/devcontainer/build/**"] } # --- integration: Podman on Linux --- - - { group: integration, label: up-provider-podman-rootless-basic, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20 } - - { group: integration, label: up-provider-podman-rootless-exec, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20 } - - { group: integration, label: up-provider-podman-rootless-lifecycle, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20 } - - { group: integration, label: up-provider-podman-rootless-config, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20 } - - { group: integration, label: up-provider-podman-rootless-features, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20 } - - { group: integration, label: up-provider-podman-rootful-basic, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20 } - - { group: integration, label: up-provider-podman-rootful-lifecycle, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20 } - - { group: integration, label: up-provider-podman-rootful-lifecycle-2, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20 } - - { group: integration, label: up-provider-podman-rootful-config, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20 } - - { group: integration, label: up-provider-podman-rootful-features, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20 } + - { group: integration, label: up-provider-podman-rootless-basic, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20, pr_smoke: true, paths: ["e2e/tests/up/*podman*", "pkg/driver/docker/**", "pkg/docker/**", "providers/docker/**", "hack/podman_ci/**"] } + - { group: integration, label: up-provider-podman-rootless-exec, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20, paths: ["e2e/tests/up/*podman*", "pkg/driver/docker/**", "pkg/docker/**", "providers/docker/**", "hack/podman_ci/**"] } + - { group: integration, label: up-provider-podman-rootless-lifecycle, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20, paths: ["e2e/tests/up/*podman*", "pkg/driver/docker/**", "pkg/docker/**", "providers/docker/**", "hack/podman_ci/**"] } + - { group: integration, label: up-provider-podman-rootless-config, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20, paths: ["e2e/tests/up/*podman*", "pkg/driver/docker/**", "pkg/docker/**", "providers/docker/**", "hack/podman_ci/**"] } + - { group: integration, label: up-provider-podman-rootless-features, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20, paths: ["e2e/tests/up/*podman*", "pkg/driver/docker/**", "pkg/docker/**", "providers/docker/**", "hack/podman_ci/**"] } + - { group: integration, label: up-provider-podman-rootful-basic, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20, pr_smoke: true, paths: ["e2e/tests/up/*podman*", "pkg/driver/docker/**", "pkg/docker/**", "providers/docker/**", "hack/podman_ci/**"] } + - { group: integration, label: up-provider-podman-rootful-lifecycle, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20, paths: ["e2e/tests/up/*podman*", "pkg/driver/docker/**", "pkg/docker/**", "providers/docker/**", "hack/podman_ci/**"] } + - { group: integration, label: up-provider-podman-rootful-lifecycle-2, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20, paths: ["e2e/tests/up/*podman*", "pkg/driver/docker/**", "pkg/docker/**", "providers/docker/**", "hack/podman_ci/**"] } + - { group: integration, label: up-provider-podman-rootful-config, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20, paths: ["e2e/tests/up/*podman*", "pkg/driver/docker/**", "pkg/docker/**", "providers/docker/**", "hack/podman_ci/**"] } + - { group: integration, label: up-provider-podman-rootful-features, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20, paths: ["e2e/tests/up/*podman*", "pkg/driver/docker/**", "pkg/docker/**", "providers/docker/**", "hack/podman_ci/**"] } # --- integration: Windows (Docker API served by Podman) --- - - { group: integration, label: provider, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20 } - - { group: integration, label: up-docker-wsl, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20 } - - { group: integration, label: up-features, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20 } - - { group: integration, label: ssh-proxy-command, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20 } - - { group: integration, label: ssh-tunnel-mode, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20 } - - { group: integration, label: task-cancel, runner: windows-latest, install_podman: rootful, test_timeout: 900s, job_timeout_minutes: 20 } + - { group: integration, label: provider, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20, paths: ["e2e/tests/provider/**", "cmd/provider/**", "pkg/provider/**", "pkg/options/**", "providers/**", "e2e/framework/dockerwsl/**", "hack/podman_ci/**"] } + - { group: integration, label: up-docker-wsl, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20, pr_smoke: true, paths: ["e2e/tests/up/docker_wsl.go", "pkg/driver/docker/**", "pkg/docker/**", "providers/docker/**", "e2e/framework/dockerwsl/**", "hack/podman_ci/**"] } + - { group: integration, label: up-features, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20, paths: ["e2e/tests/up-features/**", "pkg/devcontainer/feature/**", "pkg/devcontainer/build/**"] } + - { group: integration, label: ssh-proxy-command, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20, paths: ["e2e/tests/ssh/**", "cmd/workspace/ssh.go", "cmd/internal/ssh_*.go", "pkg/ssh/**", "e2e/framework/dockerwsl/**", "hack/podman_ci/**"] } + - { group: integration, label: ssh-tunnel-mode, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20, paths: ["e2e/tests/ssh/**", "cmd/workspace/up/**", "cmd/workspace/task*.go", "pkg/ssh/**", "pkg/task/**", "pkg/command/**", "e2e/framework/dockerwsl/**", "hack/podman_ci/**"] } + - { group: integration, label: task-cancel, runner: windows-latest, install_podman: rootful, test_timeout: 900s, job_timeout_minutes: 20, paths: ["e2e/tests/ssh/**", "cmd/workspace/task*.go", "cmd/workspace/stop.go", "pkg/task/**", "pkg/command/**"] } # --- integration: Windows with a Docker Engine in WSL2, reached over SSH --- # The workflow provisions the daemon with `devsy-ci docker-wsl setup`; the # report must show exactly expected_spec_count specs passing first try. - group: integration label: docker-internal-buildkit + paths: ["e2e/tests/dockerbuildkit/**", "e2e/framework/dockerwsl/**", "pkg/devcontainer/buildkit/**", "pkg/driver/docker/**", "pkg/docker/**"] runner: windows-latest runtime: docker-wsl expected_spec_count: 6 @@ -272,6 +331,7 @@ e2e: # selected WSL user's display, and the job also runs the Compose parity probe. - group: integration label: wslg + paths: ["e2e/tests/dockerbuildkit/wslg.go", "e2e/framework/dockerwsl/**", "pkg/wslg/**", "pkg/compose/**"] runner: windows-latest runtime: docker-wsl expected_spec_count: 2 @@ -279,19 +339,19 @@ e2e: job_timeout_minutes: 40 # --- microsandbox --- - - { group: microsandbox, label: up-provider-microsandbox, expected_spec_count: 2, performance_capture: true } - - { group: microsandbox, label: up-provider-microsandbox-images, expected_spec_count: 6 } - - { group: microsandbox, label: up-provider-microsandbox-mounts, expected_spec_count: 10 } - - { group: microsandbox, label: up-provider-microsandbox-resources, expected_spec_count: 12 } - - { group: microsandbox, label: up-provider-microsandbox-storage, expected_spec_count: 6 } - - { group: microsandbox, label: up-provider-microsandbox-ephemeral, expected_spec_count: 6 } - - { group: microsandbox, label: up-provider-microsandbox-egress, expected_spec_count: 4 } - - { group: microsandbox, label: up-provider-microsandbox-prebuilds, expected_spec_count: 4 } - - { group: microsandbox, label: up-provider-microsandbox-dockerless, expected_spec_count: 8 } - - { group: microsandbox, label: up-provider-microsandbox-recreate, expected_spec_count: 4 } - - { group: microsandbox, label: up-provider-microsandbox-managed-recreate, expected_spec_count: 4 } - - { group: microsandbox, label: up-provider-microsandbox-logs-cancel, expected_spec_count: 6 } - - { group: microsandbox, label: up-provider-microsandbox-compatibility, expected_spec_count: 2 } - - { group: microsandbox, label: up-provider-microsandbox-registry-consumer, expected_spec_count: 3, provider_mode: external } - - { group: microsandbox, label: up-provider-microsandbox-remote-agent-binaries, expected_spec_count: 1, provider_mode: external } - - { group: microsandbox, label: up-provider-microsandbox-source-migration, expected_spec_count: 2, provider_mode: external } + - { group: microsandbox, label: up-provider-microsandbox, expected_spec_count: 2, performance_capture: true, pr_smoke: true, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-images, expected_spec_count: 6, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-mounts, expected_spec_count: 10, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-resources, expected_spec_count: 12, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-storage, expected_spec_count: 6, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-ephemeral, expected_spec_count: 6, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-egress, expected_spec_count: 4, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-prebuilds, expected_spec_count: 4, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-dockerless, expected_spec_count: 8, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-recreate, expected_spec_count: 4, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-managed-recreate, expected_spec_count: 4, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-logs-cancel, expected_spec_count: 6, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-compatibility, expected_spec_count: 2, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-registry-consumer, expected_spec_count: 3, provider_mode: external, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-remote-agent-binaries, expected_spec_count: 1, provider_mode: external, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } + - { group: microsandbox, label: up-provider-microsandbox-source-migration, expected_spec_count: 2, provider_mode: external, paths: ["e2e/tests/up/*microsandbox*", "e2e/tests/up/managed_policy_fixture_linux.go", "pkg/driver/microsandbox/**", "pkg/driver/external/**", "pkg/driver/drivercreate/microsandbox_images.go", "providers/microsandbox/**", "hack/microsandbox/**", "cmd/internal/runtime_supervisor.go", "cmd/internal/managedvalue/**"] } From 3fdaab8ecefe223463d166e87036eee0221976d3 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 9 Nov 2026 22:57:21 -0600 Subject: [PATCH 2/7] feat(ci): add change-aware matrix and build planning to hack/ci --- hack/ci/matrix.go | 159 +++++++++++++++++++++++++++++++++-------- hack/ci/matrix_test.go | 156 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 277 insertions(+), 38 deletions(-) diff --git a/hack/ci/matrix.go b/hack/ci/matrix.go index 22eb566a49..0a35f6ce54 100644 --- a/hack/ci/matrix.go +++ b/hack/ci/matrix.go @@ -7,21 +7,32 @@ import ( "flag" "fmt" "io" + "os" "slices" "strings" ) -const matrixUnitGroup = "unit" +const ( + matrixUnitGroup = "unit" + matrixBuildGroup = "build" + matrixUbuntuRunner = "ubuntu-latest" + matrixMacOSRunner = "macos-latest" + matrixWindowsRunner = "windows-latest" +) // matrixGroupNames lists the job groups in the order `ci matrix` prints them. var matrixGroupNames = []string{ matrixUnitGroup, + matrixBuildGroup, groupUnprivileged, groupIntegration, groupMicrosandbox, } -var errUnknownMatrixGroup = errors.New("unknown matrix group") +var ( + errUnknownMatrixGroup = errors.New("unknown matrix group") + errUnknownMatrixMode = errors.New("unknown matrix mode") +) // matrixDoc is the GitHub Actions matrix object. type matrixDoc struct { @@ -34,6 +45,12 @@ type matrixUnitEntry struct { OS string `json:"os"` } +type matrixBuildEntry struct { + Runner string `json:"runner"` + JobTimeoutMinutes int `json:"job_timeout_minutes,omitempty"` + BuildMicrosandbox bool `json:"build_microsandbox"` +} + // matrixEntry is one e2e suite as a job of the workflow matrix. type matrixEntry struct { Name string `json:"name"` @@ -63,19 +80,47 @@ type matrixEntry struct { // matrixCLI prints the job matrices for the PR workflow. func matrixCLI(args []string, stdout, stderr io.Writer) int { + group, opts, code := parseMatrixOptions(args, stderr) + if code >= 0 { + return code + } + m, err := LoadManifest() + if err != nil { + _, _ = fmt.Fprintf(stderr, "ci matrix: %v\n", err) + return 1 + } + out, err := matrixOutput(m, group, opts) + if err != nil { + _, _ = fmt.Fprintf(stderr, "ci matrix: %v\n", err) + return matrixFailureCode(err) + } + if _, err := io.WriteString(stdout, out); err != nil { + _, _ = fmt.Fprintf(stderr, "ci matrix: write output: %v\n", err) + return 1 + } + return 0 +} + +func parseMatrixOptions(args []string, stderr io.Writer) (string, selectionOptions, int) { fs := flag.NewFlagSet("matrix", flag.ContinueOnError) fs.SetOutput(stderr) group := fs.String( "group", "", - "print only this group: unit, unprivileged, integration or microsandbox", + "print only this group: unit, build, unprivileged, integration or microsandbox", ) secretAvailable := fs.Bool("secret-available", false, "include suites that require a secret") + mode := fs.String("mode", selectionModeFull, "suite selection mode: full or pr") + changedFilesPath := fs.String( + "changed-files", + "", + "file containing newline-separated changed paths", + ) if err := fs.Parse(args); err != nil { if errors.Is(err, flag.ErrHelp) { - return 0 + return "", selectionOptions{}, 0 } - return 2 + return "", selectionOptions{}, 2 } if fs.NArg() > 0 { _, _ = fmt.Fprintf( @@ -83,23 +128,40 @@ func matrixCLI(args []string, stdout, stderr io.Writer) int { "ci matrix: unexpected arguments: %s\n", strings.Join(fs.Args(), " "), ) - return 2 + return "", selectionOptions{}, 2 } - m, err := LoadManifest() + if *mode != selectionModeFull && *mode != selectionModePR { + _, _ = fmt.Fprintf(stderr, "ci matrix: %v: %q\n", errUnknownMatrixMode, *mode) + return "", selectionOptions{}, 2 + } + changedFiles, err := readMatrixChangedFiles(*changedFilesPath) if err != nil { - _, _ = fmt.Fprintf(stderr, "ci matrix: %v\n", err) - return 1 + _, _ = fmt.Fprintf(stderr, "ci matrix: read changed files: %v\n", err) + return "", selectionOptions{}, 1 } - out, err := matrixOutput(m, *group, *secretAvailable) + return *group, selectionOptions{ + secretAvailable: *secretAvailable, + mode: *mode, + changedFiles: changedFiles, + }, -1 +} + +func readMatrixChangedFiles(path string) ([]string, error) { + if path == "" { + return nil, nil + } + data, err := os.ReadFile(path) if err != nil { - _, _ = fmt.Fprintf(stderr, "ci matrix: %v\n", err) - return matrixFailureCode(err) + return nil, err } - if _, err := io.WriteString(stdout, out); err != nil { - _, _ = fmt.Fprintf(stderr, "ci matrix: write output: %v\n", err) - return 1 + var changedFiles []string + for line := range strings.SplitSeq(string(data), "\n") { + line = strings.TrimSuffix(line, "\r") + if line != "" { + changedFiles = append(changedFiles, line) + } } - return 0 + return changedFiles, nil } // matrixFailureCode maps a matrixOutput error to an exit code. @@ -112,9 +174,9 @@ func matrixFailureCode(err error) int { // matrixOutput renders the --group JSON line, or without a group one // `name=` line per group, each terminated by a newline. -func matrixOutput(m *Manifest, group string, secretAvailable bool) (string, error) { +func matrixOutput(m *Manifest, group string, opts selectionOptions) (string, error) { if group != "" { - line, err := matrixGroupJSON(m, group, secretAvailable) + line, err := matrixGroupJSON(m, group, opts) if err != nil { return "", err } @@ -122,7 +184,7 @@ func matrixOutput(m *Manifest, group string, secretAvailable bool) (string, erro } lines := make([]string, 0, len(matrixGroupNames)) for _, name := range matrixGroupNames { - line, err := matrixGroupJSON(m, name, secretAvailable) + line, err := matrixGroupJSON(m, name, opts) if err != nil { return "", err } @@ -132,8 +194,8 @@ func matrixOutput(m *Manifest, group string, secretAvailable bool) (string, erro } // matrixGroupJSON returns the compact single-line matrix object of a group. -func matrixGroupJSON(m *Manifest, group string, secretAvailable bool) (string, error) { - include, err := matrixInclude(m, group, secretAvailable) +func matrixGroupJSON(m *Manifest, group string, opts selectionOptions) (string, error) { + include, err := matrixInclude(m, group, opts) if err != nil { return "", err } @@ -148,12 +210,14 @@ func matrixGroupJSON(m *Manifest, group string, secretAvailable bool) (string, e // matrixInclude returns the entries of one group. Entry slices are never nil // so an empty group encodes as [] rather than null. -func matrixInclude(m *Manifest, group string, secretAvailable bool) (any, error) { +func matrixInclude(m *Manifest, group string, opts selectionOptions) (any, error) { switch group { case matrixUnitGroup: return unitMatrixEntries(m), nil + case matrixBuildGroup: + return buildMatrixEntries(m, opts), nil case groupUnprivileged, groupIntegration, groupMicrosandbox: - return e2eMatrixEntries(m, group, secretAvailable), nil + return e2eMatrixEntries(m, group, opts), nil default: return nil, fmt.Errorf("%w: %q", errUnknownMatrixGroup, group) } @@ -167,18 +231,57 @@ func unitMatrixEntries(m *Manifest) []matrixUnitEntry { return entries } -func e2eMatrixEntries(m *Manifest, group string, secretAvailable bool) []matrixEntry { - suites := m.SuitesIn(group) +func e2eMatrixEntries(m *Manifest, group string, opts selectionOptions) []matrixEntry { + suites := selectSuites(m, group, opts) entries := make([]matrixEntry, 0, len(suites)) for _, s := range suites { - if s.RequiresSecret && !secretAvailable { - continue - } entries = append(entries, newMatrixEntry(s)) } return entries } +func buildMatrixEntries(m *Manifest, opts selectionOptions) []matrixBuildEntry { + if opts.mode == "" || opts.mode == selectionModeFull { + return []matrixBuildEntry{ + {Runner: matrixUbuntuRunner, JobTimeoutMinutes: 45, BuildMicrosandbox: true}, + {Runner: matrixMacOSRunner, JobTimeoutMinutes: 20}, + {Runner: matrixWindowsRunner, JobTimeoutMinutes: 45}, + } + } + if matchesAny(opts.changedFiles, m.Selection.FullRunPaths) || + slices.Contains(opts.changedFiles, ".goreleaser.yml") { + return []matrixBuildEntry{ + {Runner: matrixUbuntuRunner, JobTimeoutMinutes: 45, BuildMicrosandbox: true}, + {Runner: matrixMacOSRunner, JobTimeoutMinutes: 20, BuildMicrosandbox: true}, + {Runner: matrixWindowsRunner, JobTimeoutMinutes: 45, BuildMicrosandbox: true}, + } + } + unprivileged := selectSuites(m, groupUnprivileged, opts) + integration := selectSuites(m, groupIntegration, opts) + microsandbox := selectSuites(m, groupMicrosandbox, opts) + entries := make([]matrixBuildEntry, 0, 2) + if len(unprivileged)+len(integration)+len(microsandbox) > 0 { + entries = append( + entries, + matrixBuildEntry{ + Runner: matrixUbuntuRunner, + JobTimeoutMinutes: 45, + BuildMicrosandbox: len(microsandbox) > 0, + }, + ) + } + for _, s := range integration { + if s.OS() == osWindows { + entries = append( + entries, + matrixBuildEntry{Runner: matrixWindowsRunner, JobTimeoutMinutes: 45}, + ) + break + } + } + return entries +} + // newMatrixEntry maps a resolved suite to its matrix job, including the setup // steps the workflow must run for it. func newMatrixEntry(s Suite) matrixEntry { diff --git a/hack/ci/matrix_test.go b/hack/ci/matrix_test.go index 26431b4de8..dd7886298b 100644 --- a/hack/ci/matrix_test.go +++ b/hack/ci/matrix_test.go @@ -4,6 +4,8 @@ import ( "bytes" "encoding/json" "fmt" + "os" + "path/filepath" "strings" "testing" @@ -45,9 +47,7 @@ e2e: const ( matrixProviderLabel = "provider" - matrixUbuntuRunner = "ubuntu-latest" matrixPodmanRunner = "ubuntu-24.04" - matrixWindowsRunner = "windows-latest" matrixWindowsTimeout = "900s" ) @@ -218,9 +218,17 @@ func (s *matrixSuite) TestPlainSuiteHasNoRuntime() { func (s *matrixSuite) TestSecretSuitesOmittedUnlessAvailable() { m := s.testManifest() - withoutSecret, err := matrixOutput(m, groupIntegration, false) + withoutSecret, err := matrixOutput( + m, + groupIntegration, + selectionOptions{mode: selectionModeFull}, + ) s.Require().NoError(err) - withSecret, err := matrixOutput(m, groupIntegration, true) + withSecret, err := matrixOutput( + m, + groupIntegration, + selectionOptions{secretAvailable: true, mode: selectionModeFull}, + ) s.Require().NoError(err) names := func(line string) []string { @@ -249,21 +257,25 @@ func (s *matrixSuite) TestGroupFlagPrintsOnlyThatGroup() { s.Equal("context", entries[0].Name) } -func (s *matrixSuite) TestNoGroupPrintsFourLinesInOrder() { +func (s *matrixSuite) TestNoGroupPrintsFiveLinesInOrder() { code, stdout, stderr := s.runCLI() s.Equal(0, code, stderr) s.Empty(stderr) lines := strings.Split(strings.TrimSuffix(stdout, "\n"), "\n") - s.Require().Len(lines, 4) - for i, name := range []string{"unit", groupUnprivileged, groupIntegration, groupMicrosandbox} { + s.Require().Len(lines, 5) + for i, name := range []string{"unit", matrixBuildGroup, groupUnprivileged, groupIntegration, groupMicrosandbox} { s.True(strings.HasPrefix(lines[i], name+`={"include":[`), lines[i]) s.True(json.Valid([]byte(strings.TrimPrefix(lines[i], name+"="))), lines[i]) } } func (s *matrixSuite) TestUnitEntriesCarryOS() { - out, err := matrixOutput(s.testManifest(), matrixUnitGroup, false) + out, err := matrixOutput( + s.testManifest(), + matrixUnitGroup, + selectionOptions{mode: selectionModeFull}, + ) s.Require().NoError(err) s.Equal(`{"include":[`+ @@ -273,7 +285,11 @@ func (s *matrixSuite) TestUnitEntriesCarryOS() { func (s *matrixSuite) TestCompactJSONDoesNotEscapeHTML() { m := s.testManifest() - line, err := matrixGroupJSON(m, groupIntegration, true) + line, err := matrixGroupJSON( + m, + groupIntegration, + selectionOptions{secretAvailable: true, mode: selectionModeFull}, + ) s.Require().NoError(err) s.Contains(line, `"name":"a Date: Mon, 9 Nov 2026 22:57:21 -0600 Subject: [PATCH 4/7] ci: wire change-aware test selection and decoupled builds in pr-ci.yml --- .github/workflows/pr-ci.yml | 127 ++++++++++++++++++++++++++---------- hack/ci/README.md | 30 ++++++++- 2 files changed, 121 insertions(+), 36 deletions(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index f7ee35a25c..0e2fe23e76 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -26,35 +26,94 @@ jobs: contents: read pull-requests: read outputs: - go: ${{ steps.filter.outputs.go }} + go: ${{ github.event_name != 'pull_request' || steps.filter.outputs.full == 'true' || steps.filter.outputs.go == 'true' }} + agent_skill: ${{ github.event_name != 'pull_request' || steps.filter.outputs.full == 'true' || steps.filter.outputs.agent_skill == 'true' }} + licenses: ${{ github.event_name != 'pull_request' || steps.filter.outputs.full == 'true' || steps.filter.outputs.licenses == 'true' }} + gh_credentials: ${{ github.event_name != 'pull_request' || steps.filter.outputs.full == 'true' || steps.filter.outputs.gh_credentials == 'true' }} + provider_update: ${{ github.event_name != 'pull_request' || steps.filter.outputs.full == 'true' || steps.filter.outputs.provider_update == 'true' }} + mode: ${{ github.event_name == 'pull_request' && 'pr' || 'full' }} + changed_files: ${{ steps.filter.outputs.all_files || '[]' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4 id: filter + if: github.event_name == 'pull_request' with: + list-files: json filters: | - go: - - '**/*.go' - - 'pkg/inject/**' + full: + - '.github/workflows/pr-ci.yml' + - '.github/actions/**' + - 'hack/ci/**' - 'go.mod' - 'go.sum' - '.goreleaser.yml' + - 'e2e/framework/**' + - 'e2e/*.go' + go: + - '**/*.go' + - 'cmd/**' + - 'pkg/**' + - 'providers/**' - 'e2e/**' - - 'hack/ci/**' - 'hack/microsandbox/**' - 'hack/provider_update_ci/**' - - '.github/workflows/pr-ci.yml' - 'hack/licenses/**' - 'hack/podman_ci/**' + - '.golangci.yaml' + agent_skill: + - '.agents/skills/devsy/**' + - 'hack/validate_agent_skill/**' + licenses: + - '**/*.go' + - 'hack/licenses/**' - 'THIRD_PARTY_LICENSES.md' + gh_credentials: + - 'pkg/ghcredentials/**' + - 'pkg/credentials/**' + - 'pkg/tunnel/**' + - 'pkg/agent/tunnel/**' + - 'pkg/agent/tunnelserver/**' + - 'pkg/config/**' + - 'pkg/log/**' + - 'pkg/http/**' + - 'pkg/port/**' + - 'pkg/random/**' + - 'pkg/gitsshsigning/**' + - 'pkg/transport/**' + - 'pkg/client/**' + - 'pkg/ssh/**' + - 'pkg/flags/**' + - 'pkg/exitcode/**' + - 'e2e/cmd/ghfixture/**' + provider_update: + - 'pkg/workspace/**' + - 'pkg/provider/**' + - 'pkg/config/**' + - 'pkg/extract/**' + - 'pkg/copy/**' + - 'pkg/download/**' + - 'pkg/http/**' + - 'pkg/gitcredentials/**' + - 'pkg/hash/**' + - 'pkg/log/**' + - 'pkg/id/**' + - 'pkg/types/**' + - 'providers/**' + - 'hack/provider_update_ci/**' + all: + - '**' plan: name: Plan + needs: [changes] + if: always() runs-on: ubuntu-latest permissions: contents: read outputs: unit: ${{ steps.matrix.outputs.unit }} + build: ${{ steps.matrix.outputs.build }} unprivileged: ${{ steps.matrix.outputs.unprivileged }} integration: ${{ steps.matrix.outputs.integration }} microsandbox: ${{ steps.matrix.outputs.microsandbox }} @@ -71,7 +130,11 @@ jobs: id: matrix env: SECRET_AVAILABLE: ${{ secrets.DEVSY_GITHUB_APP_ID != '' }} - run: go run ./hack/ci matrix --secret-available="$SECRET_AVAILABLE" >> "$GITHUB_OUTPUT" + MATRIX_MODE: ${{ needs.changes.outputs.mode }} + CHANGED_FILES_JSON: ${{ needs.changes.outputs.changed_files }} + run: | + printf '%s' "$CHANGED_FILES_JSON" | jq -r '.[]' > "$RUNNER_TEMP/changed-files.txt" + go run ./hack/ci matrix --mode="$MATRIX_MODE" --changed-files="$RUNNER_TEMP/changed-files.txt" --secret-available="$SECRET_AVAILABLE" >> "$GITHUB_OUTPUT" precommit: name: Pre-commit @@ -81,10 +144,12 @@ jobs: - uses: jdx/mise-action@b44d12e7d0d8ee7461bb893e1b1993500433185b # v5.2.7 - - run: prek run --all-files + - run: SKIP=golangci-lint prek run --all-files agent-skill: name: Agent Skill + needs: [changes] + if: needs.changes.outputs.agent_skill == 'true' runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -106,6 +171,8 @@ jobs: lint: name: Lint + needs: [changes] + if: needs.changes.outputs.go == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -128,7 +195,7 @@ jobs: github-credential-host-tests: name: GitHub Credential Host Tests (${{ matrix.runner }}) needs: [changes] - if: needs.changes.outputs.go == 'true' + if: needs.changes.outputs.gh_credentials == 'true' strategy: fail-fast: false matrix: @@ -193,7 +260,7 @@ jobs: provider-update-windows: name: Provider Update Tests (windows-latest) needs: [changes] - if: needs.changes.outputs.go == 'true' + if: needs.changes.outputs.provider_update == 'true' runs-on: windows-latest # Allow cold Go compilation and the post-job cache save to complete. timeout-minutes: 30 @@ -221,17 +288,11 @@ jobs: build-cli: name: Build CLI Binary on ${{ matrix.runner }} - needs: [changes, precommit, lint] - if: needs.changes.outputs.go == 'true' + needs: [changes, plan] + if: needs.changes.outputs.go == 'true' && needs.plan.outputs.build != '{"include":[]}' strategy: - matrix: - include: - - runner: ubuntu-latest - job_timeout_minutes: 45 - - runner: macos-latest - - runner: windows-latest - # Cold builds need room to save the Go cache after artifact uploads. - job_timeout_minutes: 45 + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.build) }} runs-on: ${{ matrix.runner }} timeout-minutes: ${{ matrix.job_timeout_minutes || 20 }} steps: @@ -318,24 +379,24 @@ jobs: path: dist/devsy-${{ steps.os.outputs.runner_os }}_*/devsy-${{ steps.os.outputs.runner_os }}-* - name: build MicroSandbox CI helper - if: runner.os == 'Linux' + if: runner.os == 'Linux' && matrix.build_microsandbox env: CGO_ENABLED: 0 run: go build -trimpath -o "${RUNNER_TEMP}/devsy-ci-microsandbox" ./hack/microsandbox - name: build pinned runtime performance tools - if: runner.os == 'Linux' + if: runner.os == 'Linux' && matrix.build_microsandbox run: '"${RUNNER_TEMP}/devsy-ci-microsandbox" build-performance-tools --output-dir "${RUNNER_TEMP}/runtime-performance-tools"' - name: upload runtime performance tools - if: runner.os == 'Linux' + if: runner.os == 'Linux' && matrix.build_microsandbox uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: runtime-performance-tools-linux path: ${{ runner.temp }}/runtime-performance-tools/ - name: upload MicroSandbox CI helper - if: runner.os == 'Linux' + if: runner.os == 'Linux' && matrix.build_microsandbox uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: devsy-ci-microsandbox-linux @@ -380,8 +441,8 @@ jobs: licenses: name: Third-party licenses - needs: [changes, precommit, lint] - if: needs.changes.outputs.go == 'true' + needs: [changes] + if: needs.changes.outputs.licenses == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -397,7 +458,7 @@ jobs: integration-tests-unprivileged: name: ${{ matrix.title }} needs: [changes, plan, build-cli] - if: needs.changes.outputs.go == 'true' + if: needs.changes.outputs.go == 'true' && needs.plan.outputs.unprivileged != '{"include":[]}' strategy: fail-fast: false matrix: ${{ fromJSON(needs.plan.outputs.unprivileged) }} @@ -446,8 +507,8 @@ jobs: prepare-microsandbox-provider: name: Prepare MicroSandbox provider assets - needs: [changes, build-cli] - if: needs.changes.outputs.go == 'true' + needs: [changes, plan, build-cli] + if: needs.changes.outputs.go == 'true' && needs.plan.outputs.microsandbox != '{"include":[]}' runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -496,7 +557,7 @@ jobs: integration-tests-microsandbox: name: ${{ matrix.title }} needs: [changes, plan, build-cli, prepare-microsandbox-provider] - if: needs.changes.outputs.go == 'true' + if: needs.changes.outputs.go == 'true' && needs.plan.outputs.microsandbox != '{"include":[]}' strategy: fail-fast: false matrix: ${{ fromJSON(needs.plan.outputs.microsandbox) }} @@ -621,7 +682,7 @@ jobs: integration-tests: name: ${{ matrix.title }} needs: [changes, plan, build-cli] - if: needs.changes.outputs.go == 'true' + if: needs.changes.outputs.go == 'true' && needs.plan.outputs.integration != '{"include":[]}' strategy: fail-fast: false matrix: ${{ fromJSON(needs.plan.outputs.integration) }} @@ -933,9 +994,7 @@ jobs: with: go-version-file: go.mod - # Jobs gated on Go changes may be skipped only when no Go-relevant file - # changed; the jobs listed in --always-run must never be skipped. - name: Check required jobs env: NEEDS_JSON: ${{ toJSON(needs) }} - run: go run ./hack/ci success --always-run changes,plan,precommit,lint,agent-skill + run: go run ./hack/ci success --always-run changes,plan,precommit diff --git a/hack/ci/README.md b/hack/ci/README.md index a60e248e67..5255a1b221 100644 --- a/hack/ci/README.md +++ b/hack/ci/README.md @@ -19,6 +19,30 @@ and subcommands here decide which tests exist, where they run, and how. runs it on each runner in `unit.runners`. The workflow builds one job matrix per e2e group from `e2e.suites`. +## Change selection + +Pushes to main and manual workflow runs use `--mode=full`. They run every +suite whose required secret is available and build Linux, macOS, and Windows +binaries. + +Pull requests use `--mode=pr --changed-files FILE`. The file contains one +repository-relative path per line. Paths in `selection.full_run_paths` select +every suite. Paths in `selection.core_paths` select the 13 suites marked +`pr_smoke` plus suites with a matching `paths` entry. Other changes select +only suites with a matching entry. Patterns support `*` within one path +segment and `**` across zero or more segments. + +The build matrix follows selected suites. Every E2E run needs the Linux +binary, including Windows runs that inject it into containers. Windows +suites also need a Windows build. MicroSandbox suites enable the provider +build on Linux. Full-run paths retain all three build runners. Unmatched +changes produce empty E2E and build matrices. Unit test runners are unchanged. + +`ci success` accepts conditional skips only when successful change detection +and planning provide valid outputs that explain them. Missing or failed +planning, invalid outputs, and unexpectedly skipped required jobs fail the +gate. + ## Adding tests - **Go unit test in an ordinary package**: no change. The `packages` step finds @@ -31,7 +55,9 @@ e2e group from `e2e.suites`. - **Ginkgo spec**: add its label to `e2e.suites` and choose a `group`. Set only the fields that differ from the group defaults, such as `runner`, `test_timeout`, `install_kind`, `install_podman`, `requires_secret`, and - `host_setup`. List labels that PR CI must not select directly in + `host_setup`. Add `paths` for the suite's test files and product dependencies. + Set `pr_smoke` only for the shared core smoke tier. List labels that PR CI + must not select directly in `e2e.ungated` with a reason. These include sub-labels exercised by gated suites and labels run manually or by another workflow. @@ -43,7 +69,7 @@ workflow edits. | Command | Purpose | | --- | --- | -| `ci matrix [--group G] [--secret-available]` | Print the job matrices as GitHub outputs. Drops suites that need the GitHub App secret when it is unavailable. | +| `ci matrix [--group G] [--secret-available] [--mode full\|pr] [--changed-files FILE]` | Print unit, build, and E2E job matrices as GitHub outputs. Apply change selection and secret availability. | | `ci unit [--os GOOS] [--step NAME] [--list] [--coverprofile P]` | Run the unit tier on this machine. | | `ci e2e prepare --bin-src DIR --e2e-dir DIR` | Put the CLI binaries next to `e2e.test`. | | `ci e2e run --label L --runner R` | Run a suite, with its focused checks, sudo and environment setup, and post checks. | From a6add0727acb576c857df82a11852ffbbbf067e0 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 10 Nov 2026 11:49:40 -0600 Subject: [PATCH 5/7] fix(ci): select Docker installation tests for installer changes --- hack/ci/README.md | 3 +++ hack/ci/select_test.go | 16 ++++++++++++++++ hack/ci/suites.yaml | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/hack/ci/README.md b/hack/ci/README.md index 5255a1b221..3166a8f748 100644 --- a/hack/ci/README.md +++ b/hack/ci/README.md @@ -61,6 +61,9 @@ gate. `e2e.ungated` with a reason. These include sub-labels exercised by gated suites and labels run manually or by another workflow. +Keep suite paths aligned with implementation packages when code moves. +Include runtime helpers such as `pkg/dockerinstall/**`, not only provider wrappers. + `go test ./hack/ci` fails when a Ginkgo label or Go test package has no tier, or when the manifest names a label that no longer exists. New tests need no workflow edits. diff --git a/hack/ci/select_test.go b/hack/ci/select_test.go index 3603ac813f..146869ff7d 100644 --- a/hack/ci/select_test.go +++ b/hack/ci/select_test.go @@ -163,6 +163,22 @@ func (s *selectionSuite) TestEmbeddedLeafAndDocumentationSelection() { } } +func (s *selectionSuite) TestDockerInstallerChangesSelectInstallationSuite() { + m, err := LoadManifest() + s.Require().NoError(err) + opts := selectionOptions{ + mode: selectionModePR, + changedFiles: []string{"pkg/dockerinstall/install.go"}, + } + for _, group := range []string{groupUnprivileged, groupIntegration, groupMicrosandbox} { + want := []string{} + if group == groupIntegration { + want = []string{"docker-install"} + } + s.Equal(want, selectionLabels(selectSuites(m, group, opts)), group) + } +} + func (s *selectionSuite) TestEmbeddedCoreSmokeTier() { m, err := LoadManifest() s.Require().NoError(err) diff --git a/hack/ci/suites.yaml b/hack/ci/suites.yaml index 8faea4a2f0..602e71f6ea 100644 --- a/hack/ci/suites.yaml +++ b/hack/ci/suites.yaml @@ -259,7 +259,7 @@ e2e: - { group: integration, label: build, install_kind: true, requires_secret: true, pr_smoke: true, paths: ["e2e/tests/build/**", "cmd/workspace/build.go", "pkg/devcontainer/build/**", "pkg/devcontainer/feature/**", "pkg/dockerfile/**", "pkg/devcontainer/metadata/**", "pkg/devcontainer/buildkit/**", "pkg/driver/docker/**"] } - group: integration label: docker-install - paths: ["e2e/tests/dockerinstall/**", "pkg/docker/**", "pkg/driver/docker/**", "providers/docker/**"] + paths: ["e2e/tests/dockerinstall/**", "pkg/dockerinstall/**", "pkg/docker/**", "pkg/driver/docker/**", "providers/docker/**"] host_setup: [remove-docker] post_checks: [docker-installed] - { group: integration, label: exec, paths: ["e2e/tests/exec/**", "cmd/workspace/exec.go", "pkg/driver/docker/**", "pkg/ssh/**"] } From 5be3474f19b5fc0d8646c76e532571e89f4b5c8d Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 10 Nov 2026 11:58:27 -0600 Subject: [PATCH 6/7] fix(ci): include netstat changes in native credential checks --- .github/workflows/pr-ci.yml | 1 + hack/ci/README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 0e2fe23e76..e21132cf8e 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -78,6 +78,7 @@ jobs: - 'pkg/log/**' - 'pkg/http/**' - 'pkg/port/**' + - 'pkg/netstat/**' - 'pkg/random/**' - 'pkg/gitsshsigning/**' - 'pkg/transport/**' diff --git a/hack/ci/README.md b/hack/ci/README.md index 3166a8f748..2ff32574ee 100644 --- a/hack/ci/README.md +++ b/hack/ci/README.md @@ -63,6 +63,7 @@ gate. Keep suite paths aligned with implementation packages when code moves. Include runtime helpers such as `pkg/dockerinstall/**`, not only provider wrappers. +Include direct compile dependencies in native host job filters. `go test ./hack/ci` fails when a Ginkgo label or Go test package has no tier, or when the manifest names a label that no longer exists. New tests need no From 9c2d2c4e341e081c73d2a43f05bcf2339d964c19 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 10 Nov 2026 13:43:54 -0600 Subject: [PATCH 7/7] fix(ci): fully qualify shared E2E command changes --- .github/workflows/pr-ci.yml | 1 + hack/ci/README.md | 2 ++ hack/ci/select_test.go | 18 ++++++++++++++++++ hack/ci/suites.yaml | 1 + 4 files changed, 22 insertions(+) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index e21132cf8e..fbdf5b81cc 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -49,6 +49,7 @@ jobs: - 'go.sum' - '.goreleaser.yml' - 'e2e/framework/**' + - 'e2e/cmd/**' - 'e2e/*.go' go: - '**/*.go' diff --git a/hack/ci/README.md b/hack/ci/README.md index 2ff32574ee..a7fb7a4185 100644 --- a/hack/ci/README.md +++ b/hack/ci/README.md @@ -32,6 +32,8 @@ every suite. Paths in `selection.core_paths` select the 13 suites marked only suites with a matching entry. Patterns support `*` within one path segment and `**` across zero or more segments. +Changes under `e2e/cmd/` are shared CI dependencies and select the full matrix. + The build matrix follows selected suites. Every E2E run needs the Linux binary, including Windows runs that inject it into containers. Windows suites also need a Windows build. MicroSandbox suites enable the provider diff --git a/hack/ci/select_test.go b/hack/ci/select_test.go index 146869ff7d..3d1c85bf07 100644 --- a/hack/ci/select_test.go +++ b/hack/ci/select_test.go @@ -179,6 +179,24 @@ func (s *selectionSuite) TestDockerInstallerChangesSelectInstallationSuite() { } } +func (s *selectionSuite) TestSharedE2EHelpersSelectFullMatrix() { + m, err := LoadManifest() + s.Require().NoError(err) + for _, file := range []string{ + "e2e/cmd/ghfixture/main_linux.go", + "e2e/cmd/devsy-ci/main.go", + } { + opts := selectionOptions{ + secretAvailable: true, + mode: selectionModePR, + changedFiles: []string{file}, + } + for _, group := range []string{groupUnprivileged, groupIntegration, groupMicrosandbox} { + s.Equal(m.SuitesIn(group), selectSuites(m, group, opts), file+"/"+group) + } + } +} + func (s *selectionSuite) TestEmbeddedCoreSmokeTier() { m, err := LoadManifest() s.Require().NoError(err) diff --git a/hack/ci/suites.yaml b/hack/ci/suites.yaml index 602e71f6ea..01c55c77e7 100644 --- a/hack/ci/suites.yaml +++ b/hack/ci/suites.yaml @@ -7,6 +7,7 @@ selection: - go.sum - .goreleaser.yml - e2e/framework/** + - e2e/cmd/** - e2e/*.go core_paths: - main.go