From d4e9949fc7c0daced144731cb0719978b8c5274b Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 17 Oct 2026 12:47:05 -0600 Subject: [PATCH] refactor(e2e): add CI label coverage guard Add devsy-ci check-labels. It fails when an It spec is selected by no CI label filter or a CI label filter selects zero specs. Filters come from pr-ci.yml and docker-buildkit-windows.yml matrix labels and literal --ginkgo.label-filter values. --- .github/workflows/pr-ci.yml | 4 + e2e/README.md | 23 ++ e2e/cmd/devsy-ci/command.go | 2 +- e2e/cmd/devsy-ci/command_test.go | 2 +- e2e/cmd/devsy-ci/labels.go | 390 ++++++++++++++++++ e2e/cmd/devsy-ci/labels_test.go | 305 ++++++++++++++ e2e/cmd/devsy-ci/report.go | 21 +- e2e/cmd/devsy-ci/testdata/workflow_labels.yml | 26 ++ e2e/framework/runenv/errors.go | 1 + 9 files changed, 765 insertions(+), 9 deletions(-) create mode 100644 e2e/cmd/devsy-ci/labels.go create mode 100644 e2e/cmd/devsy-ci/labels_test.go create mode 100644 e2e/cmd/devsy-ci/testdata/workflow_labels.yml diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 48320e405e..b02fc69d4f 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -123,6 +123,10 @@ jobs: - name: test E2E runtime helpers run: go test -race -count=1 ./e2e/framework/runenv/... ./e2e/framework/dockerwsl ./e2e/cmd/devsy-ci + - name: check E2E CI label coverage + if: runner.os == 'Linux' + run: go run ./e2e/cmd/devsy-ci check-labels + - name: test Compose UID expectation matrix shell: bash run: | diff --git a/e2e/README.md b/e2e/README.md index 1c259bbbd2..14becaaf86 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -73,3 +73,26 @@ Ginkgo executes the specs directly. `devsy-ci check-report` checks its JSON report: all six BuildKit specs must pass on their first attempt, with no selected skips, dry run or programmatic focus. Teardown runs on success and failure, before the report and non-secret diagnostics are uploaded. + +## Label coverage guard + +`devsy-ci check-labels` requires every `It` spec in the e2e suite to match at least one +CI label filter, and every filter to select at least one spec. Filters come from the +`matrix.label` rows and the literal `--ginkgo.label-filter=` values in +`.github/workflows/pr-ci.yml` and `.github/workflows/docker-buildkit-windows.yml`. +Expressions such as `${{ matrix.label }}` are skipped. Filters are evaluated with +Ginkgo's own parser. + +Run it from the repository root: + +```bash +go run ./e2e/cmd/devsy-ci check-labels +``` + +The command builds the e2e test binary and lists its specs with `-ginkgo.dry-run`. Pass +`--report ` to check an existing Ginkgo JSON report instead. + +Spec sets depend on the host OS, because build-tagged files such as `*_linux.go` are +excluded elsewhere. Run the check on Linux to match the `unit-tests` CI job, which +gates the step on `runner.os == 'Linux'`. On macOS, Linux-only labels report as +selecting zero specs. diff --git a/e2e/cmd/devsy-ci/command.go b/e2e/cmd/devsy-ci/command.go index 6dc929d7f1..86e6d4e3eb 100644 --- a/e2e/cmd/devsy-ci/command.go +++ b/e2e/cmd/devsy-ci/command.go @@ -13,6 +13,6 @@ func newRootCommand(deps *runenv.Dependencies) *cobra.Command { SilenceUsage: true, SilenceErrors: true, } - root.AddCommand(dockerwsl.NewCommand(deps), newReportCommand(deps)) + root.AddCommand(dockerwsl.NewCommand(deps), newReportCommand(deps), newCheckLabelsCommand(deps)) return root } diff --git a/e2e/cmd/devsy-ci/command_test.go b/e2e/cmd/devsy-ci/command_test.go index da0e46d0a1..e7aa2ab3f4 100644 --- a/e2e/cmd/devsy-ci/command_test.go +++ b/e2e/cmd/devsy-ci/command_test.go @@ -19,5 +19,5 @@ func (s *CommandSuite) TestRootCommandSurface() { for _, c := range root.Commands() { names = append(names, c.Name()) } - s.ElementsMatch([]string{"docker-wsl", reportCommandName}, names) + s.ElementsMatch([]string{"docker-wsl", reportCommandName, checkLabelsCommandName}, names) } diff --git a/e2e/cmd/devsy-ci/labels.go b/e2e/cmd/devsy-ci/labels.go new file mode 100644 index 0000000000..c0dc41bfce --- /dev/null +++ b/e2e/cmd/devsy-ci/labels.go @@ -0,0 +1,390 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "time" + + "github.com/devsy-org/devsy/e2e/framework/runenv" + "github.com/onsi/ginkgo/v2/types" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" +) + +const ( + checkLabelsCommandName = "check-labels" + e2ePackage = "./e2e" + e2eDir = "e2e" + e2eSuiteRun = "^TestRunE2ETests$" + buildTimeout = 15 * time.Minute + dryRunTimeout = 10 * time.Minute + ginkgoExpressionMarker = "${{" +) + +// defaultLabelWorkflows lists every workflow that selects e2e specs by label. +var defaultLabelWorkflows = []string{ + ".github/workflows/pr-ci.yml", + ".github/workflows/docker-buildkit-windows.yml", +} + +// labelFilterArg matches a literal --ginkgo.label-filter= value in a run script, +// quoted with " or ', or unquoted (shell or PowerShell). +var labelFilterArg = regexp.MustCompile( + `label-filter=(?:"([^"]*)"|'([^']*)'|([^\s"'\x60]+))`, +) + +type labelFilter struct { + source string + expr string + match types.LabelFilter +} + +type workflowDoc struct { + Jobs map[string]workflowJob `yaml:"jobs"` +} + +type workflowJob struct { + Strategy workflowStrategy `yaml:"strategy"` + Steps []workflowStep `yaml:"steps"` +} + +type workflowStrategy struct { + Matrix any `yaml:"matrix"` +} + +type workflowStep struct { + Name string `yaml:"name"` + Run string `yaml:"run"` +} + +type labelCoverage struct { + specs int + uncovered []types.SpecReport + dead []labelFilter +} + +func newCheckLabelsCommand(deps *runenv.Dependencies) *cobra.Command { + var reportPath string + var workflows []string + cmd := &cobra.Command{ + Use: checkLabelsCommandName, + Short: "Require every Ginkgo spec to be selected by a CI label filter", + Long: "Requires every It spec in the e2e suite to match at least one CI label " + + "filter, and every filter to select at least one spec. Run from the repository root.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return runCheckLabels(cmd.Context(), deps, reportPath, workflows) + }, + } + cmd.Flags().StringVar(&reportPath, "report", "", + "Ginkgo JSON dry-run report; the e2e suite is built and listed when empty") + cmd.Flags().StringSliceVar(&workflows, "workflow", slices.Clone(defaultLabelWorkflows), + "workflow file to read label filters from (repeatable)") + return cmd +} + +func runCheckLabels( + ctx context.Context, + deps *runenv.Dependencies, + reportPath string, + workflows []string, +) error { + filters, err := loadLabelFilters(deps, workflows) + if err != nil { + return runenv.NewStageError(runenv.CodeLabelCoverageInvalid, checkLabelsCommandName, err) + } + data, err := readSpecReport(ctx, deps, reportPath) + if err != nil { + return runenv.NewStageError(runenv.CodeLabelCoverageInvalid, checkLabelsCommandName, err) + } + report, err := decodeSuiteReport(data) + if err != nil { + return runenv.NewStageError(runenv.CodeLabelCoverageInvalid, checkLabelsCommandName, err) + } + coverage, err := evaluateLabelCoverage(report.SpecReports, filters) + if err != nil { + return runenv.NewStageError(runenv.CodeLabelCoverageInvalid, checkLabelsCommandName, err) + } + if err := coverage.problems(); err != nil { + return runenv.NewStageError(runenv.CodeLabelCoverageInvalid, checkLabelsCommandName, err) + } + if _, err := fmt.Fprintf( + deps.Out, "label coverage: %d specs, %d filters\n", coverage.specs, len(filters), + ); err != nil { + return fmt.Errorf("write summary: %w", err) + } + return nil +} + +func loadLabelFilters(deps *runenv.Dependencies, workflows []string) ([]labelFilter, error) { + var filters []labelFilter + for _, path := range workflows { + data, err := deps.Files.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read workflow %s: %w", path, err) + } + found, err := workflowLabelFilters(path, data) + if err != nil { + return nil, err + } + filters = append(filters, found...) + } + if len(filters) == 0 { + return nil, errors.New("no CI label filters found in the workflows") + } + return filters, nil +} + +// workflowLabelFilters collects matrix labels and literal --ginkgo.label-filter +// values. Expressions such as ${{ matrix.label }} are skipped; their values +// come from the matrix rows, which are already collected. +func workflowLabelFilters(path string, data []byte) ([]labelFilter, error) { + var doc workflowDoc + if err := yaml.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("decode workflow %s: %w", path, err) + } + var filters []labelFilter + for _, jobName := range sortedKeys(doc.Jobs) { + jobFilters, err := jobLabelFilters(path, jobName, doc.Jobs[jobName]) + if err != nil { + return nil, err + } + filters = append(filters, jobFilters...) + } + return filters, nil +} + +// jobLabelFilters returns the matrix labels and literal label-filter values of +// one job. +func jobLabelFilters(path, jobName string, job workflowJob) ([]labelFilter, error) { + labels, err := matrixLabels(job.Strategy.Matrix) + if err != nil { + return nil, fmt.Errorf("%s job %s: %w", path, jobName, err) + } + var filters []labelFilter + for _, label := range labels { + filters = append(filters, labelFilter{ + source: fmt.Sprintf("%s job %s matrix", path, jobName), + expr: label, + }) + } + for _, step := range job.Steps { + for _, m := range labelFilterArg.FindAllStringSubmatch(step.Run, -1) { + expr := firstNonEmpty(m[1:]) + if expr == "" || strings.Contains(expr, ginkgoExpressionMarker) { + continue + } + filters = append(filters, labelFilter{ + source: fmt.Sprintf("%s job %s step %q", path, jobName, step.Name), + expr: expr, + }) + } + } + return filters, nil +} + +// matrixLabels returns the label of each matrix include row. Rows without a +// label (for example build-cli rows) are not e2e filters and are ignored. +func matrixLabels(matrix any) ([]string, error) { + if matrix == nil { + return nil, nil + } + m, ok := matrix.(map[string]any) + if !ok { + return nil, fmt.Errorf("strategy.matrix must be a mapping to be checked, got %T", matrix) + } + include, _ := m["include"].([]any) + var labels []string + for _, row := range include { + fields, ok := row.(map[string]any) + if !ok { + continue + } + label, ok := fields["label"].(string) + if !ok || label == "" { + continue + } + if strings.Contains(label, ginkgoExpressionMarker) { + return nil, fmt.Errorf("matrix label %q must be a literal", label) + } + labels = append(labels, label) + } + return labels, nil +} + +func compileLabelFilters(filters []labelFilter) ([]labelFilter, error) { + compiled := make([]labelFilter, len(filters)) + for i, f := range filters { + match, err := types.ParseLabelFilter(f.expr) + if err != nil { + return nil, fmt.Errorf( + "parse label filter %q from %s: %w", + f.expr, + f.source, + err, + ) + } + f.match = match + compiled[i] = f + } + return compiled, nil +} + +// recordSpecHits adds one hit to every filter that selects spec and reports +// whether any filter selected it. +func recordSpecHits(spec types.SpecReport, filters []labelFilter, hits []int) bool { + labels := spec.Labels() + selected := false + for i, f := range filters { + if f.match(labels) { + hits[i]++ + selected = true + } + } + return selected +} + +func evaluateLabelCoverage(specs types.SpecReports, filters []labelFilter) (labelCoverage, error) { + compiled, err := compileLabelFilters(filters) + if err != nil { + return labelCoverage{}, err + } + + var coverage labelCoverage + hits := make([]int, len(compiled)) + for _, spec := range specs { + if spec.LeafNodeType != types.NodeTypeIt { + continue + } + coverage.specs++ + if !recordSpecHits(spec, compiled, hits) { + coverage.uncovered = append(coverage.uncovered, spec) + } + } + if coverage.specs == 0 { + return labelCoverage{}, errors.New("report contains no It specs") + } + for i, n := range hits { + if n == 0 { + coverage.dead = append(coverage.dead, compiled[i]) + } + } + return coverage, nil +} + +func (c labelCoverage) problems() error { + if len(c.uncovered) == 0 && len(c.dead) == 0 { + return nil + } + var b strings.Builder + if len(c.uncovered) > 0 { + fmt.Fprintf(&b, "%d spec(s) selected by no CI label filter:\n", len(c.uncovered)) + for _, spec := range c.uncovered { + loc := spec.LeafNodeLocation + fmt.Fprintf(&b, " %s:%d: %s\n", loc.FileName, loc.LineNumber, spec.FullText()) + } + } + if len(c.dead) > 0 { + fmt.Fprintf(&b, "%d label filter(s) select zero specs:\n", len(c.dead)) + for _, f := range c.dead { + fmt.Fprintf(&b, " %q from %s\n", f.expr, f.source) + } + } + return errors.New(strings.TrimRight(b.String(), "\n")) +} + +func readSpecReport( + ctx context.Context, + deps *runenv.Dependencies, + reportPath string, +) ([]byte, error) { + if reportPath != "" { + data, err := deps.Files.ReadFile(reportPath) + if err != nil { + return nil, fmt.Errorf("read Ginkgo report: %w", err) + } + return data, nil + } + dir, err := os.MkdirTemp("", "devsy-ci-labels-") + if err != nil { + return nil, fmt.Errorf("create temp dir: %w", err) + } + defer func() { _ = os.RemoveAll(dir) }() + + binary := filepath.Join(dir, "e2e.test") + if deps.GOOS == runenv.GoosWindows { + binary += ".exe" + } + jsonPath := filepath.Join(dir, "report.json") + + if err := runStep(ctx, deps, runenv.Command{ + Path: "go", + Args: []string{"test", "-c", "-o", binary, e2ePackage}, + Timeout: buildTimeout, + }, "build e2e test binary"); err != nil { + return nil, err + } + if err := runStep(ctx, deps, runenv.Command{ + Path: binary, + Args: []string{ + "-test.run=" + e2eSuiteRun, + "-ginkgo.dry-run", + "-ginkgo.no-color", + "-ginkgo.json-report=" + jsonPath, + }, + Dir: e2eDir, + Timeout: dryRunTimeout, + }, "list e2e specs"); err != nil { + return nil, err + } + data, err := deps.Files.ReadFile(jsonPath) + if err != nil { + return nil, fmt.Errorf("read generated Ginkgo report: %w", err) + } + return data, nil +} + +func runStep( + ctx context.Context, + deps *runenv.Dependencies, + cmd runenv.Command, + step string, +) error { + cmd.Stdout = deps.Err + cmd.Stderr = deps.Err + result, err := deps.Run.Run(ctx, cmd) + if err != nil { + return fmt.Errorf("%s: %w", step, err) + } + if result.TimedOut { + return fmt.Errorf("%s: timed out", step) + } + if result.ExitCode != 0 { + return fmt.Errorf("%s: exit code %d", step, result.ExitCode) + } + return nil +} + +func firstNonEmpty(values []string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + slices.Sort(keys) + return keys +} diff --git a/e2e/cmd/devsy-ci/labels_test.go b/e2e/cmd/devsy-ci/labels_test.go new file mode 100644 index 0000000000..8cbcb1a6ad --- /dev/null +++ b/e2e/cmd/devsy-ci/labels_test.go @@ -0,0 +1,305 @@ +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/devsy-org/devsy/e2e/framework/runenv" + "github.com/devsy-org/devsy/e2e/framework/runenv/runenvtest" + "github.com/onsi/ginkgo/v2/types" + "github.com/stretchr/testify/suite" +) + +const ( + labelsWorkflowFixture = "testdata/workflow_labels.yml" + testWorkflow = "wf.yml" + testReport = "report.json" + jsonReportFlag = "-ginkgo.json-report=" + labelX = "up-x" + labelY = "up-y" + specTwoText = "suite two" +) + +type LabelsSuite struct{ suite.Suite } + +func TestLabelsSuite(t *testing.T) { suite.Run(t, new(LabelsSuite)) } + +func itSpec(containerLabels []string, leafLabels []string, text string) types.SpecReport { + return types.SpecReport{ + LeafNodeType: types.NodeTypeIt, + ContainerHierarchyTexts: []string{"suite"}, + ContainerHierarchyLabels: [][]string{containerLabels}, + LeafNodeLabels: leafLabels, + LeafNodeText: text, + LeafNodeLocation: types.CodeLocation{FileName: "x_test.go", LineNumber: 7}, + } +} + +func filtersFor(exprs ...string) []labelFilter { + filters := make([]labelFilter, 0, len(exprs)) + for _, e := range exprs { + filters = append(filters, labelFilter{source: "test", expr: e}) + } + return filters +} + +func (s *LabelsSuite) TestEvaluateLabelCoverage() { + specs := types.SpecReports{ + itSpec([]string{labelX}, []string{"fast"}, "one"), + itSpec(nil, []string{"slow"}, "two"), + itSpec([]string{labelY}, nil, "three"), + {LeafNodeType: types.NodeTypeBeforeAll, ContainerHierarchyLabels: [][]string{{labelX}}}, + } + tests := []struct { + name string + exprs []string + wantSpecs int + wantUncovered []string + wantDead []string + }{ + { + name: "container label selects leaf spec", + exprs: []string{labelX}, + wantSpecs: 3, + wantUncovered: []string{specTwoText, "suite three"}, + }, + { + name: "conjunction across container and leaf labels", + exprs: []string{labelX + " && fast", "!slow", labelY}, + wantSpecs: 3, + wantUncovered: []string{specTwoText}, + }, + { + name: "conjunction that matches nothing is dead", + exprs: []string{labelX + " && slow", labelX, "!slow"}, + wantSpecs: 3, + wantUncovered: []string{specTwoText}, + wantDead: []string{labelX + " && slow"}, + }, + { + name: "non-It nodes do not count as specs", + exprs: []string{labelX, labelY, "fast", "slow"}, + wantSpecs: 3, + wantUncovered: nil, + }, + } + for _, tc := range tests { + s.Run(tc.name, func() { + got, err := evaluateLabelCoverage(specs, filtersFor(tc.exprs...)) + s.Require().NoError(err) + s.Equal(tc.wantSpecs, got.specs) + + var uncovered []string + for _, spec := range got.uncovered { + uncovered = append(uncovered, spec.FullText()) + } + s.Equal(tc.wantUncovered, uncovered) + + var dead []string + for _, f := range got.dead { + dead = append(dead, f.expr) + } + s.Equal(tc.wantDead, dead) + }) + } +} + +func (s *LabelsSuite) TestEvaluateLabelCoverageErrors() { + it := types.SpecReports{itSpec([]string{labelX}, nil, "one")} + s.Run("report without It specs", func() { + _, err := evaluateLabelCoverage(types.SpecReports{ + {LeafNodeType: types.NodeTypeBeforeAll}, + }, filtersFor(labelX)) + s.ErrorContains(err, "report contains no It specs") + }) + s.Run("unparseable filter names its source", func() { + _, err := evaluateLabelCoverage(it, []labelFilter{{source: "pr-ci.yml job e2e", expr: "("}}) + s.ErrorContains(err, "pr-ci.yml job e2e") + }) +} + +func (s *LabelsSuite) TestProblemsReportsUncoveredAndDead() { + coverage, err := evaluateLabelCoverage(types.SpecReports{ + itSpec(nil, nil, "orphan"), + }, filtersFor(labelX)) + s.Require().NoError(err) + + err = coverage.problems() + s.Require().Error(err) + s.Contains(err.Error(), "x_test.go:7: suite orphan") + s.Contains(err.Error(), `"up-x" from test`) +} + +func (s *LabelsSuite) TestProblemsEmptyWhenCovered() { + coverage, err := evaluateLabelCoverage(types.SpecReports{ + itSpec([]string{labelX}, nil, "one"), + }, filtersFor(labelX)) + s.Require().NoError(err) + s.NoError(coverage.problems()) +} + +func (s *LabelsSuite) TestWorkflowLabelFilters() { + data, err := os.ReadFile(labelsWorkflowFixture) + s.Require().NoError(err) + + filters, err := workflowLabelFilters(testWorkflow, data) + s.Require().NoError(err) + + var exprs []string + for _, f := range filters { + exprs = append(exprs, f.expr) + } + // Matrix rows without a label are ignored; ${{ }} run values are skipped. + s.Equal([]string{ + "up-alpha", + "up-beta", + "up-gamma && !slow", + "up-delta", + "up-eps", + "up-zeta", + }, exprs) + s.Contains(filters[2].source, "job e2e") +} + +func (s *LabelsSuite) TestWorkflowLabelFiltersRejectsUnverifiableMatrix() { + tests := []struct { + name string + yaml string + wantErr string + }{ + { + name: "matrix expression is not a literal", + yaml: "jobs:\n e2e:\n strategy:\n matrix:\n" + + " include:\n - label: ${{ inputs.label }}\n", + wantErr: "must be a literal", + }, + { + name: "matrix is not a mapping", + yaml: "jobs:\n e2e:\n strategy:\n matrix: ${{ fromJSON(x) }}\n", + wantErr: "strategy.matrix must be a mapping", + }, + { + name: "invalid yaml", + yaml: "jobs: [", + wantErr: "decode workflow", + }, + } + for _, tc := range tests { + s.Run(tc.name, func() { + _, err := workflowLabelFilters(testWorkflow, []byte(tc.yaml)) + s.ErrorContains(err, tc.wantErr) + }) + } +} + +func (s *LabelsSuite) TestRunCheckLabelsWithReport() { + workflow, err := os.ReadFile(labelsWorkflowFixture) + s.Require().NoError(err) + + covering := types.Report{ + SuiteSucceeded: true, + SpecReports: types.SpecReports{ + itSpec([]string{"up-alpha"}, nil, "a"), + itSpec([]string{"up-beta"}, nil, "b"), + itSpec([]string{"up-gamma"}, nil, "c"), + itSpec([]string{"up-delta"}, nil, "d"), + itSpec([]string{"up-eps"}, nil, "e"), + itSpec([]string{"up-zeta"}, nil, "f"), + }, + } + + s.Run("covered specs succeed and print the count", func() { + rig := runenvtest.NewRig(runenv.GoosLinux, nil) + rig.FS.Data[testWorkflow] = workflow + rig.FS.Data[testReport] = mustJSON(s, []types.Report{covering}) + + err := runCheckLabels(s.T().Context(), rig.Dependencies, testReport, []string{testWorkflow}) + s.Require().NoError(err) + s.Contains(rig.Output.String(), "label coverage: 6 specs, 6 filters") + }) + + s.Run("uncovered spec fails with its location", func() { + rig := runenvtest.NewRig(runenv.GoosLinux, nil) + rig.FS.Data[testWorkflow] = workflow + report := covering + report.SpecReports = append( + types.SpecReports{itSpec(nil, nil, "loose")}, + report.SpecReports...) + rig.FS.Data[testReport] = mustJSON(s, []types.Report{report}) + + err := runCheckLabels(s.T().Context(), rig.Dependencies, testReport, []string{testWorkflow}) + s.ErrorContains(err, "x_test.go:7: suite loose") + }) + + s.Run("missing workflow is an error", func() { + rig := runenvtest.NewRig(runenv.GoosLinux, nil) + rig.FS.Data[testReport] = mustJSON(s, []types.Report{covering}) + + err := runCheckLabels(s.T().Context(), rig.Dependencies, testReport, []string{"absent.yml"}) + s.ErrorContains(err, "read workflow absent.yml") + }) +} + +func (s *LabelsSuite) TestReadSpecReportGeneratesReport() { + rig := runenvtest.NewRig(runenv.GoosLinux, nil) + rig.Runner.Handle = func(c runenv.Command) (runenv.CommandResult, error) { + if c.Path == "go" { + return runenv.CommandResult{}, nil + } + for _, arg := range c.Args { + if path, ok := strings.CutPrefix(arg, jsonReportFlag); ok { + rig.FS.Data[path] = []byte("[]") + } + } + return runenv.CommandResult{}, nil + } + + data, err := readSpecReport(s.T().Context(), rig.Dependencies, "") + s.Require().NoError(err) + s.Equal("[]", string(data)) + + s.Require().Len(rig.Runner.Calls, 2) + build, list := rig.Runner.Calls[0], rig.Runner.Calls[1] + s.Equal("go", build.Path) + s.Equal([]string{"test", "-c", "-o", build.Args[3], "./e2e"}, build.Args) + s.Equal(build.Args[3], list.Path) + s.Equal("e2e", list.Dir) + s.Contains(list.Args, "-test.run=^TestRunE2ETests$") + s.Contains(list.Args, "-ginkgo.dry-run") +} + +func (s *LabelsSuite) TestReadSpecReportWindowsBinaryName() { + rig := runenvtest.NewRig(runenv.GoosWindows, nil) + rig.Runner.Handle = func(c runenv.Command) (runenv.CommandResult, error) { + if c.Path != "go" { + for _, arg := range c.Args { + if path, ok := strings.CutPrefix(arg, jsonReportFlag); ok { + rig.FS.Data[path] = []byte("[]") + } + } + } + return runenv.CommandResult{}, nil + } + + _, err := readSpecReport(s.T().Context(), rig.Dependencies, "") + s.Require().NoError(err) + s.True(strings.HasSuffix(rig.Runner.Calls[0].Args[3], ".exe")) +} + +func (s *LabelsSuite) TestReadSpecReportFailsOnStepExit() { + rig := runenvtest.NewRig(runenv.GoosLinux, nil) + rig.Runner.Handle = func(runenv.Command) (runenv.CommandResult, error) { + return runenv.CommandResult{ExitCode: 1}, nil + } + + _, err := readSpecReport(s.T().Context(), rig.Dependencies, "") + s.ErrorContains(err, "build e2e test binary: exit code 1") +} + +func mustJSON(s *LabelsSuite, v any) []byte { + data, err := json.Marshal(v) + s.Require().NoError(err) + return data +} diff --git a/e2e/cmd/devsy-ci/report.go b/e2e/cmd/devsy-ci/report.go index 8419b94043..3d14a51dca 100644 --- a/e2e/cmd/devsy-ci/report.go +++ b/e2e/cmd/devsy-ci/report.go @@ -50,14 +50,10 @@ func checkReport(data []byte, label string, expected int) error { } func readReport(data []byte) (types.Report, error) { - var reports []types.Report - if err := json.Unmarshal(data, &reports); err != nil { - return types.Report{}, fmt.Errorf("decode Ginkgo report: %w", err) - } - if len(reports) != 1 { - return types.Report{}, errors.New("expected exactly one Ginkgo suite report") + report, err := decodeSuiteReport(data) + if err != nil { + return types.Report{}, err } - report := reports[0] if !report.SuiteSucceeded || report.SuiteConfig.DryRun || report.SuiteHasProgrammaticFocus { return types.Report{}, errors.New( "report must prove a successful run without dry-run or programmatic focus", @@ -66,6 +62,17 @@ func readReport(data []byte) (types.Report, error) { return report, nil } +func decodeSuiteReport(data []byte) (types.Report, error) { + var reports []types.Report + if err := json.Unmarshal(data, &reports); err != nil { + return types.Report{}, fmt.Errorf("decode Ginkgo report: %w", err) + } + if len(reports) != 1 { + return types.Report{}, errors.New("expected exactly one Ginkgo suite report") + } + return reports[0], nil +} + func checkSelectedSpecs(specs types.SpecReports, label string, expected int) error { selected := 0 for _, spec := range specs { diff --git a/e2e/cmd/devsy-ci/testdata/workflow_labels.yml b/e2e/cmd/devsy-ci/testdata/workflow_labels.yml new file mode 100644 index 0000000000..064ca668aa --- /dev/null +++ b/e2e/cmd/devsy-ci/testdata/workflow_labels.yml @@ -0,0 +1,26 @@ +name: fixture +jobs: + build: + strategy: + matrix: + include: + - runner: ubuntu-latest + e2e: + strategy: + matrix: + include: + - label: up-alpha + - label: "up-beta" + steps: + - name: quoted expression + run: go test ./e2e -- --ginkgo.label-filter="up-gamma && !slow" + - name: single quoted + run: go test ./e2e -- --ginkgo.label-filter='up-delta' + - name: bare + run: go test ./e2e -- --ginkgo.label-filter=up-eps + - name: matrix expression + run: go test ./e2e -- --ginkgo.label-filter="${{ matrix.label }}" + windows: + steps: + - name: powershell + run: go test ./e2e -- -ginkgo.label-filter=up-zeta diff --git a/e2e/framework/runenv/errors.go b/e2e/framework/runenv/errors.go index 1658f610b2..ee8c57bbbb 100644 --- a/e2e/framework/runenv/errors.go +++ b/e2e/framework/runenv/errors.go @@ -14,6 +14,7 @@ const ( CodeConfigInvalid Code = "CI_CONFIG_INVALID" CodeProfileUnsupported Code = "CI_PROFILE_UNSUPPORTED" CodeGinkgoReportInvalid Code = "CI_GINKGO_REPORT_INVALID" + CodeLabelCoverageInvalid Code = "CI_LABEL_COVERAGE_INVALID" CodeDockerWSLDistroGone Code = "CI_DOCKER_WSL_DISTRO_MISSING" CodeDockerWSLInstall Code = "CI_DOCKER_WSL_INSTALL_FAILED" CodeDockerWSLDaemon Code = "CI_DOCKER_WSL_DAEMON_NOT_READY"