Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 8 additions & 16 deletions internal/cli/spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -696,9 +696,6 @@ func specTestReport(args []string) int {
// Fold tests: parsed summary first, else explicit counts.
if ts != nil {
rep.Tests = &session.SpecTestCounts{Total: ts.Total, Passed: ts.Passed, Failed: ts.Failed, Skipped: ts.Skipped}
if ts.Source != "" {
rep.TestPaths = appendUnique(rep.TestPaths, ts.Source)
}
rep.Attentions = append(rep.Attentions, testAttentions(ts)...)
} else if total >= 0 || passed >= 0 || failed >= 0 || skipped >= 0 {
rep.Tests = &session.SpecTestCounts{Total: nz(total), Passed: nz(passed), Failed: nz(failed), Skipped: nz(skipped)}
Expand All @@ -707,9 +704,6 @@ func specTestReport(args []string) int {
// Fold coverage: parsed summary first, else explicit counts.
if cov != nil {
rep.Coverage = &session.SpecCovSummary{Pct: cov.Pct, Covered: cov.Covered, Lines: cov.Lines}
if cov.Source != "" {
rep.TestPaths = appendUnique(rep.TestPaths, cov.Source)
}
} else if pct >= 0 || covered >= 0 || lines >= 0 {
c := &session.SpecCovSummary{Covered: nz(covered), Lines: nz(lines)}
switch {
Expand All @@ -730,6 +724,14 @@ func specTestReport(args []string) int {
return 1
}

// testPaths is the spec's own folder — deterministic over (root, feature) and
// independent of where the test/coverage artifacts were found.
specRel := filepath.ToSlash(workspace.Relative(r, sdir))
if !strings.HasSuffix(specRel, "/") {
specRel += "/"
}
rep.TestPaths = []string{specRel}

b, err := json.MarshalIndent(rep, "", " ")
if err != nil {
render.Err(err.Error())
Expand Down Expand Up @@ -798,16 +800,6 @@ func testAttentions(ts *session.TestSummary) []string {
return out
}

// appendUnique appends s to xs unless already present.
func appendUnique(xs []string, s string) []string {
for _, x := range xs {
if x == s {
return xs
}
}
return append(xs, s)
}

// nz clamps an "unset" (-1) flag value to 0.
func nz(n int) int {
if n < 0 {
Expand Down
84 changes: 82 additions & 2 deletions internal/cli/test_report_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,9 +150,89 @@ func TestSpecTestReportRunPassing(t *testing.T) {
if rep.Command != cmd {
t.Errorf("command = %q", rep.Command)
}
if len(rep.TestPaths) == 0 {
t.Errorf("expected testPaths evidence, got none")
if len(rep.TestPaths) != 1 || rep.TestPaths[0] != "specs/demo/" {
t.Errorf("testPaths = %v, want [specs/demo/]", rep.TestPaths)
}
for _, p := range rep.TestPaths {
if strings.Contains(p, "coverage.out") || strings.Contains(p, "junit.xml") {
t.Errorf("testPaths leaked an artifact path: %q", p)
}
}
}

// testPaths must deterministically point at the spec folder regardless of how the
// metrics were gathered, and must never carry a discovered artifact path.
func TestSpecTestReportTestPathsAnchoredToSpec(t *testing.T) {
readPaths := func(t *testing.T, specDir string) []string {
t.Helper()
data, err := os.ReadFile(filepath.Join(specDir, "test-report.json"))
if err != nil {
t.Fatal(err)
}
var rep struct {
TestPaths []string `json:"testPaths"`
}
if err := json.Unmarshal(data, &rep); err != nil {
t.Fatal(err)
}
return rep.TestPaths
}

t.Run("explicit counts, no run", func(t *testing.T) {
root := t.TempDir()
specDir := writeSpec(t, root)
if code := specTestReport([]string{"demo", "--root", root, "--total", "3", "--passed", "3"}); code != 0 {
t.Fatalf("exit = %d", code)
}
got := readPaths(t, specDir)
if len(got) != 1 || got[0] != "specs/demo/" {
t.Errorf("testPaths = %v, want [specs/demo/]", got)
}
})

t.Run("discover by lang and path drops artifact paths", func(t *testing.T) {
root := t.TempDir()
specDir := writeSpec(t, root)
testsDir := filepath.Join(root, "tests")
if err := os.MkdirAll(testsDir, 0o755); err != nil {
t.Fatal(err)
}
os.WriteFile(filepath.Join(testsDir, "coverage.out"), []byte("mode: set\ngithub.com/a/b.go:1.1,2.2 3 1\n"), 0o644)
os.WriteFile(filepath.Join(testsDir, "junit.xml"), []byte(`<testsuites><testsuite name="s" tests="1" failures="0"><testcase name="a"/></testsuite></testsuites>`), 0o644)
if code := specTestReport([]string{"demo", "--root", root, "--lang", "go", "--path", "tests"}); code != 0 {
t.Fatalf("exit = %d", code)
}
got := readPaths(t, specDir)
if len(got) != 1 || got[0] != "specs/demo/" {
t.Errorf("testPaths = %v, want [specs/demo/]", got)
}
for _, p := range got {
if strings.Contains(p, "coverage.out") || strings.Contains(p, "junit.xml") || strings.Contains(p, "tests/") {
t.Errorf("testPaths leaked an artifact path: %q", p)
}
}
})

t.Run("explicit junit and coverage files", func(t *testing.T) {
root := t.TempDir()
specDir := writeSpec(t, root)
junit := filepath.Join(root, "junit.xml")
os.WriteFile(junit, []byte(`<testsuites><testsuite name="s" tests="2" failures="0"><testcase name="a"/><testcase name="b"/></testsuite></testsuites>`), 0o644)
cov := filepath.Join(root, "cover.out")
os.WriteFile(cov, []byte("mode: set\ngithub.com/a/b.go:1.1,2.2 3 1\n"), 0o644)
if code := specTestReport([]string{"demo", "--root", root, "--junit", junit, "--coverage", cov}); code != 0 {
t.Fatalf("exit = %d", code)
}
got := readPaths(t, specDir)
if len(got) != 1 || got[0] != "specs/demo/" {
t.Errorf("testPaths = %v, want [specs/demo/]", got)
}
for _, p := range got {
if strings.Contains(p, "junit.xml") || strings.Contains(p, "cover.out") {
t.Errorf("testPaths leaked an artifact path: %q", p)
}
}
})
}

func TestSpecTestReportRunFailing(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion internal/session/reports.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ type SpecReport struct {
Command string `json:"command,omitempty"`
Tests *SpecTestCounts `json:"tests,omitempty"`
Coverage *SpecCovSummary `json:"coverage,omitempty"`
TestPaths []string `json:"testPaths,omitempty"` // report artifacts proving the run (traceable evidence)
TestPaths []string `json:"testPaths,omitempty"` // the spec folder this evidence belongs to (deterministic: specs/<feature>/)
Attentions []string `json:"attentions,omitempty"` // key unit-test signals: failures, skips, command failure
}

Expand Down
Loading