diff --git a/hack/licenses/detector.go b/hack/licenses/detector.go new file mode 100644 index 0000000000..084f946deb --- /dev/null +++ b/hack/licenses/detector.go @@ -0,0 +1,88 @@ +package main + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" +) + +// checkInSync regenerates the report into a temp file and compares it with +// outFile. An out-of-date file prints a diff and a warning but returns nil. +func (g generator) checkInSync(hackDir, outFile string, deps []byte) error { + tmp, err := os.CreateTemp("", "third-party-licenses-*.md") + if err != nil { + return err + } + target := tmp.Name() + if err := tmp.Close(); err != nil { + return err + } + defer func() { _ = os.Remove(target) }() + + _, _ = fmt.Fprintln(g.stderr, "Generating", filepath.Base(outFile)+"...") + if err := g.runDetector(hackDir, deps, target); err != nil { + return err + } + + current, err := os.ReadFile(outFile) + if err != nil { + return fmt.Errorf("reading %s: %w", outFile, err) + } + generated, err := os.ReadFile(target) + if err != nil { + return err + } + if !bytes.Equal(current, generated) { + diff := exec.Command("git", "--no-pager", "diff", "--no-index", "--", outFile, target) + diff.Stdout = g.stderr + diff.Stderr = g.stderr + _ = g.runCmd(diff) + _, _ = fmt.Fprintln( + g.stderr, + "warning: THIRD_PARTY_LICENSES.md is out of date; run 'mise run cli:licenses' and commit the result", + ) + return nil + } + + _, _ = fmt.Fprintln(g.stderr, "THIRD_PARTY_LICENSES.md is up to date.") + return nil +} + +func (g generator) runDetector(hackDir string, deps []byte, outPath string) error { + tmpDir, err := os.MkdirTemp("", "go-licence-detector-") + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(tmpDir) }() + + cmd := exec.Command("go", "run", + "go.elastic.co/go-licence-detector@"+detectorVersion, + "-rules", filepath.Join(hackDir, "rules.json"), + "-overrides", filepath.Join(hackDir, "overrides.ndjson"), + "-depsTemplate", filepath.Join(hackDir, "third-party-licenses.md.tmpl"), + "-depsOut", outPath, + ) + cmd.Dir = tmpDir + cmd.Stdin = bytes.NewReader(deps) + cmd.Stderr = g.stderr + if err := g.runCmd(cmd); err != nil { + return fmt.Errorf("go-licence-detector: %w", err) + } + + return normalizeTrailingNewline(outPath) +} + +// normalizeTrailingNewline leaves exactly one trailing newline on path. +func normalizeTrailingNewline(path string) error { + content, err := os.ReadFile(path) + if err != nil { + return err + } + normalized := append(bytes.TrimRight(content, "\n"), '\n') + if bytes.Equal(content, normalized) { + return nil + } + return os.WriteFile(path, normalized, 0o644) +} diff --git a/hack/licenses/detector_test.go b/hack/licenses/detector_test.go new file mode 100644 index 0000000000..077712c29f --- /dev/null +++ b/hack/licenses/detector_test.go @@ -0,0 +1,126 @@ +package main + +import ( + "bytes" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +const ( + reportBody = "# Licenses\n" + inSyncMsg = "THIRD_PARTY_LICENSES.md is up to date." +) + +func TestNormalizeTrailingNewline(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {name: "no trailing newline", in: "# Licenses", want: reportBody}, + {name: "one trailing newline", in: reportBody, want: reportBody}, + {name: "many trailing newlines", in: "# Licenses\n\n\n", want: reportBody}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "out.md") + fatalOnErr(t, os.WriteFile(path, []byte(tt.in), 0o644)) + + fatalOnErr(t, normalizeTrailingNewline(path)) + + got, err := os.ReadFile(path) + fatalOnErr(t, err) + if string(got) != tt.want { + t.Errorf("content = %q, want %q", got, tt.want) + } + }) + } +} + +// TestRunDetectorFeedsDepsAndNormalizesOutput checks the detector receives the +// deps stream on stdin and that its output lands at the requested path with +// one trailing newline. +func TestRunDetectorFeedsDepsAndNormalizesOutput(t *testing.T) { + fake := &fakeProcesses{detectorOut: reportBody + "\n"} + g := generator{root: t.TempDir(), runCmd: fake.run, stderr: io.Discard} + deps := []byte("{\n\t\"Path\": \"a.example/a\"\n}\n") + outPath := filepath.Join(t.TempDir(), "THIRD_PARTY_LICENSES.md") + + fatalOnErr(t, g.runDetector(filepath.Join(g.root, "hack", "licenses"), deps, outPath)) + + if !bytes.Equal(fake.detectorIn, deps) { + t.Errorf("detector stdin = %q, want %q", fake.detectorIn, deps) + } + got, err := os.ReadFile(outPath) + fatalOnErr(t, err) + if string(got) != reportBody { + t.Errorf("output = %q, want %q", got, reportBody) + } +} + +// TestCheckInSync pins current check-mode behaviour: an out-of-date file +// prints a warning and still returns nil (see PR Notes). A missing file is +// an error. +func TestCheckInSync(t *testing.T) { + tests := []struct { + name string + current *string + detector string + wantErr bool + wantStderr string + }{ + { + name: "in sync", + current: new(reportBody), + detector: reportBody, + wantStderr: inSyncMsg, + }, + { + name: "trailing newline drift is in sync", + current: new(reportBody), + detector: "# Licenses\n\n\n", + wantStderr: inSyncMsg, + }, + { + name: "out of date warns and returns nil", + current: new("# Old\n"), + detector: "# New\n", + wantStderr: "warning: THIRD_PARTY_LICENSES.md is out of date", + }, + { + name: "missing file is an error", + current: nil, + detector: reportBody, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + outFile := filepath.Join(dir, "THIRD_PARTY_LICENSES.md") + if tt.current != nil { + fatalOnErr(t, os.WriteFile(outFile, []byte(*tt.current), 0o644)) + } + fake := &fakeProcesses{detectorOut: tt.detector} + var stderr bytes.Buffer + g := generator{root: dir, runCmd: fake.run, stderr: &stderr} + + err := g.checkInSync(filepath.Join(dir, "hack", "licenses"), outFile, []byte("{}\n")) + + if tt.wantErr { + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("err = %v, want missing-file error", err) + } + return + } + fatalOnErr(t, err) + if !strings.Contains(stderr.String(), tt.wantStderr) { + t.Errorf("stderr = %q, want it to contain %q", stderr.String(), tt.wantStderr) + } + }) + } +} diff --git a/hack/licenses/fake_runner_test.go b/hack/licenses/fake_runner_test.go new file mode 100644 index 0000000000..8b1e6041bd --- /dev/null +++ b/hack/licenses/fake_runner_test.go @@ -0,0 +1,80 @@ +package main + +import ( + "fmt" + "io" + "os" + "os/exec" + "strings" + "testing" +) + +// fakeProcesses stands in for the go and git subprocesses so tests run +// without a toolchain or network. go list output is keyed by "GOOS/GOARCH"; +// unlisted targets produce no output. The detector writes detectorOut to its +// -depsOut path and records the stdin it was given. +type fakeProcesses struct { + listOut map[string]string + listErr map[string]error + detectorOut string + detectorIn []byte +} + +func (f *fakeProcesses) run(cmd *exec.Cmd) error { + switch { + case cmd.Args[0] == "git": + return nil + case cmd.Args[1] == "list": + key := envValue(cmd.Env, "GOOS") + "/" + envValue(cmd.Env, "GOARCH") + if err := f.listErr[key]; err != nil { + return err + } + _, err := io.WriteString(cmd.Stdout, f.listOut[key]) + return err + default: + in, err := io.ReadAll(cmd.Stdin) + if err != nil { + return err + } + f.detectorIn = in + out := argValue(cmd.Args, "-depsOut") + if out == "" { + return fmt.Errorf("detector invoked without -depsOut") + } + return os.WriteFile(out, []byte(f.detectorOut), 0o644) + } +} + +// envValue returns the last value for key, matching exec's override order. +func envValue(env []string, key string) string { + val := "" + for _, kv := range env { + if k, v, ok := strings.Cut(kv, "="); ok && k == key { + val = v + } + } + return val +} + +// argValue returns the argument following flag, or "" if absent. +func argValue(args []string, flag string) string { + for i := 0; i+1 < len(args); i++ { + if args[i] == flag { + return args[i+1] + } + } + return "" +} + +// lines joins its arguments with newlines and appends a trailing newline. +func lines(parts ...string) string { + return strings.Join(parts, "\n") + "\n" +} + +// fatalOnErr fails the test when err is non-nil. +func fatalOnErr(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} diff --git a/hack/licenses/main.go b/hack/licenses/main.go index 0217c2ca4a..ad267105e5 100644 --- a/hack/licenses/main.go +++ b/hack/licenses/main.go @@ -1,16 +1,12 @@ package main import ( - "bytes" - "encoding/json" "flag" "fmt" "io" - "maps" "os" "os/exec" "path/filepath" - "sort" ) const detectorVersion = "v0.10.0" @@ -29,6 +25,21 @@ var releaseTargets = []struct{ os, arch string }{ {"windows", archARM64}, } +// commandRunner starts a subprocess. execCommand is used in production; tests +// substitute a fake so that neither go nor git runs. +type commandRunner func(cmd *exec.Cmd) error + +func execCommand(cmd *exec.Cmd) error { + return cmd.Run() +} + +// generator carries the repository root and the process seams for one run. +type generator struct { + root string + runCmd commandRunner + stderr io.Writer +} + func main() { check := flag.Bool("check", false, "verify THIRD_PARTY_LICENSES.md is in sync instead of writing it") @@ -41,76 +52,41 @@ func main() { } func run(check bool) error { - root, err := repoRoot() + cwd, err := os.Getwd() if err != nil { return err } - - hackDir := filepath.Join(root, "hack", "licenses") - outFile := filepath.Join(root, "THIRD_PARTY_LICENSES.md") - - deps, err := moduleListJSON(root) + root, err := repoRoot(cwd) if err != nil { - return fmt.Errorf("collecting module dependencies: %w", err) - } - - if check { - return checkInSync(hackDir, outFile, deps) - } - - fmt.Fprintln(os.Stderr, "Generating", filepath.Base(outFile)+"...") - if err := runDetector(hackDir, deps, outFile); err != nil { return err } - fmt.Fprintln(os.Stderr, "Wrote", outFile) - return nil + g := generator{root: root, runCmd: execCommand, stderr: os.Stderr} + return g.generate(check) } -func checkInSync(hackDir, outFile string, deps []byte) error { - tmp, err := os.CreateTemp("", "third-party-licenses-*.md") +func (g generator) generate(check bool) error { + hackDir := filepath.Join(g.root, "hack", "licenses") + outFile := filepath.Join(g.root, "THIRD_PARTY_LICENSES.md") + + deps, err := g.moduleListJSON() if err != nil { - return err - } - target := tmp.Name() - if err := tmp.Close(); err != nil { - return err + return fmt.Errorf("collecting module dependencies: %w", err) } - defer func() { _ = os.Remove(target) }() - fmt.Fprintln(os.Stderr, "Generating", filepath.Base(outFile)+"...") - if err := runDetector(hackDir, deps, target); err != nil { - return err + if check { + return g.checkInSync(hackDir, outFile, deps) } - current, err := os.ReadFile(outFile) - if err != nil { - return fmt.Errorf("reading %s: %w", outFile, err) - } - generated, err := os.ReadFile(target) - if err != nil { + _, _ = fmt.Fprintln(g.stderr, "Generating", filepath.Base(outFile)+"...") + if err := g.runDetector(hackDir, deps, outFile); err != nil { return err } - if !bytes.Equal(current, generated) { - diff := exec.Command("git", "--no-pager", "diff", "--no-index", "--", outFile, target) - diff.Stdout = os.Stderr - diff.Stderr = os.Stderr - _ = diff.Run() - fmt.Fprintln( - os.Stderr, - "warning: THIRD_PARTY_LICENSES.md is out of date; run 'mise run cli:licenses' and commit the result", - ) - return nil - } - - fmt.Fprintln(os.Stderr, "THIRD_PARTY_LICENSES.md is up to date.") + _, _ = fmt.Fprintln(g.stderr, "Wrote", outFile) return nil } -func repoRoot() (string, error) { - dir, err := os.Getwd() - if err != nil { - return "", err - } +// repoRoot returns the nearest directory at or above dir that contains go.mod. +func repoRoot(dir string) (string, error) { for { if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { return dir, nil @@ -122,106 +98,3 @@ func repoRoot() (string, error) { dir = parent } } - -type goModule struct { - Path string - Version string - Main bool - Dir string -} - -type goPackage struct { - Module *goModule -} - -func moduleListJSON(dir string) ([]byte, error) { - modules := map[string]goModule{} - for _, t := range releaseTargets { - mods, err := buildModules(dir, t.os, t.arch) - if err != nil { - return nil, fmt.Errorf("listing deps for %s/%s: %w", t.os, t.arch, err) - } - maps.Copy(modules, mods) - } - - paths := make([]string, 0, len(modules)) - for p := range modules { - paths = append(paths, p) - } - sort.Strings(paths) - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - enc.SetIndent("", "\t") - for _, p := range paths { - if err := enc.Encode(modules[p]); err != nil { - return nil, err - } - } - return buf.Bytes(), nil -} - -func buildModules(dir, goos, goarch string) (map[string]goModule, error) { - cmd := exec.Command("go", "list", "-deps", "-mod=readonly", "-json", "./...") - cmd.Dir = dir - cmd.Env = append(os.Environ(), "GOOS="+goos, "GOARCH="+goarch) - cmd.Stderr = os.Stderr - out, err := cmd.Output() - if err != nil { - return nil, err - } - - modules := map[string]goModule{} - dec := json.NewDecoder(bytes.NewReader(out)) - for { - var pkg goPackage - if err := dec.Decode(&pkg); err != nil { - if err == io.EOF { - break - } - return nil, err - } - m := pkg.Module - if m == nil || m.Main || m.Path == "" { - continue - } - modules[m.Path] = *m - } - return modules, nil -} - -func runDetector(hackDir string, deps []byte, outPath string) error { - tmpDir, err := os.MkdirTemp("", "go-licence-detector-") - if err != nil { - return err - } - defer func() { _ = os.RemoveAll(tmpDir) }() - - cmd := exec.Command("go", "run", - "go.elastic.co/go-licence-detector@"+detectorVersion, - "-rules", filepath.Join(hackDir, "rules.json"), - "-overrides", filepath.Join(hackDir, "overrides.ndjson"), - "-depsTemplate", filepath.Join(hackDir, "third-party-licenses.md.tmpl"), - "-depsOut", outPath, - ) - cmd.Dir = tmpDir - cmd.Stdin = bytes.NewReader(deps) - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return fmt.Errorf("go-licence-detector: %w", err) - } - - return normalizeTrailingNewline(outPath) -} - -func normalizeTrailingNewline(path string) error { - content, err := os.ReadFile(path) - if err != nil { - return err - } - normalized := append(bytes.TrimRight(content, "\n"), '\n') - if bytes.Equal(content, normalized) { - return nil - } - return os.WriteFile(path, normalized, 0o644) -} diff --git a/hack/licenses/main_test.go b/hack/licenses/main_test.go new file mode 100644 index 0000000000..933a50031d --- /dev/null +++ b/hack/licenses/main_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestRepoRoot(t *testing.T) { + root := t.TempDir() + nested := filepath.Join(root, "hack", "licenses") + fatalOnErr(t, os.MkdirAll(nested, 0o755)) + fatalOnErr(t, os.WriteFile(filepath.Join(root, "go.mod"), + []byte("module example.com/x\n"), 0o644)) + + tests := []struct { + name string + start string + want string + wantErr bool + }{ + {name: "start at root", start: root, want: root}, + {name: "start in nested directory", start: nested, want: root}, + {name: "no go.mod above start", start: t.TempDir(), wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := repoRoot(tt.start) + if (err != nil) != tt.wantErr { + t.Fatalf("err = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Errorf("repoRoot(%q) = %q, want %q", tt.start, got, tt.want) + } + }) + } +} diff --git a/hack/licenses/modules.go b/hack/licenses/modules.go new file mode 100644 index 0000000000..fee1b597b7 --- /dev/null +++ b/hack/licenses/modules.go @@ -0,0 +1,93 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "os" + "os/exec" + "sort" +) + +type goModule struct { + Path string + Version string + Main bool + Dir string +} + +type goPackage struct { + Module *goModule +} + +// moduleListJSON merges third-party modules from every release target and +// returns them as the sorted, indented JSON stream fed to the detector. +func (g generator) moduleListJSON() ([]byte, error) { + modules := map[string]goModule{} + for _, t := range releaseTargets { + mods, err := g.buildModules(t.os, t.arch) + if err != nil { + return nil, fmt.Errorf("listing deps for %s/%s: %w", t.os, t.arch, err) + } + maps.Copy(modules, mods) + } + return encodeModules(modules) +} + +// encodeModules writes modules sorted by path, one indented JSON object each. +func encodeModules(modules map[string]goModule) ([]byte, error) { + paths := make([]string, 0, len(modules)) + for p := range modules { + paths = append(paths, p) + } + sort.Strings(paths) + + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetIndent("", "\t") + for _, p := range paths { + if err := enc.Encode(modules[p]); err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} + +func (g generator) buildModules(goos, goarch string) (map[string]goModule, error) { + cmd := exec.Command("go", "list", "-deps", "-mod=readonly", "-json", "./...") + cmd.Dir = g.root + cmd.Env = append(os.Environ(), "GOOS="+goos, "GOARCH="+goarch) + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = g.stderr + if err := g.runCmd(cmd); err != nil { + return nil, err + } + return decodeModules(&out) +} + +// decodeModules reads a `go list -json` object stream and keeps third-party +// modules keyed by path. Packages without module info, main-module entries, +// and entries with an empty path are skipped. +func decodeModules(r io.Reader) (map[string]goModule, error) { + modules := map[string]goModule{} + dec := json.NewDecoder(r) + for { + var pkg goPackage + if err := dec.Decode(&pkg); err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, err + } + m := pkg.Module + if m == nil || m.Main || m.Path == "" { + continue + } + modules[m.Path] = *m + } + return modules, nil +} diff --git a/hack/licenses/modules_test.go b/hack/licenses/modules_test.go new file mode 100644 index 0000000000..1504081802 --- /dev/null +++ b/hack/licenses/modules_test.go @@ -0,0 +1,131 @@ +package main + +import ( + "encoding/json" + "io" + "maps" + "slices" + "strings" + "testing" +) + +// pkgStream encodes packages as `go list -deps -json` emits them. +func pkgStream(t *testing.T, pkgs ...goPackage) string { + t.Helper() + var b strings.Builder + for _, p := range pkgs { + data, err := json.Marshal(p) + fatalOnErr(t, err) + b.Write(data) + b.WriteString("\n") + } + return b.String() +} + +func mod(path, version string) *goModule { + return &goModule{Path: path, Version: version, Dir: "/gomod/" + path + "@" + version} +} + +func TestDecodeModules(t *testing.T) { + tests := []struct { + name string + stream string + want []string + wantErr bool + }{ + { + name: "keeps third-party modules only", + stream: pkgStream(t, + goPackage{Module: mod("a.example/a", "v1.0.0")}, + goPackage{}, + goPackage{Module: &goModule{Path: "example.com/main", Main: true}}, + goPackage{Module: &goModule{Path: ""}}, + ), + want: []string{"a.example/a"}, + }, + { + name: "empty stream yields no modules", + stream: "", + want: nil, + }, + { + name: "malformed json is an error", + stream: "{", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := decodeModules(strings.NewReader(tt.stream)) + if (err != nil) != tt.wantErr { + t.Fatalf("err = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if gotPaths := slices.Sorted(maps.Keys(got)); !slices.Equal(gotPaths, tt.want) { + t.Errorf("paths = %v, want %v", gotPaths, tt.want) + } + }) + } +} + +// TestModuleListJSONMergesTargetsDeterministically checks that per-target +// lists are unioned, main/empty entries never leak through, targets with no +// output contribute nothing, and output is sorted by path. +func TestModuleListJSONMergesTargetsDeterministically(t *testing.T) { + fake := &fakeProcesses{ + listOut: map[string]string{ + "linux/amd64": pkgStream(t, + goPackage{Module: mod("c.example/c", "v3.0.0")}, + goPackage{Module: mod("a.example/a", "v1.0.0")}, + ), + "linux/arm64": pkgStream(t, + goPackage{Module: mod("b.example/b", "v2.0.0")}, + goPackage{Module: mod("a.example/a", "v1.0.0")}, + goPackage{Module: &goModule{Path: "example.com/main", Main: true}}, + ), + "windows/amd64": pkgStream(t, goPackage{Module: mod("b.example/b", "v2.0.0")}), + }, + } + g := generator{root: t.TempDir(), runCmd: fake.run, stderr: io.Discard} + + got, err := g.moduleListJSON() + fatalOnErr(t, err) + + want := lines( + "{", + "\t\"Path\": \"a.example/a\",", + "\t\"Version\": \"v1.0.0\",", + "\t\"Main\": false,", + "\t\"Dir\": \"/gomod/a.example/a@v1.0.0\"", + "}", + "{", + "\t\"Path\": \"b.example/b\",", + "\t\"Version\": \"v2.0.0\",", + "\t\"Main\": false,", + "\t\"Dir\": \"/gomod/b.example/b@v2.0.0\"", + "}", + "{", + "\t\"Path\": \"c.example/c\",", + "\t\"Version\": \"v3.0.0\",", + "\t\"Main\": false,", + "\t\"Dir\": \"/gomod/c.example/c@v3.0.0\"", + "}", + ) + if string(got) != want { + t.Errorf("moduleListJSON =\n%s\nwant\n%s", got, want) + } +} + +func TestModuleListJSONReportsFailingTarget(t *testing.T) { + fake := &fakeProcesses{ + listErr: map[string]error{"darwin/arm64": io.ErrUnexpectedEOF}, + } + g := generator{root: t.TempDir(), runCmd: fake.run, stderr: io.Discard} + + _, err := g.moduleListJSON() + if err == nil || !strings.Contains(err.Error(), "listing deps for darwin/arm64") { + t.Fatalf("err = %v, want wrapped darwin/arm64 failure", err) + } +}