diff --git a/README.md b/README.md index b86befa..f6f883b 100644 --- a/README.md +++ b/README.md @@ -136,41 +136,8 @@ existing result changes or a new difference appears. ## Benchmarks -Run the matching benchmarks with: - -```bash -GOMAXPROCS=1 go test \ - -run '^$' \ - -bench . \ - -benchmem \ - -benchtime 1s \ - -count 5 \ - . -``` - -Run the repository scan benchmark with: - -```bash -go test ./cmd/licenses \ - -run '^$' \ - -bench '^BenchmarkScanRepository$' \ - -benchmem \ - -count 5 -``` - -To benchmark local checkouts with the same scan defaults: - -```bash -LICENSES_BENCH_REPOS=/path/to/repo1:/path/to/repo2 \ - go test ./cmd/licenses \ - -run '^$' \ - -bench '^BenchmarkScanRepositories$' \ - -benchmem \ - -count 5 -``` - -Use the standard `go test` flags to select benchmarks or change their duration -and sample count. +See [docs/benchmarking.md](docs/benchmarking.md) for matcher, repository +scanner, and Licensee comparison benchmarks. ## License diff --git a/cmd/licenses/benchmark_test.go b/cmd/licenses/benchmark_test.go index c5a9104..9932bea 100644 --- a/cmd/licenses/benchmark_test.go +++ b/cmd/licenses/benchmark_test.go @@ -1,9 +1,17 @@ package main import ( + "bytes" "context" + "encoding/json" + "errors" + "fmt" + "io" "os" + "os/exec" "path/filepath" + "reflect" + "runtime" "strconv" "strings" "testing" @@ -55,10 +63,7 @@ func BenchmarkScanRepository(b *testing.B) { } func BenchmarkScanRepositories(b *testing.B) { - value := os.Getenv("LICENSES_BENCH_REPOS") - if value == "" { - b.Skip("set LICENSES_BENCH_REPOS to a path-list of repositories") - } + roots := benchmarkRepositoryRoots(b) matcher, err := licenses.New() if err != nil { b.Fatal(err) @@ -69,11 +74,7 @@ func BenchmarkScanRepositories(b *testing.B) { MaxFileSize: defaultMaxFileSize, Workers: defaultWorkerCount(), } - for _, root := range filepath.SplitList(value) { - root = strings.TrimSpace(root) - if root == "" { - continue - } + for _, root := range roots { b.Run(filepath.Base(filepath.Clean(root)), func(b *testing.B) { report, err := scanRepository( context.Background(), @@ -84,8 +85,6 @@ func BenchmarkScanRepositories(b *testing.B) { if err != nil { b.Fatal(err) } - b.SetBytes(report.Summary.BytesScanned) - b.ReportMetric(float64(report.Summary.FilesScanned), "files/op") b.ResetTimer() for b.Loop() { benchmarkScanReport, err = scanRepository( @@ -98,6 +97,223 @@ func BenchmarkScanRepositories(b *testing.B) { b.Fatal(err) } } + b.StopTimer() + b.SetBytes(report.Summary.BytesScanned) + b.ReportMetric(float64(report.Summary.FilesScanned), "files/op") + }) + } +} + +func BenchmarkRepositoryCLIs(b *testing.B) { + roots := benchmarkRepositoryRoots(b) + licensesBinary := os.Getenv("LICENSES_BENCH_BIN") + if licensesBinary == "" { + b.Skip("set LICENSES_BENCH_BIN to a built cmd/licenses binary") + } + licenseeBinary := os.Getenv("LICENSEE_BENCH_BIN") + if licenseeBinary == "" { + var err error + licenseeBinary, err = exec.LookPath("licensee") + if err != nil { + b.Skip("licensee is not installed and LICENSEE_BENCH_BIN is unset") + } + } + + b.Run("licenses", func(b *testing.B) { + benchmarkLicensesCLI(b, licensesBinary, roots) + }) + + b.Run("licensee", func(b *testing.B) { + benchmarkLicenseeCLI(b, licenseeBinary, roots) + }) +} + +func benchmarkLicensesCLI(b *testing.B, binary string, roots []string) { + b.Helper() + for _, root := range roots { + b.Run(filepath.Base(filepath.Clean(root)), func(b *testing.B) { + var output bytes.Buffer + peakRSS, err := runLicensesCLI( + b.Context(), + binary, + root, + &output, + ) + if err != nil { + b.Fatal(err) + } + var report scanReport + if err := json.Unmarshal(output.Bytes(), &report); err != nil { + b.Fatalf("decode licenses output: %v", err) + } + b.ResetTimer() + for b.Loop() { + if _, err := runLicensesCLI( + b.Context(), + binary, + root, + io.Discard, + ); err != nil { + b.Fatal(err) + } + } + b.StopTimer() + b.ReportMetric(float64(report.Summary.FilesScanned), "files/op") + b.ReportMetric( + float64(report.Summary.FilesWithDetections), + "detected_files/op", + ) + b.ReportMetric(bytesToMiB(peakRSS), "peak_MiB") + b.ReportMetric(bytesToMiB(int64(output.Len())), "output_MiB") }) } } + +func benchmarkLicenseeCLI(b *testing.B, binary string, roots []string) { + b.Helper() + for _, root := range roots { + b.Run(filepath.Base(filepath.Clean(root)), func(b *testing.B) { + var output bytes.Buffer + peakRSS, err := runLicenseeCLI( + b.Context(), + binary, + root, + &output, + ) + if err != nil { + b.Fatal(err) + } + var report licenseeReport + if output.Len() > 0 { + if err := json.Unmarshal(output.Bytes(), &report); err != nil { + b.Fatalf("decode Licensee output: %v", err) + } + } + b.ResetTimer() + for b.Loop() { + if _, err := runLicenseeCLI( + b.Context(), + binary, + root, + io.Discard, + ); err != nil { + b.Fatal(err) + } + } + b.StopTimer() + b.ReportMetric(float64(len(report.MatchedFiles)), "matched_files/op") + b.ReportMetric(float64(len(report.Licenses)), "licenses/op") + b.ReportMetric(bytesToMiB(peakRSS), "peak_MiB") + b.ReportMetric(bytesToMiB(int64(output.Len())), "output_MiB") + }) + } +} + +func benchmarkRepositoryRoots(b *testing.B) []string { + b.Helper() + value := os.Getenv("LICENSES_BENCH_REPOS") + if value == "" { + b.Skip("set LICENSES_BENCH_REPOS to a path-list of repositories") + } + var roots []string + for _, root := range filepath.SplitList(value) { + root = strings.TrimSpace(root) + if root != "" { + roots = append(roots, root) + } + } + if len(roots) == 0 { + b.Skip("LICENSES_BENCH_REPOS contains no repository paths") + } + return roots +} + +func runLicensesCLI( + ctx context.Context, + binary string, + root string, + stdout io.Writer, +) (int64, error) { + command := exec.CommandContext(ctx, binary, "-json", root) + command.Stdout = stdout + var stderr bytes.Buffer + command.Stderr = &stderr + if err := command.Run(); err != nil { + var exitError *exec.ExitError + if errors.As(err, &exitError) && exitError.ExitCode() == exitNoDetections { + return processMaxRSS(command.ProcessState), nil + } + return 0, fmt.Errorf( + "run licenses for %s: %w: %s", + root, + err, + strings.TrimSpace(stderr.String()), + ) + } + return processMaxRSS(command.ProcessState), nil +} + +func runLicenseeCLI( + ctx context.Context, + binary string, + root string, + stdout io.Writer, +) (int64, error) { + command := exec.CommandContext( + ctx, + binary, + "detect", + "--json", + "--filesystem", + root, + ) + command.Stdout = stdout + var stderr bytes.Buffer + command.Stderr = &stderr + if err := command.Run(); err != nil { + var exitError *exec.ExitError + if errors.As(err, &exitError) && exitError.ExitCode() == 1 { + return processMaxRSS(command.ProcessState), nil + } + return 0, fmt.Errorf( + "run Licensee for %s: %w: %s", + root, + err, + strings.TrimSpace(stderr.String()), + ) + } + return processMaxRSS(command.ProcessState), nil +} + +func processMaxRSS(state *os.ProcessState) int64 { + if state == nil { + return 0 + } + usage := reflect.ValueOf(state.SysUsage()) + if !usage.IsValid() { + return 0 + } + if usage.Kind() == reflect.Pointer { + usage = usage.Elem() + } + if !usage.IsValid() || usage.Kind() != reflect.Struct { + return 0 + } + maxRSS := usage.FieldByName("Maxrss") + if !maxRSS.IsValid() || !maxRSS.CanInt() { + return 0 + } + return normalizeMaxRSS(runtime.GOOS, maxRSS.Int()) +} + +func normalizeMaxRSS(goos string, value int64) int64 { + if goos == "linux" { + return value * 1024 + } + return value +} + +func bytesToMiB(value int64) float64 { + const mebibyte = 1 << 20 + return float64(value) / mebibyte +} diff --git a/cmd/licenses/licensee_comparison_test.go b/cmd/licenses/licensee_comparison_test.go new file mode 100644 index 0000000..e497a7a --- /dev/null +++ b/cmd/licenses/licensee_comparison_test.go @@ -0,0 +1,326 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + pathpkg "path" + "path/filepath" + "slices" + "strings" + "testing" + + licenses "github.com/git-pkgs/licenses" + "github.com/git-pkgs/licenses/internal/corpus" + gitspdx "github.com/git-pkgs/spdx" +) + +type licenseeReport struct { + Licenses []licenseeLicense `json:"licenses"` + MatchedFiles []licenseeMatchedFile `json:"matched_files"` +} + +type licenseeLicense struct { + Key string `json:"key"` + SPDXID string `json:"spdx_id"` +} + +type licenseeMatchedFile struct { + Filename string `json:"filename"` + MatchedLicense string `json:"matched_license"` + Matcher struct { + Name string `json:"name"` + } `json:"matcher"` +} + +type licenseeComparison struct { + ProjectText []string + ProjectClues []string + OtherText []string + OtherClues []string + LicenseeText []string + LicenseeFields []string +} + +func TestCompareLicenseeRepositories(t *testing.T) { + value := os.Getenv("LICENSES_BENCH_REPOS") + if value == "" { + t.Skip("set LICENSES_BENCH_REPOS to a path-list of repositories") + } + licenseeBinary := os.Getenv("LICENSEE_BENCH_BIN") + if licenseeBinary == "" { + var err error + licenseeBinary, err = exec.LookPath("licensee") + if err != nil { + t.Skip("licensee is not installed and LICENSEE_BENCH_BIN is unset") + } + } + index, err := corpus.Load() + if err != nil { + t.Fatal(err) + } + spdxKeys := index.SPDXKeys + index = corpus.Index{} + matcher, err := licenses.New() + if err != nil { + t.Fatal(err) + } + options := scanOptions{ + MaxDepth: defaultMaxDepth, + MaxFiles: defaultMaxFiles, + MaxFileSize: defaultMaxFileSize, + Workers: defaultWorkerCount(), + } + + var repositories, exactAgreements int + for _, root := range filepath.SplitList(value) { + root = strings.TrimSpace(root) + if root == "" { + continue + } + repositories++ + ours, err := scanRepository( + context.Background(), + matcher, + root, + options, + ) + if err != nil { + t.Fatalf("scan %s: %v", root, err) + } + var raw strings.Builder + if _, err := runLicenseeCLI( + context.Background(), + licenseeBinary, + root, + &raw, + ); err != nil { + t.Fatal(err) + } + var licensee licenseeReport + if raw.Len() > 0 { + if err := json.Unmarshal([]byte(raw.String()), &licensee); err != nil { + t.Fatalf("decode Licensee output for %s: %v", root, err) + } + } + + comparison := compareLicenseResults(ours, licensee, spdxKeys) + if slices.Equal(comparison.ProjectText, comparison.LicenseeText) { + exactAgreements++ + } + t.Logf( + "%s\n licenses project text: %s\n"+ + " licenses project clues: %s\n"+ + " licenses other text: %s\n"+ + " licenses other clues: %s\n"+ + " Licensee text: %s\n Licensee manifest fields: %s", + root, + formatComparisonValues(comparison.ProjectText), + formatComparisonValues(comparison.ProjectClues), + formatComparisonValues(comparison.OtherText), + formatComparisonValues(comparison.OtherClues), + formatComparisonValues(comparison.LicenseeText), + formatComparisonValues(comparison.LicenseeFields), + ) + } + if repositories == 0 { + t.Fatal("LICENSES_BENCH_REPOS contains no repository paths") + } + t.Logf( + "text identifier agreement: %d/%d repositories", + exactAgreements, + repositories, + ) +} + +func compareLicenseResults( + ours scanReport, + licensee licenseeReport, + spdxKeys map[string]string, +) licenseeComparison { + var result licenseeComparison + for _, file := range ours.Files { + projectFile := isComparisonProjectFile(file.Path) + for _, detection := range file.Detections { + for _, match := range detection.Matches { + if projectFile { + result.ProjectText = append( + result.ProjectText, + match.LicenseIDs..., + ) + } else { + result.OtherText = append( + result.OtherText, + match.LicenseIDs..., + ) + } + } + } + for _, clue := range file.Clues { + if projectFile { + result.ProjectClues = append( + result.ProjectClues, + clue.LicenseIDs..., + ) + } else { + result.OtherClues = append( + result.OtherClues, + clue.LicenseIDs..., + ) + } + } + } + for _, file := range licensee.MatchedFiles { + expression := normalizeLicenseeExpression(file.MatchedLicense, spdxKeys) + if expression == "" { + continue + } + switch file.Matcher.Name { + case "exact", "reference": + result.LicenseeText = append(result.LicenseeText, expression) + default: + result.LicenseeFields = append(result.LicenseeFields, expression) + } + } + result.ProjectText = sortedUnique(result.ProjectText) + result.ProjectClues = sortedUnique(result.ProjectClues) + result.OtherText = sortedUnique(result.OtherText) + result.OtherClues = sortedUnique(result.OtherClues) + result.LicenseeText = sortedUnique(result.LicenseeText) + result.LicenseeFields = sortedUnique(result.LicenseeFields) + return result +} + +func isComparisonProjectFile(filePath string) bool { + cleaned := strings.TrimPrefix(filepath.ToSlash(filePath), "./") + if strings.Contains(cleaned, "/") { + return false + } + if isLegalFile(filePath) { + return true + } + name := strings.ToLower(pathpkg.Base(cleaned)) + return name == "readme" || strings.HasPrefix(name, "readme.") +} + +func normalizeLicenseeExpression( + raw string, + spdxKeys map[string]string, +) string { + expression, err := gitspdx.ParseStrict(raw) + if err != nil { + return raw + } + return gitspdx.RewriteIdentifiers(expression, func(identifier string) string { + if key := spdxKeys[strings.ToLower(identifier)]; key != "" { + return key + } + return identifier + }) +} + +func sortedUnique(values []string) []string { + slices.Sort(values) + return slices.Compact(values) +} + +func formatComparisonValues(values []string) string { + if len(values) == 0 { + return "(none)" + } + const sampleSize = 20 + if len(values) <= sampleSize { + return strings.Join(values, ", ") + } + return fmt.Sprintf( + "%s ... (%d total)", + strings.Join(values[:sampleSize], ", "), + len(values), + ) +} + +func TestCompareLicenseResultsSeparatesLicenseeSources(t *testing.T) { + var ours scanReport + ours.Files = []fileRecord{{ + Path: "LICENSE", + Detections: []detectionRecord{{ + Matches: []matchRecord{{LicenseIDs: []string{"mit"}}}, + }}, + Clues: []matchRecord{{LicenseIDs: []string{"ruby"}}}, + }} + exact := licenseeMatchedFile{MatchedLicense: "MIT"} + exact.Matcher.Name = "exact" + declared := licenseeMatchedFile{MatchedLicense: "Apache-2.0"} + declared.Matcher.Name = "gemspec" + licensee := licenseeReport{ + MatchedFiles: []licenseeMatchedFile{exact, declared}, + } + + got := compareLicenseResults( + ours, + licensee, + map[string]string{ + "mit": "mit", + "apache-2.0": "apache-2.0", + }, + ) + if !slices.Equal(got.ProjectText, []string{"mit"}) { + t.Errorf("project text = %v, want [mit]", got.ProjectText) + } + if !slices.Equal(got.ProjectClues, []string{"ruby"}) { + t.Errorf("project clues = %v, want [ruby]", got.ProjectClues) + } + if !slices.Equal(got.LicenseeText, []string{"mit"}) { + t.Errorf("Licensee text = %v, want [mit]", got.LicenseeText) + } + if !slices.Equal(got.LicenseeFields, []string{"apache-2.0"}) { + t.Errorf( + "Licensee manifest fields = %v, want [apache-2.0]", + got.LicenseeFields, + ) + } +} + +func TestIsComparisonProjectFile(t *testing.T) { + tests := map[string]bool{ + "LICENSE": true, + "LICENSES/MIT.txt": false, + "docs/README.md": false, + "README": true, + "docs/licensing.md": false, + "src/license_check.go": false, + } + for filePath, want := range tests { + if got := isComparisonProjectFile(filePath); got != want { + t.Errorf( + "isComparisonProjectFile(%q) = %t, want %t", + filePath, + got, + want, + ) + } + } +} + +func TestNormalizeLicenseeExpression(t *testing.T) { + keys := map[string]string{ + "mit": "mit", + "bsd-3-clause": "bsd-new", + } + got := normalizeLicenseeExpression("MIT OR BSD-3-Clause", keys) + if got != "mit OR bsd-new" { + t.Errorf("normalized expression = %q, want %q", got, "mit OR bsd-new") + } +} + +func TestNormalizeMaxRSS(t *testing.T) { + const value = int64(123) + if got := normalizeMaxRSS("linux", value); got != value*1024 { + t.Errorf("Linux max RSS = %d, want %d", got, value*1024) + } + if got := normalizeMaxRSS("darwin", value); got != value { + t.Errorf("Darwin max RSS = %d, want %d", got, value) + } +} diff --git a/docs/benchmarking.md b/docs/benchmarking.md new file mode 100644 index 0000000..b445493 --- /dev/null +++ b/docs/benchmarking.md @@ -0,0 +1,82 @@ +# Benchmarking + +The benchmarks use Go's `testing` package. Record the Go version, operating +system, corpus commit, and Licensee version with published results. + +## Matcher + +Run the byte-slice matching benchmarks on one logical processor: + +```bash +GOMAXPROCS=1 go test \ + -run '^$' \ + -bench . \ + -benchmem \ + -benchtime 1s \ + -count 5 \ + . +``` + +Cold `New` and repeated `New` are separate benchmarks. Matching benchmarks +reuse one loaded corpus. + +## Repository scanner + +Run the generated repository benchmark with: + +```bash +go test ./cmd/licenses \ + -run '^$' \ + -bench '^BenchmarkScanRepository$' \ + -benchmem \ + -count 5 +``` + +Set `LICENSES_BENCH_REPOS` to benchmark local checkouts with the scanner's +default project scope: + +```bash +LICENSES_BENCH_REPOS=/path/to/repo1:/path/to/repo2 \ + go test ./cmd/licenses \ + -run '^$' \ + -bench '^BenchmarkScanRepositories$' \ + -benchmem \ + -count 5 +``` + +`BenchmarkScanRepositories` measures warm in-process scans. Every repository +uses the same loaded matcher. + +## Licensee comparison + +Licensee must be installed before running the comparison. Build the current +`licenses` command once, then provide both the binary and repository list: + +```bash +go build -o /tmp/licenses-bench ./cmd/licenses + +export LICENSES_BENCH_REPOS=/path/to/repo1:/path/to/repo2 +export LICENSES_BENCH_BIN=/tmp/licenses-bench + +go test ./cmd/licenses \ + -run '^TestCompareLicenseeRepositories$' \ + -v + +go test ./cmd/licenses \ + -run '^$' \ + -bench '^BenchmarkRepositoryCLIs$' \ + -benchtime 3x \ + -count 5 +``` + +Set `LICENSEE_BENCH_BIN` when `licensee` is not on `PATH`. The result comparison +uses root legal files and README files for the direct text comparison. +Detections elsewhere in the repository and Licensee's manifest-derived +licenses are printed separately. + +The command benchmark includes process startup and JSON encoding for both +tools. It reports output size and peak resident memory from an untimed first +run. Allocation figures from `-benchmem` cover the Go benchmark driver, not +memory allocated by either child process. + +Use the standard `go test` flags to change benchmark duration and sample count.