diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 72ec7d2..8867075 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -75,6 +75,12 @@ jobs: mkdir -p dist go build -ldflags "-X main.Version=$VERSION" -o dist/compiledb ./cmd/compiledb ./dist/compiledb --help + mkdir -p dist/smoke + printf 'int main(void) { return 0; }\n' > dist/smoke/main.c + printf 'cc -c main.c -o main.o\n' > dist/smoke/build.log + (cd dist/smoke && ../compiledb --parse build.log) + grep -F '"file": "main.c"' dist/smoke/compile_commands.json + rm -rf dist/smoke tar cJf dist/compiledb-linux-amd64.txz -C dist compiledb - uses: actions/upload-artifact@v4 @@ -100,6 +106,12 @@ jobs: mkdir -p dist go build -ldflags "-X main.Version=$VERSION" -o dist/compiledb ./cmd/compiledb ./dist/compiledb --help + mkdir -p dist/smoke + printf 'int main(void) { return 0; }\n' > dist/smoke/main.c + printf 'cc -c main.c -o main.o\n' > dist/smoke/build.log + (cd dist/smoke && ../compiledb --parse build.log) + grep -F '"file": "main.c"' dist/smoke/compile_commands.json + rm -rf dist/smoke tar cJf dist/compiledb-linux-arm64.txz -C dist compiledb - uses: actions/upload-artifact@v4 @@ -128,6 +140,12 @@ jobs: mkdir -p dist go build -ldflags "-X main.Version=$VERSION" -o dist/compiledb.exe ./cmd/compiledb ./dist/compiledb.exe --help + mkdir -p dist/smoke + printf 'int main(void) { return 0; }\n' > dist/smoke/main.c + printf 'cc -c main.c -o main.o\n' > dist/smoke/build.log + (cd dist/smoke && ../compiledb.exe --parse build.log) + grep -F '"file": "main.c"' dist/smoke/compile_commands.json + rm -rf dist/smoke (cd dist && 7z a compiledb-windows-amd64.zip compiledb.exe) - uses: actions/upload-artifact@v4 @@ -153,6 +171,12 @@ jobs: mkdir -p dist go build -ldflags "-X main.Version=$VERSION" -o dist/compiledb ./cmd/compiledb ./dist/compiledb --help + mkdir -p dist/smoke + printf 'int main(void) { return 0; }\n' > dist/smoke/main.c + printf 'cc -c main.c -o main.o\n' > dist/smoke/build.log + (cd dist/smoke && ../compiledb --parse build.log) + grep -F '"file": "main.c"' dist/smoke/compile_commands.json + rm -rf dist/smoke tar cJf dist/compiledb-darwin-arm64.txz -C dist compiledb - uses: actions/upload-artifact@v4 diff --git a/README.md b/README.md index e66a633..36f8f8f 100644 --- a/README.md +++ b/README.md @@ -364,7 +364,8 @@ response files remain opaque. The parser applies fixed limits to individual physical lines and response-file expansion: -- A physical build-log line is limited to 100 MiB. +- A physical build-log line is limited to 100 MiB. An oversized line is + reported and skipped without stopping later lines from being parsed. - One response file is limited to 8 MiB. - Aggregate response-file input for one compiler command is limited to 32 MiB. - Response-file nesting is limited to 32 levels and 256 files. @@ -372,10 +373,6 @@ expansion: - Expanded arguments and their copies across multi-source entries have a 64 MiB budget. -These are not whole-process resource limits. The total build-log size and the -output produced by programs in backtick expressions do not have fixed limits; -large inputs or unbounded command output can consume substantial memory. - ## Related Projects - [nickdiego/compiledb][python-compiledb] is the Python project that originally @@ -383,21 +380,6 @@ large inputs or unbounded command output can consume substantial memory. - [clangd][clangd] and [clang-tidy][clang-tidy] are common consumers of the generated `compile_commands.json` file. -## Development - -Run the full local verification suite with: - -```sh -test -z "$(gofmt -l .)" -go vet ./... -go test ./... -go test -race ./... -go build -o /tmp/compiledb-go ./cmd/compiledb -``` - -Some tests modify the current directory, standard streams, and process-global -state, so those tests must not run in parallel. - ## License [GNU General Public License v3.0](LICENSE). diff --git a/internal/build_log_benchmark_test.go b/internal/build_log_benchmark_test.go index 2b4c4e7..6b3d09b 100644 --- a/internal/build_log_benchmark_test.go +++ b/internal/build_log_benchmark_test.go @@ -32,10 +32,7 @@ func BenchmarkBuildLogScan(b *testing.B) { b.ResetTimer() for range b.N { - lines, err := scanBuildLog(data) - if err != nil { - b.Fatal(err) - } + lines := scanBuildLog(data) runtime.KeepAlive(lines) } }) @@ -47,10 +44,7 @@ func BenchmarkBuildLogMergeLogicalLines(b *testing.B) { for _, size := range benchmarkBuildLogSizes { b.Run(size.name, func(b *testing.B) { data := buildLogBenchmarkData(size.size, chunk) - lines, err := scanBuildLog(data) - if err != nil { - b.Fatal(err) - } + lines := scanBuildLog(data) data = nil runtime.GC() @@ -58,10 +52,7 @@ func BenchmarkBuildLogMergeLogicalLines(b *testing.B) { b.ReportAllocs() b.ResetTimer() for range b.N { - logicalLines, issues := mergeLogicalLines(lines) - if len(issues) != 0 { - b.Fatalf("unexpected logical line issues: %#v", issues) - } + logicalLines := mergeLogicalLines(lines) runtime.KeepAlive(logicalLines) } }) @@ -127,11 +118,8 @@ func benchmarkDiscoveryOutput(b *testing.B, size int64, chunk []byte) { if err := writeBuildLogBenchmarkData(&stdout, size, chunk); err != nil { b.Fatal(err) } - buildLog, err := scanBuildLog(stdout.Bytes()) - if err != nil { - b.Fatal(err) - } - tool.Parse(buildLog) + buildLog := scanBuildLog(stdout.Bytes()) + tool.parseBuildLog(buildLog) if tool.StatusCode != 0 { b.Fatalf("parser status: %d", tool.StatusCode) } diff --git a/internal/init.go b/internal/init.go index d6e5f43..052aec0 100644 --- a/internal/init.go +++ b/internal/init.go @@ -1,7 +1,6 @@ package internal import ( - "bufio" "bytes" "context" "encoding/json" @@ -40,6 +39,7 @@ type Tool struct { predefinedMacros map[string][]string compilerCommand func(name string, arg ...string) *exec.Cmd compilerCommandContext func(context.Context, string, ...string) *exec.Cmd + buildLogLineLimit int makeDirectoryMarkers bool } @@ -60,6 +60,13 @@ func (t *Tool) operationContext() context.Context { return context.Background() } +func (t *Tool) physicalLineLimit() int { + if t.buildLogLineLimit > 0 { + return t.buildLogLineLimit + } + return maxBuildLogLineSize +} + func (t *Tool) compilationDatabaseBuildDir() string { if t.Config.BuildDir != "" { return compilationDatabaseBuildDir(t.Config.BuildDir) @@ -368,23 +375,42 @@ func (t *Tool) Generate() { t.Logger.Debugf("Build from stdin") } - buildLog, err := scanBuildLog(data) - if err != nil { - t.Logger.Fatalf("read build log failed: %v", err) + if t.operationContext().Err() != nil { + t.StatusCode = contextExitCode(t.operationContext()) + return } + buildLog := scanBuildLogWithLimit(data, t.physicalLineLimit()) if t.operationContext().Err() != nil { t.StatusCode = contextExitCode(t.operationContext()) return } - t.Parse(buildLog) + t.parseBuildLog(buildLog) } -func scanBuildLog(data []byte) ([]string, error) { - var lines []string - scanner := bufio.NewScanner(bytes.NewReader(data)) - scanner.Buffer(make([]byte, 1024*1024), maxBuildLogLineSize) - for scanner.Scan() { - lines = append(lines, scanner.Text()) +func scanBuildLog(data []byte) []buildLogLine { + return scanBuildLogWithLimit(data, maxBuildLogLineSize) +} + +func scanBuildLogWithLimit(data []byte, limit int) []buildLogLine { + lineCount := bytes.Count(data, []byte{'\n'}) + if len(data) > 0 && data[len(data)-1] != '\n' { + lineCount++ + } + lines := make([]buildLogLine, 0, lineCount) + for len(data) > 0 { + line := data + if newline := bytes.IndexByte(data, '\n'); newline >= 0 { + line = data[:newline] + data = data[newline+1:] + } else { + data = nil + } + line = bytes.TrimSuffix(line, []byte{'\r'}) + if len(line) > limit { + lines = append(lines, buildLogLine{raw: line, oversized: true, limit: limit}) + continue + } + lines = append(lines, buildLogLine{text: string(line)}) } - return lines, scanner.Err() + return lines } diff --git a/internal/init_test.go b/internal/init_test.go index a5e8ea4..b17d16c 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -1,6 +1,7 @@ package internal import ( + "bytes" "context" "encoding/json" "fmt" @@ -54,6 +55,66 @@ func TestGenerateFromStdinDoesNotPanic(t *testing.T) { tool.Generate() } +func TestScanBuildLogPhysicalLineLimit(t *testing.T) { + if reason := buildLogLineLimitReason(maxBuildLogLineSize); reason != "physical line exceeds 100 MiB limit" { + t.Fatalf("unexpected production line-limit diagnostic: %q", reason) + } + + const limit = 8 + for name, input := range map[string]string{ + "LF": strings.Repeat("x", limit) + "\n", + "CRLF": strings.Repeat("x", limit) + "\r\n", + "EOF": strings.Repeat("x", limit), + } { + t.Run(name, func(t *testing.T) { + lines := scanBuildLogWithLimit([]byte(input), limit) + if len(lines) != 1 || lines[0].oversized || lines[0].text != strings.Repeat("x", limit) { + t.Fatalf("line at limit was not preserved: %#v", lines) + } + }) + } + + lines := scanBuildLogWithLimit([]byte(strings.Repeat("x", limit+1)+"\r\nok\n"), limit) + if len(lines) != 2 || !lines[0].oversized || lines[0].limit != limit || lines[0].text != "" || + lines[1].oversized || lines[1].text != "ok" { + t.Fatalf("oversized line was not discarded cleanly: %#v", lines) + } +} + +func TestGenerateSkipsOversizedPhysicalLine(t *testing.T) { + projectDir := t.TempDir() + buildLog := filepath.Join(projectDir, "build.log") + outputFile := filepath.Join(projectDir, "compile_commands.json") + contents := strings.Repeat("x", 32) + "\\\ncc -c joined.c\ncc -c valid.c\n" + if err := os.WriteFile(buildLog, []byte(contents), 0o644); err != nil { + t.Fatalf("write build log failed: %v", err) + } + + var logs bytes.Buffer + tool := newTestTool(t, Config{ + InputFile: buildLog, + OutputFile: outputFile, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + tool.buildLogLineLimit = 32 + tool.Logger.SetLevel(log.ErrorLevel) + tool.Logger.SetOutput(&logs) + tool.Generate() + + commands := readCompilerTestCommands(t, outputFile) + if tool.StatusCode != 0 || len(commands) != 1 || commands[0].File != "valid.c" { + t.Fatalf("oversized build-log line stopped Generate: status=%d commands=%#v", tool.StatusCode, commands) + } + diagnostic := logs.String() + for _, want := range []string{"build log line 1", "cwd", "physical line exceeds 32 byte limit", "at byte 32"} { + if !strings.Contains(diagnostic, want) { + t.Fatalf("physical line diagnostic lacks %q: %q", want, diagnostic) + } + } +} + func TestGenerateResolvesResponseFileFromBuildLogDirectory(t *testing.T) { projectDir := t.TempDir() buildLog := filepath.Join(projectDir, "build.log") diff --git a/internal/make_wrap.go b/internal/make_wrap.go index 1320505..26baf6d 100644 --- a/internal/make_wrap.go +++ b/internal/make_wrap.go @@ -290,18 +290,12 @@ func (t *Tool) runDiscoveryMake(cmd *exec.Cmd) discoveryResult { return result } - buildLog, scanErr := scanBuildLog(stdoutBuf.Bytes()) - if scanErr != nil { - result.status = 1 - t.Logger.Errorf("read dry-run output failed: %v", scanErr) - return result - } clone := *t clone.makeDirectoryMarkers = true if !t.Config.NoBuild { clone.Logger = loggerAtLevel(t.Logger, logrus.ErrorLevel) } - clone.Parse(buildLog) + clone.parseBuildLog(scanBuildLogWithLimit(stdoutBuf.Bytes(), clone.physicalLineLimit())) result.parserStatus = clone.StatusCode return result } diff --git a/internal/make_wrap_test.go b/internal/make_wrap_test.go index 97ddb94..b399a76 100644 --- a/internal/make_wrap_test.go +++ b/internal/make_wrap_test.go @@ -43,6 +43,91 @@ func TestMakeWrapNoBuildStopsOnDryRunFailure(t *testing.T) { } } +func TestMakeWrapNoBuildDoesNotInvokeRealMake(t *testing.T) { + tmpDir := t.TempDir() + outputFile := filepath.Join(tmpDir, "compile_commands.json") + invocationsFile := filepath.Join(tmpDir, "invocations") + realInvoked := filepath.Join(tmpDir, "real-invoked") + script := filepath.Join(tmpDir, "fake-make.sh") + contents := `#!/bin/sh +printf '%s\n' "$*" >> ` + ShellJoinArgs([]string{invocationsFile}) + ` +case " $* " in + *" -Bnkw "*) echo 'cc -c no-build.c' ;; + *) : > ` + ShellJoinArgs([]string{realInvoked}) + ` ;; +esac +` + if err := os.WriteFile(script, []byte(contents), 0o755); err != nil { + t.Fatalf("write fake make failed: %v", err) + } + + oldMakePath := makePath + makePath = script + defer func() { makePath = oldMakePath }() + + tool := newTestTool(t, Config{ + OutputFile: outputFile, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoBuild: true, + NoStrict: true, + }) + tool.MakeWrap([]string{"target"}) + + commands := readCompilerTestCommands(t, outputFile) + if tool.StatusCode != 0 || len(commands) != 1 || commands[0].File != "no-build.c" { + t.Fatalf("no-build discovery failed: status=%d commands=%#v", tool.StatusCode, commands) + } + invocations, err := os.ReadFile(invocationsFile) + if err != nil { + t.Fatalf("read make invocations failed: %v", err) + } + if want := "target -Bnkw -j1 --print-directory\n"; string(invocations) != want { + t.Fatalf("expected exactly one discovery invocation %q, got %q", want, invocations) + } + if _, err := os.Stat(realInvoked); !os.IsNotExist(err) { + t.Fatalf("real Make was invoked with --no-build: %v", err) + } +} + +func TestMakeWrapSkipsOversizedDiscoveryLine(t *testing.T) { + tmpDir := t.TempDir() + outputFile := filepath.Join(tmpDir, "compile_commands.json") + script := filepath.Join(tmpDir, "fake-make.sh") + oversized := strings.Repeat("x", 32) + "\\" + contents := "#!/bin/sh\nprintf '%s\\n' " + ShellJoinArgs([]string{oversized, "cc -c joined.c", "cc -c valid.c"}) + "\n" + if err := os.WriteFile(script, []byte(contents), 0o755); err != nil { + t.Fatalf("write fake make failed: %v", err) + } + + oldMakePath := makePath + makePath = script + defer func() { makePath = oldMakePath }() + + var logs bytes.Buffer + tool := newTestTool(t, Config{ + OutputFile: outputFile, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoBuild: true, + NoStrict: true, + }) + tool.buildLogLineLimit = 32 + tool.Logger.SetLevel(logrus.ErrorLevel) + tool.Logger.SetOutput(&logs) + tool.MakeWrap(nil) + + commands := readCompilerTestCommands(t, outputFile) + if tool.StatusCode != 0 || len(commands) != 1 || commands[0].File != "valid.c" { + t.Fatalf("oversized discovery line stopped MakeWrap: status=%d commands=%#v", tool.StatusCode, commands) + } + diagnostic := logs.String() + for _, want := range []string{"build log line 1", "cwd", "physical line exceeds 32 byte limit", "at byte 32"} { + if !strings.Contains(diagnostic, want) { + t.Fatalf("physical line diagnostic lacks %q: %q", want, diagnostic) + } + } +} + func TestMakeWrapDoesNotParseDiscoveryStderr(t *testing.T) { tmpDir := t.TempDir() script := filepath.Join(tmpDir, "fake-make.sh") diff --git a/internal/parser.go b/internal/parser.go index d7db1c1..38ab037 100644 --- a/internal/parser.go +++ b/internal/parser.go @@ -2,6 +2,7 @@ package internal import ( "bytes" + "fmt" "os" "path" "path/filepath" @@ -36,8 +37,17 @@ type parserPatterns struct { } type logicalLine struct { - text string - line int + text string + line int + issue string + offset int +} + +type buildLogLine struct { + text string + raw []byte + oversized bool + limit int } type shellTokenizationError struct { @@ -225,12 +235,7 @@ func shellLexicalError(line string) *shellTokenizationError { return nil } -type logicalLineIssue struct { - line int - reason string -} - -func lineContinuation(line string, initialQuote byte, initialWordStarted bool) (string, bool, byte, bool) { +func lineContinuationState[T ~string | ~[]byte](line T, initialQuote byte, initialWordStarted bool) (bool, byte, bool) { quote := initialQuote escaped := false wordStarted := initialWordStarted || initialQuote != 0 @@ -258,7 +263,7 @@ func lineContinuation(line string, initialQuote byte, initialWordStarted bool) ( continue } if character == '#' && !wordStarted { - return line, false, 0, false + return false, 0, false } if character == '\'' || character == '"' || character == '`' { quote = character @@ -273,35 +278,67 @@ func lineContinuation(line string, initialQuote byte, initialWordStarted bool) ( } } if escaped && quote != '\'' { + return true, quote, wordStarted + } + return false, 0, false +} + +func lineContinuation(line string, initialQuote byte, initialWordStarted bool) (string, bool, byte, bool) { + continued, quote, wordStarted := lineContinuationState(line, initialQuote, initialWordStarted) + if continued { return line[:len(line)-1], true, quote, wordStarted } return line, false, 0, false } -func mergeLogicalLines(lines []string) ([]logicalLine, []logicalLineIssue) { +func mergeLogicalLines(lines []buildLogLine) []logicalLine { merged := make([]logicalLine, 0, len(lines)) - issues := []logicalLineIssue{} var builder strings.Builder var quote byte wordStarted := false continuationStart := 0 + discarding := false for index, line := range lines { - line = strings.TrimSuffix(line, "\r") - if len(line) > maxBuildLogLineSize { - start := index + 1 - if continuationStart != 0 { - start = continuationStart + if line.oversized { + var continued bool + var nextQuote byte + var nextWordStarted bool + if line.raw != nil { + continued, nextQuote, nextWordStarted = lineContinuationState(line.raw, quote, wordStarted) + } else { + continued, nextQuote, nextWordStarted = lineContinuationState(line.text, quote, wordStarted) } - issues = append(issues, logicalLineIssue{line: start, reason: "physical line exceeds 100 MiB limit"}) + merged = append(merged, logicalLine{ + line: index + 1, + issue: buildLogLineLimitReason(line.limit), + offset: line.limit, + }) builder.Reset() - quote = 0 - wordStarted = false continuationStart = 0 + discarding = continued + if continued { + quote = nextQuote + wordStarted = nextWordStarted + } else { + quote = 0 + wordStarted = false + } continue } - content, continued, nextQuote, nextWordStarted := lineContinuation(line, quote, wordStarted) + content, continued, nextQuote, nextWordStarted := lineContinuation(line.text, quote, wordStarted) + if discarding { + discarding = continued + if continued { + quote = nextQuote + wordStarted = nextWordStarted + } else { + quote = 0 + wordStarted = false + } + continue + } if continued { if continuationStart == 0 { continuationStart = index + 1 @@ -327,10 +364,18 @@ func mergeLogicalLines(lines []string) ([]logicalLine, []logicalLineIssue) { } if continuationStart != 0 { - issues = append(issues, logicalLineIssue{line: continuationStart, reason: "unterminated line continuation"}) + merged = append(merged, logicalLine{line: continuationStart, issue: "unterminated line continuation", offset: -1}) } - return merged, issues + return merged +} + +func buildLogLineLimitReason(limit int) string { + const mebibyte = 1024 * 1024 + if limit%mebibyte == 0 { + return fmt.Sprintf("physical line exceeds %d MiB limit", limit/mebibyte) + } + return fmt.Sprintf("physical line exceeds %d byte limit", limit) } func compilePatterns(cfg Config) (parserPatterns, error) { @@ -1435,6 +1480,20 @@ func (t *Tool) processCompileCommand(command string, workingDir string, line int } func (t *Tool) Parse(buildLog []string) { + lines := make([]buildLogLine, 0, len(buildLog)) + limit := t.physicalLineLimit() + for _, line := range buildLog { + line = strings.TrimSuffix(line, "\r") + if len(line) > limit { + lines = append(lines, buildLogLine{text: line, oversized: true, limit: limit}) + continue + } + lines = append(lines, buildLogLine{text: line}) + } + t.parseBuildLog(lines) +} + +func (t *Tool) parseBuildLog(buildLog []buildLogLine) { type directoryFrame struct { path string provisional bool @@ -1467,10 +1526,7 @@ func (t *Tool) Parse(buildLog []string) { dirStack := []directoryFrame{{path: workingDir}} virtualDirectories := make(map[string]struct{}) - logicalLines, lineIssues := mergeLogicalLines(buildLog) - for _, issue := range lineIssues { - t.Logger.Errorf("skip build log line %d: %s", issue.line, issue.reason) - } + logicalLines := mergeLogicalLines(buildLog) for _, logicalLine := range logicalLines { line := logicalLine.text lineNumber := logicalLine.line @@ -1478,6 +1534,20 @@ func (t *Tool) Parse(buildLog []string) { t.StatusCode = contextExitCode(t.operationContext()) return } + if logicalLine.issue != "" { + if logicalLine.offset >= 0 { + t.Logger.Errorf( + "skip build log line %d (cwd %q): %s at byte %d", + lineNumber, + workingDir, + logicalLine.issue, + logicalLine.offset, + ) + } else { + t.Logger.Errorf("skip build log line %d: %s", lineNumber, logicalLine.issue) + } + continue + } t.Logger.Debug("New command:", line) commandList, commandListErr := parseShellCommandList(line) diff --git a/internal/parser_test.go b/internal/parser_test.go index d974450..2fcf85d 100644 --- a/internal/parser_test.go +++ b/internal/parser_test.go @@ -130,6 +130,50 @@ func TestParsePreservesContinuationShellSemantics(t *testing.T) { } } +func TestParsePhysicalLineLimit(t *testing.T) { + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + buildDir := t.TempDir() + var logs bytes.Buffer + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + BuildDir: buildDir, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + tool.buildLogLineLimit = 32 + tool.Logger.SetLevel(logrus.ErrorLevel) + tool.Logger.SetOutput(&logs) + atLimit := "cc -c at-limit.c" + atLimit += strings.Repeat(" ", tool.buildLogLineLimit-len(atLimit)) + + tool.Parse([]string{ + atLimit, + "cc -DINTERRUPTED=1 \\", + strings.Repeat("x", 33), + "cc -c valid.c", + strings.Repeat("x", 32) + "\\", + "cc -c joined.c", + "cc -c after.c", + }) + + commands := readCompilerTestCommands(t, outputFile) + if tool.StatusCode != 0 || len(commands) != 3 || commands[0].File != "at-limit.c" || + commands[1].File != "valid.c" || commands[2].File != "after.c" { + t.Fatalf("physical line limit stopped parsing: status=%d commands=%#v", tool.StatusCode, commands) + } + diagnostic := logs.String() + for _, want := range []string{"build log line 3", trackedPathToSlash(buildDir), "physical line exceeds 32 byte limit", "at byte 32"} { + if !strings.Contains(diagnostic, want) { + t.Fatalf("physical line diagnostic lacks %q: %q", want, diagnostic) + } + } + if strings.Contains(diagnostic, strings.Repeat("x", 16)) { + t.Fatalf("physical line diagnostic exposed line contents: %q", diagnostic) + } +} + func TestParseCommandStyleQuotesArguments(t *testing.T) { tmpDir := t.TempDir() outputFile := filepath.Join(tmpDir, "compile_commands.json") diff --git a/internal/response_file_test.go b/internal/response_file_test.go index 3590704..9d6fb3b 100644 --- a/internal/response_file_test.go +++ b/internal/response_file_test.go @@ -240,3 +240,46 @@ func TestExpandCompilerResponseFilesRejectsCLModeFromResponseFile(t *testing.T) t.Fatalf("CL mode from response file was accepted: arguments=%#v error=%v", result, err) } } + +func TestExpandCompilerResponseFilesUsesOuterResponseQuoting(t *testing.T) { + workingDir := t.TempDir() + if err := os.WriteFile(filepath.Join(workingDir, "arguments.rsp"), []byte("--rsp-quoting=windows -c main.c"), 0o644); err != nil { + t.Fatalf("write response file failed: %v", err) + } + tool := newTestTool(t, Config{}) + + t.Run("final outer POSIX", func(t *testing.T) { + arguments := []string{"clang", "--rsp-quoting=windows", "--rsp-quoting=posix", "@arguments.rsp"} + result, err := tool.expandCompilerResponseFiles(arguments, parseCompilerInvocation(arguments), workingDir) + if err != nil { + t.Fatalf("final outer POSIX response quoting was rejected: %v", err) + } + want := []string{"clang", "--rsp-quoting=windows", "--rsp-quoting=posix", "--rsp-quoting=windows", "-c", "main.c"} + if !slices.Equal(result, want) { + t.Fatalf("unexpected expanded arguments:\nwant: %#v\ngot: %#v", want, result) + } + }) + + t.Run("final outer Windows", func(t *testing.T) { + arguments := []string{"clang", "--rsp-quoting=posix", "--rsp-quoting=windows", "@arguments.rsp"} + result, err := tool.expandCompilerResponseFiles(arguments, parseCompilerInvocation(arguments), workingDir) + if err != nil { + t.Fatalf("opaque Windows response quoting returned an error: %v", err) + } + if !slices.Equal(result, arguments) { + t.Fatalf("Windows-quoted response file was expanded:\nwant: %#v\ngot: %#v", arguments, result) + } + }) + + t.Run("response contents do not change tokenizer", func(t *testing.T) { + arguments := []string{"clang", "--rsp-quoting=posix", "@arguments.rsp"} + result, err := tool.expandCompilerResponseFiles(arguments, parseCompilerInvocation(arguments), workingDir) + if err != nil { + t.Fatalf("response-file option changed the outer tokenizer: %v", err) + } + want := []string{"clang", "--rsp-quoting=posix", "--rsp-quoting=windows", "-c", "main.c"} + if !slices.Equal(result, want) { + t.Fatalf("unexpected expanded arguments:\nwant: %#v\ngot: %#v", want, result) + } + }) +} diff --git a/justfile b/justfile index 4cffb99..86de986 100644 --- a/justfile +++ b/justfile @@ -9,6 +9,10 @@ default: build build: go run ./cmd/compiledb/main.go -v --full-path -p ./tests/build.log +[script] +test: + go test -count=1 ./... + [script] release: go install ./cmd/compiledb diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go new file mode 100644 index 0000000..bd0f7cf --- /dev/null +++ b/tests/e2e/e2e_test.go @@ -0,0 +1,452 @@ +package e2e_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strconv" + "strings" + "testing" + "time" +) + +const ( + e2eBuildTimeout = 2 * time.Minute + e2eCommandTimeout = 30 * time.Second + e2eCommandWaitDelay = 2 * time.Second + e2eHelperModeEnv = "COMPILEDB_E2E_HELPER_MODE" + e2eHelperPIDFileEnv = "COMPILEDB_E2E_HELPER_PID_FILE" +) + +type e2eHarness struct { + executable string +} + +type e2eCommandOutput struct { + stdout []byte + stderr []byte +} + +type e2eCommandResult struct { + output e2eCommandOutput + err error +} + +type e2eCompilationEntry struct { + Directory string `json:"directory"` + Command string `json:"command"` + Arguments []string `json:"arguments"` + File string `json:"file"` +} + +func TestE2ECommandWaitDelay(t *testing.T) { + switch os.Getenv(e2eHelperModeEnv) { + case "parent": + child := exec.Command(os.Args[0], "-test.run=^TestE2ECommandWaitDelay$") + child.Env = append(e2eEnvironment(), + e2eHelperModeEnv+"=child", + e2eHelperPIDFileEnv+"="+os.Getenv(e2eHelperPIDFileEnv), + ) + child.Stdout = os.Stdout + child.Stderr = os.Stderr + if err := child.Start(); err != nil { + t.Fatalf("start pipe-holding child failed: %v", err) + } + time.Sleep(10 * time.Second) + return + case "child": + pidFile := os.Getenv(e2eHelperPIDFileEnv) + temporaryPIDFile := pidFile + ".tmp" + if err := os.WriteFile(temporaryPIDFile, []byte(strconv.Itoa(os.Getpid())), 0o644); err != nil { + t.Fatalf("write pipe-holding child PID failed: %v", err) + } + if err := os.Rename(temporaryPIDFile, pidFile); err != nil { + t.Fatalf("publish pipe-holding child PID failed: %v", err) + } + time.Sleep(10 * time.Second) + return + } + + pidFile := filepath.Join(t.TempDir(), "child.pid") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + result := make(chan e2eCommandResult, 1) + go func() { + output, err := executeE2ECommand(ctx, 100*time.Millisecond, "", + append(e2eEnvironment(), + e2eHelperModeEnv+"=parent", + e2eHelperPIDFileEnv+"="+pidFile, + ), nil, os.Args[0], "-test.run=^TestE2ECommandWaitDelay$") + result <- e2eCommandResult{output: output, err: err} + }() + + childPID := waitForE2EHelperPID(t, pidFile, result) + child, err := os.FindProcess(childPID) + if err != nil { + cancel() + t.Fatalf("find pipe-holding child failed: %v", err) + } + defer func() { + _ = child.Kill() + _ = child.Release() + }() + + start := time.Now() + cancel() + command := <-result + if !errors.Is(ctx.Err(), context.Canceled) || command.err == nil { + t.Fatalf("expected command cancellation, context error=%v command error=%v", ctx.Err(), command.err) + } + if elapsed := time.Since(start); elapsed >= 3*time.Second { + t.Fatalf("command waited for a descendant-held output pipe: %s", elapsed) + } +} + +func waitForE2EHelperPID(t *testing.T, pidFile string, result <-chan e2eCommandResult) int { + t.Helper() + deadline := time.NewTimer(5 * time.Second) + defer deadline.Stop() + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + select { + case command := <-result: + t.Fatalf("helper command exited before starting child: %v\nstdout:\n%s\nstderr:\n%s", command.err, command.output.stdout, command.output.stderr) + case <-deadline.C: + t.Fatal("timed out waiting for pipe-holding child") + case <-ticker.C: + data, err := os.ReadFile(pidFile) + if os.IsNotExist(err) { + continue + } + if err != nil { + t.Fatalf("read pipe-holding child PID failed: %v", err) + } + pid, err := strconv.Atoi(string(data)) + if err != nil { + t.Fatalf("parse pipe-holding child PID failed: %v", err) + } + return pid + } + } +} + +func TestBuiltCLIEndToEnd(t *testing.T) { + harness := newE2EHarness(t) + + t.Run("parse build log", func(t *testing.T) { + logDir := t.TempDir() + runDir := t.TempDir() + writeE2EFile(t, filepath.Join(logDir, "main.c"), "int main(void) { return 0; }\n") + buildLog, err := filepath.Abs(filepath.Join(logDir, "build.log")) + if err != nil { + t.Fatalf("resolve absolute build log path failed: %v", err) + } + writeE2EFile(t, buildLog, "cc -DE2E_PARSE=1 -c main.c -o main.o\n") + outputFile := filepath.Join(runDir, "compile_commands.json") + + harness.run(t, runDir, nil, "--parse", buildLog) + + entries := readE2EDatabase(t, outputFile) + if len(entries) != 1 { + t.Fatalf("expected one compilation entry, got %#v", entries) + } + entry := entries[0] + if entry.File != "main.c" { + t.Fatalf("unexpected source file: %q", entry.File) + } + if entry.Command != "" || !slices.Equal(entry.Arguments, []string{"cc", "-DE2E_PARSE=1", "-c", "main.c", "-o", "main.o"}) { + t.Fatalf("unexpected compilation command: %#v", entry) + } + checkE2EDirectory(t, entry.Directory, logDir) + }) + + t.Run("parse stdin to stdout", func(t *testing.T) { + runDir := t.TempDir() + output := harness.run(t, runDir, []byte("cc -c 'broken.c\ncc -DE2E_STDIN=1 -c valid.c -o valid.o\n"), + "--output", "-", + "--no-strict", + ) + + entries := decodeE2EDatabase(t, output.stdout, "stdout") + if len(entries) != 1 { + t.Fatalf("expected one compilation entry, got %#v", entries) + } + entry := entries[0] + if entry.File != "valid.c" || entry.Command != "" || + !slices.Equal(entry.Arguments, []string{"cc", "-DE2E_STDIN=1", "-c", "valid.c", "-o", "valid.o"}) { + t.Fatalf("unexpected compilation entry: %#v", entry) + } + checkE2EDirectory(t, entry.Directory, runDir) + diagnostic := string(output.stderr) + if !strings.Contains(diagnostic, "skip malformed command at build log line 1") || + !strings.Contains(diagnostic, "unterminated quote") { + t.Fatalf("expected recoverable parser diagnostic on stderr, got %q", output.stderr) + } + if _, err := os.Stat(filepath.Join(runDir, "compile_commands.json")); !os.IsNotExist(err) { + t.Fatalf("stdout output created the default database: %v", err) + } + }) + + t.Run("explicit build directory", func(t *testing.T) { + logDir := t.TempDir() + buildDir := t.TempDir() + runDir := t.TempDir() + writeE2EFile(t, filepath.Join(buildDir, "main.c"), "int main(void) { return 0; }\n") + buildLog := filepath.Join(logDir, "build.log") + writeE2EFile(t, buildLog, "cc -DE2E_BUILD_DIR=1 -c main.c -o main.o\n") + + output := harness.run(t, runDir, nil, + "--build-dir", buildDir, + "--parse", buildLog, + "--output", "-", + ) + entries := decodeE2EDatabase(t, output.stdout, "stdout") + if len(entries) != 1 { + t.Fatalf("expected one compilation entry, got %#v", entries) + } + entry := entries[0] + if entry.File != "main.c" || entry.Command != "" || + !slices.Equal(entry.Arguments, []string{"cc", "-DE2E_BUILD_DIR=1", "-c", "main.c", "-o", "main.o"}) { + t.Fatalf("unexpected compilation entry: %#v", entry) + } + checkE2EDirectory(t, entry.Directory, buildDir) + }) + + t.Run("GNU Make build", func(t *testing.T) { + makeExecutable := findE2EGNUMake(t) + compiler := findE2ECompiler(t) + projectDir := t.TempDir() + marker := "COMPILEDB_E2E_MAKE_MARKER" + writeE2EFile(t, filepath.Join(projectDir, "main.c"), `#ifndef COMPILEDB_E2E +#error COMPILEDB_E2E is required +#endif +int main(void) { return 0; } +`) + writeE2EFile(t, filepath.Join(projectDir, "Makefile"), `all: main.o + +main.o: main.c + @echo `+marker+` + $(CC) -DCOMPILEDB_E2E=1 -c main.c -o main.o +`) + + output := harness.run(t, projectDir, nil, + "--build-dir", projectDir, + "--output", "-", + "make", "--cmd", makeExecutable, "CC="+compiler, + ) + checkE2ERegularFile(t, filepath.Join(projectDir, "main.o")) + if bytes.Contains(output.stdout, []byte(marker)) { + t.Fatalf("Make marker leaked to stdout: %q", output.stdout) + } + if count := bytes.Count(output.stderr, []byte(marker)); count != 1 { + t.Fatalf("expected one Make marker on stderr, got %d in %q", count, output.stderr) + } + + entries := decodeE2EDatabase(t, output.stdout, "stdout") + if len(entries) != 1 { + t.Fatalf("expected one compilation entry, got %#v", entries) + } + entry := entries[0] + if entry.File != "main.c" { + t.Fatalf("unexpected source file: %q", entry.File) + } + if entry.Command != "" || !slices.Equal(entry.Arguments, []string{compiler, "-DCOMPILEDB_E2E=1", "-c", "main.c", "-o", "main.o"}) { + t.Fatalf("unexpected compilation command: %#v", entry) + } + checkE2EDirectory(t, entry.Directory, projectDir) + + if err := os.Remove(filepath.Join(projectDir, "main.o")); err != nil { + t.Fatalf("remove real-build object failed: %v", err) + } + replayE2ECommand(t, entry) + checkE2ERegularFile(t, filepath.Join(projectDir, "main.o")) + }) +} + +func newE2EHarness(t *testing.T) e2eHarness { + t.Helper() + executable := filepath.Join(t.TempDir(), "compiledb") + if runtime.GOOS == "windows" { + executable += ".exe" + } + moduleRoot := e2eModuleRoot(t) + runE2ECommand(t, e2eBuildTimeout, moduleRoot, nil, nil, + "go", "build", "-buildvcs=false", "-o", executable, "./cmd/compiledb") + return e2eHarness{executable: executable} +} + +func e2eModuleRoot(t *testing.T) string { + t.Helper() + output := runE2ECommand(t, e2eCommandTimeout, "", nil, nil, "go", "env", "GOMOD") + goMod := strings.TrimSpace(string(output.stdout)) + if goMod == "" || goMod == os.DevNull { + t.Fatalf("E2E test is not running in a Go module: GOMOD=%q", goMod) + } + return filepath.Dir(goMod) +} + +func (h e2eHarness) run(t *testing.T, workingDir string, stdin []byte, arguments ...string) e2eCommandOutput { + t.Helper() + return runE2ECommand(t, e2eCommandTimeout, workingDir, e2eEnvironment(), stdin, h.executable, arguments...) +} + +func replayE2ECommand(t *testing.T, entry e2eCompilationEntry) { + t.Helper() + if len(entry.Arguments) == 0 { + t.Fatal("compilation entry has no arguments") + } + runE2ECommand(t, e2eCommandTimeout, filepath.FromSlash(entry.Directory), e2eEnvironment(), nil, entry.Arguments[0], entry.Arguments[1:]...) +} + +func runE2ECommand(t *testing.T, timeout time.Duration, workingDir string, environment []string, stdin []byte, name string, arguments ...string) e2eCommandOutput { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + output, err := executeE2ECommand(ctx, e2eCommandWaitDelay, workingDir, environment, stdin, name, arguments...) + invocation := append([]string{name}, arguments...) + if ctx.Err() != nil { + t.Fatalf("command timed out after %s: %v\ncommand: %q\nworking directory: %q\nstdout:\n%s\nstderr:\n%s", + timeout, ctx.Err(), invocation, workingDir, output.stdout, output.stderr) + } + if err != nil { + t.Fatalf("command failed: %v\ncommand: %q\nworking directory: %q\nstdout:\n%s\nstderr:\n%s", + err, invocation, workingDir, output.stdout, output.stderr) + } + return output +} + +func executeE2ECommand(ctx context.Context, waitDelay time.Duration, workingDir string, environment []string, stdin []byte, name string, arguments ...string) (e2eCommandOutput, error) { + command := exec.CommandContext(ctx, name, arguments...) + command.Dir = workingDir + command.WaitDelay = waitDelay + if environment != nil { + command.Env = environment + } + if stdin != nil { + command.Stdin = bytes.NewReader(stdin) + } + var stdout, stderr bytes.Buffer + command.Stdout = &stdout + command.Stderr = &stderr + err := command.Run() + output := e2eCommandOutput{stdout: stdout.Bytes(), stderr: stderr.Bytes()} + return output, err +} + +func findE2EGNUMake(t *testing.T) string { + t.Helper() + for _, name := range []string{"make", "gmake", "mingw32-make"} { + executable, err := exec.LookPath(name) + if err != nil { + continue + } + ctx, cancel := context.WithTimeout(context.Background(), e2eCommandTimeout) + command := exec.CommandContext(ctx, executable, "--version") + command.Env = e2eEnvironment() + output, err := command.CombinedOutput() + cancel() + if err == nil && bytes.Contains(output, []byte("GNU Make")) { + return executable + } + } + t.Skip("GNU Make is not available") + return "" +} + +func findE2ECompiler(t *testing.T) string { + t.Helper() + for _, name := range []string{"cc", "gcc", "clang"} { + if _, err := exec.LookPath(name); err == nil { + return name + } + } + t.Skip("a GNU-compatible C compiler is not available") + return "" +} + +func e2eEnvironment() []string { + ignored := map[string]bool{ + "COMPILEDB_ENCODING": true, + e2eHelperModeEnv: true, + e2eHelperPIDFileEnv: true, + "GNUMAKEFLAGS": true, + "MAKEFLAGS": true, + "MAKELEVEL": true, + "MFLAGS": true, + "LC_ALL": true, + } + environment := make([]string, 0, len(os.Environ())+1) + for _, variable := range os.Environ() { + name, _, _ := strings.Cut(variable, "=") + if !ignored[strings.ToUpper(name)] { + environment = append(environment, variable) + } + } + return append(environment, "LC_ALL=C") +} + +func writeE2EFile(t *testing.T, filename, contents string) { + t.Helper() + if err := os.WriteFile(filename, []byte(contents), 0o644); err != nil { + t.Fatalf("write %s failed: %v", filename, err) + } +} + +func readE2EDatabase(t *testing.T, filename string) []e2eCompilationEntry { + t.Helper() + data, err := os.ReadFile(filename) + if err != nil { + t.Fatalf("read compilation database %s failed: %v", filename, err) + } + return decodeE2EDatabase(t, data, filename) +} + +func decodeE2EDatabase(t *testing.T, data []byte, source string) []e2eCompilationEntry { + t.Helper() + if len(data) == 0 || data[len(data)-1] != '\n' { + t.Fatalf("%s must contain JSON with exactly one trailing newline: %q", source, data) + } + jsonData := data[:len(data)-1] + if !bytes.Equal(jsonData, bytes.TrimSpace(jsonData)) { + t.Fatalf("%s contains whitespace outside the JSON and final newline: %q", source, data) + } + var entries []e2eCompilationEntry + if err := json.Unmarshal(jsonData, &entries); err != nil { + t.Fatalf("decode compilation database from %s failed: %v\n%s", source, err, data) + } + return entries +} + +func checkE2EDirectory(t *testing.T, got, want string) { + t.Helper() + gotInfo, err := os.Stat(filepath.FromSlash(got)) + if err != nil { + t.Fatalf("stat generated directory %q failed: %v", got, err) + } + wantInfo, err := os.Stat(want) + if err != nil { + t.Fatalf("stat expected directory %q failed: %v", want, err) + } + if !os.SameFile(gotInfo, wantInfo) { + t.Fatalf("unexpected compilation directory: want %q, got %q", want, got) + } +} + +func checkE2ERegularFile(t *testing.T, filename string) { + t.Helper() + info, err := os.Stat(filename) + if err != nil { + t.Fatalf("stat %s failed: %v", filename, err) + } + if !info.Mode().IsRegular() { + t.Fatalf("expected %s to be a regular file, mode=%s", filename, info.Mode()) + } +}