diff --git a/builder/build.go b/builder/build.go index 104e0f3866..a6eee90592 100644 --- a/builder/build.go +++ b/builder/build.go @@ -79,17 +79,18 @@ type BuildResult struct { // key, avoiding the need for recompiling all dependencies when only the // implementation of an imported package changes. type packageAction struct { - ImportPath string - CompilerBuildID string - TinyGoVersion string - LLVMVersion string - Config *compiler.Config - CFlags []string - FileHashes map[string]string // hash of every file that's part of the package - EmbeddedFiles map[string]string // hash of all the //go:embed files in the package - Imports map[string]string // map from imported package to action ID hash - OptLevel string // LLVM optimization level (O0, O1, O2, Os, Oz) - UndefinedGlobals []string // globals that are left as external globals (no initializer) + ImportPath string + CompilerBuildID string + TinyGoVersion string + LLVMVersion string + CCompilerIdentity string + Config *compiler.Config + CFlags []string + FileHashes map[string]string // hash of every file that's part of the package + EmbeddedFiles map[string]string // hash of all the //go:embed files in the package + Imports map[string]string // map from imported package to action ID hash + OptLevel string // LLVM optimization level (O0, O1, O2, Os, Oz) + UndefinedGlobals []string // globals that are left as external globals (no initializer) } // Build performs a single package to executable Go build. It takes in a package @@ -145,53 +146,33 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe // As a side effect, this also creates the headers for the given libc, if // the libc needs them. root := goenv.Get("TINYGOROOT") + libraries, err := configuredLibraries(config) + if err != nil { + return BuildResult{}, err + } + libraryInputs, libraryKeys, err := makeLibraryCacheInputs(config, libraries) + if err != nil { + return BuildResult{}, err + } + config.LibraryKeys = libraryKeys var libcDependencies []*compileJob - switch config.Target.Libc { - case "darwin-libSystem": + if config.Target.Libc == "darwin-libSystem" { libcJob := makeDarwinLibSystemJob(config, tmpdir) libcDependencies = append(libcDependencies, libcJob) - case "musl": - var unlock func() - libcJob, unlock, err := libMusl.load(config, tmpdir) - if err != nil { - return BuildResult{}, err - } - defer unlock() - libcDependencies = append(libcDependencies, dummyCompileJob(filepath.Join(filepath.Dir(libcJob.result), "crt1.o"))) - libcDependencies = append(libcDependencies, libcJob) - case "picolibc": - libcJob, unlock, err := libPicolibc.load(config, tmpdir) - if err != nil { - return BuildResult{}, err - } - defer unlock() - libcDependencies = append(libcDependencies, libcJob) - case "wasi-libc": - libcJob, unlock, err := libWasiLibc.load(config, tmpdir) + } + if libraries.libc != nil { + libcJob, unlock, err := libraries.libc.load(config, tmpdir, libraryInputs[libraries.libc.name]) if err != nil { return BuildResult{}, err } defer unlock() - libcDependencies = append(libcDependencies, libcJob) - case "wasmbuiltins": - libcJob, unlock, err := libWasmBuiltins.load(config, tmpdir) - if err != nil { - return BuildResult{}, err + if libraries.libc.crt1Source != "" { + libcDependencies = append(libcDependencies, dummyCompileJob(filepath.Join(filepath.Dir(libcJob.result), "crt1.o"))) } - defer unlock() - libcDependencies = append(libcDependencies, libcJob) - case "mingw-w64": - libcJob, unlock, err := libMinGW.load(config, tmpdir) - if err != nil { - return BuildResult{}, err - } - defer unlock() libcDependencies = append(libcDependencies, libcJob) + } + if config.Target.Libc == "mingw-w64" { libcDependencies = append(libcDependencies, makeMinGWExtraLibs(tmpdir, config.GOARCH())...) - case "": - // no library specified, so nothing to do - default: - return BuildResult{}, fmt.Errorf("unknown libc: %s", config.Target.Libc) } optLevel, speedLevel, sizeLevel := config.OptLevel() @@ -362,6 +343,13 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe OptLevel: optLevel, UndefinedGlobals: undefinedGlobals, } + if len(pkg.CGoHeaders) != 0 { + compilerID, err := clangCompilerIdentity() + if err != nil { + return err + } + actionID.CCompilerIdentity = compilerID + } for filePath, hash := range pkg.FileHashes { actionID.FileHashes[filePath] = hex.EncodeToString(hash) } @@ -411,9 +399,6 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe } // Load bitcode of CGo headers and join the modules together. - // This may seem vulnerable to cache problems, but this is not - // the case: the Go code that was just compiled already tracks - // all C files that are read and hashes them. // These headers could be compiled in parallel but the benefit // is so small that it's probably not worth parallelizing. // Packages are compiled independently anyway. @@ -423,14 +408,16 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe if err != nil { return err } - _, err = f.Write([]byte(cgoHeader)) - if err != nil { + if _, err := f.Write([]byte(cgoHeader)); err != nil { + return err + } + if err := f.Close(); err != nil { return err } - f.Close() + output := f.Name() + ".bc" // Compile the code (if there is any) to bitcode. - flags := append([]string{"-c", "-emit-llvm", "-o", f.Name() + ".bc", f.Name()}, pkg.CFlags...) + flags := cgoHeaderCompileArgs(f.Name(), output, pkg.CFlags) if config.Options.PrintCommands != nil { config.Options.PrintCommands("clang", flags...) } @@ -444,7 +431,7 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe // in the header together with the Go code. In particular, // this allows inlining. It also ensures there is only one // file per package to cache. - headerMod, err := mod.Context().ParseBitcodeFile(f.Name() + ".bc") + headerMod, err := mod.Context().ParseBitcodeFile(output) if err != nil { return fmt.Errorf("failed to load bitcode file: %w", err) } @@ -732,10 +719,9 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe } } - // Add compiler-rt dependency if needed. Usually this is a simple load from - // a cache. - if config.Target.RTLib == "compiler-rt" { - job, unlock, err := libCompilerRT.load(config, tmpdir) + // Add library dependencies needed by the linker, usually from the cache. + for _, library := range libraries.linker { + job, unlock, err := library.load(config, tmpdir, libraryInputs[library.name]) if err != nil { return result, err } @@ -743,16 +729,6 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe linkerDependencies = append(linkerDependencies, job) } - // The Boehm collector is stored in a separate C library. - if config.GC() == "boehm" { - job, unlock, err := BoehmGC.load(config, tmpdir) - if err != nil { - return BuildResult{}, err - } - defer unlock() - linkerDependencies = append(linkerDependencies, job) - } - // Add jobs to compile extra files. These files are in C or assembly and // contain things like the interrupt vector table and low level operations // such as stack switching. @@ -1098,6 +1074,15 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe return result, nil } +func cgoHeaderCompileArgs(source, output string, cflags []string) []string { + flags := append([]string{"-c", "-emit-llvm", "-o", output, source}, cflags...) + flags = append(flags, + "-ffile-prefix-map="+source+"=tinygo-cgo.c", + "-fdebug-prefix-map="+source+"=tinygo-cgo.c", + ) + return appendCacheStableCFlags(flags) +} + // createEmbedObjectFile creates a new object file with the given contents, for // the embed package. func createEmbedObjectFile(data, hexSum, sourceFile, sourceDir, tmpdir string, compilerConfig *compiler.Config) (string, error) { diff --git a/builder/build_test.go b/builder/build_test.go new file mode 100644 index 0000000000..4ad9c7827b --- /dev/null +++ b/builder/build_test.go @@ -0,0 +1,50 @@ +package builder + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "tinygo.org/x/go-llvm" +) + +func TestCGoHeaderCompileIsPathIndependent(t *testing.T) { + var outputs [][]byte + for _, dirName := range []string{"first", "second"} { + dir := filepath.Join(t.TempDir(), dirName) + if err := os.Mkdir(dir, 0o777); err != nil { + t.Fatal(err) + } + source := filepath.Join(dir, "snippet.c") + output := filepath.Join(dir, "snippet.bc") + if err := os.WriteFile(source, []byte("const char *sourceName = __FILE__;\n"), 0o666); err != nil { + t.Fatal(err) + } + flags := cgoHeaderCompileArgs(source, output, []string{ + "-gdwarf-4", + "--target=x86_64-unknown-linux-gnu", + }) + if err := runCCompiler(flags...); err != nil { + t.Fatal(err) + } + + ctx := llvm.NewContext() + mod := ctx.NewModule("package") + headerMod, err := ctx.ParseBitcodeFile(output) + if err != nil { + t.Fatal(err) + } + if err := llvm.LinkModules(mod, headerMod); err != nil { + t.Fatal(err) + } + buf := llvm.WriteBitcodeToMemoryBuffer(mod) + outputs = append(outputs, bytes.Clone(buf.Bytes())) + buf.Dispose() + mod.Dispose() + ctx.Dispose() + } + if !bytes.Equal(outputs[0], outputs[1]) { + t.Fatal("CGo header bitcode depends on its temporary source path") + } +} diff --git a/builder/cc.go b/builder/cc.go index 9cc03790b6..f0010e6827 100644 --- a/builder/cc.go +++ b/builder/cc.go @@ -10,7 +10,6 @@ import ( "errors" "fmt" "io" - "io/fs" "os" "path/filepath" "sort" @@ -25,37 +24,21 @@ import ( // Compiling the same file again (if nothing changed, including included header // files) the output is loaded from the build cache instead. // -// Its operation is a bit complex (more complex than Go package build caching) -// because the list of file dependencies is only known after the file is -// compiled. However, luckily compilers have a flag to write a list of file -// dependencies in Makefile syntax which can be used for caching. +// Its operation is a bit complex (more complex than Go package build caching), +// because the list of file dependencies depends on C include path resolution. +// TinyGo asks Clang for the current dependency list before looking for an object +// cache hit, then uses the hashes of those dependencies in the object key. // -// Because of this complexity, every file has in fact two cached build outputs: -// the file itself, and the list of dependencies. Its operation is as follows: -// -// depfile = hash(path, compiler, cflags, ...) -// if depfile exists: -// outfile = hash of all files and depfile name -// if outfile exists: -// # cache hit -// return outfile -// # cache miss +// dependencies = clang -M source +// outfile = hash(path, compiler, cflags, dependencies, ...) +// if outfile exists: +// # cache hit +// return outfile // tmpfile = compile file -// read dependencies (side effect of compile) -// write depfile -// outfile = hash of all files and depfile name // rename tmpfile to outfile // -// There are a few edge cases that are not handled: -// - If a file is added to an include path, that file may be included instead of -// some other file. This would be fixed by also including lookup failures in the -// dependencies file, but I'm not aware of a compiler which does that. -// - The Makefile syntax that compilers output has issues, see readDepFile for -// details. -// - A header file may be changed to add/remove an include. This invalidates the -// depfile but without invalidating its name. For this reason, the depfile is -// written on each new compilation (even when it seems unnecessary). However, it -// could in rare cases lead to a stale file fetched from the cache. +// The Makefile syntax that compilers output has issues, see readDepFile for +// details. func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands func(string, ...string)) (string, error) { // Hash input file. fileHash, err := hashFile(abspath) @@ -67,65 +50,47 @@ func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands unlock := lock(filepath.Join(goenv.Get("GOCACHE"), fileHash+".c.lock")) defer unlock() - // Create cache key for the dependencies file. - buf, err := json.Marshal(struct { - Path string - Hash string - Flags []string - LLVMVersion string - }{ - Path: abspath, - Hash: fileHash, - Flags: cflags, - LLVMVersion: llvm.Version, - }) + compilerID, err := clangCompilerIdentity() if err != nil { - panic(err) // shouldn't happen + return "", err } - depfileNameHashBuf := sha512.Sum512_224(buf) - depfileNameHash := hex.EncodeToString(depfileNameHashBuf[:]) - - // Load dependencies file, if possible. - depfileName := "dep-" + depfileNameHash + ".json" - depfileCachePath := filepath.Join(goenv.Get("GOCACHE"), depfileName) - depfileBuf, err := os.ReadFile(depfileCachePath) - var dependencies []string // sorted list of dependency paths - if err == nil { - // There is a dependency file, that's great! - // Parse it first. - err := json.Unmarshal(depfileBuf, &dependencies) - if err != nil { - return "", fmt.Errorf("could not parse dependencies JSON: %w", err) - } - // Obtain hashes of all the files listed as a dependency. - outpath, err := makeCFileCachePath(dependencies, depfileNameHash) - if err == nil { - if _, err := os.Stat(outpath); err == nil { - return outpath, nil - } else if !errors.Is(err, fs.ErrNotExist) { - return "", err - } - } - } else if !errors.Is(err, fs.ErrNotExist) { - // expected either nil or IsNotExist + dependencies, err := scanCFileDependencies(abspath, tmpdir, cflags, printCommands) + if err != nil { + return "", err + } + outpath, err := makeCFileCachePath(abspath, cFileCompileArgs(abspath, "$OBJ", cflags), compilerID, dependencies) + if err != nil { + return "", err + } + if _, err := os.Stat(outpath); err == nil { + return outpath, nil + } else if !errors.Is(err, os.ErrNotExist) { return "", err } - objTmpFile, err := os.CreateTemp(goenv.Get("GOCACHE"), "tmp-*.bc") + objTmpFile, err := compileCFile(goenv.Get("GOCACHE"), abspath, cflags, printCommands) if err != nil { return "", err } - objTmpFile.Close() + if err := os.Rename(objTmpFile, outpath); err != nil { + os.Remove(objTmpFile) + return "", err + } + return outpath, nil +} + +func scanCFileDependencies(abspath, tmpdir string, cflags []string, printCommands func(string, ...string)) ([]string, error) { depTmpFile, err := os.CreateTemp(tmpdir, "dep-*.d") if err != nil { - return "", err + return nil, err } depTmpFile.Close() - flags := append([]string{}, cflags...) // copy cflags - flags = append(flags, "-MD", "-MV", "-MTdeps", "-MF", depTmpFile.Name(), "-flto=thin") // autogenerate dependencies - flags = append(flags, "-c", "-o", objTmpFile.Name(), abspath) - if strings.ToLower(filepath.Ext(abspath)) == ".s" { + defer os.Remove(depTmpFile.Name()) + + flags := appendCacheStableCFlags(append([]string{}, cflags...)) + flags = append(flags, "-M", "-MV", "-MTdeps", "-MF", depTmpFile.Name(), abspath) + if isAssemblyFile(abspath) { // If this is an assembly file (.s or .S, lowercase or uppercase), then // we'll need to add -Qunused-arguments because many parameters are // relevant to C, not assembly. And with -Werror, having meaningless @@ -137,13 +102,12 @@ func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands } err = runCCompiler(flags...) if err != nil { - return "", &commandError{"failed to build", abspath, err} + return nil, &commandError{"failed to scan dependencies", abspath, err} } - // Create sorted and uniqued slice of dependencies. dependencyPaths, err := readDepFile(depTmpFile.Name()) if err != nil { - return "", err + return nil, err } dependencyPaths = append(dependencyPaths, abspath) // necessary for .s files dependencySet := make(map[string]struct{}, len(dependencyPaths)) @@ -156,50 +120,45 @@ func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands dependencySlice = append(dependencySlice, path) } sort.Strings(dependencySlice) + return dependencySlice, nil +} - // Write dependencies file. - f, err := os.CreateTemp(filepath.Dir(depfileCachePath), depfileName) - if err != nil { - return "", err +func cFileCompileArgs(abspath, objpath string, cflags []string) []string { + flags := append([]string{}, cflags...) + flags = append(flags, "-flto=thin") + flags = append(flags, "-c", "-o", objpath, abspath) + if isAssemblyFile(abspath) { + flags = append(flags, "-Qunused-arguments") } + return appendCacheStableCFlags(flags) +} - buf, err = json.MarshalIndent(dependencySlice, "", "\t") - if err != nil { - panic(err) // shouldn't happen - } - _, err = f.Write(buf) - if err != nil { - return "", err - } - err = f.Close() - if err != nil { - return "", err - } - err = os.Rename(f.Name(), depfileCachePath) +func compileCFile(cacheDir, abspath string, cflags []string, printCommands func(string, ...string)) (string, error) { + objTmpFile, err := os.CreateTemp(cacheDir, "tmp-*.bc") if err != nil { return "", err } + objTmpFile.Close() - // Move temporary object file to final location. - outpath, err := makeCFileCachePath(dependencySlice, depfileNameHash) - if err != nil { - return "", err + flags := cFileCompileArgs(abspath, objTmpFile.Name(), cflags) + if printCommands != nil { + printCommands("clang", flags...) } - err = os.Rename(objTmpFile.Name(), outpath) + err = runCCompiler(flags...) if err != nil { - return "", err + os.Remove(objTmpFile.Name()) + return "", &commandError{"failed to build", abspath, err} } - - return outpath, nil + return objTmpFile.Name(), nil } // Create a cache path (a path in GOCACHE) to store the output of a compiler -// job. This path is based on the dep file name (which is a hash of metadata -// including compiler flags) and the hash of all input files in the paths slice. -func makeCFileCachePath(paths []string, depfileNameHash string) (string, error) { +// job. This path is based on the compiler identity, compiler flags, and the +// hash of all dependency files. +func makeCFileCachePath(path string, flags []string, compilerID string, dependencies []string) (string, error) { // Hash all input files. - fileHashes := make(map[string]string, len(paths)) - for _, path := range paths { + fileHashes := make(map[string]string, len(dependencies)) + for _, path := range dependencies { hash, err := hashFile(path) if err != nil { return "", err @@ -209,11 +168,17 @@ func makeCFileCachePath(paths []string, depfileNameHash string) (string, error) // Calculate a cache key based on the above hashes. buf, err := json.Marshal(struct { - DepfileHash string - FileHashes map[string]string + Path string + Flags []string + LLVMVersion string + CompilerIdentity string + FileHashes map[string]string }{ - DepfileHash: depfileNameHash, - FileHashes: fileHashes, + Path: path, + Flags: flags, + LLVMVersion: llvm.Version, + CompilerIdentity: compilerID, + FileHashes: fileHashes, }) if err != nil { panic(err) // shouldn't happen @@ -225,6 +190,10 @@ func makeCFileCachePath(paths []string, depfileNameHash string) (string, error) return outpath, nil } +func isAssemblyFile(path string) bool { + return strings.ToLower(filepath.Ext(path)) == ".s" +} + // hashFile hashes the given file path and returns the hash as a hex string. func hashFile(path string) (string, error) { f, err := os.Open(path) diff --git a/builder/cc_test.go b/builder/cc_test.go index 085528060e..fc9117b133 100644 --- a/builder/cc_test.go +++ b/builder/cc_test.go @@ -1,6 +1,8 @@ package builder import ( + "os" + "path/filepath" "reflect" "testing" ) @@ -31,3 +33,53 @@ func TestSplitDepFile(t *testing.T) { } } } + +func TestCFileCacheIncludePathShadowing(t *testing.T) { + t.Setenv("GOCACHEPROG", "") + + dir := t.TempDir() + + include1 := filepath.Join(dir, "include1") + include2 := filepath.Join(dir, "include2") + if err := os.Mkdir(include1, 0o777); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(include2, 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(include2, "value.h"), []byte("#define VALUE 1\n"), 0o666); err != nil { + t.Fatal(err) + } + source := filepath.Join(dir, "test.c") + if err := os.WriteFile(source, []byte("#include \"value.h\"\nint value(void) { return VALUE; }\n"), 0o666); err != nil { + t.Fatal(err) + } + + flags := []string{ + "-I", include1, + "-I", include2, + "--target=x86_64-unknown-linux-gnu", + } + first, err := compileAndCacheCFile(source, dir, flags, nil) + if err != nil { + t.Fatal(err) + } + second, err := compileAndCacheCFile(source, dir, flags, nil) + if err != nil { + t.Fatal(err) + } + if first != second { + t.Fatalf("unchanged compile did not hit cache: %s != %s", first, second) + } + + if err := os.WriteFile(filepath.Join(include1, "value.h"), []byte("#define VALUE 2\n"), 0o666); err != nil { + t.Fatal(err) + } + shadowed, err := compileAndCacheCFile(source, dir, flags, nil) + if err != nil { + t.Fatal(err) + } + if shadowed == first { + t.Fatal("include path shadowing reused stale cached object") + } +} diff --git a/builder/library.go b/builder/library.go index 722f3c8371..02dfed1703 100644 --- a/builder/library.go +++ b/builder/library.go @@ -1,7 +1,11 @@ package builder import ( + "crypto/sha512" + "encoding/hex" + "encoding/json" "errors" + "fmt" "io/fs" "os" "path/filepath" @@ -11,13 +15,11 @@ import ( "github.com/tinygo-org/tinygo/compileopts" "github.com/tinygo-org/tinygo/goenv" + "tinygo.org/x/go-llvm" ) // Library is a container for information about a single C library, such as a // compiler runtime or libc. -// -// Note: whenever a library gets changed, the version in compileopts/config.go -// probably also needs to be incremented. type Library struct { // The library name, such as compiler-rt or picolibc. name string @@ -37,6 +39,9 @@ type Library struct { // The source directory. sourceDir func() string + // The input directory that contains headers and other non-source inputs. + inputDir func() string + // The source files, relative to sourceDir. librarySources func(target string, libcNeedsMalloc bool) ([]string, error) @@ -44,6 +49,403 @@ type Library struct { crt1Source string } +const ( + libraryHeaderPathPlaceholder = "$HEADER" + libraryBuildDirPlaceholder = "$BUILDDIR" + // Increment when the archive construction changes in a way that can affect linking. + libraryArchiveFormatVersion = 1 +) + +type librarySourceInput struct { + Path string + Hash string + CFlags []string +} + +type libraryCacheInput struct { + Name string + Target string + LibcNeedsMalloc bool + ArchiveFormat int + LLVMVersion string + CompilerIdentity string + ResourceDir string + SourceDir string + InputDir string + InputFiles map[string]string + CompileInputs map[string]map[string]string + GeneratedHeaders map[string]string + CompileArgs []string + Sources []librarySourceInput + Crt1Source string + Crt1Hash string +} + +type configuredLibrarySet struct { + libc *Library + linker []*Library +} + +func configuredLibraries(config *compileopts.Config) (configuredLibrarySet, error) { + var libraries configuredLibrarySet + switch config.Target.Libc { + case "musl": + libraries.libc = &libMusl + case "picolibc": + libraries.libc = &libPicolibc + case "wasi-libc": + libraries.libc = &libWasiLibc + case "wasmbuiltins": + libraries.libc = &libWasmBuiltins + case "mingw-w64": + libraries.libc = &libMinGW + case "darwin-libSystem", "": + // These libc configurations don't use a Library-backed cache. + default: + return configuredLibrarySet{}, fmt.Errorf("unknown libc: %s", config.Target.Libc) + } + if config.Target.RTLib == "compiler-rt" { + libraries.linker = append(libraries.linker, &libCompilerRT) + } + if config.GC() == "boehm" { + libraries.linker = append(libraries.linker, &BoehmGC) + } + return libraries, nil +} + +func (libraries configuredLibrarySet) all() []*Library { + result := make([]*Library, 0, 1+len(libraries.linker)) + if libraries.libc != nil { + result = append(result, libraries.libc) + } + result = append(result, libraries.linker...) + return result +} + +func makeLibraryCacheInputs(config *compileopts.Config, libraries configuredLibrarySet) (map[string]*libraryCacheInput, map[string]string, error) { + inputs := make(map[string]*libraryCacheInput) + keys := make(map[string]string) + keyConfig := *config + keyConfig.LibraryKeys = keys + + setKey := func(l *Library) error { + if _, ok := keys[l.name]; ok { + return nil + } + input, err := l.cacheInput(&keyConfig) + if err != nil { + return err + } + inputs[l.name] = input + keys[l.name] = input.key() + return nil + } + + for _, library := range libraries.all() { + if err := setKey(library); err != nil { + return nil, nil, err + } + } + return inputs, keys, nil +} + +func (l *Library) cacheInput(config *compileopts.Config) (*libraryCacheInput, error) { + target := config.Triple() + sourceDir := l.sourceDir() + sources, err := l.librarySources(target, config.LibcNeedsMalloc()) + if err != nil { + return nil, err + } + + inputDir := sourceDir + if l.inputDir != nil { + inputDir = l.inputDir() + } + compilerID, err := clangCompilerIdentity() + if err != nil { + return nil, err + } + compileArgs := l.compileArgs(config, target, libraryHeaderPathPlaceholder, libraryBuildDirPlaceholder) + + input := libraryCacheInput{ + Name: l.name, + Target: target, + LibcNeedsMalloc: config.LibcNeedsMalloc(), + ArchiveFormat: libraryArchiveFormatVersion, + LLVMVersion: llvm.Version, + CompilerIdentity: compilerID, + ResourceDir: goenv.ClangResourceDir(false), + SourceDir: sourceDir, + InputDir: inputDir, + CompileArgs: compileArgs, + Sources: make([]librarySourceInput, 0, len(sources)), + Crt1Source: l.crt1Source, + } + for _, source := range sources { + hash, err := hashFile(filepath.Join(sourceDir, source)) + if err != nil { + return nil, err + } + sourceInput := librarySourceInput{ + Path: filepath.ToSlash(source), + Hash: hash, + } + if l.cflagsForFile != nil { + sourceInput.CFlags = l.cflagsForFile(source) + } + input.Sources = append(input.Sources, sourceInput) + } + if l.crt1Source != "" { + hash, err := hashFile(filepath.Join(sourceDir, l.crt1Source)) + if err != nil { + return nil, err + } + input.Crt1Source = filepath.ToSlash(l.crt1Source) + input.Crt1Hash = hash + } + inputFiles, err := hashLibraryInputFiles(inputDir) + if err != nil { + return nil, err + } + input.InputFiles = inputFiles + compileInputs, err := hashLibraryCompileInputs(compileArgs, sourceDir, inputDir) + if err != nil { + return nil, err + } + input.CompileInputs = compileInputs + generatedHeaders, err := l.hashGeneratedHeaders(target) + if err != nil { + return nil, err + } + input.GeneratedHeaders = generatedHeaders + + return &input, nil +} + +func (input *libraryCacheInput) key() string { + data, err := json.Marshal(input) + if err != nil { + panic(err) + } + sum := sha512.Sum512_224(data) + return hex.EncodeToString(sum[:]) +} + +func (l *Library) hashGeneratedHeaders(target string) (map[string]string, error) { + if l.makeHeaders == nil { + return nil, nil + } + dir, err := os.MkdirTemp("", "tinygo-lib-headers-*") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + if err := l.makeHeaders(target, dir); err != nil { + return nil, err + } + return hashLibraryInputFiles(dir) +} + +func hashLibraryInputFiles(root string) (map[string]string, error) { + resolvedRoot, err := filepath.EvalSymlinks(root) + if err != nil { + return nil, err + } + hashes := map[string]string{} + info, err := os.Stat(resolvedRoot) + if err != nil { + return nil, err + } + if info.Mode().IsRegular() { + hash, err := hashFile(resolvedRoot) + if err != nil { + return nil, err + } + hashes["."] = hash + return hashes, nil + } + if !info.IsDir() { + return hashes, nil + } + err = hashLibraryInputDir(resolvedRoot, "", hashes, make(map[string]bool)) + return hashes, err +} + +func hashLibraryInputDir(dir, prefix string, hashes map[string]string, ancestors map[string]bool) error { + resolvedDir, err := filepath.EvalSymlinks(dir) + if err != nil { + return err + } + if ancestors[resolvedDir] { + return nil + } + ancestors[resolvedDir] = true + defer delete(ancestors, resolvedDir) + + entries, err := os.ReadDir(resolvedDir) + if err != nil { + return err + } + for _, entry := range entries { + path := filepath.Join(resolvedDir, entry.Name()) + rel := filepath.Join(prefix, entry.Name()) + info, err := os.Stat(path) + if err != nil { + return err + } + if info.IsDir() { + if err := hashLibraryInputDir(path, rel, hashes, ancestors); err != nil { + return err + } + continue + } + if !info.Mode().IsRegular() { + continue + } + hash, err := hashFile(path) + if err != nil { + return err + } + hashes[filepath.ToSlash(rel)] = hash + } + return nil +} + +func hashLibraryCompileInputs(args []string, coveredRoots ...string) (map[string]map[string]string, error) { + paths := compilerInputPaths(args) + inputs := make(map[string]map[string]string, len(paths)) + cacheDir := filepath.Clean(goenv.Get("GOCACHE")) + for _, path := range paths { + if strings.Contains(path, libraryHeaderPathPlaceholder) || + strings.Contains(path, libraryBuildDirPlaceholder) { + continue + } + path = filepath.Clean(path) + covered := false + for _, root := range coveredRoots { + rel, err := filepath.Rel(filepath.Clean(root), path) + if err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + covered = true + break + } + } + if covered { + continue + } + if rel, err := filepath.Rel(cacheDir, path); err == nil && + rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + // Cached library include paths contain the dependency's content key. + continue + } + hashes, err := hashLibraryInputFiles(path) + if errors.Is(err, fs.ErrNotExist) { + inputs[path] = nil + continue + } + if err != nil { + return nil, err + } + inputs[path] = hashes + } + return inputs, nil +} + +func compilerInputPaths(args []string) []string { + var paths []string + for i := 0; i < len(args); i++ { + arg := args[i] + switch arg { + case "-I", "-isystem", "-iquote", "-idirafter", "-include", "-imacros", + "-resource-dir", "--sysroot", "-isysroot": + if i+1 < len(args) { + i++ + paths = append(paths, args[i]) + } + default: + for _, prefix := range []string{ + "-I", "-isystem", "-iquote", "-idirafter", "-include", "-imacros", + "-resource-dir=", "--sysroot=", "-isysroot", + } { + if strings.HasPrefix(arg, prefix) && len(arg) != len(prefix) { + paths = append(paths, strings.TrimPrefix(arg, prefix)) + break + } + } + } + } + return paths +} + +func (l *Library) compileArgs(config *compileopts.Config, target, headerPath, dir string) []string { + remapDir := filepath.Join(os.TempDir(), "tinygo-"+l.name) + args := append(l.cflags(target, headerPath), "-c", "-Oz", "-gdwarf-4", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target="+compileopts.ClangTriple(target), "-fdebug-prefix-map="+dir+"="+remapDir) + resourceDir := goenv.ClangResourceDir(false) + if resourceDir != "" { + args = append(args, "-resource-dir="+resourceDir) + } + cpu := config.CPU() + if cpu != "" { + // X86 has deprecated the -mcpu flag, so we need to use -march instead. + // However, ARM has not done this. + if strings.HasPrefix(target, "i386") || strings.HasPrefix(target, "x86_64") { + args = append(args, "-march="+cpu) + } else if strings.HasPrefix(target, "avr") { + args = append(args, "-mmcu="+cpu) + } else { + args = append(args, "-mcpu="+cpu) + } + } + if config.ABI() != "" { + args = append(args, "-mabi="+config.ABI()) + } + switch compileopts.CanonicalArchName(target) { + case "arm": + if strings.Split(target, "-")[2] == "linux" { + args = append(args, "-fno-unwind-tables", "-fno-asynchronous-unwind-tables") + } else { + args = append(args, "-fshort-enums", "-fomit-frame-pointer", "-mfloat-abi=soft", "-fno-unwind-tables", "-fno-asynchronous-unwind-tables") + } + case "avr": + // AVR defaults to C float and double both being 32-bit. This deviates + // from what most code (and certainly compiler-rt) expects. So we need + // to force the compiler to use 64-bit floating point numbers for + // double. + args = append(args, "-mdouble=64") + case "riscv32": + args = append(args, "-march="+riscvMarch(config, "rv32imac"), "-fforce-enable-int128") + case "riscv64": + args = append(args, "-march="+riscvMarch(config, "rv64gc")) + case "mips": + args = append(args, "-fno-pic") + } + if config.Target.SoftFloat { + // Use softfloat instead of floating point instructions. This is + // supported on many architectures. + args = append(args, "-msoft-float") + } else { + if strings.HasPrefix(target, "armv5") { + // On ARMv5 we need to explicitly enable hardware floating point + // instructions: Clang appears to assume the hardware doesn't have a + // FPU otherwise. + args = append(args, "-mfpu=vfpv2") + } + } + if l.needsLibc { + args = append(args, config.LibcCFlags()...) + } + return appendCacheStableCFlags(args) +} + +func expandLibraryCompileArgs(args []string, headerPath, dir string) []string { + expanded := append([]string(nil), args...) + for i, arg := range expanded { + arg = strings.ReplaceAll(arg, libraryHeaderPathPlaceholder, headerPath) + arg = strings.ReplaceAll(arg, libraryBuildDirPlaceholder, dir) + expanded[i] = arg + } + return expanded +} + // load returns a compile job to build this library file for the given target // and CPU. It may return a dummy compileJob if the library build is already // cached. The path is stored as job.result but is only valid after the job has @@ -52,7 +454,15 @@ type Library struct { // output archive file, it is expected to be removed after use. // As a side effect, this call creates the library header files if they didn't // exist yet. -func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJob, abortLock func(), err error) { +func (l *Library) load(config *compileopts.Config, tmpdir string, input *libraryCacheInput) (job *compileJob, abortLock func(), err error) { + key := input.key() + if existingKey, ok := config.LibraryKeys[l.name]; ok { + if existingKey != key { + return nil, nil, fmt.Errorf("library cache key changed for %s", l.name) + } + } else { + return nil, nil, fmt.Errorf("library cache key missing for %s", l.name) + } outdir := config.LibraryPath(l.name) archiveFilePath := filepath.Join(outdir, "lib.a") @@ -122,7 +532,6 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ } } - remapDir := filepath.Join(os.TempDir(), "tinygo-"+l.name) dir := filepath.Join(tmpdir, "build-lib-"+l.name) err = os.Mkdir(dir, 0777) if err != nil { @@ -133,61 +542,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ // Note: -fdebug-prefix-map is necessary to make the output archive // reproducible. Otherwise the temporary directory is stored in the archive // itself, which varies each run. - args := append(l.cflags(target, headerPath), "-c", "-Oz", "-gdwarf-4", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target="+compileopts.ClangTriple(target), "-fdebug-prefix-map="+dir+"="+remapDir) - resourceDir := goenv.ClangResourceDir(false) - if resourceDir != "" { - args = append(args, "-resource-dir="+resourceDir) - } - cpu := config.CPU() - if cpu != "" { - // X86 has deprecated the -mcpu flag, so we need to use -march instead. - // However, ARM has not done this. - if strings.HasPrefix(target, "i386") || strings.HasPrefix(target, "x86_64") { - args = append(args, "-march="+cpu) - } else if strings.HasPrefix(target, "avr") { - args = append(args, "-mmcu="+cpu) - } else { - args = append(args, "-mcpu="+cpu) - } - } - if config.ABI() != "" { - args = append(args, "-mabi="+config.ABI()) - } - switch compileopts.CanonicalArchName(target) { - case "arm": - if strings.Split(target, "-")[2] == "linux" { - args = append(args, "-fno-unwind-tables", "-fno-asynchronous-unwind-tables") - } else { - args = append(args, "-fshort-enums", "-fomit-frame-pointer", "-mfloat-abi=soft", "-fno-unwind-tables", "-fno-asynchronous-unwind-tables") - } - case "avr": - // AVR defaults to C float and double both being 32-bit. This deviates - // from what most code (and certainly compiler-rt) expects. So we need - // to force the compiler to use 64-bit floating point numbers for - // double. - args = append(args, "-mdouble=64") - case "riscv32": - args = append(args, "-march="+riscvMarch(config, "rv32imac"), "-fforce-enable-int128") - case "riscv64": - args = append(args, "-march="+riscvMarch(config, "rv64gc")) - case "mips": - args = append(args, "-fno-pic") - } - if config.Target.SoftFloat { - // Use softfloat instead of floating point instructions. This is - // supported on many architectures. - args = append(args, "-msoft-float") - } else { - if strings.HasPrefix(target, "armv5") { - // On ARMv5 we need to explicitly enable hardware floating point - // instructions: Clang appears to assume the hardware doesn't have a - // FPU otherwise. - args = append(args, "-mfpu=vfpv2") - } - } - if l.needsLibc { - args = append(args, config.LibcCFlags()...) - } + args := expandLibraryCompileArgs(input.CompileArgs, headerPath, dir) var once sync.Once @@ -222,16 +577,14 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ }, } - sourceDir := l.sourceDir() + sourceDir := input.SourceDir // Create jobs to compile all sources. These jobs are depended upon by the // archive job above, so must be run first. - paths, err := l.librarySources(target, config.LibcNeedsMalloc()) - if err != nil { - return nil, nil, err - } - for _, path := range paths { + for _, source := range input.Sources { // Strip leading "../" parts off the path. + source := source + path := filepath.FromSlash(source.Path) cleanpath := path for strings.HasPrefix(cleanpath, "../") { cleanpath = cleanpath[3:] @@ -245,9 +598,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ run: func(*compileJob) error { var compileArgs []string compileArgs = append(compileArgs, args...) - if l.cflagsForFile != nil { - compileArgs = append(compileArgs, l.cflagsForFile(path)...) - } + compileArgs = append(compileArgs, source.CFlags...) compileArgs = append(compileArgs, "-o", objpath, srcpath) if config.Options.PrintCommands != nil { config.Options.PrintCommands("clang", compileArgs...) diff --git a/builder/library_test.go b/builder/library_test.go new file mode 100644 index 0000000000..df2b091c65 --- /dev/null +++ b/builder/library_test.go @@ -0,0 +1,101 @@ +package builder + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +func TestHashLibraryInputFilesIncludesNonHeaders(t *testing.T) { + dir := t.TempDir() + for name, contents := range map[string]string{ + "header.h": "header", + "source.c": "source", + "data": "data", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0o666); err != nil { + t.Fatal(err) + } + } + + hashes, err := hashLibraryInputFiles(dir) + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"header.h", "source.c", "data"} { + if _, ok := hashes[name]; !ok { + t.Errorf("input file %q was not hashed", name) + } + } +} + +func TestCompilerInputPaths(t *testing.T) { + args := []string{ + "-I", "include", + "-Iinclude2", + "-isystem", "system", + "-iquotequoted", + "-idirafter", "after", + "-include", "config.h", + "-imacrosmacros.h", + "-resource-dir=resource", + "--sysroot", "sysroot", + "-isysrootsdk", + "-DVALUE=1", + } + want := []string{ + "include", + "include2", + "system", + "quoted", + "after", + "config.h", + "macros.h", + "resource", + "sysroot", + "sdk", + } + if got := compilerInputPaths(args); !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected compiler input paths:\nwant: %#v\ngot: %#v", want, got) + } +} + +func TestHashLibraryCompileInputsTracksIncludeDirectories(t *testing.T) { + includeDir := t.TempDir() + args := []string{"-I", includeDir} + + before, err := hashLibraryCompileInputs(args) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(includeDir, "transitive.c"), []byte("input"), 0o666); err != nil { + t.Fatal(err) + } + after, err := hashLibraryCompileInputs(args) + if err != nil { + t.Fatal(err) + } + if reflect.DeepEqual(before, after) { + t.Fatal("adding a file to an include directory did not change its cache input") + } +} + +func TestHashLibraryInputFilesFollowsSymlinkDirectories(t *testing.T) { + dir := t.TempDir() + target := t.TempDir() + if err := os.WriteFile(filepath.Join(target, "input.c"), []byte("input"), 0o666); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(dir, "linked")); err != nil { + t.Skipf("cannot create symlink: %v", err) + } + + hashes, err := hashLibraryInputFiles(dir) + if err != nil { + t.Fatal(err) + } + if _, ok := hashes["linked/input.c"]; !ok { + t.Fatal("file in symlinked input directory was not hashed") + } +} diff --git a/builder/musl.go b/builder/musl.go index c3f4fe99cb..40cec5a969 100644 --- a/builder/musl.go +++ b/builder/musl.go @@ -121,6 +121,7 @@ var libMusl = Library{ return cflags }, sourceDir: func() string { return filepath.Join(goenv.Get("TINYGOROOT"), "lib/musl/src") }, + inputDir: func() string { return filepath.Join(goenv.Get("TINYGOROOT"), "lib/musl") }, librarySources: func(target string, _ bool) ([]string, error) { arch := compileopts.MuslArchitecture(target) globs := []string{ diff --git a/builder/picolibc.go b/builder/picolibc.go index 43837fa1dc..82fa631995 100644 --- a/builder/picolibc.go +++ b/builder/picolibc.go @@ -43,6 +43,7 @@ var libPicolibc = Library{ } }, sourceDir: func() string { return filepath.Join(goenv.Get("TINYGOROOT"), "lib/picolibc/newlib") }, + inputDir: func() string { return filepath.Join(goenv.Get("TINYGOROOT"), "lib/picolibc") }, librarySources: func(target string, _ bool) ([]string, error) { sources := append([]string(nil), picolibcSources...) if !strings.HasPrefix(target, "avr") { diff --git a/builder/tools.go b/builder/tools.go index 33a26ec37e..2b7e12e446 100644 --- a/builder/tools.go +++ b/builder/tools.go @@ -7,27 +7,96 @@ import ( "go/token" "os" "os/exec" + "path/filepath" "regexp" "strconv" "strings" + "sync" + + "tinygo.org/x/go-llvm" +) + +func appendCacheStableCFlags(flags []string) []string { + return append(flags, "-fdebug-compilation-dir=.") +} + +var ( + clangCompilerOnce sync.Once + clangCompilerPath string + clangCompilerID string + clangCompilerInfoErr error ) +func loadClangCompiler() { + if hasBuiltinTools { + cmd := exec.Command(os.Args[0], "clang", "--version") + cmd.Env = []string{} + out, err := cmd.CombinedOutput() + if err != nil { + clangCompilerInfoErr = fmt.Errorf("failed to identify builtin clang: %w", err) + return + } + clangCompilerPath = os.Args[0] + clangCompilerID = "builtin clang llvm " + llvm.Version + "\n" + string(out) + return + } + + name, err := LookupCommand("clang") + if err != nil { + clangCompilerInfoErr = err + return + } + path, err := exec.LookPath(name) + if err != nil { + clangCompilerInfoErr = err + return + } + path, err = filepath.Abs(path) + if err != nil { + clangCompilerInfoErr = err + return + } + compilerHash, err := hashFile(path) + if err != nil { + clangCompilerInfoErr = err + return + } + cmd := exec.Command(path, "--version") + cmd.Env = []string{} + out, err := cmd.CombinedOutput() + if err != nil { + clangCompilerInfoErr = fmt.Errorf("failed to identify clang: %w", err) + return + } + clangCompilerPath = path + clangCompilerID = path + "\n" + compilerHash + "\n" + string(out) +} + +func clangCompilerIdentity() (string, error) { + clangCompilerOnce.Do(loadClangCompiler) + return clangCompilerID, clangCompilerInfoErr +} + // runCCompiler invokes a C compiler with the given arguments. func runCCompiler(flags ...string) error { - // Find the right command to run Clang. + cmd, err := cCompilerCommand(flags...) + if err != nil { + return err + } + return cmd.Run() +} + +func cCompilerCommand(flags ...string) (*exec.Cmd, error) { + clangCompilerOnce.Do(loadClangCompiler) + if clangCompilerInfoErr != nil { + return nil, clangCompilerInfoErr + } var cmd *exec.Cmd if hasBuiltinTools { - // Compile this with the internal Clang compiler. - cmd = exec.Command(os.Args[0], append([]string{"clang"}, flags...)...) + cmd = exec.Command(clangCompilerPath, append([]string{"clang"}, flags...)...) } else { - // Compile this with an external invocation of the Clang compiler. - name, err := LookupCommand("clang") - if err != nil { - return err - } - cmd = exec.Command(name, flags...) + cmd = exec.Command(clangCompilerPath, flags...) } - cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -52,7 +121,7 @@ func runCCompiler(flags ...string) error { } } - return cmd.Run() + return cmd, nil } // link invokes a linker with the given name and flags. diff --git a/compileopts/config.go b/compileopts/config.go index 7786cd2178..6bcfe1e441 100644 --- a/compileopts/config.go +++ b/compileopts/config.go @@ -9,30 +9,19 @@ import ( "path/filepath" "regexp" "slices" - "strconv" "strings" "github.com/google/shlex" "github.com/tinygo-org/tinygo/goenv" ) -// Library versions. Whenever an existing library is changed, this number should -// be added/increased so that existing caches are invalidated. -// -// (This is a bit of a layering violation, this should really be part of the -// builder.Library struct but that's hard to do since we want to know the -// library path in advance in several places). -var libVersions = map[string]int{ - "musl": 3, - "bdwgc": 2, -} - // Config keeps all configuration affecting the build in a single struct. type Config struct { Options *Options Target *TargetSpec GoMinorVersion int TestConfig TestConfig + LibraryKeys map[string]string } // Triple returns the LLVM target triple, like armv6m-unknown-unknown-eabi. @@ -288,9 +277,8 @@ func (c *Config) LibraryPath(name string) string { archname += "-" + c.Target.Libc } - // Append a version string, if this library has a version. - if v, ok := libVersions[name]; ok { - archname += "-v" + strconv.Itoa(v) + if key, ok := c.LibraryKeys[name]; ok { + archname += "-h" + key } options := "" @@ -371,8 +359,8 @@ func (c *Config) CFlags(libclang bool) []string { } // LibcCFlags returns the C compiler flags for the configured libc. -// It only uses flags that are part of the libc path (triple, cpu, abi, libc -// name) so it can safely be used to compile another C library. +// It only uses flags that are part of the libc path, so it can safely be used +// to compile another C library. func (c *Config) LibcCFlags() []string { switch c.Target.Libc { case "darwin-libSystem":