Proposal: trim compile-only packages from compiled releases

go.mod

@@ -7,7 +7,7 @@ require (
 	code.cloudfoundry.org/tlsconfig v0.69.0
 	github.com/Microsoft/hcsshim v0.15.0
 	github.com/charlievieth/fs v0.0.3
-	github.com/cloudfoundry/bosh-cli/v7 v7.11.2
+	github.com/cloudfoundry/bosh-cli/v7 v7.11.3-0.20261026181525-2cf7369ee21e
 	github.com/cloudfoundry/bosh-davcli v0.0.519
 	github.com/cloudfoundry/bosh-utils v0.0.666
 	github.com/cloudfoundry/gosigar v1.3.128
@@ -29,6 +29,7 @@ require (
 	golang.org/x/net v0.61.0
 	golang.org/x/sys v0.49.0
 	golang.org/x/tools v0.52.0
+	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 	inet.af/wf v0.0.0-20221017222439-36129f591884
 )

go.sum

@@ -22,8 +22,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/charlievieth/fs v0.0.3 h1:3lZQXTj4PbE81CVPwALSn+JoyCNXkZgORHN6h2XHGlg=
 github.com/charlievieth/fs v0.0.3/go.mod h1:hD4sRzto1Hw8zCua76tNVKZxaeZZr1RiKftjAJQRLLo=
-github.com/cloudfoundry/bosh-cli/v7 v7.11.2 h1://i69MRCyXq6I9gAbq5Q1Ro2/sMJaFltu8DGenaY02Y=
-github.com/cloudfoundry/bosh-cli/v7 v7.11.2/go.mod h1:2DMYkwX9TxRHvQXYIdENGJEpobjQiqb2kaHc/kG1E7k=
+github.com/cloudfoundry/bosh-cli/v7 v7.11.3-0.20261026181525-2cf7369ee21e h1:J+X2uOPlZigaWQKfBWVPgUe5YtaNxEssl7VaV8fU8m0=
+github.com/cloudfoundry/bosh-cli/v7 v7.11.3-0.20261026181525-2cf7369ee21e/go.mod h1:2DMYkwX9TxRHvQXYIdENGJEpobjQiqb2kaHc/kG1E7k=
 github.com/cloudfoundry/bosh-davcli v0.0.519 h1:d4DlSQvh076efPl1XNU3+j9Gn8CpyeL0krj91AiFYlg=
 github.com/cloudfoundry/bosh-davcli v0.0.519/go.mod h1:w3yoiGdDfIxhuBJwexfLT3FIP+dZQKVxXwzPg3uAiT4=
 github.com/cloudfoundry/bosh-utils v0.0.666 h1:i1VF8qngBHANc4Pg3Z0Sz8QwYVPuPNIjwgAtmF44vwc=
@@ -265,6 +265,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

main/compile.go

@@ -41,7 +41,7 @@ func compileTarball(command string, args []string) {
 		log.Fatal(err)
 	}
 	for _, releaseTarballPath := range options.SourceReleases {
-		compiledReleaseTarballPath, err := releasetarball.Compile(compiler, releaseTarballPath, dirProvider.BlobsDir(), options.OutputDirectory, compiledReleaseFileSuffix)
+		compiledReleaseTarballPath, err := releasetarball.Compile(compiler, releaseTarballPath, dirProvider.BlobsDir(), options.OutputDirectory, compiledReleaseFileSuffix, releasetarball.CompileOptions{ExcludeCompileTimePackages: options.ExcludeCompileTimePackages})
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -50,14 +50,23 @@ func compileTarball(command string, args []string) {
 }
 
 type CompileTarballOptions struct {
-	OutputDirectory string
-	SourceReleases  []string
+	OutputDirectory            string
+	SourceReleases             []string
+	ExcludeCompileTimePackages bool
 }
 
 func newCompileTarballOptions(command string, args []string) (CompileTarballOptions, error) {
 	var options CompileTarballOptions
+	flags := newCompileTarballFlagSet(command, &options)
+	err := flags.Parse(args)
+	options.SourceReleases = flags.Args()
+	return options, err
+}
+
+func newCompileTarballFlagSet(command string, options *CompileTarballOptions) *flag.FlagSet {
 	flags := flag.NewFlagSet(command, flag.ExitOnError)
 	flags.StringVar(&options.OutputDirectory, "output-directory", "/tmp", "the directory to put the compiled release tarball")
+	flags.BoolVar(&options.ExcludeCompileTimePackages, "exclude-compile-time-packages", false, "omit compiled packages that no job lists (e.g. Go/Rust toolchains); see notes below")
 	flags.Usage = func() {
 		_, _ = fmt.Fprintf(flags.Output(), //nolint:errcheck
 			`The BOSH Agent %[1]s command creates BOSH Release tarballs with compiled packages from tarballs with source packages.
@@ -69,8 +78,10 @@ Usage:
 Flags:
 `, command)
 		flags.PrintDefaults()
+		_, _ = fmt.Fprintln(flags.Output(), //nolint:errcheck
+			`Notes on --exclude-compile-time-packages:
+	- Compiled output gets a distinct version: +trimmed is appended, or .trimmed if the version already contains +; the filename changes to match. If nothing is excluded, the version and output are unchanged.
+	- Every package listed by any job's job.MF is kept; others are recorded under compile_time_packages. Keep the untrimmed release to roll back. Export trimmed releases only on the stemcell they are compiled for.`)
 	}
-	err := flags.Parse(args)
-	options.SourceReleases = flags.Args()
-	return options, err
+	return flags
 }

main/compile_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"bytes"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("newCompileTarballOptions", func() {
+	It("parses --exclude-compile-time-packages", func() {
+		options, err := newCompileTarballOptions("compile", []string{"--exclude-compile-time-packages", "--output-directory", "/x", "a.tgz"})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(options.ExcludeCompileTimePackages).To(BeTrue())
+		Expect(options.OutputDirectory).To(Equal("/x"))
+		Expect(options.SourceReleases).To(Equal([]string{"a.tgz"}))
+	})
+
+	It("leaves ExcludeCompileTimePackages false without the flag", func() {
+		options, err := newCompileTarballOptions("compile", []string{"a.tgz"})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(options.ExcludeCompileTimePackages).To(BeFalse())
+	})
+})
+
+var _ = Describe("compile usage", func() {
+	It("describes the trimmed output", func() {
+		var options CompileTarballOptions
+		flags := newCompileTarballFlagSet("compile", &options)
+		var usage bytes.Buffer
+		flags.SetOutput(&usage)
+		flags.Usage()
+
+		for _, want := range []string{"+trimmed", ".trimmed", "job.MF", "roll back", "stemcell"} {
+			Expect(usage.String()).To(ContainSubstring(want))
+		}
+	})
+})

main/main_suite_test.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestMainPackage(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Main Suite")
+}

releasetarball/compile.go

@@ -53,6 +53,10 @@ const (
 	defaultMode             = 0o0644
 )
 
+type CompileOptions struct {
+	ExcludeCompileTimePackages bool
+}
+
 // NewCompiler can be used for multiple compilations and should be passed to Compile
 // It expects to be used in a stemcell image and has not been tested on non-warden stemcells.
 func NewCompiler(dirProvider directories.Provider) (boshcomp.Compiler, error) {
@@ -75,7 +79,7 @@ func NewCompiler(dirProvider directories.Provider) (boshcomp.Compiler, error) {
 }
 
 // Compile expects the compiler returned by NewCompiler and may not work with compilers constructed differently.
-func Compile(compiler boshcomp.Compiler, boshReleaseTarballPath, blobsDirectory, outputDirectory, stemcellSlug string) (string, error) {
+func Compile(compiler boshcomp.Compiler, boshReleaseTarballPath, blobsDirectory, outputDirectory, stemcellSlug string, options CompileOptions) (string, error) {
 	log.Printf("Reading BOSH Release Manifest from tarball %s", boshReleaseTarballPath)
 
 	m, err := Manifest(boshReleaseTarballPath)
@@ -96,6 +100,13 @@ func Compile(compiler boshcomp.Compiler, boshReleaseTarballPath, blobsDirectory,
 	if err := topologicalSort(packages, func(p manifest.PackageRef) string { return p.Name }, func(p manifest.PackageRef) []string { return slices.Clone(p.Dependencies) }); err != nil {
 		return "", err
 	}
+	var shipped map[string]bool
+	if options.ExcludeCompileTimePackages {
+		shipped, err = shippedPackages(boshReleaseTarballPath, m)
+		if err != nil {
+			return "", fmt.Errorf("failed to identify shipped packages in release %q: %w", boshReleaseTarballPath, err)
+		}
+	}
 	var compiledPackages []boshmodels.Package
 	for _, p := range packages {
 		compiledPackages, err = compilePackage(compiledPackages, p, blobstoreIDs, compiler)
@@ -106,7 +117,7 @@ func Compile(compiler boshcomp.Compiler, boshReleaseTarballPath, blobsDirectory,
 	log.Printf("Finished packages compilation after %s", time.Since(start))
 
 	log.Printf("Archiving compiled BOSH Release %s/%s with stemcell %s", m.Name, m.Version, stemcellSlug)
-	return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, boshReleaseTarballPath, m.Packages, compiledPackages)
+	return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, boshReleaseTarballPath, m.Packages, compiledPackages, options, shipped)
 }
 
 func compilePackage(compiledPackages []boshmodels.Package, p manifest.PackageRef, blobstoreIDs map[string]string, compiler boshcomp.Compiler) ([]boshmodels.Package, error) {
@@ -198,13 +209,51 @@ func newCompiledPackageRef(pkg boshmodels.Package, src manifest.PackageRef, stem
 	return compiled
 }
 
-func writeCompiledRelease(m manifest.Manifest, outputDirectory, stemcellFilenameSuffix, blobsDirectory, initialTarball string, sourcePackages []manifest.PackageRef, compiledPackages []boshmodels.Package) (string, error) {
+func writeCompiledRelease(m manifest.Manifest, outputDirectory, stemcellFilenameSuffix, blobsDirectory, initialTarball string, sourcePackages []manifest.PackageRef, compiledPackages []boshmodels.Package, options CompileOptions, shipped map[string]bool) (string, error) {
+	identities := map[string]rewrittenPkg{}
+	if options.ExcludeCompileTimePackages {
+		m.CompileTimePackages = nil
+		var err error
+		identities, err = rewriteIdentities(sourcePackages, shipped)
+		if err != nil {
+			return "", err
+		}
+		for _, p := range sourcePackages {
+			if shipped[p.Name] {
+				continue
+			}
+			m.CompileTimePackages = append(m.CompileTimePackages, manifest.CompileTimePackageRef{
+				Name: p.Name, Version: p.Version, Fingerprint: p.Fingerprint, Dependencies: slices.Clone(p.Dependencies),
+			})
+		}
+		slices.SortFunc(m.CompileTimePackages, func(a, b manifest.CompileTimePackageRef) int { return strings.Compare(a.Name, b.Name) })
+		if len(m.CompileTimePackages) > 0 {
+			if strings.Contains(m.Version, "+") {
+				m.Version += ".trimmed"
+			} else {
+				m.Version += "+trimmed"
+			}
+		}
+	}
+
+	keptCompiledPackages := make([]boshmodels.Package, 0, len(compiledPackages))
 	m.CompiledPkgs = make([]manifest.CompiledPackageRef, 0, len(compiledPackages))
 	for _, p := range compiledPackages {
+		if options.ExcludeCompileTimePackages && !shipped[p.Name] {
+			continue
+		}
+		keptCompiledPackages = append(keptCompiledPackages, p)
 		srcIndex := slices.IndexFunc(sourcePackages, func(ref manifest.PackageRef) bool {
 			return ref.Name == p.Name && ref.Version == p.Version
 		})
-		m.CompiledPkgs = append(m.CompiledPkgs, newCompiledPackageRef(p, sourcePackages[srcIndex], stemcellFilenameSuffix))
+		compiledRef := newCompiledPackageRef(p, sourcePackages[srcIndex], stemcellFilenameSuffix)
+		if identity := identities[p.Name]; identity.Rewritten {
+			compiledRef.Version = identity.Fingerprint
+			compiledRef.Fingerprint = identity.Fingerprint
+			compiledRef.OriginalFingerprint = identity.OriginalFingerprint
+			compiledRef.Dependencies = slices.Clone(identity.Dependencies)
+		}
+		m.CompiledPkgs = append(m.CompiledPkgs, compiledRef)
 	}
 	empty := [0]manifest.PackageRef{}
 	m.Packages = empty[:]
@@ -232,7 +281,7 @@ func writeCompiledRelease(m manifest.Manifest, outputDirectory, stemcellFilename
 	}
 	defer closeAndIgnoreErr(tw)
 
-	err = walkTarballFiles(initialTarball, writeCompiledTarballFiles(tw, compiledPackages, releaseManifestBuffer, blobsDirectory))
+	err = walkTarballFiles(initialTarball, writeCompiledTarballFiles(tw, keptCompiledPackages, releaseManifestBuffer, blobsDirectory))
 	if err != nil {
 		return "", errors.Join(err, os.RemoveAll(filePath))
 	}