From cc1fd64e733ac4c155fb2623393566f142cb75cb Mon Sep 17 00:00:00 2001 From: Max Proske Date: Thu, 15 Oct 2026 20:50:07 -0700 Subject: [PATCH] feat(build): pass parallel limit to bake as --jobs Signed-off-by: Max Proske --- docs/reference/compose.md | 8 +++-- docs/reference/docker_compose.yaml | 8 +++-- pkg/compose/api_versions.go | 3 ++ pkg/compose/build_bake.go | 47 ++++++++++++++++++++++++++++++ pkg/compose/build_bake_test.go | 44 ++++++++++++++++++++++++++++ 5 files changed, 106 insertions(+), 4 deletions(-) diff --git a/docs/reference/compose.md b/docs/reference/compose.md index 50ebfba473d..d709074e1f5 100644 --- a/docs/reference/compose.md +++ b/docs/reference/compose.md @@ -215,9 +215,13 @@ Profiles can also be set by `COMPOSE_PROFILES` environment variable. ### Configuring parallelism -Use `--parallel` to specify the maximum level of parallelism for concurrent engine calls. +Use `--parallel` to specify the maximum level of parallelism for concurrent engine calls or builds. Calling `docker compose --parallel 1 pull` pulls the pullable images defined in the Compose file -one at a time. This can also be used to control build concurrency. +one at a time. + +Limiting build concurrency requires Buildx 0.38.0 or later. When services are linked by +`additional_contexts` (`service:`), Buildx rejects a limit lower than the number of images to +build, so Compose ignores it with a warning. Parallelism can also be set by the `COMPOSE_PARALLEL_LIMIT` environment variable. diff --git a/docs/reference/docker_compose.yaml b/docs/reference/docker_compose.yaml index c5fdb937510..e1fe5f1a57e 100644 --- a/docs/reference/docker_compose.yaml +++ b/docs/reference/docker_compose.yaml @@ -389,9 +389,13 @@ examples: |- ### Configuring parallelism - Use `--parallel` to specify the maximum level of parallelism for concurrent engine calls. + Use `--parallel` to specify the maximum level of parallelism for concurrent engine calls or builds. Calling `docker compose --parallel 1 pull` pulls the pullable images defined in the Compose file - one at a time. This can also be used to control build concurrency. + one at a time. + + Limiting build concurrency requires Buildx 0.38.0 or later. When services are linked by + `additional_contexts` (`service:`), Buildx rejects a limit lower than the number of images to + build, so Compose ignores it with a warning. Parallelism can also be set by the `COMPOSE_PARALLEL_LIMIT` environment variable. diff --git a/pkg/compose/api_versions.go b/pkg/compose/api_versions.go index d39aef8aacd..a8d3102c7b9 100644 --- a/pkg/compose/api_versions.go +++ b/pkg/compose/api_versions.go @@ -64,4 +64,7 @@ const ( const ( // buildxMinVersion is the minimum required version of buildx for compose build buildxMinVersion = "0.17.0" + + // buildxJobsMinVersion is the minimum version of buildx supporting `bake --jobs` + buildxJobsMinVersion = "0.38.0" ) diff --git a/pkg/compose/build_bake.go b/pkg/compose/build_bake.go index 09722d26fcc..4ae20b303f9 100644 --- a/pkg/compose/build_bake.go +++ b/pkg/compose/build_bake.go @@ -31,6 +31,7 @@ import ( "os/exec" "path/filepath" "slices" + "strconv" "strings" "github.com/compose-spec/compose-go/v2/types" @@ -131,6 +132,7 @@ type bakeBuild struct { localWritePaths []string // local cache destinations bake needs `--allow fs.write` for entitlements []string secretsEnv []string + jobs int // bake `--jobs` limit, 0 for unlimited } func (s *composeService) doBuildBake(ctx context.Context, project *types.Project, serviceToBeBuild types.Services, options api.BuildOptions) (map[string]string, error) { @@ -177,6 +179,7 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project if err != nil { return nil, err } + bake.jobs = bakeJobs(bake.cfg, s.maxConcurrency, buildx.Version) args := bakeArgs(bake, metadataFile, options) logrus.Debugf("Executing bake with args: %v", args) @@ -498,12 +501,56 @@ func bakeArgs(bake *bakeBuild, metadataFile string, options api.BuildOptions) [] if options.Builder != "" { args = append(args, "--builder", options.Builder) } + if bake.jobs > 0 { + args = append(args, "--jobs", strconv.Itoa(bake.jobs)) + } if options.Quiet { args = append(args, "--progress=quiet") } return args } +// bakeJobs returns how many targets bake can build concurrently +func bakeJobs(cfg bakeConfig, maxConcurrency int, buildxVersion string) int { + // no limit + if maxConcurrency <= 0 { + return 0 + } + // limit doesn't hold target back + targets, linked := bakeTargets(cfg) + if maxConcurrency >= targets { + return 0 + } + if versions.LessThan(buildxVersion[1:], buildxJobsMinVersion) { + logrus.Warnf("limiting build concurrency requires buildx %s or later, ignoring parallel limit", buildxJobsMinVersion) + return 0 + } + // bake rejects a limit lower than its target count once any are linked by `target:` contexts + if linked { + logrus.Warnf("bake does not support a parallel limit lower than its %d linked build targets, ignoring parallel limit", targets) + return 0 + } + return maxConcurrency +} + +// bakeTargets counts the targets bake builds for the default group, those +// linked in by `target:` contexts included, and reports whether any link exists. +func bakeTargets(cfg bakeConfig) (int, bool) { + targets := slices.Clone(cfg.Groups["default"].Targets) + linked := false + for i := 0; i < len(targets); i++ { + for _, source := range cfg.Targets[targets[i]].Contexts { + if target, ok := strings.CutPrefix(source, "target:"); ok { + linked = true + if !slices.Contains(targets, target) { + targets = append(targets, target) + } + } + } + } + return len(targets), linked +} + // waitBake forwards the solve statuses of a started bake command's rawjson // stderr stream to ch, then waits for the command to exit. It returns the // lines of the stream that are not solve statuses — where bake tells why it diff --git a/pkg/compose/build_bake_test.go b/pkg/compose/build_bake_test.go index cc80e116206..da77eb944a8 100644 --- a/pkg/compose/build_bake_test.go +++ b/pkg/compose/build_bake_test.go @@ -33,6 +33,7 @@ import ( "github.com/compose-spec/compose-go/v2/types" "github.com/docker/cli/cli/streams" "github.com/moby/buildkit/client" + logrustest "github.com/sirupsen/logrus/hooks/test" "gotest.tools/v3/assert" "github.com/docker/compose/v5/pkg/api" @@ -149,6 +150,49 @@ func TestBakeReadGrants(t *testing.T) { }) } +func TestBakeJobs(t *testing.T) { + group := func(targets ...string) bakeConfig { + return bakeConfig{ + Groups: map[string]bakeGroup{"default": {Targets: targets}}, + Targets: map[string]bakeTarget{ + "app": {Contexts: map[string]string{"base": "target:base"}}, + "base": {}, + "db": {}, + "web": {}, + }, + } + } + for _, tc := range []struct { + name string + cfg bakeConfig + maxConcurrency int + buildx string + want int + warn bool + }{ + {name: "unlimited", cfg: group("base", "db", "web"), maxConcurrency: -1, buildx: "v0.38.0"}, + {name: "limited, link not built", cfg: group("base", "db", "web"), maxConcurrency: 2, buildx: "v0.38.0", want: 2}, + {name: "limit holds no target back", cfg: group("base", "db"), maxConcurrency: 2, buildx: "v0.38.0"}, + {name: "buildx too old", cfg: group("base", "db", "web"), maxConcurrency: 2, buildx: "v0.37.2", warn: true}, + {name: "linked targets", cfg: group("app", "base", "db"), maxConcurrency: 2, buildx: "v0.38.0", warn: true}, + {name: "linked targets, limit holds no target back", cfg: group("app", "base", "db"), maxConcurrency: 3, buildx: "v0.38.0"}, + {name: "link outside group, limit too low", cfg: group("app", "db"), maxConcurrency: 2, buildx: "v0.38.0", warn: true}, + {name: "link outside group, limit holds no target back", cfg: group("app", "db"), maxConcurrency: 3, buildx: "v0.38.0"}, + } { + t.Run(tc.name, func(t *testing.T) { + hook := logrustest.NewGlobal() + assert.Equal(t, bakeJobs(tc.cfg, tc.maxConcurrency, tc.buildx), tc.want) + assert.Equal(t, len(hook.AllEntries()) > 0, tc.warn) + }) + } +} + +func TestBakeArgsJobs(t *testing.T) { + want := []string{"bake", "--file", "-", "--progress", "rawjson", "--metadata-file", "metadata.json"} + assert.DeepEqual(t, bakeArgs(&bakeBuild{}, "metadata.json", api.BuildOptions{}), want) + assert.DeepEqual(t, bakeArgs(&bakeBuild{jobs: 2}, "metadata.json", api.BuildOptions{}), append(want, "--jobs", "2")) +} + func TestPrepareBakeBuildGrants(t *testing.T) { root := t.TempDir() contextDir := filepath.Join(root, "app")