remotecache/s3: honor compression attributes

README.md

@@ -596,6 +596,9 @@ Beware, these configurations must be available at buildkit daemon level, not at
 * `ignore-error=<false|true>`: specify if error is ignored in case cache export fails (default: `false`)
 * `touch_refresh=24h`: Instead of being uploaded again when not changed, blobs files will be "touched" on s3 every `touch_refresh`, default is 24h. Due to this, an expiration policy can be set on the S3 bucket to cleanup useless files automatically. Manifests files are systematically rewritten, there is no need to touch them.
 * `upload_parallelism=4`: This parameter changes the number of layers uploaded to s3 in parallel. Each individual layer is uploaded with 5 threads, using the Upload manager provided by the AWS SDK.
+* `compression=<uncompressed|gzip|estargz|zstd>`: choose compression type for layers newly created and cached, gzip is default value. `estargz` layers are recorded as plain `gzip` layers in the s3 cache manifest (their eStargz annotations are not preserved), so they cannot be lazily pulled from s3
+* `compression-level=<value>`: compression level for gzip, estargz (0-9) and zstd (0-22)
+* `force-compression=true`: forcibly apply `compression` option to all layers
 * `retry_mode=<standard|adaptive>`: sets the AWS SDK retry mode (default: `standard`). `standard` uses exponential backoff, `adaptive` adds client-side rate limiting. See [AWS retry documentation](https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html).
 * `retry_max_attempts=<int>`: sets the maximum number of attempts for each S3 request, including the initial request and all retries (default: 3). Must be a positive integer.
 

cache/remotecache/s3/s3.go

@@ -208,20 +208,26 @@ func ResolveCacheExporterFunc() remotecache.ResolveCacheExporterFunc {
 			return nil, err
 		}
 
+		compressionConfig, err := compression.ParseAttributes(attrs)
+		if err != nil {
+			return nil, err
+		}
+
 		s3Client, err := newS3Client(ctx, config)
 		if err != nil {
 			return nil, err
 		}
 		cc := v1.NewCacheChains()
-		return &exporter{CacheExporterTarget: cc, chains: cc, s3Client: s3Client, config: config}, nil
+		return &exporter{CacheExporterTarget: cc, chains: cc, s3Client: s3Client, config: config, compression: compressionConfig}, nil
 	}
 }
 
 type exporter struct {
 	solver.CacheExporterTarget
-	chains   *v1.CacheChains
-	s3Client *s3Client
-	config   Config
+	chains      *v1.CacheChains
+	s3Client    *s3Client
+	config      Config
+	compression compression.Config
 }
 
 func (*exporter) Name() string {
@@ -230,7 +236,7 @@ func (*exporter) Name() string {
 
 func (e *exporter) Config() remotecache.Config {
 	return remotecache.Config{
-		Compression: compression.New(compression.Default),
+		Compression: e.compression,
 	}
 }
 

cache/remotecache/s3/s3_test.go

@@ -0,0 +1,111 @@
+package s3
+
+import (
+	"maps"
+	"path/filepath"
+	"testing"
+
+	"github.com/moby/buildkit/cache/remotecache"
+	"github.com/moby/buildkit/util/compression"
+	"github.com/stretchr/testify/require"
+)
+
+// TestExporterCompressionAttributes checks that the compression attributes
+// passed to --export-cache are reflected in the exporter config instead of
+// always falling back to the default compression, and that invalid values are
+// rejected when the exporter is resolved.
+func TestExporterCompressionAttributes(t *testing.T) {
+	tests := []struct {
+		name    string
+		attrs   map[string]string
+		want    compression.Config
+		wantErr string
+	}{
+		{
+			name:  "default",
+			attrs: map[string]string{},
+			want:  compression.New(compression.Default),
+		},
+		{
+			name:  "gzip",
+			attrs: map[string]string{"compression": "gzip"},
+			want:  compression.New(compression.Gzip),
+		},
+		{
+			name:  "zstd",
+			attrs: map[string]string{"compression": "zstd"},
+			want:  compression.New(compression.Zstd),
+		},
+		{
+			name:  "uncompressed",
+			attrs: map[string]string{"compression": "uncompressed"},
+			want:  compression.New(compression.Uncompressed),
+		},
+		{
+			name:  "estargz",
+			attrs: map[string]string{"compression": "estargz"},
+			want:  compression.New(compression.EStargz),
+		},
+		{
+			name:  "level",
+			attrs: map[string]string{"compression": "zstd", "compression-level": "12"},
+			want:  compression.New(compression.Zstd).SetLevel(12),
+		},
+		{
+			name:  "force",
+			attrs: map[string]string{"compression": "zstd", "force-compression": "true"},
+			want:  compression.New(compression.Zstd).SetForce(true),
+		},
+		{
+			name:  "force without value",
+			attrs: map[string]string{"force-compression": ""},
+			want:  compression.New(compression.Default).SetForce(true),
+		},
+		{
+			name:    "unknown compression type",
+			attrs:   map[string]string{"compression": "lzma"},
+			wantErr: "unsupported compression type lzma",
+		},
+		{
+			name:    "non-integer compression level",
+			attrs:   map[string]string{"compression-level": "fastest"},
+			wantErr: "non-integer value fastest specified for compression-level",
+		},
+		{
+			name:    "non-bool force compression",
+			attrs:   map[string]string{"force-compression": "yes please"},
+			wantErr: "non-bool value yes please specified for force-compression",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			exp, err := resolveTestExporter(t, tt.attrs)
+			if tt.wantErr != "" {
+				require.ErrorContains(t, err, tt.wantErr)
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, tt.want, exp.Config().Compression)
+		})
+	}
+}
+
+// resolveTestExporter resolves the s3 exporter for attrs on top of the
+// minimal required ones. Resolving builds a real AWS SDK client whose config
+// loader reads the ambient AWS environment, so point it at nothing: otherwise
+// an AWS_PROFILE or ~/.aws/config on the developer's machine can fail the
+// test for reasons unrelated to the attributes.
+func resolveTestExporter(t *testing.T, attrs map[string]string) (remotecache.Exporter, error) {
+	t.Helper()
+	t.Setenv("AWS_PROFILE", "")
+	t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "config"))
+	t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "credentials"))
+
+	all := map[string]string{
+		attrBucket: "bucket",
+		attrRegion: "us-east-1",
+	}
+	maps.Copy(all, attrs)
+	return ResolveCacheExporterFunc()(t.Context(), nil, all)
+}