remotecache: split the import blob size limits (manifest 4 MiB, cache config 16 MiB)

cache/remotecache/import.go

@@ -46,7 +46,7 @@ type contentCacheImporter struct {
 }
 
 func (ci *contentCacheImporter) Resolve(ctx context.Context, desc ocispecs.Descriptor, id string, w worker.Worker) (solver.CacheManager, error) {
-	dt, err := readBlob(ctx, ci.provider, desc)
+	dt, err := readBlob(ctx, ci.provider, desc, maxManifestBlobSize)
 	if err != nil {
 		return nil, err
 	}
@@ -112,7 +112,7 @@ func (ci *contentCacheImporter) Resolve(ctx context.Context, desc ocispecs.Descr
 		return ci.importInlineCache(ctx, dt, id, w)
 	}
 
-	dt, err = readBlob(ctx, ci.provider, configDesc)
+	dt, err = readBlob(ctx, ci.provider, configDesc, maxCacheConfigBlobSize)
 	if err != nil {
 		return nil, err
 	}
@@ -129,8 +129,18 @@ func (ci *contentCacheImporter) Resolve(ctx context.Context, desc ocispecs.Descr
 	return solver.NewCacheManager(ctx, id, keysStorage, resultStorage), nil
 }
 
-func readBlob(ctx context.Context, provider content.Provider, desc ocispecs.Descriptor) ([]byte, error) {
-	maxBlobSize := int64(1 << 20)
+const (
+	// maxManifestBlobSize bounds the cache manifest (image manifest or index)
+	// read from the registry's manifest endpoint, which registries cap at a few MiB.
+	maxManifestBlobSize = int64(4 << 20)
+	// maxCacheConfigBlobSize bounds the cache config blob (records + layers,
+	// application/vnd.buildkit.cacheconfig.v0). It is served from the blob
+	// endpoint, so the manifest ceiling does not apply; a mode=max export of a
+	// large multi-stage build on a busy shared daemon reaches 1-2 MiB.
+	maxCacheConfigBlobSize = int64(16 << 20)
+)
+
+func readBlob(ctx context.Context, provider content.Provider, desc ocispecs.Descriptor, maxBlobSize int64) ([]byte, error) {
 	if desc.Size > maxBlobSize {
 		return nil, errors.Errorf("blob %s is too large (%d > %d)", desc.Digest, desc.Size, maxBlobSize)
 	}

cache/remotecache/import_test.go

@@ -0,0 +1,86 @@
+package remotecache
+
+import (
+	"bytes"
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/containerd/containerd/v2/core/content"
+	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/stretchr/testify/require"
+)
+
+// memProvider serves one blob from memory.
+type memProvider struct {
+	dt []byte
+}
+
+func (p *memProvider) ReaderAt(_ context.Context, _ ocispecs.Descriptor) (content.ReaderAt, error) {
+	return &memReaderAt{Reader: bytes.NewReader(p.dt), size: int64(len(p.dt))}, nil
+}
+
+type memReaderAt struct {
+	*bytes.Reader
+	size int64
+}
+
+func (r *memReaderAt) Size() int64  { return r.size }
+func (r *memReaderAt) Close() error { return nil }
+
+func descFor(dt []byte, mediaType string) ocispecs.Descriptor {
+	return ocispecs.Descriptor{
+		MediaType: mediaType,
+		Digest:    digest.FromBytes(dt),
+		Size:      int64(len(dt)),
+	}
+}
+
+func TestReadBlobLimits(t *testing.T) {
+	ctx := context.Background()
+
+	// The limits must both sit above the 1 MiB ceiling that refused real
+	// cache configs (mode=max exports of large builds, buildkit issues #3719
+	// and #4916); the config ceiling is the larger of the two.
+	require.Greater(t, maxManifestBlobSize, int64(1<<20))
+	require.Greater(t, maxCacheConfigBlobSize, maxManifestBlobSize)
+
+	t.Run("cache config just over 1 MiB imports", func(t *testing.T) {
+		dt := bytes.Repeat([]byte{'x'}, 1<<20+1)
+		desc := descFor(dt, "application/vnd.buildkit.cacheconfig.v0")
+		got, err := readBlob(ctx, &memProvider{dt: dt}, desc, maxCacheConfigBlobSize)
+		require.NoError(t, err)
+		require.Equal(t, dt, got)
+	})
+
+	t.Run("cache config over its ceiling is refused before reading", func(t *testing.T) {
+		desc := ocispecs.Descriptor{
+			MediaType: "application/vnd.buildkit.cacheconfig.v0",
+			Digest:    digest.FromString("unread"),
+			Size:      maxCacheConfigBlobSize + 1,
+		}
+		_, err := readBlob(ctx, &memProvider{dt: nil}, desc, maxCacheConfigBlobSize)
+		require.Error(t, err)
+		require.True(t, strings.Contains(err.Error(), "is too large"), err.Error())
+	})
+
+	t.Run("manifest over its ceiling is refused before reading", func(t *testing.T) {
+		desc := ocispecs.Descriptor{
+			MediaType: ocispecs.MediaTypeImageManifest,
+			Digest:    digest.FromString("unread"),
+			Size:      maxManifestBlobSize + 1,
+		}
+		_, err := readBlob(ctx, &memProvider{dt: nil}, desc, maxManifestBlobSize)
+		require.Error(t, err)
+		require.True(t, strings.Contains(err.Error(), "is too large"), err.Error())
+	})
+
+	t.Run("manifest within its ceiling reads", func(t *testing.T) {
+		dt := []byte(`{"schemaVersion":2}`)
+		desc := descFor(dt, ocispecs.MediaTypeImageManifest)
+		got, err := readBlob(ctx, &memProvider{dt: dt}, desc, maxManifestBlobSize)
+		require.NoError(t, err)
+		require.Equal(t, dt, got)
+	})
+}