Refuse package mirroring from the CLI while scanning is enabled

README.md

@@ -1012,6 +1012,8 @@ proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20
 
 The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped.
 
+With `scanning.enabled`, the mirror command refuses every package and container image, reporting each one as failed, so nothing is cached unscanned. Scanners fetch the staged bytes from the running server, which this command does not start. Use the mirror API below instead, since its jobs run inside the server and go through the pre-cache scan. `--dry-run` still lists what would be mirrored.
+
 A mirror API is also available when the server is running:
 
 ```bash

cmd/proxy/main.go

@@ -558,9 +558,7 @@ func runMirror() {
 	m := mirror.New(proxy, db, store, logger, *concurrency)
 	m.Images = images
 	if cfg.Scanning.Enabled {
-		// Scanners fetch staged layers from the running server, which this
-		// command does not start.
-		m.Images = mirror.UnavailableImages{Reason: "container images cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server"}
+		refuseUnscannedMirroring(m)
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -601,6 +599,23 @@ func runMirror() {
 	}
 }
 
+// Reasons the mirror command gives for refusing items while scanning is
+// enabled.
+const (
+	mirrorScanningImagesReason   = "container images cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server"
+	mirrorScanningPackagesReason = "packages cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server"
+)
+
+// refuseUnscannedMirroring makes m fail every item instead of caching it.
+// Scanners fetch the staged bytes from the running server, which this command
+// does not start, so anything it cached would skip the pre-cache scan and
+// later be served from the cache unscanned. The mirror API's jobs run inside
+// the server with its scanners and are unaffected.
+func refuseUnscannedMirroring(m *mirror.Mirror) {
+	m.Images = mirror.UnavailableImages{Reason: mirrorScanningImagesReason}
+	m.RefusePackages = mirrorScanningPackagesReason
+}
+
 func printStats(db *database.DB, popular, recent int, asJSON bool) error {
 	defer func() { _ = db.Close() }()
 

cmd/proxy/mirror_test.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/git-pkgs/proxy/internal/database"
+	"github.com/git-pkgs/proxy/internal/mirror"
+)
+
+// mirrorTestEnv carries the mirror command's arguments into the helper
+// process, one per line.
+const mirrorTestEnv = "PROXY_TEST_MIRROR_ARGS"
+
+// writeScanningMirrorConfig writes a config with scanning enabled and storage
+// and database under dir. The scanner address is never reached.
+func writeScanningMirrorConfig(t *testing.T, dir string) string {
+	t.Helper()
+	cfg := "storage:\n" +
+		"  url: file://" + filepath.Join(dir, "artifacts") + "\n" +
+		"database:\n" +
+		"  path: " + filepath.Join(dir, "proxy.db") + "\n" +
+		"scanning:\n" +
+		"  enabled: true\n" +
+		"  signing_key: test-signing-key\n" +
+		"  scanners:\n" +
+		"    - name: test\n" +
+		"      url: http://127.0.0.1:1/scan\n"
+	path := filepath.Join(dir, "config.yaml")
+	if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
+		t.Fatalf("writing config: %v", err)
+	}
+	return path
+}
+
+func runMirrorProcess(t *testing.T, args ...string) string {
+	t.Helper()
+	cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorHelperProcess$")
+	cmd.Env = append(os.Environ(), mirrorTestEnv+"="+strings.Join(args, "\n"))
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("proxy mirror %v failed: %v\n%s", args, err, output)
+	}
+	return string(output)
+}
+
+// With scanning enabled, proxy mirror refuses packages rather than caching
+// them unscanned: scanners fetch staged bytes from the running server, which
+// the command does not start.
+func TestMirrorCommandRefusesPackagesWhileScanning(t *testing.T) {
+	dir := t.TempDir()
+	configPath := writeScanningMirrorConfig(t, dir)
+
+	output := runMirrorProcess(t, "-config", configPath, "pkg:npm/lodash@4.17.21")
+
+	for _, want := range []string{mirrorScanningPackagesReason, "0 downloaded", "1 failed"} {
+		if !strings.Contains(output, want) {
+			t.Errorf("output missing %q:\n%s", want, output)
+		}
+	}
+
+	db, err := database.Open(filepath.Join(dir, "proxy.db"))
+	if err != nil {
+		t.Fatalf("opening database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+	stats, err := db.GetCacheStats()
+	if err != nil {
+		t.Fatalf("GetCacheStats: %v", err)
+	}
+	if stats.TotalArtifacts != 0 {
+		t.Errorf("cached artifacts = %d, want 0", stats.TotalArtifacts)
+	}
+
+	var stored []string
+	_ = filepath.WalkDir(filepath.Join(dir, "artifacts"), func(path string, d os.DirEntry, err error) error {
+		if err == nil && !d.IsDir() {
+			stored = append(stored, path)
+		}
+		return nil
+	})
+	if len(stored) != 0 {
+		t.Errorf("files written to storage: %v", stored)
+	}
+}
+
+// A dry run caches nothing, so it still lists what would be mirrored.
+func TestMirrorCommandDryRunWhileScanning(t *testing.T) {
+	configPath := writeScanningMirrorConfig(t, t.TempDir())
+
+	output := runMirrorProcess(t, "-config", configPath, "-dry-run", "pkg:npm/lodash@4.17.21")
+
+	if !strings.Contains(output, "Would mirror 1 package versions") {
+		t.Errorf("dry run output:\n%s", output)
+	}
+	if strings.Contains(output, mirrorScanningPackagesReason) {
+		t.Errorf("dry run reported the refusal:\n%s", output)
+	}
+}
+
+func TestRefuseUnscannedMirroring(t *testing.T) {
+	m := mirror.New(nil, nil, nil, nil, 1)
+	refuseUnscannedMirroring(m)
+
+	if m.RefusePackages != mirrorScanningPackagesReason {
+		t.Errorf("RefusePackages = %q, want %q", m.RefusePackages, mirrorScanningPackagesReason)
+	}
+	images, ok := m.Images.(mirror.UnavailableImages)
+	if !ok || images.Reason != mirrorScanningImagesReason {
+		t.Errorf("Images = %#v, want UnavailableImages with the scanning reason", m.Images)
+	}
+}
+
+func TestMirrorHelperProcess(*testing.T) {
+	args := os.Getenv(mirrorTestEnv)
+	if args == "" {
+		return
+	}
+	os.Args = append([]string{"proxy", "mirror"}, strings.Split(args, "\n")...)
+	main()
+}

docs/configuration.md

@@ -860,6 +860,8 @@ Scanners configured for the same ecosystem all run concurrently, never sequentia
 
 A blocked download surfaces to the client as `403 Forbidden` with the scanner's reason, across every ecosystem handler.
 
+Because scanners pull the staged bytes from the running server, the `proxy mirror` command cannot scan what it fetches. While scanning is enabled it refuses every package and container image instead of caching them unscanned. Mirror through the `/api/mirror` endpoints of the running server, whose jobs are scanned like any other download.
+
 ### Scanner HTTP contract
 
 Any external service that implements this contract can act as a scanner — a trivy wrapper, a clamav-rest bridge, a Wiz connector, or an in-house service. The proxy POSTs a notify request to `scanning.scanners[].url` and waits for a JSON verdict.

internal/mirror/mirror.go

@@ -26,6 +26,10 @@ type Mirror struct {
 	// Images mirrors docker and oci items. Without it they fail like any
 	// other unsupported ecosystem.
 	Images ImageMirrorer
+
+	// RefusePackages, when set, fails every item Images does not take with
+	// this reason instead of fetching it, so nothing is cached.
+	RefusePackages string
 }
 
 // New creates a new Mirror with the given dependencies.
@@ -211,6 +215,13 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog
 		m.mirrorImage(ctx, pv, tracker)
 		return
 	}
+	if m.RefusePackages != "" {
+		tracker.failed.Add(1)
+		tracker.addError(pv.Ecosystem, pv.Name, pv.Version, m.RefusePackages)
+		m.logger.Warn("mirror refused",
+			"ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "reason", m.RefusePackages)
+		return
+	}
 	result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "")
 	if err != nil {
 		tracker.failed.Add(1)

internal/mirror/mirror_test.go

@@ -3,10 +3,12 @@ package mirror
 import (
 	"context"
 	"database/sql"
+	"errors"
 	"log/slog"
 	"net/http"
 	"os"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -261,3 +263,90 @@ func TestNewMirrorDefaultWorkers(t *testing.T) {
 		t.Errorf("workers = %d, want 1 (minimum)", m.workers)
 	}
 }
+
+// recordingFetcher fails every fetch and records the URLs it was asked for, so
+// a test can show nothing reached upstream.
+type recordingFetcher struct {
+	mu   sync.Mutex
+	urls []string
+}
+
+func (f *recordingFetcher) record(url string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.urls = append(f.urls, url)
+	return errors.New("recordingFetcher: no upstream in this test")
+}
+
+func (f *recordingFetcher) Fetch(_ context.Context, url string) (*fetch.Artifact, error) {
+	return nil, f.record(url)
+}
+
+func (f *recordingFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) {
+	return nil, f.record(url)
+}
+
+func (f *recordingFetcher) Head(_ context.Context, url string) (int64, string, error) {
+	return 0, "", f.record(url)
+}
+
+func (f *recordingFetcher) fetched() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.urls...)
+}
+
+func TestMirrorRunRefusesPackages(t *testing.T) {
+	m := setupTestMirror(t, 2)
+	fetcher := &recordingFetcher{}
+	m.proxy.Fetcher = fetcher
+	const reason = "packages cannot be mirrored here"
+	m.RefusePackages = reason
+
+	source := &PURLSource{PURLs: []string{"pkg:npm/lodash@4.17.21", "pkg:cargo/serde@1.0.0"}}
+	progress, err := m.Run(context.Background(), source)
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	if progress.Failed != 2 || progress.Completed != 0 || progress.Skipped != 0 {
+		t.Errorf("progress = %d failed, %d completed, %d skipped; want 2 failed only",
+			progress.Failed, progress.Completed, progress.Skipped)
+	}
+	if len(progress.Errors) != 2 {
+		t.Fatalf("errors = %+v, want one per package", progress.Errors)
+	}
+	for _, e := range progress.Errors {
+		if e.Error != reason {
+			t.Errorf("%s/%s error = %q, want %q", e.Ecosystem, e.Name, e.Error, reason)
+		}
+	}
+	if urls := fetcher.fetched(); len(urls) != 0 {
+		t.Errorf("refused packages were fetched from upstream: %v", urls)
+	}
+
+	stats, err := m.db.GetCacheStats()
+	if err != nil {
+		t.Fatalf("GetCacheStats: %v", err)
+	}
+	if stats.TotalArtifacts != 0 {
+		t.Errorf("cached artifacts = %d, want 0", stats.TotalArtifacts)
+	}
+}
+
+// Images keep their own refusal, so a mirror refusing packages still hands
+// docker and oci items to Images.
+func TestMirrorRefusePackagesLeavesImagesToImages(t *testing.T) {
+	m := setupTestMirror(t, 1)
+	m.proxy.Fetcher = &recordingFetcher{}
+	m.RefusePackages = "packages refused"
+	m.Images = UnavailableImages{Reason: "images refused"}
+
+	progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:docker/library/alpine@3.20"}})
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if len(progress.Errors) != 1 || progress.Errors[0].Error != "images refused" {
+		t.Errorf("errors = %+v, want the image refusal", progress.Errors)
+	}
+}