diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index e02c688fb9..4c5c026158 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -134,6 +134,13 @@ jobs: go test -race -short -count=1 -v ./e2e/tests/up-docker-compose \ -run '^TestExpectedUIDMapping$' + - name: test workspace fixture helpers + run: go test -race -short -count=1 -timeout 90s ./e2e/tests/up + + - name: test Linux registry fixture + if: runner.os == 'Linux' + run: go test -race -count=1 -timeout 90s ./e2e/framework/registryfixture + provider-update-windows: name: Provider Update Tests (windows-latest) needs: [changes] @@ -257,6 +264,17 @@ jobs: CGO_ENABLED: 0 run: go build -trimpath -o "${RUNNER_TEMP}/devsy-ci-microsandbox" ./hack/microsandbox + - name: build pinned runtime performance tools + if: runner.os == 'Linux' + run: '"${RUNNER_TEMP}/devsy-ci-microsandbox" build-performance-tools --output-dir "${RUNNER_TEMP}/runtime-performance-tools"' + + - name: upload runtime performance tools + if: runner.os == 'Linux' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runtime-performance-tools-linux + path: ${{ runner.temp }}/runtime-performance-tools/ + - name: upload MicroSandbox CI helper if: runner.os == 'Linux' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -407,6 +425,9 @@ jobs: expected-spec-count: 6 - label: up-provider-microsandbox-compatibility expected-spec-count: 2 + - label: up-provider-microsandbox-registry-consumer + expected-spec-count: 3 + provider-mode: external runs-on: ubuntu-latest timeout-minutes: 25 steps: @@ -428,6 +449,28 @@ jobs: name: e2e-test-linux path: ./e2e/ + - name: download runtime performance tools + if: matrix.label == 'up-provider-microsandbox' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runtime-performance-tools-linux + path: ${{ runner.temp }}/runtime-performance-tools/ + + - name: prepare runtime performance capture + if: matrix.label == 'up-provider-microsandbox' + shell: bash + run: | + chmod +x "${RUNNER_TEMP}/runtime-performance-tools/devsy-runtime-spawn-probe" \ + "${RUNNER_TEMP}/runtime-performance-tools/devsy-runtime-supervisor" + install -d -m 700 "${RUNNER_TEMP}/microsandbox-performance" + { + echo "DEVSY_TEST_RUNTIME_SPAWN_PROBE=${RUNNER_TEMP}/runtime-performance-tools/devsy-runtime-spawn-probe" + echo "DEVSY_TEST_RUNTIME_SUPERVISOR=${RUNNER_TEMP}/runtime-performance-tools/devsy-runtime-supervisor" + echo "DEVSY_TEST_RUNTIME_PERFORMANCE_DIR=${RUNNER_TEMP}/microsandbox-performance" + echo "DEVSY_TEST_RUNTIME_REVISION=${{ github.sha }}" + echo "DEVSY_TEST_RUNTIME_SDK_VERSION=$(cat "${RUNNER_TEMP}/runtime-performance-tools/sdk-version")" + } >> "$GITHUB_ENV" + - name: download MicroSandbox CI helper uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -466,7 +509,7 @@ jobs: GH_CREDENTIAL_USERNAME: x-access-token TEST_TIMEOUT: 1200s DEVSY_REQUIRE_MICROSANDBOX: "true" - run: '"${RUNNER_TEMP}/devsy-ci-microsandbox" run --label "${{ matrix.label }}" --expected-count "${{ matrix.expected-spec-count }}"' + run: '"${RUNNER_TEMP}/devsy-ci-microsandbox" run --label "${{ matrix.label }}" --expected-count "${{ matrix.expected-spec-count }}" --provider-mode "${{ matrix.provider-mode || ''parity'' }}"' - name: upload MicroSandbox test evidence if: always() @@ -479,6 +522,15 @@ jobs: if-no-files-found: warn retention-days: 14 + - name: upload runtime performance evidence + if: always() && matrix.label == 'up-provider-microsandbox' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: microsandbox-runtime-performance + path: ${{ runner.temp }}/microsandbox-performance/provider-performance.json + if-no-files-found: error + retention-days: 14 + integration-tests: name: Test ${{ matrix.label }}${{ (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && format(' ({0})', matrix.install-podman) || '' }} on ${{ matrix.runner }} needs: [changes, can-read-secret, build-cli] diff --git a/e2e/credential_helper_linux_test.go b/e2e/credential_helper_linux_test.go new file mode 100644 index 0000000000..00643edb90 --- /dev/null +++ b/e2e/credential_helper_linux_test.go @@ -0,0 +1,9 @@ +//go:build linux + +package e2e + +import "github.com/devsy-org/devsy/e2e/framework/registryfixture" + +func dispatchFixtureCredentialHelper() (bool, int) { + return registryfixture.DispatchCredentialHelper() +} diff --git a/e2e/credential_helper_other_test.go b/e2e/credential_helper_other_test.go new file mode 100644 index 0000000000..90fe4ab4bc --- /dev/null +++ b/e2e/credential_helper_other_test.go @@ -0,0 +1,7 @@ +//go:build !linux + +package e2e + +func dispatchFixtureCredentialHelper() (bool, int) { + return false, 0 +} diff --git a/e2e/e2e_suite_test.go b/e2e/e2e_suite_test.go index abe2f0af1c..09878ce04c 100644 --- a/e2e/e2e_suite_test.go +++ b/e2e/e2e_suite_test.go @@ -50,6 +50,9 @@ func TestMain(m *testing.M) { cmd.Execute() return } + if handled, code := dispatchFixtureCredentialHelper(); handled { + os.Exit(code) + } os.Exit(m.Run()) } diff --git a/e2e/framework/registryfixture/credential_linux.go b/e2e/framework/registryfixture/credential_linux.go new file mode 100644 index 0000000000..9eb8e490dd --- /dev/null +++ b/e2e/framework/registryfixture/credential_linux.go @@ -0,0 +1,517 @@ +//go:build linux + +package registryfixture + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "syscall" +) + +const ( + // #nosec G101 -- This is an environment variable name, not a credential value. + CredentialConfigEnv = "CASE_A_CREDENTIAL_CONFIG" + // #nosec G101 -- This is an environment variable name, not a credential value. + CredentialLogEnv = "CASE_A_CREDENTIAL_OBSERVATIONS" + // #nosec G101 -- This is an environment variable name, not a credential value. + CredentialRootEnv = "CASE_A_CREDENTIAL_ROOT" + CredentialOperationEnv = "CASE_A_OPERATION_ID" + HelperBinaryName = "docker-credential-fixture" +) + +// CredentialConfig fields are stored only in the private fixture root. +type CredentialConfig struct { + OperationID string `json:"operation_id"` + ServerKey string `json:"server_key"` + Username string `json:"username,omitempty"` + Secret string `json:"secret,omitempty"` + CredentialsAvailable bool `json:"credentials_available"` + ExpectedProviderSHA256 string `json:"expected_provider_sha256"` +} + +// CredentialObservation contains secret-free helper invocation evidence. +type CredentialObservation struct { + OperationID string `json:"operation_id"` + ServerKey string `json:"server_key"` + Action string `json:"action"` + Parent ProcessIdentity `json:"parent"` + Outcome string `json:"outcome"` +} + +// ProcessIdentity records a live process identity without cmdline or env. +type ProcessIdentity struct { + PID int `json:"pid"` + StartTimeTicks uint64 `json:"start_time_ticks"` + Executable string `json:"executable"` + SHA256 string `json:"sha256"` +} + +const credentialsNotFoundMessage = "credentials not found in native keychain" + +var ( + errCredentialRequestRead = errors.New("credential request could not be read") + errCredentialRequestKey = errors.New("credential request did not name the owned registry") +) + +type credentialSession struct { + root, log string + config CredentialConfig +} + +// CredentialHelperMain implements Docker's credential-helper `get` wire +// protocol. The input is recorded only as an owned server key and parent PID; +// the secret is read from a private fixture file and never logged. +func CredentialHelperMain(args []string, stdin io.Reader, stdout, stderr io.Writer) int { + if !validCredentialHelperArgs(args) { + _, _ = fmt.Fprintln(stderr, "fixture credential helper supports get only") + return 1 + } + if !credentialEnvironmentConfigured() { + _, _ = fmt.Fprintln(stderr, "fixture credential helper is not configured") + return 1 + } + session, err := loadCredentialConfig() + if err != nil { + _, _ = fmt.Fprintln(stderr, "fixture credential configuration is invalid") + return 1 + } + outcome, ok := session.observeRequest(stdin, stderr) + if !ok { + return 1 + } + if outcome == "parent_hash_mismatch" { + _, _ = fmt.Fprintln(stderr, "credential helper parent is not the expected provider") + return 1 + } + if !session.config.CredentialsAvailable { + _, _ = fmt.Fprintln(stdout, credentialsNotFoundMessage) + return 1 + } + if err := writeCredentials(stdout, session.config); err != nil { + return 1 + } + return 0 +} + +func validCredentialHelperArgs(args []string) bool { + return len(args) == 1 && args[0] == "get" +} + +func (s credentialSession) observeRequest(stdin io.Reader, stderr io.Writer) (string, bool) { + serverKey, err := readCredentialRequest(stdin, s.config.ServerKey) + if err != nil { + if errors.Is(err, errCredentialRequestRead) { + _, _ = fmt.Fprintln(stderr, "fixture credential request could not be read") + } else { + _, _ = fmt.Fprintln(stderr, "credential request did not name the owned registry") + } + return "", false + } + parent, err := captureParentIdentity(os.Getppid()) + if err != nil { + _, _ = fmt.Fprintln(stderr, "credential helper could not verify its live Linux parent") + return "", false + } + outcome := credentialOutcome(parent.SHA256, s.config) + if err := s.recordObservation(serverKey, parent, outcome); err != nil { + _, _ = fmt.Fprintln(stderr, "fixture observation could not be recorded") + return "", false + } + return outcome, true +} + +func credentialEnvironmentConfigured() bool { + return os.Getenv(CredentialRootEnv) != "" && os.Getenv(CredentialConfigEnv) != "" && + os.Getenv(CredentialLogEnv) != "" && os.Getenv(CredentialOperationEnv) != "" +} + +func loadCredentialConfig() (credentialSession, error) { + root, configPath := os.Getenv(CredentialRootEnv), os.Getenv(CredentialConfigEnv) + logPath, operationID := os.Getenv(CredentialLogEnv), os.Getenv(CredentialOperationEnv) + if root == "" || configPath == "" || logPath == "" || operationID == "" { + return credentialSession{}, errors.New("credential helper is not configured") + } + cfg, err := ReadCredentialConfig(root, configPath) + if err != nil { + return credentialSession{}, err + } + if err := validateCredentialConfig(cfg, operationID); err != nil { + return credentialSession{}, errors.New("credential configuration is invalid") + } + return credentialSession{root: root, log: logPath, config: cfg}, nil +} + +func validateCredentialConfig(cfg CredentialConfig, operationID string) error { + if cfg.OperationID == "" || cfg.OperationID != operationID || cfg.ServerKey == "" || + !validSHA256(cfg.ExpectedProviderSHA256) { + return errors.New("credential configuration is invalid") + } + if cfg.CredentialsAvailable && (cfg.Username == "" || cfg.Secret == "") { + return errors.New("available fixture credentials are incomplete") + } + return nil +} + +func readCredentialRequest(stdin io.Reader, expected string) (string, error) { + serverKey, err := bufio.NewReader(stdin).ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return "", fmt.Errorf("%w: %w", errCredentialRequestRead, err) + } + serverKey = strings.TrimSpace(serverKey) + if serverKey != expected { + return "", errCredentialRequestKey + } + return serverKey, nil +} + +func credentialOutcome(parentHash string, cfg CredentialConfig) string { + switch { + case !equalSHA256(parentHash, cfg.ExpectedProviderSHA256): + return "parent_hash_mismatch" + case cfg.CredentialsAvailable: + return "credentials_provided" + default: + return "credentials_not_found" + } +} + +func (s credentialSession) recordObservation( + serverKey string, + parent ProcessIdentity, + outcome string, +) error { + line, err := json.Marshal(CredentialObservation{ + OperationID: s.config.OperationID, + ServerKey: serverKey, + Action: "get", + Parent: parent, + Outcome: outcome, + }) + if err != nil { + return err + } + return appendOwnedLine(s.root, s.log, append(line, '\n')) +} + +func writeCredentials(stdout io.Writer, cfg CredentialConfig) error { + result, err := json.Marshal(map[string]string{"Username": cfg.Username, "Secret": cfg.Secret}) + if err != nil { + return err + } + _, err = stdout.Write(append(result, '\n')) + return err +} + +func validSHA256(value string) bool { + if len(value) != sha256.Size*2 { + return false + } + _, err := hex.DecodeString(value) + return err == nil +} + +func equalSHA256(a, b string) bool { return strings.EqualFold(a, b) } + +// captureParentIdentity binds an observation to the live direct parent without +// reading its command line or environment, which may contain credentials. +func captureParentIdentity(pid int) (ProcessIdentity, error) { + if runtime.GOOS != "linux" || pid <= 0 { + return ProcessIdentity{}, errors.New("linux /proc identity is required") + } + proc := filepath.Join("/proc", fmt.Sprint(pid)) + startBefore, err := processStartTime(filepath.Join(proc, "stat")) + if err != nil { + return ProcessIdentity{}, err + } + executable, err := os.Readlink(filepath.Join(proc, "exe")) + if err != nil { + return ProcessIdentity{}, err + } + digest, err := hashFile(filepath.Join(proc, "exe")) + if err != nil { + return ProcessIdentity{}, err + } + startAfter, err := processStartTime(filepath.Join(proc, "stat")) + if err != nil { + return ProcessIdentity{}, err + } + if startBefore != startAfter { + return ProcessIdentity{}, errors.New("parent process changed during identity capture") + } + return ProcessIdentity{ + PID: pid, StartTimeTicks: startBefore, Executable: executable, SHA256: digest, + }, nil +} + +func hashFile(path string) (string, error) { + // Parent hashing receives only /proc//exe. + //nolint:gosec // fixed procfs executable entry for the verified parent PID + file, err := os.Open(path) + if err != nil { + return "", err + } + h := sha256.New() + _, copyErr := io.Copy(h, file) + closeErr := file.Close() + if err := errors.Join(copyErr, closeErr); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func processStartTime(path string) (uint64, error) { + // The caller supplies only /proc//stat. + //nolint:gosec // fixed procfs stat entry for the verified parent PID + b, err := os.ReadFile(path) + if err != nil { + return 0, err + } + // /proc//stat field 2 is parenthesized and may contain spaces or ')'. + end := strings.LastIndexByte(string(b), ')') + if end < 0 || end+1 >= len(b) { + return 0, errors.New("invalid Linux process stat") + } + fields := strings.Fields(string(b[end+1:])) // starts at field 3 (state) + if len(fields) <= 19 { // field 22 is index 19 from field 3 + return 0, errors.New("linux process stat lacks start time") + } + var start uint64 + if _, err := fmt.Sscan(fields[19], &start); err != nil || start == 0 { + return 0, errors.New("invalid Linux process start time") + } + return start, nil +} + +func ownedChild(root, path string) (string, error) { + if err := validatePrivateRoot(root); err != nil { + return "", err + } + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return "", errors.New("owned file path must be absolute and clean") + } + if filepath.Dir(path) != root || filepath.Base(path) == "." || + filepath.Base(path) == string(filepath.Separator) { + return "", errors.New("owned file must be a direct child of its private root") + } + return path, nil +} + +func validatePrivateRoot(root string) error { + if !filepath.IsAbs(root) || filepath.Clean(root) != root { + return errors.New("owned root must be absolute and clean") + } + resolved, err := filepath.EvalSymlinks(root) + if err != nil || resolved != root { + return errors.New("owned root must be canonical and symlink-free") + } + return nil +} + +func openOwnedFile(root, path string, flags int, create bool) (file *os.File, resultErr error) { + defer closeFileOnError(&file, &resultErr) + if _, err := ownedChild(root, path); err != nil { + return nil, err + } + openedRoot, err := os.OpenRoot(root) + if err != nil { + return nil, err + } + defer func() { resultErr = errors.Join(resultErr, openedRoot.Close()) }() + rootFile, err := openedRoot.Open(".") + if err != nil { + return nil, errors.New("owned root must be a mode-0700 directory") + } + defer func() { resultErr = errors.Join(resultErr, rootFile.Close()) }() + if err := validateOpenedRoot(rootFile); err != nil { + return nil, err + } + flags = ownedFileOpenFlags(flags, create) + file, err = openedRoot.OpenFile(filepath.Base(path), flags, 0o600) + if err != nil { + return nil, err + } + if err := validateOpenedFile(file); err != nil { + return nil, errors.Join(err, file.Close()) + } + return file, nil +} + +func closeFileOnError(file **os.File, resultErr *error) { + if *resultErr == nil || *file == nil { + return + } + *resultErr = errors.Join(*resultErr, (*file).Close()) + *file = nil +} + +func ownedFileOpenFlags(flags int, create bool) int { + flags |= syscall.O_NOFOLLOW | syscall.O_CLOEXEC + if create { + flags |= syscall.O_CREAT | syscall.O_EXCL + } + return flags +} + +func validateOpenedRoot(root *os.File) error { + info, err := root.Stat() + if err != nil || !info.IsDir() || info.Mode().Perm() != 0o700 { + return errors.New("owned root must be a mode-0700 directory") + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || int(stat.Uid) != os.Geteuid() { + return errors.New("owned root has an unexpected owner") + } + return nil +} + +func validateOpenedFile(file *os.File) error { + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 { + return errors.New("owned file must be a mode-0600 regular file") + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || int(stat.Uid) != os.Geteuid() { + return errors.New("owned file has an unexpected owner") + } + return nil +} + +// ReadCredentialConfig reads a validated private fixture config. The path must +// be a direct child of a canonical, current-user-owned mode-0700 root. +func ReadCredentialConfig(root, path string) (CredentialConfig, error) { + f, err := openOwnedFile(root, path, syscall.O_RDONLY, false) + if err != nil { + return CredentialConfig{}, err + } + var cfg CredentialConfig + err = json.NewDecoder(io.LimitReader(f, 64*1024)).Decode(&cfg) + return cfg, errors.Join(err, f.Close()) +} + +// IsCredentialHelperInvocation recognizes only the fixed helper path under +// the configured private root, when it resolves to this running test binary. +func IsCredentialHelperInvocation(argv0 string) bool { + root := os.Getenv(CredentialRootEnv) + if root == "" { + return false + } + path, ok := helperInvocationPath(argv0) + if !ok || validatePrivateRoot(root) != nil { + return false + } + helperPath := filepath.Join(root, "bin", HelperBinaryName) + if path != helperPath || !privateDirectory(root) || + !privateDirectory(filepath.Dir(helperPath)) { + return false + } + return sameAsCurrentExecutable(helperPath) +} + +func helperInvocationPath(argv0 string) (string, bool) { + path := argv0 + if !filepath.IsAbs(argv0) { + if argv0 != HelperBinaryName { + return "", false + } + var err error + path, err = exec.LookPath(argv0) + if err != nil { + return "", false + } + } + return path, filepath.IsAbs(path) && filepath.Clean(path) == path +} + +func sameAsCurrentExecutable(path string) bool { + // path is constructed from the validated private fixture root and fixed name. + //nolint:gosec // the helper path is constrained by IsCredentialHelperInvocation + linkInfo, err := os.Stat(path) + if err != nil { + return false + } + executable, err := os.Executable() + if err != nil { + return false + } + executableInfo, err := os.Stat(executable) + return err == nil && os.SameFile(linkInfo, executableInfo) +} + +// DispatchCredentialHelper runs the helper only for the fixed private-root +// path when it resolves to the current executable. handled is false for the +// normal test-binary invocation. +func DispatchCredentialHelper() (handled bool, exitCode int) { + if !IsCredentialHelperInvocation(os.Args[0]) { + return false, 0 + } + return true, CredentialHelperMain(os.Args[1:], os.Stdin, os.Stdout, os.Stderr) +} + +func privateDirectory(path string) bool { + if !validPrivateDirectoryPath(path) { + return false + } + root, err := os.OpenRoot(path) + if err != nil { + return false + } + rootFile, err := root.Open(".") + if err != nil { + _ = root.Close() + return false + } + valid := privateDirectoryOwner(rootFile) + fileCloseErr := rootFile.Close() + rootCloseErr := root.Close() + return valid && fileCloseErr == nil && rootCloseErr == nil +} + +func validPrivateDirectoryPath(path string) bool { + resolved, err := filepath.EvalSymlinks(path) + if err != nil || resolved != path { + return false + } + // path is either the validated fixture root or its fixed bin child. + //nolint:gosec // constrained to the private root/bin pair by the caller + info, err := os.Lstat(path) + return err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 && + info.Mode().Perm() == 0o700 +} + +func privateDirectoryOwner(rootFile *os.File) bool { + info, err := rootFile.Stat() + if err != nil || !info.IsDir() || info.Mode().Perm() != 0o700 { + return false + } + stat, ok := info.Sys().(*syscall.Stat_t) + return ok && int(stat.Uid) == os.Geteuid() +} + +func appendOwnedLine(root, path string, line []byte) error { + f, err := openOwnedFile(root, path, syscall.O_WRONLY|syscall.O_APPEND, false) + if errors.Is(err, os.ErrNotExist) { + f, err = openOwnedFile(root, path, syscall.O_WRONLY|syscall.O_APPEND, true) + if errors.Is(err, os.ErrExist) { + f, err = openOwnedFile(root, path, syscall.O_WRONLY|syscall.O_APPEND, false) + } + } + if err != nil { + return err + } + if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + return errors.Join(err, f.Close()) + } + _, writeErr := f.Write(line) + unlockErr := syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + return errors.Join(writeErr, unlockErr, f.Close()) +} diff --git a/e2e/framework/registryfixture/server_linux.go b/e2e/framework/registryfixture/server_linux.go new file mode 100644 index 0000000000..9093ae3bad --- /dev/null +++ b/e2e/framework/registryfixture/server_linux.go @@ -0,0 +1,476 @@ +//go:build linux + +// Package registryfixture owns the bounded Linux registry/proxy and helper +// fixtures used by native provider-consumer tests. +package registryfixture + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "hash" + "io" + "math/big" + "net" + "net/http" + "strings" + "sync" + "time" +) + +const RegistryHost = "registry.invalid" + +type Config struct { + Username string + Password string +} + +type Request struct { + Method string `json:"method"` + Path string `json:"path"` // normalized route class, never the raw URL + Status int `json:"status"` + Authenticated bool `json:"authenticated"` + Bytes int64 `json:"bytes"` + BodyDigest string `json:"body_digest,omitempty"` + Digest string `json:"docker_content_digest,omitempty"` +} + +type Snapshot struct { + ConnectAllowed int `json:"connect_allowed"` + ConnectDenied int `json:"connect_denied"` + Authenticated int `json:"authenticated_requests"` + AuthRejected int `json:"auth_rejected_requests"` + Completed int `json:"completed_responses"` + ManifestGets int `json:"manifest_gets"` + BlobGets int `json:"blob_gets"` + ResponseBytes int64 `json:"response_bytes"` + Requests []Request `json:"requests"` +} + +// Server owns one TLS registry listener and one CONNECT listener. The proxy +// accepts only the exact registry authority and dials the owned listener IP; +// it never resolves or forwards caller-selected destinations. +type Server struct { + Authority string + ProxyURL string + CAPEM []byte + Username string + Password string + + registry *http.Server + regLn net.Listener + proxy *http.Server + proxyLn net.Listener + mu sync.Mutex + stats Snapshot + close sync.Once + closeErr error + tunnelMu sync.Mutex + closing bool + active int + idle chan struct{} + tunnels map[net.Conn]struct{} +} + +func Start(next http.Handler, cfg Config) (*Server, error) { + if next == nil || cfg.Username == "" || cfg.Password == "" { + return nil, fmt.Errorf("registry handler and fixture credentials are required") + } + regLn, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, fmt.Errorf("listen owned registry: %w", err) + } + cert, caPEM, err := makeCertificate() + if err != nil { + return nil, errors.Join(err, closeListener(regLn)) + } + s := &Server{ + Authority: RegistryHost + ":" + fmt.Sprint(regLn.Addr().(*net.TCPAddr).Port), + CAPEM: caPEM, Username: cfg.Username, Password: cfg.Password, regLn: regLn, + tunnels: make(map[net.Conn]struct{}), idle: make(chan struct{}), + } + close(s.idle) + regTLS := tls.NewListener(regLn, &tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{cert}, + }) + s.registry = &http.Server{Handler: s.authenticated(next), ReadHeaderTimeout: 5 * time.Second} + s.tunnelMu.Lock() + s.trackStartLocked() + s.tunnelMu.Unlock() + go func() { defer s.trackDone(); _ = s.registry.Serve(regTLS) }() + + proxyLn, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + s.tunnelMu.Lock() + s.closing = true + s.tunnelMu.Unlock() + registryErr := s.registry.Close() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + return nil, errors.Join( + fmt.Errorf("listen owned proxy: %w", err), + registryErr, + s.waitIdle(ctx), + closeListener(regLn), + ) + } + s.proxyLn = proxyLn + s.ProxyURL = "http://" + proxyLn.Addr().String() + s.proxy = &http.Server{Handler: http.HandlerFunc(s.connect), ReadHeaderTimeout: 5 * time.Second} + s.tunnelMu.Lock() + s.trackStartLocked() + s.tunnelMu.Unlock() + go func() { defer s.trackDone(); _ = s.proxy.Serve(proxyLn) }() + return s, nil +} + +func isSuccessfulGet(r *http.Request, status int, path, expectedPath string) bool { + return r.Method == http.MethodGet && status >= 200 && status < 300 && path == expectedPath +} + +func routeClass(path string) string { + switch { + case path == "/v2/": + return "v2" + case strings.Contains(path, "/manifests/"): + return "manifests" + case strings.Contains(path, "/blobs/uploads/"): + return "uploads" + case strings.Contains(path, "/blobs/"): + return "blobs" + default: + return "other" + } +} + +type countWriter struct { + http.ResponseWriter + status int + bytes int64 + digest hash.Hash +} + +func (w *countWriter) WriteHeader(code int) { + if w.status != 0 { + return + } + w.status = code + w.ResponseWriter.WriteHeader(code) +} + +func (w *countWriter) Write(p []byte) (int, error) { + if w.status == 0 { + // Match net/http's implicit status before forwarding the first body write. + // A later WriteHeader is ignored by the underlying writer and must be + // ignored by our observation too. + w.WriteHeader(http.StatusOK) + } + n, e := w.ResponseWriter.Write(p) + w.bytes += int64(n) + if n > 0 && w.digest != nil { + _, _ = w.digest.Write(p[:n]) + } + return n, e +} + +func (w *countWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } + +func (w *countWriter) Flush() { + if w.status == 0 { + w.WriteHeader(http.StatusOK) + } + if f, ok := w.ResponseWriter.(http.Flusher); ok { + f.Flush() + } +} + +// hijackConnect is split out to keep CONNECT tunneling explicit and small. +func hijackConnect(w http.ResponseWriter) (net.Conn, error) { + h, ok := w.(http.Hijacker) + if !ok { + return nil, fmt.Errorf("HTTP server does not support hijacking") + } + c, rw, err := h.Hijack() + if err != nil { + return nil, err + } + if _, err = rw.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil { + _ = c.Close() + return nil, err + } + if err = rw.Flush(); err != nil { + _ = c.Close() + return nil, err + } + return c, nil +} + +func (s *Server) Snapshot() Snapshot { + s.mu.Lock() + defer s.mu.Unlock() + out := s.stats + out.Requests = append([]Request(nil), s.stats.Requests...) + return out +} + +// Close shuts down both listeners and joins servers and active tunnels. +func (s *Server) Close(ctx context.Context) error { + s.close.Do(func() { + s.tunnelMu.Lock() + s.closing = true + owned := make([]net.Conn, 0, len(s.tunnels)) + for conn := range s.tunnels { + owned = append(owned, conn) + } + idle := s.idle + s.tunnelMu.Unlock() + + // Closing registered tunnels first releases both copy directions; the + // close fence prevents any later CONNECT from registering new work. + var tunnelCloseErr error + for _, conn := range owned { + if err := conn.Close(); err != nil && !errors.Is(err, net.ErrClosed) { + tunnelCloseErr = errors.Join(tunnelCloseErr, err) + } + } + s.closeErr = errors.Join(tunnelCloseErr, + s.registry.Shutdown(ctx), s.proxy.Shutdown(ctx), + s.registry.Close(), s.proxy.Close(), closeListener(s.regLn), closeListener(s.proxyLn), + ) + select { + case <-idle: + case <-ctx.Done(): + s.closeErr = errors.Join(s.closeErr, ctx.Err()) + } + }) + return s.closeErr +} + +func (s *Server) trackStartLocked() { + if s.active == 0 { + s.idle = make(chan struct{}) + } + s.active++ +} + +func (s *Server) trackDone() { + s.tunnelMu.Lock() + s.active-- + if s.active == 0 { + close(s.idle) + } + s.tunnelMu.Unlock() +} + +func (s *Server) waitIdle(ctx context.Context) error { + s.tunnelMu.Lock() + idle := s.idle + s.tunnelMu.Unlock() + select { + case <-idle: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func closeListener(ln net.Listener) error { + err := ln.Close() + if errors.Is(err, net.ErrClosed) { + return nil + } + return err +} + +func makeCertificate() (tls.Certificate, []byte, error) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return tls.Certificate{}, nil, err + } + now := time.Now() + tmpl := x509.Certificate{ + SerialNumber: bigIntSerial(), + Subject: pkix.Name{CommonName: RegistryHost}, + DNSNames: []string{RegistryHost}, + NotBefore: now.Add(-time.Minute), + NotAfter: now.Add(2 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key) + if err != nil { + return tls.Certificate{}, nil, err + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM := pem.EncodeToMemory( + &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}, + ) + cert, err := tls.X509KeyPair(certPEM, keyPEM) + return cert, certPEM, err +} + +func bigIntSerial() *big.Int { + n, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 120)) + if err != nil { + return big.NewInt(time.Now().UnixNano()) + } + return n +} + +func (s *Server) authenticated(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.tunnelMu.Lock() + if s.closing { + s.tunnelMu.Unlock() + http.Error(w, "owned fixture is closing", http.StatusServiceUnavailable) + return + } + s.trackStartLocked() + s.tunnelMu.Unlock() + defer s.trackDone() + path := routeClass(r.URL.Path) + authenticated := s.validBasic(r) + cw := &countWriter{ResponseWriter: w, digest: sha256.New()} + defer s.recordResponse(r, cw, path, authenticated) + if !authenticated { + w.Header().Set("WWW-Authenticate", `Basic realm="case-a-owned-registry"`) + http.Error(cw, "authentication required", http.StatusUnauthorized) + return + } + next.ServeHTTP(cw, r) + }) +} + +func (s *Server) recordResponse(r *http.Request, cw *countWriter, path string, authenticated bool) { + status := cw.status + if status == 0 { + status = http.StatusOK + } + bodyDigest := "" + if cw.bytes > 0 { + bodyDigest = fmt.Sprintf("sha256:%x", cw.digest.Sum(nil)) + } + s.mu.Lock() + defer s.mu.Unlock() + if authenticated { + s.stats.Authenticated++ + } else { + s.stats.AuthRejected++ + } + s.stats.Completed++ + s.stats.ResponseBytes += cw.bytes + if authenticated && isSuccessfulGet(r, status, path, "manifests") { + s.stats.ManifestGets++ + } + if authenticated && isSuccessfulGet(r, status, path, "blobs") { + s.stats.BlobGets++ + } + s.stats.Requests = append(s.stats.Requests, Request{ + Method: r.Method, Path: path, Status: status, Authenticated: authenticated, + Bytes: cw.bytes, BodyDigest: bodyDigest, Digest: cw.Header().Get("Docker-Content-Digest"), + }) +} + +func (s *Server) validBasic(r *http.Request) bool { + u, p, ok := r.BasicAuth() + return ok && u == s.Username && p == s.Password +} + +func (s *Server) connect(w http.ResponseWriter, r *http.Request) { + if !s.registerConnectHandler() { + http.Error(w, "owned fixture is closing", http.StatusServiceUnavailable) + return + } + defer s.trackDone() + if !s.ownsConnectTarget(r) { + s.denyConnect(w) + return + } + client, err := hijackConnect(w) + if err != nil { + return + } + s.openTunnel(client) +} + +func (s *Server) registerConnectHandler() bool { + s.tunnelMu.Lock() + defer s.tunnelMu.Unlock() + if s.closing { + return false + } + // Register before Shutdown can ignore a handler that later hijacks. + s.trackStartLocked() + return true +} + +func (s *Server) ownsConnectTarget(r *http.Request) bool { + return r.Method == http.MethodConnect && r.Host == s.Authority +} + +func (s *Server) denyConnect(w http.ResponseWriter) { + s.mu.Lock() + s.stats.ConnectDenied++ + s.mu.Unlock() + http.Error(w, "CONNECT destination is not the owned fixture registry", http.StatusForbidden) +} + +func (s *Server) openTunnel(client net.Conn) { + upstream, err := net.DialTimeout("tcp", s.regLn.Addr().String(), 3*time.Second) + if err != nil { + _ = client.Close() + return + } + s.mu.Lock() + s.stats.ConnectAllowed++ + s.mu.Unlock() + if !s.registerTunnel(client, upstream) { + _ = client.Close() + _ = upstream.Close() + return + } + go s.copyTunnel(client, upstream) +} + +func (s *Server) registerTunnel(client, upstream net.Conn) bool { + s.tunnelMu.Lock() + defer s.tunnelMu.Unlock() + if s.closing { + return false + } + s.tunnels[client] = struct{}{} + s.tunnels[upstream] = struct{}{} + s.trackStartLocked() // register the tunnel under the same close fence + return true +} + +func (s *Server) copyTunnel(client, upstream net.Conn) { + defer s.trackDone() + defer func() { + _ = client.Close() + _ = upstream.Close() + s.tunnelMu.Lock() + delete(s.tunnels, client) + delete(s.tunnels, upstream) + s.tunnelMu.Unlock() + }() + var copyWG sync.WaitGroup + copyWG.Go(func() { + _, _ = io.Copy(upstream, client) + if c, ok := upstream.(*net.TCPConn); ok { + _ = c.CloseWrite() + } + }) + _, _ = io.Copy(client, upstream) + copyWG.Wait() +} diff --git a/e2e/framework/registryfixture/server_linux_test.go b/e2e/framework/registryfixture/server_linux_test.go new file mode 100644 index 0000000000..3ed0cbdd29 --- /dev/null +++ b/e2e/framework/registryfixture/server_linux_test.go @@ -0,0 +1,648 @@ +//go:build linux + +package registryfixture + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const ( + fixturePassword = "fixture-password" + fixtureServerKey = "registry.invalid:43210" + fixtureUser = "fixture-user" + fixtureOperationID = "attempt-0001" +) + +// This exercises only the owned loopback TLS/proxy/auth fixture. It does not +// execute MicroSandbox or claim that the published provider consumed it. +func TestOwnedLoopbackTLSAndAuthorityLimitedCONNECT(t *testing.T) { + s, err := Start(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Docker-Content-Digest", "sha256:fixture") + _, _ = io.WriteString(w, "fixture-response") + }), Config{Username: fixtureUser, Password: fixturePassword}) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + defer func() { + if err := s.Close(ctx); err != nil { + t.Errorf("close owned fixture: %v", err) + } + }() + client := fixtureHTTPClient(t, s) + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + "https://"+s.Authority+"/v2/fixture/manifests/tag", + nil, + ) + require.NoError(t, err) + req.SetBasicAuth(s.Username, s.Password) + resp, err := client.Do(req) + require.NoError(t, err) + body, readErr := io.ReadAll(resp.Body) + closeErr := resp.Body.Close() + require.NoError(t, readErr) + require.NoError(t, closeErr) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Contains(t, string(body), "fixture-response") + denied, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://outside.invalid/", nil) + require.NoError(t, err) + _, err = client.Do(denied) + require.Error(t, err, "proxy unexpectedly allowed a non-owned CONNECT authority") + got := s.Snapshot() + require.Equal(t, 1, got.ConnectAllowed) + require.Equal(t, 1, got.ConnectDenied) + require.Equal(t, 1, got.Authenticated) + require.Zero(t, got.AuthRejected) + require.Equal(t, 1, got.Completed) + require.Equal(t, 1, got.ManifestGets) + require.Positive(t, got.ResponseBytes) + require.Len(t, got.Requests, 1) + require.Equal( + t, + fmt.Sprintf("sha256:%x", sha256.Sum256([]byte("fixture-response"))), + got.Requests[0].BodyDigest, + ) +} + +func TestRegistryRecordsRejected401AndBodyDigests(t *testing.T) { + const body = "manifest-body-for-digest" + s, err := Start(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Docker-Content-Digest", "sha256:expected-header") + _, _ = io.WriteString(w, body) + }), Config{Username: fixtureUser, Password: fixturePassword}) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + cleanupRegistryServer(t, s) + client := fixtureHTTPClient(t, s) + endpoint := "https://" + s.Authority + "/v2/fixture/manifests/sha256:expected" + for _, authenticated := range []bool{false, true} { + req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + require.NoError(t, reqErr) + if authenticated { + req.SetBasicAuth(s.Username, s.Password) + } + resp, requestErr := client.Do(req) + require.NoError(t, requestErr, "owned registry request failed") + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + wantStatus := http.StatusUnauthorized + if authenticated { + wantStatus = http.StatusOK + } + require.Equal(t, wantStatus, resp.StatusCode) + } + got := s.Snapshot() + wantBodyDigest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(body))) + require.Equal(t, 1, got.AuthRejected) + require.Equal(t, 1, got.Authenticated) + require.Equal(t, 2, got.Completed) + require.Equal(t, 1, got.ManifestGets) + require.Len(t, got.Requests, 2) + rejected, success := got.Requests[0], got.Requests[1] + require.Equal(t, http.StatusUnauthorized, rejected.Status) + require.False(t, rejected.Authenticated) + require.Equal(t, "manifests", rejected.Path) + require.Positive(t, rejected.Bytes) + require.NotEmpty(t, rejected.BodyDigest) + require.Equal(t, http.StatusOK, success.Status) + require.True(t, success.Authenticated) + require.Equal(t, "manifests", success.Path) + require.Equal(t, int64(len(body)), success.Bytes) + require.Equal(t, wantBodyDigest, success.BodyDigest) + require.Equal(t, "sha256:expected-header", success.Digest) +} + +func TestRegistryObservesImplicitOKBeforeLateWriteHeader(t *testing.T) { + const body = "body commits implicit status" + s, err := Start(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, body) + w.WriteHeader(http.StatusInternalServerError) + }), Config{Username: fixtureUser, Password: fixturePassword}) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + cleanupRegistryServer(t, s) + client := fixtureHTTPClient(t, s) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + "https://"+s.Authority+"/v2/fixture/manifests/implicit-status", nil) + require.NoError(t, err) + req.SetBasicAuth(s.Username, s.Password) + resp, err := client.Do(req) + require.NoError(t, err, "owned registry request failed") + gotBody, readErr := io.ReadAll(resp.Body) + closeErr := resp.Body.Close() + require.NoError(t, readErr) + require.NoError(t, closeErr) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, body, string(gotBody)) + + got := s.Snapshot() + wantDigest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(body))) + require.Equal(t, 1, got.Completed) + require.Equal(t, 1, got.Authenticated) + require.Equal(t, int64(len(body)), got.ResponseBytes) + require.Len(t, got.Requests, 1) + observed := got.Requests[0] + require.Equal(t, resp.StatusCode, observed.Status) + require.Equal(t, int64(len(body)), observed.Bytes) + require.Equal(t, wantDigest, observed.BodyDigest) +} + +func TestCredentialHelperGetSubprocessProtocol(t *testing.T) { + dir := privateCredentialRoot(t) + configPath := filepath.Join(dir, "credentials.json") + logPath := filepath.Join(dir, "helper-observations.jsonl") + const secret = "private-fixture-secret" + const operationID = fixtureOperationID + parentHash, err := executableSHA256() + require.NoError(t, err, "hash test executable") + // #nosec G117 -- Synthetic credentials are written only to this private test fixture. + config, err := json.Marshal(CredentialConfig{ + OperationID: operationID, + ServerKey: fixtureServerKey, + Username: fixtureUser, + Secret: secret, + CredentialsAvailable: true, + ExpectedProviderSHA256: parentHash, + }) + require.NoError(t, err) + require.NoError(t, os.WriteFile(configPath, config, 0o600)) + helperPath := linkCredentialHelper(t, dir) + t.Setenv(CredentialRootEnv, dir) + t.Setenv("PATH", filepath.Dir(helperPath)+string(os.PathListSeparator)+os.Getenv("PATH")) + require.True( + t, + IsCredentialHelperInvocation(HelperBinaryName), + "bare helper must resolve to the private helper", + ) + cmd := credentialHelperCommand(t) + cmd.Env = append( + os.Environ(), + CredentialRootEnv+"="+dir, + CredentialConfigEnv+"="+configPath, + CredentialLogEnv+"="+logPath, + CredentialOperationEnv+"="+operationID, + ) + cmd.Stdin = strings.NewReader("registry.invalid:43210\n") + stdout, err := cmd.Output() + require.NoError(t, err, "credential helper get subprocess failed") + var got map[string]string + require.NoError(t, json.Unmarshal(stdout, &got), "credential helper returned invalid JSON") + require.True(t, got["Username"] == fixtureUser, "credential helper returned an unexpected user") + require.True(t, got["Secret"] == secret, "credential helper returned an unexpected secret") + // #nosec G304 -- The path is created beneath the test-owned temporary directory. + line, err := os.ReadFile(logPath) + require.NoError(t, err, "credential helper did not record its owned invocation") + var observation CredentialObservation + require.NoError( + t, + json.Unmarshal([]byte(strings.TrimSpace(string(line))), &observation), + "credential helper observation is invalid JSON", + ) + assertSuccessfulHelperObservation(t, observation, parentHash) + require.False( + t, + strings.Contains(string(line), secret), + "observation leaked the fixture secret", + ) + assertCredentialNonceRejected(t, dir, secret) + + assertCredentialParentMismatch(t, dir, secret) +} + +func assertCredentialNonceRejected(t *testing.T, dir, secret string) { + t.Helper() + configPath := filepath.Join(dir, "credentials.json") + logPath := filepath.Join(dir, "helper-observations.jsonl") + cmd := credentialHelperCommand(t) + cmd.Env = append(os.Environ(), CredentialRootEnv+"="+dir, CredentialConfigEnv+"="+configPath, + CredentialLogEnv+"="+logPath, CredentialOperationEnv+"=different-attempt") + cmd.Stdin = strings.NewReader("registry.invalid:43210\n") + output, err := cmd.Output() + require.Error(t, err, "helper accepted a config from a different attempt") + require.False( + t, + strings.Contains(string(output), secret), + "helper exposed credentials for a different attempt", + ) + require.Len(t, readCredentialObservations(t, logPath), 1) +} + +func assertSuccessfulHelperObservation( + t *testing.T, + observation CredentialObservation, + parentHash string, +) { + t.Helper() + require.Equal(t, fixtureOperationID, observation.OperationID) + require.Equal(t, fixtureServerKey, observation.ServerKey) + require.Equal(t, "get", observation.Action) + require.Equal(t, os.Getpid(), observation.Parent.PID) + require.Positive(t, observation.Parent.StartTimeTicks) + require.Equal(t, parentHash, observation.Parent.SHA256) + require.Equal(t, "credentials_provided", observation.Outcome) +} + +func assertCredentialParentMismatch(t *testing.T, dir, secret string) { + t.Helper() + const operationID = fixtureOperationID + configPath := filepath.Join(dir, "credentials.json") + logPath := filepath.Join(dir, "helper-observations.jsonl") + // The fixture test process can invoke the helper, but cannot satisfy an + // unrelated expected provider hash merely by being its recorded parent. + // #nosec G117 -- Synthetic credentials stay within this private test fixture. + config, err := json.Marshal(CredentialConfig{ + OperationID: operationID, + ServerKey: fixtureServerKey, + Username: fixtureUser, + Secret: secret, + CredentialsAvailable: true, + ExpectedProviderSHA256: strings.Repeat("0", 64), + }) + require.NoError(t, err, "encode mismatch config") + require.NoError(t, os.WriteFile(configPath, config, 0o600), "write mismatch config") + cmd := credentialHelperCommand(t) + cmd.Env = append( + os.Environ(), + CredentialRootEnv+"="+dir, + CredentialConfigEnv+"="+configPath, + CredentialLogEnv+"="+logPath, + CredentialOperationEnv+"="+operationID, + ) + cmd.Stdin = strings.NewReader("registry.invalid:43210\n") + stdout, err := cmd.Output() + require.Error(t, err, "test process satisfied a mismatched pinned-provider hash") + require.False( + t, + strings.Contains(string(stdout), secret), + "mismatched parent received credentials", + ) + observations := readCredentialObservations(t, logPath) + require.Len(t, observations, 2) + require.Equal(t, "parent_hash_mismatch", observations[1].Outcome) + require.Equal(t, os.Getpid(), observations[1].Parent.PID) + require.NotEqual(t, strings.Repeat("0", 64), observations[1].Parent.SHA256) + // #nosec G304 -- The path is an observation file in the test-owned temporary directory. + allObservations, err := os.ReadFile(logPath) + require.NoError(t, err) + require.False( + t, + strings.Contains(string(allObservations), secret), + "observations leaked the fixture secret", + ) +} + +func TestCredentialHelperExplicitNotFoundDoesNotReturnCredentials(t *testing.T) { + dir := privateCredentialRoot(t) + configPath, logPath := filepath.Join( + dir, + "credentials.json", + ), filepath.Join( + dir, + "observations.jsonl", + ) + parentHash, err := executableSHA256() + require.NoError(t, err, "hash test executable") + // #nosec G117 -- The deliberate not-found fixture contains no secret. + config, err := json.Marshal(CredentialConfig{ + OperationID: "attempt-not-found", + ServerKey: fixtureServerKey, + CredentialsAvailable: false, + ExpectedProviderSHA256: parentHash, + }) + require.NoError(t, err, "encode no-credentials config") + require.NoError(t, os.WriteFile(configPath, config, 0o600), "write no-credentials config") + helperPath := linkCredentialHelper(t, dir) + t.Setenv(CredentialRootEnv, dir) + t.Setenv("PATH", filepath.Dir(helperPath)+string(os.PathListSeparator)+os.Getenv("PATH")) + require.True( + t, + IsCredentialHelperInvocation(HelperBinaryName), + "bare helper must resolve to the private helper", + ) + cmd := credentialHelperCommand(t) + cmd.Env = append( + os.Environ(), + CredentialRootEnv+"="+dir, + CredentialConfigEnv+"="+configPath, + CredentialLogEnv+"="+logPath, + CredentialOperationEnv+"=attempt-not-found", + ) + cmd.Stdin = strings.NewReader("registry.invalid:43210\n") + stdout, err := cmd.Output() + require.Error(t, err) + require.True( + t, + strings.TrimSpace(string(stdout)) == credentialsNotFoundMessage, + "unexpected not-found response", + ) + observations := readCredentialObservations(t, logPath) + require.Len(t, observations, 1) + require.Equal(t, "attempt-not-found", observations[0].OperationID) + require.Equal(t, "credentials_not_found", observations[0].Outcome) + require.Equal(t, os.Getpid(), observations[0].Parent.PID) + require.False( + t, + strings.Contains(string(stdout), "Secret"), + "not-found response contains credential fields", + ) +} + +func TestCredentialFilesRequirePrivateOwnedRootAndRegularFiles(t *testing.T) { + root := t.TempDir() + root, err := filepath.EvalSymlinks(root) + require.NoError(t, err, "resolve private root") + // #nosec G302 -- This is a private directory, so owner execute permission is required. + require.NoError(t, os.Chmod(root, 0o700), "chmod private root") + configPath, logPath := filepath.Join( + root, + "config.json", + ), filepath.Join( + root, + "observations.jsonl", + ) + // #nosec G117 -- This configuration contains no credentials and stays in the test-owned root. + validConfig, err := json.Marshal( + CredentialConfig{OperationID: "attempt", ServerKey: "registry.invalid:1234"}, + ) + require.NoError(t, err, "encode config") + writeConfig := func(path string, mode os.FileMode) { + t.Helper() + require.NoError(t, os.WriteFile(path, validConfig, mode), "write config") + require.NoError(t, os.Chmod(path, mode), "set config mode") + } + writeConfig(configPath, 0o600) + cfg, err := ReadCredentialConfig(root, configPath) + require.NoError(t, err, "valid owned config rejected") + require.Equal(t, "attempt", cfg.OperationID) + + loose := filepath.Join(root, "loose.json") + writeConfig(loose, 0o644) + _, err = ReadCredentialConfig(root, loose) + require.Error(t, err, "accepted group/world-readable config") + link := filepath.Join(root, "linked.json") + require.NoError(t, os.Symlink(configPath, link), "create config symlink") + _, err = ReadCredentialConfig(root, link) + require.Error(t, err, "accepted symlink config") + outside := filepath.Join(t.TempDir(), "outside.json") + writeConfig(outside, 0o600) + _, err = ReadCredentialConfig(root, outside) + require.Error(t, err, "accepted config outside private root") + require.NoError(t, os.Symlink(outside, logPath), "create log symlink") + require.Error( + t, + appendOwnedLine(root, logPath, []byte("{}\n")), + "accepted symlink observation log", + ) + require.Error( + t, + appendOwnedLine( + root, + filepath.Join(filepath.Dir(root), "outside-log.jsonl"), + []byte("{}\n"), + ), + "accepted observation log outside private root", + ) +} + +func TestCredentialHelperDispatchRequiresOwnedPathAndSameExecutable(t *testing.T) { + root := privateCredentialRoot(t) + binDir := filepath.Join(root, "bin") + require.NoError(t, os.Mkdir(binDir, 0o700), "create helper bin directory") + helpPath := filepath.Join(binDir, HelperBinaryName) + require.NoError(t, os.Symlink("/bin/true", helpPath), "create foreign helper link") + t.Setenv(CredentialRootEnv, root) + require.False(t, IsCredentialHelperInvocation(helpPath), "dispatch accepted another executable") + require.False( + t, + IsCredentialHelperInvocation(os.Args[0]), + "dispatch accepted the ordinary test binary", + ) +} + +func TestCredentialHelperDispatchResolvesOnlyPrivateBareHelper(t *testing.T) { + root := privateCredentialRoot(t) + helperPath := linkCredentialHelper(t, root) + t.Setenv(CredentialRootEnv, root) + privateBin := filepath.Dir(helperPath) + t.Setenv("PATH", privateBin+string(os.PathListSeparator)+os.Getenv("PATH")) + require.True( + t, + IsCredentialHelperInvocation(HelperBinaryName), + "dispatch rejected the private bare name", + ) + require.True( + t, + IsCredentialHelperInvocation(helperPath), + "dispatch rejected the private absolute path", + ) + require.False( + t, + IsCredentialHelperInvocation(filepath.Join("bin", HelperBinaryName)), + "accepted another relative path", + ) + + foreignBin := t.TempDir() + executable, err := os.Executable() + require.NoError(t, err, "locate current executable") + require.NoError( + t, + os.Symlink(executable, filepath.Join(foreignBin, HelperBinaryName)), + "create foreign PATH helper", + ) + t.Setenv( + "PATH", + foreignBin+string( + os.PathListSeparator, + )+privateBin+string( + os.PathListSeparator, + )+os.Getenv( + "PATH", + ), + ) + require.False( + t, + IsCredentialHelperInvocation(HelperBinaryName), + "accepted a helper from a foreign PATH directory", + ) +} + +func privateCredentialRoot(t *testing.T) string { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err, "resolve helper root") + // #nosec G302 -- This private directory requires owner execute permission. + require.NoError(t, os.Chmod(dir, 0o700), "set helper root mode") + return dir +} + +func linkCredentialHelper(t *testing.T, dir string) string { + t.Helper() + binDir := filepath.Join(dir, "bin") + require.NoError(t, os.Mkdir(binDir, 0o700), "create private helper bin directory") + executable, err := os.Executable() + require.NoError(t, err, "locate fixture test executable") + helperPath := filepath.Join(binDir, HelperBinaryName) + require.NoError( + t, + os.Symlink(executable, helperPath), + "link fixture test executable as credential helper", + ) + return helperPath +} + +func executableSHA256() (string, error) { + executable, err := os.Executable() + if err != nil { + return "", err + } + // #nosec G304 -- This is the current test executable returned by os.Executable. + file, err := os.Open(executable) + if err != nil { + return "", err + } + h := sha256.New() + _, copyErr := io.Copy(h, file) + closeErr := file.Close() + if err := errors.Join(copyErr, closeErr); err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +func readCredentialObservations(t *testing.T, path string) []CredentialObservation { + t.Helper() + // #nosec G304 -- Callers pass the observation file in their private temporary fixture. + data, err := os.ReadFile(path) + require.NoError(t, err, "read helper observations") + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + observations := make([]CredentialObservation, 0, len(lines)) + for _, line := range lines { + var observation CredentialObservation + require.NoError(t, json.Unmarshal([]byte(line), &observation), "decode helper observation") + observations = append(observations, observation) + } + return observations +} + +func credentialHelperCommand(t *testing.T) *exec.Cmd { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + t.Cleanup(cancel) + return exec.CommandContext(ctx, HelperBinaryName, "get") +} + +func fixtureHTTPClient(t *testing.T, server *Server) *http.Client { + t.Helper() + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM(server.CAPEM), "owned CA PEM was not parseable") + proxyURL, err := url.Parse(server.ProxyURL) + require.NoError(t, err) + transport := &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, + } + t.Cleanup(transport.CloseIdleConnections) + return &http.Client{Transport: transport} +} + +func cleanupRegistryServer(t *testing.T, server *Server) { + t.Helper() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.WithoutCancel(t.Context()), 3*time.Second) + defer cancel() + require.NoError(t, server.Close(ctx), "close owned registry fixture") + }) +} + +func TestCloseRetainsFirstResult(t *testing.T) { + s, err := Start( + http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}), + Config{Username: "u", Password: "p"}, + ) + require.NoError(t, err) + // Hold an owned activity slot so an already-canceled first close must + // report its bounded-wait error deterministically. + s.tunnelMu.Lock() + s.trackStartLocked() + s.tunnelMu.Unlock() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + first := s.Close(ctx) + s.trackDone() + second := s.Close(context.Background()) + require.Error(t, first) + require.Error(t, second) + require.EqualError(t, second, first.Error(), "Close did not retain its first cleanup result") +} + +func TestCloseRacingOwnedCONNECTsLeavesNoTrackedWork(t *testing.T) { + s, err := Start(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "owned") + }), Config{Username: "u", Password: "p"}) + require.NoError(t, err) + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cleanupCancel() + _ = s.Close(cleanupCtx) + }) + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) + defer cancel() + client := fixtureHTTPClient(t, s) + start := make(chan struct{}) + done := make(chan struct{}, 24) + for range 24 { + go func() { + defer func() { done <- struct{}{} }() + <-start + req, reqErr := http.NewRequestWithContext( + ctx, + http.MethodGet, + "https://"+s.Authority+"/v2/fixture/manifests/tag", + nil, + ) + if reqErr != nil { + return + } + req.SetBasicAuth(s.Username, s.Password) + resp, reqErr := client.Do(req) + if reqErr == nil { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } + }() + } + close(start) + require.NoError(t, s.Close(ctx), "close owned fixture") + for range 24 { + select { + case <-done: + case <-ctx.Done(): + t.Fatal("owned client requests did not finish before the bound") + } + } + s.tunnelMu.Lock() + active, tunnels := s.active, len(s.tunnels) + s.tunnelMu.Unlock() + require.Zero(t, active, "fixture retained work after close") + require.Zero(t, tunnels, "fixture retained tunnels after close") +} diff --git a/e2e/framework/registryfixture/testmain_linux_test.go b/e2e/framework/registryfixture/testmain_linux_test.go new file mode 100644 index 0000000000..597c6f9db6 --- /dev/null +++ b/e2e/framework/registryfixture/testmain_linux_test.go @@ -0,0 +1,15 @@ +//go:build linux + +package registryfixture + +import ( + "os" + "testing" +) + +func TestMain(m *testing.M) { + if handled, code := DispatchCredentialHelper(); handled { + os.Exit(code) + } + os.Exit(m.Run()) +} diff --git a/e2e/tests/up/MICROSANDBOX.md b/e2e/tests/up/MICROSANDBOX.md index 2ec8de2598..7600bd72d3 100644 --- a/e2e/tests/up/MICROSANDBOX.md +++ b/e2e/tests/up/MICROSANDBOX.md @@ -1,7 +1,7 @@ # MicroSandbox integration tests -Each capability has a Ginkgo label and scenario file. The built-in and external -providers run the same contracts. Keep scenario assertions in those files so +Each capability has a Ginkgo label and scenario file. Parity suites run the same +contracts against the built-in and external providers. Keep scenario assertions in those files so resource, ownership and lifecycle expectations remain visible. ## Shared setup @@ -29,16 +29,22 @@ The dedicated `integration-tests-microsandbox` job in artifact uploads. The Go utility in `hack/microsandbox` installs and checks the pinned Linux bundle and owns test execution, managed fixture selection, output capture and report validation. The Linux build job supplies its CI-only executable; -it is not a Devsy release artifact. Each suite still gets a fresh runner, a 25-minute job limit and a 20-minute test limit, -with one attempt. The checked runtime bundle, build artifacts, required KVM gate +it is not a Devsy release artifact. Each suite still gets a fresh runner, a +25-minute job limit and a 20-minute test limit, with one attempt. The checked +runtime bundle, build artifacts, required KVM gate and credentials are configured once in the job definition. Each matrix entry names its Ginkgo label and expected scenario count. The helper's `verify-report` command requires a real successful run with that exact count, split equally between the explicit `microsandbox-builtin` and -`microsandbox-external` labels. Missing cases, skips, retries, dry runs and -programmatic focus fail the gate. Reports and test output are retained even when -the test command fails. +`microsandbox-external` labels by default. A contract specific to the published +provider uses an explicit `provider-mode: external` catalogue entry and requires +every selected case to carry only the external provider label. Missing cases, +skips, retries, dry runs and programmatic focus fail the gate. Reports and test +output are retained even when the test command fails. + +Fixture and performance unit tests run by package in the existing unit-test jobs, +so additions do not require maintaining a second test-name list. To add a scenario: @@ -51,6 +57,60 @@ A new suite needs a catalogue entry only when it has a separate contract or isolation requirement. Avoid copying runner setup or adding a job for each individual assertion. +## Published-provider registry consumption + +The Linux-only `up-provider-microsandbox-registry-consumer` suite exercises the +production external host's Runtime Protocol `RunImage` path with the installed, +checksum-verified provider. Its three external-only cases cover an owned TLS +registry through a restricted CONNECT proxy, refusal without the fixture CA, +and refusal without registry credentials. A private credential helper records +the direct parent executable digest and operation identity without recording +credentials. Successful image responses are checked against their actual body +digests and byte counts, and the positive case verifies a guest marker. + +This contract calls `RunImage` directly because ordinary `up` resolves image +metadata before invoking the provider. It proves the published provider's image +consumer; ordinary workspace orchestration and remote agent binary preparation +have separate contracts. Each case owns fresh runtime state and an immutable +image reference. Registry seeding finishes before the measured listeners start, +and fixture shutdown joins active handlers before observations are inspected. + +The previous 74-case native parity baseline remains evidence for those existing +rows only. This external-only row brings the configured catalogue to 77 cases; +the earlier 74-case run does not prove the new consumer cases or the expanded +catalogue. Acceptance requires fresh current-head native reports for all rows. + +On Linux, the E2E test binary also serves as the private Docker credential helper. +Its `TestMain` preserves the production runtime-supervisor dispatch and recognizes +the helper only at its canonical, current-user-owned private path. Other test +invocations retain the normal test runner path. + +## Runtime startup measurements + +The existing two-case lifecycle label optionally captures its ordinary fresh +and stop/resume `up` commands for the external provider when CI supplies all five +performance inputs: `DEVSY_TEST_RUNTIME_SPAWN_PROBE`, +`DEVSY_TEST_RUNTIME_SUPERVISOR`, `DEVSY_TEST_RUNTIME_PERFORMANCE_DIR`, +`DEVSY_TEST_RUNTIME_REVISION`, and `DEVSY_TEST_RUNTIME_SDK_VERSION`. The +built-in case retains its existing path. No new suite, VM, deadline or latency +threshold is added. After both commands succeed, the helper stages the published +runtime from `Agent.Binaries`, verifies its manifest checksum and compares direct +and supervised Info/Find launches in direct/supervised/supervised/direct order. +Each run records one first launch and 100 warm samples with process-reaping checks. +The same Go CI helper builds both tools through `build-performance-tools` from +the unreplaced Runtime SDK version in `go.mod`. The producer uploads the tools +and their SDK-version marker for native jobs to reuse. Their build metadata and +selected supervisor digest are verified before accepting reports. + +The retained JSON includes the provider source, runtime digest, checkout revision, +SDK version, command wall time and filtered startup-event durations. Startup events +measure launch/handshake/dispense before RPC; they exclude binary resolution, RPC +and reaping and must not be described as total plugin overhead. The comparison +runs after both ordinary `up` captures so it does not prewarm their provider path. +It measures warm host caches, not a cold-cache or end-to-end speedup guarantee. +Raw probe stderr and environment values are not included in the performance artifact. +Normal suites remain inactive when these optional CI paths are unset. + ## Managed policy boundary Managed-policy recreation is Linux-only and runs through diff --git a/e2e/tests/up/provider_microsandbox.go b/e2e/tests/up/provider_microsandbox.go index bcce747670..bb691d26bc 100644 --- a/e2e/tests/up/provider_microsandbox.go +++ b/e2e/tests/up/provider_microsandbox.go @@ -50,8 +50,11 @@ var _ = ginkgo.Describe( }) ginkgo.DeferCleanup(f.CleanupWorkspace, tempDir) + performance, err := newMicrosandboxPerformanceCapture(source, name, f) + framework.ExpectNoError(err) + // full up: boots the microVM, streams in the agent, opens the tunnel - err = f.DevsyUp(ctx, tempDir, "--devcontainer", ".devcontainer.json") + err = performance.up(ctx, tempDir, "fresh", "--devcontainer", ".devcontainer.json") framework.ExpectNoError(err) // the workspace is reachable over SSH @@ -96,10 +99,11 @@ var _ = ginkgo.Describe( assertMicrosandboxHostEntries(ctx, f, tempDir, "fresh") err = f.DevsyWorkspaceStop(ctx, tempDir) framework.ExpectNoError(err) - err = f.DevsyUp(ctx, tempDir) + err = performance.up(ctx, tempDir, "reused") framework.ExpectNoError(err) gomega.Expect(microsandboxCreationTime(ctx, sandbox)).To(gomega.Equal(createdAt)) assertMicrosandboxHostEntries(ctx, f, tempDir, "restarted") + framework.ExpectNoError(performance.compare(ctx)) err = f.DevsyUpRecreate(ctx, tempDir) framework.ExpectNoError(err) gomega.Expect(microsandboxCreationTime(ctx, sandbox)).NotTo(gomega.Equal(createdAt)) diff --git a/e2e/tests/up/provider_microsandbox_consumer_linux.go b/e2e/tests/up/provider_microsandbox_consumer_linux.go new file mode 100644 index 0000000000..cd1c9adb5a --- /dev/null +++ b/e2e/tests/up/provider_microsandbox_consumer_linux.go @@ -0,0 +1,586 @@ +//go:build linux + +package up + +import ( + "archive/tar" + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/devsy-org/devsy-runtime-sdk/runtimev1" + "github.com/devsy-org/devsy/e2e/framework" + "github.com/devsy-org/devsy/e2e/framework/registryfixture" + devsyconfig "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/driver" + "github.com/devsy-org/devsy/pkg/driver/external" + "github.com/devsy-org/devsy/pkg/image" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +type microsandboxConsumerAttempt struct { + name string + credentials bool + trustFixture bool + wantErrorText string +} + +type microsandboxConsumerSeed struct { + handler http.Handler + repository string + imageDigest string + markerPath string + markerContent string + expectedBodies map[string]int64 +} + +type microsandboxConsumerHelper struct { + root, config, log, operationID string +} + +type microsandboxConsumerEnvironment struct { + fixture *registryfixture.Server + home, msbHome string + helper microsandboxConsumerHelper + trustFixture bool +} + +type microsandboxConsumerReceiptExpectation struct { + path, operationID, serverKey, providerBinary, providerHash string + required bool + outcome string +} + +type microsandboxConsumerProvider struct { + config provider.ProviderAgentConfig + origin string + runtimeBinary string + runtimeHash string +} + +type microsandboxConsumerHostRun struct { + host *external.Host + workspaceID string + imageRef string + attempt microsandboxConsumerAttempt + markerPath string + markerContent string +} + +const ( + microsandboxConsumerPositive = "positive" + microsandboxConsumerNoCA = "missing-ca" + microsandboxConsumerNoCredentials = "missing-credentials" +) + +var _ = ginkgo.Describe( + "published MicroSandbox provider registry consumption", + ginkgo.Label("up-provider-microsandbox-registry-consumer", "microsandbox-external"), + func() { + ginkgo.DescribeTable( + "uses the published provider's registry client", + runMicrosandboxConsumerAttempt, + ginkgo.Entry( + "downloads an authenticated image over fixture TLS", + microsandboxConsumerAttempt{ + name: microsandboxConsumerPositive, credentials: true, trustFixture: true, + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ), + ginkgo.Entry("rejects an untrusted registry certificate", microsandboxConsumerAttempt{ + name: microsandboxConsumerNoCA, credentials: true, wantErrorText: "certificate", + }, ginkgo.SpecTimeout(framework.TimeoutLong())), + ginkgo.Entry("rejects a registry without credentials", microsandboxConsumerAttempt{ + name: microsandboxConsumerNoCredentials, + trustFixture: true, + wantErrorText: "unauthorized", + }, ginkgo.SpecTimeout(framework.TimeoutLong())), + ) + }, +) + +func runMicrosandboxConsumerAttempt(ctx context.Context, attempt microsandboxConsumerAttempt) { + session := newMicrosandboxTestSession(ctx) + providerName := "case-a-consumer" + session.addProvider(ctx, microsandboxExternalProvider, providerName) + consumerProvider := prepareMicrosandboxConsumerProvider(ctx, providerName) + actualProviderHash := consumerProvider.runtimeHash + runtimeBinary := consumerProvider.runtimeBinary + + seed := seedMicrosandboxConsumerImage(ctx, attempt.name) + + fixture := newMicrosandboxConsumerRegistry(seed.handler) + imageRef := fixture.Authority + "/" + seed.repository + "@" + seed.imageDigest + assertMicrosandboxImageAbsentFromDocker(ctx, imageRef) + + workspaceID := microsandboxConsumerID(attempt.name) + privateHome := microsandboxConsumerDirectory(ginkgo.GinkgoT().TempDir()) + msbHome := microsandboxConsumerDirectory(ginkgo.GinkgoT().TempDir()) + cacheDir := filepath.Join(msbHome, "cache") + _, err := os.Stat(cacheDir) + gomega.Expect(err). + To(gomega.MatchError(os.ErrNotExist), "fresh MSB_HOME cache must not contain a prior image") + helper := prepareMicrosandboxConsumerHelper( + fixture, privateHome, actualProviderHash, attempt.credentials, + ) + setMicrosandboxConsumerEnvironment(microsandboxConsumerEnvironment{ + fixture: fixture, home: privateHome, msbHome: msbHome, + helper: helper, trustFixture: attempt.trustFixture, + }) + + host := newMicrosandboxConsumerHost(ctx, providerName, workspaceID, consumerProvider) + ginkgo.DeferCleanup(func(cleanupCtx ginkgo.SpecContext) error { + deleteCtx, cancel := context.WithTimeout(context.WithoutCancel(cleanupCtx), time.Minute) + defer cancel() + return host.DeleteDevContainer(deleteCtx, workspaceID) + }) + runMicrosandboxConsumerHost(ctx, microsandboxConsumerHostRun{ + host: host, workspaceID: workspaceID, imageRef: imageRef, attempt: attempt, + markerPath: seed.markerPath, markerContent: seed.markerContent, + }) + + framework.ExpectNoError(closeMicrosandboxConsumerRegistry(ctx, fixture)) + snapshot := fixture.Snapshot() + assertMicrosandboxConsumerSnapshot(attempt, snapshot, seed.expectedBodies) + receiptRequired := attempt.name == microsandboxConsumerPositive || + attempt.name == microsandboxConsumerNoCredentials + expectedOutcome := "" + switch attempt.name { + case microsandboxConsumerPositive: + expectedOutcome = "credentials_provided" + case microsandboxConsumerNoCredentials: + expectedOutcome = "credentials_not_found" + } + assertMicrosandboxConsumerReceipts(microsandboxConsumerReceiptExpectation{ + path: helper.log, operationID: helper.operationID, serverKey: fixture.Authority, + providerBinary: runtimeBinary, providerHash: actualProviderHash, + required: receiptRequired, outcome: expectedOutcome, + }) +} + +func newMicrosandboxConsumerRegistry(handler http.Handler) *registryfixture.Server { + fixture, err := registryfixture.Start(handler, registryfixture.Config{ + Username: "case-a-fixture-user", Password: "case-a-private-fixture-secret", + }) + framework.ExpectNoError(err) + ginkgo.DeferCleanup(func(ctx ginkgo.SpecContext) error { + return closeMicrosandboxConsumerRegistry(ctx, fixture) + }) + return fixture +} + +func closeMicrosandboxConsumerRegistry(ctx context.Context, fixture *registryfixture.Server) error { + closeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + return fixture.Close(closeCtx) +} + +func prepareMicrosandboxConsumerProvider( + ctx context.Context, + name string, +) microsandboxConsumerProvider { + installed, err := provider.LoadProviderConfig(devsyconfig.DefaultContext, name) + framework.ExpectNoError(err) + origin := ginkgo.GinkgoT().TempDir() + binaryDir := filepath.Join(origin, "binaries") + framework.ExpectNoError(os.MkdirAll(binaryDir, 0o700)) + binaryPaths, err := provider.DownloadBinaries(ctx, installed.Agent.Binaries, binaryDir) + framework.ExpectNoError(err) + binary, expectedHash := microsandboxConsumerBinary(installed.Agent, binaryPaths) + actualHash := microsandboxConsumerFileSHA256(binary) + gomega.Expect(actualHash).To(gomega.Equal(expectedHash)) + return microsandboxConsumerProvider{ + config: installed.Agent, origin: origin, runtimeBinary: binary, runtimeHash: actualHash, + } +} + +func newMicrosandboxConsumerHost( + ctx context.Context, + name, workspaceID string, + consumerProvider microsandboxConsumerProvider, +) *external.Host { + devsyConfig, err := devsyconfig.LoadConfig("", "") + framework.ExpectNoError(err) + workspace := &provider.Workspace{ + ID: workspaceID, UID: workspaceID, Context: devsyconfig.DefaultContext, + Provider: provider.WorkspaceProviderConfig{Name: name}, Origin: consumerProvider.origin, + } + info := &provider.AgentWorkspaceInfo{ + WorkspaceOrigin: consumerProvider.origin, + Workspace: workspace, + Agent: consumerProvider.config, + Options: provider.GetProviderOptions( + workspace, + nil, + devsyConfig, + ), + Origin: consumerProvider.origin, + } + host, err := external.New(ctx, info) + framework.ExpectNoError(err) + return host +} + +func runMicrosandboxConsumerHost(ctx context.Context, run microsandboxConsumerHostRun) { + framework.ExpectNoError(run.host.ProvisioningPreflight(ctx)) + found, err := run.host.FindDevContainer(ctx, run.workspaceID) + framework.ExpectNoError(err) + gomega.Expect(found).To(gomega.BeNil(), "workspace ID must be fresh before RunImage") + err = run.host.RunImage(ctx, &runtimev1.RunImageRequest{ + WorkspaceId: run.workspaceID, Image: run.imageRef, + User: microsandboxRootUser, RemoteUser: microsandboxRootUser, + Entrypoint: "/bin/sh", Args: []string{"-c", "sleep 600"}, + }) + if run.attempt.wantErrorText != "" { + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(strings.ToLower(err.Error())). + To(gomega.ContainSubstring(run.attempt.wantErrorText)) + found, err = run.host.FindDevContainer(ctx, run.workspaceID) + framework.ExpectNoError(err) + gomega.Expect(found).To(gomega.BeNil(), "failed RunImage must not leave a workspace") + return + } + framework.ExpectNoError(err) + var stdout strings.Builder + framework.ExpectNoError(run.host.CommandDevContainer(ctx, &driver.CommandParams{ + WorkspaceID: run.workspaceID, Command: "cat " + run.markerPath, Stdout: &stdout, + })) + gomega.Expect(stdout.String()).To(gomega.Equal(run.markerContent)) + found, err = run.host.FindDevContainer(ctx, run.workspaceID) + framework.ExpectNoError(err) + gomega.Expect(found).NotTo(gomega.BeNil()) +} + +func seedMicrosandboxConsumerImage(ctx context.Context, attempt string) microsandboxConsumerSeed { + img, err := image.GetImageForArch( + ctx, + "ghcr.io/devsy-org/test-images/base:alpine", + runtime.GOARCH, + ) + framework.ExpectNoError(err) + img, markerPath, markerContent := appendMicrosandboxConsumerMarker(img, attempt) + imageDigest, err := img.Digest() + framework.ExpectNoError(err) + + seedHandler := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + seed := httptest.NewServer(seedHandler) + seedClosed := false + ginkgo.DeferCleanup(func() { + if !seedClosed { + seed.Close() + } + }) + operationID := microsandboxConsumerID(attempt) + repository := "case-a/" + strings.TrimPrefix(operationID, "case-a-") + seedRef, err := name.NewTag( + strings.TrimPrefix(seed.URL, "http://")+"/"+repository+":fixture", + name.Insecure, + ) + framework.ExpectNoError(err) + framework.ExpectNoError(remote.Write(seedRef, img, remote.WithContext(ctx))) + seed.Close() + seedClosed = true + expected := microsandboxConsumerImageBodies(img) + return microsandboxConsumerSeed{ + handler: seedHandler, repository: repository, imageDigest: imageDigest.String(), + markerPath: markerPath, markerContent: markerContent, expectedBodies: expected, + } +} + +func appendMicrosandboxConsumerMarker(img v1.Image, attempt string) (v1.Image, string, string) { + markerToken := microsandboxConsumerID(attempt) + markerPath := "/case-a-marker-" + strings.TrimPrefix(markerToken, "case-a-") + markerContent := "published-provider-image-" + markerToken + "\n" + var markerTar bytes.Buffer + tarWriter := tar.NewWriter(&markerTar) + framework.ExpectNoError(tarWriter.WriteHeader(&tar.Header{ + Name: strings.TrimPrefix(markerPath, "/"), Mode: 0o644, + Size: int64(len(markerContent)), Typeflag: tar.TypeReg, Format: tar.FormatUSTAR, + })) + _, err := io.WriteString(tarWriter, markerContent) + framework.ExpectNoError(err) + framework.ExpectNoError(tarWriter.Close()) + markerLayer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(markerTar.Bytes())), nil + }) + framework.ExpectNoError(err) + imageWithMarker, err := mutate.AppendLayers(img, markerLayer) + framework.ExpectNoError(err) + return imageWithMarker, markerPath, markerContent +} + +func microsandboxConsumerImageBodies(img v1.Image) map[string]int64 { + manifestBytes, err := img.RawManifest() + framework.ExpectNoError(err) + manifestHash := sha256.Sum256(manifestBytes) + expected := map[string]int64{ + "sha256:" + hex.EncodeToString(manifestHash[:]): int64(len(manifestBytes)), + } + configBytes, err := img.RawConfigFile() + framework.ExpectNoError(err) + configDigest, err := img.ConfigName() + framework.ExpectNoError(err) + expected[configDigest.String()] = int64(len(configBytes)) + layers, err := img.Layers() + framework.ExpectNoError(err) + for _, layer := range layers { + digest, digestErr := layer.Digest() + framework.ExpectNoError(digestErr) + size, sizeErr := layer.Size() + framework.ExpectNoError(sizeErr) + expected[digest.String()] = size + } + return expected +} + +func microsandboxConsumerBinary( + agent provider.ProviderAgentConfig, + binaryPaths map[string]string, +) (string, string) { + binaryName := agent.External.Binary + binaryPath := binaryPaths[binaryName] + gomega.Expect(binaryPath). + NotTo(gomega.BeEmpty(), "published provider runtime binary was not downloaded") + var expected string + for _, binary := range agent.Binaries[binaryName] { + if binary.OS == runtime.GOOS && binary.Arch == runtime.GOARCH { + expected = strings.TrimPrefix(strings.ToLower(binary.Checksum), "sha256:") + break + } + } + gomega.Expect(expected). + To(gomega.HaveLen(sha256.Size*2), "published runtime checksum must be pinned") + return binaryPath, expected +} + +func microsandboxConsumerFileSHA256(path string) string { + // #nosec G304 -- DownloadBinaries returns this path inside the spec's private temporary origin. + file, err := os.Open(path) + framework.ExpectNoError(err) + defer func() { framework.ExpectNoError(file.Close()) }() + h := sha256.New() + _, err = io.Copy(h, file) + framework.ExpectNoError(err) + return hex.EncodeToString(h.Sum(nil)) +} + +func microsandboxConsumerID(suffix string) string { + var token [8]byte + _, err := rand.Read(token[:]) + framework.ExpectNoError(err) + return "case-a-" + suffix + "-" + hex.EncodeToString(token[:]) +} + +func microsandboxConsumerDirectory(path string) string { + // #nosec G302 -- Ginkgo owns this fresh test directory; private 0700 access is required. + framework.ExpectNoError(os.Chmod(path, 0o700)) + return path +} + +func prepareMicrosandboxConsumerHelper( + fixture *registryfixture.Server, + home, providerHash string, + credentials bool, +) microsandboxConsumerHelper { + root, err := os.MkdirTemp(home, "helper-") + framework.ExpectNoError(err) + root, err = filepath.EvalSymlinks(root) + framework.ExpectNoError(err) + // #nosec G302 -- the helper's executable/config/log tree is deliberately private to this spec. + framework.ExpectNoError(os.Chmod(root, 0o700)) + binDir := filepath.Join(root, "bin") + framework.ExpectNoError(os.Mkdir(binDir, 0o700)) + executable, err := os.Executable() + framework.ExpectNoError(err) + framework.ExpectNoError( + os.Symlink(executable, filepath.Join(binDir, registryfixture.HelperBinaryName)), + ) + operationID := microsandboxConsumerID("operation") + configPath, logPath := filepath.Join( + root, + "credentials.json", + ), filepath.Join( + root, + "observations.jsonl", + ) + config := registryfixture.CredentialConfig{ + OperationID: operationID, ServerKey: fixture.Authority, + CredentialsAvailable: credentials, ExpectedProviderSHA256: providerHash, + } + if credentials { + config.Username, config.Secret = fixture.Username, fixture.Password + } + // #nosec G117 -- private helper config intentionally holds fixture credentials and is written mode 0600. + data, err := json.Marshal(config) + framework.ExpectNoError(err) + framework.ExpectNoError(os.WriteFile(configPath, data, 0o600)) + dockerDir := filepath.Join(home, ".docker") + framework.ExpectNoError(os.Mkdir(dockerDir, 0o700)) + framework.ExpectNoError(os.Mkdir(filepath.Join(home, ".config"), 0o700)) + dockerConfig, err := json.Marshal(struct { + CredHelpers map[string]string `json:"credHelpers"` + }{CredHelpers: map[string]string{fixture.Authority: "fixture"}}) + framework.ExpectNoError(err) + framework.ExpectNoError( + os.WriteFile(filepath.Join(dockerDir, "config.json"), dockerConfig, 0o600), + ) + return microsandboxConsumerHelper{ + root: root, + config: configPath, + log: logPath, + operationID: operationID, + } +} + +func setMicrosandboxConsumerEnvironment(environment microsandboxConsumerEnvironment) { + fixture, home, msbHome := environment.fixture, environment.home, environment.msbHome + helper := environment.helper + binDir := filepath.Join(helper.root, "bin") + for name, value := range map[string]string{ + "HOME": home, "XDG_CONFIG_HOME": filepath.Join(home, ".config"), + "DOCKER_CONFIG": filepath.Join(home, ".docker"), "MSB_HOME": msbHome, + "MSB_CONFIG_PATH": filepath.Join(msbHome, "config.json"), + "CASE_A_CREDENTIAL_ROOT": helper.root, "CASE_A_CREDENTIAL_CONFIG": helper.config, + "CASE_A_CREDENTIAL_OBSERVATIONS": helper.log, "CASE_A_OPERATION_ID": helper.operationID, + "HTTPS_PROXY": fixture.ProxyURL, "HTTP_PROXY": fixture.ProxyURL, + "https_proxy": fixture.ProxyURL, "http_proxy": fixture.ProxyURL, + "NO_PROXY": "", "no_proxy": "", "REGISTRY_AUTH_FILE": "", + "PATH": binDir + string(os.PathListSeparator) + os.Getenv("PATH"), + } { + ginkgo.GinkgoT().Setenv(name, value) + } + certPath := filepath.Join(home, "fixture-ca.pem") + cert := []byte(nil) + if environment.trustFixture { + cert = fixture.CAPEM + } + framework.ExpectNoError(os.WriteFile(certPath, cert, 0o600)) + framework.ExpectNoError(os.Mkdir(filepath.Join(home, "empty-cert-dir"), 0o700)) + ginkgo.GinkgoT().Setenv("SSL_CERT_FILE", certPath) + ginkgo.GinkgoT().Setenv("SSL_CERT_DIR", filepath.Join(home, "empty-cert-dir")) +} + +func assertMicrosandboxConsumerSnapshot( + attempt microsandboxConsumerAttempt, + snapshot registryfixture.Snapshot, + expected map[string]int64, +) { + gomega.Expect(snapshot.ConnectAllowed).To(gomega.BeNumerically(">", 0)) + gomega.Expect(snapshot.ConnectDenied).To(gomega.BeZero()) + switch attempt.name { + case microsandboxConsumerPositive: + gomega.Expect(snapshot.Authenticated).To(gomega.BeNumerically(">", 0)) + gomega.Expect(snapshot.ManifestGets).To(gomega.BeNumerically(">", 0)) + gomega.Expect(snapshot.BlobGets).To(gomega.BeNumerically(">", 0)) + assertMicrosandboxConsumerPositiveRequests(snapshot.Requests, expected) + case microsandboxConsumerNoCA: + gomega.Expect(snapshot.Authenticated).To(gomega.BeZero()) + gomega.Expect(snapshot.AuthRejected).To(gomega.BeZero()) + gomega.Expect(snapshot.Requests). + To(gomega.BeEmpty(), "TLS rejection must happen before HTTP requests") + case microsandboxConsumerNoCredentials: + gomega.Expect(snapshot.Authenticated).To(gomega.BeZero()) + gomega.Expect(snapshot.AuthRejected).To(gomega.BeNumerically(">", 0)) + gomega.Expect(snapshot.ManifestGets).To(gomega.BeZero()) + gomega.Expect(snapshot.BlobGets).To(gomega.BeZero()) + assertMicrosandboxConsumerChallenges(snapshot.Requests) + default: + ginkgo.Fail(fmt.Sprintf("unknown consumer attempt %q", attempt.name)) + } +} + +func assertMicrosandboxConsumerPositiveRequests( + requests []registryfixture.Request, + expected map[string]int64, +) { + observed := map[string]int64{} + for _, request := range requests { + if !request.Authenticated { + assertMicrosandboxConsumerChallenge(request) + continue + } + gomega.Expect(request.Status).To(gomega.BeNumerically(">=", 200)) + gomega.Expect(request.Status).To(gomega.BeNumerically("<", 300)) + if request.Method != http.MethodGet || + (request.Path != "manifests" && request.Path != "blobs") { + continue + } + gomega.Expect(request.BodyDigest).NotTo(gomega.BeEmpty()) + expectedSize, knownDigest := expected[request.BodyDigest] + gomega.Expect(knownDigest). + To(gomega.BeTrue(), "provider fetched an unexpected image body %s", request.BodyDigest) + gomega.Expect(request.Bytes).To(gomega.Equal(expectedSize)) + observed[request.BodyDigest] = request.Bytes + } + for digest, size := range expected { + gomega.Expect(observed[digest]). + To(gomega.Equal(size), "published image component %s was not received byte-for-byte", digest) + } +} + +func assertMicrosandboxConsumerChallenges(requests []registryfixture.Request) { + for _, request := range requests { + gomega.Expect(request.Authenticated).To(gomega.BeFalse()) + gomega.Expect(request.Method).To(gomega.Equal(http.MethodGet)) + // An anonymous client can receive the Basic challenge, then be refused + // again when it requests the protected manifest without credentials. + gomega.Expect(request.Path).To(gomega.Or(gomega.Equal("v2"), gomega.Equal("manifests"))) + gomega.Expect(request.Status).To(gomega.Equal(http.StatusUnauthorized)) + } +} + +func assertMicrosandboxConsumerChallenge(request registryfixture.Request) { + gomega.Expect(request.Method).To(gomega.Equal(http.MethodGet)) + gomega.Expect(request.Path).To(gomega.Equal("v2")) + gomega.Expect(request.Status).To(gomega.Equal(http.StatusUnauthorized)) +} + +func assertMicrosandboxConsumerReceipts(expected microsandboxConsumerReceiptExpectation) { + data, err := os.ReadFile(expected.path) + if os.IsNotExist(err) && !expected.required { + return + } + framework.ExpectNoError(err) + if len(data) == 0 { + gomega.Expect(expected.required). + To(gomega.BeFalse(), "provider did not invoke its credential helper") + return + } + if strings.Contains(string(data), "case-a-private-fixture-secret") { + ginkgo.Fail("credential observations included secret material") + } + for line := range strings.SplitSeq(strings.TrimSpace(string(data)), "\n") { + var observation registryfixture.CredentialObservation + framework.ExpectNoError(json.Unmarshal([]byte(line), &observation)) + gomega.Expect(observation.OperationID).To(gomega.Equal(expected.operationID)) + gomega.Expect(observation.ServerKey).To(gomega.Equal(expected.serverKey)) + gomega.Expect(observation.Action).To(gomega.Equal("get")) + gomega.Expect(observation.Parent.Executable).To(gomega.Equal(expected.providerBinary)) + gomega.Expect(observation.Parent.SHA256).To(gomega.Equal(expected.providerHash)) + gomega.Expect(observation.Parent.StartTimeTicks).To(gomega.BeNumerically(">", 0)) + if expected.outcome != "" { + gomega.Expect(observation.Outcome).To(gomega.Equal(expected.outcome)) + } + } +} diff --git a/e2e/tests/up/provider_microsandbox_performance.go b/e2e/tests/up/provider_microsandbox_performance.go new file mode 100644 index 0000000000..1f8be9170f --- /dev/null +++ b/e2e/tests/up/provider_microsandbox_performance.go @@ -0,0 +1,683 @@ +package up + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "debug/buildinfo" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/docker" + "github.com/devsy-org/devsy/pkg/provider" +) + +const ( + spawnProbeEnv = "DEVSY_TEST_RUNTIME_SPAWN_PROBE" + supervisorEnv = "DEVSY_TEST_RUNTIME_SUPERVISOR" + performanceDir = "DEVSY_TEST_RUNTIME_PERFORMANCE_DIR" + revisionEnv = "DEVSY_TEST_RUNTIME_REVISION" + sdkVersionEnv = "DEVSY_TEST_RUNTIME_SDK_VERSION" + performanceFile = "provider-performance.json" + runtimeSDKModule = "github.com/devsy-org/devsy-runtime-sdk" + probeModeDirect = "direct" + probeModeSupervised = "supervised" + performanceUpFresh = "fresh" + performanceUpReused = "reused" +) + +var startupEvent = regexp.MustCompile(`External runtime ([^ ]+) startup completed in ([^\s]+)`) + +type microsandboxPerformanceCapture struct { + active bool + probe string + supervisor string + directory string + config *provider.ProviderConfig + framework *framework.Framework + workspace *provider.Workspace + options map[string]config.OptionValue + upRuns []upPerformanceRun + probeRuns []probePerformanceRun + expectedSHA string + supervisorSHA string + revision string + sdkVersion string +} + +type performanceToolPaths struct { + probe, supervisor, directory, revision, sdkVersion string +} + +type performanceSDKTool struct { + name, path, packagePath string +} + +type stagedPerformanceRuntime struct { + name, path, checksum string +} + +type probeSequence struct { + environment []string + runtime stagedPerformanceRuntime + workspaceID string +} + +type probeInvocation struct { + sequence probeSequence + order int + mode string +} + +type upPerformanceRun struct { + Phase string `json:"phase"` + Succeeded bool `json:"succeeded"` + WallTimeNS int64 `json:"wall_time_ns"` + Startup []startupPerformance `json:"startup_events"` +} + +type startupPerformance struct { + Operation string `json:"operation"` + DurationNS int64 `json:"duration_ns"` +} + +type probePerformanceRun struct { + Order int `json:"order"` + Mode string `json:"launch_mode"` + Report json.RawMessage `json:"report"` +} + +type microsandboxPerformanceReport struct { + ProviderSource string `json:"provider_source"` + ExpectedBinarySHA256 string `json:"expected_binary_sha256"` + TestedCheckoutRevision string `json:"tested_checkout_revision"` + RuntimeSDKVersion string `json:"runtime_sdk_version"` + StartupMeasurement string `json:"startup_measurement"` + UpRuns []upPerformanceRun `json:"up_runs"` + ProbeRuns []probePerformanceRun `json:"probe_runs"` +} + +// newMicrosandboxPerformanceCapture enables the optional measurements only for +// the published external provider and only when all native-job paths exist. +func newMicrosandboxPerformanceCapture( + source, providerName string, + f *framework.Framework, +) (*microsandboxPerformanceCapture, error) { + capture := µsandboxPerformanceCapture{framework: f} + paths, active, err := configuredPerformanceTools(source) + if err != nil || !active { + return capture, err + } + config, err := installedExternalPerformanceProvider(providerName, source) + if err != nil { + return nil, err + } + capture.active = true + capture.probe = paths.probe + capture.supervisor = paths.supervisor + capture.directory = paths.directory + capture.config = config + capture.revision = paths.revision + capture.sdkVersion = paths.sdkVersion + capture.supervisorSHA, err = fileSHA256(paths.supervisor) + if err != nil { + return nil, fmt.Errorf("hash SDK runtime supervisor: %w", err) + } + return capture, nil +} + +func configuredPerformanceTools(source string) (performanceToolPaths, bool, error) { + if source != microsandboxExternalProvider { + return performanceToolPaths{}, false, nil + } + paths := performanceToolPaths{ + probe: os.Getenv(spawnProbeEnv), supervisor: os.Getenv(supervisorEnv), + directory: os.Getenv(performanceDir), revision: os.Getenv(revisionEnv), + sdkVersion: os.Getenv(sdkVersionEnv), + } + set := 0 + for _, value := range []string{ + paths.probe, paths.supervisor, paths.directory, paths.revision, paths.sdkVersion, + } { + if value != "" { + set++ + } + } + if set == 0 { + return performanceToolPaths{}, false, nil + } + if set != 5 { + return performanceToolPaths{}, false, errors.New( + "MicroSandbox performance tools are only partially configured", + ) + } + if runtime.GOOS != osLinux { + return performanceToolPaths{}, false, errors.New( + "MicroSandbox real-provider performance capture requires Linux", + ) + } + if err := validatePerformanceToolPaths(paths); err != nil { + return performanceToolPaths{}, false, err + } + return paths, true, nil +} + +func validatePerformanceToolPaths(paths performanceToolPaths) error { + if err := validatePerformanceSDKTools(paths); err != nil { + return err + } + return validatePerformanceReportDirectory(paths.directory) +} + +func validatePerformanceSDKTools(paths performanceToolPaths) error { + for _, tool := range []performanceSDKTool{ + {spawnProbeEnv, paths.probe, runtimeSDKModule + "/cmd/devsy-runtime-spawn-probe"}, + {supervisorEnv, paths.supervisor, runtimeSDKModule + "/cmd/devsy-runtime-supervisor"}, + } { + if err := validatePerformanceSDKTool(tool, paths.sdkVersion); err != nil { + return err + } + } + return nil +} + +func validatePerformanceSDKTool(tool performanceSDKTool, version string) error { + if !filepath.IsAbs(tool.path) { + return fmt.Errorf("%s must be an absolute path", tool.name) + } + info, err := os.Stat(tool.path) + if err != nil { + return fmt.Errorf("%s is unavailable: %w", tool.name, err) + } + if !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 { + return fmt.Errorf("%s must name an executable file", tool.name) + } + return validateSDKTool(tool.path, tool.packagePath, version) +} + +func validatePerformanceReportDirectory(directory string) error { + if !filepath.IsAbs(directory) { + return errors.New("performance report directory must be absolute") + } + // #nosec G703 -- this absolute report path is supplied by the trusted native CI job. + dirInfo, err := os.Stat(directory) + if err != nil { + return fmt.Errorf("performance report directory is unavailable: %w", err) + } + if !dirInfo.IsDir() { + return errors.New("performance report path is not a directory") + } + return nil +} + +func installedExternalPerformanceProvider( + providerName, source string, +) (*provider.ProviderConfig, error) { + installed, err := provider.LoadProviderConfig(config.DefaultContext, providerName) + if err != nil { + return nil, fmt.Errorf("load installed MicroSandbox provider configuration: %w", err) + } + if installed.Source.Raw != source || installed.Agent.Driver != provider.ExternalDriver || + installed.Agent.External.Binary == "" { + return nil, errors.New( + "installed provider does not match the expected external MicroSandbox source", + ) + } + return installed, nil +} + +func validateSDKTool(path, packagePath, version string) error { + info, err := buildinfo.ReadFile(path) + if err != nil { + return fmt.Errorf("read runtime probe tool build information: %w", err) + } + if info.Path != packagePath || info.Main.Path != runtimeSDKModule || + info.Main.Version != version || + info.Main.Replace != nil { + return fmt.Errorf("runtime probe tool is not the expected unmodified SDK %s build", version) + } + return nil +} + +func (c *microsandboxPerformanceCapture) up( + ctx context.Context, + workspace, phase string, + additionalArgs ...string, +) error { + if !c.active { + args := append([]string{workspace}, additionalArgs...) + return c.framework.DevsyUp(ctx, args...) + } + if err := c.validateUpPhase(phase); err != nil { + return err + } + _, err := c.captureUp(ctx, workspace, phase, additionalArgs...) + if err != nil { + return err + } + if phase == performanceUpFresh { + return c.captureFreshWorkspace(ctx, workspace) + } + return nil +} + +func (c *microsandboxPerformanceCapture) validateUpPhase(phase string) error { + if phase != performanceUpFresh && phase != performanceUpReused { + return fmt.Errorf("unsupported MicroSandbox performance up phase %q", phase) + } + if phase == performanceUpFresh && len(c.upRuns) == 0 { + return nil + } + if phase == performanceUpReused && len(c.upRuns) == 1 && + c.upRuns[0].Phase == performanceUpFresh { + return nil + } + return fmt.Errorf("MicroSandbox performance up phase %q is out of order", phase) +} + +func (c *microsandboxPerformanceCapture) captureUp( + ctx context.Context, + workspace, phase string, + additionalArgs ...string, +) (upPerformanceRun, error) { + started := time.Now() + stdout, stderr, err := c.framework.DevsyUpStreams(ctx, workspace, additionalArgs...) + run := upPerformanceRun{ + Phase: phase, + Succeeded: err == nil, + WallTimeNS: time.Since(started).Nanoseconds(), + } + run.Startup = append(run.Startup, parseStartupEvents(stdout)...) + run.Startup = append(run.Startup, parseStartupEvents(stderr)...) + c.upRuns = append(c.upRuns, run) + if writeErr := c.writeReport(); writeErr != nil { + return run, errors.Join( + err, + fmt.Errorf("write MicroSandbox performance report: %w", writeErr), + ) + } + if err != nil { + return run, err + } + if len(run.Startup) == 0 { + return run, errors.New( + "successful MicroSandbox up emitted no valid external runtime startup timing events", + ) + } + return run, nil +} + +func (c *microsandboxPerformanceCapture) captureFreshWorkspace( + ctx context.Context, + workspace string, +) error { + var err error + c.workspace, err = c.framework.FindWorkspace(ctx, workspace) + if err != nil { + return fmt.Errorf("find fresh MicroSandbox workspace for runtime probe: %w", err) + } + devsyConfig, err := config.LoadConfig("", "") + if err != nil { + return fmt.Errorf("load options for runtime probe environment: %w", err) + } + c.options = provider.GetProviderOptions(c.workspace, nil, devsyConfig) + return nil +} + +func parseStartupEvents(output string) []startupPerformance { + var events []startupPerformance + for line := range strings.SplitSeq(output, "\n") { + match := startupEvent.FindStringSubmatch(line) + if len(match) != 3 { + continue + } + duration, err := time.ParseDuration(match[2]) + if err == nil && duration > 0 { + events = append( + events, + startupPerformance{Operation: match[1], DurationNS: duration.Nanoseconds()}, + ) + } + } + return events +} + +func (c *microsandboxPerformanceCapture) compare(ctx context.Context) (retErr error) { + if !c.active { + return nil + } + if !c.capturesReady() { + return errors.New( + "runtime comparison requires successful fresh and reused provider up captures", + ) + } + stage, err := os.MkdirTemp("", "devsy-runtime-performance-") + if err != nil { + return fmt.Errorf("create private runtime staging directory: %w", err) + } + defer func() { + if err := os.RemoveAll(stage); err != nil { + retErr = errors.Join( + retErr, + fmt.Errorf("remove private runtime staging directory: %w", err), + ) + } + }() + return c.compareWithStage(ctx, stage) +} + +func (c *microsandboxPerformanceCapture) capturesReady() bool { + return len(c.upRuns) == 2 && c.upRuns[0].Phase == performanceUpFresh && + c.upRuns[0].Succeeded && c.upRuns[1].Phase == performanceUpReused && + c.upRuns[1].Succeeded && c.workspace != nil && c.options != nil +} + +func (c *microsandboxPerformanceCapture) compareWithStage(ctx context.Context, stage string) error { + paths, err := provider.DownloadBinaries(ctx, c.config.Agent.Binaries, stage) + if err != nil { + return fmt.Errorf("stage published external runtime from Agent.Binaries: %w", err) + } + staged, err := c.validateStagedRuntime(paths) + if err != nil { + return err + } + c.expectedSHA = staged.checksum + workspaceID, err := absentWorkspaceID() + if err != nil { + return err + } + probeEnv := provider.ToEnvironment( + c.workspace, + nil, + c.options, + map[string]string{staged.name: staged.path}, + ) + return c.runProbeSequence(ctx, probeSequence{ + environment: probeEnv, runtime: staged, workspaceID: workspaceID, + }) +} + +func (c *microsandboxPerformanceCapture) validateStagedRuntime( + paths map[string]string, +) (stagedPerformanceRuntime, error) { + binaryName := c.config.Agent.External.Binary + binaryPath := paths[binaryName] + if binaryPath == "" || !filepath.IsAbs(binaryPath) { + return stagedPerformanceRuntime{}, errors.New( + "published external runtime did not resolve to an absolute staged binary", + ) + } + if err := validateStagedExecutable(binaryPath); err != nil { + return stagedPerformanceRuntime{}, err + } + expected, err := selectedRuntimeChecksum(c.config, binaryName) + if err != nil { + return stagedPerformanceRuntime{}, err + } + actual, err := fileSHA256(binaryPath) + if err != nil { + return stagedPerformanceRuntime{}, fmt.Errorf("hash staged external runtime: %w", err) + } + if actual != expected { + return stagedPerformanceRuntime{}, fmt.Errorf( + "staged external runtime checksum differs from provider manifest: got %s", + actual, + ) + } + return stagedPerformanceRuntime{name: binaryName, path: binaryPath, checksum: expected}, nil +} + +func validateStagedExecutable(path string) error { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("stat staged external runtime: %w", err) + } + if info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0 { + return nil + } + return errors.New("staged external runtime is not an executable regular file") +} + +func (c *microsandboxPerformanceCapture) runProbeSequence( + ctx context.Context, + sequence probeSequence, +) error { + for index, mode := range []string{ + probeModeDirect, probeModeSupervised, probeModeSupervised, probeModeDirect, + } { + invocation := probeInvocation{sequence: sequence, order: index + 1, mode: mode} + if err := c.runProbe(ctx, invocation); err != nil { + return err + } + } + return nil +} + +func (c *microsandboxPerformanceCapture) runProbe( + ctx context.Context, + invocation probeInvocation, +) error { + sequence := invocation.sequence + args := []string{ + "--binary", sequence.runtime.path, "--samples", "100", "--timeout", "15s", + "--workspace-id", sequence.workspaceID, "--revision", c.revision, + } + if invocation.mode == probeModeSupervised { + args = append(args, "--supervisor-binary", c.supervisor) + } + args = append(args, "--") + args = append(args, c.config.Agent.External.Args...) + // #nosec G204 -- SDK tools and runtime checksum are verified; arguments are fixed flags plus manifest argv. + command := exec.CommandContext(ctx, c.probe, args...) + docker.PrepareForGroupCancellation(command) + command.WaitDelay = 5 * time.Second + command.Env = sequence.environment + var stdout strings.Builder + command.Stdout = &stdout + command.Stderr = io.Discard + if err := command.Run(); err != nil { + return fmt.Errorf( + "runtime spawn probe run %d (%s) failed: %w", invocation.order, invocation.mode, err, + ) + } + raw := []byte(stdout.String()) + expectation := probeExpectations{ + mode: invocation.mode, binarySHA: sequence.runtime.checksum, + supervisorSHA: c.supervisorSHA, revision: c.revision, + } + if err := expectation.validate(raw); err != nil { + return fmt.Errorf( + "runtime spawn probe run %d (%s): %w", invocation.order, invocation.mode, err, + ) + } + c.probeRuns = append( + c.probeRuns, + probePerformanceRun{ + Order: invocation.order, Mode: invocation.mode, Report: json.RawMessage(raw), + }, + ) + if err := c.writeReport(); err != nil { + return fmt.Errorf("write MicroSandbox performance report: %w", err) + } + return nil +} + +func selectedRuntimeChecksum(config *provider.ProviderConfig, name string) (string, error) { + for _, binary := range config.Agent.Binaries[name] { + if binary == nil { + continue + } + if binary.OS == runtime.GOOS && binary.Arch == runtime.GOARCH { + checksum := strings.TrimPrefix(strings.ToLower(binary.Checksum), "sha256:") + decoded, err := hex.DecodeString(checksum) + if err != nil || len(decoded) != sha256.Size { + return "", errors.New("provider runtime manifest has an invalid SHA-256 checksum") + } + return checksum, nil + } + } + return "", fmt.Errorf( + "provider runtime manifest has no %s/%s entry", + runtime.GOOS, + runtime.GOARCH, + ) +} + +type runtimeProbeSample struct { + Reaped bool `json:"reaped"` +} + +type runtimeProbeMeasurements struct { + Operation string `json:"operation"` + First runtimeProbeSample `json:"first_launch"` + Warm []runtimeProbeSample `json:"warm_samples"` +} + +type runtimeProbeReport struct { + SchemaVersion int `json:"schema_version"` + LaunchMode string `json:"launch_mode"` + SupervisorSHA256 string `json:"supervisor_sha256"` + GOOS string `json:"goos"` + GOARCH string `json:"goarch"` + GoVersion string `json:"go_version"` + Revision string `json:"revision"` + BinarySHA256 string `json:"binary_sha256"` + ColdCacheMeasured *bool `json:"cold_cache_measured"` + Info runtimeProbeMeasurements `json:"info"` + Find runtimeProbeMeasurements `json:"find"` +} + +type probeExpectations struct { + mode, binarySHA, supervisorSHA, revision string +} + +func (expected probeExpectations) validate(raw []byte) error { + var report runtimeProbeReport + if err := json.Unmarshal(raw, &report); err != nil { + return fmt.Errorf("decode probe JSON: %w", err) + } + if err := report.validateMetadata(expected); err != nil { + return err + } + if err := validateMeasurements("Info", report.Info); err != nil { + return err + } + return validateMeasurements("Find", report.Find) +} + +func (report runtimeProbeReport) validateMetadata(expected probeExpectations) error { + if err := report.validateLaunch(expected); err != nil { + return err + } + if err := report.validatePlatform(expected); err != nil { + return err + } + return report.validateSupervisor(expected) +} + +func (report runtimeProbeReport) validateLaunch(expected probeExpectations) error { + if expected.mode != probeModeDirect && expected.mode != probeModeSupervised { + return errors.New("probe launch mode is unsupported") + } + if report.SchemaVersion != 1 || report.LaunchMode != expected.mode || + report.BinarySHA256 != expected.binarySHA || report.ColdCacheMeasured == nil || + *report.ColdCacheMeasured { + return errors.New( + "probe report does not match requested launch mode, expected binary, or cache claim", + ) + } + return nil +} + +func (report runtimeProbeReport) validatePlatform(expected probeExpectations) error { + if report.GOOS == runtime.GOOS && report.GOARCH == runtime.GOARCH && + report.GoVersion != "" && report.Revision == expected.revision { + return nil + } + return errors.New( + "probe report platform or tested checkout revision does not match the current run", + ) +} + +func (report runtimeProbeReport) validateSupervisor(expected probeExpectations) error { + wantSupervisorSHA := "" + if expected.mode == probeModeSupervised { + wantSupervisorSHA = expected.supervisorSHA + } + if report.SupervisorSHA256 != wantSupervisorSHA { + return errors.New( + "probe report supervisor checksum does not match the selected launch mode", + ) + } + return nil +} + +func validateMeasurements(label string, measurement runtimeProbeMeasurements) error { + if measurement.Operation != label || !measurement.First.Reaped || len(measurement.Warm) != 100 { + return fmt.Errorf( + "%s report is missing the expected first launch or 100 warm samples", + label, + ) + } + for _, sample := range measurement.Warm { + if !sample.Reaped { + return fmt.Errorf("%s report includes a process that was not reaped", label) + } + } + return nil +} + +func (c *microsandboxPerformanceCapture) writeReport() error { + report := microsandboxPerformanceReport{ + ProviderSource: microsandboxExternalProvider, + ExpectedBinarySHA256: c.expectedSHA, + TestedCheckoutRevision: c.revision, + RuntimeSDKVersion: c.sdkVersion, + StartupMeasurement: "external runtime startup event; excludes binary resolution, RPC, and process reaping", + UpRuns: c.upRuns, + ProbeRuns: c.probeRuns, + } + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(c.directory, performanceFile), append(data, '\n'), 0o600) +} + +func fileSHA256(path string) (string, error) { + // #nosec G304 -- caller selects only the validated SDK supervisor or downloaded manifest binary in private staging. + // #nosec G703 -- absolute SDK path is CI-validated; runtime path is the manifest download in private staging. + file, err := os.Open(path) + if err != nil { + return "", err + } + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + _ = file.Close() + return "", err + } + if err := file.Close(); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func absentWorkspaceID() (string, error) { + var suffix [16]byte + if _, err := rand.Read(suffix[:]); err != nil { + return "", fmt.Errorf("generate absent workspace ID: %w", err) + } + return "runtime-probe-" + hex.EncodeToString(suffix[:]), nil +} diff --git a/e2e/tests/up/provider_microsandbox_performance_test.go b/e2e/tests/up/provider_microsandbox_performance_test.go new file mode 100644 index 0000000000..1e87a66bbd --- /dev/null +++ b/e2e/tests/up/provider_microsandbox_performance_test.go @@ -0,0 +1,264 @@ +package up + +import ( + "encoding/json" + "os" + "runtime" + "strings" + "testing" + "time" +) + +type probeValidationCase struct { + name string + mode string + supervisor string + mutate func(map[string]any) + malformed bool + wantError string +} + +func TestProbeExpectationsValidateSDKReports(t *testing.T) { + runProbeValidationCases(t, append( + append(probeMetadataCases(), probeOperationCases()...), probeProcessCases()..., + )) +} + +func probeMetadataCases() []probeValidationCase { + return append(probeIdentityCases(), probeCacheCases()...) +} + +func probeIdentityCases() []probeValidationCase { + return []probeValidationCase{ + {name: "valid direct report", mode: probeModeDirect}, + {name: "valid supervised report", mode: probeModeSupervised, supervisor: "supervisor-hash"}, + { + name: "malformed JSON", + mode: probeModeDirect, + malformed: true, + wantError: "decode probe JSON", + }, + { + name: "wrong runtime checksum", + mode: probeModeDirect, + mutate: func(r map[string]any) { r["binary_sha256"] = "other-hash" }, + wantError: "expected binary", + }, + { + name: "wrong launch mode", + mode: probeModeDirect, + mutate: func(r map[string]any) { r["launch_mode"] = probeModeSupervised }, + wantError: "launch mode", + }, + { + name: "unsupported expected launch mode", + mode: "other", + wantError: "launch mode is unsupported", + }, + { + name: "wrong platform", mode: probeModeDirect, + mutate: func(r map[string]any) { r["goos"] = "other" }, wantError: "platform", + }, + { + name: "missing Go version", mode: probeModeDirect, + mutate: func(r map[string]any) { r["go_version"] = "" }, wantError: "platform", + }, + { + name: "wrong supervised helper hash", + mode: probeModeSupervised, + supervisor: "supervisor-hash", + mutate: func(r map[string]any) { r["supervisor_sha256"] = "other-helper" }, + wantError: "supervisor checksum", + }, + { + name: "direct report with helper hash", + mode: probeModeDirect, + mutate: func(r map[string]any) { r["supervisor_sha256"] = "unexpected-helper" }, + wantError: "supervisor checksum", + }, + } +} + +func probeCacheCases() []probeValidationCase { + return []probeValidationCase{ + { + name: "missing cache field", + mode: probeModeDirect, + mutate: func(r map[string]any) { delete(r, "cold_cache_measured") }, + wantError: "cache claim", + }, + { + name: "true cache flag", + mode: probeModeDirect, + mutate: func(r map[string]any) { r["cold_cache_measured"] = true }, + wantError: "cache claim", + }, + { + name: "ill-typed cache flag", + mode: probeModeDirect, + mutate: func(r map[string]any) { r["cold_cache_measured"] = "false" }, + wantError: "decode probe JSON", + }, + { + name: "wrong tested revision", + mode: probeModeDirect, + mutate: func(r map[string]any) { r["revision"] = "different-revision" }, + wantError: "revision", + }, + } +} + +func probeOperationCases() []probeValidationCase { + return []probeValidationCase{ + { + name: "wrong Info operation", + mode: probeModeDirect, + mutate: func(r map[string]any) { probeSection(r, "info")["operation"] = "RunImage" }, + wantError: "Info report", + }, + { + name: "wrong warm count", + mode: probeModeDirect, + mutate: func(r map[string]any) { probeSection(r, "info")["warm_samples"] = []any{} }, + wantError: "Info report", + }, + } +} + +func probeProcessCases() []probeValidationCase { + return []probeValidationCase{ + { + name: "unreaped first launch", + mode: probeModeDirect, + mutate: func(r map[string]any) { probeSection(r, "find")["first_launch"].(map[string]any)["reaped"] = false }, + wantError: "Find report", + }, + { + name: "unreaped warm sample", mode: probeModeDirect, + mutate: func(r map[string]any) { + probeSection(r, "find")["warm_samples"].([]any)[0].(map[string]any)["reaped"] = false + }, + wantError: "process that was not reaped", + }, + } +} + +func runProbeValidationCases(t *testing.T, tests []probeValidationCase) { + t.Helper() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { runProbeValidationCase(t, test) }) + } +} + +func runProbeValidationCase(t *testing.T, test probeValidationCase) { + t.Helper() + raw, err := test.reportJSON() + if err != nil { + t.Fatal(err) + } + wantSupervisor := test.supervisorHash() + expectation := probeExpectations{ + mode: test.mode, binarySHA: "runtime-hash", supervisorSHA: wantSupervisor, + revision: "tested-revision", + } + assertProbeError(t, expectation.validate(raw), test.wantError) +} + +func (test probeValidationCase) reportJSON() ([]byte, error) { + if test.malformed { + return []byte("{"), nil + } + report := validProbeReport(test.mode, test.supervisor) + if test.mutate != nil { + test.mutate(report) + } + return json.Marshal(report) +} + +func (test probeValidationCase) supervisorHash() string { + if test.mode == probeModeSupervised { + return test.supervisor + } + return "" +} + +func assertProbeError(t *testing.T, err error, expected string) { + t.Helper() + if expected == "" { + if err != nil { + t.Fatalf("expected a valid probe report, got %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), expected) { + t.Fatalf("expected error containing %q, got %v", expected, err) + } +} + +func TestValidateSDKToolRejectsDifferentBuildSource(t *testing.T) { + path, err := os.Executable() + if err != nil { + t.Fatal(err) + } + err = validateSDKTool( + path, + runtimeSDKModule+"/cmd/devsy-runtime-spawn-probe", + "v1.6.0", + ) + if err == nil { + t.Fatal("expected the test binary to fail SDK probe provenance validation") + } +} + +func TestParseStartupEvents(t *testing.T) { + output := strings.Join([]string{ + "DEBU External runtime Info startup completed in 4.2ms", + "DEBU External runtime Find startup completed in 1s", + "DEBU External runtime Info startup completed in someday", + "DEBU External runtime Info startup completed in 0s", + "DEBU External runtime Info startup completed in -3ms", + }, "\n") + + events := parseStartupEvents(output) + if len(events) != 2 { + t.Fatalf("expected two positive startup timings, got %#v", events) + } + if events[0].Operation != "Info" || events[0].DurationNS != int64(4200*time.Microsecond) { + t.Errorf("unexpected first startup event: %#v", events[0]) + } + if events[1].Operation != "Find" || events[1].DurationNS != int64(time.Second) { + t.Errorf("unexpected second startup event: %#v", events[1]) + } +} + +func validProbeReport(mode, supervisorSHA string) map[string]any { + return map[string]any{ + "schema_version": 1, + "launch_mode": mode, + "supervisor_sha256": supervisorSHA, + "goos": runtime.GOOS, + "goarch": runtime.GOARCH, + "go_version": "go1.26.0", + "revision": "tested-revision", + "binary_sha256": "runtime-hash", + "cold_cache_measured": false, + "info": validMeasurement("Info"), + "find": validMeasurement("Find"), + } +} + +func validMeasurement(operation string) map[string]any { + warm := make([]any, 100) + for index := range warm { + warm[index] = map[string]any{"reaped": true} + } + return map[string]any{ + "operation": operation, + "first_launch": map[string]any{"reaped": true}, + "warm_samples": warm, + } +} + +func probeSection(report map[string]any, name string) map[string]any { + return report[name].(map[string]any) +} diff --git a/hack/microsandbox/command.go b/hack/microsandbox/command.go index 09c73d4e33..e7f765192c 100644 --- a/hack/microsandbox/command.go +++ b/hack/microsandbox/command.go @@ -9,6 +9,7 @@ import ( type suiteConfig struct { label string expectedCount int + providerMode string repositoryDir string tempDir string testTimeout string diff --git a/hack/microsandbox/main.go b/hack/microsandbox/main.go index 464a1d0549..59762e1486 100644 --- a/hack/microsandbox/main.go +++ b/hack/microsandbox/main.go @@ -11,9 +11,10 @@ import ( ) const ( - installLinuxCommand = "install-linux" - runSuiteCommand = "run" - verifyReportCommand = "verify-report" + installLinuxCommand = "install-linux" + runSuiteCommand = "run" + verifyReportCommand = "verify-report" + buildPerformanceToolsCommand = "build-performance-tools" ) func main() { @@ -24,7 +25,7 @@ func runCLI(args []string, stdout, stderr io.Writer) int { if len(args) == 0 { _, _ = fmt.Fprintln( stderr, - "usage: devsy-ci-microsandbox ", + "usage: devsy-ci-microsandbox ", ) return 2 } @@ -35,6 +36,8 @@ func runCLI(args []string, stdout, stderr io.Writer) int { return runSuiteCLI(args[1:], stdout, stderr) case verifyReportCommand: return verifyReportCLI(args[1:], stdout, stderr) + case buildPerformanceToolsCommand: + return buildPerformanceToolsCLI(args[1:], stdout, stderr) default: _, _ = fmt.Fprintf(stderr, "unknown command: %s\n", args[0]) return 2 @@ -55,11 +58,21 @@ func installLinuxCLI(args []string, stdout, stderr io.Writer) int { } func runSuiteCLI(args []string, stdout, stderr io.Writer) int { + return runSuiteCLIWithRunner(args, stdout, stderr, commandRunner((*exec.Cmd).Run)) +} + +func runSuiteCLIWithRunner(args []string, stdout, stderr io.Writer, run commandRunner) int { flags := flag.NewFlagSet(runSuiteCommand, flag.ContinueOnError) flags.SetOutput(stderr) cfg := suiteConfig{} flags.StringVar(&cfg.label, "label", "", "Ginkgo suite label") flags.IntVar(&cfg.expectedCount, "expected-count", 0, "required scenario count") + flags.StringVar( + &cfg.providerMode, + "provider-mode", + providerModeParity, + "provider label validation mode", + ) flags.StringVar(&cfg.repositoryDir, "repository-dir", ".", "Devsy checkout directory") flags.StringVar( &cfg.tempDir, @@ -83,7 +96,7 @@ func runSuiteCLI(args []string, stdout, stderr io.Writer) int { return runSuite( context.Background(), cfg, - commandRunner((*exec.Cmd).Run), + run, commandOutput{stdout: stdout, stderr: stderr}, ) } @@ -94,17 +107,18 @@ func verifyReportCLI(args []string, stdout, stderr io.Writer) int { report := flags.String("report", "", "Ginkgo JSON report path") label := flags.String("label", "", "Ginkgo suite label") count := flags.Int("expected-count", 0, "required scenario count") + mode := flags.String("provider-mode", providerModeParity, "provider label validation mode") if err := flags.Parse(args); err != nil { return 2 } - if flags.NArg() != 0 || !validReportArguments(*report, *label, *count) { + if flags.NArg() != 0 || !validReportArguments(*report, *label, *count, *mode) { _, _ = fmt.Fprintln( stderr, - "verify-report requires a report, label and positive even expected-count", + "verify-report requires a report, label, valid provider-mode and valid expected-count", ) return 2 } - if err := validateReport(*report, *label, *count); err != nil { + if err := validateReport(*report, *label, *count, *mode); err != nil { _, _ = fmt.Fprintf(stderr, "::error::%v\n", err) return 1 } @@ -119,6 +133,6 @@ func verifyReportCLI(args []string, stdout, stderr io.Writer) int { return 0 } -func validReportArguments(report, label string, count int) bool { - return report != "" && label != "" && isPositiveEven(count) +func validReportArguments(report, label string, count int, mode string) bool { + return report != "" && label != "" && validProviderExpectation(mode, count) } diff --git a/hack/microsandbox/main_test.go b/hack/microsandbox/main_test.go index acb273a8bb..83160ba806 100644 --- a/hack/microsandbox/main_test.go +++ b/hack/microsandbox/main_test.go @@ -3,28 +3,46 @@ package main import ( "bytes" "os" + "os/exec" "path/filepath" "testing" "github.com/stretchr/testify/require" ) +const ( + labelFlag = "--label" + unknownProviderMode = "unknown" + expectedCountFlag = "--expected-count" + reportFlag = "--report" + providerModeFlag = "--provider-mode" + cliReportLabel = "cli-report" +) + func TestCLIRejectsInvalidArguments(t *testing.T) { cases := [][]string{ nil, - {"unknown"}, + {"invalid-command"}, {runSuiteCommand, "--unknown-flag"}, + {runSuiteCommand, labelFlag, "suite", expectedCountFlag, "3"}, {runSuiteCommand, "unexpected-positional"}, {verifyReportCommand}, { verifyReportCommand, - "--report", + reportFlag, "missing.json", - "--label", - "cli-report", - "--expected-count", + labelFlag, + cliReportLabel, + expectedCountFlag, "3", }, + { + verifyReportCommand, + reportFlag, "report.json", + labelFlag, cliReportLabel, + expectedCountFlag, "3", + providerModeFlag, unknownProviderMode, + }, {installLinuxCommand, "unexpected-positional"}, } for _, args := range cases { @@ -40,9 +58,47 @@ func TestCLIVerifyReportRejectsMalformedJSON(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte("{"), 0o600)) var stdout, stderr bytes.Buffer code := runCLI([]string{ - verifyReportCommand, "--report", path, "--label", "cli-report", "--expected-count", "2", + verifyReportCommand, reportFlag, path, labelFlag, cliReportLabel, expectedCountFlag, "2", }, &stdout, &stderr) require.Equal(t, 1, code) require.Contains(t, stderr.String(), "::error::") require.Empty(t, stdout.String()) } + +func TestCLIVerifyReportAcceptsExternalMode(t *testing.T) { + path := writeReportFixture(t, validExternalReportFor(reportTestLabel)) + var stdout, stderr bytes.Buffer + code := runCLI([]string{ + verifyReportCommand, + reportFlag, path, + labelFlag, reportTestLabel, + expectedCountFlag, "3", + providerModeFlag, providerModeExternal, + }, &stdout, &stderr) + require.Equal(t, 0, code, stderr.String()) + require.Contains(t, stdout.String(), "3 specs passed") +} + +func TestCLIRunForwardsExternalModeIntoReportValidation(t *testing.T) { + cfg, tempDir := newSuiteRunFixture(t, reportTestLabel) + var stdout, stderr bytes.Buffer + var testCommandSeen bool + run := func(cmd *exec.Cmd) error { + if suiteCommandIsChown(cmd) { + return nil + } + testCommandSeen = true + return writeExternalSuiteReportFromCommand(cmd) + } + code := runSuiteCLIWithRunner([]string{ + labelFlag, cfg.label, + expectedCountFlag, "3", + providerModeFlag, providerModeExternal, + "--repository-dir", cfg.repositoryDir, + "--artifacts-dir", tempDir, + "--test-timeout", cfg.testTimeout, + }, &stdout, &stderr, run) + require.Equal(t, 0, code, stderr.String()) + require.True(t, testCommandSeen) + require.Contains(t, stdout.String(), "3 specs passed") +} diff --git a/hack/microsandbox/performance_tools.go b/hack/microsandbox/performance_tools.go new file mode 100644 index 0000000000..3072a063d5 --- /dev/null +++ b/hack/microsandbox/performance_tools.go @@ -0,0 +1,198 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/blang/semver/v4" +) + +const runtimeSDKModulePath = "github.com/devsy-org/devsy-runtime-sdk" + +var runtimeSDKTools = []string{ + "devsy-runtime-spawn-probe", + "devsy-runtime-supervisor", +} + +type runtimeSDKModule struct { + Path string `json:"Path"` + Version string `json:"Version"` + Replace *runtimeSDKReplacement `json:"Replace"` +} + +type runtimeSDKReplacement struct { + Path string `json:"Path"` + Version string `json:"Version"` +} + +func buildPerformanceToolsCLI(args []string, stdout, stderr io.Writer) int { + if runtime.GOOS != "linux" { + _, _ = fmt.Fprintln(stderr, "build-performance-tools requires Linux") + return 2 + } + flags := flag.NewFlagSet(buildPerformanceToolsCommand, flag.ContinueOnError) + flags.SetOutput(stderr) + outputDir := flags.String("output-dir", "", "absolute directory for SDK tools") + if err := flags.Parse(args); err != nil { + return 2 + } + if flags.NArg() != 0 || !filepath.IsAbs(*outputDir) { + _, _ = fmt.Fprintln( + stderr, + "build-performance-tools requires an absolute --output-dir and no positional arguments", + ) + return 2 + } + err := buildPerformanceTools( + context.Background(), *outputDir, commandRunner((*exec.Cmd).Run), + commandOutput{stdout: stdout, stderr: stderr}, + ) + if err != nil { + _, _ = fmt.Fprintf(stderr, "::error::%v\n", err) + return commandExitCode(err) + } + return 0 +} + +func buildPerformanceTools( + ctx context.Context, + outputDir string, + run commandRunner, + output commandOutput, +) error { + output = normalizedCommandOutput(output) + if !filepath.IsAbs(outputDir) { + return errors.New("performance tool output directory must be absolute") + } + if run == nil { + return errors.New("command runner is required") + } + metadataPath, err := preparePerformanceToolOutput(outputDir) + if err != nil { + return err + } + + environment := replaceCommandEnvironment(os.Environ(), map[string]string{ + "GOBIN": outputDir, "CGO_ENABLED": "0", + }) + version, err := runtimeSDKVersion(ctx, run, environment, output.stderr) + if err != nil { + return err + } + for _, tool := range runtimeSDKTools { + module := runtimeSDKModulePath + "/cmd/" + tool + "@" + version + cmd := exec.CommandContext(ctx, "go", "install", "-trimpath", module) + cmd.Env = environment + cmd.Stdout = output.stdout + cmd.Stderr = output.stderr + if err := run(cmd); err != nil { + return fmt.Errorf("build pinned SDK tool %s: %w", tool, err) + } + } + if err := writeRuntimeSDKVersion(metadataPath, version); err != nil { + return fmt.Errorf("write SDK version metadata: %w", err) + } + _, err = fmt.Fprintf( + output.stdout, + "built runtime SDK tools at %s for %s\n", + outputDir, + version, + ) + return err +} + +func preparePerformanceToolOutput(outputDir string) (string, error) { + if err := os.MkdirAll(outputDir, 0o755); err != nil { + return "", fmt.Errorf("create performance tool output directory: %w", err) + } + info, err := os.Stat(outputDir) + if err != nil { + return "", fmt.Errorf("inspect performance tool output directory: %w", err) + } + if !info.IsDir() { + return "", errors.New("performance tool output path must be a directory") + } + metadataPath := filepath.Join(outputDir, "sdk-version") + if err := os.Remove(metadataPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("remove stale SDK version metadata: %w", err) + } + return metadataPath, nil +} + +func runtimeSDKVersion( + ctx context.Context, + run commandRunner, + environment []string, + stderr io.Writer, +) (string, error) { + cmd := exec.CommandContext(ctx, "go", "list", "-m", "-json", runtimeSDKModulePath) + cmd.Env = environment + var stdout strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = stderr + if err := run(cmd); err != nil { + return "", fmt.Errorf("resolve pinned runtime SDK module: %w", err) + } + var module runtimeSDKModule + if err := json.Unmarshal([]byte(stdout.String()), &module); err != nil { + return "", fmt.Errorf("decode runtime SDK module metadata: %w", err) + } + _, versionErr := semver.Parse(strings.TrimPrefix(module.Version, "v")) + if module.Path != runtimeSDKModulePath || !strings.HasPrefix(module.Version, "v") || + versionErr != nil || module.Replace != nil { + return "", errors.New("runtime SDK must resolve to its pinned version without a replace") + } + return module.Version, nil +} + +func replaceCommandEnvironment(environment []string, replacements map[string]string) []string { + result := make([]string, 0, len(environment)+len(replacements)) + for _, entry := range environment { + key, _, ok := strings.Cut(entry, "=") + if _, replace := replacements[key]; !ok || !replace { + result = append(result, entry) + } + } + for key, value := range replacements { + result = append(result, key+"="+value) + } + return result +} + +func writeRuntimeSDKVersion(path, version string) (err error) { + file, err := os.CreateTemp(filepath.Dir(path), ".sdk-version-*") + if err != nil { + return err + } + tempPath := file.Name() + defer func() { + if removeErr := os.Remove( + tempPath, + ); removeErr != nil && + !errors.Is(removeErr, os.ErrNotExist) { + err = errors.Join(err, removeErr) + } + }() + if err = file.Chmod(0o644); err != nil { + return errors.Join(err, file.Close()) + } + if _, err = io.WriteString(file, version+"\n"); err != nil { + return errors.Join(err, file.Close()) + } + if err = file.Sync(); err != nil { + return errors.Join(err, file.Close()) + } + if err = file.Close(); err != nil { + return err + } + return os.Rename(tempPath, path) +} diff --git a/hack/microsandbox/performance_tools_test.go b/hack/microsandbox/performance_tools_test.go new file mode 100644 index 0000000000..c9851ca1b3 --- /dev/null +++ b/hack/microsandbox/performance_tools_test.go @@ -0,0 +1,214 @@ +package main + +import ( + "context" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" +) + +const installCommand = "install" + +type performanceToolsRunner struct { + moduleJSON string + failList bool + failInstallAt int + commands [][]string + environments [][]string +} + +func (runner *performanceToolsRunner) run(command *exec.Cmd) error { + runner.commands = append(runner.commands, append([]string(nil), command.Args[1:]...)) + runner.environments = append(runner.environments, append([]string(nil), command.Env...)) + if len(command.Args) > 1 && command.Args[1] == "list" { + if runner.failList { + return errors.New("module query failed") + } + _, err := io.WriteString(command.Stdout, runner.moduleJSON) + return err + } + installCount := countPerformanceToolInstalls(runner.commands) + if runner.failInstallAt == installCount { + return errors.New("tool build failed") + } + return nil +} + +func TestBuildPerformanceToolsCommandRejectsInvalidInvocation(t *testing.T) { + if code := runCLI( + []string{"build-performance-tools", "--output-dir", "relative"}, + io.Discard, + io.Discard, + ); code != 2 { + t.Fatalf("invalid tool build invocation exit = %d, want 2", code) + } +} + +func TestBuildPerformanceToolsCommandRequiresLinux(t *testing.T) { + if runtime.GOOS == "linux" { + t.Skip("command is available on Linux") + } + if code := buildPerformanceToolsCLI(nil, io.Discard, io.Discard); code != 2 { + t.Fatalf("non-Linux build command exit = %d, want 2", code) + } +} + +func TestBuildPerformanceToolsRejectsUnpinnedModuleBeforeInstall(t *testing.T) { + for _, test := range []struct { + name string + moduleJSON string + }{ + { + name: "missing version", + moduleJSON: `{"Path":"github.com/devsy-org/devsy-runtime-sdk"}`, + }, + { + name: "invalid version", + moduleJSON: `{"Path":"github.com/devsy-org/devsy-runtime-sdk","Version":"vnot-semver"}`, + }, + { + name: "incomplete semantic version", + moduleJSON: `{"Path":"github.com/devsy-org/devsy-runtime-sdk","Version":"v1"}`, + }, + { + name: "replacement", + moduleJSON: strings.Join([]string{ + `{"Path":"github.com/devsy-org/devsy-runtime-sdk",`, + `"Version":"v1.6.0","Replace":{"Path":"../runtime-sdk"}}`, + }, ""), + }, + { + name: "different module", + moduleJSON: `{"Path":"example.com/runtime-sdk","Version":"v1.6.0"}`, + }, + } { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + runner := &performanceToolsRunner{moduleJSON: test.moduleJSON} + err := buildPerformanceTools(context.Background(), dir, runner.run, commandOutput{}) + if err == nil { + t.Fatal("expected SDK module validation error") + } + if got := countPerformanceToolInstalls(runner.commands); got != 0 { + t.Fatalf("ran %d installs before rejecting module metadata", got) + } + assertNoSDKVersion(t, dir) + }) + } +} + +func TestBuildPerformanceToolsDoesNotWriteMetadataAfterFailure(t *testing.T) { + for _, failAt := range []int{1, 2} { + t.Run(strconv.Itoa(failAt), func(t *testing.T) { + dir := t.TempDir() + metadata := filepath.Join(dir, "sdk-version") + if err := os.WriteFile(metadata, []byte("stale\n"), 0o600); err != nil { + t.Fatal(err) + } + runner := &performanceToolsRunner{ + moduleJSON: pinnedSDKModuleJSON, + failInstallAt: failAt, + } + err := buildPerformanceTools(context.Background(), dir, runner.run, commandOutput{}) + if err == nil { + t.Fatal("expected tool build error") + } + assertNoSDKVersion(t, dir) + }) + } +} + +func TestBuildPerformanceToolsUsesPinnedToolsAndWritesVersionLast(t *testing.T) { + dir := t.TempDir() + oldPath := os.Getenv("PATH") + t.Setenv("PATH", oldPath) + t.Setenv("GOBIN", "/old/bin") + t.Setenv("CGO_ENABLED", "1") + runner := &performanceToolsRunner{moduleJSON: pinnedSDKModuleJSON} + if err := buildPerformanceTools( + context.Background(), + dir, + runner.run, + commandOutput{}, + ); err != nil { + t.Fatal(err) + } + assertPinnedToolInstalls(t, runner, dir, oldPath) + version, err := os.ReadFile(filepath.Join(dir, "sdk-version")) + if err != nil { + t.Fatal(err) + } + if string(version) != "v1.6.0\n" { + t.Fatalf("SDK version metadata = %q", version) + } +} + +func assertPinnedToolInstalls(t *testing.T, runner *performanceToolsRunner, dir, path string) { + t.Helper() + wantPackages := []string{ + "github.com/devsy-org/devsy-runtime-sdk/cmd/devsy-runtime-spawn-probe@v1.6.0", + "github.com/devsy-org/devsy-runtime-sdk/cmd/devsy-runtime-supervisor@v1.6.0", + } + if got := countPerformanceToolInstalls(runner.commands); got != len(wantPackages) { + t.Fatalf("install count = %d, want %d", got, len(wantPackages)) + } + for index, want := range wantPackages { + args := runner.commands[index+1] + if len(args) != 3 || args[0] != installCommand || args[1] != "-trimpath" || + args[2] != want { + t.Fatalf("install command = %#v, want go install -trimpath %s", args, want) + } + assertBuildEnvironment(t, runner.environments[index+1], dir, path) + } +} + +func TestBuildPerformanceToolsRejectsRelativeOutputDirectory(t *testing.T) { + runner := &performanceToolsRunner{moduleJSON: pinnedSDKModuleJSON} + err := buildPerformanceTools(context.Background(), "relative", runner.run, commandOutput{}) + if err == nil || !strings.Contains(err.Error(), "must be absolute") { + t.Fatalf("relative output dir error = %v", err) + } + if len(runner.commands) != 0 { + t.Fatalf("ran commands with invalid output dir: %#v", runner.commands) + } +} + +const pinnedSDKModuleJSON = `{"Path":"github.com/devsy-org/devsy-runtime-sdk","Version":"v1.6.0"}` + +func countPerformanceToolInstalls(commands [][]string) int { + count := 0 + for _, args := range commands { + if len(args) > 0 && args[0] == installCommand { + count++ + } + } + return count +} + +func assertBuildEnvironment(t *testing.T, environment []string, outputDir, path string) { + t.Helper() + values := make(map[string]string, len(environment)) + for _, entry := range environment { + key, value, found := strings.Cut(entry, "=") + if !found { + continue + } + values[key] = value + } + if values["GOBIN"] != outputDir || values["CGO_ENABLED"] != "0" || values["PATH"] != path { + t.Fatalf("unexpected build environment: %#v", values) + } +} + +func assertNoSDKVersion(t *testing.T, dir string) { + t.Helper() + if _, err := os.Stat(filepath.Join(dir, "sdk-version")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("sdk-version metadata exists after failed build: %v", err) + } +} diff --git a/hack/microsandbox/report.go b/hack/microsandbox/report.go index 3d6f62ab2d..0b0bcea898 100644 --- a/hack/microsandbox/report.go +++ b/hack/microsandbox/report.go @@ -32,9 +32,14 @@ type selectedGinkgoSpec struct { labels map[string]struct{} } -func validateReport(path, label string, expectedCount int) error { - if !isPositiveEven(expectedCount) { - return errors.New("expected-count must be a positive even number") +const ( + providerModeParity = "parity" + providerModeExternal = "external" +) + +func validateReport(path, label string, expectedCount int, mode string) error { + if !validProviderExpectation(mode, expectedCount) { + return fmt.Errorf("expected-count %d is invalid for provider-mode %q", expectedCount, mode) } specs, err := readSuiteSpecs(path) if err != nil { @@ -44,7 +49,18 @@ func validateReport(path, label string, expectedCount int) error { if err != nil { return err } - return validateSelectedSpecs(selected, expectedCount) + return validateSelectedSpecs(selected, expectedCount, mode) +} + +func validProviderExpectation(mode string, expectedCount int) bool { + switch mode { + case providerModeParity: + return isPositiveEven(expectedCount) + case providerModeExternal: + return expectedCount > 0 + default: + return false + } } func selectLabeledSpecs( @@ -154,7 +170,7 @@ func validateSuiteExecution(suite ginkgoSuiteReport) error { return nil } -func validateSelectedSpecs(selected []selectedGinkgoSpec, expectedCount int) error { +func validateSelectedSpecs(selected []selectedGinkgoSpec, expectedCount int, mode string) error { providers := [2]int{} for index, item := range selected { provider, err := validateSelectedSpec(item, index) @@ -163,6 +179,17 @@ func validateSelectedSpecs(selected []selectedGinkgoSpec, expectedCount int) err } providers[provider]++ } + if mode == providerModeExternal { + if providers[0] != 0 || providers[1] != expectedCount { + return fmt.Errorf( + "provider label counts (builtin=%d, external=%d), expected all %d external", + providers[0], + providers[1], + expectedCount, + ) + } + return nil + } if providers[0] != expectedCount/2 || providers[1] != expectedCount/2 { return fmt.Errorf( "provider label counts (builtin=%d, external=%d), expected %d each", diff --git a/hack/microsandbox/report_test.go b/hack/microsandbox/report_test.go index f01630a602..7c485404b9 100644 --- a/hack/microsandbox/report_test.go +++ b/hack/microsandbox/report_test.go @@ -9,8 +9,12 @@ import ( ) const ( + skippedState = "skipped" reportTestLabel = "up-provider-microsandbox-images" numAttemptsKey = "NumAttempts" + stateKey = "State" + leafNodeTypeKey = "LeafNodeType" + exactlyOneProviderErr = "exactly one provider label" realRunError = "real test run" focusError = "programmatic focus" builtinProviderLabel = "microsandbox-builtin" @@ -19,7 +23,7 @@ const ( func TestValidateReportAcceptsExactSuccessfulProviderMatrix(t *testing.T) { path := writeReportFixture(t, validReportFor(reportTestLabel)) - if err := validateReport(path, reportTestLabel, 2); err != nil { + if err := validateReport(path, reportTestLabel, 2, providerModeParity); err != nil { t.Fatal(err) } } @@ -28,9 +32,14 @@ func TestValidateReportAllowsUnselectedSkippedSpecs(t *testing.T) { report := validReportFor(reportTestLabel) suite := report[0].(map[string]any) specs := suite["SpecReports"].([]any) - specs = append(specs, map[string]any{"State": "skipped", numAttemptsKey: 0}) + specs = append(specs, map[string]any{stateKey: skippedState, numAttemptsKey: 0}) suite["SpecReports"] = specs - if err := validateReport(writeReportFixture(t, report), reportTestLabel, 2); err != nil { + if err := validateReport( + writeReportFixture(t, report), + reportTestLabel, + 2, + providerModeParity, + ); err != nil { t.Fatal(err) } } @@ -54,7 +63,7 @@ func TestValidateReportRejectsMissingMalformedAndWrongRoot(t *testing.T) { func assertReportErrorContains(t *testing.T, path, want string) { t.Helper() - err := validateReport(path, reportTestLabel, 2) + err := validateReport(path, reportTestLabel, 2, providerModeParity) if err == nil || !strings.Contains(err.Error(), want) { t.Fatalf("validateReport error = %v, want %q", err, want) } @@ -100,7 +109,12 @@ func TestValidateReportRequiresExplicitSuiteEvidence(t *testing.T) { t.Run(test.name, func(t *testing.T) { report := validReportFor(reportTestLabel) test.edit(report[0].(map[string]any)) - err := validateReport(writeReportFixture(t, report), reportTestLabel, 2) + err := validateReport( + writeReportFixture(t, report), + reportTestLabel, + 2, + providerModeParity, + ) if err == nil || !strings.Contains(err.Error(), test.want) { t.Fatalf("validateReport() error = %v, want %q", err, test.want) } @@ -115,7 +129,7 @@ func TestValidateReportRejectsMalformedSpecRecords(t *testing.T) { func(s map[string]any) { s["ContainerHierarchyLabels"] = []any{"not-a-label-list"} }, func(s map[string]any) { s["ContainerHierarchyLabels"] = []any{nil} }, func(s map[string]any) { s["ContainerHierarchyLabels"] = []any{[]any{"suite-label", nil}} }, - func(s map[string]any) { s["LeafNodeType"] = 7 }, + func(s map[string]any) { s[leafNodeTypeKey] = 7 }, } { report := validReportFor(reportTestLabel) mutate(report[0].(map[string]any)["SpecReports"].([]any)[0].(map[string]any)) @@ -129,6 +143,7 @@ func TestValidateReportRejectsMalformedSpecRecords(t *testing.T) { path, reportTestLabel, 2, + providerModeParity, ); err == nil || !strings.Contains(err.Error(), "malformed spec") { t.Fatalf("non-object spec error = %v", err) @@ -136,10 +151,10 @@ func TestValidateReportRejectsMalformedSpecRecords(t *testing.T) { } func TestValidateReportRejectsSkippedPendingAndFailedSelectedSpecs(t *testing.T) { - for _, state := range []string{"skipped", "pending", "failed"} { + for _, state := range []string{skippedState, "pending", "failed"} { t.Run(state, func(t *testing.T) { report := validReportFor(reportTestLabel) - report[0].(map[string]any)["SpecReports"].([]any)[0].(map[string]any)["State"] = state + report[0].(map[string]any)["SpecReports"].([]any)[0].(map[string]any)[stateKey] = state requireReportFailure(t, report, "not passed") }) } @@ -152,7 +167,7 @@ func TestValidateReportRejectsRetryAndNonItSelectedSpec(t *testing.T) { want string }{ "retry": {key: numAttemptsKey, value: 2, want: "expected one"}, - "not an It": {key: "LeafNodeType", value: "ReportEntry", want: "not an It spec"}, + "not an It": {key: leafNodeTypeKey, value: "ReportEntry", want: "not an It spec"}, } { t.Run(name, func(t *testing.T) { report := validReportFor(reportTestLabel) @@ -176,6 +191,7 @@ func TestValidateReportRejectsWrongLabelAndCount(t *testing.T) { path, test.label, test.count, + providerModeParity, ); err == nil || !strings.Contains(err.Error(), test.want) { t.Errorf("validateReport(%q, %d) = %v", test.label, test.count, err) @@ -189,11 +205,11 @@ func TestValidateReportRejectsProviderCoverageErrors(t *testing.T) { labels []string want string }{ - {"missing", nil, "exactly one provider label"}, + {"missing", nil, exactlyOneProviderErr}, { "ambiguous", []string{builtinProviderLabel, externalProviderLabel}, - "exactly one provider label", + exactlyOneProviderErr, }, {"unbalanced", []string{builtinProviderLabel}, "provider label counts"}, } @@ -208,7 +224,7 @@ func TestValidateReportRejectsProviderCoverageErrors(t *testing.T) { func requireReportFailure(t *testing.T, report any, want string) { t.Helper() - err := validateReport(writeReportFixture(t, report), reportTestLabel, 2) + err := validateReport(writeReportFixture(t, report), reportTestLabel, 2, providerModeParity) if err == nil || !strings.Contains(err.Error(), want) { t.Fatalf("validateReport error = %v, want %q", err, want) } @@ -220,13 +236,103 @@ func TestValidateReportRequiresPositiveEvenCount(t *testing.T) { "", reportTestLabel, count, + providerModeParity, ); err == nil || - !strings.Contains(err.Error(), "positive even") { + !strings.Contains(err.Error(), "invalid") { t.Errorf("expected count %d error = %v", count, err) } } } +func TestValidateReportAcceptsThreeExternalSpecs(t *testing.T) { + path := writeReportFixture(t, validExternalReportFor(reportTestLabel)) + if err := validateReport(path, reportTestLabel, 3, providerModeExternal); err != nil { + t.Fatal(err) + } +} + +func TestValidateReportExternalModeRejectsProviderLabelErrors(t *testing.T) { + cases := []struct { + name string + labels []string + want string + }{ + {"builtin", []string{builtinProviderLabel}, "expected all 3 external"}, + {"mixed", []string{builtinProviderLabel}, "expected all 3 external"}, + {"missing", nil, exactlyOneProviderErr}, + { + "ambiguous", + []string{builtinProviderLabel, externalProviderLabel}, + exactlyOneProviderErr, + }, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + report := validExternalReportFor(reportTestLabel) + specs := report[0].(map[string]any)["SpecReports"].([]any) + if test.name == "builtin" { + for _, raw := range specs { + raw.(map[string]any)["LeafNodeLabels"] = test.labels + } + } else { + specs[0].(map[string]any)["LeafNodeLabels"] = test.labels + } + err := validateReport( + writeReportFixture(t, report), + reportTestLabel, + 3, + providerModeExternal, + ) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("validateReport() error = %v, want %q", err, test.want) + } + }) + } +} + +func TestValidateReportExternalModeRejectsSkippedOrRetriedSpecs(t *testing.T) { + for _, test := range []struct { + name string + field string + value any + want string + }{ + {skippedState, stateKey, skippedState, "not passed"}, + {"retried", numAttemptsKey, 2, "expected one"}, + } { + t.Run(test.name, func(t *testing.T) { + report := validExternalReportFor(reportTestLabel) + selected := report[0].(map[string]any)["SpecReports"].([]any)[0].(map[string]any) + selected[test.field] = test.value + err := validateReport( + writeReportFixture(t, report), + reportTestLabel, + 3, + providerModeExternal, + ) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("validateReport() error = %v, want %q", err, test.want) + } + }) + } +} + +func TestValidateReportRejectsInvalidProviderModeAndCount(t *testing.T) { + path := writeReportFixture(t, validExternalReportFor(reportTestLabel)) + for _, test := range []struct { + mode string + count int + }{ + {providerModeParity, 3}, + {providerModeExternal, 0}, + {unknownProviderMode, 3}, + } { + if err := validateReport(path, reportTestLabel, test.count, test.mode); err == nil { + t.Errorf("validateReport(%q, %d) unexpectedly succeeded", test.mode, test.count) + } + } +} + func writeReportFixture(t *testing.T, report any) string { t.Helper() data, err := json.Marshal(report) @@ -247,8 +353,8 @@ func validReportFor(label string) []any { specs[index] = map[string]any{ "ContainerHierarchyLabels": [][]string{{label}}, "LeafNodeLabels": []string{providers[index%len(providers)]}, - "LeafNodeType": "It", - "State": "passed", + leafNodeTypeKey: "It", + stateKey: "passed", numAttemptsKey: 1, } } @@ -259,3 +365,20 @@ func validReportFor(label string) []any { "SpecReports": specs, }} } + +func validExternalReportFor(label string) []any { + report := validReportFor(label) + suite := report[0].(map[string]any) + specs := suite["SpecReports"].([]any) + specs = append(specs, map[string]any{ + "ContainerHierarchyLabels": [][]string{{label}}, + "LeafNodeLabels": []string{externalProviderLabel}, + leafNodeTypeKey: "It", + stateKey: "passed", + numAttemptsKey: 1, + }) + specs[0].(map[string]any)["LeafNodeLabels"] = []string{externalProviderLabel} + specs[1].(map[string]any)["LeafNodeLabels"] = []string{externalProviderLabel} + suite["SpecReports"] = specs + return report +} diff --git a/hack/microsandbox/run.go b/hack/microsandbox/run.go index 9dc5a9d342..9f863447a5 100644 --- a/hack/microsandbox/run.go +++ b/hack/microsandbox/run.go @@ -12,7 +12,26 @@ import ( "strconv" ) -const managedPolicyLabel = "up-provider-microsandbox-managed-recreate" +const ( + managedPolicyLabel = "up-provider-microsandbox-managed-recreate" + performanceSuiteLabel = "up-provider-microsandbox" +) + +var performanceEnvironmentNames = []string{ + "DEVSY_TEST_RUNTIME_SPAWN_PROBE", + "DEVSY_TEST_RUNTIME_SUPERVISOR", + "DEVSY_TEST_RUNTIME_PERFORMANCE_DIR", + "DEVSY_TEST_RUNTIME_REVISION", + "DEVSY_TEST_RUNTIME_SDK_VERSION", +} + +var requiredSuiteEnvironmentNames = []string{ + "DEVSY_REQUIRE_MICROSANDBOX", + "GH_USERNAME", + "GH_ACCESS_TOKEN", + "GH_CREDENTIAL_USERNAME", + "PATH", +} var suiteLabelPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9-]*$`) @@ -120,8 +139,12 @@ func validateSuiteConfig(cfg suiteConfig) error { if cfg.label == "" || !isValidSuiteLabel(cfg.label) { return fmt.Errorf("invalid Ginkgo label: %s", cfg.label) } - if !isPositiveEven(cfg.expectedCount) { - return errors.New("expected-spec-count must be a positive even integer") + if !validProviderExpectation(providerModeOrDefault(cfg.providerMode), cfg.expectedCount) { + return fmt.Errorf( + "expected-spec-count %d is invalid for provider-mode %q", + cfg.expectedCount, + providerModeOrDefault(cfg.providerMode), + ) } if !directoryExists(cfg.repositoryDir) { return errors.New("repository directory is required") @@ -139,6 +162,13 @@ func isPositiveEven(value int) bool { return value > 0 && value%2 == 0 } +func providerModeOrDefault(mode string) string { + if mode == "" { + return providerModeParity + } + return mode +} + func directoryExists(path string) bool { if path == "" { return false @@ -152,37 +182,61 @@ func isValidSuiteLabel(label string) bool { } func suiteSudoArguments(cfg suiteConfig) ([]string, error) { - names := []string{ - "DEVSY_REQUIRE_MICROSANDBOX", "GH_USERNAME", "GH_ACCESS_TOKEN", - "GH_CREDENTIAL_USERNAME", "PATH", + values, err := requiredSuiteEnvironment() + if err != nil { + return nil, err + } + kubeconfig, err := configuredKubeconfig() + if err != nil { + return nil, err + } + values = append(values, "KUBECONFIG="+kubeconfig) + if dockerHost := os.Getenv("DOCKER_HOST"); dockerHost != "" { + values = append(values, "DOCKER_HOST="+dockerHost) } - values := make([]string, 0, len(names)+3) - for _, name := range names { + return append(values, suiteSpecificEnvironment(cfg)...), nil +} + +func requiredSuiteEnvironment() ([]string, error) { + values := make([]string, 0, len(requiredSuiteEnvironmentNames)+3) + for _, name := range requiredSuiteEnvironmentNames { value, ok := os.LookupEnv(name) if !ok || value == "" { return nil, fmt.Errorf("%s must be set by the workflow", name) } values = append(values, name+"="+value) } + return values, nil +} + +func configuredKubeconfig() (string, error) { kubeconfig := os.Getenv("KUBECONFIG") if kubeconfig == "" { home := os.Getenv("HOME") if home == "" { - return nil, errors.New("HOME must be set to resolve the default KUBECONFIG") + return "", errors.New("HOME must be set to resolve the default KUBECONFIG") } kubeconfig = filepath.Join(home, ".kube", "config") } - values = append(values, "KUBECONFIG="+kubeconfig) - if dockerHost := os.Getenv("DOCKER_HOST"); dockerHost != "" { - values = append(values, "DOCKER_HOST="+dockerHost) - } + return kubeconfig, nil +} + +func suiteSpecificEnvironment(cfg suiteConfig) []string { + var values []string if cfg.label == managedPolicyLabel { values = append(values, "DEVSY_REQUIRE_MSB_MANAGED_POLICY_FIXTURE=true", "RUNNER_TEMP="+cfg.tempDir, ) } - return values, nil + if cfg.label == performanceSuiteLabel { + for _, name := range performanceEnvironmentNames { + if value := os.Getenv(name); value != "" { + values = append(values, name+"="+value) + } + } + } + return values } func validateSuiteReport( @@ -190,6 +244,10 @@ func validateSuiteReport( cfg suiteConfig, output commandOutput, ) int { + var performanceChownErr error + if cfg.label == performanceSuiteLabel { + performanceChownErr = chownPerformanceArtifact(run, cfg.tempDir, output) + } reportPath := filepath.Join(cfg.tempDir, "microsandbox-"+cfg.label+".json") uidGid := strconv.Itoa(os.Getuid()) + ":" + strconv.Itoa(os.Getgid()) if _, err := os.Stat(reportPath); err == nil { @@ -205,10 +263,18 @@ func validateSuiteReport( _, _ = fmt.Fprintf(output.stderr, "::error::could not inspect Ginkgo report: %v\n", err) return 1 } - if err := validateReport(reportPath, cfg.label, cfg.expectedCount); err != nil { + if err := validateReport( + reportPath, + cfg.label, + cfg.expectedCount, + providerModeOrDefault(cfg.providerMode), + ); err != nil { _, _ = fmt.Fprintf(output.stderr, "::error::%v\n", err) return 1 } + if performanceChownErr != nil { + return commandExitCode(performanceChownErr) + } if _, err := fmt.Fprintf( output.stdout, "%s: %d specs passed in one attempt each\n", @@ -220,6 +286,62 @@ func validateSuiteReport( return 0 } +func chownPerformanceArtifact(run commandRunner, tempDir string, output commandOutput) error { + directory := filepath.Join(tempDir, "microsandbox-performance") + info, err := os.Lstat(directory) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + _, _ = fmt.Fprintf( + output.stderr, + "::error::could not inspect performance artifact: %v\n", + err, + ) + return err + } + if !info.IsDir() { + err := errors.New("performance artifact directory is not a directory") + _, _ = fmt.Fprintf( + output.stderr, + "::error::could not inspect performance artifact: %v\n", + err, + ) + return err + } + path := filepath.Join(directory, "provider-performance.json") + info, err = os.Lstat(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + _, _ = fmt.Fprintf( + output.stderr, + "::error::could not inspect performance artifact: %v\n", + err, + ) + return err + } + if !info.Mode().IsRegular() { + err := errors.New("performance artifact is not a regular file") + _, _ = fmt.Fprintf( + output.stderr, + "::error::could not inspect performance artifact: %v\n", + err, + ) + return err + } + uidGid := strconv.Itoa(os.Getuid()) + ":" + strconv.Itoa(os.Getgid()) + if err := run(exec.Command("sudo", "chown", uidGid, path)); err != nil { + _, _ = fmt.Fprintln( + output.stderr, + "::error::could not make performance artifact readable for artifact upload", + ) + return err + } + return nil +} + type suiteOutputCapture struct { console io.Writer log io.Writer diff --git a/hack/microsandbox/run_test.go b/hack/microsandbox/run_test.go index 2b121b5442..9c8fa884d1 100644 --- a/hack/microsandbox/run_test.go +++ b/hack/microsandbox/run_test.go @@ -361,6 +361,164 @@ func writeSuiteReportFromCommand(cmd *exec.Cmd) error { return os.WriteFile(reportPath, data, 0o600) } +func writeExternalSuiteReportFromCommand(cmd *exec.Cmd) error { + var reportPath, label string + for _, arg := range cmd.Args { + if value, ok := strings.CutPrefix(arg, "-ginkgo.json-report="); ok { + reportPath = value + } + if value, ok := strings.CutPrefix(arg, "--ginkgo.label-filter="); ok { + label = value + } + } + if reportPath == "" || label == "" { + return errors.New("Ginkgo report or label argument missing") + } + data, err := json.Marshal(validExternalReportFor(label)) + if err != nil { + return err + } + return os.WriteFile(reportPath, data, 0o600) +} + +func TestRunSuitePerformanceEnvironmentAndArtifactOwnership(t *testing.T) { + cfg, tempDir := newSuiteRunFixture(t, performanceSuiteLabel) + for index, name := range performanceEnvironmentNames { + t.Setenv(name, "fixture-value-"+strconv.Itoa(index)) + } + performancePath := createPerformanceArtifact(t, tempDir) + var commands [][]string + run := func(cmd *exec.Cmd) error { + commands = append(commands, append([]string(nil), cmd.Args...)) + if suiteCommandIsChown(cmd) { + return nil + } + return writeSuiteReportFromCommand(cmd) + } + var stdout, stderr bytes.Buffer + if code := runSuite( + context.Background(), + cfg, + run, + commandOutput{stdout: &stdout, stderr: &stderr}, + ); code != 0 { + t.Fatalf("runSuite() = %d; stderr=%s", code, stderr.String()) + } + requirePerformanceEnvironment(t, commands[0]) + requirePerformanceChownTargets(t, commands, performancePath, filepath.Join( + tempDir, + "microsandbox-"+cfg.label+".json", + )) +} + +func TestRunSuitePerformanceChownFailurePreservesNativeExit(t *testing.T) { + cfg, tempDir := newSuiteRunFixture(t, performanceSuiteLabel) + performancePath := createPerformanceArtifact(t, tempDir) + var stderr bytes.Buffer + run := func(cmd *exec.Cmd) error { + if suiteCommandIsChown(cmd) { + if slices.Contains(cmd.Args, performancePath) { + return exitWithStatus(47) + } + return nil + } + if err := writeSuiteReportFromCommand(cmd); err != nil { + return err + } + return exitWithStatus(23) + } + code := runSuite(context.Background(), cfg, run, commandOutput{stderr: &stderr}) + if code != 23 { + t.Fatalf("runSuite() = %d, want native exit status 23; stderr=%s", code, stderr.String()) + } + if !strings.Contains(stderr.String(), "could not make performance artifact readable") { + t.Fatalf("performance chown failure was not reported: %s", stderr.String()) + } +} + +func TestRunSuiteChownsPerformanceArtifactWhenGinkgoReportIsMissing(t *testing.T) { + cfg, tempDir := newSuiteRunFixture(t, performanceSuiteLabel) + performancePath := createPerformanceArtifact(t, tempDir) + var chownTargets []string + var stderr bytes.Buffer + run := func(cmd *exec.Cmd) error { + if suiteCommandIsChown(cmd) { + chownTargets = append(chownTargets, cmd.Args[len(cmd.Args)-1]) + return nil + } + return exitWithStatus(23) + } + code := runSuite(context.Background(), cfg, run, commandOutput{stderr: &stderr}) + if code != 23 { + t.Fatalf("runSuite() = %d, want native exit status 23; stderr=%s", code, stderr.String()) + } + if !slices.Contains(chownTargets, performancePath) { + t.Fatalf("performance artifact was not chowned: %v", chownTargets) + } + if slices.Contains(chownTargets, filepath.Join(tempDir, "microsandbox-"+cfg.label+".json")) { + t.Fatalf("missing Ginkgo report was chowned: %v", chownTargets) + } + if !strings.Contains(stderr.String(), "cannot read Ginkgo JSON report") { + t.Fatalf("missing report error not retained: %s", stderr.String()) + } +} + +func createPerformanceArtifact(t *testing.T, tempDir string) string { + t.Helper() + path := filepath.Join(tempDir, "microsandbox-performance", "provider-performance.json") + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(`{"captured":true}`), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func requirePerformanceEnvironment(t *testing.T, args []string) { + t.Helper() + for index, name := range performanceEnvironmentNames { + want := name + "=fixture-value-" + strconv.Itoa(index) + if !slices.Contains(args, want) { + t.Errorf("performance invocation lacks %q: %v", want, args) + } + } +} + +func requirePerformanceChownTargets( + t *testing.T, + commands [][]string, + performancePath, reportPath string, +) { + t.Helper() + want := []string{performancePath, reportPath} + if len(commands) != len(want)+1 { + t.Fatalf("commands = %v, want test plus report and performance chown", commands) + } + for index, path := range want { + command := commands[index+1] + if !suiteCommandIsChownArgs(command) || command[len(command)-1] != path { + t.Errorf("chown command %d = %v, want target %q", index+1, command, path) + } + } +} + +func TestPerformanceEnvironmentIsNotForwardedToOtherSuites(t *testing.T) { + cfg, _ := newSuiteRunFixture(t, "up-provider-microsandbox-images") + for index, name := range performanceEnvironmentNames { + t.Setenv(name, "fixture-value-"+strconv.Itoa(index)) + } + args, err := suiteSudoArguments(cfg) + if err != nil { + t.Fatal(err) + } + for index, name := range performanceEnvironmentNames { + if slices.Contains(args, name+"=fixture-value-"+strconv.Itoa(index)) { + t.Errorf("performance environment %s was forwarded to unrelated suite", name) + } + } +} + func suiteCommandIsChown(cmd *exec.Cmd) bool { return suiteCommandIsChownArgs(cmd.Args) } diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx index 756fdac693..8c4f6873cb 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx @@ -53,7 +53,9 @@ For SDK usage and development commands, see the [SDK README](https://github.com/ ## MicroSandbox parity gate -MicroSandbox integration suites share a provider matrix and test-session setup. Their dedicated CI job defines the suite catalogue and common runtime installation once, while keeping each suite on an independent runner. Every native run must report the exact expected count, split equally between built-in and external provider cases, with no skips or retries. Dry-run and programmatic-focus reports cannot satisfy this gate. See [the test architecture guide](https://github.com/devsy-org/devsy/blob/main/e2e/tests/up/MICROSANDBOX.md) for fixture ownership and extension rules. +MicroSandbox integration suites share a provider matrix and test-session setup. The dedicated `integration-tests-microsandbox` job defines the suite catalogue and common runtime installation once, while keeping each suite on an independent Linux runner. A CI-only Go utility built from the tested checkout installs and checks the pinned MicroSandbox bundle, executes one labelled E2E suite and retains its log and Ginkgo JSON report; it is not a Devsy release artifact. The utility's `verify-report` command checks the selected label in that report. + +Each native run must report the exact expected count, with every selected case passed once and no selected skips or retries. The default `provider-mode: parity` requires equal built-in and external provider counts. An explicitly external-only catalogue row uses `provider-mode: external` and requires every selected case to carry only the external provider label. Dry-run and programmatic-focus reports cannot satisfy this gate. Prior reports prove only the labels and cases they contain; the earlier 74-case parity baseline does not prove the added three-case consumer contract or the expanded 77-case catalogue. See [the test architecture guide](https://github.com/devsy-org/devsy/blob/main/e2e/tests/up/MICROSANDBOX.md) for fixture ownership and extension rules. The `up-provider-microsandbox` E2E label runs the same lifecycle and ownership scenario against the built-in provider and the external v0.1.8 release, each with isolated Devsy configuration. CI pins MicroSandbox v0.7.7 by checksum and requires access to KVM; unavailable virtualization fails this job instead of producing a passing skipped test. Each provider scenario has a ten-minute deadline and the CI job has a 25-minute deadline. @@ -161,6 +163,14 @@ DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsand For built-in Exec, cancellation closes non-file `io.ReadCloser` input whose `Close` unblocks `Read`; ordinary completion and caller-owned file input retain their ownership. Agent binary delivery owns its acquired source and closes it when the individual transfer attempt is canceled, before retrying with a fresh source. Cleanup joins cancellation callbacks before returning. +The Linux-only `up-provider-microsandbox-registry-consumer` label selects exactly three cases from the published external provider's Runtime Protocol `RunImage` path. Its positive case uses an owned TLS registry, fixture CA, restricted CONNECT proxy and private Docker credential helper, verifies the provider executable digest and exact authenticated manifest/blob response bytes, then checks a guest marker. Independent missing-CA and missing-credential cases require concrete refusal and no created VM. Each case owns fresh image and runtime state. This suite does not prove ordinary `up` image metadata resolution, remote agent binary preparation, or general egress denial. Reports from the earlier 74-case parity baseline remain useful for those existing cases, but do not count as runs of this external-only row. + +```sh +DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox-registry-consumer +``` + +For the existing two-case lifecycle label, the external-provider case can capture ordinary fresh and reused `up` wall time and filtered runtime startup events when CI supplies all five explicit inputs: `DEVSY_TEST_RUNTIME_SPAWN_PROBE`, `DEVSY_TEST_RUNTIME_SUPERVISOR`, `DEVSY_TEST_RUNTIME_PERFORMANCE_DIR`, `DEVSY_TEST_RUNTIME_REVISION`, and `DEVSY_TEST_RUNTIME_SDK_VERSION`. The built-in case keeps its existing path. After both ordinary commands pass, the helper stages a checksum-verified copy of the published runtime and runs direct/supervised/supervised/direct Info/Find comparisons, each with a first launch and 100 warm samples. The JSON artifact records tool/runtime provenance and reaped processes. Startup events measure launch, handshake and dispense before RPC; they exclude binary resolution, RPC and reaping. These measurements use warm host caches, add no latency threshold or VM case, and do not claim cold-cache performance or total plugin overhead. + These scenarios do not establish complete parity. Compatibility shims verify preservation under injected version faults but do not establish behavior against an actual historical MicroSandbox release. Green lifecycle, image, mount, resource, persistent-storage, ephemeral-root, TCP-egress, prebuild, Dockerless, request-invalid and managed-policy recreation, logs, and cancellation checks alone do not authorize that cutover. Dockerless refusal checks cover fresh creation, not preservation after an invalid recreation request. Egress coverage does not establish UDP, DNS, TLS, ingress, or live-policy modification behavior. Prebuild invalidation coverage changes the Dockerfile; it does not independently establish hashing of local Feature script contents.