From 5d21c0afc299a88201ef834a9afae3c7b005e945 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 13 Oct 2026 14:46:20 -0600 Subject: [PATCH 1/8] test(runtime): verify published MicroSandbox registry consumption --- .github/workflows/pr-ci.yml | 19 +- e2e/credential_helper_linux_test.go | 9 + e2e/credential_helper_other_test.go | 7 + e2e/e2e_suite_test.go | 3 + .../registryfixture/credential_linux.go | 517 ++++++++++++++ e2e/framework/registryfixture/server_linux.go | 476 +++++++++++++ .../registryfixture/server_linux_test.go | 648 ++++++++++++++++++ .../registryfixture/testmain_linux_test.go | 15 + e2e/tests/up/MICROSANDBOX.md | 34 +- .../provider_microsandbox_consumer_linux.go | 585 ++++++++++++++++ hack/microsandbox/test_verify_report.py | 60 +- hack/microsandbox/verify_report.py | 33 +- .../developing-providers/runtime-protocol.mdx | 8 +- 13 files changed, 2397 insertions(+), 17 deletions(-) create mode 100644 e2e/credential_helper_linux_test.go create mode 100644 e2e/credential_helper_other_test.go create mode 100644 e2e/framework/registryfixture/credential_linux.go create mode 100644 e2e/framework/registryfixture/server_linux.go create mode 100644 e2e/framework/registryfixture/server_linux_test.go create mode 100644 e2e/framework/registryfixture/testmain_linux_test.go create mode 100644 e2e/tests/up/provider_microsandbox_consumer_linux.go diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 883ae64237..889d7a2a73 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -358,6 +358,9 @@ jobs: expected-spec-count: 4 - label: up-provider-microsandbox-logs-cancel expected-spec-count: 6 + - label: up-provider-microsandbox-registry-consumer + expected-spec-count: 3 + provider-mode: external runs-on: ubuntu-latest timeout-minutes: 25 steps: @@ -393,6 +396,19 @@ jobs: - name: test Ginkgo report validator run: python3 -m unittest discover -s hack/microsandbox -p 'test_*.py' + - name: setup Go for registry fixture tests + if: matrix.label == 'up-provider-microsandbox-registry-consumer' + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: go.mod + cache: false + + - name: test Linux registry fixture + if: matrix.label == 'up-provider-microsandbox-registry-consumer' + env: + GOTOOLCHAIN: local + run: go test -race -count=1 -timeout 90s ./e2e/framework/registryfixture + - name: cache microsandbox release bundle (Linux) uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -468,7 +484,8 @@ jobs: else python3 ../hack/microsandbox/verify_report.py \ --report "$report" --label "${{ matrix.label }}" \ - --expected-count "${{ matrix.expected-spec-count }}" || report_status=$? + --expected-count "${{ matrix.expected-spec-count }}" \ + --provider-mode "${{ matrix.provider-mode || 'parity' }}" || report_status=$? fi if [ "$test_status" -ne 0 ]; then exit "$test_status" diff --git a/e2e/credential_helper_linux_test.go b/e2e/credential_helper_linux_test.go new file mode 100644 index 0000000000..00643edb90 --- /dev/null +++ b/e2e/credential_helper_linux_test.go @@ -0,0 +1,9 @@ +//go:build linux + +package e2e + +import "github.com/devsy-org/devsy/e2e/framework/registryfixture" + +func dispatchFixtureCredentialHelper() (bool, int) { + return registryfixture.DispatchCredentialHelper() +} diff --git a/e2e/credential_helper_other_test.go b/e2e/credential_helper_other_test.go new file mode 100644 index 0000000000..90fe4ab4bc --- /dev/null +++ b/e2e/credential_helper_other_test.go @@ -0,0 +1,7 @@ +//go:build !linux + +package e2e + +func dispatchFixtureCredentialHelper() (bool, int) { + return false, 0 +} diff --git a/e2e/e2e_suite_test.go b/e2e/e2e_suite_test.go index abe2f0af1c..09878ce04c 100644 --- a/e2e/e2e_suite_test.go +++ b/e2e/e2e_suite_test.go @@ -50,6 +50,9 @@ func TestMain(m *testing.M) { cmd.Execute() return } + if handled, code := dispatchFixtureCredentialHelper(); handled { + os.Exit(code) + } os.Exit(m.Run()) } diff --git a/e2e/framework/registryfixture/credential_linux.go b/e2e/framework/registryfixture/credential_linux.go new file mode 100644 index 0000000000..9eb8e490dd --- /dev/null +++ b/e2e/framework/registryfixture/credential_linux.go @@ -0,0 +1,517 @@ +//go:build linux + +package registryfixture + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "syscall" +) + +const ( + // #nosec G101 -- This is an environment variable name, not a credential value. + CredentialConfigEnv = "CASE_A_CREDENTIAL_CONFIG" + // #nosec G101 -- This is an environment variable name, not a credential value. + CredentialLogEnv = "CASE_A_CREDENTIAL_OBSERVATIONS" + // #nosec G101 -- This is an environment variable name, not a credential value. + CredentialRootEnv = "CASE_A_CREDENTIAL_ROOT" + CredentialOperationEnv = "CASE_A_OPERATION_ID" + HelperBinaryName = "docker-credential-fixture" +) + +// CredentialConfig fields are stored only in the private fixture root. +type CredentialConfig struct { + OperationID string `json:"operation_id"` + ServerKey string `json:"server_key"` + Username string `json:"username,omitempty"` + Secret string `json:"secret,omitempty"` + CredentialsAvailable bool `json:"credentials_available"` + ExpectedProviderSHA256 string `json:"expected_provider_sha256"` +} + +// CredentialObservation contains secret-free helper invocation evidence. +type CredentialObservation struct { + OperationID string `json:"operation_id"` + ServerKey string `json:"server_key"` + Action string `json:"action"` + Parent ProcessIdentity `json:"parent"` + Outcome string `json:"outcome"` +} + +// ProcessIdentity records a live process identity without cmdline or env. +type ProcessIdentity struct { + PID int `json:"pid"` + StartTimeTicks uint64 `json:"start_time_ticks"` + Executable string `json:"executable"` + SHA256 string `json:"sha256"` +} + +const credentialsNotFoundMessage = "credentials not found in native keychain" + +var ( + errCredentialRequestRead = errors.New("credential request could not be read") + errCredentialRequestKey = errors.New("credential request did not name the owned registry") +) + +type credentialSession struct { + root, log string + config CredentialConfig +} + +// CredentialHelperMain implements Docker's credential-helper `get` wire +// protocol. The input is recorded only as an owned server key and parent PID; +// the secret is read from a private fixture file and never logged. +func CredentialHelperMain(args []string, stdin io.Reader, stdout, stderr io.Writer) int { + if !validCredentialHelperArgs(args) { + _, _ = fmt.Fprintln(stderr, "fixture credential helper supports get only") + return 1 + } + if !credentialEnvironmentConfigured() { + _, _ = fmt.Fprintln(stderr, "fixture credential helper is not configured") + return 1 + } + session, err := loadCredentialConfig() + if err != nil { + _, _ = fmt.Fprintln(stderr, "fixture credential configuration is invalid") + return 1 + } + outcome, ok := session.observeRequest(stdin, stderr) + if !ok { + return 1 + } + if outcome == "parent_hash_mismatch" { + _, _ = fmt.Fprintln(stderr, "credential helper parent is not the expected provider") + return 1 + } + if !session.config.CredentialsAvailable { + _, _ = fmt.Fprintln(stdout, credentialsNotFoundMessage) + return 1 + } + if err := writeCredentials(stdout, session.config); err != nil { + return 1 + } + return 0 +} + +func validCredentialHelperArgs(args []string) bool { + return len(args) == 1 && args[0] == "get" +} + +func (s credentialSession) observeRequest(stdin io.Reader, stderr io.Writer) (string, bool) { + serverKey, err := readCredentialRequest(stdin, s.config.ServerKey) + if err != nil { + if errors.Is(err, errCredentialRequestRead) { + _, _ = fmt.Fprintln(stderr, "fixture credential request could not be read") + } else { + _, _ = fmt.Fprintln(stderr, "credential request did not name the owned registry") + } + return "", false + } + parent, err := captureParentIdentity(os.Getppid()) + if err != nil { + _, _ = fmt.Fprintln(stderr, "credential helper could not verify its live Linux parent") + return "", false + } + outcome := credentialOutcome(parent.SHA256, s.config) + if err := s.recordObservation(serverKey, parent, outcome); err != nil { + _, _ = fmt.Fprintln(stderr, "fixture observation could not be recorded") + return "", false + } + return outcome, true +} + +func credentialEnvironmentConfigured() bool { + return os.Getenv(CredentialRootEnv) != "" && os.Getenv(CredentialConfigEnv) != "" && + os.Getenv(CredentialLogEnv) != "" && os.Getenv(CredentialOperationEnv) != "" +} + +func loadCredentialConfig() (credentialSession, error) { + root, configPath := os.Getenv(CredentialRootEnv), os.Getenv(CredentialConfigEnv) + logPath, operationID := os.Getenv(CredentialLogEnv), os.Getenv(CredentialOperationEnv) + if root == "" || configPath == "" || logPath == "" || operationID == "" { + return credentialSession{}, errors.New("credential helper is not configured") + } + cfg, err := ReadCredentialConfig(root, configPath) + if err != nil { + return credentialSession{}, err + } + if err := validateCredentialConfig(cfg, operationID); err != nil { + return credentialSession{}, errors.New("credential configuration is invalid") + } + return credentialSession{root: root, log: logPath, config: cfg}, nil +} + +func validateCredentialConfig(cfg CredentialConfig, operationID string) error { + if cfg.OperationID == "" || cfg.OperationID != operationID || cfg.ServerKey == "" || + !validSHA256(cfg.ExpectedProviderSHA256) { + return errors.New("credential configuration is invalid") + } + if cfg.CredentialsAvailable && (cfg.Username == "" || cfg.Secret == "") { + return errors.New("available fixture credentials are incomplete") + } + return nil +} + +func readCredentialRequest(stdin io.Reader, expected string) (string, error) { + serverKey, err := bufio.NewReader(stdin).ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return "", fmt.Errorf("%w: %w", errCredentialRequestRead, err) + } + serverKey = strings.TrimSpace(serverKey) + if serverKey != expected { + return "", errCredentialRequestKey + } + return serverKey, nil +} + +func credentialOutcome(parentHash string, cfg CredentialConfig) string { + switch { + case !equalSHA256(parentHash, cfg.ExpectedProviderSHA256): + return "parent_hash_mismatch" + case cfg.CredentialsAvailable: + return "credentials_provided" + default: + return "credentials_not_found" + } +} + +func (s credentialSession) recordObservation( + serverKey string, + parent ProcessIdentity, + outcome string, +) error { + line, err := json.Marshal(CredentialObservation{ + OperationID: s.config.OperationID, + ServerKey: serverKey, + Action: "get", + Parent: parent, + Outcome: outcome, + }) + if err != nil { + return err + } + return appendOwnedLine(s.root, s.log, append(line, '\n')) +} + +func writeCredentials(stdout io.Writer, cfg CredentialConfig) error { + result, err := json.Marshal(map[string]string{"Username": cfg.Username, "Secret": cfg.Secret}) + if err != nil { + return err + } + _, err = stdout.Write(append(result, '\n')) + return err +} + +func validSHA256(value string) bool { + if len(value) != sha256.Size*2 { + return false + } + _, err := hex.DecodeString(value) + return err == nil +} + +func equalSHA256(a, b string) bool { return strings.EqualFold(a, b) } + +// captureParentIdentity binds an observation to the live direct parent without +// reading its command line or environment, which may contain credentials. +func captureParentIdentity(pid int) (ProcessIdentity, error) { + if runtime.GOOS != "linux" || pid <= 0 { + return ProcessIdentity{}, errors.New("linux /proc identity is required") + } + proc := filepath.Join("/proc", fmt.Sprint(pid)) + startBefore, err := processStartTime(filepath.Join(proc, "stat")) + if err != nil { + return ProcessIdentity{}, err + } + executable, err := os.Readlink(filepath.Join(proc, "exe")) + if err != nil { + return ProcessIdentity{}, err + } + digest, err := hashFile(filepath.Join(proc, "exe")) + if err != nil { + return ProcessIdentity{}, err + } + startAfter, err := processStartTime(filepath.Join(proc, "stat")) + if err != nil { + return ProcessIdentity{}, err + } + if startBefore != startAfter { + return ProcessIdentity{}, errors.New("parent process changed during identity capture") + } + return ProcessIdentity{ + PID: pid, StartTimeTicks: startBefore, Executable: executable, SHA256: digest, + }, nil +} + +func hashFile(path string) (string, error) { + // Parent hashing receives only /proc//exe. + //nolint:gosec // fixed procfs executable entry for the verified parent PID + file, err := os.Open(path) + if err != nil { + return "", err + } + h := sha256.New() + _, copyErr := io.Copy(h, file) + closeErr := file.Close() + if err := errors.Join(copyErr, closeErr); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func processStartTime(path string) (uint64, error) { + // The caller supplies only /proc//stat. + //nolint:gosec // fixed procfs stat entry for the verified parent PID + b, err := os.ReadFile(path) + if err != nil { + return 0, err + } + // /proc//stat field 2 is parenthesized and may contain spaces or ')'. + end := strings.LastIndexByte(string(b), ')') + if end < 0 || end+1 >= len(b) { + return 0, errors.New("invalid Linux process stat") + } + fields := strings.Fields(string(b[end+1:])) // starts at field 3 (state) + if len(fields) <= 19 { // field 22 is index 19 from field 3 + return 0, errors.New("linux process stat lacks start time") + } + var start uint64 + if _, err := fmt.Sscan(fields[19], &start); err != nil || start == 0 { + return 0, errors.New("invalid Linux process start time") + } + return start, nil +} + +func ownedChild(root, path string) (string, error) { + if err := validatePrivateRoot(root); err != nil { + return "", err + } + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return "", errors.New("owned file path must be absolute and clean") + } + if filepath.Dir(path) != root || filepath.Base(path) == "." || + filepath.Base(path) == string(filepath.Separator) { + return "", errors.New("owned file must be a direct child of its private root") + } + return path, nil +} + +func validatePrivateRoot(root string) error { + if !filepath.IsAbs(root) || filepath.Clean(root) != root { + return errors.New("owned root must be absolute and clean") + } + resolved, err := filepath.EvalSymlinks(root) + if err != nil || resolved != root { + return errors.New("owned root must be canonical and symlink-free") + } + return nil +} + +func openOwnedFile(root, path string, flags int, create bool) (file *os.File, resultErr error) { + defer closeFileOnError(&file, &resultErr) + if _, err := ownedChild(root, path); err != nil { + return nil, err + } + openedRoot, err := os.OpenRoot(root) + if err != nil { + return nil, err + } + defer func() { resultErr = errors.Join(resultErr, openedRoot.Close()) }() + rootFile, err := openedRoot.Open(".") + if err != nil { + return nil, errors.New("owned root must be a mode-0700 directory") + } + defer func() { resultErr = errors.Join(resultErr, rootFile.Close()) }() + if err := validateOpenedRoot(rootFile); err != nil { + return nil, err + } + flags = ownedFileOpenFlags(flags, create) + file, err = openedRoot.OpenFile(filepath.Base(path), flags, 0o600) + if err != nil { + return nil, err + } + if err := validateOpenedFile(file); err != nil { + return nil, errors.Join(err, file.Close()) + } + return file, nil +} + +func closeFileOnError(file **os.File, resultErr *error) { + if *resultErr == nil || *file == nil { + return + } + *resultErr = errors.Join(*resultErr, (*file).Close()) + *file = nil +} + +func ownedFileOpenFlags(flags int, create bool) int { + flags |= syscall.O_NOFOLLOW | syscall.O_CLOEXEC + if create { + flags |= syscall.O_CREAT | syscall.O_EXCL + } + return flags +} + +func validateOpenedRoot(root *os.File) error { + info, err := root.Stat() + if err != nil || !info.IsDir() || info.Mode().Perm() != 0o700 { + return errors.New("owned root must be a mode-0700 directory") + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || int(stat.Uid) != os.Geteuid() { + return errors.New("owned root has an unexpected owner") + } + return nil +} + +func validateOpenedFile(file *os.File) error { + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 { + return errors.New("owned file must be a mode-0600 regular file") + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || int(stat.Uid) != os.Geteuid() { + return errors.New("owned file has an unexpected owner") + } + return nil +} + +// ReadCredentialConfig reads a validated private fixture config. The path must +// be a direct child of a canonical, current-user-owned mode-0700 root. +func ReadCredentialConfig(root, path string) (CredentialConfig, error) { + f, err := openOwnedFile(root, path, syscall.O_RDONLY, false) + if err != nil { + return CredentialConfig{}, err + } + var cfg CredentialConfig + err = json.NewDecoder(io.LimitReader(f, 64*1024)).Decode(&cfg) + return cfg, errors.Join(err, f.Close()) +} + +// IsCredentialHelperInvocation recognizes only the fixed helper path under +// the configured private root, when it resolves to this running test binary. +func IsCredentialHelperInvocation(argv0 string) bool { + root := os.Getenv(CredentialRootEnv) + if root == "" { + return false + } + path, ok := helperInvocationPath(argv0) + if !ok || validatePrivateRoot(root) != nil { + return false + } + helperPath := filepath.Join(root, "bin", HelperBinaryName) + if path != helperPath || !privateDirectory(root) || + !privateDirectory(filepath.Dir(helperPath)) { + return false + } + return sameAsCurrentExecutable(helperPath) +} + +func helperInvocationPath(argv0 string) (string, bool) { + path := argv0 + if !filepath.IsAbs(argv0) { + if argv0 != HelperBinaryName { + return "", false + } + var err error + path, err = exec.LookPath(argv0) + if err != nil { + return "", false + } + } + return path, filepath.IsAbs(path) && filepath.Clean(path) == path +} + +func sameAsCurrentExecutable(path string) bool { + // path is constructed from the validated private fixture root and fixed name. + //nolint:gosec // the helper path is constrained by IsCredentialHelperInvocation + linkInfo, err := os.Stat(path) + if err != nil { + return false + } + executable, err := os.Executable() + if err != nil { + return false + } + executableInfo, err := os.Stat(executable) + return err == nil && os.SameFile(linkInfo, executableInfo) +} + +// DispatchCredentialHelper runs the helper only for the fixed private-root +// path when it resolves to the current executable. handled is false for the +// normal test-binary invocation. +func DispatchCredentialHelper() (handled bool, exitCode int) { + if !IsCredentialHelperInvocation(os.Args[0]) { + return false, 0 + } + return true, CredentialHelperMain(os.Args[1:], os.Stdin, os.Stdout, os.Stderr) +} + +func privateDirectory(path string) bool { + if !validPrivateDirectoryPath(path) { + return false + } + root, err := os.OpenRoot(path) + if err != nil { + return false + } + rootFile, err := root.Open(".") + if err != nil { + _ = root.Close() + return false + } + valid := privateDirectoryOwner(rootFile) + fileCloseErr := rootFile.Close() + rootCloseErr := root.Close() + return valid && fileCloseErr == nil && rootCloseErr == nil +} + +func validPrivateDirectoryPath(path string) bool { + resolved, err := filepath.EvalSymlinks(path) + if err != nil || resolved != path { + return false + } + // path is either the validated fixture root or its fixed bin child. + //nolint:gosec // constrained to the private root/bin pair by the caller + info, err := os.Lstat(path) + return err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 && + info.Mode().Perm() == 0o700 +} + +func privateDirectoryOwner(rootFile *os.File) bool { + info, err := rootFile.Stat() + if err != nil || !info.IsDir() || info.Mode().Perm() != 0o700 { + return false + } + stat, ok := info.Sys().(*syscall.Stat_t) + return ok && int(stat.Uid) == os.Geteuid() +} + +func appendOwnedLine(root, path string, line []byte) error { + f, err := openOwnedFile(root, path, syscall.O_WRONLY|syscall.O_APPEND, false) + if errors.Is(err, os.ErrNotExist) { + f, err = openOwnedFile(root, path, syscall.O_WRONLY|syscall.O_APPEND, true) + if errors.Is(err, os.ErrExist) { + f, err = openOwnedFile(root, path, syscall.O_WRONLY|syscall.O_APPEND, false) + } + } + if err != nil { + return err + } + if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + return errors.Join(err, f.Close()) + } + _, writeErr := f.Write(line) + unlockErr := syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + return errors.Join(writeErr, unlockErr, f.Close()) +} diff --git a/e2e/framework/registryfixture/server_linux.go b/e2e/framework/registryfixture/server_linux.go new file mode 100644 index 0000000000..9093ae3bad --- /dev/null +++ b/e2e/framework/registryfixture/server_linux.go @@ -0,0 +1,476 @@ +//go:build linux + +// Package registryfixture owns the bounded Linux registry/proxy and helper +// fixtures used by native provider-consumer tests. +package registryfixture + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "hash" + "io" + "math/big" + "net" + "net/http" + "strings" + "sync" + "time" +) + +const RegistryHost = "registry.invalid" + +type Config struct { + Username string + Password string +} + +type Request struct { + Method string `json:"method"` + Path string `json:"path"` // normalized route class, never the raw URL + Status int `json:"status"` + Authenticated bool `json:"authenticated"` + Bytes int64 `json:"bytes"` + BodyDigest string `json:"body_digest,omitempty"` + Digest string `json:"docker_content_digest,omitempty"` +} + +type Snapshot struct { + ConnectAllowed int `json:"connect_allowed"` + ConnectDenied int `json:"connect_denied"` + Authenticated int `json:"authenticated_requests"` + AuthRejected int `json:"auth_rejected_requests"` + Completed int `json:"completed_responses"` + ManifestGets int `json:"manifest_gets"` + BlobGets int `json:"blob_gets"` + ResponseBytes int64 `json:"response_bytes"` + Requests []Request `json:"requests"` +} + +// Server owns one TLS registry listener and one CONNECT listener. The proxy +// accepts only the exact registry authority and dials the owned listener IP; +// it never resolves or forwards caller-selected destinations. +type Server struct { + Authority string + ProxyURL string + CAPEM []byte + Username string + Password string + + registry *http.Server + regLn net.Listener + proxy *http.Server + proxyLn net.Listener + mu sync.Mutex + stats Snapshot + close sync.Once + closeErr error + tunnelMu sync.Mutex + closing bool + active int + idle chan struct{} + tunnels map[net.Conn]struct{} +} + +func Start(next http.Handler, cfg Config) (*Server, error) { + if next == nil || cfg.Username == "" || cfg.Password == "" { + return nil, fmt.Errorf("registry handler and fixture credentials are required") + } + regLn, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, fmt.Errorf("listen owned registry: %w", err) + } + cert, caPEM, err := makeCertificate() + if err != nil { + return nil, errors.Join(err, closeListener(regLn)) + } + s := &Server{ + Authority: RegistryHost + ":" + fmt.Sprint(regLn.Addr().(*net.TCPAddr).Port), + CAPEM: caPEM, Username: cfg.Username, Password: cfg.Password, regLn: regLn, + tunnels: make(map[net.Conn]struct{}), idle: make(chan struct{}), + } + close(s.idle) + regTLS := tls.NewListener(regLn, &tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{cert}, + }) + s.registry = &http.Server{Handler: s.authenticated(next), ReadHeaderTimeout: 5 * time.Second} + s.tunnelMu.Lock() + s.trackStartLocked() + s.tunnelMu.Unlock() + go func() { defer s.trackDone(); _ = s.registry.Serve(regTLS) }() + + proxyLn, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + s.tunnelMu.Lock() + s.closing = true + s.tunnelMu.Unlock() + registryErr := s.registry.Close() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + return nil, errors.Join( + fmt.Errorf("listen owned proxy: %w", err), + registryErr, + s.waitIdle(ctx), + closeListener(regLn), + ) + } + s.proxyLn = proxyLn + s.ProxyURL = "http://" + proxyLn.Addr().String() + s.proxy = &http.Server{Handler: http.HandlerFunc(s.connect), ReadHeaderTimeout: 5 * time.Second} + s.tunnelMu.Lock() + s.trackStartLocked() + s.tunnelMu.Unlock() + go func() { defer s.trackDone(); _ = s.proxy.Serve(proxyLn) }() + return s, nil +} + +func isSuccessfulGet(r *http.Request, status int, path, expectedPath string) bool { + return r.Method == http.MethodGet && status >= 200 && status < 300 && path == expectedPath +} + +func routeClass(path string) string { + switch { + case path == "/v2/": + return "v2" + case strings.Contains(path, "/manifests/"): + return "manifests" + case strings.Contains(path, "/blobs/uploads/"): + return "uploads" + case strings.Contains(path, "/blobs/"): + return "blobs" + default: + return "other" + } +} + +type countWriter struct { + http.ResponseWriter + status int + bytes int64 + digest hash.Hash +} + +func (w *countWriter) WriteHeader(code int) { + if w.status != 0 { + return + } + w.status = code + w.ResponseWriter.WriteHeader(code) +} + +func (w *countWriter) Write(p []byte) (int, error) { + if w.status == 0 { + // Match net/http's implicit status before forwarding the first body write. + // A later WriteHeader is ignored by the underlying writer and must be + // ignored by our observation too. + w.WriteHeader(http.StatusOK) + } + n, e := w.ResponseWriter.Write(p) + w.bytes += int64(n) + if n > 0 && w.digest != nil { + _, _ = w.digest.Write(p[:n]) + } + return n, e +} + +func (w *countWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } + +func (w *countWriter) Flush() { + if w.status == 0 { + w.WriteHeader(http.StatusOK) + } + if f, ok := w.ResponseWriter.(http.Flusher); ok { + f.Flush() + } +} + +// hijackConnect is split out to keep CONNECT tunneling explicit and small. +func hijackConnect(w http.ResponseWriter) (net.Conn, error) { + h, ok := w.(http.Hijacker) + if !ok { + return nil, fmt.Errorf("HTTP server does not support hijacking") + } + c, rw, err := h.Hijack() + if err != nil { + return nil, err + } + if _, err = rw.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil { + _ = c.Close() + return nil, err + } + if err = rw.Flush(); err != nil { + _ = c.Close() + return nil, err + } + return c, nil +} + +func (s *Server) Snapshot() Snapshot { + s.mu.Lock() + defer s.mu.Unlock() + out := s.stats + out.Requests = append([]Request(nil), s.stats.Requests...) + return out +} + +// Close shuts down both listeners and joins servers and active tunnels. +func (s *Server) Close(ctx context.Context) error { + s.close.Do(func() { + s.tunnelMu.Lock() + s.closing = true + owned := make([]net.Conn, 0, len(s.tunnels)) + for conn := range s.tunnels { + owned = append(owned, conn) + } + idle := s.idle + s.tunnelMu.Unlock() + + // Closing registered tunnels first releases both copy directions; the + // close fence prevents any later CONNECT from registering new work. + var tunnelCloseErr error + for _, conn := range owned { + if err := conn.Close(); err != nil && !errors.Is(err, net.ErrClosed) { + tunnelCloseErr = errors.Join(tunnelCloseErr, err) + } + } + s.closeErr = errors.Join(tunnelCloseErr, + s.registry.Shutdown(ctx), s.proxy.Shutdown(ctx), + s.registry.Close(), s.proxy.Close(), closeListener(s.regLn), closeListener(s.proxyLn), + ) + select { + case <-idle: + case <-ctx.Done(): + s.closeErr = errors.Join(s.closeErr, ctx.Err()) + } + }) + return s.closeErr +} + +func (s *Server) trackStartLocked() { + if s.active == 0 { + s.idle = make(chan struct{}) + } + s.active++ +} + +func (s *Server) trackDone() { + s.tunnelMu.Lock() + s.active-- + if s.active == 0 { + close(s.idle) + } + s.tunnelMu.Unlock() +} + +func (s *Server) waitIdle(ctx context.Context) error { + s.tunnelMu.Lock() + idle := s.idle + s.tunnelMu.Unlock() + select { + case <-idle: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func closeListener(ln net.Listener) error { + err := ln.Close() + if errors.Is(err, net.ErrClosed) { + return nil + } + return err +} + +func makeCertificate() (tls.Certificate, []byte, error) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return tls.Certificate{}, nil, err + } + now := time.Now() + tmpl := x509.Certificate{ + SerialNumber: bigIntSerial(), + Subject: pkix.Name{CommonName: RegistryHost}, + DNSNames: []string{RegistryHost}, + NotBefore: now.Add(-time.Minute), + NotAfter: now.Add(2 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key) + if err != nil { + return tls.Certificate{}, nil, err + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM := pem.EncodeToMemory( + &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}, + ) + cert, err := tls.X509KeyPair(certPEM, keyPEM) + return cert, certPEM, err +} + +func bigIntSerial() *big.Int { + n, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 120)) + if err != nil { + return big.NewInt(time.Now().UnixNano()) + } + return n +} + +func (s *Server) authenticated(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.tunnelMu.Lock() + if s.closing { + s.tunnelMu.Unlock() + http.Error(w, "owned fixture is closing", http.StatusServiceUnavailable) + return + } + s.trackStartLocked() + s.tunnelMu.Unlock() + defer s.trackDone() + path := routeClass(r.URL.Path) + authenticated := s.validBasic(r) + cw := &countWriter{ResponseWriter: w, digest: sha256.New()} + defer s.recordResponse(r, cw, path, authenticated) + if !authenticated { + w.Header().Set("WWW-Authenticate", `Basic realm="case-a-owned-registry"`) + http.Error(cw, "authentication required", http.StatusUnauthorized) + return + } + next.ServeHTTP(cw, r) + }) +} + +func (s *Server) recordResponse(r *http.Request, cw *countWriter, path string, authenticated bool) { + status := cw.status + if status == 0 { + status = http.StatusOK + } + bodyDigest := "" + if cw.bytes > 0 { + bodyDigest = fmt.Sprintf("sha256:%x", cw.digest.Sum(nil)) + } + s.mu.Lock() + defer s.mu.Unlock() + if authenticated { + s.stats.Authenticated++ + } else { + s.stats.AuthRejected++ + } + s.stats.Completed++ + s.stats.ResponseBytes += cw.bytes + if authenticated && isSuccessfulGet(r, status, path, "manifests") { + s.stats.ManifestGets++ + } + if authenticated && isSuccessfulGet(r, status, path, "blobs") { + s.stats.BlobGets++ + } + s.stats.Requests = append(s.stats.Requests, Request{ + Method: r.Method, Path: path, Status: status, Authenticated: authenticated, + Bytes: cw.bytes, BodyDigest: bodyDigest, Digest: cw.Header().Get("Docker-Content-Digest"), + }) +} + +func (s *Server) validBasic(r *http.Request) bool { + u, p, ok := r.BasicAuth() + return ok && u == s.Username && p == s.Password +} + +func (s *Server) connect(w http.ResponseWriter, r *http.Request) { + if !s.registerConnectHandler() { + http.Error(w, "owned fixture is closing", http.StatusServiceUnavailable) + return + } + defer s.trackDone() + if !s.ownsConnectTarget(r) { + s.denyConnect(w) + return + } + client, err := hijackConnect(w) + if err != nil { + return + } + s.openTunnel(client) +} + +func (s *Server) registerConnectHandler() bool { + s.tunnelMu.Lock() + defer s.tunnelMu.Unlock() + if s.closing { + return false + } + // Register before Shutdown can ignore a handler that later hijacks. + s.trackStartLocked() + return true +} + +func (s *Server) ownsConnectTarget(r *http.Request) bool { + return r.Method == http.MethodConnect && r.Host == s.Authority +} + +func (s *Server) denyConnect(w http.ResponseWriter) { + s.mu.Lock() + s.stats.ConnectDenied++ + s.mu.Unlock() + http.Error(w, "CONNECT destination is not the owned fixture registry", http.StatusForbidden) +} + +func (s *Server) openTunnel(client net.Conn) { + upstream, err := net.DialTimeout("tcp", s.regLn.Addr().String(), 3*time.Second) + if err != nil { + _ = client.Close() + return + } + s.mu.Lock() + s.stats.ConnectAllowed++ + s.mu.Unlock() + if !s.registerTunnel(client, upstream) { + _ = client.Close() + _ = upstream.Close() + return + } + go s.copyTunnel(client, upstream) +} + +func (s *Server) registerTunnel(client, upstream net.Conn) bool { + s.tunnelMu.Lock() + defer s.tunnelMu.Unlock() + if s.closing { + return false + } + s.tunnels[client] = struct{}{} + s.tunnels[upstream] = struct{}{} + s.trackStartLocked() // register the tunnel under the same close fence + return true +} + +func (s *Server) copyTunnel(client, upstream net.Conn) { + defer s.trackDone() + defer func() { + _ = client.Close() + _ = upstream.Close() + s.tunnelMu.Lock() + delete(s.tunnels, client) + delete(s.tunnels, upstream) + s.tunnelMu.Unlock() + }() + var copyWG sync.WaitGroup + copyWG.Go(func() { + _, _ = io.Copy(upstream, client) + if c, ok := upstream.(*net.TCPConn); ok { + _ = c.CloseWrite() + } + }) + _, _ = io.Copy(client, upstream) + copyWG.Wait() +} diff --git a/e2e/framework/registryfixture/server_linux_test.go b/e2e/framework/registryfixture/server_linux_test.go new file mode 100644 index 0000000000..3ed0cbdd29 --- /dev/null +++ b/e2e/framework/registryfixture/server_linux_test.go @@ -0,0 +1,648 @@ +//go:build linux + +package registryfixture + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const ( + fixturePassword = "fixture-password" + fixtureServerKey = "registry.invalid:43210" + fixtureUser = "fixture-user" + fixtureOperationID = "attempt-0001" +) + +// This exercises only the owned loopback TLS/proxy/auth fixture. It does not +// execute MicroSandbox or claim that the published provider consumed it. +func TestOwnedLoopbackTLSAndAuthorityLimitedCONNECT(t *testing.T) { + s, err := Start(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Docker-Content-Digest", "sha256:fixture") + _, _ = io.WriteString(w, "fixture-response") + }), Config{Username: fixtureUser, Password: fixturePassword}) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + defer func() { + if err := s.Close(ctx); err != nil { + t.Errorf("close owned fixture: %v", err) + } + }() + client := fixtureHTTPClient(t, s) + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + "https://"+s.Authority+"/v2/fixture/manifests/tag", + nil, + ) + require.NoError(t, err) + req.SetBasicAuth(s.Username, s.Password) + resp, err := client.Do(req) + require.NoError(t, err) + body, readErr := io.ReadAll(resp.Body) + closeErr := resp.Body.Close() + require.NoError(t, readErr) + require.NoError(t, closeErr) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Contains(t, string(body), "fixture-response") + denied, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://outside.invalid/", nil) + require.NoError(t, err) + _, err = client.Do(denied) + require.Error(t, err, "proxy unexpectedly allowed a non-owned CONNECT authority") + got := s.Snapshot() + require.Equal(t, 1, got.ConnectAllowed) + require.Equal(t, 1, got.ConnectDenied) + require.Equal(t, 1, got.Authenticated) + require.Zero(t, got.AuthRejected) + require.Equal(t, 1, got.Completed) + require.Equal(t, 1, got.ManifestGets) + require.Positive(t, got.ResponseBytes) + require.Len(t, got.Requests, 1) + require.Equal( + t, + fmt.Sprintf("sha256:%x", sha256.Sum256([]byte("fixture-response"))), + got.Requests[0].BodyDigest, + ) +} + +func TestRegistryRecordsRejected401AndBodyDigests(t *testing.T) { + const body = "manifest-body-for-digest" + s, err := Start(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Docker-Content-Digest", "sha256:expected-header") + _, _ = io.WriteString(w, body) + }), Config{Username: fixtureUser, Password: fixturePassword}) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + cleanupRegistryServer(t, s) + client := fixtureHTTPClient(t, s) + endpoint := "https://" + s.Authority + "/v2/fixture/manifests/sha256:expected" + for _, authenticated := range []bool{false, true} { + req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + require.NoError(t, reqErr) + if authenticated { + req.SetBasicAuth(s.Username, s.Password) + } + resp, requestErr := client.Do(req) + require.NoError(t, requestErr, "owned registry request failed") + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + wantStatus := http.StatusUnauthorized + if authenticated { + wantStatus = http.StatusOK + } + require.Equal(t, wantStatus, resp.StatusCode) + } + got := s.Snapshot() + wantBodyDigest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(body))) + require.Equal(t, 1, got.AuthRejected) + require.Equal(t, 1, got.Authenticated) + require.Equal(t, 2, got.Completed) + require.Equal(t, 1, got.ManifestGets) + require.Len(t, got.Requests, 2) + rejected, success := got.Requests[0], got.Requests[1] + require.Equal(t, http.StatusUnauthorized, rejected.Status) + require.False(t, rejected.Authenticated) + require.Equal(t, "manifests", rejected.Path) + require.Positive(t, rejected.Bytes) + require.NotEmpty(t, rejected.BodyDigest) + require.Equal(t, http.StatusOK, success.Status) + require.True(t, success.Authenticated) + require.Equal(t, "manifests", success.Path) + require.Equal(t, int64(len(body)), success.Bytes) + require.Equal(t, wantBodyDigest, success.BodyDigest) + require.Equal(t, "sha256:expected-header", success.Digest) +} + +func TestRegistryObservesImplicitOKBeforeLateWriteHeader(t *testing.T) { + const body = "body commits implicit status" + s, err := Start(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, body) + w.WriteHeader(http.StatusInternalServerError) + }), Config{Username: fixtureUser, Password: fixturePassword}) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + cleanupRegistryServer(t, s) + client := fixtureHTTPClient(t, s) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + "https://"+s.Authority+"/v2/fixture/manifests/implicit-status", nil) + require.NoError(t, err) + req.SetBasicAuth(s.Username, s.Password) + resp, err := client.Do(req) + require.NoError(t, err, "owned registry request failed") + gotBody, readErr := io.ReadAll(resp.Body) + closeErr := resp.Body.Close() + require.NoError(t, readErr) + require.NoError(t, closeErr) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, body, string(gotBody)) + + got := s.Snapshot() + wantDigest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(body))) + require.Equal(t, 1, got.Completed) + require.Equal(t, 1, got.Authenticated) + require.Equal(t, int64(len(body)), got.ResponseBytes) + require.Len(t, got.Requests, 1) + observed := got.Requests[0] + require.Equal(t, resp.StatusCode, observed.Status) + require.Equal(t, int64(len(body)), observed.Bytes) + require.Equal(t, wantDigest, observed.BodyDigest) +} + +func TestCredentialHelperGetSubprocessProtocol(t *testing.T) { + dir := privateCredentialRoot(t) + configPath := filepath.Join(dir, "credentials.json") + logPath := filepath.Join(dir, "helper-observations.jsonl") + const secret = "private-fixture-secret" + const operationID = fixtureOperationID + parentHash, err := executableSHA256() + require.NoError(t, err, "hash test executable") + // #nosec G117 -- Synthetic credentials are written only to this private test fixture. + config, err := json.Marshal(CredentialConfig{ + OperationID: operationID, + ServerKey: fixtureServerKey, + Username: fixtureUser, + Secret: secret, + CredentialsAvailable: true, + ExpectedProviderSHA256: parentHash, + }) + require.NoError(t, err) + require.NoError(t, os.WriteFile(configPath, config, 0o600)) + helperPath := linkCredentialHelper(t, dir) + t.Setenv(CredentialRootEnv, dir) + t.Setenv("PATH", filepath.Dir(helperPath)+string(os.PathListSeparator)+os.Getenv("PATH")) + require.True( + t, + IsCredentialHelperInvocation(HelperBinaryName), + "bare helper must resolve to the private helper", + ) + cmd := credentialHelperCommand(t) + cmd.Env = append( + os.Environ(), + CredentialRootEnv+"="+dir, + CredentialConfigEnv+"="+configPath, + CredentialLogEnv+"="+logPath, + CredentialOperationEnv+"="+operationID, + ) + cmd.Stdin = strings.NewReader("registry.invalid:43210\n") + stdout, err := cmd.Output() + require.NoError(t, err, "credential helper get subprocess failed") + var got map[string]string + require.NoError(t, json.Unmarshal(stdout, &got), "credential helper returned invalid JSON") + require.True(t, got["Username"] == fixtureUser, "credential helper returned an unexpected user") + require.True(t, got["Secret"] == secret, "credential helper returned an unexpected secret") + // #nosec G304 -- The path is created beneath the test-owned temporary directory. + line, err := os.ReadFile(logPath) + require.NoError(t, err, "credential helper did not record its owned invocation") + var observation CredentialObservation + require.NoError( + t, + json.Unmarshal([]byte(strings.TrimSpace(string(line))), &observation), + "credential helper observation is invalid JSON", + ) + assertSuccessfulHelperObservation(t, observation, parentHash) + require.False( + t, + strings.Contains(string(line), secret), + "observation leaked the fixture secret", + ) + assertCredentialNonceRejected(t, dir, secret) + + assertCredentialParentMismatch(t, dir, secret) +} + +func assertCredentialNonceRejected(t *testing.T, dir, secret string) { + t.Helper() + configPath := filepath.Join(dir, "credentials.json") + logPath := filepath.Join(dir, "helper-observations.jsonl") + cmd := credentialHelperCommand(t) + cmd.Env = append(os.Environ(), CredentialRootEnv+"="+dir, CredentialConfigEnv+"="+configPath, + CredentialLogEnv+"="+logPath, CredentialOperationEnv+"=different-attempt") + cmd.Stdin = strings.NewReader("registry.invalid:43210\n") + output, err := cmd.Output() + require.Error(t, err, "helper accepted a config from a different attempt") + require.False( + t, + strings.Contains(string(output), secret), + "helper exposed credentials for a different attempt", + ) + require.Len(t, readCredentialObservations(t, logPath), 1) +} + +func assertSuccessfulHelperObservation( + t *testing.T, + observation CredentialObservation, + parentHash string, +) { + t.Helper() + require.Equal(t, fixtureOperationID, observation.OperationID) + require.Equal(t, fixtureServerKey, observation.ServerKey) + require.Equal(t, "get", observation.Action) + require.Equal(t, os.Getpid(), observation.Parent.PID) + require.Positive(t, observation.Parent.StartTimeTicks) + require.Equal(t, parentHash, observation.Parent.SHA256) + require.Equal(t, "credentials_provided", observation.Outcome) +} + +func assertCredentialParentMismatch(t *testing.T, dir, secret string) { + t.Helper() + const operationID = fixtureOperationID + configPath := filepath.Join(dir, "credentials.json") + logPath := filepath.Join(dir, "helper-observations.jsonl") + // The fixture test process can invoke the helper, but cannot satisfy an + // unrelated expected provider hash merely by being its recorded parent. + // #nosec G117 -- Synthetic credentials stay within this private test fixture. + config, err := json.Marshal(CredentialConfig{ + OperationID: operationID, + ServerKey: fixtureServerKey, + Username: fixtureUser, + Secret: secret, + CredentialsAvailable: true, + ExpectedProviderSHA256: strings.Repeat("0", 64), + }) + require.NoError(t, err, "encode mismatch config") + require.NoError(t, os.WriteFile(configPath, config, 0o600), "write mismatch config") + cmd := credentialHelperCommand(t) + cmd.Env = append( + os.Environ(), + CredentialRootEnv+"="+dir, + CredentialConfigEnv+"="+configPath, + CredentialLogEnv+"="+logPath, + CredentialOperationEnv+"="+operationID, + ) + cmd.Stdin = strings.NewReader("registry.invalid:43210\n") + stdout, err := cmd.Output() + require.Error(t, err, "test process satisfied a mismatched pinned-provider hash") + require.False( + t, + strings.Contains(string(stdout), secret), + "mismatched parent received credentials", + ) + observations := readCredentialObservations(t, logPath) + require.Len(t, observations, 2) + require.Equal(t, "parent_hash_mismatch", observations[1].Outcome) + require.Equal(t, os.Getpid(), observations[1].Parent.PID) + require.NotEqual(t, strings.Repeat("0", 64), observations[1].Parent.SHA256) + // #nosec G304 -- The path is an observation file in the test-owned temporary directory. + allObservations, err := os.ReadFile(logPath) + require.NoError(t, err) + require.False( + t, + strings.Contains(string(allObservations), secret), + "observations leaked the fixture secret", + ) +} + +func TestCredentialHelperExplicitNotFoundDoesNotReturnCredentials(t *testing.T) { + dir := privateCredentialRoot(t) + configPath, logPath := filepath.Join( + dir, + "credentials.json", + ), filepath.Join( + dir, + "observations.jsonl", + ) + parentHash, err := executableSHA256() + require.NoError(t, err, "hash test executable") + // #nosec G117 -- The deliberate not-found fixture contains no secret. + config, err := json.Marshal(CredentialConfig{ + OperationID: "attempt-not-found", + ServerKey: fixtureServerKey, + CredentialsAvailable: false, + ExpectedProviderSHA256: parentHash, + }) + require.NoError(t, err, "encode no-credentials config") + require.NoError(t, os.WriteFile(configPath, config, 0o600), "write no-credentials config") + helperPath := linkCredentialHelper(t, dir) + t.Setenv(CredentialRootEnv, dir) + t.Setenv("PATH", filepath.Dir(helperPath)+string(os.PathListSeparator)+os.Getenv("PATH")) + require.True( + t, + IsCredentialHelperInvocation(HelperBinaryName), + "bare helper must resolve to the private helper", + ) + cmd := credentialHelperCommand(t) + cmd.Env = append( + os.Environ(), + CredentialRootEnv+"="+dir, + CredentialConfigEnv+"="+configPath, + CredentialLogEnv+"="+logPath, + CredentialOperationEnv+"=attempt-not-found", + ) + cmd.Stdin = strings.NewReader("registry.invalid:43210\n") + stdout, err := cmd.Output() + require.Error(t, err) + require.True( + t, + strings.TrimSpace(string(stdout)) == credentialsNotFoundMessage, + "unexpected not-found response", + ) + observations := readCredentialObservations(t, logPath) + require.Len(t, observations, 1) + require.Equal(t, "attempt-not-found", observations[0].OperationID) + require.Equal(t, "credentials_not_found", observations[0].Outcome) + require.Equal(t, os.Getpid(), observations[0].Parent.PID) + require.False( + t, + strings.Contains(string(stdout), "Secret"), + "not-found response contains credential fields", + ) +} + +func TestCredentialFilesRequirePrivateOwnedRootAndRegularFiles(t *testing.T) { + root := t.TempDir() + root, err := filepath.EvalSymlinks(root) + require.NoError(t, err, "resolve private root") + // #nosec G302 -- This is a private directory, so owner execute permission is required. + require.NoError(t, os.Chmod(root, 0o700), "chmod private root") + configPath, logPath := filepath.Join( + root, + "config.json", + ), filepath.Join( + root, + "observations.jsonl", + ) + // #nosec G117 -- This configuration contains no credentials and stays in the test-owned root. + validConfig, err := json.Marshal( + CredentialConfig{OperationID: "attempt", ServerKey: "registry.invalid:1234"}, + ) + require.NoError(t, err, "encode config") + writeConfig := func(path string, mode os.FileMode) { + t.Helper() + require.NoError(t, os.WriteFile(path, validConfig, mode), "write config") + require.NoError(t, os.Chmod(path, mode), "set config mode") + } + writeConfig(configPath, 0o600) + cfg, err := ReadCredentialConfig(root, configPath) + require.NoError(t, err, "valid owned config rejected") + require.Equal(t, "attempt", cfg.OperationID) + + loose := filepath.Join(root, "loose.json") + writeConfig(loose, 0o644) + _, err = ReadCredentialConfig(root, loose) + require.Error(t, err, "accepted group/world-readable config") + link := filepath.Join(root, "linked.json") + require.NoError(t, os.Symlink(configPath, link), "create config symlink") + _, err = ReadCredentialConfig(root, link) + require.Error(t, err, "accepted symlink config") + outside := filepath.Join(t.TempDir(), "outside.json") + writeConfig(outside, 0o600) + _, err = ReadCredentialConfig(root, outside) + require.Error(t, err, "accepted config outside private root") + require.NoError(t, os.Symlink(outside, logPath), "create log symlink") + require.Error( + t, + appendOwnedLine(root, logPath, []byte("{}\n")), + "accepted symlink observation log", + ) + require.Error( + t, + appendOwnedLine( + root, + filepath.Join(filepath.Dir(root), "outside-log.jsonl"), + []byte("{}\n"), + ), + "accepted observation log outside private root", + ) +} + +func TestCredentialHelperDispatchRequiresOwnedPathAndSameExecutable(t *testing.T) { + root := privateCredentialRoot(t) + binDir := filepath.Join(root, "bin") + require.NoError(t, os.Mkdir(binDir, 0o700), "create helper bin directory") + helpPath := filepath.Join(binDir, HelperBinaryName) + require.NoError(t, os.Symlink("/bin/true", helpPath), "create foreign helper link") + t.Setenv(CredentialRootEnv, root) + require.False(t, IsCredentialHelperInvocation(helpPath), "dispatch accepted another executable") + require.False( + t, + IsCredentialHelperInvocation(os.Args[0]), + "dispatch accepted the ordinary test binary", + ) +} + +func TestCredentialHelperDispatchResolvesOnlyPrivateBareHelper(t *testing.T) { + root := privateCredentialRoot(t) + helperPath := linkCredentialHelper(t, root) + t.Setenv(CredentialRootEnv, root) + privateBin := filepath.Dir(helperPath) + t.Setenv("PATH", privateBin+string(os.PathListSeparator)+os.Getenv("PATH")) + require.True( + t, + IsCredentialHelperInvocation(HelperBinaryName), + "dispatch rejected the private bare name", + ) + require.True( + t, + IsCredentialHelperInvocation(helperPath), + "dispatch rejected the private absolute path", + ) + require.False( + t, + IsCredentialHelperInvocation(filepath.Join("bin", HelperBinaryName)), + "accepted another relative path", + ) + + foreignBin := t.TempDir() + executable, err := os.Executable() + require.NoError(t, err, "locate current executable") + require.NoError( + t, + os.Symlink(executable, filepath.Join(foreignBin, HelperBinaryName)), + "create foreign PATH helper", + ) + t.Setenv( + "PATH", + foreignBin+string( + os.PathListSeparator, + )+privateBin+string( + os.PathListSeparator, + )+os.Getenv( + "PATH", + ), + ) + require.False( + t, + IsCredentialHelperInvocation(HelperBinaryName), + "accepted a helper from a foreign PATH directory", + ) +} + +func privateCredentialRoot(t *testing.T) string { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err, "resolve helper root") + // #nosec G302 -- This private directory requires owner execute permission. + require.NoError(t, os.Chmod(dir, 0o700), "set helper root mode") + return dir +} + +func linkCredentialHelper(t *testing.T, dir string) string { + t.Helper() + binDir := filepath.Join(dir, "bin") + require.NoError(t, os.Mkdir(binDir, 0o700), "create private helper bin directory") + executable, err := os.Executable() + require.NoError(t, err, "locate fixture test executable") + helperPath := filepath.Join(binDir, HelperBinaryName) + require.NoError( + t, + os.Symlink(executable, helperPath), + "link fixture test executable as credential helper", + ) + return helperPath +} + +func executableSHA256() (string, error) { + executable, err := os.Executable() + if err != nil { + return "", err + } + // #nosec G304 -- This is the current test executable returned by os.Executable. + file, err := os.Open(executable) + if err != nil { + return "", err + } + h := sha256.New() + _, copyErr := io.Copy(h, file) + closeErr := file.Close() + if err := errors.Join(copyErr, closeErr); err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +func readCredentialObservations(t *testing.T, path string) []CredentialObservation { + t.Helper() + // #nosec G304 -- Callers pass the observation file in their private temporary fixture. + data, err := os.ReadFile(path) + require.NoError(t, err, "read helper observations") + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + observations := make([]CredentialObservation, 0, len(lines)) + for _, line := range lines { + var observation CredentialObservation + require.NoError(t, json.Unmarshal([]byte(line), &observation), "decode helper observation") + observations = append(observations, observation) + } + return observations +} + +func credentialHelperCommand(t *testing.T) *exec.Cmd { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + t.Cleanup(cancel) + return exec.CommandContext(ctx, HelperBinaryName, "get") +} + +func fixtureHTTPClient(t *testing.T, server *Server) *http.Client { + t.Helper() + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM(server.CAPEM), "owned CA PEM was not parseable") + proxyURL, err := url.Parse(server.ProxyURL) + require.NoError(t, err) + transport := &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, + } + t.Cleanup(transport.CloseIdleConnections) + return &http.Client{Transport: transport} +} + +func cleanupRegistryServer(t *testing.T, server *Server) { + t.Helper() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.WithoutCancel(t.Context()), 3*time.Second) + defer cancel() + require.NoError(t, server.Close(ctx), "close owned registry fixture") + }) +} + +func TestCloseRetainsFirstResult(t *testing.T) { + s, err := Start( + http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}), + Config{Username: "u", Password: "p"}, + ) + require.NoError(t, err) + // Hold an owned activity slot so an already-canceled first close must + // report its bounded-wait error deterministically. + s.tunnelMu.Lock() + s.trackStartLocked() + s.tunnelMu.Unlock() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + first := s.Close(ctx) + s.trackDone() + second := s.Close(context.Background()) + require.Error(t, first) + require.Error(t, second) + require.EqualError(t, second, first.Error(), "Close did not retain its first cleanup result") +} + +func TestCloseRacingOwnedCONNECTsLeavesNoTrackedWork(t *testing.T) { + s, err := Start(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "owned") + }), Config{Username: "u", Password: "p"}) + require.NoError(t, err) + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cleanupCancel() + _ = s.Close(cleanupCtx) + }) + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) + defer cancel() + client := fixtureHTTPClient(t, s) + start := make(chan struct{}) + done := make(chan struct{}, 24) + for range 24 { + go func() { + defer func() { done <- struct{}{} }() + <-start + req, reqErr := http.NewRequestWithContext( + ctx, + http.MethodGet, + "https://"+s.Authority+"/v2/fixture/manifests/tag", + nil, + ) + if reqErr != nil { + return + } + req.SetBasicAuth(s.Username, s.Password) + resp, reqErr := client.Do(req) + if reqErr == nil { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } + }() + } + close(start) + require.NoError(t, s.Close(ctx), "close owned fixture") + for range 24 { + select { + case <-done: + case <-ctx.Done(): + t.Fatal("owned client requests did not finish before the bound") + } + } + s.tunnelMu.Lock() + active, tunnels := s.active, len(s.tunnels) + s.tunnelMu.Unlock() + require.Zero(t, active, "fixture retained work after close") + require.Zero(t, tunnels, "fixture retained tunnels after close") +} diff --git a/e2e/framework/registryfixture/testmain_linux_test.go b/e2e/framework/registryfixture/testmain_linux_test.go new file mode 100644 index 0000000000..597c6f9db6 --- /dev/null +++ b/e2e/framework/registryfixture/testmain_linux_test.go @@ -0,0 +1,15 @@ +//go:build linux + +package registryfixture + +import ( + "os" + "testing" +) + +func TestMain(m *testing.M) { + if handled, code := DispatchCredentialHelper(); handled { + os.Exit(code) + } + os.Exit(m.Run()) +} diff --git a/e2e/tests/up/MICROSANDBOX.md b/e2e/tests/up/MICROSANDBOX.md index 54d42bba27..6c7a0d7f5f 100644 --- a/e2e/tests/up/MICROSANDBOX.md +++ b/e2e/tests/up/MICROSANDBOX.md @@ -1,7 +1,7 @@ # MicroSandbox integration tests -Each capability has a Ginkgo label and scenario file. The built-in and external -providers run the same contracts. Keep scenario assertions in those files so +Each capability has a Ginkgo label and scenario file. Parity suites run the same +contracts against the built-in and external providers. Keep scenario assertions in those files so resource, ownership and lifecycle expectations remain visible. ## Shared setup @@ -33,8 +33,11 @@ and credentials are configured once in the job definition. Each matrix entry names its Ginkgo label and expected scenario count. `hack/microsandbox/verify_report.py` requires a real successful run with that exact count, split equally between the explicit `microsandbox-builtin` and -`microsandbox-external` labels. Missing cases, skips, retries, dry runs and -programmatic focus fail the gate. Reports and test output are retained even when +`microsandbox-external` labels by default. A contract specific to the published +provider uses an explicit `provider-mode: external` catalogue entry and requires +every selected case to carry only the external provider label. Missing cases, +skips, retries, dry runs and programmatic focus fail the gate. Reports and test +output are retained even when the test command fails. To add a scenario: @@ -48,6 +51,29 @@ A new suite needs a catalogue entry only when it has a separate contract or isolation requirement. Avoid copying runner setup or adding a job for each individual assertion. +## Published-provider registry consumption + +The Linux-only `up-provider-microsandbox-registry-consumer` suite exercises the +production external host's Runtime Protocol `RunImage` path with the installed, +checksum-verified provider. Its three external-only cases cover an owned TLS +registry through a restricted CONNECT proxy, refusal without the fixture CA, +and refusal without registry credentials. A private credential helper records +the direct parent executable digest and operation identity without recording +credentials. Successful image responses are checked against their actual body +digests and byte counts, and the positive case verifies a guest marker. + +This contract calls `RunImage` directly because ordinary `up` resolves image +metadata before invoking the provider. It proves the published provider's image +consumer; ordinary workspace orchestration and remote agent binary preparation +have separate contracts. Each case owns fresh runtime state and an immutable +image reference. Registry seeding finishes before the measured listeners start, +and fixture shutdown joins active handlers before observations are inspected. + +On Linux, the E2E test binary also serves as the private Docker credential helper. +Its `TestMain` preserves the production runtime-supervisor dispatch and recognizes +the helper only at its canonical, current-user-owned private path. Other test +invocations retain the normal test runner path. + ## Managed policy boundary Managed-policy recreation is Linux-only and runs through diff --git a/e2e/tests/up/provider_microsandbox_consumer_linux.go b/e2e/tests/up/provider_microsandbox_consumer_linux.go new file mode 100644 index 0000000000..587bde8200 --- /dev/null +++ b/e2e/tests/up/provider_microsandbox_consumer_linux.go @@ -0,0 +1,585 @@ +//go:build linux + +package up + +import ( + "archive/tar" + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/devsy-org/devsy-runtime-sdk/runtimev1" + "github.com/devsy-org/devsy/e2e/framework" + "github.com/devsy-org/devsy/e2e/framework/registryfixture" + devsyconfig "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/driver" + "github.com/devsy-org/devsy/pkg/driver/external" + "github.com/devsy-org/devsy/pkg/image" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +type microsandboxConsumerAttempt struct { + name string + credentials bool + trustFixture bool + wantErrorText string +} + +type microsandboxConsumerSeed struct { + handler http.Handler + repository string + imageDigest string + markerPath string + markerContent string + expectedBodies map[string]int64 +} + +type microsandboxConsumerHelper struct { + root, config, log, operationID string +} + +type microsandboxConsumerEnvironment struct { + fixture *registryfixture.Server + home, msbHome string + helper microsandboxConsumerHelper + trustFixture bool +} + +type microsandboxConsumerReceiptExpectation struct { + path, operationID, serverKey, providerBinary, providerHash string + required bool + outcome string +} + +type microsandboxConsumerProvider struct { + config provider.ProviderAgentConfig + origin string + runtimeBinary string + runtimeHash string +} + +type microsandboxConsumerHostRun struct { + host *external.Host + workspaceID string + imageRef string + attempt microsandboxConsumerAttempt + markerPath string + markerContent string +} + +const ( + microsandboxConsumerPositive = "positive" + microsandboxConsumerNoCA = "missing-ca" + microsandboxConsumerNoCredentials = "missing-credentials" +) + +var _ = ginkgo.Describe( + "published MicroSandbox provider registry consumption", + ginkgo.Label("up-provider-microsandbox-registry-consumer", "microsandbox-external"), + func() { + ginkgo.DescribeTable( + "uses the published provider's registry client", + runMicrosandboxConsumerAttempt, + ginkgo.Entry( + "downloads an authenticated image over fixture TLS", + microsandboxConsumerAttempt{ + name: microsandboxConsumerPositive, credentials: true, trustFixture: true, + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ), + ginkgo.Entry("rejects an untrusted registry certificate", microsandboxConsumerAttempt{ + name: microsandboxConsumerNoCA, credentials: true, wantErrorText: "certificate", + }, ginkgo.SpecTimeout(framework.TimeoutLong())), + ginkgo.Entry("rejects a registry without credentials", microsandboxConsumerAttempt{ + name: microsandboxConsumerNoCredentials, + trustFixture: true, + wantErrorText: "unauthorized", + }, ginkgo.SpecTimeout(framework.TimeoutLong())), + ) + }, +) + +func runMicrosandboxConsumerAttempt(ctx context.Context, attempt microsandboxConsumerAttempt) { + session := newMicrosandboxTestSession(ctx) + providerName := "case-a-consumer" + session.addProvider(ctx, microsandboxExternalProvider, providerName) + consumerProvider := prepareMicrosandboxConsumerProvider(ctx, providerName) + actualProviderHash := consumerProvider.runtimeHash + runtimeBinary := consumerProvider.runtimeBinary + + seed := seedMicrosandboxConsumerImage(ctx, attempt.name) + + fixture := newMicrosandboxConsumerRegistry(seed.handler) + imageRef := fixture.Authority + "/" + seed.repository + "@" + seed.imageDigest + assertMicrosandboxImageAbsentFromDocker(ctx, imageRef) + + workspaceID := microsandboxConsumerID(attempt.name) + privateHome := microsandboxConsumerDirectory(ginkgo.GinkgoT().TempDir()) + msbHome := microsandboxConsumerDirectory(ginkgo.GinkgoT().TempDir()) + cacheDir := filepath.Join(msbHome, "cache") + _, err := os.Stat(cacheDir) + gomega.Expect(err). + To(gomega.MatchError(os.ErrNotExist), "fresh MSB_HOME cache must not contain a prior image") + helper := prepareMicrosandboxConsumerHelper( + fixture, privateHome, actualProviderHash, attempt.credentials, + ) + setMicrosandboxConsumerEnvironment(microsandboxConsumerEnvironment{ + fixture: fixture, home: privateHome, msbHome: msbHome, + helper: helper, trustFixture: attempt.trustFixture, + }) + + host := newMicrosandboxConsumerHost(ctx, providerName, workspaceID, consumerProvider) + ginkgo.DeferCleanup(func(cleanupCtx ginkgo.SpecContext) error { + deleteCtx, cancel := context.WithTimeout(context.WithoutCancel(cleanupCtx), time.Minute) + defer cancel() + return host.DeleteDevContainer(deleteCtx, workspaceID) + }) + runMicrosandboxConsumerHost(ctx, microsandboxConsumerHostRun{ + host: host, workspaceID: workspaceID, imageRef: imageRef, attempt: attempt, + markerPath: seed.markerPath, markerContent: seed.markerContent, + }) + + framework.ExpectNoError(closeMicrosandboxConsumerRegistry(ctx, fixture)) + snapshot := fixture.Snapshot() + assertMicrosandboxConsumerSnapshot(attempt, snapshot, seed.expectedBodies) + receiptRequired := attempt.name == microsandboxConsumerPositive || + attempt.name == microsandboxConsumerNoCredentials + expectedOutcome := "" + switch attempt.name { + case microsandboxConsumerPositive: + expectedOutcome = "credentials_provided" + case microsandboxConsumerNoCredentials: + expectedOutcome = "credentials_not_found" + } + assertMicrosandboxConsumerReceipts(microsandboxConsumerReceiptExpectation{ + path: helper.log, operationID: helper.operationID, serverKey: fixture.Authority, + providerBinary: runtimeBinary, providerHash: actualProviderHash, + required: receiptRequired, outcome: expectedOutcome, + }) +} + +func newMicrosandboxConsumerRegistry(handler http.Handler) *registryfixture.Server { + fixture, err := registryfixture.Start(handler, registryfixture.Config{ + Username: "case-a-fixture-user", Password: "case-a-private-fixture-secret", + }) + framework.ExpectNoError(err) + ginkgo.DeferCleanup(func(ctx ginkgo.SpecContext) error { + return closeMicrosandboxConsumerRegistry(ctx, fixture) + }) + return fixture +} + +func closeMicrosandboxConsumerRegistry(ctx context.Context, fixture *registryfixture.Server) error { + closeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + return fixture.Close(closeCtx) +} + +func prepareMicrosandboxConsumerProvider( + ctx context.Context, + name string, +) microsandboxConsumerProvider { + installed, err := provider.LoadProviderConfig(devsyconfig.DefaultContext, name) + framework.ExpectNoError(err) + origin := ginkgo.GinkgoT().TempDir() + binaryDir := filepath.Join(origin, "binaries") + framework.ExpectNoError(os.MkdirAll(binaryDir, 0o700)) + binaryPaths, err := provider.DownloadBinaries(ctx, installed.Agent.Binaries, binaryDir) + framework.ExpectNoError(err) + binary, expectedHash := microsandboxConsumerBinary(installed.Agent, binaryPaths) + actualHash := microsandboxConsumerFileSHA256(binary) + gomega.Expect(actualHash).To(gomega.Equal(expectedHash)) + return microsandboxConsumerProvider{ + config: installed.Agent, origin: origin, runtimeBinary: binary, runtimeHash: actualHash, + } +} + +func newMicrosandboxConsumerHost( + ctx context.Context, + name, workspaceID string, + consumerProvider microsandboxConsumerProvider, +) *external.Host { + devsyConfig, err := devsyconfig.LoadConfig("", "") + framework.ExpectNoError(err) + workspace := &provider.Workspace{ + ID: workspaceID, UID: workspaceID, Context: devsyconfig.DefaultContext, + Provider: provider.WorkspaceProviderConfig{Name: name}, Origin: consumerProvider.origin, + } + info := &provider.AgentWorkspaceInfo{ + WorkspaceOrigin: consumerProvider.origin, + Workspace: workspace, + Agent: consumerProvider.config, + Options: provider.GetProviderOptions( + workspace, + nil, + devsyConfig, + ), + Origin: consumerProvider.origin, + } + host, err := external.New(ctx, info) + framework.ExpectNoError(err) + return host +} + +func runMicrosandboxConsumerHost(ctx context.Context, run microsandboxConsumerHostRun) { + framework.ExpectNoError(run.host.ProvisioningPreflight(ctx)) + found, err := run.host.FindDevContainer(ctx, run.workspaceID) + framework.ExpectNoError(err) + gomega.Expect(found).To(gomega.BeNil(), "workspace ID must be fresh before RunImage") + err = run.host.RunImage(ctx, &runtimev1.RunImageRequest{ + WorkspaceId: run.workspaceID, Image: run.imageRef, + User: microsandboxRootUser, RemoteUser: microsandboxRootUser, + Entrypoint: "/bin/sh", Args: []string{"-c", "sleep 600"}, + }) + if run.attempt.wantErrorText != "" { + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(strings.ToLower(err.Error())). + To(gomega.ContainSubstring(run.attempt.wantErrorText)) + found, err = run.host.FindDevContainer(ctx, run.workspaceID) + framework.ExpectNoError(err) + gomega.Expect(found).To(gomega.BeNil(), "failed RunImage must not leave a workspace") + return + } + framework.ExpectNoError(err) + var stdout strings.Builder + framework.ExpectNoError(run.host.CommandDevContainer(ctx, &driver.CommandParams{ + WorkspaceID: run.workspaceID, Command: "cat " + run.markerPath, Stdout: &stdout, + })) + gomega.Expect(stdout.String()).To(gomega.Equal(run.markerContent)) + found, err = run.host.FindDevContainer(ctx, run.workspaceID) + framework.ExpectNoError(err) + gomega.Expect(found).NotTo(gomega.BeNil()) +} + +func seedMicrosandboxConsumerImage(ctx context.Context, attempt string) microsandboxConsumerSeed { + img, err := image.GetImageForArch( + ctx, + "ghcr.io/devsy-org/test-images/base:alpine", + runtime.GOARCH, + ) + framework.ExpectNoError(err) + img, markerPath, markerContent := appendMicrosandboxConsumerMarker(img, attempt) + imageDigest, err := img.Digest() + framework.ExpectNoError(err) + + seedHandler := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + seed := httptest.NewServer(seedHandler) + seedClosed := false + ginkgo.DeferCleanup(func() { + if !seedClosed { + seed.Close() + } + }) + operationID := microsandboxConsumerID(attempt) + repository := "case-a/" + strings.TrimPrefix(operationID, "case-a-") + seedRef, err := name.NewTag( + strings.TrimPrefix(seed.URL, "http://")+"/"+repository+":fixture", + name.Insecure, + ) + framework.ExpectNoError(err) + framework.ExpectNoError(remote.Write(seedRef, img, remote.WithContext(ctx))) + seed.Close() + seedClosed = true + expected := microsandboxConsumerImageBodies(img) + return microsandboxConsumerSeed{ + handler: seedHandler, repository: repository, imageDigest: imageDigest.String(), + markerPath: markerPath, markerContent: markerContent, expectedBodies: expected, + } +} + +func appendMicrosandboxConsumerMarker(img v1.Image, attempt string) (v1.Image, string, string) { + markerToken := microsandboxConsumerID(attempt) + markerPath := "/case-a-marker-" + strings.TrimPrefix(markerToken, "case-a-") + markerContent := "published-provider-image-" + markerToken + "\n" + var markerTar bytes.Buffer + tarWriter := tar.NewWriter(&markerTar) + framework.ExpectNoError(tarWriter.WriteHeader(&tar.Header{ + Name: strings.TrimPrefix(markerPath, "/"), Mode: 0o644, + Size: int64(len(markerContent)), Typeflag: tar.TypeReg, Format: tar.FormatUSTAR, + })) + _, err := io.WriteString(tarWriter, markerContent) + framework.ExpectNoError(err) + framework.ExpectNoError(tarWriter.Close()) + markerLayer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(markerTar.Bytes())), nil + }) + framework.ExpectNoError(err) + imageWithMarker, err := mutate.AppendLayers(img, markerLayer) + framework.ExpectNoError(err) + return imageWithMarker, markerPath, markerContent +} + +func microsandboxConsumerImageBodies(img v1.Image) map[string]int64 { + manifestBytes, err := img.RawManifest() + framework.ExpectNoError(err) + manifestHash := sha256.Sum256(manifestBytes) + expected := map[string]int64{ + "sha256:" + hex.EncodeToString(manifestHash[:]): int64(len(manifestBytes)), + } + configBytes, err := img.RawConfigFile() + framework.ExpectNoError(err) + configDigest, err := img.ConfigName() + framework.ExpectNoError(err) + expected[configDigest.String()] = int64(len(configBytes)) + layers, err := img.Layers() + framework.ExpectNoError(err) + for _, layer := range layers { + digest, digestErr := layer.Digest() + framework.ExpectNoError(digestErr) + size, sizeErr := layer.Size() + framework.ExpectNoError(sizeErr) + expected[digest.String()] = size + } + return expected +} + +func microsandboxConsumerBinary( + agent provider.ProviderAgentConfig, + binaryPaths map[string]string, +) (string, string) { + binaryName := agent.External.Binary + binaryPath := binaryPaths[binaryName] + gomega.Expect(binaryPath). + NotTo(gomega.BeEmpty(), "published provider runtime binary was not downloaded") + var expected string + for _, binary := range agent.Binaries[binaryName] { + if binary.OS == runtime.GOOS && binary.Arch == runtime.GOARCH { + expected = strings.TrimPrefix(strings.ToLower(binary.Checksum), "sha256:") + break + } + } + gomega.Expect(expected). + To(gomega.HaveLen(sha256.Size*2), "published runtime checksum must be pinned") + return binaryPath, expected +} + +func microsandboxConsumerFileSHA256(path string) string { + // #nosec G304 -- DownloadBinaries returns this path inside the spec's private temporary origin. + file, err := os.Open(path) + framework.ExpectNoError(err) + defer func() { framework.ExpectNoError(file.Close()) }() + h := sha256.New() + _, err = io.Copy(h, file) + framework.ExpectNoError(err) + return hex.EncodeToString(h.Sum(nil)) +} + +func microsandboxConsumerID(suffix string) string { + var token [8]byte + _, err := rand.Read(token[:]) + framework.ExpectNoError(err) + return "case-a-" + suffix + "-" + hex.EncodeToString(token[:]) +} + +func microsandboxConsumerDirectory(path string) string { + // #nosec G302 -- Ginkgo owns this fresh test directory; private 0700 access is required. + framework.ExpectNoError(os.Chmod(path, 0o700)) + return path +} + +func prepareMicrosandboxConsumerHelper( + fixture *registryfixture.Server, + home, providerHash string, + credentials bool, +) microsandboxConsumerHelper { + root, err := os.MkdirTemp(home, "helper-") + framework.ExpectNoError(err) + root, err = filepath.EvalSymlinks(root) + framework.ExpectNoError(err) + // #nosec G302 -- the helper's executable/config/log tree is deliberately private to this spec. + framework.ExpectNoError(os.Chmod(root, 0o700)) + binDir := filepath.Join(root, "bin") + framework.ExpectNoError(os.Mkdir(binDir, 0o700)) + executable, err := os.Executable() + framework.ExpectNoError(err) + framework.ExpectNoError( + os.Symlink(executable, filepath.Join(binDir, registryfixture.HelperBinaryName)), + ) + operationID := microsandboxConsumerID("operation") + configPath, logPath := filepath.Join( + root, + "credentials.json", + ), filepath.Join( + root, + "observations.jsonl", + ) + config := registryfixture.CredentialConfig{ + OperationID: operationID, ServerKey: fixture.Authority, + CredentialsAvailable: credentials, ExpectedProviderSHA256: providerHash, + } + if credentials { + config.Username, config.Secret = fixture.Username, fixture.Password + } + // #nosec G117 -- private helper config intentionally holds fixture credentials and is written mode 0600. + data, err := json.Marshal(config) + framework.ExpectNoError(err) + framework.ExpectNoError(os.WriteFile(configPath, data, 0o600)) + dockerDir := filepath.Join(home, ".docker") + framework.ExpectNoError(os.Mkdir(dockerDir, 0o700)) + framework.ExpectNoError(os.Mkdir(filepath.Join(home, ".config"), 0o700)) + dockerConfig, err := json.Marshal(struct { + CredHelpers map[string]string `json:"credHelpers"` + }{CredHelpers: map[string]string{fixture.Authority: "fixture"}}) + framework.ExpectNoError(err) + framework.ExpectNoError( + os.WriteFile(filepath.Join(dockerDir, "config.json"), dockerConfig, 0o600), + ) + return microsandboxConsumerHelper{ + root: root, + config: configPath, + log: logPath, + operationID: operationID, + } +} + +func setMicrosandboxConsumerEnvironment(environment microsandboxConsumerEnvironment) { + fixture, home, msbHome := environment.fixture, environment.home, environment.msbHome + helper := environment.helper + binDir := filepath.Join(helper.root, "bin") + for name, value := range map[string]string{ + "HOME": home, "XDG_CONFIG_HOME": filepath.Join(home, ".config"), + "DOCKER_CONFIG": filepath.Join(home, ".docker"), "MSB_HOME": msbHome, + "CASE_A_CREDENTIAL_ROOT": helper.root, "CASE_A_CREDENTIAL_CONFIG": helper.config, + "CASE_A_CREDENTIAL_OBSERVATIONS": helper.log, "CASE_A_OPERATION_ID": helper.operationID, + "HTTPS_PROXY": fixture.ProxyURL, "HTTP_PROXY": fixture.ProxyURL, + "https_proxy": fixture.ProxyURL, "http_proxy": fixture.ProxyURL, + "NO_PROXY": "", "no_proxy": "", "REGISTRY_AUTH_FILE": "", + "PATH": binDir + string(os.PathListSeparator) + os.Getenv("PATH"), + } { + ginkgo.GinkgoT().Setenv(name, value) + } + certPath := filepath.Join(home, "fixture-ca.pem") + cert := []byte(nil) + if environment.trustFixture { + cert = fixture.CAPEM + } + framework.ExpectNoError(os.WriteFile(certPath, cert, 0o600)) + framework.ExpectNoError(os.Mkdir(filepath.Join(home, "empty-cert-dir"), 0o700)) + ginkgo.GinkgoT().Setenv("SSL_CERT_FILE", certPath) + ginkgo.GinkgoT().Setenv("SSL_CERT_DIR", filepath.Join(home, "empty-cert-dir")) +} + +func assertMicrosandboxConsumerSnapshot( + attempt microsandboxConsumerAttempt, + snapshot registryfixture.Snapshot, + expected map[string]int64, +) { + gomega.Expect(snapshot.ConnectAllowed).To(gomega.BeNumerically(">", 0)) + gomega.Expect(snapshot.ConnectDenied).To(gomega.BeZero()) + switch attempt.name { + case microsandboxConsumerPositive: + gomega.Expect(snapshot.Authenticated).To(gomega.BeNumerically(">", 0)) + gomega.Expect(snapshot.ManifestGets).To(gomega.BeNumerically(">", 0)) + gomega.Expect(snapshot.BlobGets).To(gomega.BeNumerically(">", 0)) + assertMicrosandboxConsumerPositiveRequests(snapshot.Requests, expected) + case microsandboxConsumerNoCA: + gomega.Expect(snapshot.Authenticated).To(gomega.BeZero()) + gomega.Expect(snapshot.AuthRejected).To(gomega.BeZero()) + gomega.Expect(snapshot.Requests). + To(gomega.BeEmpty(), "TLS rejection must happen before HTTP requests") + case microsandboxConsumerNoCredentials: + gomega.Expect(snapshot.Authenticated).To(gomega.BeZero()) + gomega.Expect(snapshot.AuthRejected).To(gomega.BeNumerically(">", 0)) + gomega.Expect(snapshot.ManifestGets).To(gomega.BeZero()) + gomega.Expect(snapshot.BlobGets).To(gomega.BeZero()) + assertMicrosandboxConsumerChallenges(snapshot.Requests) + default: + ginkgo.Fail(fmt.Sprintf("unknown consumer attempt %q", attempt.name)) + } +} + +func assertMicrosandboxConsumerPositiveRequests( + requests []registryfixture.Request, + expected map[string]int64, +) { + observed := map[string]int64{} + for _, request := range requests { + if !request.Authenticated { + assertMicrosandboxConsumerChallenge(request) + continue + } + gomega.Expect(request.Status).To(gomega.BeNumerically(">=", 200)) + gomega.Expect(request.Status).To(gomega.BeNumerically("<", 300)) + if request.Method != http.MethodGet || + (request.Path != "manifests" && request.Path != "blobs") { + continue + } + gomega.Expect(request.BodyDigest).NotTo(gomega.BeEmpty()) + expectedSize, knownDigest := expected[request.BodyDigest] + gomega.Expect(knownDigest). + To(gomega.BeTrue(), "provider fetched an unexpected image body %s", request.BodyDigest) + gomega.Expect(request.Bytes).To(gomega.Equal(expectedSize)) + observed[request.BodyDigest] = request.Bytes + } + for digest, size := range expected { + gomega.Expect(observed[digest]). + To(gomega.Equal(size), "published image component %s was not received byte-for-byte", digest) + } +} + +func assertMicrosandboxConsumerChallenges(requests []registryfixture.Request) { + for _, request := range requests { + gomega.Expect(request.Authenticated).To(gomega.BeFalse()) + gomega.Expect(request.Method).To(gomega.Equal(http.MethodGet)) + // An anonymous client can receive the Basic challenge, then be refused + // again when it requests the protected manifest without credentials. + gomega.Expect(request.Path).To(gomega.Or(gomega.Equal("v2"), gomega.Equal("manifests"))) + gomega.Expect(request.Status).To(gomega.Equal(http.StatusUnauthorized)) + } +} + +func assertMicrosandboxConsumerChallenge(request registryfixture.Request) { + gomega.Expect(request.Method).To(gomega.Equal(http.MethodGet)) + gomega.Expect(request.Path).To(gomega.Equal("v2")) + gomega.Expect(request.Status).To(gomega.Equal(http.StatusUnauthorized)) +} + +func assertMicrosandboxConsumerReceipts(expected microsandboxConsumerReceiptExpectation) { + data, err := os.ReadFile(expected.path) + if os.IsNotExist(err) && !expected.required { + return + } + framework.ExpectNoError(err) + if len(data) == 0 { + gomega.Expect(expected.required). + To(gomega.BeFalse(), "provider did not invoke its credential helper") + return + } + if strings.Contains(string(data), "case-a-private-fixture-secret") { + ginkgo.Fail("credential observations included secret material") + } + for line := range strings.SplitSeq(strings.TrimSpace(string(data)), "\n") { + var observation registryfixture.CredentialObservation + framework.ExpectNoError(json.Unmarshal([]byte(line), &observation)) + gomega.Expect(observation.OperationID).To(gomega.Equal(expected.operationID)) + gomega.Expect(observation.ServerKey).To(gomega.Equal(expected.serverKey)) + gomega.Expect(observation.Action).To(gomega.Equal("get")) + gomega.Expect(observation.Parent.Executable).To(gomega.Equal(expected.providerBinary)) + gomega.Expect(observation.Parent.SHA256).To(gomega.Equal(expected.providerHash)) + gomega.Expect(observation.Parent.StartTimeTicks).To(gomega.BeNumerically(">", 0)) + if expected.outcome != "" { + gomega.Expect(observation.Outcome).To(gomega.Equal(expected.outcome)) + } + } +} diff --git a/hack/microsandbox/test_verify_report.py b/hack/microsandbox/test_verify_report.py index 15a2720cfb..7cbf13f2d0 100644 --- a/hack/microsandbox/test_verify_report.py +++ b/hack/microsandbox/test_verify_report.py @@ -10,7 +10,9 @@ PROVIDERS = ("microsandbox-builtin", "microsandbox-external") -def report(count=2): +def report(count=2, provider_labels=None): + if provider_labels is None: + provider_labels = [PROVIDERS[index % 2] for index in range(count)] return [{ "SuiteConfig": {"DryRun": False}, "SuiteSucceeded": True, @@ -18,7 +20,7 @@ def report(count=2): "SpecReports": [ { "ContainerHierarchyLabels": [[LABEL]], - "LeafNodeLabels": [PROVIDERS[index % 2]], + "LeafNodeLabels": [provider_labels[index]], "LeafNodeType": "It", "State": "passed", "NumAttempts": 1, @@ -43,6 +45,56 @@ def test_accepts_exact_successful_label_count(self): self.write(report()) self.assertEqual(validate_report(self.path, LABEL, 2), 2) + def test_accepts_odd_external_only_count(self): + self.write(report(3, [PROVIDERS[1]] * 3)) + self.assertEqual(validate_report(self.path, LABEL, 3, provider_mode="external"), 3) + + def test_external_mode_rejects_builtin_or_mixed_provider_labels(self): + cases = ( + ( + "builtin", + [[PROVIDERS[0]]] * 3, + "exactly the microsandbox-external", + ), + ( + "mixed", + [[PROVIDERS[1]], [PROVIDERS[0]], [PROVIDERS[1]]], + "exactly the microsandbox-external", + ), + ( + "ambiguous", + [list(PROVIDERS), [PROVIDERS[1]], [PROVIDERS[1]]], + "exactly one provider label", + ), + ) + for name, labels, message in cases: + data = report(3) + for spec, provider_labels in zip(data[0]["SpecReports"], labels): + spec["LeafNodeLabels"] = provider_labels + self.write(data) + with self.subTest(name=name), self.assertRaisesRegex(ValueError, message): + validate_report(self.path, LABEL, 3, provider_mode="external") + + def test_external_mode_rejects_skipped_or_retried_specs(self): + for field, value, message in ( + ("State", "skipped", "not passed"), + ("NumAttempts", 2, "expected one"), + ): + data = report(3, [PROVIDERS[1]] * 3) + data[0]["SpecReports"][1][field] = value + self.write(data) + with self.subTest(field=field), self.assertRaisesRegex(ValueError, message): + validate_report(self.path, LABEL, 3, provider_mode="external") + + def test_provider_mode_and_expected_count_are_validated(self): + self.write(report(1, [PROVIDERS[1]])) + with self.assertRaisesRegex(ValueError, "unsupported provider mode"): + validate_report(self.path, LABEL, 1, provider_mode="unknown") + with self.assertRaisesRegex(ValueError, "must be positive"): + validate_report(self.path, LABEL, 0, provider_mode="external") + with self.assertRaisesRegex(ValueError, "must be even in parity mode"): + validate_report(self.path, LABEL, 1) + def test_rejects_missing_report(self): with self.assertRaisesRegex(ValueError, "cannot read"): validate_report(self.path, LABEL, 2) @@ -100,8 +152,8 @@ def test_rejects_wrong_label(self): def test_rejects_wrong_count(self): self.write(report()) - with self.assertRaisesRegex(ValueError, "expected 3 specs, found 2"): - validate_report(self.path, LABEL, 3) + with self.assertRaisesRegex(ValueError, "expected 4 specs, found 2"): + validate_report(self.path, LABEL, 4) def test_rejects_missing_provider_coverage(self): data = report() diff --git a/hack/microsandbox/verify_report.py b/hack/microsandbox/verify_report.py index e6f8b90d99..ebd4e91718 100644 --- a/hack/microsandbox/verify_report.py +++ b/hack/microsandbox/verify_report.py @@ -56,13 +56,26 @@ def _select_specs(specs, expected_label, expected_count): return selected -def _validate_selected_specs(selected, expected_count): +def _validate_expected_count(expected_count, provider_mode): + if provider_mode not in ("parity", "external"): + raise ValueError(f"unsupported provider mode {provider_mode!r}") + if expected_count <= 0: + raise ValueError("expected spec count must be positive") + if provider_mode == "parity" and expected_count % 2: + raise ValueError("expected spec count must be even in parity mode") + + +def _validate_selected_specs(selected, expected_count, provider_mode): providers = {"microsandbox-builtin": 0, "microsandbox-external": 0} for index, spec in enumerate(selected, start=1): spec_labels = _spec_labels(spec) provider_labels = spec_labels & providers.keys() if len(provider_labels) != 1: raise ValueError(f"selected spec {index} must have exactly one provider label") + if provider_mode == "external" and provider_labels != {"microsandbox-external"}: + raise ValueError( + f"selected spec {index} must have exactly the microsandbox-external provider label" + ) providers[next(iter(provider_labels))] += 1 if spec.get("LeafNodeType") != "It": raise ValueError(f"selected report {index} is not an It spec") @@ -72,17 +85,20 @@ def _validate_selected_specs(selected, expected_count): raise ValueError( f"selected spec {index} used {spec.get('NumAttempts')!r} attempts, expected one" ) - if any(count != expected_count // 2 for count in providers.values()): + if provider_mode == "parity" and any( + count != expected_count // 2 for count in providers.values() + ): raise ValueError( f"provider label counts are {providers}, expected {expected_count // 2} each" ) return len(selected) -def validate_report(path, expected_label, expected_count): +def validate_report(path, expected_label, expected_count, provider_mode="parity"): + _validate_expected_count(expected_count, provider_mode) specs = _read_suite_specs(path) selected = _select_specs(specs, expected_label, expected_count) - return _validate_selected_specs(selected, expected_count) + return _validate_selected_specs(selected, expected_count, provider_mode) def main(argv=None): @@ -90,11 +106,14 @@ def main(argv=None): parser.add_argument("--report", required=True, type=Path) parser.add_argument("--label", required=True) parser.add_argument("--expected-count", required=True, type=int) + parser.add_argument("--provider-mode", choices=("parity", "external"), default="parity") args = parser.parse_args(argv) - if args.expected_count <= 0 or args.expected_count % 2: - parser.error("--expected-count must be a positive even number") try: - count = validate_report(args.report, args.label, args.expected_count) + _validate_expected_count(args.expected_count, args.provider_mode) + except ValueError as exc: + parser.error(str(exc)) + try: + count = validate_report(args.report, args.label, args.expected_count, args.provider_mode) except ValueError as exc: print(f"::error::{exc}", file=sys.stderr) return 1 diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx index 114107a1bb..4eb284b3c8 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx @@ -53,7 +53,7 @@ For SDK usage and development commands, see the [SDK README](https://github.com/ ## MicroSandbox parity gate -MicroSandbox integration suites share a provider matrix and test-session setup. Their dedicated CI job defines the suite catalogue and common runtime installation once, while keeping each suite on an independent runner. Every native run must report the exact expected count, split equally between built-in and external provider cases, with no skips or retries. Dry-run and programmatic-focus reports cannot satisfy this gate. See [the test architecture guide](https://github.com/devsy-org/devsy/blob/main/e2e/tests/up/MICROSANDBOX.md) for fixture ownership and extension rules. +MicroSandbox integration suites share a provider matrix and test-session setup. Their dedicated CI job defines the suite catalogue and common runtime installation once, while keeping each suite on an independent runner. Every native run must report the exact expected count, with no skips or retries. Parity rows require equal built-in and external provider counts; explicitly external-only contracts require every selected case to carry the external provider label. Dry-run and programmatic-focus reports cannot satisfy this gate. See [the test architecture guide](https://github.com/devsy-org/devsy/blob/main/e2e/tests/up/MICROSANDBOX.md) for fixture ownership and extension rules. The `up-provider-microsandbox` E2E label runs the same lifecycle and ownership scenario against the built-in provider and the external v0.1.8 release, each with isolated Devsy configuration. CI pins MicroSandbox v0.7.7 by checksum and requires access to KVM; unavailable virtualization fails this job instead of producing a passing skipped test. Each provider scenario has a ten-minute deadline and the CI job has a 25-minute deadline. @@ -155,6 +155,12 @@ DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsand For built-in Exec, cancellation closes non-file `io.ReadCloser` input whose `Close` unblocks `Read`; ordinary completion and caller-owned file input retain their ownership. Agent binary delivery owns its acquired source and closes it when the individual transfer attempt is canceled, before retrying with a fresh source. Cleanup joins cancellation callbacks before returning. +The Linux-only `up-provider-microsandbox-registry-consumer` label exercises the published external provider's Runtime Protocol `RunImage` path directly. Its positive case uses an owned TLS registry, fixture CA, restricted CONNECT proxy and private Docker credential helper, verifies the provider executable digest and exact authenticated manifest/blob response bytes, then checks a guest marker. Independent missing-CA and missing-credential cases require concrete refusal and no created VM. Each case owns fresh image and runtime state. This suite does not prove ordinary `up` image metadata resolution, remote agent binary preparation, or general egress denial. + +```sh +DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox-registry-consumer +``` + These scenarios do not establish complete parity. Runtime compatibility failures still require coverage before replacing the built-in provider. Green lifecycle, image, mount, resource, persistent-storage, ephemeral-root, TCP-egress, prebuild, Dockerless, request-invalid and managed-policy recreation, logs, and cancellation checks alone do not authorize that cutover. Dockerless refusal checks cover fresh creation, not preservation after an invalid recreation request. Egress coverage does not establish UDP, DNS, TLS, ingress, or live-policy modification behavior. Prebuild invalidation coverage changes the Dockerfile; it does not independently establish hashing of local Feature script contents. From 5f2466431ba3600ba4299aadd70169c34a7a8e4e Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 13 Oct 2026 18:52:20 -0600 Subject: [PATCH 2/8] test(runtime): prepare remote-agent binary reuse coverage --- ...rovider_microsandbox_remote_agent_linux.go | 465 ++++++++++++++++++ 1 file changed, 465 insertions(+) create mode 100644 e2e/tests/up/provider_microsandbox_remote_agent_linux.go diff --git a/e2e/tests/up/provider_microsandbox_remote_agent_linux.go b/e2e/tests/up/provider_microsandbox_remote_agent_linux.go new file mode 100644 index 0000000000..346de55ed7 --- /dev/null +++ b/e2e/tests/up/provider_microsandbox_remote_agent_linux.go @@ -0,0 +1,465 @@ +//go:build linux + +package up + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "time" + + "al.essio.dev/pkg/shellescape" + "github.com/devsy-org/devsy/e2e/framework" + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/types" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "sigs.k8s.io/yaml" +) + +type remoteAgentArtifactServer struct { + server *http.Server + listener net.Listener + done chan error + requests atomic.Int32 + offline atomic.Bool + runtime []byte + agent []byte +} + +const ( + remoteAgentDockerRunFlag = "run" + remoteAgentDockerMountFlag = "--mount" + remoteAgentDockerEnvFlag = "--env" +) + +var _ = ginkgo.Describe("published MicroSandbox provider remote agent binaries", + ginkgo.Label("up-provider-microsandbox-remote-agent-binaries", "microsandbox-external"), + ginkgo.It("downloads the external runtime on the remote agent and reuses its workspace copy", + func(ctx ginkgo.SpecContext) { + session := newMicrosandboxTestSession(ctx) + framework.ExpectNoError(requireRemoteAgentHostPrerequisites()) + + providerName := "case-a-remote-agent" + stageProviderName := "case-a-remote-stage" + session.addProvider(ctx, microsandboxExternalProvider, stageProviderName) + staged := prepareMicrosandboxConsumerProvider(ctx, stageProviderName) + runtimeBytes, err := os.ReadFile(staged.runtimeBinary) + framework.ExpectNoError(err) + runtimeRelativePath, err := remoteAgentRuntimeRelativePath(staged) + framework.ExpectNoError(err) + descriptor, err := provider.LoadProviderConfig(config.DefaultContext, stageProviderName) + framework.ExpectNoError(err) + + containerName := "devsy-" + microsandboxConsumerID("agent") + agentHome := filepath.Join( + "/tmp/devsy-remote-agent-fixture", + microsandboxConsumerID("home"), + "agent", + ) + bundleRoot, err := remoteAgentBundleRoot() + framework.ExpectNoError(err) + dockerPath, err := exec.LookPath("docker") + framework.ExpectNoError(err) + dockerPath, err = filepath.Abs(dockerPath) + framework.ExpectNoError(err) + preflightRemoteAgentContainer(ctx, bundleRoot, dockerPath, filepath.Dir(agentHome)) + + binaryName := descriptor.Agent.External.Binary + checksums := descriptor.Agent.Binaries[binaryName] + binaryEntry := remoteAgentLinuxBinary(checksums) + descriptorHash := strings.TrimPrefix(strings.ToLower(binaryEntry.Checksum), "sha256:") + gomega.Expect(descriptorHash).To(gomega.Equal(staged.runtimeHash)) + agentPath := filepath.Join( + session.framework.DevsyBinDir, + session.framework.DevsyBinName, + ) + server := startRemoteAgentArtifactServer(runtimeBytes, agentPath) + ginkgo.DeferCleanup(func(cleanupCtx ginkgo.SpecContext) error { + closeCtx, cancel := context.WithTimeout( + context.WithoutCancel(cleanupCtx), 10*time.Second, + ) + defer cancel() + return server.close(closeCtx) + }) + + descriptor.Name = providerName + descriptor.Agent.Local = types.StrBool("false") + descriptor.Agent.DataPath = agentHome + descriptor.Agent.DownloadURL = server.agentURL() + binaryEntry.Path = server.runtimeURL() + descriptor.Agent.Binaries[binaryName] = checksums + descriptor.Exec = remoteAgentCommands(containerName, bundleRoot, dockerPath, agentHome) + configPath := filepath.Join(ginkgo.GinkgoT().TempDir(), "provider.yaml") + configBytes, err := yaml.Marshal(descriptor) + framework.ExpectNoError(err) + framework.ExpectNoError(os.WriteFile(configPath, configBytes, 0o600)) + session.addProvider(ctx, configPath, providerName) + framework.ExpectNoError(session.framework.DevsyProviderUse(ctx, providerName)) + + workspace := session.newWorkspace() + ginkgo.DeferCleanup(func(cleanupCtx ginkgo.SpecContext) error { + boundedCtx, cancel := context.WithTimeout( + context.WithoutCancel(cleanupCtx), 20*time.Second, + ) + defer cancel() + return removeRemoteAgentContainer(boundedCtx, containerName) + }) + ginkgo.DeferCleanup(session.framework.CleanupWorkspace, workspace) + framework.ExpectNoError(session.framework.DevsyUp(ctx, workspace)) + gomega.Expect(server.requests.Load()).To(gomega.Equal(int32(1)), + "remote agent must fetch the published provider binary exactly once") + workspaceRecord, err := session.framework.FindWorkspace(ctx, workspace) + framework.ExpectNoError(err) + remoteBinaryPath := filepath.Join( + agentHome, "contexts", config.DefaultContext, "workspaces", workspaceRecord.ID, + "binaries", runtimeRelativePath, + ) + firstBinary := assertRemoteAgentBinary( + ctx, containerName, remoteBinaryPath, staged.runtimeHash, + ) + assertRemoteAgentGuest(ctx, session.framework, workspace) + + clearRemoteAgentBinaryCache(ctx, containerName, filepath.Dir(agentHome)) + framework.ExpectNoError(session.framework.DevsyStop(ctx, workspace)) + server.offline.Store(true) + framework.ExpectNoError(session.framework.DevsyUp(ctx, workspace)) + gomega.Expect(server.requests.Load()).To(gomega.Equal(int32(1)), + "existing workspace binary must be reused when its source returns 404") + secondBinary := assertRemoteAgentBinary( + ctx, containerName, remoteBinaryPath, staged.runtimeHash, + ) + gomega.Expect(secondBinary).To( + gomega.Equal(firstBinary), + "workspace binary identity must survive stop and restart", + ) + assertRemoteAgentGuest(ctx, session.framework, workspace) + }, ginkgo.SpecTimeout(framework.TimeoutLong()), + ), +) + +func remoteAgentRuntimeRelativePath(staged microsandboxConsumerProvider) (string, error) { + binaryRoot := filepath.Join(staged.origin, "binaries") + relative, err := filepath.Rel(binaryRoot, staged.runtimeBinary) + if err != nil { + return "", fmt.Errorf("resolve staged provider binary path: %w", err) + } + if relative == "." || filepath.IsAbs(relative) || relative == ".." || + strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", fmt.Errorf( + "staged provider binary path escapes its binaries directory: %q", relative, + ) + } + return relative, nil +} + +func remoteAgentLinuxBinary(entries []*provider.ProviderBinary) *provider.ProviderBinary { + for _, entry := range entries { + if entry.OS == runtime.GOOS && entry.Arch == runtime.GOARCH { + return entry + } + } + ginkgo.Fail("published MicroSandbox descriptor has no Linux/amd64 runtime binary") + return nil +} + +func startRemoteAgentArtifactServer( + runtimeBinary []byte, + agentPath string, +) *remoteAgentArtifactServer { + // #nosec G304 -- agentPath is built from Framework.DevsyBinDir and its configured binary name. + agentBinary, err := os.ReadFile(agentPath) + framework.ExpectNoError(err) + // #nosec G102 -- Docker bridge containers need the ephemeral host port; cleanup closes this test server. + listener, err := net.Listen("tcp", "0.0.0.0:0") + framework.ExpectNoError(err) + server := &remoteAgentArtifactServer{ + listener: listener, runtime: runtimeBinary, done: make(chan error, 1), + } + server.agent = agentBinary + mux := http.NewServeMux() + mux.HandleFunc("/agent/devsy-linux-amd64", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + _, _ = w.Write(server.agent) + }) + mux.HandleFunc("/provider/runtime", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + server.requests.Add(1) + if server.offline.Load() { + http.NotFound(w, r) + return + } + _, _ = w.Write(server.runtime) + }) + server.server = &http.Server{ + Handler: mux, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 15 * time.Second, + WriteTimeout: 3 * time.Minute, IdleTimeout: 30 * time.Second, + } + go func() { server.done <- server.server.Serve(listener) }() + return server +} + +func (s *remoteAgentArtifactServer) baseURL() string { + return "http://host.docker.internal:" + strings.TrimPrefix( + s.listener.Addr().String(), "0.0.0.0:", + ) +} + +func (s *remoteAgentArtifactServer) runtimeURL() string { return s.baseURL() + "/provider/runtime" } +func (s *remoteAgentArtifactServer) agentURL() string { return s.baseURL() + "/agent" } + +func (s *remoteAgentArtifactServer) close(ctx context.Context) error { + shutdownErr := s.server.Shutdown(ctx) + closeErr := s.server.Close() + select { + case serveErr := <-s.done: + if errors.Is(serveErr, http.ErrServerClosed) { + serveErr = nil + } + return errors.Join(shutdownErr, closeErr, serveErr) + case <-ctx.Done(): + return errors.Join(shutdownErr, closeErr, ctx.Err()) + } +} + +func remoteAgentCommands( + containerName, bundleRoot, dockerPath, agentHome string, +) provider.ProviderCommands { + devsyHome := filepath.Dir(agentHome) + create := remoteAgentContainerCreateCommand(containerName, bundleRoot, dockerPath, devsyHome) + return provider.ProviderCommands{ + Create: types.StrArray{ + create, + }, + Command: types.StrArray{ + fmt.Sprintf("docker exec -i %s sh -c \"${COMMAND}\"", containerName), + }, + Stop: types.StrArray{ + "docker stop " + shellescape.Quote(containerName), + }, + Start: types.StrArray{ + "docker start " + shellescape.Quote(containerName), + }, + Status: types.StrArray{ + fmt.Sprintf( + `state=$(docker inspect -f '{{.State.Status}}' %s 2>/dev/null) || { echo NOTFOUND; exit 0; } +case "$state" in + running) echo RUNNING ;; + exited) echo STOPPED ;; + *) echo BUSY ;; +esac`, + containerName, + ), + }, + Delete: types.StrArray{ + "docker rm -f " + shellescape.Quote(containerName), + }, + } +} + +const ( + remoteAgentContainerImage = "ubuntu:24.04" + remoteAgentContainerInit = `apt-get update && +apt-get install -y --no-install-recommends curl && +rm -rf /var/lib/apt/lists/* && +touch /tmp/devsy-agent-ready && +sleep infinity` + remoteAgentContainerReady = `test -e /tmp/devsy-agent-ready && +test "$(uname -s)" = Linux && +test "$(uname -m)" = x86_64 && +grep -Eq '^(ID=ubuntu|ID=debian)$' /etc/os-release && +test -c /dev/kvm && +test -r /dev/kvm && +test -w /dev/kvm && +command -v curl >/dev/null && +/opt/devsy-microsandbox/bin/msb --version && +/usr/local/bin/docker version >/dev/null` +) + +func remoteAgentContainerArgs(name, bundleRoot, dockerPath, devsyHome string) []string { + return []string{ + remoteAgentDockerRunFlag, + "-d", + "--name", + name, + "--add-host=host.docker.internal:host-gateway", + "--device=/dev/kvm:/dev/kvm", + remoteAgentDockerMountFlag, + "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock", + remoteAgentDockerMountFlag, + "type=bind,src=" + bundleRoot + ",dst=/opt/devsy-microsandbox,readonly", + remoteAgentDockerMountFlag, + "type=bind,src=" + dockerPath + ",dst=/usr/local/bin/docker,readonly", + remoteAgentDockerEnvFlag, + "HOME=/root", + remoteAgentDockerEnvFlag, + "DEVSY_HOME=" + devsyHome, + remoteAgentDockerEnvFlag, + "PATH=/opt/devsy-microsandbox/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + remoteAgentDockerEnvFlag, + "LD_LIBRARY_PATH=/opt/devsy-microsandbox/lib", + remoteAgentContainerImage, + "sh", + "-c", + remoteAgentContainerInit, + } +} + +func remoteAgentContainerCreateCommand(name, bundleRoot, dockerPath, devsyHome string) string { + args := remoteAgentContainerArgs(name, bundleRoot, dockerPath, devsyHome) + quotedArgs := make([]string, len(args)) + for i, arg := range args { + quotedArgs[i] = shellescape.Quote(arg) + } + waitForSetup := fmt.Sprintf(`i=0 +while [ "$i" -lt 180 ]; do + if docker exec %s test -e /tmp/devsy-agent-ready; then break; fi + i=$((i + 1)) + sleep 1 +done`, shellescape.Quote(name)) + checkReady := "docker exec " + shellescape.Quote(name) + + " sh -c " + shellescape.Quote(remoteAgentContainerReady) + return strings.Join([]string{ + "docker " + strings.Join(quotedArgs, " "), + "&& { " + waitForSetup + "; }", + "&& " + checkReady, + }, " ") +} + +func requireRemoteAgentHostPrerequisites() error { + if runtime.GOOS != osLinux || runtime.GOARCH != "amd64" { + return fmt.Errorf("remote MicroSandbox agent proof requires Linux/amd64") + } + if _, err := os.Stat("/dev/kvm"); err != nil { + return fmt.Errorf("remote MicroSandbox agent proof requires host /dev/kvm: %w", err) + } + if _, err := os.Stat("/var/run/docker.sock"); err != nil { + return fmt.Errorf("remote MicroSandbox agent proof requires Docker socket: %w", err) + } + if _, err := exec.LookPath("docker"); err != nil { + return err + } + return nil +} + +func remoteAgentBundleRoot() (string, error) { + msb, err := exec.LookPath("msb") + if err != nil { + return "", err + } + resolved, err := filepath.EvalSymlinks(msb) + if err != nil { + return "", err + } + if filepath.Base(filepath.Dir(resolved)) != "bin" || + filepath.Base(filepath.Dir(filepath.Dir(resolved))) != ".microsandbox" { + return "", fmt.Errorf("msb is not in the pinned .microsandbox bundle: %s", resolved) + } + return filepath.Dir(filepath.Dir(resolved)), nil +} + +func preflightRemoteAgentContainer(ctx context.Context, bundleRoot, dockerPath, devsyHome string) { + name := "devsy-agent-preflight-" + microsandboxConsumerID("container") + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if err := removeRemoteAgentContainer(cleanupCtx, name); err != nil { + ginkgo.Fail("remove remote agent preflight container: " + err.Error()) + } + }() + cmdCtx, cancel := context.WithTimeout(ctx, 4*time.Minute) + defer cancel() + args := remoteAgentContainerArgs(name, bundleRoot, dockerPath, devsyHome) + // #nosec G204 -- args are fixed by the builder and use test-owned paths. + output, err := exec.CommandContext(cmdCtx, "docker", args...).CombinedOutput() + framework.ExpectNoError(err, string(output)) + framework.ExpectNoError(waitRemoteAgentContainer(cmdCtx, dockerPath, name)) + // #nosec G204 -- script is fixed and name is a generated fixture nonce. + preflightOutput, err := exec.CommandContext( + cmdCtx, "docker", "exec", name, "sh", "-c", remoteAgentContainerReady, + ).CombinedOutput() + framework.ExpectNoError(err, "remote agent container prerequisites: "+string(preflightOutput)) +} + +func waitRemoteAgentContainer(ctx context.Context, dockerPath, name string) error { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + // #nosec G204 -- dockerPath is from exec.LookPath; arguments are fixed or nonce-generated. + cmd := exec.CommandContext( + ctx, dockerPath, "exec", name, "test", "-e", "/tmp/devsy-agent-ready", + ) + if err := cmd.Run(); err == nil { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("wait for remote agent container setup: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +func removeRemoteAgentContainer(ctx context.Context, name string) error { + // #nosec G204 -- docker is fixed; name is a nonce generated within this test. + cmd := exec.CommandContext(ctx, "docker", "rm", "-f", name) + output, err := cmd.CombinedOutput() + if err != nil && !strings.Contains(string(output), "No such container") { + return fmt.Errorf("remove remote agent container: %w: %s", err, output) + } + return nil +} + +func assertRemoteAgentBinary(ctx context.Context, containerName, path, expectedHash string) string { + command := fmt.Sprintf( + "test -x %s && stat -c '%%a %%i' %s && sha256sum %s", + shellescape.Quote(path), shellescape.Quote(path), shellescape.Quote(path), + ) + cmdCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + // #nosec G204 -- docker is fixed and command uses shell-quoted test-owned paths. + output, err := exec.CommandContext( + cmdCtx, "docker", "exec", containerName, "sh", "-c", command, + ).CombinedOutput() + framework.ExpectNoError(err, string(output)) + fields := strings.Fields(string(output)) + gomega.Expect(fields).To(gomega.HaveLen(4)) + gomega.Expect(fields[0]).To(gomega.Equal("755")) + gomega.Expect(fields[2]).To(gomega.Equal(expectedHash)) + return fields[1] +} + +func clearRemoteAgentBinaryCache(ctx context.Context, containerName, devsyHome string) { + cmdCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + cacheDir := filepath.Join(devsyHome, "cache", "providers") + command := "rm -rf " + shellescape.Quote(cacheDir) + + " && test ! -e " + shellescape.Quote(cacheDir) + // #nosec G204 -- command removes only the nonce-scoped private provider cache. + output, err := exec.CommandContext( + cmdCtx, "docker", "exec", containerName, "sh", "-c", command, + ).CombinedOutput() + framework.ExpectNoError(err, string(output)) +} + +func assertRemoteAgentGuest(ctx context.Context, f *framework.Framework, workspace string) { + marker := "remote-agent-" + microsandboxConsumerID("guest") + out, err := f.DevsySSHOnce(ctx, workspace, "printf %s "+shellescape.Quote(marker)) + framework.ExpectNoError(err) + gomega.Expect(strings.TrimSpace(out)).To(gomega.Equal(marker)) +} From dbfd4f54c53f18839ea11dc721fb59bee6e4053f Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 13 Oct 2026 19:01:19 -0600 Subject: [PATCH 3/8] test(runtime): isolate MicroSandbox consumer configuration --- e2e/tests/up/provider_microsandbox_consumer_linux.go | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/tests/up/provider_microsandbox_consumer_linux.go b/e2e/tests/up/provider_microsandbox_consumer_linux.go index 587bde8200..cd1c9adb5a 100644 --- a/e2e/tests/up/provider_microsandbox_consumer_linux.go +++ b/e2e/tests/up/provider_microsandbox_consumer_linux.go @@ -460,6 +460,7 @@ func setMicrosandboxConsumerEnvironment(environment microsandboxConsumerEnvironm for name, value := range map[string]string{ "HOME": home, "XDG_CONFIG_HOME": filepath.Join(home, ".config"), "DOCKER_CONFIG": filepath.Join(home, ".docker"), "MSB_HOME": msbHome, + "MSB_CONFIG_PATH": filepath.Join(msbHome, "config.json"), "CASE_A_CREDENTIAL_ROOT": helper.root, "CASE_A_CREDENTIAL_CONFIG": helper.config, "CASE_A_CREDENTIAL_OBSERVATIONS": helper.log, "CASE_A_OPERATION_ID": helper.operationID, "HTTPS_PROXY": fixture.ProxyURL, "HTTP_PROXY": fixture.ProxyURL, From a6380ade24603099df5247f85e919fad7aa4a03f Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 13 Oct 2026 21:24:41 -0600 Subject: [PATCH 4/8] test(runtime): capture real-provider startup measurements --- .github/workflows/pr-ci.yml | 75 +- e2e/tests/up/MICROSANDBOX.md | 21 + e2e/tests/up/provider_microsandbox.go | 8 +- .../up/provider_microsandbox_performance.go | 683 ++++++++++++++++++ .../provider_microsandbox_performance_test.go | 264 +++++++ .../developing-providers/runtime-protocol.mdx | 2 + 6 files changed, 1050 insertions(+), 3 deletions(-) create mode 100644 e2e/tests/up/provider_microsandbox_performance.go create mode 100644 e2e/tests/up/provider_microsandbox_performance_test.go diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 1d6dee5f93..a348bb05a6 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -133,6 +133,17 @@ jobs: go test -race -short -count=1 -v ./e2e/tests/up-docker-compose \ -run '^TestExpectedUIDMapping$' + - name: test runtime performance report checks + shell: bash + run: | + set -euo pipefail + tests="$(go test ./e2e/tests/up -list '^Test(ProbeExpectationsValidateSDKReports|ValidateSDKToolRejectsDifferentBuildSource|ParseStartupEvents)$')" + for test_name in TestProbeExpectationsValidateSDKReports TestValidateSDKToolRejectsDifferentBuildSource TestParseStartupEvents; do + grep -Fx "$test_name" <<< "$tests" + done + go test -race -count=1 ./e2e/tests/up \ + -run '^Test(ProbeExpectationsValidateSDKReports|ValidateSDKToolRejectsDifferentBuildSource|ParseStartupEvents)$' + can-read-secret: name: Can Read Secret runs-on: ubuntu-latest @@ -199,6 +210,31 @@ jobs: fi go build -trimpath -o "$OUT" ./hack/podman_ci + - name: build pinned runtime performance tools + if: runner.os == 'Linux' + shell: bash + env: + CGO_ENABLED: 0 + run: | + sdk_module=github.com/devsy-org/devsy-runtime-sdk + sdk_version="$(go list -m -f '{{if .Replace}}replaced{{else}}{{.Version}}{{end}}' "$sdk_module")" + case "$sdk_version" in + v*) ;; + *) echo "::error::runtime performance tools require an unreplaced SDK version"; exit 1 ;; + esac + export GOBIN="${RUNNER_TEMP}/runtime-performance-tools" + mkdir -p "$GOBIN" + go install -trimpath "$sdk_module/cmd/devsy-runtime-spawn-probe@$sdk_version" + go install -trimpath "$sdk_module/cmd/devsy-runtime-supervisor@$sdk_version" + printf '%s\n' "$sdk_version" > "$GOBIN/sdk-version" + + - name: upload runtime performance tools + if: runner.os == 'Linux' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runtime-performance-tools-linux + path: ${{ runner.temp }}/runtime-performance-tools/ + - name: upload Podman CI helper if: runner.os == 'Linux' || runner.os == 'Windows' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -386,6 +422,20 @@ jobs: name: e2e-test-linux path: ./e2e/ + - name: download runtime performance tools + if: matrix.label == 'up-provider-microsandbox' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runtime-performance-tools-linux + path: ${{ runner.temp }}/runtime-performance-tools/ + + - name: prepare runtime performance capture + if: matrix.label == 'up-provider-microsandbox' + run: | + chmod +x "${RUNNER_TEMP}/runtime-performance-tools/devsy-runtime-spawn-probe" \ + "${RUNNER_TEMP}/runtime-performance-tools/devsy-runtime-supervisor" + install -d -m 700 "${RUNNER_TEMP}/microsandbox-performance" + - name: setup executable shell: bash working-directory: ./e2e @@ -463,6 +513,17 @@ jobs: command=(../hack/run-managed-policy-e2e.sh "${command[@]}") fixture_environment=(DEVSY_REQUIRE_MSB_MANAGED_POLICY_FIXTURE=true "RUNNER_TEMP=${RUNNER_TEMP}") fi + performance_environment=() + if [ "${{ matrix.label }}" = "up-provider-microsandbox" ]; then + tools="${RUNNER_TEMP}/runtime-performance-tools" + performance_environment=( + "DEVSY_TEST_RUNTIME_SPAWN_PROBE=$tools/devsy-runtime-spawn-probe" + "DEVSY_TEST_RUNTIME_SUPERVISOR=$tools/devsy-runtime-supervisor" + "DEVSY_TEST_RUNTIME_PERFORMANCE_DIR=${RUNNER_TEMP}/microsandbox-performance" + "DEVSY_TEST_RUNTIME_REVISION=${{ github.sha }}" + "DEVSY_TEST_RUNTIME_SDK_VERSION=$(cat "$tools/sdk-version")" + ) + fi set +e sudo \ DEVSY_REQUIRE_MICROSANDBOX="${DEVSY_REQUIRE_MICROSANDBOX}" \ @@ -472,7 +533,7 @@ jobs: KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" \ ${DOCKER_HOST:+DOCKER_HOST="${DOCKER_HOST}"} \ PATH="${PATH}" \ - "${fixture_environment[@]}" "${command[@]}" 2>&1 | tee "$test_log" + "${fixture_environment[@]}" "${performance_environment[@]}" "${command[@]}" 2>&1 | tee "$test_log" pipeline_status=("${PIPESTATUS[@]}") set -e test_status=${pipeline_status[0]} @@ -481,6 +542,9 @@ jobs: if [ -e "$report" ]; then sudo chown "$(id -u):$(id -g)" "$report" || chown_status=$? fi + if [ "${{ matrix.label }}" = "up-provider-microsandbox" ]; then + sudo chown -R "$(id -u):$(id -g)" "${RUNNER_TEMP}/microsandbox-performance" || chown_status=$? + fi report_status=0 if [ "$chown_status" -ne 0 ]; then echo "::error::could not make Ginkgo report readable for artifact upload" @@ -510,6 +574,15 @@ jobs: if-no-files-found: warn retention-days: 14 + - name: upload runtime performance evidence + if: always() && matrix.label == 'up-provider-microsandbox' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: microsandbox-runtime-performance + path: ${{ runner.temp }}/microsandbox-performance/provider-performance.json + if-no-files-found: error + retention-days: 14 + integration-tests: name: Test ${{ matrix.label }}${{ (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && format(' ({0})', matrix.install-podman) || '' }} on ${{ matrix.runner }} needs: [changes, can-read-secret, build-cli] diff --git a/e2e/tests/up/MICROSANDBOX.md b/e2e/tests/up/MICROSANDBOX.md index 6c7a0d7f5f..c6e77f93d9 100644 --- a/e2e/tests/up/MICROSANDBOX.md +++ b/e2e/tests/up/MICROSANDBOX.md @@ -74,6 +74,27 @@ Its `TestMain` preserves the production runtime-supervisor dispatch and recogniz the helper only at its canonical, current-user-owned private path. Other test invocations retain the normal test runner path. +## Runtime startup measurements + +The existing external lifecycle case optionally captures its ordinary fresh and +stop/resume `up` commands when CI supplies the runtime performance tools. The +built-in case retains its existing path. No new suite, VM, deadline or latency +threshold is added. After both commands succeed, the helper stages the published +runtime from `Agent.Binaries`, verifies its manifest checksum and compares direct +and supervised Info/Find launches in direct/supervised/supervised/direct order. +Each run records one first launch and 100 warm samples with process-reaping checks. +The tools are built from the checkout's pinned Runtime SDK version and their build +metadata and selected supervisor digest are verified before accepting reports. + +The retained JSON includes the provider source, runtime digest, checkout revision, +SDK version, command wall time and filtered startup-event durations. Startup events +measure launch/handshake/dispense before RPC; they exclude binary resolution, RPC +and reaping and must not be described as total plugin overhead. The comparison +runs after both ordinary `up` captures so it does not prewarm their provider path. +It measures warm host caches, not a cold-cache or end-to-end speedup guarantee. +Raw probe stderr and environment values are not included in the performance artifact. +Normal suites remain inactive when these optional CI paths are unset. + ## Managed policy boundary Managed-policy recreation is Linux-only and runs through diff --git a/e2e/tests/up/provider_microsandbox.go b/e2e/tests/up/provider_microsandbox.go index bcce747670..bb691d26bc 100644 --- a/e2e/tests/up/provider_microsandbox.go +++ b/e2e/tests/up/provider_microsandbox.go @@ -50,8 +50,11 @@ var _ = ginkgo.Describe( }) ginkgo.DeferCleanup(f.CleanupWorkspace, tempDir) + performance, err := newMicrosandboxPerformanceCapture(source, name, f) + framework.ExpectNoError(err) + // full up: boots the microVM, streams in the agent, opens the tunnel - err = f.DevsyUp(ctx, tempDir, "--devcontainer", ".devcontainer.json") + err = performance.up(ctx, tempDir, "fresh", "--devcontainer", ".devcontainer.json") framework.ExpectNoError(err) // the workspace is reachable over SSH @@ -96,10 +99,11 @@ var _ = ginkgo.Describe( assertMicrosandboxHostEntries(ctx, f, tempDir, "fresh") err = f.DevsyWorkspaceStop(ctx, tempDir) framework.ExpectNoError(err) - err = f.DevsyUp(ctx, tempDir) + err = performance.up(ctx, tempDir, "reused") framework.ExpectNoError(err) gomega.Expect(microsandboxCreationTime(ctx, sandbox)).To(gomega.Equal(createdAt)) assertMicrosandboxHostEntries(ctx, f, tempDir, "restarted") + framework.ExpectNoError(performance.compare(ctx)) err = f.DevsyUpRecreate(ctx, tempDir) framework.ExpectNoError(err) gomega.Expect(microsandboxCreationTime(ctx, sandbox)).NotTo(gomega.Equal(createdAt)) diff --git a/e2e/tests/up/provider_microsandbox_performance.go b/e2e/tests/up/provider_microsandbox_performance.go new file mode 100644 index 0000000000..1f8be9170f --- /dev/null +++ b/e2e/tests/up/provider_microsandbox_performance.go @@ -0,0 +1,683 @@ +package up + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "debug/buildinfo" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/docker" + "github.com/devsy-org/devsy/pkg/provider" +) + +const ( + spawnProbeEnv = "DEVSY_TEST_RUNTIME_SPAWN_PROBE" + supervisorEnv = "DEVSY_TEST_RUNTIME_SUPERVISOR" + performanceDir = "DEVSY_TEST_RUNTIME_PERFORMANCE_DIR" + revisionEnv = "DEVSY_TEST_RUNTIME_REVISION" + sdkVersionEnv = "DEVSY_TEST_RUNTIME_SDK_VERSION" + performanceFile = "provider-performance.json" + runtimeSDKModule = "github.com/devsy-org/devsy-runtime-sdk" + probeModeDirect = "direct" + probeModeSupervised = "supervised" + performanceUpFresh = "fresh" + performanceUpReused = "reused" +) + +var startupEvent = regexp.MustCompile(`External runtime ([^ ]+) startup completed in ([^\s]+)`) + +type microsandboxPerformanceCapture struct { + active bool + probe string + supervisor string + directory string + config *provider.ProviderConfig + framework *framework.Framework + workspace *provider.Workspace + options map[string]config.OptionValue + upRuns []upPerformanceRun + probeRuns []probePerformanceRun + expectedSHA string + supervisorSHA string + revision string + sdkVersion string +} + +type performanceToolPaths struct { + probe, supervisor, directory, revision, sdkVersion string +} + +type performanceSDKTool struct { + name, path, packagePath string +} + +type stagedPerformanceRuntime struct { + name, path, checksum string +} + +type probeSequence struct { + environment []string + runtime stagedPerformanceRuntime + workspaceID string +} + +type probeInvocation struct { + sequence probeSequence + order int + mode string +} + +type upPerformanceRun struct { + Phase string `json:"phase"` + Succeeded bool `json:"succeeded"` + WallTimeNS int64 `json:"wall_time_ns"` + Startup []startupPerformance `json:"startup_events"` +} + +type startupPerformance struct { + Operation string `json:"operation"` + DurationNS int64 `json:"duration_ns"` +} + +type probePerformanceRun struct { + Order int `json:"order"` + Mode string `json:"launch_mode"` + Report json.RawMessage `json:"report"` +} + +type microsandboxPerformanceReport struct { + ProviderSource string `json:"provider_source"` + ExpectedBinarySHA256 string `json:"expected_binary_sha256"` + TestedCheckoutRevision string `json:"tested_checkout_revision"` + RuntimeSDKVersion string `json:"runtime_sdk_version"` + StartupMeasurement string `json:"startup_measurement"` + UpRuns []upPerformanceRun `json:"up_runs"` + ProbeRuns []probePerformanceRun `json:"probe_runs"` +} + +// newMicrosandboxPerformanceCapture enables the optional measurements only for +// the published external provider and only when all native-job paths exist. +func newMicrosandboxPerformanceCapture( + source, providerName string, + f *framework.Framework, +) (*microsandboxPerformanceCapture, error) { + capture := µsandboxPerformanceCapture{framework: f} + paths, active, err := configuredPerformanceTools(source) + if err != nil || !active { + return capture, err + } + config, err := installedExternalPerformanceProvider(providerName, source) + if err != nil { + return nil, err + } + capture.active = true + capture.probe = paths.probe + capture.supervisor = paths.supervisor + capture.directory = paths.directory + capture.config = config + capture.revision = paths.revision + capture.sdkVersion = paths.sdkVersion + capture.supervisorSHA, err = fileSHA256(paths.supervisor) + if err != nil { + return nil, fmt.Errorf("hash SDK runtime supervisor: %w", err) + } + return capture, nil +} + +func configuredPerformanceTools(source string) (performanceToolPaths, bool, error) { + if source != microsandboxExternalProvider { + return performanceToolPaths{}, false, nil + } + paths := performanceToolPaths{ + probe: os.Getenv(spawnProbeEnv), supervisor: os.Getenv(supervisorEnv), + directory: os.Getenv(performanceDir), revision: os.Getenv(revisionEnv), + sdkVersion: os.Getenv(sdkVersionEnv), + } + set := 0 + for _, value := range []string{ + paths.probe, paths.supervisor, paths.directory, paths.revision, paths.sdkVersion, + } { + if value != "" { + set++ + } + } + if set == 0 { + return performanceToolPaths{}, false, nil + } + if set != 5 { + return performanceToolPaths{}, false, errors.New( + "MicroSandbox performance tools are only partially configured", + ) + } + if runtime.GOOS != osLinux { + return performanceToolPaths{}, false, errors.New( + "MicroSandbox real-provider performance capture requires Linux", + ) + } + if err := validatePerformanceToolPaths(paths); err != nil { + return performanceToolPaths{}, false, err + } + return paths, true, nil +} + +func validatePerformanceToolPaths(paths performanceToolPaths) error { + if err := validatePerformanceSDKTools(paths); err != nil { + return err + } + return validatePerformanceReportDirectory(paths.directory) +} + +func validatePerformanceSDKTools(paths performanceToolPaths) error { + for _, tool := range []performanceSDKTool{ + {spawnProbeEnv, paths.probe, runtimeSDKModule + "/cmd/devsy-runtime-spawn-probe"}, + {supervisorEnv, paths.supervisor, runtimeSDKModule + "/cmd/devsy-runtime-supervisor"}, + } { + if err := validatePerformanceSDKTool(tool, paths.sdkVersion); err != nil { + return err + } + } + return nil +} + +func validatePerformanceSDKTool(tool performanceSDKTool, version string) error { + if !filepath.IsAbs(tool.path) { + return fmt.Errorf("%s must be an absolute path", tool.name) + } + info, err := os.Stat(tool.path) + if err != nil { + return fmt.Errorf("%s is unavailable: %w", tool.name, err) + } + if !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 { + return fmt.Errorf("%s must name an executable file", tool.name) + } + return validateSDKTool(tool.path, tool.packagePath, version) +} + +func validatePerformanceReportDirectory(directory string) error { + if !filepath.IsAbs(directory) { + return errors.New("performance report directory must be absolute") + } + // #nosec G703 -- this absolute report path is supplied by the trusted native CI job. + dirInfo, err := os.Stat(directory) + if err != nil { + return fmt.Errorf("performance report directory is unavailable: %w", err) + } + if !dirInfo.IsDir() { + return errors.New("performance report path is not a directory") + } + return nil +} + +func installedExternalPerformanceProvider( + providerName, source string, +) (*provider.ProviderConfig, error) { + installed, err := provider.LoadProviderConfig(config.DefaultContext, providerName) + if err != nil { + return nil, fmt.Errorf("load installed MicroSandbox provider configuration: %w", err) + } + if installed.Source.Raw != source || installed.Agent.Driver != provider.ExternalDriver || + installed.Agent.External.Binary == "" { + return nil, errors.New( + "installed provider does not match the expected external MicroSandbox source", + ) + } + return installed, nil +} + +func validateSDKTool(path, packagePath, version string) error { + info, err := buildinfo.ReadFile(path) + if err != nil { + return fmt.Errorf("read runtime probe tool build information: %w", err) + } + if info.Path != packagePath || info.Main.Path != runtimeSDKModule || + info.Main.Version != version || + info.Main.Replace != nil { + return fmt.Errorf("runtime probe tool is not the expected unmodified SDK %s build", version) + } + return nil +} + +func (c *microsandboxPerformanceCapture) up( + ctx context.Context, + workspace, phase string, + additionalArgs ...string, +) error { + if !c.active { + args := append([]string{workspace}, additionalArgs...) + return c.framework.DevsyUp(ctx, args...) + } + if err := c.validateUpPhase(phase); err != nil { + return err + } + _, err := c.captureUp(ctx, workspace, phase, additionalArgs...) + if err != nil { + return err + } + if phase == performanceUpFresh { + return c.captureFreshWorkspace(ctx, workspace) + } + return nil +} + +func (c *microsandboxPerformanceCapture) validateUpPhase(phase string) error { + if phase != performanceUpFresh && phase != performanceUpReused { + return fmt.Errorf("unsupported MicroSandbox performance up phase %q", phase) + } + if phase == performanceUpFresh && len(c.upRuns) == 0 { + return nil + } + if phase == performanceUpReused && len(c.upRuns) == 1 && + c.upRuns[0].Phase == performanceUpFresh { + return nil + } + return fmt.Errorf("MicroSandbox performance up phase %q is out of order", phase) +} + +func (c *microsandboxPerformanceCapture) captureUp( + ctx context.Context, + workspace, phase string, + additionalArgs ...string, +) (upPerformanceRun, error) { + started := time.Now() + stdout, stderr, err := c.framework.DevsyUpStreams(ctx, workspace, additionalArgs...) + run := upPerformanceRun{ + Phase: phase, + Succeeded: err == nil, + WallTimeNS: time.Since(started).Nanoseconds(), + } + run.Startup = append(run.Startup, parseStartupEvents(stdout)...) + run.Startup = append(run.Startup, parseStartupEvents(stderr)...) + c.upRuns = append(c.upRuns, run) + if writeErr := c.writeReport(); writeErr != nil { + return run, errors.Join( + err, + fmt.Errorf("write MicroSandbox performance report: %w", writeErr), + ) + } + if err != nil { + return run, err + } + if len(run.Startup) == 0 { + return run, errors.New( + "successful MicroSandbox up emitted no valid external runtime startup timing events", + ) + } + return run, nil +} + +func (c *microsandboxPerformanceCapture) captureFreshWorkspace( + ctx context.Context, + workspace string, +) error { + var err error + c.workspace, err = c.framework.FindWorkspace(ctx, workspace) + if err != nil { + return fmt.Errorf("find fresh MicroSandbox workspace for runtime probe: %w", err) + } + devsyConfig, err := config.LoadConfig("", "") + if err != nil { + return fmt.Errorf("load options for runtime probe environment: %w", err) + } + c.options = provider.GetProviderOptions(c.workspace, nil, devsyConfig) + return nil +} + +func parseStartupEvents(output string) []startupPerformance { + var events []startupPerformance + for line := range strings.SplitSeq(output, "\n") { + match := startupEvent.FindStringSubmatch(line) + if len(match) != 3 { + continue + } + duration, err := time.ParseDuration(match[2]) + if err == nil && duration > 0 { + events = append( + events, + startupPerformance{Operation: match[1], DurationNS: duration.Nanoseconds()}, + ) + } + } + return events +} + +func (c *microsandboxPerformanceCapture) compare(ctx context.Context) (retErr error) { + if !c.active { + return nil + } + if !c.capturesReady() { + return errors.New( + "runtime comparison requires successful fresh and reused provider up captures", + ) + } + stage, err := os.MkdirTemp("", "devsy-runtime-performance-") + if err != nil { + return fmt.Errorf("create private runtime staging directory: %w", err) + } + defer func() { + if err := os.RemoveAll(stage); err != nil { + retErr = errors.Join( + retErr, + fmt.Errorf("remove private runtime staging directory: %w", err), + ) + } + }() + return c.compareWithStage(ctx, stage) +} + +func (c *microsandboxPerformanceCapture) capturesReady() bool { + return len(c.upRuns) == 2 && c.upRuns[0].Phase == performanceUpFresh && + c.upRuns[0].Succeeded && c.upRuns[1].Phase == performanceUpReused && + c.upRuns[1].Succeeded && c.workspace != nil && c.options != nil +} + +func (c *microsandboxPerformanceCapture) compareWithStage(ctx context.Context, stage string) error { + paths, err := provider.DownloadBinaries(ctx, c.config.Agent.Binaries, stage) + if err != nil { + return fmt.Errorf("stage published external runtime from Agent.Binaries: %w", err) + } + staged, err := c.validateStagedRuntime(paths) + if err != nil { + return err + } + c.expectedSHA = staged.checksum + workspaceID, err := absentWorkspaceID() + if err != nil { + return err + } + probeEnv := provider.ToEnvironment( + c.workspace, + nil, + c.options, + map[string]string{staged.name: staged.path}, + ) + return c.runProbeSequence(ctx, probeSequence{ + environment: probeEnv, runtime: staged, workspaceID: workspaceID, + }) +} + +func (c *microsandboxPerformanceCapture) validateStagedRuntime( + paths map[string]string, +) (stagedPerformanceRuntime, error) { + binaryName := c.config.Agent.External.Binary + binaryPath := paths[binaryName] + if binaryPath == "" || !filepath.IsAbs(binaryPath) { + return stagedPerformanceRuntime{}, errors.New( + "published external runtime did not resolve to an absolute staged binary", + ) + } + if err := validateStagedExecutable(binaryPath); err != nil { + return stagedPerformanceRuntime{}, err + } + expected, err := selectedRuntimeChecksum(c.config, binaryName) + if err != nil { + return stagedPerformanceRuntime{}, err + } + actual, err := fileSHA256(binaryPath) + if err != nil { + return stagedPerformanceRuntime{}, fmt.Errorf("hash staged external runtime: %w", err) + } + if actual != expected { + return stagedPerformanceRuntime{}, fmt.Errorf( + "staged external runtime checksum differs from provider manifest: got %s", + actual, + ) + } + return stagedPerformanceRuntime{name: binaryName, path: binaryPath, checksum: expected}, nil +} + +func validateStagedExecutable(path string) error { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("stat staged external runtime: %w", err) + } + if info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0 { + return nil + } + return errors.New("staged external runtime is not an executable regular file") +} + +func (c *microsandboxPerformanceCapture) runProbeSequence( + ctx context.Context, + sequence probeSequence, +) error { + for index, mode := range []string{ + probeModeDirect, probeModeSupervised, probeModeSupervised, probeModeDirect, + } { + invocation := probeInvocation{sequence: sequence, order: index + 1, mode: mode} + if err := c.runProbe(ctx, invocation); err != nil { + return err + } + } + return nil +} + +func (c *microsandboxPerformanceCapture) runProbe( + ctx context.Context, + invocation probeInvocation, +) error { + sequence := invocation.sequence + args := []string{ + "--binary", sequence.runtime.path, "--samples", "100", "--timeout", "15s", + "--workspace-id", sequence.workspaceID, "--revision", c.revision, + } + if invocation.mode == probeModeSupervised { + args = append(args, "--supervisor-binary", c.supervisor) + } + args = append(args, "--") + args = append(args, c.config.Agent.External.Args...) + // #nosec G204 -- SDK tools and runtime checksum are verified; arguments are fixed flags plus manifest argv. + command := exec.CommandContext(ctx, c.probe, args...) + docker.PrepareForGroupCancellation(command) + command.WaitDelay = 5 * time.Second + command.Env = sequence.environment + var stdout strings.Builder + command.Stdout = &stdout + command.Stderr = io.Discard + if err := command.Run(); err != nil { + return fmt.Errorf( + "runtime spawn probe run %d (%s) failed: %w", invocation.order, invocation.mode, err, + ) + } + raw := []byte(stdout.String()) + expectation := probeExpectations{ + mode: invocation.mode, binarySHA: sequence.runtime.checksum, + supervisorSHA: c.supervisorSHA, revision: c.revision, + } + if err := expectation.validate(raw); err != nil { + return fmt.Errorf( + "runtime spawn probe run %d (%s): %w", invocation.order, invocation.mode, err, + ) + } + c.probeRuns = append( + c.probeRuns, + probePerformanceRun{ + Order: invocation.order, Mode: invocation.mode, Report: json.RawMessage(raw), + }, + ) + if err := c.writeReport(); err != nil { + return fmt.Errorf("write MicroSandbox performance report: %w", err) + } + return nil +} + +func selectedRuntimeChecksum(config *provider.ProviderConfig, name string) (string, error) { + for _, binary := range config.Agent.Binaries[name] { + if binary == nil { + continue + } + if binary.OS == runtime.GOOS && binary.Arch == runtime.GOARCH { + checksum := strings.TrimPrefix(strings.ToLower(binary.Checksum), "sha256:") + decoded, err := hex.DecodeString(checksum) + if err != nil || len(decoded) != sha256.Size { + return "", errors.New("provider runtime manifest has an invalid SHA-256 checksum") + } + return checksum, nil + } + } + return "", fmt.Errorf( + "provider runtime manifest has no %s/%s entry", + runtime.GOOS, + runtime.GOARCH, + ) +} + +type runtimeProbeSample struct { + Reaped bool `json:"reaped"` +} + +type runtimeProbeMeasurements struct { + Operation string `json:"operation"` + First runtimeProbeSample `json:"first_launch"` + Warm []runtimeProbeSample `json:"warm_samples"` +} + +type runtimeProbeReport struct { + SchemaVersion int `json:"schema_version"` + LaunchMode string `json:"launch_mode"` + SupervisorSHA256 string `json:"supervisor_sha256"` + GOOS string `json:"goos"` + GOARCH string `json:"goarch"` + GoVersion string `json:"go_version"` + Revision string `json:"revision"` + BinarySHA256 string `json:"binary_sha256"` + ColdCacheMeasured *bool `json:"cold_cache_measured"` + Info runtimeProbeMeasurements `json:"info"` + Find runtimeProbeMeasurements `json:"find"` +} + +type probeExpectations struct { + mode, binarySHA, supervisorSHA, revision string +} + +func (expected probeExpectations) validate(raw []byte) error { + var report runtimeProbeReport + if err := json.Unmarshal(raw, &report); err != nil { + return fmt.Errorf("decode probe JSON: %w", err) + } + if err := report.validateMetadata(expected); err != nil { + return err + } + if err := validateMeasurements("Info", report.Info); err != nil { + return err + } + return validateMeasurements("Find", report.Find) +} + +func (report runtimeProbeReport) validateMetadata(expected probeExpectations) error { + if err := report.validateLaunch(expected); err != nil { + return err + } + if err := report.validatePlatform(expected); err != nil { + return err + } + return report.validateSupervisor(expected) +} + +func (report runtimeProbeReport) validateLaunch(expected probeExpectations) error { + if expected.mode != probeModeDirect && expected.mode != probeModeSupervised { + return errors.New("probe launch mode is unsupported") + } + if report.SchemaVersion != 1 || report.LaunchMode != expected.mode || + report.BinarySHA256 != expected.binarySHA || report.ColdCacheMeasured == nil || + *report.ColdCacheMeasured { + return errors.New( + "probe report does not match requested launch mode, expected binary, or cache claim", + ) + } + return nil +} + +func (report runtimeProbeReport) validatePlatform(expected probeExpectations) error { + if report.GOOS == runtime.GOOS && report.GOARCH == runtime.GOARCH && + report.GoVersion != "" && report.Revision == expected.revision { + return nil + } + return errors.New( + "probe report platform or tested checkout revision does not match the current run", + ) +} + +func (report runtimeProbeReport) validateSupervisor(expected probeExpectations) error { + wantSupervisorSHA := "" + if expected.mode == probeModeSupervised { + wantSupervisorSHA = expected.supervisorSHA + } + if report.SupervisorSHA256 != wantSupervisorSHA { + return errors.New( + "probe report supervisor checksum does not match the selected launch mode", + ) + } + return nil +} + +func validateMeasurements(label string, measurement runtimeProbeMeasurements) error { + if measurement.Operation != label || !measurement.First.Reaped || len(measurement.Warm) != 100 { + return fmt.Errorf( + "%s report is missing the expected first launch or 100 warm samples", + label, + ) + } + for _, sample := range measurement.Warm { + if !sample.Reaped { + return fmt.Errorf("%s report includes a process that was not reaped", label) + } + } + return nil +} + +func (c *microsandboxPerformanceCapture) writeReport() error { + report := microsandboxPerformanceReport{ + ProviderSource: microsandboxExternalProvider, + ExpectedBinarySHA256: c.expectedSHA, + TestedCheckoutRevision: c.revision, + RuntimeSDKVersion: c.sdkVersion, + StartupMeasurement: "external runtime startup event; excludes binary resolution, RPC, and process reaping", + UpRuns: c.upRuns, + ProbeRuns: c.probeRuns, + } + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(c.directory, performanceFile), append(data, '\n'), 0o600) +} + +func fileSHA256(path string) (string, error) { + // #nosec G304 -- caller selects only the validated SDK supervisor or downloaded manifest binary in private staging. + // #nosec G703 -- absolute SDK path is CI-validated; runtime path is the manifest download in private staging. + file, err := os.Open(path) + if err != nil { + return "", err + } + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + _ = file.Close() + return "", err + } + if err := file.Close(); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func absentWorkspaceID() (string, error) { + var suffix [16]byte + if _, err := rand.Read(suffix[:]); err != nil { + return "", fmt.Errorf("generate absent workspace ID: %w", err) + } + return "runtime-probe-" + hex.EncodeToString(suffix[:]), nil +} diff --git a/e2e/tests/up/provider_microsandbox_performance_test.go b/e2e/tests/up/provider_microsandbox_performance_test.go new file mode 100644 index 0000000000..1e87a66bbd --- /dev/null +++ b/e2e/tests/up/provider_microsandbox_performance_test.go @@ -0,0 +1,264 @@ +package up + +import ( + "encoding/json" + "os" + "runtime" + "strings" + "testing" + "time" +) + +type probeValidationCase struct { + name string + mode string + supervisor string + mutate func(map[string]any) + malformed bool + wantError string +} + +func TestProbeExpectationsValidateSDKReports(t *testing.T) { + runProbeValidationCases(t, append( + append(probeMetadataCases(), probeOperationCases()...), probeProcessCases()..., + )) +} + +func probeMetadataCases() []probeValidationCase { + return append(probeIdentityCases(), probeCacheCases()...) +} + +func probeIdentityCases() []probeValidationCase { + return []probeValidationCase{ + {name: "valid direct report", mode: probeModeDirect}, + {name: "valid supervised report", mode: probeModeSupervised, supervisor: "supervisor-hash"}, + { + name: "malformed JSON", + mode: probeModeDirect, + malformed: true, + wantError: "decode probe JSON", + }, + { + name: "wrong runtime checksum", + mode: probeModeDirect, + mutate: func(r map[string]any) { r["binary_sha256"] = "other-hash" }, + wantError: "expected binary", + }, + { + name: "wrong launch mode", + mode: probeModeDirect, + mutate: func(r map[string]any) { r["launch_mode"] = probeModeSupervised }, + wantError: "launch mode", + }, + { + name: "unsupported expected launch mode", + mode: "other", + wantError: "launch mode is unsupported", + }, + { + name: "wrong platform", mode: probeModeDirect, + mutate: func(r map[string]any) { r["goos"] = "other" }, wantError: "platform", + }, + { + name: "missing Go version", mode: probeModeDirect, + mutate: func(r map[string]any) { r["go_version"] = "" }, wantError: "platform", + }, + { + name: "wrong supervised helper hash", + mode: probeModeSupervised, + supervisor: "supervisor-hash", + mutate: func(r map[string]any) { r["supervisor_sha256"] = "other-helper" }, + wantError: "supervisor checksum", + }, + { + name: "direct report with helper hash", + mode: probeModeDirect, + mutate: func(r map[string]any) { r["supervisor_sha256"] = "unexpected-helper" }, + wantError: "supervisor checksum", + }, + } +} + +func probeCacheCases() []probeValidationCase { + return []probeValidationCase{ + { + name: "missing cache field", + mode: probeModeDirect, + mutate: func(r map[string]any) { delete(r, "cold_cache_measured") }, + wantError: "cache claim", + }, + { + name: "true cache flag", + mode: probeModeDirect, + mutate: func(r map[string]any) { r["cold_cache_measured"] = true }, + wantError: "cache claim", + }, + { + name: "ill-typed cache flag", + mode: probeModeDirect, + mutate: func(r map[string]any) { r["cold_cache_measured"] = "false" }, + wantError: "decode probe JSON", + }, + { + name: "wrong tested revision", + mode: probeModeDirect, + mutate: func(r map[string]any) { r["revision"] = "different-revision" }, + wantError: "revision", + }, + } +} + +func probeOperationCases() []probeValidationCase { + return []probeValidationCase{ + { + name: "wrong Info operation", + mode: probeModeDirect, + mutate: func(r map[string]any) { probeSection(r, "info")["operation"] = "RunImage" }, + wantError: "Info report", + }, + { + name: "wrong warm count", + mode: probeModeDirect, + mutate: func(r map[string]any) { probeSection(r, "info")["warm_samples"] = []any{} }, + wantError: "Info report", + }, + } +} + +func probeProcessCases() []probeValidationCase { + return []probeValidationCase{ + { + name: "unreaped first launch", + mode: probeModeDirect, + mutate: func(r map[string]any) { probeSection(r, "find")["first_launch"].(map[string]any)["reaped"] = false }, + wantError: "Find report", + }, + { + name: "unreaped warm sample", mode: probeModeDirect, + mutate: func(r map[string]any) { + probeSection(r, "find")["warm_samples"].([]any)[0].(map[string]any)["reaped"] = false + }, + wantError: "process that was not reaped", + }, + } +} + +func runProbeValidationCases(t *testing.T, tests []probeValidationCase) { + t.Helper() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { runProbeValidationCase(t, test) }) + } +} + +func runProbeValidationCase(t *testing.T, test probeValidationCase) { + t.Helper() + raw, err := test.reportJSON() + if err != nil { + t.Fatal(err) + } + wantSupervisor := test.supervisorHash() + expectation := probeExpectations{ + mode: test.mode, binarySHA: "runtime-hash", supervisorSHA: wantSupervisor, + revision: "tested-revision", + } + assertProbeError(t, expectation.validate(raw), test.wantError) +} + +func (test probeValidationCase) reportJSON() ([]byte, error) { + if test.malformed { + return []byte("{"), nil + } + report := validProbeReport(test.mode, test.supervisor) + if test.mutate != nil { + test.mutate(report) + } + return json.Marshal(report) +} + +func (test probeValidationCase) supervisorHash() string { + if test.mode == probeModeSupervised { + return test.supervisor + } + return "" +} + +func assertProbeError(t *testing.T, err error, expected string) { + t.Helper() + if expected == "" { + if err != nil { + t.Fatalf("expected a valid probe report, got %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), expected) { + t.Fatalf("expected error containing %q, got %v", expected, err) + } +} + +func TestValidateSDKToolRejectsDifferentBuildSource(t *testing.T) { + path, err := os.Executable() + if err != nil { + t.Fatal(err) + } + err = validateSDKTool( + path, + runtimeSDKModule+"/cmd/devsy-runtime-spawn-probe", + "v1.6.0", + ) + if err == nil { + t.Fatal("expected the test binary to fail SDK probe provenance validation") + } +} + +func TestParseStartupEvents(t *testing.T) { + output := strings.Join([]string{ + "DEBU External runtime Info startup completed in 4.2ms", + "DEBU External runtime Find startup completed in 1s", + "DEBU External runtime Info startup completed in someday", + "DEBU External runtime Info startup completed in 0s", + "DEBU External runtime Info startup completed in -3ms", + }, "\n") + + events := parseStartupEvents(output) + if len(events) != 2 { + t.Fatalf("expected two positive startup timings, got %#v", events) + } + if events[0].Operation != "Info" || events[0].DurationNS != int64(4200*time.Microsecond) { + t.Errorf("unexpected first startup event: %#v", events[0]) + } + if events[1].Operation != "Find" || events[1].DurationNS != int64(time.Second) { + t.Errorf("unexpected second startup event: %#v", events[1]) + } +} + +func validProbeReport(mode, supervisorSHA string) map[string]any { + return map[string]any{ + "schema_version": 1, + "launch_mode": mode, + "supervisor_sha256": supervisorSHA, + "goos": runtime.GOOS, + "goarch": runtime.GOARCH, + "go_version": "go1.26.0", + "revision": "tested-revision", + "binary_sha256": "runtime-hash", + "cold_cache_measured": false, + "info": validMeasurement("Info"), + "find": validMeasurement("Find"), + } +} + +func validMeasurement(operation string) map[string]any { + warm := make([]any, 100) + for index := range warm { + warm[index] = map[string]any{"reaped": true} + } + return map[string]any{ + "operation": operation, + "first_launch": map[string]any{"reaped": true}, + "warm_samples": warm, + } +} + +func probeSection(report map[string]any, name string) map[string]any { + return report[name].(map[string]any) +} diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx index d05e8a488a..2e64048eae 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx @@ -167,6 +167,8 @@ The Linux-only `up-provider-microsandbox-registry-consumer` label exercises the DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox-registry-consumer ``` +The existing external lifecycle case also captures ordinary fresh and reused `up` wall time and filtered runtime startup events when CI supplies the pinned SDK performance tools. After both commands pass, a checksum-verified copy of the published runtime runs direct/supervised/supervised/direct Info/Find comparisons, each with a first launch and 100 warm samples. The JSON artifact records tool/runtime provenance and reaped processes. Startup events measure launch, handshake and dispense before RPC; they exclude binary resolution, RPC and reaping. These measurements use warm host caches, add no latency threshold or VM case, and do not claim cold-cache performance or total plugin overhead. + These scenarios do not establish complete parity. Compatibility shims verify preservation under injected version faults but do not establish behavior against an actual historical MicroSandbox release. Green lifecycle, image, mount, resource, persistent-storage, ephemeral-root, TCP-egress, prebuild, Dockerless, request-invalid and managed-policy recreation, logs, and cancellation checks alone do not authorize that cutover. Dockerless refusal checks cover fresh creation, not preservation after an invalid recreation request. Egress coverage does not establish UDP, DNS, TLS, ingress, or live-policy modification behavior. Prebuild invalidation coverage changes the Dockerfile; it does not independently establish hashing of local Feature script contents. From 9f251809df6933d4c81df303ab5ea5b66f0672a8 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 13 Oct 2026 23:04:37 -0600 Subject: [PATCH 5/8] test(runtime): register remote agent binary acceptance case --- .github/workflows/pr-ci.yml | 3 + e2e/tests/up/MICROSANDBOX.md | 10 + ...rovider_microsandbox_remote_agent_linux.go | 200 ++++++++++-------- .../developing-providers/runtime-protocol.mdx | 6 + 4 files changed, 126 insertions(+), 93 deletions(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index a348bb05a6..e2fb9a5297 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -401,6 +401,9 @@ jobs: - label: up-provider-microsandbox-registry-consumer expected-spec-count: 3 provider-mode: external + - label: up-provider-microsandbox-remote-agent-binaries + expected-spec-count: 1 + provider-mode: external runs-on: ubuntu-latest timeout-minutes: 25 steps: diff --git a/e2e/tests/up/MICROSANDBOX.md b/e2e/tests/up/MICROSANDBOX.md index c6e77f93d9..2999c79ff0 100644 --- a/e2e/tests/up/MICROSANDBOX.md +++ b/e2e/tests/up/MICROSANDBOX.md @@ -74,6 +74,16 @@ Its `TestMain` preserves the production runtime-supervisor dispatch and recogniz the helper only at its canonical, current-user-owned private path. Other test invocations retain the normal test runner path. +The Linux-only `up-provider-microsandbox-remote-agent-binaries` case uses the +published external provider with `agent.local: false` and a test-owned remote +agent container. It verifies one runtime-binary download into that agent's +workspace copy, then clears the agent's provider cache, makes the source return +404, and requires a subsequent `up` to reuse the retained workspace binary. +The guest check verifies the agent can still operate the workspace. This is a +source-prepared remote-agent contract; it does not establish ordinary `up` +image metadata resolution, general agent transfer behavior, or native results +until the required Linux/KVM CI case passes. + ## Runtime startup measurements The existing external lifecycle case optionally captures its ordinary fresh and diff --git a/e2e/tests/up/provider_microsandbox_remote_agent_linux.go b/e2e/tests/up/provider_microsandbox_remote_agent_linux.go index 346de55ed7..de8f98613f 100644 --- a/e2e/tests/up/provider_microsandbox_remote_agent_linux.go +++ b/e2e/tests/up/provider_microsandbox_remote_agent_linux.go @@ -44,107 +44,121 @@ const ( var _ = ginkgo.Describe("published MicroSandbox provider remote agent binaries", ginkgo.Label("up-provider-microsandbox-remote-agent-binaries", "microsandbox-external"), - ginkgo.It("downloads the external runtime on the remote agent and reuses its workspace copy", - func(ctx ginkgo.SpecContext) { - session := newMicrosandboxTestSession(ctx) - framework.ExpectNoError(requireRemoteAgentHostPrerequisites()) + func() { + ginkgo.It( + "downloads the external runtime on the remote agent and reuses its workspace copy", + func(ctx ginkgo.SpecContext) { + session := newMicrosandboxTestSession(ctx) + framework.ExpectNoError(requireRemoteAgentHostPrerequisites()) - providerName := "case-a-remote-agent" - stageProviderName := "case-a-remote-stage" - session.addProvider(ctx, microsandboxExternalProvider, stageProviderName) - staged := prepareMicrosandboxConsumerProvider(ctx, stageProviderName) - runtimeBytes, err := os.ReadFile(staged.runtimeBinary) - framework.ExpectNoError(err) - runtimeRelativePath, err := remoteAgentRuntimeRelativePath(staged) - framework.ExpectNoError(err) - descriptor, err := provider.LoadProviderConfig(config.DefaultContext, stageProviderName) - framework.ExpectNoError(err) + providerName := "case-a-remote-agent" + stageProviderName := "case-a-remote-stage" + session.addProvider(ctx, microsandboxExternalProvider, stageProviderName) + staged := prepareMicrosandboxConsumerProvider(ctx, stageProviderName) + runtimeBytes, err := os.ReadFile(staged.runtimeBinary) + framework.ExpectNoError(err) + runtimeRelativePath, err := remoteAgentRuntimeRelativePath(staged) + framework.ExpectNoError(err) + descriptor, err := provider.LoadProviderConfig( + config.DefaultContext, + stageProviderName, + ) + framework.ExpectNoError(err) - containerName := "devsy-" + microsandboxConsumerID("agent") - agentHome := filepath.Join( - "/tmp/devsy-remote-agent-fixture", - microsandboxConsumerID("home"), - "agent", - ) - bundleRoot, err := remoteAgentBundleRoot() - framework.ExpectNoError(err) - dockerPath, err := exec.LookPath("docker") - framework.ExpectNoError(err) - dockerPath, err = filepath.Abs(dockerPath) - framework.ExpectNoError(err) - preflightRemoteAgentContainer(ctx, bundleRoot, dockerPath, filepath.Dir(agentHome)) + containerName := "devsy-" + microsandboxConsumerID("agent") + agentHome := filepath.Join( + "/tmp/devsy-remote-agent-fixture", + microsandboxConsumerID("home"), + "agent", + ) + bundleRoot, err := remoteAgentBundleRoot() + framework.ExpectNoError(err) + dockerPath, err := exec.LookPath("docker") + framework.ExpectNoError(err) + dockerPath, err = filepath.Abs(dockerPath) + framework.ExpectNoError(err) + preflightRemoteAgentContainer(ctx, bundleRoot, dockerPath, filepath.Dir(agentHome)) - binaryName := descriptor.Agent.External.Binary - checksums := descriptor.Agent.Binaries[binaryName] - binaryEntry := remoteAgentLinuxBinary(checksums) - descriptorHash := strings.TrimPrefix(strings.ToLower(binaryEntry.Checksum), "sha256:") - gomega.Expect(descriptorHash).To(gomega.Equal(staged.runtimeHash)) - agentPath := filepath.Join( - session.framework.DevsyBinDir, - session.framework.DevsyBinName, - ) - server := startRemoteAgentArtifactServer(runtimeBytes, agentPath) - ginkgo.DeferCleanup(func(cleanupCtx ginkgo.SpecContext) error { - closeCtx, cancel := context.WithTimeout( - context.WithoutCancel(cleanupCtx), 10*time.Second, + binaryName := descriptor.Agent.External.Binary + checksums := descriptor.Agent.Binaries[binaryName] + binaryEntry := remoteAgentLinuxBinary(checksums) + descriptorHash := strings.TrimPrefix( + strings.ToLower(binaryEntry.Checksum), + "sha256:", ) - defer cancel() - return server.close(closeCtx) - }) + gomega.Expect(descriptorHash).To(gomega.Equal(staged.runtimeHash)) + agentPath := filepath.Join( + session.framework.DevsyBinDir, + session.framework.DevsyBinName, + ) + server := startRemoteAgentArtifactServer(runtimeBytes, agentPath) + ginkgo.DeferCleanup(func(cleanupCtx ginkgo.SpecContext) error { + closeCtx, cancel := context.WithTimeout( + context.WithoutCancel(cleanupCtx), 10*time.Second, + ) + defer cancel() + return server.close(closeCtx) + }) - descriptor.Name = providerName - descriptor.Agent.Local = types.StrBool("false") - descriptor.Agent.DataPath = agentHome - descriptor.Agent.DownloadURL = server.agentURL() - binaryEntry.Path = server.runtimeURL() - descriptor.Agent.Binaries[binaryName] = checksums - descriptor.Exec = remoteAgentCommands(containerName, bundleRoot, dockerPath, agentHome) - configPath := filepath.Join(ginkgo.GinkgoT().TempDir(), "provider.yaml") - configBytes, err := yaml.Marshal(descriptor) - framework.ExpectNoError(err) - framework.ExpectNoError(os.WriteFile(configPath, configBytes, 0o600)) - session.addProvider(ctx, configPath, providerName) - framework.ExpectNoError(session.framework.DevsyProviderUse(ctx, providerName)) + descriptor.Name = providerName + descriptor.Agent.Local = types.StrBool("false") + descriptor.Agent.DataPath = agentHome + descriptor.Agent.DownloadURL = server.agentURL() + binaryEntry.Path = server.runtimeURL() + descriptor.Agent.Binaries[binaryName] = checksums + descriptor.Exec = remoteAgentCommands( + containerName, + bundleRoot, + dockerPath, + agentHome, + ) + configPath := filepath.Join(ginkgo.GinkgoT().TempDir(), "provider.yaml") + configBytes, err := yaml.Marshal(descriptor) + framework.ExpectNoError(err) + framework.ExpectNoError(os.WriteFile(configPath, configBytes, 0o600)) + session.addProvider(ctx, configPath, providerName) + framework.ExpectNoError(session.framework.DevsyProviderUse(ctx, providerName)) - workspace := session.newWorkspace() - ginkgo.DeferCleanup(func(cleanupCtx ginkgo.SpecContext) error { - boundedCtx, cancel := context.WithTimeout( - context.WithoutCancel(cleanupCtx), 20*time.Second, + workspace := session.newWorkspace() + ginkgo.DeferCleanup(func(cleanupCtx ginkgo.SpecContext) error { + boundedCtx, cancel := context.WithTimeout( + context.WithoutCancel(cleanupCtx), 20*time.Second, + ) + defer cancel() + return removeRemoteAgentContainer(boundedCtx, containerName) + }) + ginkgo.DeferCleanup(session.framework.CleanupWorkspace, workspace) + framework.ExpectNoError(session.framework.DevsyUp(ctx, workspace)) + gomega.Expect(server.requests.Load()).To(gomega.Equal(int32(1)), + "remote agent must fetch the published provider binary exactly once") + workspaceRecord, err := session.framework.FindWorkspace(ctx, workspace) + framework.ExpectNoError(err) + remoteBinaryPath := filepath.Join( + agentHome, "contexts", config.DefaultContext, "workspaces", workspaceRecord.ID, + "binaries", runtimeRelativePath, ) - defer cancel() - return removeRemoteAgentContainer(boundedCtx, containerName) - }) - ginkgo.DeferCleanup(session.framework.CleanupWorkspace, workspace) - framework.ExpectNoError(session.framework.DevsyUp(ctx, workspace)) - gomega.Expect(server.requests.Load()).To(gomega.Equal(int32(1)), - "remote agent must fetch the published provider binary exactly once") - workspaceRecord, err := session.framework.FindWorkspace(ctx, workspace) - framework.ExpectNoError(err) - remoteBinaryPath := filepath.Join( - agentHome, "contexts", config.DefaultContext, "workspaces", workspaceRecord.ID, - "binaries", runtimeRelativePath, - ) - firstBinary := assertRemoteAgentBinary( - ctx, containerName, remoteBinaryPath, staged.runtimeHash, - ) - assertRemoteAgentGuest(ctx, session.framework, workspace) + firstBinary := assertRemoteAgentBinary( + ctx, containerName, remoteBinaryPath, staged.runtimeHash, + ) + assertRemoteAgentGuest(ctx, session.framework, workspace) - clearRemoteAgentBinaryCache(ctx, containerName, filepath.Dir(agentHome)) - framework.ExpectNoError(session.framework.DevsyStop(ctx, workspace)) - server.offline.Store(true) - framework.ExpectNoError(session.framework.DevsyUp(ctx, workspace)) - gomega.Expect(server.requests.Load()).To(gomega.Equal(int32(1)), - "existing workspace binary must be reused when its source returns 404") - secondBinary := assertRemoteAgentBinary( - ctx, containerName, remoteBinaryPath, staged.runtimeHash, - ) - gomega.Expect(secondBinary).To( - gomega.Equal(firstBinary), - "workspace binary identity must survive stop and restart", - ) - assertRemoteAgentGuest(ctx, session.framework, workspace) - }, ginkgo.SpecTimeout(framework.TimeoutLong()), - ), + clearRemoteAgentBinaryCache(ctx, containerName, filepath.Dir(agentHome)) + framework.ExpectNoError(session.framework.DevsyStop(ctx, workspace)) + server.offline.Store(true) + framework.ExpectNoError(session.framework.DevsyUp(ctx, workspace)) + gomega.Expect(server.requests.Load()).To(gomega.Equal(int32(1)), + "existing workspace binary must be reused when its source returns 404") + secondBinary := assertRemoteAgentBinary( + ctx, containerName, remoteBinaryPath, staged.runtimeHash, + ) + gomega.Expect(secondBinary).To( + gomega.Equal(firstBinary), + "workspace binary identity must survive stop and restart", + ) + assertRemoteAgentGuest(ctx, session.framework, workspace) + }, ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + }, ) func remoteAgentRuntimeRelativePath(staged microsandboxConsumerProvider) (string, error) { diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx index 2e64048eae..06974fd05f 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx @@ -167,6 +167,12 @@ The Linux-only `up-provider-microsandbox-registry-consumer` label exercises the DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox-registry-consumer ``` +The Linux-only `up-provider-microsandbox-remote-agent-binaries` label uses the published external provider with `agent.local: false` and a test-owned remote agent container. It checks that `up` downloads the runtime binary into the remote agent's workspace copy once, then clears the agent's provider cache, makes the source return 404, and requires a later `up` to reuse the retained workspace binary. A guest command verifies the workspace remains usable. This source-prepared case does not establish general agent delivery behavior or ordinary `up` image metadata resolution. Its external-provider behavior still requires native Linux/KVM CI evidence; documentation and catalogue changes are not runtime proof. + +```sh +DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox-remote-agent-binaries +``` + The existing external lifecycle case also captures ordinary fresh and reused `up` wall time and filtered runtime startup events when CI supplies the pinned SDK performance tools. After both commands pass, a checksum-verified copy of the published runtime runs direct/supervised/supervised/direct Info/Find comparisons, each with a first launch and 100 warm samples. The JSON artifact records tool/runtime provenance and reaped processes. Startup events measure launch, handshake and dispense before RPC; they exclude binary resolution, RPC and reaping. These measurements use warm host caches, add no latency threshold or VM case, and do not claim cold-cache performance or total plugin overhead. These scenarios do not establish complete parity. Compatibility shims verify preservation under injected version faults but do not establish behavior against an actual historical MicroSandbox release. Green lifecycle, image, mount, resource, persistent-storage, ephemeral-root, TCP-egress, prebuild, Dockerless, request-invalid and managed-policy recreation, logs, and cancellation checks alone do not authorize that cutover. Dockerless refusal checks cover fresh creation, not preservation after an invalid recreation request. Egress coverage does not establish UDP, DNS, TLS, ingress, or live-policy modification behavior. From 93d709fb6ccddfba0910ba6b5ca89169ccc6dfe8 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Wed, 14 Oct 2026 08:47:45 -0600 Subject: [PATCH 6/8] test(runtime): distinguish remote agent and provider downloads --- ...rovider_microsandbox_remote_agent_linux.go | 77 +++++++++------ ...er_microsandbox_remote_agent_linux_test.go | 94 +++++++++++++++++++ 2 files changed, 140 insertions(+), 31 deletions(-) create mode 100644 e2e/tests/up/provider_microsandbox_remote_agent_linux_test.go diff --git a/e2e/tests/up/provider_microsandbox_remote_agent_linux.go b/e2e/tests/up/provider_microsandbox_remote_agent_linux.go index de8f98613f..78ae60ed10 100644 --- a/e2e/tests/up/provider_microsandbox_remote_agent_linux.go +++ b/e2e/tests/up/provider_microsandbox_remote_agent_linux.go @@ -27,13 +27,14 @@ import ( ) type remoteAgentArtifactServer struct { - server *http.Server - listener net.Listener - done chan error - requests atomic.Int32 - offline atomic.Bool - runtime []byte - agent []byte + server *http.Server + listener net.Listener + done chan error + runtimeRequests atomic.Int32 + agentRequests atomic.Int32 + offline atomic.Bool + runtime []byte + agent []byte } const ( @@ -129,8 +130,10 @@ var _ = ginkgo.Describe("published MicroSandbox provider remote agent binaries", }) ginkgo.DeferCleanup(session.framework.CleanupWorkspace, workspace) framework.ExpectNoError(session.framework.DevsyUp(ctx, workspace)) - gomega.Expect(server.requests.Load()).To(gomega.Equal(int32(1)), + gomega.Expect(server.runtimeRequests.Load()).To(gomega.Equal(int32(1)), "remote agent must fetch the published provider binary exactly once") + gomega.Expect(server.agentRequests.Load()).To(gomega.BeNumerically(">", 0), + "remote agent must fetch its agent binary from the fixture endpoint") workspaceRecord, err := session.framework.FindWorkspace(ctx, workspace) framework.ExpectNoError(err) remoteBinaryPath := filepath.Join( @@ -146,7 +149,7 @@ var _ = ginkgo.Describe("published MicroSandbox provider remote agent binaries", framework.ExpectNoError(session.framework.DevsyStop(ctx, workspace)) server.offline.Store(true) framework.ExpectNoError(session.framework.DevsyUp(ctx, workspace)) - gomega.Expect(server.requests.Load()).To(gomega.Equal(int32(1)), + gomega.Expect(server.runtimeRequests.Load()).To(gomega.Equal(int32(1)), "existing workspace binary must be reused when its source returns 404") secondBinary := assertRemoteAgentBinary( ctx, containerName, remoteBinaryPath, staged.runtimeHash, @@ -200,34 +203,46 @@ func startRemoteAgentArtifactServer( listener: listener, runtime: runtimeBinary, done: make(chan error, 1), } server.agent = agentBinary - mux := http.NewServeMux() - mux.HandleFunc("/agent/devsy-linux-amd64", func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } - _, _ = w.Write(server.agent) - }) - mux.HandleFunc("/provider/runtime", func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } - server.requests.Add(1) - if server.offline.Load() { - http.NotFound(w, r) - return - } - _, _ = w.Write(server.runtime) - }) server.server = &http.Server{ - Handler: mux, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 15 * time.Second, - WriteTimeout: 3 * time.Minute, IdleTimeout: 30 * time.Second, + Handler: server.artifactHandler(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: 3 * time.Minute, + IdleTimeout: 30 * time.Second, } go func() { server.done <- server.server.Serve(listener) }() return server } +func (s *remoteAgentArtifactServer) artifactHandler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/agent/devsy-linux-amd64", s.serveAgent) + mux.HandleFunc("/provider/runtime", s.serveRuntime) + return mux +} + +func (s *remoteAgentArtifactServer) serveAgent(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + s.agentRequests.Add(1) + _, _ = w.Write(s.agent) +} + +func (s *remoteAgentArtifactServer) serveRuntime(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + s.runtimeRequests.Add(1) + if s.offline.Load() { + http.NotFound(w, r) + return + } + _, _ = w.Write(s.runtime) +} + func (s *remoteAgentArtifactServer) baseURL() string { return "http://host.docker.internal:" + strings.TrimPrefix( s.listener.Addr().String(), "0.0.0.0:", diff --git a/e2e/tests/up/provider_microsandbox_remote_agent_linux_test.go b/e2e/tests/up/provider_microsandbox_remote_agent_linux_test.go new file mode 100644 index 0000000000..950e8e197f --- /dev/null +++ b/e2e/tests/up/provider_microsandbox_remote_agent_linux_test.go @@ -0,0 +1,94 @@ +//go:build linux + +package up + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" +) + +const ( + runtimeEndpointPath = "/provider/runtime" + agentEndpointPath = "/agent/devsy-linux-amd64" +) + +func TestRemoteAgentArtifactEndpointsTrackIndependentGETs(t *testing.T) { + server := &remoteAgentArtifactServer{ + runtime: []byte("runtime binary"), + agent: []byte("agent binary"), + } + handler := server.artifactHandler() + + assertArtifactResponse(t, handler, artifactResponseExpectation{ + method: http.MethodGet, + path: runtimeEndpointPath, + status: http.StatusOK, + body: server.runtime, + }) + assertArtifactResponse(t, handler, artifactResponseExpectation{ + method: http.MethodGet, path: agentEndpointPath, status: http.StatusOK, body: server.agent, + }) + + if got := server.runtimeRequests.Load(); got != 1 { + t.Fatalf("runtime GET count = %d, want 1", got) + } + if got := server.agentRequests.Load(); got != 1 { + t.Fatalf("agent GET count = %d, want 1", got) + } +} + +func TestRemoteAgentArtifactEndpointMethodsAndOfflineRuntime(t *testing.T) { + server := &remoteAgentArtifactServer{ + runtime: []byte("runtime binary"), + agent: []byte("agent binary"), + } + handler := server.artifactHandler() + server.offline.Store(true) + + assertArtifactResponse(t, handler, artifactResponseExpectation{ + method: http.MethodGet, path: runtimeEndpointPath, status: http.StatusNotFound, + }) + assertArtifactResponse(t, handler, artifactResponseExpectation{ + method: http.MethodGet, path: agentEndpointPath, status: http.StatusOK, body: server.agent, + }) + assertArtifactResponse(t, handler, artifactResponseExpectation{ + method: http.MethodPost, path: runtimeEndpointPath, status: http.StatusMethodNotAllowed, + }) + assertArtifactResponse(t, handler, artifactResponseExpectation{ + method: http.MethodPost, path: agentEndpointPath, status: http.StatusMethodNotAllowed, + }) + + if got := server.runtimeRequests.Load(); got != 1 { + t.Fatalf("runtime GET count = %d, want the offline GET only", got) + } + if got := server.agentRequests.Load(); got != 1 { + t.Fatalf("agent GET count = %d, want the successful GET only", got) + } +} + +type artifactResponseExpectation struct { + method, path string + status int + body []byte +} + +func assertArtifactResponse(t *testing.T, handler http.Handler, want artifactResponseExpectation) { + t.Helper() + request := httptest.NewRequest(want.method, want.path, nil) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != want.status { + t.Fatalf("%s %s status = %d, want %d", want.method, want.path, response.Code, want.status) + } + if want.body != nil && !bytes.Equal(response.Body.Bytes(), want.body) { + t.Fatalf( + "%s %s body = %q, want %q", + want.method, + want.path, + response.Body.Bytes(), + want.body, + ) + } +} From 7db5060d11b1fc25c6342887dd611ab0588259ca Mon Sep 17 00:00:00 2001 From: Samuel K Date: Wed, 14 Oct 2026 10:33:54 -0600 Subject: [PATCH 7/8] fix(e2e): normalize remote agent artifact listener addresses --- ...rovider_microsandbox_remote_agent_linux.go | 14 ++++++++++--- ...er_microsandbox_remote_agent_linux_test.go | 20 +++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/e2e/tests/up/provider_microsandbox_remote_agent_linux.go b/e2e/tests/up/provider_microsandbox_remote_agent_linux.go index 78ae60ed10..bbe7f10223 100644 --- a/e2e/tests/up/provider_microsandbox_remote_agent_linux.go +++ b/e2e/tests/up/provider_microsandbox_remote_agent_linux.go @@ -244,9 +244,17 @@ func (s *remoteAgentArtifactServer) serveRuntime(w http.ResponseWriter, r *http. } func (s *remoteAgentArtifactServer) baseURL() string { - return "http://host.docker.internal:" + strings.TrimPrefix( - s.listener.Addr().String(), "0.0.0.0:", - ) + address, err := remoteAgentArtifactBaseURL(s.listener.Addr().String()) + framework.ExpectNoError(err) + return address +} + +func remoteAgentArtifactBaseURL(address string) (string, error) { + _, port, err := net.SplitHostPort(address) + if err != nil { + return "", fmt.Errorf("parse remote agent artifact listener: %w", err) + } + return "http://" + net.JoinHostPort("host.docker.internal", port), nil } func (s *remoteAgentArtifactServer) runtimeURL() string { return s.baseURL() + "/provider/runtime" } diff --git a/e2e/tests/up/provider_microsandbox_remote_agent_linux_test.go b/e2e/tests/up/provider_microsandbox_remote_agent_linux_test.go index 950e8e197f..36e45c37f1 100644 --- a/e2e/tests/up/provider_microsandbox_remote_agent_linux_test.go +++ b/e2e/tests/up/provider_microsandbox_remote_agent_linux_test.go @@ -14,6 +14,26 @@ const ( agentEndpointPath = "/agent/devsy-linux-amd64" ) +func TestRemoteAgentArtifactBaseURLUsesListenerPort(t *testing.T) { + for _, address := range []string{"0.0.0.0:35437", "[::]:35437", "127.0.0.1:35437", "[::1]:35437"} { + t.Run(address, func(t *testing.T) { + got, err := remoteAgentArtifactBaseURL(address) + if err != nil { + t.Fatal(err) + } + if want := "http://host.docker.internal:35437"; got != want { + t.Fatalf("artifact base URL = %q, want %q", got, want) + } + }) + } +} + +func TestRemoteAgentArtifactBaseURLRejectsInvalidAddress(t *testing.T) { + if _, err := remoteAgentArtifactBaseURL("missing-port"); err == nil { + t.Fatal("expected an invalid listener address error") + } +} + func TestRemoteAgentArtifactEndpointsTrackIndependentGETs(t *testing.T) { server := &remoteAgentArtifactServer{ runtime: []byte("runtime binary"), From 35c518fa3094fffb320f0cc4c07715aae244fccc Mon Sep 17 00:00:00 2001 From: Samuel K Date: Wed, 14 Oct 2026 12:09:58 -0600 Subject: [PATCH 8/8] fix(e2e): prepare system trust for remote agent HTTPS --- e2e/tests/up/MICROSANDBOX.md | 3 +++ e2e/tests/up/provider_microsandbox_remote_agent_linux.go | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/e2e/tests/up/MICROSANDBOX.md b/e2e/tests/up/MICROSANDBOX.md index 012b8f0afc..01b4809e7c 100644 --- a/e2e/tests/up/MICROSANDBOX.md +++ b/e2e/tests/up/MICROSANDBOX.md @@ -94,6 +94,9 @@ transfer and exactly one runtime download through `Agent.Binaries`. The runtime is checked against the published manifest and the workspace copy's checksum, executable mode and inode. +Container preflight requires KVM access, Docker connectivity, curl and a +populated system CA store for ordinary HTTPS image metadata resolution. + After the first successful `up`, the case removes only the private agent's provider-download cache and makes the runtime endpoint return 404. Stop/start must retain the workspace binary's checksum and inode without another runtime diff --git a/e2e/tests/up/provider_microsandbox_remote_agent_linux.go b/e2e/tests/up/provider_microsandbox_remote_agent_linux.go index bbe7f10223..05e0f4dcb4 100644 --- a/e2e/tests/up/provider_microsandbox_remote_agent_linux.go +++ b/e2e/tests/up/provider_microsandbox_remote_agent_linux.go @@ -312,7 +312,7 @@ esac`, const ( remoteAgentContainerImage = "ubuntu:24.04" remoteAgentContainerInit = `apt-get update && -apt-get install -y --no-install-recommends curl && +apt-get install -y --no-install-recommends ca-certificates curl && rm -rf /var/lib/apt/lists/* && touch /tmp/devsy-agent-ready && sleep infinity` @@ -324,6 +324,7 @@ test -c /dev/kvm && test -r /dev/kvm && test -w /dev/kvm && command -v curl >/dev/null && +test -s /etc/ssl/certs/ca-certificates.crt && /opt/devsy-microsandbox/bin/msb --version && /usr/local/bin/docker version >/dev/null` )