diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml index 6b63cecd6..d2ad89e08 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/native.yml @@ -73,6 +73,8 @@ jobs: run: | go build -ldflags "-X github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/cli.Version=$(git rev-parse HEAD)" -o "$RUNNER_TEMP/oac-daemon${{ runner.os == 'Windows' && '.exe' || '' }}" ./apps/daemon/cmd/oac-daemon node --test scripts/build-native-installer.test.mjs + - name: Verify native download bootstrap and recovery + run: go test ./services/core/internal/nativeinstaller -count=1 -timeout=3m - name: Native filesystem, authentication and process lifecycle run: >- go test -race -count=1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4b18c3f7d..c6eb58aa8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,6 +31,7 @@ This guide owns how to work in the repository: documentation ownership, the repo | Runtime telemetry responses | [Runtime telemetry API](contracts/agents-api/runtime-observability-api.md) | | Runtime observation, sampling, retention and export | [Runtime observability](contracts/agents-api/runtime-observability.md) | | Distribution builds, Runtime image builds, CI and publication | [Maintainer guide](docs/maintainers.md) | +| Self-hosted Runtime installation, recovery and local operation | [Self-hosted execution](docs/getting-started/self-hosted.md) | | Installer lifecycle, locking, generated state, managed HTTPS and downloads | [Installer design rules](deploy/install/README.md) | | Operator installation and alternatives | [Installation](docs/getting-started/install.md), [installation options](docs/getting-started/install-options.md) | | Settings, defaults, files and installation layout | [Configuration](docs/configuration.md) | diff --git a/apps/daemon/internal/cli/native_bundle.go b/apps/daemon/internal/cli/native_bundle.go index 51f6adb34..3f2126ce4 100644 --- a/apps/daemon/internal/cli/native_bundle.go +++ b/apps/daemon/internal/cli/native_bundle.go @@ -92,7 +92,9 @@ func componentReceipt(root string) (nativeComponent, error) { return c, err } -func checkComponentFiles(directory string, c nativeComponent) error { +var errNativeComponentMismatch = errors.New("installed files do not match the verified release") + +func checkComponentFiles(ctx context.Context, directory string, c nativeComponent) error { root, err := os.OpenRoot(directory) if err != nil { return err @@ -106,12 +108,23 @@ func checkComponentFiles(directory string, c nativeComponent) error { if err != nil { return err } - info, e := f.Stat() + info, statErr := f.Stat() + if statErr != nil { + f.Close() + return statErr + } + if !info.Mode().IsRegular() { + f.Close() + return errNativeComponentMismatch + } h := sha256.New() - _, copyErr := io.Copy(h, f) + _, copyErr := nativeCopy(ctx, h, f) f.Close() - if e != nil || !info.Mode().IsRegular() || copyErr != nil || hex.EncodeToString(h.Sum(nil)) != expected.SHA256 || (runtime.GOOS != "windows" && expected.Executable && info.Mode().Perm()&0100 == 0) { - return errors.New("installed files do not match the verified release") + if copyErr != nil { + return copyErr + } + if hex.EncodeToString(h.Sum(nil)) != expected.SHA256 || (runtime.GOOS != "windows" && expected.Executable && info.Mode().Perm()&0100 == 0) { + return errNativeComponentMismatch } } return nil @@ -126,8 +139,11 @@ func installNativeComponent(ctx context.Context, source, root, name string, expe if e != nil || !reflect.DeepEqual(got, expected) { return fmt.Errorf("install: existing %s is incompatible; preserve it and use a separate installation directory", name) } - if checkComponentFiles(dest, got) != nil { - return fmt.Errorf("install: existing %s is incomplete or modified; reinstall separately", name) + if err := checkComponentFiles(ctx, dest, got); err != nil { + if errors.Is(err, errNativeComponentMismatch) || errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("install: existing %s is incomplete or modified; reinstall separately", name) + } + return fmt.Errorf("install: cannot verify existing %s: %w", name, err) } return nil } else if !errors.Is(err, os.ErrNotExist) { @@ -147,6 +163,23 @@ func installNativeComponent(ctx context.Context, source, root, name string, expe return err } defer src.Close() + var required uint64 + for name := range expected.Files { + if !validBundlePath(name) { + return errors.New("install: invalid component path") + } + info, err := src.Stat(filepath.FromSlash(name)) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() < 0 || uint64(info.Size()) > ^uint64(0)-required { + return errors.New("install: invalid component size") + } + required += uint64(info.Size()) + } + if err = requireNativeSpace(parent, required); err != nil { + return err + } for name, expectedFile := range expected.Files { if err := ctx.Err(); err != nil { return err @@ -184,7 +217,13 @@ func installNativeComponent(ctx context.Context, source, root, name string, expe err = out.Sync() } closeErr := out.Close() - if err != nil || closeErr != nil || hex.EncodeToString(h.Sum(nil)) != expectedFile.SHA256 { + if err != nil { + return fmt.Errorf("install: component copy failed: %w", err) + } + if closeErr != nil { + return fmt.Errorf("install: component write failed: %w", closeErr) + } + if hex.EncodeToString(h.Sum(nil)) != expectedFile.SHA256 { return errors.New("install: component checksum failed") } } diff --git a/apps/daemon/internal/cli/native_install.go b/apps/daemon/internal/cli/native_install.go index 60d120344..9e0f87695 100644 --- a/apps/daemon/internal/cli/native_install.go +++ b/apps/daemon/internal/cli/native_install.go @@ -9,7 +9,6 @@ import ( "errors" "flag" "fmt" - "io" "os" "path/filepath" "slices" @@ -98,7 +97,7 @@ func runInstall(rc *runContext, args []string) error { ctx, stop := daemonize.NotifyContext(context.Background()) defer stop() if err := installNativeOptions(ctx, rc, &o); err != nil { - return err + return nativeInstallError(err) } if o.OnboardURL != "" { return finishOnboarding(ctx, rc, o) @@ -130,6 +129,9 @@ func installNativeOptions(ctx context.Context, rc *runContext, o *nativeInstallO return err } defer unlock() + if err = cleanNativeTemporaryFiles(o.Directory); err != nil { + return err + } if o.OnboardURL != "" { if runtimefs.ValidateLocalPath(o.Workspace) != nil { return errors.New("install: Session workspace is invalid on this platform") @@ -172,19 +174,19 @@ func installNativeOptions(ctx context.Context, rc *runContext, o *nativeInstallO // Verify existing components before adding any new one; never repair or // upgrade an installed dependency as a side effect of adding a Harness. if len(previous.Harnesses) > 0 { - if err = verifyNativeComponents(o.Directory, previous.Harnesses); err != nil { + if err = nativeInstallPhase(rc.stdout, "Verifying installed components", func() error { return verifyNativeComponents(ctx, o.Directory, previous.Harnesses) }); err != nil { return err } } - if err = installNativeBinary(o.Directory, len(previous.Harnesses) > 0); err != nil { + if err = nativeInstallPhase(rc.stdout, "Installing Runtime", func() error { return installNativeBinary(ctx, o.Directory, len(previous.Harnesses) > 0) }); err != nil { return err } for _, name := range append([]string{"node"}, selected...) { - if err = installNativeComponent(ctx, o.Bundle, o.Directory, name, bundle.Components[name]); err != nil { + if err = nativeInstallPhase(rc.stdout, "Installing "+name, func() error { return installNativeComponent(ctx, o.Bundle, o.Directory, name, bundle.Components[name]) }); err != nil { return err } } - if err = probeNativeInstallation(ctx, o.Directory, all); err != nil { + if err = nativeInstallPhase(rc.stdout, "Checking installed programs", func() error { return probeNativeInstallation(ctx, o.Directory, all) }); err != nil { return err } if err = ctx.Err(); err != nil { @@ -203,20 +205,26 @@ func installNativeOptions(ctx context.Context, rc *runContext, o *nativeInstallO return nil } -func verifyNativeComponents(root string, selected []string) error { +func verifyNativeComponents(ctx context.Context, root string, selected []string) error { for _, name := range append([]string{"node"}, selected...) { if _, ok := nativePins[name]; !ok { return errors.New("installation contains an unsupported Harness") } c, err := componentReceipt(nativeComponentRoot(root, name)) - if err != nil || c.Version != nativePins[name] || len(c.Files) == 0 || checkComponentFiles(nativeComponentRoot(root, name), c) != nil { + if err != nil || c.Version != nativePins[name] || len(c.Files) == 0 { return fmt.Errorf("installed %s is missing, modified or incompatible; reinstall separately (no automatic repair or upgrade)", name) } + if err = checkComponentFiles(ctx, nativeComponentRoot(root, name), c); err != nil { + if errors.Is(err, errNativeComponentMismatch) || errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("installed %s is missing or modified; reinstall separately", name) + } + return fmt.Errorf("install: cannot verify installed %s: %w", name, err) + } } return nil } -func installNativeBinary(root string, existing bool) error { +func installNativeBinary(ctx context.Context, root string, existing bool) error { exe, err := os.Executable() if err != nil { return err @@ -233,7 +241,7 @@ func installNativeBinary(root string, existing bool) error { } defer f.Close() h := sha256.New() - _, e = io.Copy(h, f) + _, e = nativeCopy(ctx, h, f) return hex.EncodeToString(h.Sum(nil)), e } want, err := digest(exe) @@ -256,12 +264,19 @@ func installNativeBinary(root string, existing bool) error { return err } defer in.Close() + info, err := in.Stat() + if err != nil { + return err + } + if err = requireNativeSpace(dir, uint64(info.Size())); err != nil { + return err + } out, err := os.CreateTemp(dir, ".oac-daemon-") if err != nil { return err } defer os.Remove(out.Name()) - _, err = io.Copy(out, in) + _, err = nativeCopy(ctx, out, in) if err == nil { err = out.Chmod(0700) } @@ -309,7 +324,7 @@ func runStart(rc *runContext, args []string) error { err = errors.New("start: no installed Harnesses; rerun install with --harness") } if err == nil { - err = verifyNativeComponents(root, config.Harnesses) + err = verifyNativeComponents(ctx, root, config.Harnesses) } if err == nil { err = probeNativeInstallation(ctx, root, config.Harnesses) diff --git a/apps/daemon/internal/cli/native_install_io.go b/apps/daemon/internal/cli/native_install_io.go new file mode 100644 index 000000000..aa2060641 --- /dev/null +++ b/apps/daemon/internal/cli/native_install_io.go @@ -0,0 +1,105 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "time" +) + +// These exact temporary names are reserved by the installer, never workspace data. +var nativeTemporaryNames = map[string]*regexp.Regexp{ + "components": regexp.MustCompile(`^\.install-(node|codex|claude|minimax)-[0-9]+$`), + "bin": regexp.MustCompile(`^\.oac-daemon-[0-9]+$`), + "daemon": regexp.MustCompile(`^\.(installation\.json|executor-credential\.json)-[0-9a-f]{24}\.tmp$`), +} + +// The caller holds the installation lock, including while recovering a failed copy. +func cleanNativeTemporaryFiles(root string) error { + for directory, pattern := range nativeTemporaryNames { + parent := filepath.Join(root, directory) + info, err := os.Lstat(parent) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return err + } + if !info.IsDir() { + return fmt.Errorf("install: %s must be a directory, not a link", directory) + } + entries, err := os.ReadDir(parent) + if err != nil { + return err + } + for _, entry := range entries { + if !pattern.MatchString(entry.Name()) { + continue + } + // RemoveAll unlinks a link itself and never follows its target. + if err = os.RemoveAll(filepath.Join(parent, entry.Name())); err != nil { + return fmt.Errorf("install: cannot remove interrupted staging: %w", err) + } + } + } + return nil +} + +func requireNativeSpace(directory string, size uint64) error { + available, err := nativeAvailableSpace(directory) + if err != nil { + return fmt.Errorf("install: cannot check free space: %w", err) + } + const reserve = 64 << 20 + if available < reserve || size > available-reserve { + return errors.New("install: not enough disk space; free space in the installation directory and retry") + } + return nil +} + +func nativeInstallError(err error) error { + if err == nil { + return nil + } + if nativeDiskFull(err) { + return errors.New("install: disk space or quota exhausted; free space and retry (completed components and credentials were preserved)") + } + if errors.Is(err, os.ErrPermission) { + return errors.New("install: filesystem access denied; check directory permissions and files in use, then retry with the same account") + } + return err +} + +// Non-terminal output contains ordinary phase lines, with no redraws or escapes. +func nativeInstallPhase(output io.Writer, label string, operation func() error) error { + fmt.Fprintln(output, label+"...") + file, ok := output.(*os.File) + if !ok || !nativeTerminal(file) { + return operation() + } + done, stopped := make(chan struct{}), make(chan struct{}) + start := time.Now() + go func() { + defer close(stopped) + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-done: + return + case <-ticker.C: + fmt.Fprintf(output, "\r%s... %ds", label, int(time.Since(start).Seconds())) + } + } + }() + defer func() { close(done); <-stopped; fmt.Fprintln(output) }() + return operation() +} + +func nativeCopy(ctx context.Context, out io.Writer, in io.Reader) (int64, error) { + return io.Copy(out, nativeCopyReader{ctx: ctx, Reader: in}) +} diff --git a/apps/daemon/internal/cli/native_install_io_unix.go b/apps/daemon/internal/cli/native_install_io_unix.go new file mode 100644 index 000000000..7cb72e0f6 --- /dev/null +++ b/apps/daemon/internal/cli/native_install_io_unix.go @@ -0,0 +1,25 @@ +//go:build linux || darwin + +package cli + +import ( + "errors" + "golang.org/x/sys/unix" + "os" + "syscall" +) + +func nativeAvailableSpace(directory string) (uint64, error) { + var s unix.Statfs_t + if err := unix.Statfs(directory, &s); err != nil { + return 0, err + } + return uint64(s.Bavail) * uint64(s.Bsize), nil +} +func nativeDiskFull(err error) bool { + return errors.Is(err, syscall.ENOSPC) || errors.Is(err, syscall.EDQUOT) +} +func nativeTerminal(f *os.File) bool { + _, err := unix.IoctlGetWinsize(int(f.Fd()), unix.TIOCGWINSZ) + return err == nil +} diff --git a/apps/daemon/internal/cli/native_install_io_windows.go b/apps/daemon/internal/cli/native_install_io_windows.go new file mode 100644 index 000000000..3c43c0152 --- /dev/null +++ b/apps/daemon/internal/cli/native_install_io_windows.go @@ -0,0 +1,24 @@ +package cli + +import ( + "errors" + "golang.org/x/sys/windows" + "os" +) + +func nativeAvailableSpace(directory string) (uint64, error) { + name, err := windows.UTF16PtrFromString(directory) + if err != nil { + return 0, err + } + var free, total, available uint64 + err = windows.GetDiskFreeSpaceEx(name, &available, &total, &free) + return available, err +} +func nativeDiskFull(err error) bool { + return errors.Is(err, windows.ERROR_DISK_FULL) || errors.Is(err, windows.ERROR_HANDLE_DISK_FULL) || errors.Is(err, windows.ERROR_DISK_QUOTA_EXCEEDED) +} +func nativeTerminal(f *os.File) bool { + var mode uint32 + return windows.GetConsoleMode(windows.Handle(f.Fd()), &mode) == nil +} diff --git a/apps/daemon/internal/cli/native_install_recovery_test.go b/apps/daemon/internal/cli/native_install_recovery_test.go new file mode 100644 index 000000000..12dabf52f --- /dev/null +++ b/apps/daemon/internal/cli/native_install_recovery_test.go @@ -0,0 +1,132 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestNativeInstallationCleansOnlyReservedPartialFiles(t *testing.T) { + rc, args, root, _ := nativeInstallFixture(t) + names := []string{"components/.install-codex-12345/partial", "bin/.oac-daemon-9876", "daemon/.installation.json-0123456789abcdef01234567.tmp"} + for _, name := range names { + path := filepath.Join(root, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("partial"), 0600); err != nil { + t.Fatal(err) + } + } + keep := filepath.Join(root, "components", ".install-codex-user-notes") + if err := os.WriteFile(keep, []byte("keep"), 0600); err != nil { + t.Fatal(err) + } + // Cleanup must not happen before obtaining the operation lock. + _, unlock, err := lockNativeInstallation(root) + if err != nil { + t.Fatal(err) + } + if err = runInstall(rc, args); err == nil { + t.Fatal("concurrent installation accepted") + } + for _, name := range names { + if _, err = os.Stat(filepath.Join(root, name)); err != nil { + t.Fatal("active staging was removed") + } + } + unlock() + if err = runInstall(rc, args); err != nil { + t.Fatal(err) + } + for _, name := range names { + if _, err = os.Stat(filepath.Join(root, name)); !os.IsNotExist(err) { + t.Fatalf("stale staging remains: %s", name) + } + } + if data, err := os.ReadFile(keep); err != nil || string(data) != "keep" { + t.Fatal("unrelated file changed") + } + output := rc.stdout.(*bytes.Buffer).String() + if !strings.Contains(output, "Installing codex") || strings.Contains(output, "\r") || strings.Contains(output, "\x1b") { + t.Fatalf("invalid redirected progress: %q", output) + } +} + +func TestNativeCopyPreservesIOErrorAndCancellation(t *testing.T) { + failure := errors.New("synthetic filesystem failure") + _, err := nativeCopy(context.Background(), nativeFailingWriter{failure}, strings.NewReader("component")) + if !errors.Is(err, failure) { + t.Fatalf("lost write error: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = nativeCopy(ctx, new(bytes.Buffer), strings.NewReader("component")) + if !errors.Is(err, context.Canceled) { + t.Fatalf("lost cancellation: %v", err) + } + if err = requireNativeSpace(t.TempDir(), ^uint64(0)); err == nil || !strings.Contains(err.Error(), "not enough disk space") { + t.Fatalf("space overflow accepted: %v", err) + } +} + +type nativeFailingWriter struct{ err error } + +func (w nativeFailingWriter) Write([]byte) (int, error) { return 0, w.err } + +func TestNativeStagingCleanupDoesNotFollowLinks(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + keep := filepath.Join(outside, "keep") + if err := os.WriteFile(keep, []byte("keep"), 0600); err != nil { + t.Fatal(err) + } + parent := filepath.Join(root, "components") + if err := os.Mkdir(parent, 0700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(parent, ".install-codex-123")); err != nil { + t.Skip("symlink permission unavailable") + } + if err := cleanNativeTemporaryFiles(root); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(keep); err != nil { + t.Fatal("removed link target") + } +} + +func TestNativeVerificationCancellationPreservesInstallation(t *testing.T) { + rc, args, root, bundle := nativeInstallFixture(t) + if err := runInstall(rc, args); err != nil { + t.Fatal(err) + } + config := filepath.Join(root, "daemon", "installation.json") + before, err := os.ReadFile(config) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err = verifyNativeComponents(ctx, root, []string{"codex"}); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled verification reported damage: %v", err) + } + b, err := readNativeBundle(bundle, []string{"codex"}) + if err != nil { + t.Fatal(err) + } + if err = installNativeComponent(ctx, bundle, root, "codex", b.Components["codex"]); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled reuse reported damage: %v", err) + } + after, err := os.ReadFile(config) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("cancellation changed installation") + } + if err = verifyNativeComponents(context.Background(), root, []string{"codex"}); err != nil { + t.Fatal(err) + } +} diff --git a/docs/getting-started/self-hosted.md b/docs/getting-started/self-hosted.md index bf650fa83..b032e81e6 100644 --- a/docs/getting-started/self-hosted.md +++ b/docs/getting-started/self-hosted.md @@ -23,7 +23,7 @@ The machine needs: - Python and pip when the Session's packages need them; - any system packages your setup needs. The daemon never runs apt, sudo or another elevation command, so install them through the host's normal administration. -No administrator privileges or Docker are needed. +No administrator privileges or Docker are needed. The Unix download command also uses `curl`, `tar`, `gzip`, a SHA-256 tool and the system file-lock command (`flock` on Linux, `lockf` on macOS). ## Connect a machine @@ -67,6 +67,8 @@ The installer reports three results: | **Daemon connection** | Core confirmed the daemon's authenticated connection | | **Model configuration** | Not checked; the first Turn uses the Session's model provider | +Downloads retry temporary network failures up to three attempts and show progress in a terminal. Disk space is checked before downloading, extracting and copying components. If a download or installation is interrupted, rerun the command: it clears unfinished temporary copies while preserving completed components, credentials and the workspace. Download staging lives in `native-download` under the Runtime home; its small `download.lock` file remains for concurrency control. An active download or installation is never cleared by another run. If the command expires, copy a fresh one from the Session. + If the connection is not confirmed within 45 seconds, the installer prints the path of the daemon's log. The daemon keeps reconnecting. Fix the cause and run the install command again with the same installation directory, copying a fresh one from Web if it has expired: completed components and the credential are kept and a running daemon is reused. Do not remove the workspace or the Session to retry. ### Options for automation diff --git a/services/core/internal/nativeinstaller/assets/bootstrap.ps1 b/services/core/internal/nativeinstaller/assets/bootstrap.ps1 index bca81cdf4..92c0b8094 100644 --- a/services/core/internal/nativeinstaller/assets/bootstrap.ps1 +++ b/services/core/internal/nativeinstaller/assets/bootstrap.ps1 @@ -4,27 +4,171 @@ param( [Parameter(ValueFromRemainingArguments=$true)][string[]]$InstallArguments ) $ErrorActionPreference = 'Stop' -$ProgressPreference = 'SilentlyContinue' -$architecture = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant() -$architecture = @{x64='amd64';arm64='arm64'}[$architecture] +Add-Type -AssemblyName System.Net.Http +if (!("OacNativeDownloadSpace" -as [type])) { + Add-Type -TypeDefinition @' +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; +public static class OacNativeDownloadSpace { + [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] + static extern bool GetDiskFreeSpaceEx(string path, out ulong available, out ulong total, out ulong free); + public static ulong Available(string path) { + ulong available, total, free; + if (!GetDiskFreeSpaceEx(path, out available, out total, out free)) throw new Win32Exception(Marshal.GetLastWin32Error()); + return available; + } +} +'@ +} +$architecture = @{x64='amd64';arm64='arm64'}[[System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant()] if (!$architecture) { throw 'Unsupported processor architecture.' } -$work = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) -New-Item -ItemType Directory -Path $work | Out-Null +$root = if ($env:OAC_RUNTIME_HOME) { $env:OAC_RUNTIME_HOME } else { Join-Path $HOME '.oac' } +if (![IO.Path]::IsPathRooted($root)) { throw 'OAC_RUNTIME_HOME must be an absolute directory.' } +$cache = Join-Path ([IO.Path]::GetFullPath($root)) 'native-download' +$work = Join-Path $cache 'staging' +$lock = $null +$client = $null +$ownsStaging = $false +$terminal = ![Console]::IsOutputRedirected +$ProgressPreference = if ($terminal) { 'Continue' } else { 'SilentlyContinue' } +function Assert-OrdinaryPath([string]$Path) { + if ((Test-Path -LiteralPath $Path) -and ((Get-Item -Force -LiteralPath $Path).Attributes -band [IO.FileAttributes]::ReparsePoint)) { + throw 'Native download paths must not be symbolic links or junctions.' + } +} +function Assert-Space([long]$Required) { + $available = [OacNativeDownloadSpace]::Available($cache) + if ($available -lt ($Required + 64MB)) { throw 'Not enough disk space for the native installer. Free space in the Runtime home and retry.' } +} +function Show-Bytes([string]$Label, [long]$Bytes, [long]$Total) { + if (!$terminal) { return } + $percent = if ($Total -gt 0) { [Math]::Min(100, [int](100.0 * $Bytes / $Total)) } else { -1 } + Write-Progress -Activity $Label -Status ("{0:N1} MiB" -f ($Bytes / 1MB)) -PercentComplete $percent +} +function Get-NativeFile([string]$Uri, [string]$Destination, [long]$Limit) { + for ($attempt=1; $attempt -le 3; $attempt++) { + $response = $null; $inputStream = $null; $outputStream = $null; $retryable = $true + try { + $address = [Uri]$Uri + for ($redirect=0; $redirect -le 5; $redirect++) { + $response = $client.GetAsync($address, [Net.Http.HttpCompletionOption]::ResponseHeadersRead).GetAwaiter().GetResult() + $status = [int]$response.StatusCode + if ($status -notin @(301,302,303,307,308)) { break } + if ($redirect -eq 5 -or !$response.Headers.Location) { $retryable=$false; throw 'Installer download has too many or invalid redirects.' } + $next = [Uri]::new($address, $response.Headers.Location) + if ($next.Scheme -ne 'https' -or $next.UserInfo) { $retryable=$false; throw 'Installer download redirect must use HTTPS without credentials.' } + $response.Dispose(); $response=$null; $address=$next + } + if ($status -ne 200) { + $retryable = $status -in @(408,429,500,502,503,504) + if ($status -eq 404) { throw 'This Core has no qualified installer for this platform.' } + throw "Installer download failed (HTTP $status). Check Core and the download host." + } + $total = $response.Content.Headers.ContentLength + if ($null -eq $total) { $total = 0 } + $retryable=$false + if ($total -gt $Limit) { throw 'Installer download exceeds the available space or metadata size limit.' } + Assert-Space $total + $outputStream = [IO.File]::Open($Destination, [IO.FileMode]::Create, [IO.FileAccess]::Write, [IO.FileShare]::None) + $retryable=$true + $inputStream = $response.Content.ReadAsStreamAsync().GetAwaiter().GetResult() + $buffer = New-Object byte[] 65536 + [long]$received = 0 + $clock = [Diagnostics.Stopwatch]::StartNew() + [long]$lastProgress=0 + while ($true) { + if ($clock.Elapsed.TotalSeconds -gt 1200) { throw 'Installer download timed out; retry with a fresh command if it expired.' } + $cancel = [Threading.CancellationTokenSource]::new() + try { + $cancel.CancelAfter(60000) + $pending = $inputStream.ReadAsync($buffer, 0, $buffer.Length, $cancel.Token) + if (!$pending.Wait(60000)) { throw 'Installer download stalled; retry when the connection is available.' } + $read = $pending.GetAwaiter().GetResult() + } finally { $cancel.Dispose() } + if ($read -eq 0) { break } + $received += $read + $retryable=$false + if ($received -gt $Limit) { throw 'Installer download exceeds the available space or metadata size limit.' } + $outputStream.Write($buffer,0,$read) + $retryable=$true + if ($clock.ElapsedMilliseconds - $lastProgress -ge 200) { Show-Bytes 'Downloading installer' $received $total; $lastProgress=$clock.ElapsedMilliseconds } + } + if ($total -gt 0 -and $received -ne $total) { throw 'Installer download was interrupted.' } + $retryable=$false + $outputStream.Flush($true) + return + } catch { + if (!$retryable) { throw } + if ($attempt -eq 3) { throw 'Installer download failed after three attempts. Check network, proxy and certificates, then retry.' } + Write-Host "Download interrupted; retrying ($attempt/2)..." + Start-Sleep -Seconds $attempt + } finally { + if ($inputStream) { $inputStream.Dispose() } + if ($outputStream) { $outputStream.Dispose() } + if ($response) { $response.Dispose() } + if ($terminal) { Write-Progress -Activity 'Downloading installer' -Completed } + } + } +} try { + Assert-OrdinaryPath $cache + [IO.Directory]::CreateDirectory($cache) | Out-Null + $lockPath = Join-Path $cache 'download.lock' + Assert-OrdinaryPath $lockPath + try { $lock = [IO.File]::Open($lockPath, [IO.FileMode]::OpenOrCreate, [IO.FileAccess]::ReadWrite, [IO.FileShare]::None) } + catch { throw 'Cannot lock native downloads. Another download may be running; check permissions or wait and retry.' } + Assert-OrdinaryPath $work + if (Test-Path -LiteralPath $work) { + # A killed PowerShell host can leave its native child alive. Do not remove its source bundle. + $children = Get-CimInstance Win32_Process -Filter "Name = 'tar.exe' OR Name = 'oac-daemon.exe'" + foreach ($child in $children) { + if (($child.ExecutablePath -and $child.ExecutablePath.StartsWith($work, [StringComparison]::OrdinalIgnoreCase)) -or ($child.CommandLine -and $child.CommandLine.IndexOf($work, [StringComparison]::OrdinalIgnoreCase) -ge 0)) { + throw 'A native installer is still using the download directory; wait for it to finish and retry.' + } + } + Remove-Item -LiteralPath $work -Recurse -Force + } + [IO.Directory]::CreateDirectory($work) | Out-Null + $ownsStaging = $true + $handler = [Net.Http.HttpClientHandler]::new() + $handler.AllowAutoRedirect = $false + $client = [Net.Http.HttpClient]::new($handler) + $client.Timeout = [TimeSpan]::FromSeconds(15) + Assert-Space 0 Write-Host 'Downloading the installer matched to Core...' - try { $expected = (Invoke-WebRequest -UseBasicParsing "$Base/windows-$architecture.sha256").Content.Trim() } - catch { throw 'This Core has no qualified installer for this platform.' } + $checksum = Join-Path $work 'checksum' + Get-NativeFile "$Base/windows-$architecture.sha256" $checksum 1024 + $expected = [IO.File]::ReadAllText($checksum).Trim() + if ($expected -cnotmatch '^[0-9a-f]{64}$') { throw 'Core returned an invalid installer checksum.' } $archive = Join-Path $work 'bundle.tar.gz' - Invoke-WebRequest -UseBasicParsing "$Base/windows-$architecture.tar.gz" -OutFile $archive + $available = [OacNativeDownloadSpace]::Available($cache) + Get-NativeFile "$Base/windows-$architecture.tar.gz" $archive ($available-64MB) Write-Host 'Verifying the installer archive...' if ((Get-FileHash -Algorithm SHA256 $archive).Hash.ToLowerInvariant() -ne $expected) { throw 'Installer checksum mismatch; download again.' } + Write-Host 'Checking extraction space...' + $file = [IO.File]::OpenRead($archive) + $gzip = [IO.Compression.GZipStream]::new($file,[IO.Compression.CompressionMode]::Decompress) + try { + $buffer = New-Object byte[] 65536 + [long]$unpacked=0 + while (($count=$gzip.Read($buffer,0,$buffer.Length)) -gt 0) { $unpacked += $count } + Assert-Space $unpacked + } finally { $gzip.Dispose(); $file.Dispose() } $bundle = Join-Path $work 'bundle' - New-Item -ItemType Directory -Path $bundle | Out-Null + [IO.Directory]::CreateDirectory($bundle) | Out-Null Write-Host 'Extracting the installer...' & (Join-Path $env:SystemRoot 'System32\tar.exe') -xzf $archive -C $bundle - if ($LASTEXITCODE -ne 0) { throw 'Installer extraction failed.' } + if ($LASTEXITCODE -ne 0) { throw 'Installer extraction failed. Check disk space, quota and filesystem permissions.' } $endpoint = $Base -replace '/install/[^/]+$', '/installation' Write-Host 'Starting installation...' & (Join-Path $bundle 'oac-daemon.exe') install --onboard-url $endpoint --authorization $Authorization @InstallArguments if ($LASTEXITCODE -ne 0) { throw 'Installation or connection failed; follow the installer guidance and retry.' } -} finally { Remove-Item -LiteralPath $work -Recurse -Force } +} finally { + if ($client) { $client.Dispose() } + if ($lock) { + # Only clean staging created by this invocation, after its native child returned. + try { if ($ownsStaging -and (Test-Path -LiteralPath $work)) { Remove-Item -LiteralPath $work -Recurse -Force } } + finally { $lock.Dispose() } + } +} diff --git a/services/core/internal/nativeinstaller/assets/bootstrap.sh b/services/core/internal/nativeinstaller/assets/bootstrap.sh index f97f98929..e14d88759 100644 --- a/services/core/internal/nativeinstaller/assets/bootstrap.sh +++ b/services/core/internal/nativeinstaller/assets/bootstrap.sh @@ -3,21 +3,91 @@ set -euo pipefail base=$1 authorization=$2 shift 2 -case "$(uname -s)" in Linux) os=linux;; Darwin) os=darwin;; *) echo 'Unsupported operating system.' >&2; exit 1;; esac -case "$(uname -m)" in x86_64) arch=amd64;; arm64|aarch64) arch=arm64;; *) echo 'Unsupported processor architecture.' >&2; exit 1;; esac +fail() { printf '%s\n' "$*" >&2; exit 1; } +case "$(uname -s)" in Linux) os=linux;; Darwin) os=darwin;; *) fail 'Unsupported operating system.';; esac +case "$(uname -m)" in x86_64) arch=amd64;; arm64|aarch64) arch=arm64;; *) fail 'Unsupported processor architecture.';; esac +for tool in curl tar gzip df awk wc; do command -v "$tool" >/dev/null || fail "Required command missing: $tool"; done +if command -v sha256sum >/dev/null; then hash=(sha256sum); else hash=(shasum -a 256); command -v shasum >/dev/null || fail 'Required command missing: shasum'; fi +if [ "$os" = darwin ]; then command -v lockf >/dev/null || fail 'Required command missing: lockf'; else command -v flock >/dev/null || fail 'Required command missing: flock (util-linux)'; fi umask 077 -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT -echo 'Downloading the installer matched to Core...' -curl -fsS "$base/$os-$arch.sha256" -o "$work/checksum" || { echo 'This Core has no qualified installer for this platform.' >&2; exit 1; } -curl -fsSL --proto-redir =https "$base/$os-$arch.tar.gz" -o "$work/bundle.tar.gz" -if command -v sha256sum >/dev/null; then - actual=$(sha256sum "$work/bundle.tar.gz"); actual=${actual%% *} -else - actual=$(shasum -a 256 "$work/bundle.tar.gz"); actual=${actual%% *} -fi +root=${OAC_RUNTIME_HOME:-${HOME:?HOME must be set}/.oac} +case "$root" in /*) ;; *) fail 'OAC_RUNTIME_HOME must be an absolute directory.';; esac +mkdir -p "$root" || fail 'Cannot write the Runtime home with the current account.' +root=$(cd "$root" && pwd -P) +cache=$root/native-download +[ ! -L "$cache" ] || fail 'Native download directory must not be a symbolic link.' +mkdir -p "$cache" +[ -d "$cache" ] && [ -O "$cache" ] || fail 'Native download directory must belong to the current account.' +chmod 700 "$cache" +lock=$cache/download.lock +[ ! -L "$lock" ] && { [ ! -e "$lock" ] || { [ -f "$lock" ] && [ -O "$lock" ]; }; } || fail 'Invalid native download lock.' +# Children inherit this descriptor, so killing only the shell cannot expose an active download. +exec 9>>"$lock" +if [ "$os" = darwin ]; then lockf -s -t 0 9; else flock -n 9; fi || fail 'Another native download is running; wait for it to finish and retry.' +work=$cache/staging +[ ! -L "$work" ] && { [ ! -e "$work" ] || { [ -d "$work" ] && [ -O "$work" ]; }; } || fail 'Invalid native download staging directory.' +rm -rf "$work" +mkdir "$work" +active= +cleanup() { rm -rf "$work"; } +interrupt() { + trap '' INT TERM HUP + if [ -n "$active" ]; then kill -TERM "$active" 2>/dev/null || :; wait "$active" 2>/dev/null || :; fi + exit 130 +} +trap cleanup EXIT +trap interrupt INT TERM HUP +run() { + "$@" <&0 & active=$! + local result=0 + wait "$active" || result=$? + active= + return "$result" +} +space() { + local available + available=$(df -Pk "$work" | awk 'NR==2 {print $4}') + case "$available" in ''|*[!0-9]*) fail 'Cannot determine free disk space.';; esac + [ "$available" -ge "$(( ($1 + 67108864 + 1023) / 1024 ))" ] || fail 'Not enough disk space for the native installer. Free space in the Runtime home and retry.' +} +progress=(-sS) +if [ -t 2 ]; then progress=(--progress-bar --show-error); fi +fetch() { + local url=$1 destination=$2 limit=$3 status code attempt + for attempt in 1 2 3; do + code=0 + run curl "${progress[@]}" --fail --location --proto-redir =https --connect-timeout 15 --max-time 1200 --speed-time 60 --speed-limit 1024 --max-filesize "$limit" --write-out '%{http_code}' "$url" -o "$destination" >"$work/http-status" || code=$? + status=$(cat "$work/http-status") + [ "$code" -ne 0 ] || return 0 + case "$status" in 404) fail 'This Core has no qualified installer for this platform.';; esac + case "$code" in + 23) fail 'Cannot write the installer download. Check disk space, quota and directory permissions.';; + 63) fail 'Installer download exceeds the available space or metadata size limit.';; + esac + case "$code:$status" in + 6:*|7:*|18:*|28:*|52:*|55:*|56:*|22:408|22:429|22:500|22:502|22:503|22:504) + if [ "$attempt" -lt 3 ]; then printf 'Download interrupted; retrying (%s/2)...\n' "$attempt" >&2; run sleep "$attempt"; continue; fi;; + esac + fail "Installer download failed (HTTP ${status:-unknown}, curl $code). Check network, proxy and certificates, then retry." + done +} +printf 'Downloading the installer matched to Core...\n' +space 0 +fetch "$base/$os-$arch.sha256" "$work/checksum" 1024 expected=$(cat "$work/checksum") -if [ "$actual" != "$expected" ]; then echo 'Installer checksum mismatch; download again.' >&2; exit 1; fi +[[ "$expected" =~ ^[0-9a-f]{64}$ ]] || fail 'Core returned an invalid installer checksum.' +available=$(df -Pk "$work" | awk 'NR==2 {print $4}') +fetch "$base/$os-$arch.tar.gz" "$work/bundle.tar.gz" "$((available * 1024 - 67108864))" +printf 'Verifying the installer archive...\n' +run "${hash[@]}" "$work/bundle.tar.gz" >"$work/digest" +read -r actual rest <"$work/digest" +[ "$actual" = "$expected" ] || fail 'Installer checksum mismatch; download again.' +printf 'Checking extraction space...\n' +# Count the verified tar stream without storing another copy, including its headers and padding. +run bash -o pipefail -c 'gzip -dc "$1" | wc -c' -- "$work/bundle.tar.gz" >"$work/unpacked-size" || fail 'Installer archive is incomplete or invalid.' +unpacked=$(cat "$work/unpacked-size") +space "$unpacked" +printf 'Extracting the installer...\n' mkdir "$work/bundle" -tar -xzf "$work/bundle.tar.gz" -C "$work/bundle" -"$work/bundle/oac-daemon" install --onboard-url "${base%/install/*}/installation" --authorization "$authorization" "$@" +run tar -xzf "$work/bundle.tar.gz" -C "$work/bundle" || fail 'Installer extraction failed. Check disk space, quota and filesystem permissions.' +run "$work/bundle/oac-daemon" install --onboard-url "${base%/install/*}/installation" --authorization "$authorization" "$@" diff --git a/services/core/internal/nativeinstaller/bootstrap_interrupt_unix_test.go b/services/core/internal/nativeinstaller/bootstrap_interrupt_unix_test.go new file mode 100644 index 000000000..bb6766c52 --- /dev/null +++ b/services/core/internal/nativeinstaller/bootstrap_interrupt_unix_test.go @@ -0,0 +1,110 @@ +//go:build unix + +package nativeinstaller + +import ( + "bytes" + "crypto/sha256" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" +) + +func TestBootstrapParentDeathKeepsActiveDownloadLocked(t *testing.T) { + archive := bootstrapFixtureArchive(t) + started := make(chan struct{}) + release := make(chan struct{}) + var downloads atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".sha256") { + fmt.Fprintf(w, "%x\n", sha256.Sum256(archive)) + return + } + w.Header().Set("Content-Length", fmt.Sprint(len(archive))) + if downloads.Add(1) == 1 { + w.Write(archive[:1]) + w.(http.Flusher).Flush() + close(started) + <-release + return + } + w.Write(archive) + })) + defer server.Close() + defer close(release) + home := t.TempDir() + command := bootstrapCommand(t, server.URL, home) + command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + var output bytes.Buffer + command.Stdout = &output + command.Stderr = &output + if err := command.Start(); err != nil { + t.Fatal(err) + } + // Kill only this fixture's process group, even if an assertion fails. + defer syscall.Kill(-command.Process.Pid, syscall.SIGKILL) + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatal("download did not start") + } + if err := command.Process.Kill(); err != nil { + t.Fatal(err) + } + retry, err := bootstrapCommand(t, server.URL, home).CombinedOutput() + if err == nil || !bytes.Contains(retry, []byte("Another native download")) { + t.Fatalf("active download was not protected: %v %s", err, retry) + } + if _, err := os.Stat(filepath.Join(home, "native-download", "staging", "bundle.tar.gz")); err != nil { + t.Fatal("active staging removed") + } + syscall.Kill(-command.Process.Pid, syscall.SIGKILL) + _ = command.Wait() + retry, err = bootstrapCommand(t, server.URL, home).CombinedOutput() + if err != nil { + t.Fatalf("recovery failed: %v %s", err, retry) + } + if _, err := os.Stat(filepath.Join(home, "native-download", "staging")); !os.IsNotExist(err) { + t.Fatal("staging left after recovery") + } +} + +func TestBootstrapCommandDiscardsTimedOutResponse(t *testing.T) { + curl, err := exec.LookPath("curl") + if err != nil { + t.Skip("curl unavailable") + } + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) == 1 { + w.Header().Set("Content-Length", "1000") + fmt.Fprint(w, "printf 'incomplete response") + w.(http.Flusher).Flush() + time.Sleep(700 * time.Millisecond) + return + } + w.Write([]byte("printf '%s\\n' entry-success\n")) + })) + defer server.Close() + // Shorten only the fixture's real curl timeout so the actual generated command is exercised. + bin := t.TempDir() + wrapper := "#!/bin/sh\nexec " + shellQuote(curl) + " \"$@\" --max-time 0.3\n" + if err = os.WriteFile(filepath.Join(bin, "curl"), []byte(wrapper), 0700); err != nil { + t.Fatal(err) + } + catalog := Catalog{Version: "test"} + command := exec.Command("bash", "-c", catalog.Commands(server.URL, "fixture-grant")["posix"]) + command.Env = append(os.Environ(), "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"), "NO_PROXY=127.0.0.1", "no_proxy=127.0.0.1") + output, err := command.CombinedOutput() + if err != nil || !bytes.Contains(output, []byte("entry-success")) || bytes.Contains(output, []byte("incomplete response")) || requests.Load() != 2 { + t.Fatalf("partial bootstrap executed: %v requests=%d %s", err, requests.Load(), output) + } +} diff --git a/services/core/internal/nativeinstaller/bootstrap_recovery_test.go b/services/core/internal/nativeinstaller/bootstrap_recovery_test.go new file mode 100644 index 000000000..a0d112925 --- /dev/null +++ b/services/core/internal/nativeinstaller/bootstrap_recovery_test.go @@ -0,0 +1,165 @@ +package nativeinstaller + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "testing" +) + +func bootstrapFixtureArchive(t *testing.T) []byte { + t.Helper() + var buffer bytes.Buffer + z := gzip.NewWriter(&buffer) + tarball := tar.NewWriter(z) + data := []byte("#!/bin/sh\nexit 0\n") + if err := tarball.WriteHeader(&tar.Header{Name: "oac-daemon", Mode: 0700, Size: int64(len(data))}); err != nil { + t.Fatal(err) + } + if _, err := tarball.Write(data); err != nil { + t.Fatal(err) + } + if err := tarball.Close(); err != nil { + t.Fatal(err) + } + if err := z.Close(); err != nil { + t.Fatal(err) + } + return buffer.Bytes() +} + +func bootstrapCommand(t *testing.T, base, home string) *exec.Cmd { + t.Helper() + var command *exec.Cmd + if runtime.GOOS == "windows" { + command = exec.Command("powershell.exe", "-NoProfile", "-NonInteractive", "-File", "assets/bootstrap.ps1", "-Base", base, "-Authorization", "fixture-grant") + } else { + command = exec.Command("bash", "assets/bootstrap.sh", base, "fixture-grant") + } + command.Env = append(os.Environ(), "OAC_RUNTIME_HOME="+home, "NO_PROXY=127.0.0.1", "no_proxy=127.0.0.1") + return command +} + +func TestBootstrapRecovery(t *testing.T) { + archive := bootstrapFixtureArchive(t) + for _, scenario := range []string{"metadata-503", "archive-truncated", "missing", "corrupt"} { + t.Run(scenario, func(t *testing.T) { + var metadata, downloads atomic.Int32 + sum := fmt.Sprintf("%x", sha256.Sum256(archive)) + // Windows exercises the real downloader, then rejects the fixture before execution. + if scenario == "corrupt" || runtime.GOOS == "windows" { + sum = strings.Repeat("0", 64) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".sha256") { + request := metadata.Add(1) + if scenario == "missing" { + w.WriteHeader(404) + return + } + if scenario == "metadata-503" && request == 1 { + w.WriteHeader(503) + return + } + fmt.Fprintln(w, sum) + return + } + request := downloads.Add(1) + w.Header().Set("Content-Length", fmt.Sprint(len(archive))) + if scenario == "archive-truncated" && request == 1 { + w.Write(archive[:len(archive)/2]) + return + } + w.Write(archive) + })) + defer server.Close() + home := filepath.Join(t.TempDir(), "Runtime home with spaces") + output, err := bootstrapCommand(t, server.URL+"/api/v1/agent-daemon/install/test", home).CombinedOutput() + if scenario == "missing" { + if err == nil || !bytes.Contains(output, []byte("no qualified installer")) || downloads.Load() != 0 || metadata.Load() != 1 { + t.Fatalf("missing: %v %s", err, output) + } + } else if scenario == "corrupt" || runtime.GOOS == "windows" { + if err == nil || !bytes.Contains(output, []byte("checksum mismatch")) { + t.Fatalf("corrupt: %v %s", err, output) + } + } else if err != nil { + t.Fatalf("recovery failed: %v %s", err, output) + } + if scenario == "metadata-503" && metadata.Load() != 2 { + t.Fatalf("metadata requests=%d", metadata.Load()) + } + if scenario == "archive-truncated" && downloads.Load() != 2 { + t.Fatalf("download requests=%d", downloads.Load()) + } + if _, err := os.Stat(filepath.Join(home, "native-download", "staging")); !os.IsNotExist(err) { + t.Fatalf("left staging: %v", err) + } + if bytes.Contains(output, []byte("fixture-grant")) || bytes.Contains(output, []byte("\x1b")) { + t.Fatal("secret or terminal escape in captured output") + } + }) + } +} + +func TestBootstrapDiscardsStaleStagingOnly(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX executable fixture; Windows recovery is covered separately") + } + home := t.TempDir() + staging := filepath.Join(home, "native-download", "staging") + if err := os.MkdirAll(staging, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(staging, "interrupted"), []byte("partial"), 0600); err != nil { + t.Fatal(err) + } + keep := filepath.Join(home, "workspace.txt") + if err := os.WriteFile(keep, []byte("keep"), 0600); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(404) })) + defer server.Close() + if output, err := bootstrapCommand(t, server.URL, home).CombinedOutput(); err == nil { + t.Fatalf("unexpected success %s", output) + } + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Fatal("stale staging remains") + } + if data, err := os.ReadFile(keep); err != nil || string(data) != "keep" { + t.Fatal("unrelated file changed") + } +} + +func TestBootstrapRejectsLowSpaceBeforeNetwork(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX disk-space fixture") + } + home := t.TempDir() + bin := t.TempDir() + if err := os.WriteFile(filepath.Join(bin, "df"), []byte("#!/bin/sh\nprintf 'Filesystem 1024-blocks Used Available Capacity Mounted\\nfixture 100 99 1 99%% /\\n'\n"), 0700); err != nil { + t.Fatal(err) + } + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(500) })) + defer server.Close() + command := bootstrapCommand(t, server.URL, home) + command.Env = append(command.Env, "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH")) + output, err := command.CombinedOutput() + if err == nil || !bytes.Contains(output, []byte("Not enough disk space")) || requests.Load() != 0 { + t.Fatalf("low-space guard: %v %s requests=%d", err, output, requests.Load()) + } + if _, err = os.Stat(filepath.Join(home, "native-download", "staging")); !os.IsNotExist(err) { + t.Fatal("low-space failure left staging") + } +} diff --git a/services/core/internal/nativeinstaller/bootstrap_test.go b/services/core/internal/nativeinstaller/bootstrap_test.go index 5bdfa6820..35348770b 100644 --- a/services/core/internal/nativeinstaller/bootstrap_test.go +++ b/services/core/internal/nativeinstaller/bootstrap_test.go @@ -83,7 +83,7 @@ func TestBootstrapDownloadsVerifiedPlatformAcrossHTTPSRedirect(t *testing.T) { defer core.Close() result := filepath.Join(t.TempDir(), "result") command := exec.Command("bash", "assets/bootstrap.sh", core.URL+"/api/v1/agent-daemon/install/build", "private-grant", "--harness", "codex") - command.Env = append(os.Environ(), "CURL_CA_BUNDLE="+ca, "OAC_BOOTSTRAP_TEST_RESULT="+result, "NO_PROXY=127.0.0.1", "no_proxy=127.0.0.1") + command.Env = append(os.Environ(), "OAC_RUNTIME_HOME="+t.TempDir(), "CURL_CA_BUNDLE="+ca, "OAC_BOOTSTRAP_TEST_RESULT="+result, "NO_PROXY=127.0.0.1", "no_proxy=127.0.0.1") output, err := command.CombinedOutput() if valid { if err != nil { diff --git a/services/core/internal/nativeinstaller/catalog.go b/services/core/internal/nativeinstaller/catalog.go index 6c448b321..2a40dc0a5 100644 --- a/services/core/internal/nativeinstaller/catalog.go +++ b/services/core/internal/nativeinstaller/catalog.go @@ -125,11 +125,11 @@ func psQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") func (c *Catalog) Commands(origin, authorization string) map[string]string { base := origin + "/api/v1/agent-daemon/install/" + c.Version - // Download to a private temporary file so failure cannot become an empty, - // successful shell program. Keep stdin available for installer interaction. - posix := "set -e; f=$(mktemp); trap 'rm -f \"$f\"' EXIT; curl -fsS " + shellQuote(base+"/bootstrap.sh") + " -o \"$f\"; bash \"$f\" \"$@\"" + // Hold the small bootstrap in memory so an interrupted fetch leaves no file. + // Only execute a complete successful response; preserve interactive stdin. + posix := `set -e; script=; for attempt in 1 2 3; do if script=$(curl -fsS --connect-timeout 15 --max-time 60 --max-filesize 1048576 ` + shellQuote(base+"/bootstrap.sh") + `); then break; fi; [ "$attempt" -lt 3 ] || exit 1; sleep "$attempt"; done; bash -c "$script" -- "$@"` return map[string]string{ "posix": "bash -c " + shellQuote(posix) + " -- " + shellQuote(base) + " " + shellQuote(authorization), - "powershell": "& ([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing " + psQuote(base+"/bootstrap.ps1") + " -ErrorAction Stop).Content)) -Base " + psQuote(base) + " -Authorization " + psQuote(authorization), + "powershell": "& { $source=$null; for ($attempt=1; $attempt -le 3; $attempt++) { try { $source=(Invoke-WebRequest -UseBasicParsing " + psQuote(base+"/bootstrap.ps1") + " -TimeoutSec 60 -ErrorAction Stop).Content; break } catch { if ($attempt -eq 3) { throw }; Start-Sleep -Seconds $attempt } }; & ([scriptblock]::Create($source)) -Base " + psQuote(base) + " -Authorization " + psQuote(authorization) + " @args }", } }