Harden self-hosted Runtime installation and interrupted recovery

.github/workflows/native.yml

@@ -73,6 +73,8 @@ jobs:
         run: |
           go build -ldflags "-X github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/cli.Version=$(git rev-parse HEAD)" -o "$RUNNER_TEMP/oac-daemon${{ runner.os == 'Windows' && '.exe' || '' }}" ./apps/daemon/cmd/oac-daemon
           node --test scripts/build-native-installer.test.mjs
+      - name: Verify native download bootstrap and recovery
+        run: go test ./services/core/internal/nativeinstaller -count=1 -timeout=3m
       - name: Native filesystem, authentication and process lifecycle
         run: >-
           go test -race -count=1

CONTRIBUTING.md

@@ -31,6 +31,7 @@ This guide owns how to work in the repository: documentation ownership, the repo
 | Runtime telemetry responses | [Runtime telemetry API](contracts/agents-api/runtime-observability-api.md) |
 | Runtime observation, sampling, retention and export | [Runtime observability](contracts/agents-api/runtime-observability.md) |
 | Distribution builds, Runtime image builds, CI and publication | [Maintainer guide](docs/maintainers.md) |
+| Self-hosted Runtime installation, recovery and local operation | [Self-hosted execution](docs/getting-started/self-hosted.md) |
 | Installer lifecycle, locking, generated state, managed HTTPS and downloads | [Installer design rules](deploy/install/README.md) |
 | Operator installation and alternatives | [Installation](docs/getting-started/install.md), [installation options](docs/getting-started/install-options.md) |
 | Settings, defaults, files and installation layout | [Configuration](docs/configuration.md) |

apps/daemon/internal/cli/native_bundle.go

@@ -92,7 +92,9 @@ func componentReceipt(root string) (nativeComponent, error) {
 	return c, err
 }
 
-func checkComponentFiles(directory string, c nativeComponent) error {
+var errNativeComponentMismatch = errors.New("installed files do not match the verified release")
+
+func checkComponentFiles(ctx context.Context, directory string, c nativeComponent) error {
 	root, err := os.OpenRoot(directory)
 	if err != nil {
 		return err
@@ -106,12 +108,23 @@ func checkComponentFiles(directory string, c nativeComponent) error {
 		if err != nil {
 			return err
 		}
-		info, e := f.Stat()
+		info, statErr := f.Stat()
+		if statErr != nil {
+			f.Close()
+			return statErr
+		}
+		if !info.Mode().IsRegular() {
+			f.Close()
+			return errNativeComponentMismatch
+		}
 		h := sha256.New()
-		_, copyErr := io.Copy(h, f)
+		_, copyErr := nativeCopy(ctx, h, f)
 		f.Close()
-		if e != nil || !info.Mode().IsRegular() || copyErr != nil || hex.EncodeToString(h.Sum(nil)) != expected.SHA256 || (runtime.GOOS != "windows" && expected.Executable && info.Mode().Perm()&0100 == 0) {
-			return errors.New("installed files do not match the verified release")
+		if copyErr != nil {
+			return copyErr
+		}
+		if hex.EncodeToString(h.Sum(nil)) != expected.SHA256 || (runtime.GOOS != "windows" && expected.Executable && info.Mode().Perm()&0100 == 0) {
+			return errNativeComponentMismatch
 		}
 	}
 	return nil
@@ -126,8 +139,11 @@ func installNativeComponent(ctx context.Context, source, root, name string, expe
 		if e != nil || !reflect.DeepEqual(got, expected) {
 			return fmt.Errorf("install: existing %s is incompatible; preserve it and use a separate installation directory", name)
 		}
-		if checkComponentFiles(dest, got) != nil {
-			return fmt.Errorf("install: existing %s is incomplete or modified; reinstall separately", name)
+		if err := checkComponentFiles(ctx, dest, got); err != nil {
+			if errors.Is(err, errNativeComponentMismatch) || errors.Is(err, os.ErrNotExist) {
+				return fmt.Errorf("install: existing %s is incomplete or modified; reinstall separately", name)
+			}
+			return fmt.Errorf("install: cannot verify existing %s: %w", name, err)
 		}
 		return nil
 	} else if !errors.Is(err, os.ErrNotExist) {
@@ -147,6 +163,23 @@ func installNativeComponent(ctx context.Context, source, root, name string, expe
 		return err
 	}
 	defer src.Close()
+	var required uint64
+	for name := range expected.Files {
+		if !validBundlePath(name) {
+			return errors.New("install: invalid component path")
+		}
+		info, err := src.Stat(filepath.FromSlash(name))
+		if err != nil {
+			return err
+		}
+		if !info.Mode().IsRegular() || info.Size() < 0 || uint64(info.Size()) > ^uint64(0)-required {
+			return errors.New("install: invalid component size")
+		}
+		required += uint64(info.Size())
+	}
+	if err = requireNativeSpace(parent, required); err != nil {
+		return err
+	}
 	for name, expectedFile := range expected.Files {
 		if err := ctx.Err(); err != nil {
 			return err
@@ -184,7 +217,13 @@ func installNativeComponent(ctx context.Context, source, root, name string, expe
 			err = out.Sync()
 		}
 		closeErr := out.Close()
-		if err != nil || closeErr != nil || hex.EncodeToString(h.Sum(nil)) != expectedFile.SHA256 {
+		if err != nil {
+			return fmt.Errorf("install: component copy failed: %w", err)
+		}
+		if closeErr != nil {
+			return fmt.Errorf("install: component write failed: %w", closeErr)
+		}
+		if hex.EncodeToString(h.Sum(nil)) != expectedFile.SHA256 {
 			return errors.New("install: component checksum failed")
 		}
 	}

apps/daemon/internal/cli/native_install.go

@@ -9,7 +9,6 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 	"slices"
@@ -98,7 +97,7 @@ func runInstall(rc *runContext, args []string) error {
 	ctx, stop := daemonize.NotifyContext(context.Background())
 	defer stop()
 	if err := installNativeOptions(ctx, rc, &o); err != nil {
-		return err
+		return nativeInstallError(err)
 	}
 	if o.OnboardURL != "" {
 		return finishOnboarding(ctx, rc, o)
@@ -130,6 +129,9 @@ func installNativeOptions(ctx context.Context, rc *runContext, o *nativeInstallO
 		return err
 	}
 	defer unlock()
+	if err = cleanNativeTemporaryFiles(o.Directory); err != nil {
+		return err
+	}
 	if o.OnboardURL != "" {
 		if runtimefs.ValidateLocalPath(o.Workspace) != nil {
 			return errors.New("install: Session workspace is invalid on this platform")
@@ -172,19 +174,19 @@ func installNativeOptions(ctx context.Context, rc *runContext, o *nativeInstallO
 	// Verify existing components before adding any new one; never repair or
 	// upgrade an installed dependency as a side effect of adding a Harness.
 	if len(previous.Harnesses) > 0 {
-		if err = verifyNativeComponents(o.Directory, previous.Harnesses); err != nil {
+		if err = nativeInstallPhase(rc.stdout, "Verifying installed components", func() error { return verifyNativeComponents(ctx, o.Directory, previous.Harnesses) }); err != nil {
 			return err
 		}
 	}
-	if err = installNativeBinary(o.Directory, len(previous.Harnesses) > 0); err != nil {
+	if err = nativeInstallPhase(rc.stdout, "Installing Runtime", func() error { return installNativeBinary(ctx, o.Directory, len(previous.Harnesses) > 0) }); err != nil {
 		return err
 	}
 	for _, name := range append([]string{"node"}, selected...) {
-		if err = installNativeComponent(ctx, o.Bundle, o.Directory, name, bundle.Components[name]); err != nil {
+		if err = nativeInstallPhase(rc.stdout, "Installing "+name, func() error { return installNativeComponent(ctx, o.Bundle, o.Directory, name, bundle.Components[name]) }); err != nil {
 			return err
 		}
 	}
-	if err = probeNativeInstallation(ctx, o.Directory, all); err != nil {
+	if err = nativeInstallPhase(rc.stdout, "Checking installed programs", func() error { return probeNativeInstallation(ctx, o.Directory, all) }); err != nil {
 		return err
 	}
 	if err = ctx.Err(); err != nil {
@@ -203,20 +205,26 @@ func installNativeOptions(ctx context.Context, rc *runContext, o *nativeInstallO
 	return nil
 }
 
-func verifyNativeComponents(root string, selected []string) error {
+func verifyNativeComponents(ctx context.Context, root string, selected []string) error {
 	for _, name := range append([]string{"node"}, selected...) {
 		if _, ok := nativePins[name]; !ok {
 			return errors.New("installation contains an unsupported Harness")
 		}
 		c, err := componentReceipt(nativeComponentRoot(root, name))
-		if err != nil || c.Version != nativePins[name] || len(c.Files) == 0 || checkComponentFiles(nativeComponentRoot(root, name), c) != nil {
+		if err != nil || c.Version != nativePins[name] || len(c.Files) == 0 {
 			return fmt.Errorf("installed %s is missing, modified or incompatible; reinstall separately (no automatic repair or upgrade)", name)
 		}
+		if err = checkComponentFiles(ctx, nativeComponentRoot(root, name), c); err != nil {
+			if errors.Is(err, errNativeComponentMismatch) || errors.Is(err, os.ErrNotExist) {
+				return fmt.Errorf("installed %s is missing or modified; reinstall separately", name)
+			}
+			return fmt.Errorf("install: cannot verify installed %s: %w", name, err)
+		}
 	}
 	return nil
 }
 
-func installNativeBinary(root string, existing bool) error {
+func installNativeBinary(ctx context.Context, root string, existing bool) error {
 	exe, err := os.Executable()
 	if err != nil {
 		return err
@@ -233,7 +241,7 @@ func installNativeBinary(root string, existing bool) error {
 		}
 		defer f.Close()
 		h := sha256.New()
-		_, e = io.Copy(h, f)
+		_, e = nativeCopy(ctx, h, f)
 		return hex.EncodeToString(h.Sum(nil)), e
 	}
 	want, err := digest(exe)
@@ -256,12 +264,19 @@ func installNativeBinary(root string, existing bool) error {
 		return err
 	}
 	defer in.Close()
+	info, err := in.Stat()
+	if err != nil {
+		return err
+	}
+	if err = requireNativeSpace(dir, uint64(info.Size())); err != nil {
+		return err
+	}
 	out, err := os.CreateTemp(dir, ".oac-daemon-")
 	if err != nil {
 		return err
 	}
 	defer os.Remove(out.Name())
-	_, err = io.Copy(out, in)
+	_, err = nativeCopy(ctx, out, in)
 	if err == nil {
 		err = out.Chmod(0700)
 	}
@@ -309,7 +324,7 @@ func runStart(rc *runContext, args []string) error {
 		err = errors.New("start: no installed Harnesses; rerun install with --harness")
 	}
 	if err == nil {
-		err = verifyNativeComponents(root, config.Harnesses)
+		err = verifyNativeComponents(ctx, root, config.Harnesses)
 	}
 	if err == nil {
 		err = probeNativeInstallation(ctx, root, config.Harnesses)

apps/daemon/internal/cli/native_install_io.go

@@ -0,0 +1,105 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"time"
+)
+
+// These exact temporary names are reserved by the installer, never workspace data.
+var nativeTemporaryNames = map[string]*regexp.Regexp{
+	"components": regexp.MustCompile(`^\.install-(node|codex|claude|minimax)-[0-9]+$`),
+	"bin":        regexp.MustCompile(`^\.oac-daemon-[0-9]+$`),
+	"daemon":     regexp.MustCompile(`^\.(installation\.json|executor-credential\.json)-[0-9a-f]{24}\.tmp$`),
+}
+
+// The caller holds the installation lock, including while recovering a failed copy.
+func cleanNativeTemporaryFiles(root string) error {
+	for directory, pattern := range nativeTemporaryNames {
+		parent := filepath.Join(root, directory)
+		info, err := os.Lstat(parent)
+		if errors.Is(err, os.ErrNotExist) {
+			continue
+		}
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			return fmt.Errorf("install: %s must be a directory, not a link", directory)
+		}
+		entries, err := os.ReadDir(parent)
+		if err != nil {
+			return err
+		}
+		for _, entry := range entries {
+			if !pattern.MatchString(entry.Name()) {
+				continue
+			}
+			// RemoveAll unlinks a link itself and never follows its target.
+			if err = os.RemoveAll(filepath.Join(parent, entry.Name())); err != nil {
+				return fmt.Errorf("install: cannot remove interrupted staging: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+func requireNativeSpace(directory string, size uint64) error {
+	available, err := nativeAvailableSpace(directory)
+	if err != nil {
+		return fmt.Errorf("install: cannot check free space: %w", err)
+	}
+	const reserve = 64 << 20
+	if available < reserve || size > available-reserve {
+		return errors.New("install: not enough disk space; free space in the installation directory and retry")
+	}
+	return nil
+}
+
+func nativeInstallError(err error) error {
+	if err == nil {
+		return nil
+	}
+	if nativeDiskFull(err) {
+		return errors.New("install: disk space or quota exhausted; free space and retry (completed components and credentials were preserved)")
+	}
+	if errors.Is(err, os.ErrPermission) {
+		return errors.New("install: filesystem access denied; check directory permissions and files in use, then retry with the same account")
+	}
+	return err
+}
+
+// Non-terminal output contains ordinary phase lines, with no redraws or escapes.
+func nativeInstallPhase(output io.Writer, label string, operation func() error) error {
+	fmt.Fprintln(output, label+"...")
+	file, ok := output.(*os.File)
+	if !ok || !nativeTerminal(file) {
+		return operation()
+	}
+	done, stopped := make(chan struct{}), make(chan struct{})
+	start := time.Now()
+	go func() {
+		defer close(stopped)
+		ticker := time.NewTicker(250 * time.Millisecond)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-done:
+				return
+			case <-ticker.C:
+				fmt.Fprintf(output, "\r%s... %ds", label, int(time.Since(start).Seconds()))
+			}
+		}
+	}()
+	defer func() { close(done); <-stopped; fmt.Fprintln(output) }()
+	return operation()
+}
+
+func nativeCopy(ctx context.Context, out io.Writer, in io.Reader) (int64, error) {
+	return io.Copy(out, nativeCopyReader{ctx: ctx, Reader: in})
+}

apps/daemon/internal/cli/native_install_io_unix.go

@@ -0,0 +1,25 @@
+//go:build linux || darwin
+
+package cli
+
+import (
+	"errors"
+	"golang.org/x/sys/unix"
+	"os"
+	"syscall"
+)
+
+func nativeAvailableSpace(directory string) (uint64, error) {
+	var s unix.Statfs_t
+	if err := unix.Statfs(directory, &s); err != nil {
+		return 0, err
+	}
+	return uint64(s.Bavail) * uint64(s.Bsize), nil
+}
+func nativeDiskFull(err error) bool {
+	return errors.Is(err, syscall.ENOSPC) || errors.Is(err, syscall.EDQUOT)
+}
+func nativeTerminal(f *os.File) bool {
+	_, err := unix.IoctlGetWinsize(int(f.Fd()), unix.TIOCGWINSZ)
+	return err == nil
+}

apps/daemon/internal/cli/native_install_io_windows.go

@@ -0,0 +1,24 @@
+package cli
+
+import (
+	"errors"
+	"golang.org/x/sys/windows"
+	"os"
+)
+
+func nativeAvailableSpace(directory string) (uint64, error) {
+	name, err := windows.UTF16PtrFromString(directory)
+	if err != nil {
+		return 0, err
+	}
+	var free, total, available uint64
+	err = windows.GetDiskFreeSpaceEx(name, &available, &total, &free)
+	return available, err
+}
+func nativeDiskFull(err error) bool {
+	return errors.Is(err, windows.ERROR_DISK_FULL) || errors.Is(err, windows.ERROR_HANDLE_DISK_FULL) || errors.Is(err, windows.ERROR_DISK_QUOTA_EXCEEDED)
+}
+func nativeTerminal(f *os.File) bool {
+	var mode uint32
+	return windows.GetConsoleMode(windows.Handle(f.Fd()), &mode) == nil
+}