diff --git a/cmd/ateom-microvm/checkpoint.go b/cmd/ateom-microvm/checkpoint.go index f95e5a7ee..b5ff7a714 100644 --- a/cmd/ateom-microvm/checkpoint.go +++ b/cmd/ateom-microvm/checkpoint.go @@ -21,6 +21,7 @@ import ( "fmt" "log/slog" "os" + "os/exec" "path/filepath" "time" @@ -29,19 +30,31 @@ import ( "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/proto/ateompb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -// CheckpointWorkload suspends the actor and writes a portable CH snapshot. +// CheckpointWorkload suspends the actor and writes a portable snapshot. // // Contract with atelet: after we return, atelet uploads the checkpoint dir to object // storage, then tears down bundles and resets the actor dir. // -// ateom drives the CH REST api-socket: pause -> snapshot file:// -// (config.json + state.json + sparse memory-ranges) -> tear the VMM down. Each -// container's rootfs is overlay(virtio-fs RO lower + guest-tmpfs upper), so the -// writable upper lives in guest RAM and is captured by the memory snapshot — process -// memory and rootfs writes both persist across suspend/resume. The RO lower is -// reconstructed from the OCI image at restore, so nothing rootfs-related ships here. +// What the snapshot holds depends on the requested scope: +// +// - FULL: the whole guest. ateom drives the CH REST api-socket: pause -> snapshot +// file:// (config.json + state.json + sparse memory-ranges) +// -> tear the VMM down. Each container's rootfs is overlay(virtio-fs RO lower + +// guest-tmpfs upper), so the writable upper lives in guest RAM and is captured by +// the memory snapshot — process memory and rootfs writes both persist across +// suspend/resume. The RO lower is reconstructed from the OCI image at restore, so +// nothing rootfs-related ships. Durable-dir volumes are host-backed rather than in +// guest RAM, so they ship alongside as a tar. +// - DATA: the durable-dir volumes only, as that same tar. The guest is discarded, so +// the actor cold-starts on restore with its volumes re-materialized. +// +// Either way the guest is paused first, which is what makes the tar coherent: the +// durable share is served write-through, so every completed guest write is already on +// the host and no further ones can arrive. func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.CheckpointWorkloadRequest) (*ateompb.CheckpointWorkloadResponse, error) { s.lock.Lock() defer s.lock.Unlock() @@ -54,6 +67,25 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec s.actorLogger.EmitLifecycleLog("Actor checkpointing", atespace, name, actorUID, templateNS, templateName) + // Check what the request asks for BEFORE touching the guest: these are + // properties of the request, and pausing first would leave the actor + // suspended mid-flight for a call that could never have succeeded. + // + // Durable-dir volumes are host-backed, so they are captured the same way + // under either scope — and are the ONLY thing a Data-scope snapshot captures. + durable := hasDurableVolumes(req.GetSpec().GetContainers()) + scope := req.GetScope() + switch scope { + case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: + case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: + if !durable { + return nil, status.Error(codes.FailedPrecondition, + "no durable-dir volumes found for a Data-scope snapshot") + } + default: + return nil, status.Errorf(codes.InvalidArgument, "unsupported snapshot scope: %v", scope) + } + // The actor's CH was booted by RunWorkload or relaunched by RestoreWorkload; // either way ateom owns it and tracks its api-socket. ra := s.running[actorUID] @@ -81,6 +113,61 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec return nil, fmt.Errorf("while creating checkpoint dir %q: %w", checkpointDir, err) } + // Only a Full snapshot captures the guest. A Data snapshot deliberately + // captures no VM state — no memory image, and no base-id, since nothing will + // reattach to the frozen virtio-fs lower: the actor cold-boots from the OCI + // image and gets its durable-dir volumes back from the tar below. + var dSnapshot time.Duration + if scope == ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL { + var err error + if dSnapshot, err = s.snapshotVMState(ctx, client, ra, actorUID, checkpointDir); err != nil { + return nil, err + } + } + + var dDurable time.Duration + if durable { + tDurable := time.Now() + if err := tarDurableVolumes(ctx, ateompath.DurableDirVolumeMountsDir(actorUID), checkpointDir); err != nil { + return nil, err + } + dDurable = time.Since(tDurable) + } + + // Report exactly the files we wrote so atelet ships precisely this snapshot: for + // Full, the CH snapshot (config.json + state.json + memory-ranges + base-id) plus + // any durable-dir tar; for Data, that tar alone. + snapshotFiles, err := listFiles(checkpointDir) + if err != nil { + return nil, fmt.Errorf("while listing snapshot files: %w", err) + } + + // Tear down: the actor returns to "available". Best-effort; the snapshot is + // already on disk for atelet to ship. + tTeardown := time.Now() + s.teardownActor(ctx, actorUID, ra, client) + dTeardown := time.Since(tTeardown) + delete(s.running, actorUID) + + // Tear down the per-activation actor network. + if err := s.cleanupActorNetwork(ctx); err != nil { + slog.WarnContext(ctx, "Failed to clean up actor network after checkpoint", slog.Any("err", err)) + } + + s.actorLogger.EmitLifecycleLog("Actor checkpointed", atespace, name, actorUID, templateNS, templateName) + slog.InfoContext(ctx, "Actor checkpointed", slog.String("id", actorUID), slog.Any("snapshot_files", snapshotFiles), + slog.String("scope", scope.String()), slog.Duration("pause", dPause), + slog.Duration("snapshot", dSnapshot), + // The durable-dir tar runs while the guest is paused, so its cost is part + // of the suspend latency and scales with the volume's contents. + slog.Duration("durable_dir", dDurable), slog.Duration("teardown", dTeardown)) + return &ateompb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles}, nil +} + +// snapshotVMState captures the paused guest into checkpointDir: the CH snapshot +// (config.json + state.json + memory-ranges) plus the base-id the restore side +// needs, and returns how long the snapshot itself took. +func (s *AteomService) snapshotVMState(ctx context.Context, client *ch.Client, ra *runningActor, actorUID, checkpointDir string) (time.Duration, error) { // Record the FROZEN base id (the id the guest's virtio-fs find-paths are pinned // to, /rootfs). For a cold-run actor this is its own id; for a restored // actor it is the golden id propagated via ra.baseID (set from the snapshot we @@ -93,13 +180,13 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec baseID = ra.baseID } if err := os.WriteFile(filepath.Join(checkpointDir, baseIDFile), []byte(baseID), 0o600); err != nil { - return nil, fmt.Errorf("while writing %s: %w", baseIDFile, err) + return 0, fmt.Errorf("while writing %s: %w", baseIDFile, err) } slog.InfoContext(ctx, "Snapshotting guest", slog.String("id", actorUID), slog.String("dir", checkpointDir)) tSnapshot := time.Now() if err := client.Snapshot(ctx, checkpointDir); err != nil { - return nil, fmt.Errorf("while snapshotting guest: %w", err) + return 0, fmt.Errorf("while snapshotting guest: %w", err) } dSnapshot := time.Since(tSnapshot) @@ -117,7 +204,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // CH is paused and about to be torn down, and base is discarded after. See // MergeDeltaIntoBase. (Falls back to the copying merge across filesystems.) if err := ch.MergeDeltaIntoBase(ctx, base, delta); err != nil { - return nil, fmt.Errorf("while merging OnDemand delta into restore source: %w", err) + return 0, fmt.Errorf("while merging OnDemand delta into restore source: %w", err) } slog.InfoContext(ctx, "Merged OnDemand delta into base (complete snapshot)", slog.String("id", actorUID), slog.Duration("merge", time.Since(tMerge))) @@ -126,32 +213,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // Nothing rootfs-related ships: the overlay's writable upper is a guest tmpfs, so // the actor's rootfs writes are already in the memory snapshot above, and the RO // lower is reconstructed from the OCI image at restore (it never changes). - - // Report exactly the files we wrote so atelet ships precisely the CH snapshot - // (config.json + state.json + memory-ranges + base-id). The RO base is - // reconstructed from the OCI image at restore. - snapshotFiles, err := listFiles(checkpointDir) - if err != nil { - return nil, fmt.Errorf("while listing snapshot files: %w", err) - } - - // Tear down: the actor returns to "available". Best-effort; the snapshot is - // already on disk for atelet to ship. - tTeardown := time.Now() - s.teardownActor(ctx, actorUID, ra, client) - dTeardown := time.Since(tTeardown) - delete(s.running, actorUID) - - // Tear down the per-activation actor network. - if err := s.cleanupActorNetwork(ctx); err != nil { - slog.WarnContext(ctx, "Failed to clean up actor network after checkpoint", slog.Any("err", err)) - } - - s.actorLogger.EmitLifecycleLog("Actor checkpointed", atespace, name, actorUID, templateNS, templateName) - slog.InfoContext(ctx, "Actor checkpointed", slog.String("id", actorUID), slog.Any("snapshot_files", snapshotFiles), - slog.Duration("pause", dPause), - slog.Duration("snapshot", dSnapshot), slog.Duration("teardown", dTeardown)) - return &ateompb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles}, nil + return dSnapshot, nil } // listFiles returns the (relative) names of regular files directly under dir. @@ -198,10 +260,13 @@ func (s *AteomService) teardownActor(ctx context.Context, id string, ra *running _ = ra.chCmd.Process.Kill() _, _ = ra.chCmd.Process.Wait() } - // Kill the virtiofsd serving the overlay RO lower (after CH, its only client). - if ra.vfsdCmd != nil && ra.vfsdCmd.Process != nil { - _ = ra.vfsdCmd.Process.Kill() - _, _ = ra.vfsdCmd.Process.Wait() + // Kill the virtiofsds (after CH, their only client): the overlay RO lower's + // and, when the actor has durable-dir volumes, the writable share's. + for _, cmd := range []*exec.Cmd{ra.vfsdCmd, ra.durableVfsdCmd} { + if cmd != nil && cmd.Process != nil { + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + } } } diff --git a/cmd/ateom-microvm/durable.go b/cmd/ateom-microvm/durable.go new file mode 100644 index 000000000..58676d365 --- /dev/null +++ b/cmd/ateom-microvm/durable.go @@ -0,0 +1,194 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +// Durable-dir volumes for the micro-VM runtime. +// +// A durable-dir volume is a directory whose contents outlive the actor's process +// state: it survives suspend/resume and, under the Data snapshot scope, is the +// ONLY thing captured (the workload cold-starts on restore). The host side is +// owned by atelet, which creates one directory per volume under +// ateompath.DurableDirVolumeMountsDir(actorUID) and wipes them when the actor's +// directories are reset. +// +// ateom exposes that host directory to the guest over a SECOND virtiofsd — the +// kataShared share stays strictly read-only (it is the overlay lower, served +// with cache=always), so writable volumes get their own share, mounted at +// kata.GuestDurableVolumeDir(volume) and bind-mounted from there into each +// container that declares the volume. +// +// Snapshots carry the contents as a tar of the whole per-actor directory, so the +// volume names round-trip without ateom having to learn them from the wire (the +// ateom protocol carries mount paths only). virtiofsd serves the share +// write-through (no --writeback), so once the guest is paused every completed +// guest write is already visible on the host and the tar is complete. + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/tarutil" + "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/proto/ateompb" + specs "github.com/opencontainers/runtime-spec/specs-go" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// durableTarFile is the snapshot file holding the tar of the actor's durable-dir +// volumes. Its entries are /... relative to +// ateompath.DurableDirVolumeMountsDir, so extraction restores the same layout. +const durableTarFile = "durable-dir.tar" + +// hasDurableVolumes reports whether any container mounts a durable-dir volume. +func hasDurableVolumes(containers []*ateompb.Container) bool { + for _, c := range containers { + if len(c.GetDurableDirVolumes()) > 0 { + return true + } + } + return false +} + +// resolveDurableVolumeName returns the name of the actor's durable-dir volume, +// read from the directory atelet created for it (dir is +// ateompath.DurableDirVolumeMountsDir for the actor). +// +// The ateom protocol carries only the guest mount paths, not volume names, so +// the name comes from the host layout: the directory holds exactly one +// subdirectory per volume. The ActorTemplate API allows at most one durable-dir +// volume per template, so anything other than exactly one directory means ateom +// and atelet disagree about the actor — fail rather than guess. +// +// That makes this the one place ateom depends on atelet's on-disk layout, and +// the first thing to change if more than one durable-dir volume is ever +// supported: ateompb.Container would need to carry each mount's volume name +// alongside its path, and this lookup would go away. +func resolveDurableVolumeName(dir string) (string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return "", fmt.Errorf("while reading durable-dir volumes dir %q: %w", dir, err) + } + var names []string + for _, e := range entries { + if e.IsDir() { + names = append(names, e.Name()) + } + } + switch len(names) { + case 1: + return names[0], nil + case 0: + return "", status.Errorf(codes.FailedPrecondition, + "actor declares durable-dir volume mounts but %q holds no volume directory", dir) + default: + return "", status.Errorf(codes.Unimplemented, + "ateom-microvm supports at most one durable-dir volume, found %d in %q", len(names), dir) + } +} + +// durableMounts returns the OCI mounts that expose one durable volume at each of +// a container's declared mount paths. The source is the volume's directory inside +// the guest's durable share, which the agent mounts at sandbox creation. +func durableMounts(volumeName string, mountPaths []string) []specs.Mount { + src := kata.GuestDurableVolumeDir(volumeName) + mounts := make([]specs.Mount, 0, len(mountPaths)) + for _, p := range mountPaths { + mounts = append(mounts, specs.Mount{ + Destination: p, + Source: src, + Type: "bind", + Options: []string{"rbind", "rw"}, + }) + } + return mounts +} + +// workloadSpec returns the OCI spec to start a container's overlay workload with: +// the prepared spec, plus the durable-dir binds when the actor has a volume and +// this container mounts it. +// +// The spec is copied rather than mutated so the bundle's on-disk config.json and +// the carrier's view stay as prepared — only the workload sees the binds. +func workloadSpec(c actorContainer, durableVolume string) *specs.Spec { + if durableVolume == "" || len(c.durableMountPaths) == 0 { + return c.spec + } + spec := *c.spec + spec.Mounts = append(append([]specs.Mount(nil), c.spec.Mounts...), durableMounts(durableVolume, c.durableMountPaths)...) + return &spec +} + +// stageDurableShare starts the virtiofsd serving the actor's durable-dir volumes. +// +// It serves ateompath.DurableDirVolumeMountsDir directly — no bind into the +// kataShared tree — so teardown has nothing extra to unmount. Unlike the RO +// lower's virtiofsd this one runs with cache=auto: the host contents change +// underneath the guest whenever a snapshot is restored into them. +// +// The returned cmd outlives this call (CH talks to it for the VM's lifetime); +// the caller owns it (tracked on runningActor, killed in teardownActor). +func (s *AteomService) stageDurableShare(ctx context.Context, rr resolvedRuntime, actorUID string) (*exec.Cmd, error) { + shared := ateompath.DurableDirVolumeMountsDir(actorUID) + if _, err := os.Stat(shared); err != nil { + return nil, fmt.Errorf("while checking durable-dir volumes dir %q: %w", shared, err) + } + log, _ := os.OpenFile(filepath.Join(kata.VMDir(actorUID), "virtiofsd-durable.log"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + cmd, err := kata.StartVirtiofsd(ctx, kata.VirtiofsdOptions{ + Binary: rr.virtiofsd, + SocketPath: kata.DurableVirtiofsdSocketPath(actorUID), + SharedDir: shared, + Cache: "auto", + Log: log, + }) + if err != nil { + return nil, fmt.Errorf("while starting durable-dir virtiofsd: %w", err) + } + return cmd, nil +} + +// tarDurableVolumes archives the actor's durable-dir volumes (dir) into the +// checkpoint directory. The caller must have paused the guest first: virtiofsd is +// write-through, so a completed guest write is on the host by then, but a +// running guest could still add more after the walk. +// +// Sockets the workload left behind are skipped rather than archived (tarutil +// logs them); they hold no data and the workload recreates them on start. +func tarDurableVolumes(ctx context.Context, dir, checkpointDir string) error { + if err := tarutil.Create(ctx, filepath.Join(checkpointDir, durableTarFile), dir); err != nil { + return fmt.Errorf("while archiving durable-dir volumes from %q: %w", dir, err) + } + return nil +} + +// untarDurableVolumes restores the durable-dir volumes from a snapshot into the +// actor's host directory (dir, which atelet has already created, empty). It must +// run before the durable share's virtiofsd starts, so the guest never observes +// the directory mid-restore. +func untarDurableVolumes(dir, snapshotDir string) error { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("while creating durable-dir volumes dir %q: %w", dir, err) + } + if err := tarutil.Extract(filepath.Join(snapshotDir, durableTarFile), dir); err != nil { + return fmt.Errorf("while restoring durable-dir volumes into %q: %w", dir, err) + } + return nil +} diff --git a/cmd/ateom-microvm/durable_test.go b/cmd/ateom-microvm/durable_test.go new file mode 100644 index 000000000..f8c8b1f20 --- /dev/null +++ b/cmd/ateom-microvm/durable_test.go @@ -0,0 +1,232 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" + "github.com/agent-substrate/substrate/internal/proto/ateompb" + specs "github.com/opencontainers/runtime-spec/specs-go" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestHasDurableVolumes(t *testing.T) { + tests := []struct { + name string + containers []*ateompb.Container + want bool + }{ + {name: "no containers"}, + { + name: "container without durable volumes", + containers: []*ateompb.Container{{Name: "app"}}, + }, + { + name: "one of several containers has a durable volume", + containers: []*ateompb.Container{ + {Name: "sidecar"}, + {Name: "app", DurableDirVolumes: []string{"/home/counter"}}, + }, + want: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := hasDurableVolumes(tc.containers); got != tc.want { + t.Errorf("hasDurableVolumes() = %v, want %v", got, tc.want) + } + }) + } +} + +// durableDirWith returns a durable-dir volumes directory laid out the way atelet +// prepares one: a subdirectory per volume, plus optionally a stray regular file. +func durableDirWith(t *testing.T, volumes []string, strayFile bool) string { + t.Helper() + dir := t.TempDir() + for _, v := range volumes { + if err := os.Mkdir(filepath.Join(dir, v), 0o700); err != nil { + t.Fatalf("creating volume dir %q: %v", v, err) + } + } + if strayFile { + if err := os.WriteFile(filepath.Join(dir, "not-a-volume"), nil, 0o600); err != nil { + t.Fatalf("creating stray file: %v", err) + } + } + return dir +} + +func TestResolveDurableVolumeName(t *testing.T) { + tests := []struct { + name string + volumes []string + strayFile bool + want string + wantCode codes.Code + }{ + { + name: "single volume", + volumes: []string{"data"}, + want: "data", + }, + { + name: "regular files are not volumes", + volumes: []string{"data"}, + strayFile: true, + want: "data", + }, + { + name: "no volume directory", + wantCode: codes.FailedPrecondition, + }, + { + name: "more than one volume is unsupported", + volumes: []string{"data", "other"}, + wantCode: codes.Unimplemented, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := resolveDurableVolumeName(durableDirWith(t, tc.volumes, tc.strayFile)) + if tc.wantCode != codes.OK { + if status.Code(err) != tc.wantCode { + t.Fatalf("error = %v (code %v), want code %v", err, status.Code(err), tc.wantCode) + } + return + } + if err != nil { + t.Fatalf("resolveDurableVolumeName: %v", err) + } + if got != tc.want { + t.Errorf("resolveDurableVolumeName() = %q, want %q", got, tc.want) + } + }) + } +} + +func TestResolveDurableVolumeNameMissingDir(t *testing.T) { + missing := filepath.Join(t.TempDir(), "no-such-dir") + if _, err := resolveDurableVolumeName(missing); err == nil { + t.Fatal("resolveDurableVolumeName succeeded with no durable-dir directory, want an error") + } +} + +func TestDurableVolumesRoundTrip(t *testing.T) { + // Checkpoint: a volume with data, archived while the guest is paused. + src := durableDirWith(t, []string{"data"}, false) + if err := os.WriteFile(filepath.Join(src, "data", "a.txt"), []byte("42"), 0o644); err != nil { + t.Fatalf("writing volume content: %v", err) + } + checkpointDir := t.TempDir() + if err := tarDurableVolumes(t.Context(), src, checkpointDir); err != nil { + t.Fatalf("tarDurableVolumes: %v", err) + } + if _, err := os.Stat(filepath.Join(checkpointDir, durableTarFile)); err != nil { + t.Fatalf("checkpoint is missing %s: %v", durableTarFile, err) + } + + // Restore: onto the empty directory atelet re-creates for the actor. + dst := t.TempDir() + if err := untarDurableVolumes(dst, checkpointDir); err != nil { + t.Fatalf("untarDurableVolumes: %v", err) + } + got, err := os.ReadFile(filepath.Join(dst, "data", "a.txt")) + if err != nil { + t.Fatalf("reading restored content: %v", err) + } + if string(got) != "42" { + t.Errorf("restored content = %q, want %q", got, "42") + } + // The volume name must survive the round trip: it is how the guest mount + // path is resolved after a restore onto another node. + name, err := resolveDurableVolumeName(dst) + if err != nil { + t.Fatalf("resolveDurableVolumeName after restore: %v", err) + } + if name != "data" { + t.Errorf("resolved volume name = %q, want %q", name, "data") + } +} + +func TestDurableMounts(t *testing.T) { + got := durableMounts("data", []string{"/home/counter", "/var/data"}) + want := []specs.Mount{ + {Destination: "/home/counter", Source: "/run/ateom-durable/data", Type: "bind", Options: []string{"rbind", "rw"}}, + {Destination: "/var/data", Source: "/run/ateom-durable/data", Type: "bind", Options: []string{"rbind", "rw"}}, + } + if len(got) != len(want) { + t.Fatalf("durableMounts() returned %d mounts, want %d", len(got), len(want)) + } + for i := range want { + if got[i].Destination != want[i].Destination || got[i].Source != want[i].Source || + got[i].Type != want[i].Type || !slices.Equal(got[i].Options, want[i].Options) { + t.Errorf("mount %d = %+v, want %+v", i, got[i], want[i]) + } + } + // The source must match what the agent mounts the durable share at. + if want[0].Source != kata.GuestDurableVolumeDir("data") { + t.Errorf("mount source %q does not match kata.GuestDurableVolumeDir", want[0].Source) + } +} + +func TestWorkloadSpec(t *testing.T) { + base := []specs.Mount{{Destination: "/proc", Type: "proc", Source: "proc"}} + newContainer := func(paths []string) actorContainer { + return actorContainer{ + name: "app", + spec: &specs.Spec{Mounts: slices.Clone(base)}, + durableMountPaths: paths, + } + } + + t.Run("no durable volume returns the spec unchanged", func(t *testing.T) { + c := newContainer(nil) + if got := workloadSpec(c, ""); got != c.spec { + t.Error("workloadSpec() copied the spec when there was nothing to add") + } + }) + + t.Run("container without mounts is unchanged", func(t *testing.T) { + c := newContainer(nil) + if got := workloadSpec(c, "data"); got != c.spec { + t.Error("workloadSpec() copied the spec for a container with no durable mounts") + } + }) + + t.Run("durable mounts are appended without mutating the source spec", func(t *testing.T) { + c := newContainer([]string{"/home/counter"}) + got := workloadSpec(c, "data") + if len(got.Mounts) != len(base)+1 { + t.Fatalf("workload spec has %d mounts, want %d", len(got.Mounts), len(base)+1) + } + last := got.Mounts[len(got.Mounts)-1] + if last.Destination != "/home/counter" || last.Source != kata.GuestDurableVolumeDir("data") { + t.Errorf("appended mount = %+v, want the durable volume bound at /home/counter", last) + } + // The prepared spec (shared with the carrier and the on-disk bundle) must + // not gain the bind. + if len(c.spec.Mounts) != len(base) { + t.Errorf("source spec now has %d mounts, want %d", len(c.spec.Mounts), len(base)) + } + }) +} diff --git a/cmd/ateom-microvm/internal/kata/overlay_linux.go b/cmd/ateom-microvm/internal/kata/overlay_linux.go index 4a7864519..abf0995df 100644 --- a/cmd/ateom-microvm/internal/kata/overlay_linux.go +++ b/cmd/ateom-microvm/internal/kata/overlay_linux.go @@ -46,8 +46,22 @@ const ( // guestSharedDir is where the agent mounts the kataShared tag in the guest; // per-container rootfs then lives at //rootfs. guestSharedDir = "/run/kata-containers/shared/containers/" + + // DurableFsTag is the virtio-fs tag for the actor's WRITABLE durable-dir + // share, served by a second virtiofsd (the kataShared share stays read-only). + DurableFsTag = "ateDurable" + // guestDurableDir is where the agent mounts DurableFsTag in the guest; each + // volume's contents live at / and are bind-mounted + // from there into the containers that declare the volume. + guestDurableDir = "/run/ateom-durable" ) +// GuestDurableVolumeDir is the in-guest path holding one durable volume's +// contents, i.e. the bind source for that volume's container mount points. +func GuestDurableVolumeDir(volumeName string) string { + return guestDurableDir + "/" + volumeName +} + // SharedDir is the host directory virtiofsd serves into the guest as the RO base. // Its layout (/rootfs) is what find-paths re-opens by path on restore. func SharedDir(id string) string { @@ -74,7 +88,32 @@ type VirtiofsdOptions struct { Binary string // virtiofsd executable; defaults to "virtiofsd" SocketPath string // vhost-user socket CH connects to (VirtiofsdSocketPath) SharedDir string // directory to serve (SharedDir(id)) - Log io.Writer + // Cache is virtiofsd's --cache mode. Empty defaults to "always", which is + // only correct for a strictly read-only share (see virtiofsdArgs). + Cache string + Log io.Writer +} + +// virtiofsdArgs builds the virtiofsd command line for o. +func virtiofsdArgs(o VirtiofsdOptions) []string { + cache := o.Cache + if cache == "" { + // The overlay RO lower is served strictly read-only (the carrier remounts it + // ro and the guest's overlayfs lowerdir is immutable), so aggressively cache + // it in the guest for read performance — there is no host<>guest write churn + // to invalidate. A WRITABLE share (the durable-dir volumes) must instead pass + // Cache: "auto", because cache=always would serve stale data once the host + // side changes underneath the guest (e.g. contents restored from a snapshot). + cache = "always" + } + return []string{ + "--socket-path=" + o.SocketPath, + "--shared-dir=" + o.SharedDir, + "--cache=" + cache, + "--thread-pool-size=1", + "--announce-submounts", + "--migration-mode", "find-paths", + } } // StartVirtiofsd launches virtiofsd in find-paths migration mode serving o.SharedDir @@ -86,22 +125,7 @@ func StartVirtiofsd(ctx context.Context, o VirtiofsdOptions) (*exec.Cmd, error) bin = "virtiofsd" } _ = os.Remove(o.SocketPath) - cmd := exec.Command(bin, - "--socket-path="+o.SocketPath, - "--shared-dir="+o.SharedDir, - // The shared dir is served strictly read-only (the overlay's RO lower; the - // carrier remounts it ro and the guest's overlayfs lowerdir is immutable), so - // aggressively cache it in the guest for read performance — there is no - // host<>guest write churn to invalidate. - // TODO: cache=always serves stale data for any writable virtio-fs mount. If we - // later need one (e.g. projected volumes), prefer keeping this mount fully - // read-only and writing such volumes via a separate mechanism (e.g. a writer - // that execs into the sandbox) rather than dropping back to cache=auto/none. - "--cache=always", - "--thread-pool-size=1", - "--announce-submounts", - "--migration-mode", "find-paths", - ) + cmd := exec.Command(bin, virtiofsdArgs(o)...) cmd.Stdout = o.Log cmd.Stderr = o.Log if err := cmd.Start(); err != nil { @@ -166,16 +190,28 @@ func ReconstructSharedDirFromImage(ctx context.Context, bundleRootfs, restoreID, // CreateSandboxForActor creates the guest sandbox with the kataShared virtio-fs mount // (the RO base backing every container's rootfs). Mirrors kata startSandbox. -func (a *AgentClient) CreateSandboxForActor(ctx context.Context, sandboxID, hostname string) error { +// +// withDurableShare additionally mounts the writable durable-dir share, whose +// per-volume subdirectories the containers bind-mount at their declared paths. +func (a *AgentClient) CreateSandboxForActor(ctx context.Context, sandboxID, hostname string, withDurableShare bool) error { + storages := []*agentpb.Storage{{ + Driver: virtioFSDriver, + Source: FsTag, + Fstype: typeVirtioFS, + MountPoint: guestSharedDir, + }} + if withDurableShare { + storages = append(storages, &agentpb.Storage{ + Driver: virtioFSDriver, + Source: DurableFsTag, + Fstype: typeVirtioFS, + MountPoint: guestDurableDir, + }) + } return a.CreateSandbox(ctx, &agentpb.CreateSandboxRequest{ Hostname: hostname, SandboxId: sandboxID, - Storages: []*agentpb.Storage{{ - Driver: virtioFSDriver, - Source: FsTag, - Fstype: typeVirtioFS, - MountPoint: guestSharedDir, - }}, + Storages: storages, }) } diff --git a/cmd/ateom-microvm/internal/kata/overlay_linux_test.go b/cmd/ateom-microvm/internal/kata/overlay_linux_test.go new file mode 100644 index 000000000..c6dfb56a8 --- /dev/null +++ b/cmd/ateom-microvm/internal/kata/overlay_linux_test.go @@ -0,0 +1,65 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kata + +import ( + "slices" + "testing" +) + +func TestVirtiofsdArgs(t *testing.T) { + tests := []struct { + name string + opts VirtiofsdOptions + wantCache string + }{ + { + name: "RO lower defaults to cache=always", + opts: VirtiofsdOptions{SocketPath: "/run/vm/virtiofsd.sock", SharedDir: "/run/shared"}, + wantCache: "--cache=always", + }, + { + name: "writable durable share overrides the cache mode", + opts: VirtiofsdOptions{ + SocketPath: "/run/vm/virtiofsd-durable.sock", + SharedDir: "/var/lib/ateom-gvisor/actors/uid/durable-dir", + Cache: "auto", + }, + wantCache: "--cache=auto", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + args := virtiofsdArgs(tc.opts) + if !slices.Contains(args, tc.wantCache) { + t.Errorf("args %v do not contain %q", args, tc.wantCache) + } + for _, want := range []string{ + "--socket-path=" + tc.opts.SocketPath, + "--shared-dir=" + tc.opts.SharedDir, + // find-paths re-opens the guest's open files by path on + // restore; both shares depend on it. + "--migration-mode", + "find-paths", + } { + if !slices.Contains(args, want) { + t.Errorf("args %v do not contain %q", args, want) + } + } + }) + } +} diff --git a/cmd/ateom-microvm/internal/kata/restore.go b/cmd/ateom-microvm/internal/kata/restore.go index 2dccbca8f..0b71bea9d 100644 --- a/cmd/ateom-microvm/internal/kata/restore.go +++ b/cmd/ateom-microvm/internal/kata/restore.go @@ -29,3 +29,9 @@ func VMDir(id string) string { return filepath.Join(vcVMDir, id) } // VsockSocketPath is the hybrid-vsock socket the CH snapshot's vsock device // references; CH recreates the listener here on restore. func VsockSocketPath(id string) string { return filepath.Join(VMDir(id), "clh.sock") } + +// DurableVirtiofsdSocketPath is the vhost-user-fs socket for the actor's writable +// durable-dir share, served by a second virtiofsd alongside the RO lower's. +func DurableVirtiofsdSocketPath(id string) string { + return filepath.Join(VMDir(id), "virtiofsd-durable.sock") +} diff --git a/cmd/ateom-microvm/internal/tarutil/fifo_linux.go b/cmd/ateom-microvm/internal/tarutil/fifo_linux.go new file mode 100644 index 000000000..c7496fc9e --- /dev/null +++ b/cmd/ateom-microvm/internal/tarutil/fifo_linux.go @@ -0,0 +1,48 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarutil + +import ( + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +// createFifo creates a FIFO at name, a path relative to root, with mode's +// permission bits. +// +// os.Root has no Mkfifo, so this opens name's parent directory THROUGH root — +// which refuses to traverse a symlink out of the tree, the same guarantee the +// rest of extraction relies on — and creates the FIFO relative to that +// directory's descriptor. The final element is a single path component, so it +// cannot walk anywhere from there. +func createFifo(root *os.Root, name string, mode os.FileMode) error { + dir, base := filepath.Split(name) + if dir == "" { + dir = "." + } + parent, err := root.Open(filepath.Clean(dir)) + if err != nil { + return fmt.Errorf("opening parent directory of %q: %w", name, err) + } + defer parent.Close() + + if err := unix.Mkfifoat(int(parent.Fd()), base, uint32(mode.Perm())); err != nil { + return fmt.Errorf("creating fifo %q: %w", name, err) + } + return nil +} diff --git a/cmd/ateom-microvm/internal/tarutil/owner_linux.go b/cmd/ateom-microvm/internal/tarutil/owner_linux.go new file mode 100644 index 000000000..9d33a0785 --- /dev/null +++ b/cmd/ateom-microvm/internal/tarutil/owner_linux.go @@ -0,0 +1,85 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarutil + +import ( + "archive/tar" + "errors" + "fmt" + "os" + "syscall" +) + +// inodeKey identifies an inode within one filesystem, so hardlinks to the same +// file can be recognized while distinct files that share an inode number across +// devices are not conflated. +type inodeKey struct { + dev uint64 + ino uint64 +} + +// inodeOf returns info's inode identity, or ok=false on a platform or file +// where it is unavailable (the caller then archives the contents rather than +// emitting a hardlink). +func inodeOf(info os.FileInfo) (inodeKey, bool) { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return inodeKey{}, false + } + return inodeKey{dev: uint64(st.Dev), ino: uint64(st.Ino)}, true +} + +// nlinkOf returns the file's link count, or 1 when unavailable. +func nlinkOf(info os.FileInfo) uint64 { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 1 + } + return uint64(st.Nlink) +} + +// setOwner records the on-disk uid/gid in the header. tar.FileInfoHeader leaves +// them zero, which would silently re-own every restored file to root. +func setOwner(hdr *tar.Header, info os.FileInfo) { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return + } + hdr.Uid = int(st.Uid) + hdr.Gid = int(st.Gid) + // Names are deliberately left empty: the guest's passwd/group namespace is + // not the host's, so only the numeric ids are meaningful. + hdr.Uname = "" + hdr.Gname = "" +} + +// lchownEntry restores an entry's ownership without following symlinks. +// +// Changing a file's owner requires privilege. The production caller (ateom) is +// root in its worker pod, so ownership is restored faithfully and any EPERM +// there signals a real problem worth failing on. An unprivileged process cannot +// chown at all, so rather than making the package unusable outside a root +// context (tests, local tooling), EPERM is tolerated there and the extracted +// files simply belong to the extracting user. +func lchownEntry(root *os.Root, name string, hdr *tar.Header) error { + err := root.Lchown(name, hdr.Uid, hdr.Gid) + if err == nil { + return nil + } + if errors.Is(err, os.ErrPermission) && os.Geteuid() != 0 { + return nil + } + return fmt.Errorf("restoring ownership of %q to %d:%d: %w", name, hdr.Uid, hdr.Gid, err) +} diff --git a/cmd/ateom-microvm/internal/tarutil/tarutil.go b/cmd/ateom-microvm/internal/tarutil/tarutil.go new file mode 100644 index 000000000..7b1a6d319 --- /dev/null +++ b/cmd/ateom-microvm/internal/tarutil/tarutil.go @@ -0,0 +1,374 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tarutil archives and restores a directory tree as a tar file, +// preserving the metadata a workload's data directory depends on: modes, +// ownership, modification times, symlinks, hardlinks, and FIFOs. +// +// It exists for snapshotting durable-dir volumes (see cmd/ateom-microvm): the +// contents are written by the sandboxed workload under arbitrary uids, shipped +// to object storage, and restored — possibly onto another node — where the +// workload must see them unchanged. +// +// Extraction is confined to the destination with os.Root, so a crafted archive +// cannot write outside it via "..", an absolute path, or a symlink. +package tarutil + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + "sort" + "strings" +) + +// Create writes a tar archive of srcDir's contents to tarPath. Entry names are +// relative to srcDir, so extracting into another directory reproduces the tree. +// srcDir itself is not an entry. +// +// Regular files, directories, symlinks, and FIFOs are archived with their mode, +// ownership, and modification time. A file with multiple links inside srcDir is +// archived once and referenced as a hardlink thereafter. Sockets are skipped +// (see writeTree). A device node is an error rather than silent data loss. +func Create(ctx context.Context, tarPath, srcDir string) error { + f, err := os.Create(tarPath) + if err != nil { + return fmt.Errorf("creating tar %q: %w", tarPath, err) + } + defer f.Close() + + tw := tar.NewWriter(f) + if err := writeTree(ctx, tw, srcDir); err != nil { + return err + } + if err := tw.Close(); err != nil { + return fmt.Errorf("closing tar %q: %w", tarPath, err) + } + // Durable-dir tars are handed to atelet for upload as soon as we return, so + // flush to disk rather than trusting the page cache to outlive us. + if err := f.Sync(); err != nil { + return fmt.Errorf("syncing tar %q: %w", tarPath, err) + } + return nil +} + +// writeTree walks srcDir in lexical order (filepath.WalkDir) and writes one +// entry per path. The deterministic order keeps archives of identical trees +// byte-comparable, which makes snapshot diffs meaningful. +func writeTree(ctx context.Context, tw *tar.Writer, srcDir string) error { + // Maps an already-archived multi-link inode to the name it was archived + // under, so later links become tar hardlink entries instead of copies. + linked := map[inodeKey]string{} + + return filepath.WalkDir(srcDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(srcDir, path) + if err != nil { + return err + } + if rel == "." { + return nil + } + + info, err := d.Info() + if err != nil { + return err + } + + // Skipped before the header is built, because archive/tar cannot + // represent a socket at all. A socket carries no data and is meaningless + // without the process listening on it, which is gone by the time this + // archive is read; workloads recreate theirs on start. GNU tar and + // containerd's archiver both skip sockets for that reason. Failing + // instead would be worse than useless here: agents leave sockets lying + // around in their data directories (ssh-agent, gpg-agent, language + // servers), and one of them would make every checkpoint fail, stranding + // the actor on its worker with no way to suspend it. + if info.Mode()&os.ModeSocket != 0 { + slog.WarnContext(ctx, "Skipping socket while archiving directory", + slog.String("path", path), slog.String("root", srcDir)) + return nil + } + + link := "" + if info.Mode()&os.ModeSymlink != 0 { + if link, err = os.Readlink(path); err != nil { + return fmt.Errorf("reading symlink %q: %w", path, err) + } + } + hdr, err := tar.FileInfoHeader(info, link) + if err != nil { + return fmt.Errorf("building tar header for %q: %w", path, err) + } + hdr.Name = filepath.ToSlash(rel) + if d.IsDir() { + hdr.Name += "/" + } + setOwner(hdr, info) + + switch { + case info.Mode().IsRegular(): + // A second link to an already-archived inode: record the link and + // skip the contents. + if key, ok := inodeOf(info); ok && info.Sys() != nil && nlinkOf(info) > 1 { + if target, seen := linked[key]; seen { + hdr.Typeflag = tar.TypeLink + hdr.Linkname = target + hdr.Size = 0 + return tw.WriteHeader(hdr) + } + linked[key] = hdr.Name + } + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("writing tar header for %q: %w", path, err) + } + return copyFileInto(tw, path) + + case d.IsDir(), info.Mode()&os.ModeSymlink != 0, info.Mode()&os.ModeNamedPipe != 0: + // FileInfoHeader already gave a FIFO Typeflag TypeFifo and size 0. + return tw.WriteHeader(hdr) + + default: + // Device nodes reach here. They need privilege to create, so one in a + // workload's data directory means something unexpected happened — + // worth failing on rather than silently dropping or re-creating it + // under a later restore. + return fmt.Errorf("unsupported file type %v at %q", info.Mode().Type(), path) + } + }) +} + +// copyFileInto streams path's contents into the archive. +func copyFileInto(tw *tar.Writer, path string) error { + in, err := os.Open(path) + if err != nil { + return fmt.Errorf("opening %q: %w", path, err) + } + defer in.Close() + if _, err := io.Copy(tw, in); err != nil { + return fmt.Errorf("archiving contents of %q: %w", path, err) + } + return nil +} + +// Extract unpacks tarPath into dstDir, which must already exist. Modes, +// ownership, modification times, symlinks, and hardlinks are restored. +// +// All writes go through an os.Root rooted at dstDir, so entries naming a path +// outside it — via "..", an absolute path, or a symlink planted earlier in the +// same archive — are refused rather than followed. An entry that collides with +// an existing path replaces it ("later entry wins", standard tar semantics), +// except that an existing directory is kept when the entry is also a directory. +func Extract(tarPath, dstDir string) error { + f, err := os.Open(tarPath) + if err != nil { + return fmt.Errorf("opening tar %q: %w", tarPath, err) + } + defer f.Close() + + root, err := os.OpenRoot(dstDir) + if err != nil { + return fmt.Errorf("opening destination %q: %w", dstDir, err) + } + defer root.Close() + + // Directories are created owner-writable so their children can be written + // even when the archive marks them read-only; the recorded modes and times + // are applied after every child exists (see restoreDirMeta). + dirs := map[string]*tar.Header{} + + tr := tar.NewReader(f) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("reading tar %q: %w", tarPath, err) + } + name, skip, err := cleanTarName(hdr.Name) + if err != nil { + return fmt.Errorf("invalid entry in tar %q: %w", tarPath, err) + } + if skip { + continue + } + if err := extractEntry(root, tr, hdr, name, dirs); err != nil { + return err + } + } + return restoreDirMeta(root, dirs) +} + +// extractEntry materializes one archive entry under root. +func extractEntry(root *os.Root, tr *tar.Reader, hdr *tar.Header, name string, dirs map[string]*tar.Header) error { + mode := hdr.FileInfo().Mode().Perm() + + switch hdr.Typeflag { + case tar.TypeDir: + if err := root.Mkdir(name, mode|0o700); err != nil && !errors.Is(err, os.ErrExist) { + return fmt.Errorf("creating directory %q: %w", name, err) + } + dirs[name] = hdr + return nil + + case tar.TypeReg: + if err := replaceExisting(root, name); err != nil { + return err + } + out, err := root.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + if err != nil { + return fmt.Errorf("creating file %q: %w", name, err) + } + _, copyErr := io.Copy(out, tr) + closeErr := out.Close() + if copyErr != nil { + return fmt.Errorf("writing contents of %q: %w", name, copyErr) + } + if closeErr != nil { + return fmt.Errorf("closing %q: %w", name, closeErr) + } + return restoreMeta(root, name, hdr) + + case tar.TypeSymlink: + if err := replaceExisting(root, name); err != nil { + return err + } + if err := root.Symlink(hdr.Linkname, name); err != nil { + return fmt.Errorf("creating symlink %q -> %q: %w", name, hdr.Linkname, err) + } + // Only ownership applies to a symlink itself; its mode is meaningless + // and its times would follow the target. + return lchownEntry(root, name, hdr) + + case tar.TypeFifo: + if err := replaceExisting(root, name); err != nil { + return err + } + if err := createFifo(root, name, mode); err != nil { + return err + } + return restoreMeta(root, name, hdr) + + case tar.TypeLink: + target, skip, err := cleanTarName(hdr.Linkname) + if err != nil || skip { + return fmt.Errorf("invalid hardlink target %q for %q", hdr.Linkname, name) + } + if err := replaceExisting(root, name); err != nil { + return err + } + if err := root.Link(target, name); err != nil { + return fmt.Errorf("creating hardlink %q -> %q: %w", name, target, err) + } + return nil + + default: + return fmt.Errorf("unsupported tar entry type %q at %q", string([]byte{hdr.Typeflag}), name) + } +} + +// replaceExisting removes whatever currently occupies name so the new entry +// does not write through a symlink, truncate a shared inode, or collide with a +// directory. +func replaceExisting(root *os.Root, name string) error { + if _, err := root.Lstat(name); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("checking existing path %q: %w", name, err) + } + if err := root.RemoveAll(name); err != nil { + return fmt.Errorf("replacing existing path %q: %w", name, err) + } + return nil +} + +// restoreDirMeta applies the archived modes, ownership, and times to extracted +// directories, deepest first: a directory's path is always longer than its +// parent's, so length-descending order restores children before the parent's +// mode can make them unreachable. +func restoreDirMeta(root *os.Root, dirs map[string]*tar.Header) error { + names := make([]string, 0, len(dirs)) + for name := range dirs { + names = append(names, name) + } + sort.Slice(names, func(i, j int) bool { return len(names[i]) > len(names[j]) }) + for _, name := range names { + if err := restoreMeta(root, name, dirs[name]); err != nil { + return err + } + } + return nil +} + +// restoredMode is the mode an extracted entry ends up with: the permission bits +// plus setuid, setgid, and sticky, which FileMode.Perm() alone would drop. +// +// The archive records them, so dropping them would make the round trip silently +// lossy. The case that bites is a setgid data directory (0o2775): the workload +// relies on group inheritance for files its different uids create, and losing +// the bit only surfaces later, when the next file lands with the wrong group. +// These bits govern behavior inside the sandbox — the host never executes a +// workload's files — and a running workload can set them on its own data +// anyway, so carrying them across a suspend/resume grants it nothing new. +func restoredMode(hdr *tar.Header) os.FileMode { + return hdr.FileInfo().Mode() & (fs.ModePerm | fs.ModeSetuid | fs.ModeSetgid | fs.ModeSticky) +} + +// restoreMeta applies ownership, mode, and modification time to an extracted +// path. Ownership is applied first because chowning a file clears its setuid +// and setgid bits, which the chmod below then puts back. +func restoreMeta(root *os.Root, name string, hdr *tar.Header) error { + if err := lchownEntry(root, name, hdr); err != nil { + return err + } + if err := root.Chmod(name, restoredMode(hdr)); err != nil { + return fmt.Errorf("restoring mode on %q: %w", name, err) + } + if !hdr.ModTime.IsZero() { + if err := root.Chtimes(name, hdr.ModTime, hdr.ModTime); err != nil { + return fmt.Errorf("restoring times on %q: %w", name, err) + } + } + return nil +} + +// cleanTarName validates an archive entry name and returns it relative and +// slash-free of surprises. skip is true for entries that name nothing ("" or +// "."), which some tar writers emit for the archive root. +func cleanTarName(name string) (cleaned string, skip bool, err error) { + if name == "" { + return "", true, nil + } + cleaned = filepath.Clean(filepath.FromSlash(name)) + cleaned = strings.TrimPrefix(cleaned, string(filepath.Separator)) + if cleaned == "" || cleaned == "." { + return "", true, nil + } + if !filepath.IsLocal(cleaned) { + return "", false, fmt.Errorf("not a local path: %q", name) + } + return cleaned, false, nil +} diff --git a/cmd/ateom-microvm/internal/tarutil/tarutil_test.go b/cmd/ateom-microvm/internal/tarutil/tarutil_test.go new file mode 100644 index 000000000..d8b72c4ae --- /dev/null +++ b/cmd/ateom-microvm/internal/tarutil/tarutil_test.go @@ -0,0 +1,499 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarutil + +import ( + "archive/tar" + "errors" + "net" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/roottest" + "golang.org/x/sys/unix" +) + +// writeTar builds a tar file at path from the given headers; a header with a +// non-empty body is written as a regular file with that content. +func writeTar(t *testing.T, path string, entries ...tar.Header) { + t.Helper() + f, err := os.Create(path) + if err != nil { + t.Fatalf("creating tar: %v", err) + } + defer f.Close() + tw := tar.NewWriter(f) + for _, hdr := range entries { + if err := tw.WriteHeader(&hdr); err != nil { + t.Fatalf("writing header %q: %v", hdr.Name, err) + } + if hdr.Size > 0 { + if _, err := tw.Write([]byte(strings.Repeat("x", int(hdr.Size)))); err != nil { + t.Fatalf("writing body %q: %v", hdr.Name, err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatalf("closing tar: %v", err) + } +} + +func TestRoundTrip(t *testing.T) { + src := t.TempDir() + mustWrite := func(rel, content string, mode os.FileMode) string { + t.Helper() + p := filepath.Join(src, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatalf("mkdir for %q: %v", rel, err) + } + if err := os.WriteFile(p, []byte(content), mode); err != nil { + t.Fatalf("writing %q: %v", rel, err) + } + if err := os.Chmod(p, mode); err != nil { + t.Fatalf("chmod %q: %v", rel, err) + } + return p + } + + mustWrite("a.txt", "hello", 0o644) + mustWrite("sub/b.txt", "nested", 0o600) + // A read-only directory: extraction must still populate it, then restore + // the mode. + mustWrite("ro/c.txt", "in-readonly", 0o444) + if err := os.Chmod(filepath.Join(src, "ro"), 0o500); err != nil { + t.Fatalf("chmod ro dir: %v", err) + } + // TempDir cleanup cannot unlink children of a non-writable directory, so + // make both copies writable again once the test is done. + restoreWritable := func(dir string) { + t.Cleanup(func() { _ = os.Chmod(filepath.Join(dir, "ro"), 0o700) }) + } + restoreWritable(src) + if err := os.Mkdir(filepath.Join(src, "empty"), 0o755); err != nil { + t.Fatalf("mkdir empty: %v", err) + } + if err := os.Symlink("a.txt", filepath.Join(src, "link")); err != nil { + t.Fatalf("symlink: %v", err) + } + if err := os.Link(filepath.Join(src, "a.txt"), filepath.Join(src, "hard.txt")); err != nil { + t.Fatalf("hardlink: %v", err) + } + mtime := time.Unix(1600000000, 0) + if err := os.Chtimes(filepath.Join(src, "a.txt"), mtime, mtime); err != nil { + t.Fatalf("chtimes: %v", err) + } + + tarPath := filepath.Join(t.TempDir(), "out.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + restoreWritable(dst) + + t.Run("contents", func(t *testing.T) { + for rel, want := range map[string]string{ + "a.txt": "hello", + "sub/b.txt": "nested", + "ro/c.txt": "in-readonly", + "hard.txt": "hello", + } { + got, err := os.ReadFile(filepath.Join(dst, rel)) + if err != nil { + t.Errorf("reading %q: %v", rel, err) + continue + } + if string(got) != want { + t.Errorf("%q = %q, want %q", rel, got, want) + } + } + }) + + t.Run("modes", func(t *testing.T) { + for rel, want := range map[string]os.FileMode{ + "a.txt": 0o644, + "sub/b.txt": 0o600, + "ro/c.txt": 0o444, + "ro": 0o500, + "empty": 0o755, + } { + st, err := os.Lstat(filepath.Join(dst, rel)) + if err != nil { + t.Errorf("lstat %q: %v", rel, err) + continue + } + if got := st.Mode().Perm(); got != want { + t.Errorf("%q mode = %v, want %v", rel, got, want) + } + } + }) + + t.Run("mtime", func(t *testing.T) { + st, err := os.Stat(filepath.Join(dst, "a.txt")) + if err != nil { + t.Fatalf("stat: %v", err) + } + if !st.ModTime().Equal(mtime) { + t.Errorf("a.txt mtime = %v, want %v", st.ModTime(), mtime) + } + }) + + t.Run("symlink", func(t *testing.T) { + got, err := os.Readlink(filepath.Join(dst, "link")) + if err != nil { + t.Fatalf("readlink: %v", err) + } + if got != "a.txt" { + t.Errorf("link target = %q, want %q", got, "a.txt") + } + }) + + t.Run("hardlink shares inode", func(t *testing.T) { + a, err := os.Stat(filepath.Join(dst, "a.txt")) + if err != nil { + t.Fatalf("stat a.txt: %v", err) + } + h, err := os.Stat(filepath.Join(dst, "hard.txt")) + if err != nil { + t.Fatalf("stat hard.txt: %v", err) + } + if !os.SameFile(a, h) { + t.Error("hard.txt is a copy, want the same inode as a.txt") + } + }) +} + +func TestCreateEmptyDir(t *testing.T) { + tarPath := filepath.Join(t.TempDir(), "empty.tar") + if err := Create(t.Context(), tarPath, t.TempDir()); err != nil { + t.Fatalf("Create: %v", err) + } + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + entries, err := os.ReadDir(dst) + if err != nil { + t.Fatalf("readdir: %v", err) + } + if len(entries) != 0 { + t.Errorf("extracted %d entries from an empty archive, want 0", len(entries)) + } +} + +func TestRoundTripFifo(t *testing.T) { + src := t.TempDir() + if err := unix.Mkfifo(filepath.Join(src, "pipe"), 0o640); err != nil { + t.Fatalf("creating fifo: %v", err) + } + + tarPath := filepath.Join(t.TempDir(), "fifo.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + + st, err := os.Lstat(filepath.Join(dst, "pipe")) + if err != nil { + t.Fatalf("lstat restored fifo: %v", err) + } + if st.Mode()&os.ModeNamedPipe == 0 { + t.Errorf("restored entry mode = %v, want a fifo", st.Mode()) + } + if got := st.Mode().Perm(); got != 0o640 { + t.Errorf("restored fifo mode = %v, want %v", got, os.FileMode(0o640)) + } +} + +// TestCreateSkipsSockets covers the availability property that matters most for +// snapshots: a socket in the archived tree — which agents leave behind routinely +// — must not fail the archive, because that would make the actor impossible to +// suspend. +func TestCreateSkipsSockets(t *testing.T) { + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "keep.txt"), []byte("data"), 0o644); err != nil { + t.Fatalf("writing file: %v", err) + } + l, err := net.Listen("unix", filepath.Join(src, "agent.sock")) + if err != nil { + t.Fatalf("creating socket: %v", err) + } + defer l.Close() + + tarPath := filepath.Join(t.TempDir(), "sock.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + + // The rest of the tree survives; the socket itself is dropped. + got, err := os.ReadFile(filepath.Join(dst, "keep.txt")) + if err != nil { + t.Fatalf("reading archived file: %v", err) + } + if string(got) != "data" { + t.Errorf("keep.txt = %q, want %q", got, "data") + } + if _, err := os.Lstat(filepath.Join(dst, "agent.sock")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("socket was archived (lstat err = %v), want it skipped", err) + } +} + +// TestRoundTripSpecialModeBits pins setuid/setgid/sticky across a round trip. +// FileMode.Perm() silently drops them, so without this the archive would record +// bits that extraction throws away — and a setgid data directory would come back +// from a suspend having quietly lost its group inheritance. +func TestRoundTripSpecialModeBits(t *testing.T) { + src := t.TempDir() + // Modes are applied after creation because MkdirAll and WriteFile mask the + // special bits against the umask. + setgidDir := filepath.Join(src, "shared") + if err := os.Mkdir(setgidDir, 0o775); err != nil { + t.Fatalf("mkdir: %v", err) + } + stickyDir := filepath.Join(src, "tmp") + if err := os.Mkdir(stickyDir, 0o777); err != nil { + t.Fatalf("mkdir: %v", err) + } + setuidFile := filepath.Join(src, "tool") + if err := os.WriteFile(setuidFile, []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatalf("writing file: %v", err) + } + for path, mode := range map[string]os.FileMode{ + setgidDir: os.ModeDir | os.ModeSetgid | 0o775, + stickyDir: os.ModeDir | os.ModeSticky | 0o777, + setuidFile: os.ModeSetuid | 0o755, + } { + if err := os.Chmod(path, mode); err != nil { + t.Fatalf("chmod %q: %v", path, err) + } + } + + tarPath := filepath.Join(t.TempDir(), "modes.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + + for name, want := range map[string]os.FileMode{ + "shared": os.ModeDir | os.ModeSetgid | 0o775, + "tmp": os.ModeDir | os.ModeSticky | 0o777, + "tool": os.ModeSetuid | 0o755, + } { + st, err := os.Lstat(filepath.Join(dst, name)) + if err != nil { + t.Errorf("lstat %q: %v", name, err) + continue + } + if got := st.Mode(); got != want { + t.Errorf("%q mode = %v, want %v", name, got, want) + } + } +} + +// TestRoundTripOwnership covers the reason this package exists rather than a +// plain tar: durable-dir contents are written by the sandboxed workload under +// its own uids, and a restore that re-owned everything to root would hand the +// workload back files it cannot write. Needs root, since only root can give a +// file away. +func TestRoundTripOwnership(t *testing.T) { + roottest.Require(t, "restoring file ownership requires CAP_CHOWN") + + const uid, gid = 1234, 5678 + src := t.TempDir() + path := filepath.Join(src, "owned.txt") + if err := os.WriteFile(path, []byte("data"), 0o644); err != nil { + t.Fatalf("writing file: %v", err) + } + if err := os.Chown(path, uid, gid); err != nil { + t.Fatalf("chown: %v", err) + } + + tarPath := filepath.Join(t.TempDir(), "owned.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + + st, err := os.Stat(filepath.Join(dst, "owned.txt")) + if err != nil { + t.Fatalf("stat restored file: %v", err) + } + sys, ok := st.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal("stat has no syscall.Stat_t") + } + if sys.Uid != uid || sys.Gid != gid { + t.Errorf("restored ownership = %d:%d, want %d:%d", sys.Uid, sys.Gid, uid, gid) + } +} + +func TestCreateRejectsDeviceNode(t *testing.T) { + roottest.Require(t, "creating a device node requires root") + + src := t.TempDir() + // /dev/null, the cheapest character device to plant. + if err := unix.Mknod(filepath.Join(src, "null"), unix.S_IFCHR|0o666, int(unix.Mkdev(1, 3))); err != nil { + t.Fatalf("creating device node: %v", err) + } + err := Create(t.Context(), filepath.Join(t.TempDir(), "dev.tar"), src) + if err == nil { + t.Fatal("Create succeeded on a device node, want an error") + } + if !strings.Contains(err.Error(), "unsupported file type") { + t.Errorf("error = %v, want it to mention an unsupported file type", err) + } +} + +func TestExtractRejectsEscapes(t *testing.T) { + tests := []struct { + name string + entries []tar.Header + }{ + { + name: "parent traversal", + entries: []tar.Header{{Name: "../escape.txt", Mode: 0o644, Size: 3, Typeflag: tar.TypeReg}}, + }, + { + name: "nested parent traversal", + entries: []tar.Header{{Name: "sub/../../escape.txt", Mode: 0o644, Size: 3, Typeflag: tar.TypeReg}}, + }, + { + name: "write through symlink out of tree", + entries: []tar.Header{ + {Name: "out", Linkname: "/tmp", Mode: 0o777, Typeflag: tar.TypeSymlink}, + {Name: "out/escape.txt", Mode: 0o644, Size: 3, Typeflag: tar.TypeReg}, + }, + }, + { + name: "hardlink to a path outside the tree", + entries: []tar.Header{{Name: "link", Linkname: "../../etc/passwd", Typeflag: tar.TypeLink}}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tarPath := filepath.Join(t.TempDir(), "bad.tar") + writeTar(t, tarPath, tc.entries...) + dst := t.TempDir() + if err := Extract(tarPath, dst); err == nil { + t.Fatal("Extract succeeded, want an error") + } + // Whatever the archive intended, nothing may exist outside dst. + if _, err := os.Lstat(filepath.Join(filepath.Dir(dst), "escape.txt")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("escape.txt exists outside the destination (err=%v)", err) + } + }) + } +} + +func TestExtractAbsoluteNameStaysInside(t *testing.T) { + tarPath := filepath.Join(t.TempDir(), "abs.tar") + writeTar(t, tarPath, tar.Header{Name: "/abs.txt", Mode: 0o644, Size: 2, Typeflag: tar.TypeReg}) + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + // The leading slash is stripped, so the entry lands inside the destination. + if _, err := os.Stat(filepath.Join(dst, "abs.txt")); err != nil { + t.Errorf("abs.txt not extracted inside the destination: %v", err) + } +} + +func TestExtractReplacesExisting(t *testing.T) { + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "a.txt"), []byte("new"), 0o644); err != nil { + t.Fatalf("writing source: %v", err) + } + tarPath := filepath.Join(t.TempDir(), "out.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + + // The destination already holds a stale file and a symlink where a regular + // file is about to land — extraction must replace both, not write through. + dst := t.TempDir() + if err := os.WriteFile(filepath.Join(dst, "a.txt"), []byte("stale"), 0o600); err != nil { + t.Fatalf("seeding destination: %v", err) + } + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + got, err := os.ReadFile(filepath.Join(dst, "a.txt")) + if err != nil { + t.Fatalf("reading result: %v", err) + } + if string(got) != "new" { + t.Errorf("a.txt = %q, want %q", got, "new") + } +} + +func TestExtractIntoPrecreatedVolumeDir(t *testing.T) { + // The durable-dir case: the destination's / subdirectory already + // exists (atelet pre-creates it) when the archive is extracted over it. + src := t.TempDir() + if err := os.MkdirAll(filepath.Join(src, "data"), 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(src, "data", "counter"), []byte("7"), 0o644); err != nil { + t.Fatalf("writing counter: %v", err) + } + tarPath := filepath.Join(t.TempDir(), "durable.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + + dst := t.TempDir() + if err := os.MkdirAll(filepath.Join(dst, "data"), 0o700); err != nil { + t.Fatalf("pre-creating volume dir: %v", err) + } + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + got, err := os.ReadFile(filepath.Join(dst, "data", "counter")) + if err != nil { + t.Fatalf("reading counter: %v", err) + } + if string(got) != "7" { + t.Errorf("counter = %q, want %q", got, "7") + } +} + +func TestExtractRejectsUnsupportedType(t *testing.T) { + tarPath := filepath.Join(t.TempDir(), "dev.tar") + writeTar(t, tarPath, tar.Header{Name: "null", Typeflag: tar.TypeChar, Devmajor: 1, Devminor: 3}) + if err := Extract(tarPath, t.TempDir()); err == nil { + t.Fatal("Extract succeeded on a device entry, want an error") + } +} diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index 3a98860e1..c939af669 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -22,6 +22,7 @@ import ( "fmt" "log/slog" "os" + "os/exec" "path/filepath" "strings" "time" @@ -36,41 +37,90 @@ import ( "google.golang.org/grpc/status" ) -// RestoreWorkload restores the actor on a (possibly different) pod by relaunching -// cloud-hypervisor directly from the downloaded snapshot and resuming. +// RestoreWorkload brings the actor back from a snapshot, on a possibly different +// pod. What that means depends on the scope the snapshot was taken with: // -// Contract with atelet: the snapshot dir (config.json + state.json + memory-ranges + -// base-id) has been downloaded to RestoreStateDir. +// - FULL: relaunch cloud-hypervisor from the snapshot and resume the guest +// (restoreFullScope). +// - DATA: there is no guest to resume — re-materialize the durable-dir volumes and +// cold-boot the actor, which starts its containers afresh from the OCI image. +// +// Contract with atelet: the snapshot's files have been downloaded to RestoreStateDir, +// and the durable-dir volume directories re-created (empty). +func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.RestoreWorkloadRequest) (*ateompb.RestoreWorkloadResponse, error) { + s.lock.Lock() + defer s.lock.Unlock() + + p := actorBootParams{ + atespace: req.GetAtespace(), + actorName: req.GetActorName(), + actorUID: req.GetActorUid(), + templateNS: req.GetActorTemplateNamespace(), + templateName: req.GetActorTemplateName(), + containers: req.GetSpec().GetContainers(), + assetPaths: req.GetRuntimeAssetPaths(), + } + restoreDir := ateompath.RestoreStateDir(p.actorUID) + durableDir := ateompath.DurableDirVolumeMountsDir(p.actorUID) + tStart := time.Now() + + s.actorLogger.EmitLifecycleLog("Actor restoring", p.atespace, p.actorName, p.actorUID, p.templateNS, p.templateName) + + // Restore the durable-dir volumes before anything can observe them: for Full + // that means before the share's virtiofsd starts, for Data before the workload + // cold-starts. The snapshot must carry them — the actor declares the volume, and + // every scope captures it. + if hasDurableVolumes(p.containers) { + if err := untarDurableVolumes(durableDir, restoreDir); err != nil { + return nil, err + } + } + + switch scope := req.GetScope(); scope { + case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: + if err := s.restoreFullScope(ctx, p, restoreDir, tStart); err != nil { + return nil, err + } + case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: + // A Data snapshot holds no guest state, so this is a cold boot that + // happens to start with the volumes already populated. readyz gating comes + // with the cold-boot path, so the actor is serving when we return. + if err := s.coldBootActor(ctx, p); err != nil { + return nil, err + } + slog.InfoContext(ctx, "Actor restored (durable-dir volumes, cold boot)", + slog.String("id", p.actorUID), slog.Duration("total", time.Since(tStart))) + default: + return nil, status.Errorf(codes.InvalidArgument, "unsupported snapshot scope: %v", scope) + } + + s.actorLogger.EmitLifecycleLog("Actor restored", p.atespace, p.actorName, p.actorUID, p.templateNS, p.templateName) + return &ateompb.RestoreWorkloadResponse{}, nil +} + +// restoreFullScope restores a whole-guest snapshot: relaunch cloud-hypervisor +// directly from it and resume. // // Each container's rootfs is overlay(virtio-fs RO lower + guest-tmpfs upper). Steps: // reconstruct each RO lower from the local OCI bundle (atelet re-unpacked the golden // image) at the frozen find-paths path and start the virtiofsd serving them; rewrite -// the snapshot config's per-VMDir paths (vsock + serial + fs socket) to this actor's; +// the snapshot config's per-VMDir paths (vsock + serial + fs sockets) to this actor's; // rebuild the tap (the snapshot's virtio-net is fd-backed → fresh net_fds); relaunch // CH with --restore (OnDemand), and resume. Guest RAM — incl. the actor's in-memory // state, the tmpfs rootfs upper (so rootfs writes PERSIST), and the frozen network -// config — comes back from the memory snapshot. -func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.RestoreWorkloadRequest) (resp *ateompb.RestoreWorkloadResponse, retErr error) { - s.lock.Lock() - defer s.lock.Unlock() - - atespace := req.GetAtespace() - name := req.GetActorName() - actorUID := req.GetActorUid() - templateNS := req.GetActorTemplateNamespace() - templateName := req.GetActorTemplateName() - restoreDir := ateompath.RestoreStateDir(actorUID) - tStart := time.Now() - - s.actorLogger.EmitLifecycleLog("Actor restoring", atespace, name, actorUID, templateNS, templateName) +// config — comes back from the memory snapshot. Durable-dir volumes are host-backed +// instead, and the caller has already restored them from the snapshot's tar. +func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, restoreDir string, tStart time.Time) (retErr error) { + atespace, name, actorUID := p.atespace, p.actorName, p.actorUID + templateNS, templateName := p.templateNS, p.templateName - rr := s.resolveRuntime(req.GetRuntimeAssetPaths()) + rr := s.resolveRuntime(p.assetPaths) kata.CleanupSandboxState(ctx, actorUID) // Repoint the snapshot's vsock socket to this actor's VMDir (the disk + kernel // paths are content-addressed/per-actor and already line up on the same node). if err := rewriteSnapshotSocketPaths(restoreDir, actorUID); err != nil { - return nil, fmt.Errorf("while rewriting snapshot socket paths: %w", err) + return fmt.Errorf("while rewriting snapshot socket paths: %w", err) } srcID := actorUID if b, rerr := os.ReadFile(filepath.Join(restoreDir, baseIDFile)); rerr == nil { @@ -79,7 +129,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } } if err := os.MkdirAll(kata.VMDir(actorUID), 0o700); err != nil { - return nil, fmt.Errorf("while creating VM dir: %w", err) + return fmt.Errorf("while creating VM dir: %w", err) } // Reconstruct each container's overlay RO lower from the LOCAL OCI bundle (atelet @@ -90,20 +140,20 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore // fs socket in the snapshot config is repointed to this VMDir by // rewriteSnapshotSocketPaths above. cross-node consistency relies on a deterministic // unpack of the same image at the same /rootfs path. - containers := req.GetSpec().GetContainers() + containers := p.containers if len(containers) == 0 { - return nil, status.Error(codes.InvalidArgument, "actor spec has no containers") + return status.Error(codes.InvalidArgument, "actor spec has no containers") } if len(containers) > maxActorContainers { - return nil, status.Errorf(codes.Unimplemented, "ateom-microvm supports at most %d containers, got %d", maxActorContainers, len(containers)) + return status.Errorf(codes.Unimplemented, "ateom-microvm supports at most %d containers, got %d", maxActorContainers, len(containers)) } ctrs, err := s.buildActorContainers(actorUID, containers) if err != nil { - return nil, err + return err } vfsdCmd, err := s.stageOverlayLowers(ctx, rr, actorUID, ctrs) if err != nil { - return nil, err + return err } defer func() { if retErr != nil && vfsdCmd.Process != nil { @@ -112,10 +162,27 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } }() + // Restart the durable-dir share's virtiofsd over the contents the caller + // restored. The guest reattaches to it by the socket path rewritten into the + // snapshot config below; find-paths re-opens whatever files it still holds open + // against the same paths, which the restored tar reproduces exactly. + var durableVfsdCmd *exec.Cmd + if hasDurableVolumes(containers) { + if durableVfsdCmd, err = s.stageDurableShare(ctx, rr, actorUID); err != nil { + return err + } + defer func() { + if retErr != nil && durableVfsdCmd.Process != nil { + _ = durableVfsdCmd.Process.Kill() + _, _ = durableVfsdCmd.Process.Wait() + } + }() + } + // Networking: rebuild the per-activation veth + tap; the snapshot's virtio-net // is fd-backed, so CH needs fresh tap FDs (net_fds) on restore. if err := s.setupActorNetwork(ctx); err != nil { - return nil, fmt.Errorf("while setting up actor network: %w", err) + return fmt.Errorf("while setting up actor network: %w", err) } defer func() { if retErr != nil { @@ -131,7 +198,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore }() netDevs, err := ch.SnapshotNetDevices(restoreDir) if err != nil { - return nil, fmt.Errorf("while reading snapshot net devices: %w", err) + return fmt.Errorf("while reading snapshot net devices: %w", err) } var restoredNets []ch.RestoredNet var tapFiles []*os.File @@ -143,7 +210,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore for i, nd := range netDevs { files, terr := s.setupRestoreTap(ctx, fmt.Sprintf("tap%d_kata", i), nd.QueuePairs) if terr != nil { - return nil, fmt.Errorf("while building restore tap for %s: %w", nd.ID, terr) + return fmt.Errorf("while building restore tap for %s: %w", nd.ID, terr) } tapFiles = append(tapFiles, files...) rn := ch.RestoredNet{ID: nd.ID} @@ -160,7 +227,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore Binary: rr.chBinary, APISocket: apiSocket, Stdout: slogWriter{ctx}, Stderr: slogWriter{ctx}, }) if err != nil { - return nil, fmt.Errorf("while launching VMM for restore: %w", err) + return fmt.Errorf("while launching VMM for restore: %w", err) } defer func() { if retErr != nil && chCmd.Process != nil { @@ -175,18 +242,21 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore // rebuild a complete snapshot. CH demand-pages from restoreDir for the VM's whole // lifetime, so it must persist until teardown (atelet keeps it until reset). if err := client.RestoreWithNetFDs(ctx, restoreDir, restoredNets, "OnDemand"); err != nil { - return nil, fmt.Errorf("while restoring VM with net FDs: %w", err) + return fmt.Errorf("while restoring VM with net FDs: %w", err) } if err := client.Resume(ctx); err != nil { - return nil, fmt.Errorf("while resuming restored guest: %w", err) + return fmt.Errorf("while resuming restored guest: %w", err) } // Block until every readyz-enabled container reports 200. if err := readyz.WaitAll(ctx, containers, actorVethIP); err != nil { - return nil, fmt.Errorf("while waiting for container readyz: %w", err) + return fmt.Errorf("while waiting for container readyz: %w", err) } - ra := &runningActor{chCmd: chCmd, vfsdCmd: vfsdCmd, apiSocket: apiSocket, baseID: srcID, restoreSourceDir: restoreDir} + ra := &runningActor{ + chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, + apiSocket: apiSocket, baseID: srcID, restoreSourceDir: restoreDir, + } // Re-attach stdout/stderr forwarding for each container: the restored guest's // containers + kata-agent are alive, so a fresh dial over this actor's vsock @@ -206,18 +276,17 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } s.running[actorUID] = ra - s.actorLogger.EmitLifecycleLog("Actor restored", atespace, name, actorUID, templateNS, templateName) slog.InfoContext(ctx, "Actor restored (overlay rootfs)", slog.String("id", actorUID), slog.Duration("total", time.Since(tStart))) - return &ateompb.RestoreWorkloadResponse{}, nil + return nil } // rewriteSnapshotSocketPaths repoints the snapshot config.json's per-VMDir paths from // the source actor's VMDir to the restoring actor's: the hybrid-vsock socket, the -// File serial console, and each virtio-fs (overlay RO lower) socket, so the sockets/ -// files we create are the ones CH reopens. The kernel and /dev/vda kata image are -// content-addressed static files with identical paths on every node, so they need no -// rewrite, and the overlay has no per-actor disk to repoint. +// File serial console, and each virtio-fs socket, so the sockets/files we create are +// the ones CH reopens. The kernel and /dev/vda kata image are content-addressed static +// files with identical paths on every node, so they need no rewrite, and the overlay +// has no per-actor disk to repoint. func rewriteSnapshotSocketPaths(snapshotDir, id string) error { cfgPath := filepath.Join(snapshotDir, "config.json") b, err := os.ReadFile(cfgPath) @@ -240,12 +309,24 @@ func rewriteSnapshotSocketPaths(snapshotDir, id string) error { serial["file"] = filepath.Join(kata.VMDir(id), "serial.log") } } - // The overlay RO lower is served by a per-VMDir virtiofsd socket; the snapshot - // recorded the golden actor's, so repoint each fs device at this actor's VMDir. + // Each virtio-fs share is served by its own per-VMDir virtiofsd socket; the + // snapshot recorded the golden actor's, so repoint them at this actor's VMDir. + // Match on the device tag: the shares have separate sockets (the overlay RO + // lower's and, when the actor has durable-dir volumes, the writable share's), and + // crossing them would hand the guest the wrong filesystem. if fss, ok := cfg["fs"].([]any); ok { for _, f := range fss { - if fm, ok := f.(map[string]any); ok { + fm, ok := f.(map[string]any) + if !ok { + return fmt.Errorf("snapshot config %q has a malformed fs device", cfgPath) + } + switch tag, _ := fm["tag"].(string); tag { + case kata.FsTag: fm["socket"] = kata.VirtiofsdSocketPath(id) + case kata.DurableFsTag: + fm["socket"] = kata.DurableVirtiofsdSocketPath(id) + default: + return fmt.Errorf("snapshot config %q has fs device with unknown tag %q", cfgPath, tag) } } } diff --git a/cmd/ateom-microvm/restore_test.go b/cmd/ateom-microvm/restore_test.go new file mode 100644 index 000000000..4ea92e102 --- /dev/null +++ b/cmd/ateom-microvm/restore_test.go @@ -0,0 +1,146 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" +) + +// writeSnapshotConfig writes a config.json holding the given fs devices (plus the +// vsock and serial entries every snapshot has) into a fresh snapshot dir. +func writeSnapshotConfig(t *testing.T, fsDevices []map[string]any) string { + t.Helper() + dir := t.TempDir() + cfg := map[string]any{ + "vsock": map[string]any{"cid": 3, "socket": "/run/vc/vm/golden/clh.sock"}, + "serial": map[string]any{"mode": "File", "file": "/run/vc/vm/golden/serial.log"}, + "fs": fsDevices, + } + b, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("marshaling config: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "config.json"), b, 0o600); err != nil { + t.Fatalf("writing config.json: %v", err) + } + return dir +} + +// readFsSockets returns the rewritten config's fs sockets keyed by device tag. +func readFsSockets(t *testing.T, dir string) map[string]string { + t.Helper() + b, err := os.ReadFile(filepath.Join(dir, "config.json")) + if err != nil { + t.Fatalf("reading config.json: %v", err) + } + var cfg struct { + Vsock map[string]any `json:"vsock"` + Serial map[string]any `json:"serial"` + Fs []struct { + Tag string `json:"tag"` + Socket string `json:"socket"` + } `json:"fs"` + } + if err := json.Unmarshal(b, &cfg); err != nil { + t.Fatalf("parsing rewritten config: %v", err) + } + got := map[string]string{} + for _, f := range cfg.Fs { + got[f.Tag] = f.Socket + } + return got +} + +func TestRewriteSnapshotSocketPaths(t *testing.T) { + const id = "actor-uid" + + t.Run("overlay lower only", func(t *testing.T) { + dir := writeSnapshotConfig(t, []map[string]any{ + {"tag": kata.FsTag, "socket": "/run/vc/vm/golden/virtiofsd.sock"}, + }) + if err := rewriteSnapshotSocketPaths(dir, id); err != nil { + t.Fatalf("rewriteSnapshotSocketPaths: %v", err) + } + if got, want := readFsSockets(t, dir)[kata.FsTag], kata.VirtiofsdSocketPath(id); got != want { + t.Errorf("%s socket = %q, want %q", kata.FsTag, got, want) + } + }) + + t.Run("each share keeps its own socket", func(t *testing.T) { + // Ordered durable-first to catch a rewrite that assumes the RO lower comes + // first, which would hand the guest the wrong filesystem. + dir := writeSnapshotConfig(t, []map[string]any{ + {"tag": kata.DurableFsTag, "socket": "/run/vc/vm/golden/virtiofsd-durable.sock"}, + {"tag": kata.FsTag, "socket": "/run/vc/vm/golden/virtiofsd.sock"}, + }) + if err := rewriteSnapshotSocketPaths(dir, id); err != nil { + t.Fatalf("rewriteSnapshotSocketPaths: %v", err) + } + got := readFsSockets(t, dir) + for tag, want := range map[string]string{ + kata.FsTag: kata.VirtiofsdSocketPath(id), + kata.DurableFsTag: kata.DurableVirtiofsdSocketPath(id), + } { + if got[tag] != want { + t.Errorf("%s socket = %q, want %q", tag, got[tag], want) + } + } + if got[kata.FsTag] == got[kata.DurableFsTag] { + t.Error("both shares were pointed at the same socket") + } + }) + + t.Run("unknown tag is an error", func(t *testing.T) { + dir := writeSnapshotConfig(t, []map[string]any{ + {"tag": "somethingElse", "socket": "/run/vc/vm/golden/other.sock"}, + }) + if err := rewriteSnapshotSocketPaths(dir, id); err == nil { + t.Fatal("rewriteSnapshotSocketPaths accepted an unknown fs tag, want an error") + } + }) + + t.Run("vsock and serial are repointed", func(t *testing.T) { + dir := writeSnapshotConfig(t, []map[string]any{ + {"tag": kata.FsTag, "socket": "/run/vc/vm/golden/virtiofsd.sock"}, + }) + if err := rewriteSnapshotSocketPaths(dir, id); err != nil { + t.Fatalf("rewriteSnapshotSocketPaths: %v", err) + } + b, err := os.ReadFile(filepath.Join(dir, "config.json")) + if err != nil { + t.Fatalf("reading config.json: %v", err) + } + var cfg struct { + Vsock struct{ Socket string } `json:"vsock"` + Serial struct{ File string } `json:"serial"` + } + if err := json.Unmarshal(b, &cfg); err != nil { + t.Fatalf("parsing rewritten config: %v", err) + } + if want := kata.VsockSocketPath(id); cfg.Vsock.Socket != want { + t.Errorf("vsock socket = %q, want %q", cfg.Vsock.Socket, want) + } + if want := filepath.Join(kata.VMDir(id), "serial.log"); cfg.Serial.File != want { + t.Errorf("serial file = %q, want %q", cfg.Serial.File, want) + } + }) +} diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 7c323e787..5047f271f 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -56,6 +56,10 @@ type runningActor struct { // demand-pages from it for the actor's lifetime). ateom owns it; teardownActor // kills it after the CH process. vfsdCmd *exec.Cmd + // durableVfsdCmd is the second virtiofsd, serving the actor's writable + // durable-dir volumes. nil when the actor declares none. Owned and torn down + // exactly like vfsdCmd. + durableVfsdCmd *exec.Cmd // apiSocket is the CH api-socket for this ateom-owned VMM. apiSocket string @@ -120,6 +124,10 @@ type actorContainer struct { name string bundleRootfs string spec *specs.Spec + // durableMountPaths are the in-container paths where this container mounts + // the actor's durable-dir volume (see durable.go). Empty for containers that + // declare none. + durableMountPaths []string } // resolvedRuntime holds the concrete binary/config paths for a request, taken @@ -185,43 +193,74 @@ func writeGuestResolvConf(rootfs string) error { // - The runtime assets (guest kernel, guest OS image, cloud-hypervisor, virtiofsd, // base kata config) are on disk and passed as runtime asset paths. // - The OCI bundle (config.json + populated rootfs/) is prepared per container. -func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkloadRequest) (resp *ateompb.RunWorkloadResponse, retErr error) { +func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkloadRequest) (*ateompb.RunWorkloadResponse, error) { s.lock.Lock() defer s.lock.Unlock() - atespace := req.GetAtespace() - name := req.GetActorName() - actorUID := req.GetActorUid() - templateNS := req.GetActorTemplateNamespace() - templateName := req.GetActorTemplateName() + p := actorBootParams{ + atespace: req.GetAtespace(), + actorName: req.GetActorName(), + actorUID: req.GetActorUid(), + templateNS: req.GetActorTemplateNamespace(), + templateName: req.GetActorTemplateName(), + containers: req.GetSpec().GetContainers(), + assetPaths: req.GetRuntimeAssetPaths(), + } - s.actorLogger.EmitLifecycleLog("Actor starting", atespace, name, actorUID, templateNS, templateName) + s.actorLogger.EmitLifecycleLog("Actor starting", p.atespace, p.actorName, p.actorUID, p.templateNS, p.templateName) + if err := s.coldBootActor(ctx, p); err != nil { + return nil, err + } + s.actorLogger.EmitLifecycleLog("Actor started", p.atespace, p.actorName, p.actorUID, p.templateNS, p.templateName) + slog.InfoContext(ctx, "Actor started (overlay rootfs)", slog.String("id", p.actorUID)) + return &ateompb.RunWorkloadResponse{}, nil +} + +// actorBootParams is what a cold boot needs about an actor. It comes from a Run +// request, or from a Restore request whose snapshot scope covers only the +// durable-dir volumes (the workload itself cold-starts). +type actorBootParams struct { + atespace string + actorName string + actorUID string + templateNS string + templateName string + containers []*ateompb.Container + assetPaths map[string]string +} + +// coldBootActor boots the actor's micro-VM from scratch and starts its +// containers, registering the result in s.running. The caller holds s.lock and +// owns the lifecycle logging. +func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (retErr error) { + atespace, name, actorUID := p.atespace, p.actorName, p.actorUID + templateNS, templateName := p.templateNS, p.templateName // All of the actor's containers share the one micro-VM (which is the pod // sandbox): each gets its own overlay rootfs and its own kata-agent // CreateContainer/StartContainer, driven below after the shared boot + // CreateSandbox + guest networking. - containers := req.GetSpec().GetContainers() + containers := p.containers if len(containers) == 0 { - return nil, status.Error(codes.InvalidArgument, "actor spec has no containers") + return status.Error(codes.InvalidArgument, "actor spec has no containers") } if len(containers) > maxActorContainers { - return nil, status.Errorf(codes.Unimplemented, "ateom-microvm supports at most %d containers, got %d", maxActorContainers, len(containers)) + return status.Errorf(codes.Unimplemented, "ateom-microvm supports at most %d containers, got %d", maxActorContainers, len(containers)) } // ateom builds the CH vm.create itself, so it needs the guest kernel + image // paths directly. - paths := req.GetRuntimeAssetPaths() + paths := p.assetPaths kernel, image := paths[assetKernel], paths[assetImage] if kernel == "" || image == "" { - return nil, fmt.Errorf("ateom-microvm requires %q and %q asset paths", assetKernel, assetImage) + return fmt.Errorf("ateom-microvm requires %q and %q asset paths", assetKernel, assetImage) } rr := s.resolveRuntime(paths) // Networking (host side): per-activation veth into the interior netns. The // tap + TC mirror is built below (after the VM exists) so its FDs are fresh. if err := s.setupActorNetwork(ctx); err != nil { - return nil, fmt.Errorf("while setting up actor network: %w", err) + return fmt.Errorf("while setting up actor network: %w", err) } defer func() { if retErr != nil { @@ -240,19 +279,19 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // lower). No host disk — the rootfs is overlay(virtio-fs lower + guest-tmpfs upper). ctrs, err := s.buildActorContainers(actorUID, containers) if err != nil { - return nil, err + return err } // Guest sizing + agent kernel params from the kata config. memMiB, vcpus, kparams, err := s.guestConfig(rr) if err != nil { - return nil, err + return err } // Clean stale per-sandbox state + create the runtime dir for the sockets. kata.CleanupSandboxState(ctx, actorUID) if err := os.MkdirAll(kata.VMDir(actorUID), 0o700); err != nil { - return nil, fmt.Errorf("while creating VM dir: %w", err) + return fmt.Errorf("while creating VM dir: %w", err) } // Stage the overlay RO lowers (bind each image into the shared dir) + start the @@ -260,7 +299,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // the actor's lifetime, so ateom owns the process (killed in teardownActor). vfsdCmd, err := s.stageOverlayLowers(ctx, rr, actorUID, ctrs) if err != nil { - return nil, err + return err } defer func() { if retErr != nil && vfsdCmd.Process != nil { @@ -269,6 +308,25 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload } }() + // Durable-dir volumes (if any) get their own writable virtio-fs share, served + // by a second virtiofsd from the host directory atelet prepared. + var durableVolume string + var durableVfsdCmd *exec.Cmd + if hasDurableVolumes(containers) { + if durableVolume, err = resolveDurableVolumeName(ateompath.DurableDirVolumeMountsDir(actorUID)); err != nil { + return err + } + if durableVfsdCmd, err = s.stageDurableShare(ctx, rr, actorUID); err != nil { + return err + } + defer func() { + if retErr != nil && durableVfsdCmd.Process != nil { + _ = durableVfsdCmd.Process.Kill() + _, _ = durableVfsdCmd.Process.Wait() + } + }() + } + // Launch a bare VMM (CH + api-socket); ateom owns this process for teardown. apiSocket := filepath.Join(kata.VMDir(actorUID), "clh-api.sock") chCmd, client, err := ch.LaunchVMM(ctx, ch.LaunchVMMOptions{ @@ -278,7 +336,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload Stderr: slogWriter{ctx}, }) if err != nil { - return nil, fmt.Errorf("while launching VMM: %w", err) + return fmt.Errorf("while launching VMM: %w", err) } defer func() { if retErr != nil && chCmd.Process != nil { @@ -292,16 +350,16 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // writable upper is a guest tmpfs). serialLog is also read on a failed agent dial // below, so keep it here. serialLog := filepath.Join(kata.VMDir(actorUID), "serial.log") - vmCfg := buildVMConfig(actorUID, kernel, image, kparams, serialLog, memMiB, vcpus) + vmCfg := buildVMConfig(actorUID, kernel, image, kparams, serialLog, memMiB, vcpus, durableVolume != "") if err := client.CreateVM(ctx, vmCfg); err != nil { - return nil, fmt.Errorf("while creating VM: %w", err) + return fmt.Errorf("while creating VM: %w", err) } // Network device: build the tap + TC mirror against the actor veth and add a // virtio-net to the created (pre-boot) VM with the tap FDs (SCM_RIGHTS). tapFiles, err := s.setupRestoreTap(ctx, "tap0_kata", 1) if err != nil { - return nil, fmt.Errorf("while building tap: %w", err) + return fmt.Errorf("while building tap: %w", err) } defer func() { for _, f := range tapFiles { @@ -313,12 +371,12 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload fds = append(fds, int(f.Fd())) } if err := client.AddNetWithFDs(ctx, actorGuestMAC, 2*len(tapFiles), fds); err != nil { - return nil, fmt.Errorf("while adding net device: %w", err) + return fmt.Errorf("while adding net device: %w", err) } // Boot. if err := client.BootVM(ctx); err != nil { - return nil, fmt.Errorf("while booting VM: %w", err) + return fmt.Errorf("while booting VM: %w", err) } slog.InfoContext(ctx, "Micro-VM booted", slog.String("id", actorUID), slog.String("api", apiSocket)) @@ -328,14 +386,14 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // does), rather than dialing once. vsockPath := kata.VsockSocketPath(actorUID) if !waitForFile(vsockPath, 15*time.Second) { - return nil, fmt.Errorf("kata-agent vsock socket %q did not appear", vsockPath) + return fmt.Errorf("kata-agent vsock socket %q did not appear", vsockPath) } ac, err := dialAgentRetry(ctx, vsockPath, 60*time.Second) if err != nil { if b, rerr := os.ReadFile(serialLog); rerr == nil { slog.ErrorContext(ctx, "agent dial failed; guest serial tail", slog.String("serial", tailString(string(b), 3000))) } - return nil, fmt.Errorf("while dialing kata-agent: %w", err) + return fmt.Errorf("while dialing kata-agent: %w", err) } // The agent client must stay open past this RPC: the stdout/stderr forwarding // goroutines (started below) read over it for the actor's lifetime. It is stored @@ -348,16 +406,16 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload }() // Post-boot kata-agent setup: sandbox, guest networking, start each container. - if err := s.startActorContainers(ctx, ac, actorUID, vsockPath, ctrs); err != nil { - return nil, err + if err := s.startActorContainers(ctx, ac, actorUID, vsockPath, ctrs, durableVolume); err != nil { + return err } // Block until every readyz-enabled container reports 200. if err := readyz.WaitAll(ctx, containers, actorVethIP); err != nil { - return nil, fmt.Errorf("while waiting for container readyz: %w", err) + return fmt.Errorf("while waiting for container readyz: %w", err) } - ra := &runningActor{chCmd: chCmd, vfsdCmd: vfsdCmd, apiSocket: apiSocket, baseID: actorUID, logAgent: ac} + ra := &runningActor{chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, apiSocket: apiSocket, baseID: actorUID, logAgent: ac} s.running[actorUID] = ra // Forward each container's stdout/stderr into the pod logs. The overlay workload's @@ -368,9 +426,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload s.startActorLogForwarding(ac, atespace, name, actorUID, templateNS, templateName, overlayWorkloadID(c.name), c.name) } - s.actorLogger.EmitLifecycleLog("Actor started", atespace, name, actorUID, templateNS, templateName) - slog.InfoContext(ctx, "Actor started (overlay rootfs)", slog.String("id", actorUID)) - return &ateompb.RunWorkloadResponse{}, nil + return nil } // buildActorContainers prepares each of the actor's containers for the shared @@ -406,7 +462,12 @@ func (s *AteomService) buildActorContainers(actorUID string, containers []*ateom if err := writeGuestResolvConf(bundleRootfs); err != nil { return nil, fmt.Errorf("while writing guest resolv.conf for %q: %w", cn, err) } - ctrs[i] = actorContainer{name: cn, bundleRootfs: bundleRootfs, spec: spec} + ctrs[i] = actorContainer{ + name: cn, + bundleRootfs: bundleRootfs, + spec: spec, + durableMountPaths: c.GetDurableDirVolumes(), + } } return ctrs, nil } @@ -461,7 +522,10 @@ func (s *AteomService) guestConfig(rr resolvedRuntime) (memMiB, vcpus int, kpara // systemd-networkd (the agent owns eth0). The console is arch-specific: ttyAMA0 on // arm64, ttyS0 on amd64. /dev/vda is the RO guest image; the actor rootfs's RO lower is // the virtio-fs device on PCI segment 1 (hence num_pci_segments=2), with no actor disks. -func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus int) ch.VmConfig { +// +// withDurable adds a second virtio-fs device for the actor's writable durable-dir +// volumes (see durable.go), served by its own virtiofsd on the same PCI segment. +func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus int, withDurable bool) ch.VmConfig { console := "ttyS0" if runtime.GOARCH == "arm64" { console = "ttyAMA0" @@ -479,10 +543,7 @@ func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus i Disks: []ch.DiskConfig{ {Path: image, Readonly: true, ImageType: "Raw", NumQueues: int32(vcpus), QueueSize: 1024}, }, - Fs: []ch.FsConfig{{ - Tag: kata.FsTag, Socket: kata.VirtiofsdSocketPath(id), - NumQueues: 1, QueueSize: 1024, PciSegment: 1, - }}, + Fs: buildFsConfigs(id, withDurable), Platform: &ch.PlatformConfig{NumPciSegments: 2}, Rng: &ch.RngConfig{Src: "/dev/urandom"}, Serial: &ch.ConsoleConfig{Mode: "File", File: serialLog}, @@ -490,16 +551,37 @@ func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus i } } +// buildFsConfigs returns the VM's virtio-fs devices: the overlay RO lower's +// share, plus the writable durable-dir share when the actor has one. Both sit on +// PCI segment 1 (the segment buildVMConfig reserves for virtio-fs). +func buildFsConfigs(id string, withDurable bool) []ch.FsConfig { + fs := []ch.FsConfig{{ + Tag: kata.FsTag, Socket: kata.VirtiofsdSocketPath(id), + NumQueues: 1, QueueSize: 1024, PciSegment: 1, + }} + if withDurable { + fs = append(fs, ch.FsConfig{ + Tag: kata.DurableFsTag, Socket: kata.DurableVirtiofsdSocketPath(id), + NumQueues: 1, QueueSize: 1024, PciSegment: 1, + }) + } + return fs +} + // startActorContainers performs the post-boot kata-agent setup the shim normally // does at boot: establish the sandbox once (mounting the kataShared virtio-fs base), // configure guest networking (eth0 IP/MAC/MTU + routes) once, then start each // container on its own overlay rootfs. On failure it dumps guest diagnostics. -func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentClient, id, vsockPath string, ctrs []actorContainer) error { +// +// durableVolume is the actor's durable-dir volume name, or "" when it has none; +// with one, the sandbox also mounts the writable durable share and each container +// binds the volume at its declared paths. +func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentClient, id, vsockPath string, ctrs []actorContainer, durableVolume string) error { // Establish the agent sandbox + the kataShared virtio-fs mount (the RO base for // every container's overlay lower). All containers share it, so use the first // container's hostname. sbCtx, sbCancel := context.WithTimeout(ctx, 20*time.Second) - err := ac.CreateSandboxForActor(sbCtx, id, ctrs[0].spec.Hostname) + err := ac.CreateSandboxForActor(sbCtx, id, ctrs[0].spec.Hostname, durableVolume != "") sbCancel() if err != nil { return fmt.Errorf("while creating agent sandbox: %w", err) @@ -517,7 +599,7 @@ func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentC } for _, c := range ctrs { - if err := startOverlayContainer(ctx, ac, vsockPath, c); err != nil { + if err := startOverlayContainer(ctx, ac, vsockPath, c, durableVolume); err != nil { return err } } @@ -528,7 +610,11 @@ func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentC // lower + guest-tmpfs upper): a carrier container (id == name) eager-binds the RO base // to /run/kata-containers//rootfs, then the workload (id == _ovl) overlays // it with a tmpfs upper. On failure it dumps the guest overlay state. -func startOverlayContainer(ctx context.Context, ac *kata.AgentClient, vsockPath string, c actorContainer) error { +// +// With a durable-dir volume, the WORKLOAD also binds it at the container's declared +// mount paths. The carrier deliberately does not: its rootfs is the read-only lower, +// where the agent could not create a missing mount point. +func startOverlayContainer(ctx context.Context, ac *kata.AgentClient, vsockPath string, c actorContainer, durableVolume string) error { carrierCtx, carrierCancel := context.WithTimeout(ctx, 30*time.Second) err := ac.CreateCarrier(carrierCtx, c.name, c.spec) carrierCancel() @@ -540,7 +626,7 @@ func startOverlayContainer(ctx context.Context, ac *kata.AgentClient, vsockPath upperBase := kata.OverlayUpperBase(c.name) wlCtx, wlCancel := context.WithTimeout(ctx, 30*time.Second) - err = ac.StartOverlayWorkload(wlCtx, c.name, overlayWorkloadID(c.name), upperBase, c.spec) + err = ac.StartOverlayWorkload(wlCtx, c.name, overlayWorkloadID(c.name), upperBase, workloadSpec(c, durableVolume)) wlCancel() if err != nil { dump := kata.DebugConsoleDump(ctx, vsockPath, diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index d53002066..ed3cc4e31 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -121,8 +121,17 @@ spec: httpGet: path: /readyz port: 80 + # The counter writes its file counter here. With the durableDir volume below + # it lands on a host-backed virtio-fs share instead of the guest-RAM rootfs + # upper, so it survives a Data-scope snapshot (which keeps no guest memory). + volumeMounts: + - name: data + mountPath: /home/counter workerSelector: matchLabels: workload: counter-microvm snapshotsConfig: location: gs://${BUCKET_NAME}/ate-demo-counter-microvm/ + volumes: + - name: data + durableDir: {} diff --git a/docs/architecture.md b/docs/architecture.md index 62d804253..9a2a512cb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -338,7 +338,7 @@ A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class * **gVisor** (`ateom-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree. - * **micro-VM** (`ateom-microvm`): Runs the workload inside a [Kata Containers](https://katacontainers.io/) guest on the [Cloud Hypervisor](https://www.cloudhypervisor.org/) VMM. Suspend and resume capture a memory-only VM snapshot and restore it on-demand using `userfaultfd` memory demand-paging, with container rootfs writes captured in guest RAM via a `tmpfs` overlay. + * **micro-VM** (`ateom-microvm`): Runs the workload inside a [Kata Containers](https://katacontainers.io/) guest on the [Cloud Hypervisor](https://www.cloudhypervisor.org/) VMM. Suspend and resume capture a memory-only VM snapshot and restore it on-demand using `userfaultfd` memory demand-paging, with container rootfs writes captured in guest RAM via a `tmpfs` overlay. `DurableDir` volumes are host-backed instead, served over a second (writable) virtio-fs share and shipped in snapshots as a tar, so a `Data`-scope snapshot can capture them without any guest memory. ### Networking Stack (`atenet` + Envoy) diff --git a/docs/glossary.md b/docs/glossary.md index 9f1fa6fe3..aadb3f383 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -76,19 +76,18 @@ because they change too frequently for etcd. - **DurableDir volume**: a directory mounted into one or more containers whose contents are preserved by the [`Data` snapshot scope](#snapshots) and therefore survive across Suspend/Resume independently of process - memory or other rootfs writes. A single `ActorTemplate` may declare - multiple `DurableDir` volumes, and the same volume may be mounted into - multiple containers (potentially at different paths). This is the - per-Actor application-data surface. + memory or other rootfs writes. An `ActorTemplate` may declare one + `DurableDir` volume, and it may be mounted into multiple containers + (potentially at different paths). This is the per-Actor + application-data surface. ## Snapshots - **Snapshot scope**: what an `ActorTemplate`'s `SnapshotsConfig` includes in a given snapshot. Two scopes exist today: - **`Full`**: process memory plus the rootfs delta on top of the OCI - image (which also includes any attached `DurableDir` volumes, - since they live inside rootfs). Used to capture everything needed - to resume hot. + image, and any attached `DurableDir` volumes. Used to capture + everything needed to resume hot. - **`Data`**: only the contents of attached volumes that support snapshots — currently `DurableDir` volumes. Process memory and the rest of rootfs are discarded; on Resume the Actor cold-boots from diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 8fb958b37..c1c404a1b 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -83,10 +83,6 @@ func TestActorLifecycle(t *testing.T) { } func TestDurableDirLifecycle(t *testing.T) { - if isMicroVMEnvironment() { - t.Skip("Skipping TestDurableDirLifecycle for microVM environment") - } - tests := []struct { name string tc actorLifecycleTestCase @@ -380,11 +376,7 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * t.Fatalf("failed to call actor: %v", err) } - if isMicroVMEnvironment() { - validateCounterResponse(t, resp, "after creation", 1, -1) - } else { - validateCounterResponse(t, resp, "after creation", 1, 1) - } + validateCounterResponse(t, resp, "after creation", 1, 1) // Pausing the actor t.Logf("Pausing Actor %q...", actorName) @@ -408,11 +400,7 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * if err != nil { t.Fatalf("failed to call actor again: %v", err) } - if isMicroVMEnvironment() { - validateCounterResponse(t, resp, "after pause", 2, -1) - } else { - validateCounterResponse(t, resp, "after pause", 2, 2) - } + validateCounterResponse(t, resp, "after pause", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -467,11 +455,7 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj if err != nil { t.Fatalf("failed to call actor: %v", err) } - if isMicroVMEnvironment() { - validateCounterResponse(t, resp, "after creation", 1, -1) - } else { - validateCounterResponse(t, resp, "after creation", 1, 1) - } + validateCounterResponse(t, resp, "after creation", 1, 1) // Suspending the actor t.Logf("Suspending Actor %q...", actorName) @@ -495,11 +479,7 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj if err != nil { t.Fatalf("failed to call actor again: %v", err) } - if isMicroVMEnvironment() { - validateCounterResponse(t, resp, "after suspend", 2, -1) - } else { - validateCounterResponse(t, resp, "after suspend", 2, 2) - } + validateCounterResponse(t, resp, "after suspend", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -826,7 +806,9 @@ func callActorOnce(t *testing.T, atespace, actorName string) (string, error) { return string(body), nil } +// isMicroVMEnvironment reports whether the suite is running against the +// micro-VM demo template, which does not support every volume type yet (see +// TestExternalVolumeLifecycle). func isMicroVMEnvironment() bool { - // TODO(BenTheElder) remove it once https://github.com/agent-substrate/substrate/pull/313 is merged. return os.Getenv("E2E_TEMPLATE_NAMESPACE") == "ate-demo-counter-microvm" } diff --git a/manifests/ate-install/generated/ate.dev_actortemplates.yaml b/manifests/ate-install/generated/ate.dev_actortemplates.yaml index 0bebc1473..4fa151c4a 100644 --- a/manifests/ate-install/generated/ate.dev_actortemplates.yaml +++ b/manifests/ate-install/generated/ate.dev_actortemplates.yaml @@ -433,9 +433,6 @@ spec: rule: '!has(self.volumes) || self.volumes.all(v, has(self.containers) && self.containers.exists(c, has(c.volumeMounts) && c.volumeMounts.exists(vm, vm.name == v.name)))' - - message: DurableDir volumes are not supported when sandboxClass is 'microvm' - rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || - !has(self.volumes) || !self.volumes.exists(v, has(v.durableDir))' - message: ExternalVolumes are not supported when sandboxClass is 'microvm' rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))' diff --git a/pkg/api/v1alpha1/actortemplate_types.go b/pkg/api/v1alpha1/actortemplate_types.go index c6933c104..4835afbae 100644 --- a/pkg/api/v1alpha1/actortemplate_types.go +++ b/pkg/api/v1alpha1/actortemplate_types.go @@ -297,7 +297,6 @@ type SnapshotsConfig struct { // +kubebuilder:validation:XValidation:rule="!has(self.volumes) || self.volumes.filter(v, has(v.durableDir)).size() <= 1",message="At most one DurableDir-typed volume is supported per ActorTemplate" // +kubebuilder:validation:XValidation:rule="!has(self.containers) || self.containers.all(c, !has(c.volumeMounts) || c.volumeMounts.filter(vm, has(self.volumes) && self.volumes.exists(v, v.name == vm.name && has(v.durableDir))).size() <= 1)",message="A container may mount at most one DurableDir-typed volume" // +kubebuilder:validation:XValidation:rule="!has(self.volumes) || self.volumes.all(v, has(self.containers) && self.containers.exists(c, has(c.volumeMounts) && c.volumeMounts.exists(vm, vm.name == v.name)))",message="All volumes defined in spec.volumes must be mounted by at least one container" -// +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass != 'microvm' || !has(self.volumes) || !self.volumes.exists(v, has(v.durableDir))",message="DurableDir volumes are not supported when sandboxClass is 'microvm'" // +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass != 'microvm' || !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))",message="ExternalVolumes are not supported when sandboxClass is 'microvm'" type ActorTemplateSpec struct { // PauseImage is the container to use as the root sandbox container. diff --git a/pkg/api/v1alpha1/actortemplate_validation_test.go b/pkg/api/v1alpha1/actortemplate_validation_test.go index 89ccbc923..a8ecce1e4 100644 --- a/pkg/api/v1alpha1/actortemplate_validation_test.go +++ b/pkg/api/v1alpha1/actortemplate_validation_test.go @@ -998,7 +998,7 @@ func TestActorTemplateValidation(t *testing.T) { wantErr: true, errMsg: "Name must be a valid DNS label", }, { - name: "Volumes: DurableDir volume with SandboxClass microvm is invalid", + name: "Volumes: DurableDir volume with SandboxClass microvm is valid", mutate: func(at *ActorTemplate) { at.Spec.SandboxClass = SandboxClassMicroVM at.Spec.Volumes = []Volume{ @@ -1008,8 +1008,7 @@ func TestActorTemplateValidation(t *testing.T) { {Name: "vol1", MountPath: "/home/user"}, } }, - wantErr: true, - errMsg: "DurableDir volumes are not supported when sandboxClass is 'microvm'", + wantErr: false, }, { name: "Volumes: DurableDir volume with SandboxClass gvisor is valid", mutate: func(at *ActorTemplate) {