From 88d682d0cc6afc5cb427f9e94e9e85b6eac81c50 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 24 Jul 2026 15:39:13 -0700 Subject: [PATCH 1/3] ateom-microvm: mount durable-dir volumes over a writable virtio-fs share DurableDir volumes shipped for gVisor only; the micro-VM runtime dropped Container.durable_dir_volumes on the floor and the API rejected the combination outright. Give the guest the volume: - cmd/ateom-microvm/internal/tarutil: archive/restore a directory tree preserving modes, ownership, times, symlinks and hardlinks, with os.Root-confined extraction. Snapshot support (a later commit) ships the volume as a tar; no reusable helper existed, and atelet's image untar does not preserve ownership. - A SECOND virtiofsd serves the actor's durable-dir directory read-write with cache=auto. The kataShared share stays read-only at cache=always, which is what its TODO asked for: writable volumes get their own share rather than weakening the overlay lower's caching. - The guest mounts the new tag at /run/ateom-durable and each WORKLOAD container binds / at its declared paths. The carrier is deliberately left alone: its rootfs is the read-only lower, where the agent could not create a missing mount point. - Lift the CEL rule banning durableDir on sandboxClass microvm. ateom learns the volume name from the directory layout atelet prepares: the wire protocol carries mount paths only, and the API allows at most one durable-dir volume, so anything but exactly one directory is an error. Snapshot scopes are unchanged so far, so the contents live only as long as the actor. Checkpoint and restore follow. --- cmd/ateom-microvm/checkpoint.go | 12 +- cmd/ateom-microvm/durable.go | 194 +++++++ cmd/ateom-microvm/durable_test.go | 232 ++++++++ .../internal/kata/overlay_linux.go | 84 ++- .../internal/kata/overlay_linux_test.go | 65 +++ cmd/ateom-microvm/internal/kata/restore.go | 6 + .../internal/tarutil/fifo_linux.go | 48 ++ .../internal/tarutil/owner_linux.go | 85 +++ cmd/ateom-microvm/internal/tarutil/tarutil.go | 374 +++++++++++++ .../internal/tarutil/tarutil_test.go | 499 ++++++++++++++++++ cmd/ateom-microvm/run.go | 85 ++- .../generated/ate.dev_actortemplates.yaml | 3 - pkg/api/v1alpha1/actortemplate_types.go | 1 - .../v1alpha1/actortemplate_validation_test.go | 5 +- 14 files changed, 1644 insertions(+), 49 deletions(-) create mode 100644 cmd/ateom-microvm/durable.go create mode 100644 cmd/ateom-microvm/durable_test.go create mode 100644 cmd/ateom-microvm/internal/kata/overlay_linux_test.go create mode 100644 cmd/ateom-microvm/internal/tarutil/fifo_linux.go create mode 100644 cmd/ateom-microvm/internal/tarutil/owner_linux.go create mode 100644 cmd/ateom-microvm/internal/tarutil/tarutil.go create mode 100644 cmd/ateom-microvm/internal/tarutil/tarutil_test.go diff --git a/cmd/ateom-microvm/checkpoint.go b/cmd/ateom-microvm/checkpoint.go index f95e5a7ee..f65ea4aad 100644 --- a/cmd/ateom-microvm/checkpoint.go +++ b/cmd/ateom-microvm/checkpoint.go @@ -21,6 +21,7 @@ import ( "fmt" "log/slog" "os" + "os/exec" "path/filepath" "time" @@ -198,10 +199,13 @@ func (s *AteomService) teardownActor(ctx context.Context, id string, ra *running _ = ra.chCmd.Process.Kill() _, _ = ra.chCmd.Process.Wait() } - // Kill the virtiofsd serving the overlay RO lower (after CH, its only client). - if ra.vfsdCmd != nil && ra.vfsdCmd.Process != nil { - _ = ra.vfsdCmd.Process.Kill() - _, _ = ra.vfsdCmd.Process.Wait() + // Kill the virtiofsds (after CH, their only client): the overlay RO lower's + // and, when the actor has durable-dir volumes, the writable share's. + for _, cmd := range []*exec.Cmd{ra.vfsdCmd, ra.durableVfsdCmd} { + if cmd != nil && cmd.Process != nil { + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + } } } diff --git a/cmd/ateom-microvm/durable.go b/cmd/ateom-microvm/durable.go new file mode 100644 index 000000000..58676d365 --- /dev/null +++ b/cmd/ateom-microvm/durable.go @@ -0,0 +1,194 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +// Durable-dir volumes for the micro-VM runtime. +// +// A durable-dir volume is a directory whose contents outlive the actor's process +// state: it survives suspend/resume and, under the Data snapshot scope, is the +// ONLY thing captured (the workload cold-starts on restore). The host side is +// owned by atelet, which creates one directory per volume under +// ateompath.DurableDirVolumeMountsDir(actorUID) and wipes them when the actor's +// directories are reset. +// +// ateom exposes that host directory to the guest over a SECOND virtiofsd — the +// kataShared share stays strictly read-only (it is the overlay lower, served +// with cache=always), so writable volumes get their own share, mounted at +// kata.GuestDurableVolumeDir(volume) and bind-mounted from there into each +// container that declares the volume. +// +// Snapshots carry the contents as a tar of the whole per-actor directory, so the +// volume names round-trip without ateom having to learn them from the wire (the +// ateom protocol carries mount paths only). virtiofsd serves the share +// write-through (no --writeback), so once the guest is paused every completed +// guest write is already visible on the host and the tar is complete. + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/tarutil" + "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/proto/ateompb" + specs "github.com/opencontainers/runtime-spec/specs-go" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// durableTarFile is the snapshot file holding the tar of the actor's durable-dir +// volumes. Its entries are /... relative to +// ateompath.DurableDirVolumeMountsDir, so extraction restores the same layout. +const durableTarFile = "durable-dir.tar" + +// hasDurableVolumes reports whether any container mounts a durable-dir volume. +func hasDurableVolumes(containers []*ateompb.Container) bool { + for _, c := range containers { + if len(c.GetDurableDirVolumes()) > 0 { + return true + } + } + return false +} + +// resolveDurableVolumeName returns the name of the actor's durable-dir volume, +// read from the directory atelet created for it (dir is +// ateompath.DurableDirVolumeMountsDir for the actor). +// +// The ateom protocol carries only the guest mount paths, not volume names, so +// the name comes from the host layout: the directory holds exactly one +// subdirectory per volume. The ActorTemplate API allows at most one durable-dir +// volume per template, so anything other than exactly one directory means ateom +// and atelet disagree about the actor — fail rather than guess. +// +// That makes this the one place ateom depends on atelet's on-disk layout, and +// the first thing to change if more than one durable-dir volume is ever +// supported: ateompb.Container would need to carry each mount's volume name +// alongside its path, and this lookup would go away. +func resolveDurableVolumeName(dir string) (string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return "", fmt.Errorf("while reading durable-dir volumes dir %q: %w", dir, err) + } + var names []string + for _, e := range entries { + if e.IsDir() { + names = append(names, e.Name()) + } + } + switch len(names) { + case 1: + return names[0], nil + case 0: + return "", status.Errorf(codes.FailedPrecondition, + "actor declares durable-dir volume mounts but %q holds no volume directory", dir) + default: + return "", status.Errorf(codes.Unimplemented, + "ateom-microvm supports at most one durable-dir volume, found %d in %q", len(names), dir) + } +} + +// durableMounts returns the OCI mounts that expose one durable volume at each of +// a container's declared mount paths. The source is the volume's directory inside +// the guest's durable share, which the agent mounts at sandbox creation. +func durableMounts(volumeName string, mountPaths []string) []specs.Mount { + src := kata.GuestDurableVolumeDir(volumeName) + mounts := make([]specs.Mount, 0, len(mountPaths)) + for _, p := range mountPaths { + mounts = append(mounts, specs.Mount{ + Destination: p, + Source: src, + Type: "bind", + Options: []string{"rbind", "rw"}, + }) + } + return mounts +} + +// workloadSpec returns the OCI spec to start a container's overlay workload with: +// the prepared spec, plus the durable-dir binds when the actor has a volume and +// this container mounts it. +// +// The spec is copied rather than mutated so the bundle's on-disk config.json and +// the carrier's view stay as prepared — only the workload sees the binds. +func workloadSpec(c actorContainer, durableVolume string) *specs.Spec { + if durableVolume == "" || len(c.durableMountPaths) == 0 { + return c.spec + } + spec := *c.spec + spec.Mounts = append(append([]specs.Mount(nil), c.spec.Mounts...), durableMounts(durableVolume, c.durableMountPaths)...) + return &spec +} + +// stageDurableShare starts the virtiofsd serving the actor's durable-dir volumes. +// +// It serves ateompath.DurableDirVolumeMountsDir directly — no bind into the +// kataShared tree — so teardown has nothing extra to unmount. Unlike the RO +// lower's virtiofsd this one runs with cache=auto: the host contents change +// underneath the guest whenever a snapshot is restored into them. +// +// The returned cmd outlives this call (CH talks to it for the VM's lifetime); +// the caller owns it (tracked on runningActor, killed in teardownActor). +func (s *AteomService) stageDurableShare(ctx context.Context, rr resolvedRuntime, actorUID string) (*exec.Cmd, error) { + shared := ateompath.DurableDirVolumeMountsDir(actorUID) + if _, err := os.Stat(shared); err != nil { + return nil, fmt.Errorf("while checking durable-dir volumes dir %q: %w", shared, err) + } + log, _ := os.OpenFile(filepath.Join(kata.VMDir(actorUID), "virtiofsd-durable.log"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + cmd, err := kata.StartVirtiofsd(ctx, kata.VirtiofsdOptions{ + Binary: rr.virtiofsd, + SocketPath: kata.DurableVirtiofsdSocketPath(actorUID), + SharedDir: shared, + Cache: "auto", + Log: log, + }) + if err != nil { + return nil, fmt.Errorf("while starting durable-dir virtiofsd: %w", err) + } + return cmd, nil +} + +// tarDurableVolumes archives the actor's durable-dir volumes (dir) into the +// checkpoint directory. The caller must have paused the guest first: virtiofsd is +// write-through, so a completed guest write is on the host by then, but a +// running guest could still add more after the walk. +// +// Sockets the workload left behind are skipped rather than archived (tarutil +// logs them); they hold no data and the workload recreates them on start. +func tarDurableVolumes(ctx context.Context, dir, checkpointDir string) error { + if err := tarutil.Create(ctx, filepath.Join(checkpointDir, durableTarFile), dir); err != nil { + return fmt.Errorf("while archiving durable-dir volumes from %q: %w", dir, err) + } + return nil +} + +// untarDurableVolumes restores the durable-dir volumes from a snapshot into the +// actor's host directory (dir, which atelet has already created, empty). It must +// run before the durable share's virtiofsd starts, so the guest never observes +// the directory mid-restore. +func untarDurableVolumes(dir, snapshotDir string) error { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("while creating durable-dir volumes dir %q: %w", dir, err) + } + if err := tarutil.Extract(filepath.Join(snapshotDir, durableTarFile), dir); err != nil { + return fmt.Errorf("while restoring durable-dir volumes into %q: %w", dir, err) + } + return nil +} diff --git a/cmd/ateom-microvm/durable_test.go b/cmd/ateom-microvm/durable_test.go new file mode 100644 index 000000000..f8c8b1f20 --- /dev/null +++ b/cmd/ateom-microvm/durable_test.go @@ -0,0 +1,232 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" + "github.com/agent-substrate/substrate/internal/proto/ateompb" + specs "github.com/opencontainers/runtime-spec/specs-go" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestHasDurableVolumes(t *testing.T) { + tests := []struct { + name string + containers []*ateompb.Container + want bool + }{ + {name: "no containers"}, + { + name: "container without durable volumes", + containers: []*ateompb.Container{{Name: "app"}}, + }, + { + name: "one of several containers has a durable volume", + containers: []*ateompb.Container{ + {Name: "sidecar"}, + {Name: "app", DurableDirVolumes: []string{"/home/counter"}}, + }, + want: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := hasDurableVolumes(tc.containers); got != tc.want { + t.Errorf("hasDurableVolumes() = %v, want %v", got, tc.want) + } + }) + } +} + +// durableDirWith returns a durable-dir volumes directory laid out the way atelet +// prepares one: a subdirectory per volume, plus optionally a stray regular file. +func durableDirWith(t *testing.T, volumes []string, strayFile bool) string { + t.Helper() + dir := t.TempDir() + for _, v := range volumes { + if err := os.Mkdir(filepath.Join(dir, v), 0o700); err != nil { + t.Fatalf("creating volume dir %q: %v", v, err) + } + } + if strayFile { + if err := os.WriteFile(filepath.Join(dir, "not-a-volume"), nil, 0o600); err != nil { + t.Fatalf("creating stray file: %v", err) + } + } + return dir +} + +func TestResolveDurableVolumeName(t *testing.T) { + tests := []struct { + name string + volumes []string + strayFile bool + want string + wantCode codes.Code + }{ + { + name: "single volume", + volumes: []string{"data"}, + want: "data", + }, + { + name: "regular files are not volumes", + volumes: []string{"data"}, + strayFile: true, + want: "data", + }, + { + name: "no volume directory", + wantCode: codes.FailedPrecondition, + }, + { + name: "more than one volume is unsupported", + volumes: []string{"data", "other"}, + wantCode: codes.Unimplemented, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := resolveDurableVolumeName(durableDirWith(t, tc.volumes, tc.strayFile)) + if tc.wantCode != codes.OK { + if status.Code(err) != tc.wantCode { + t.Fatalf("error = %v (code %v), want code %v", err, status.Code(err), tc.wantCode) + } + return + } + if err != nil { + t.Fatalf("resolveDurableVolumeName: %v", err) + } + if got != tc.want { + t.Errorf("resolveDurableVolumeName() = %q, want %q", got, tc.want) + } + }) + } +} + +func TestResolveDurableVolumeNameMissingDir(t *testing.T) { + missing := filepath.Join(t.TempDir(), "no-such-dir") + if _, err := resolveDurableVolumeName(missing); err == nil { + t.Fatal("resolveDurableVolumeName succeeded with no durable-dir directory, want an error") + } +} + +func TestDurableVolumesRoundTrip(t *testing.T) { + // Checkpoint: a volume with data, archived while the guest is paused. + src := durableDirWith(t, []string{"data"}, false) + if err := os.WriteFile(filepath.Join(src, "data", "a.txt"), []byte("42"), 0o644); err != nil { + t.Fatalf("writing volume content: %v", err) + } + checkpointDir := t.TempDir() + if err := tarDurableVolumes(t.Context(), src, checkpointDir); err != nil { + t.Fatalf("tarDurableVolumes: %v", err) + } + if _, err := os.Stat(filepath.Join(checkpointDir, durableTarFile)); err != nil { + t.Fatalf("checkpoint is missing %s: %v", durableTarFile, err) + } + + // Restore: onto the empty directory atelet re-creates for the actor. + dst := t.TempDir() + if err := untarDurableVolumes(dst, checkpointDir); err != nil { + t.Fatalf("untarDurableVolumes: %v", err) + } + got, err := os.ReadFile(filepath.Join(dst, "data", "a.txt")) + if err != nil { + t.Fatalf("reading restored content: %v", err) + } + if string(got) != "42" { + t.Errorf("restored content = %q, want %q", got, "42") + } + // The volume name must survive the round trip: it is how the guest mount + // path is resolved after a restore onto another node. + name, err := resolveDurableVolumeName(dst) + if err != nil { + t.Fatalf("resolveDurableVolumeName after restore: %v", err) + } + if name != "data" { + t.Errorf("resolved volume name = %q, want %q", name, "data") + } +} + +func TestDurableMounts(t *testing.T) { + got := durableMounts("data", []string{"/home/counter", "/var/data"}) + want := []specs.Mount{ + {Destination: "/home/counter", Source: "/run/ateom-durable/data", Type: "bind", Options: []string{"rbind", "rw"}}, + {Destination: "/var/data", Source: "/run/ateom-durable/data", Type: "bind", Options: []string{"rbind", "rw"}}, + } + if len(got) != len(want) { + t.Fatalf("durableMounts() returned %d mounts, want %d", len(got), len(want)) + } + for i := range want { + if got[i].Destination != want[i].Destination || got[i].Source != want[i].Source || + got[i].Type != want[i].Type || !slices.Equal(got[i].Options, want[i].Options) { + t.Errorf("mount %d = %+v, want %+v", i, got[i], want[i]) + } + } + // The source must match what the agent mounts the durable share at. + if want[0].Source != kata.GuestDurableVolumeDir("data") { + t.Errorf("mount source %q does not match kata.GuestDurableVolumeDir", want[0].Source) + } +} + +func TestWorkloadSpec(t *testing.T) { + base := []specs.Mount{{Destination: "/proc", Type: "proc", Source: "proc"}} + newContainer := func(paths []string) actorContainer { + return actorContainer{ + name: "app", + spec: &specs.Spec{Mounts: slices.Clone(base)}, + durableMountPaths: paths, + } + } + + t.Run("no durable volume returns the spec unchanged", func(t *testing.T) { + c := newContainer(nil) + if got := workloadSpec(c, ""); got != c.spec { + t.Error("workloadSpec() copied the spec when there was nothing to add") + } + }) + + t.Run("container without mounts is unchanged", func(t *testing.T) { + c := newContainer(nil) + if got := workloadSpec(c, "data"); got != c.spec { + t.Error("workloadSpec() copied the spec for a container with no durable mounts") + } + }) + + t.Run("durable mounts are appended without mutating the source spec", func(t *testing.T) { + c := newContainer([]string{"/home/counter"}) + got := workloadSpec(c, "data") + if len(got.Mounts) != len(base)+1 { + t.Fatalf("workload spec has %d mounts, want %d", len(got.Mounts), len(base)+1) + } + last := got.Mounts[len(got.Mounts)-1] + if last.Destination != "/home/counter" || last.Source != kata.GuestDurableVolumeDir("data") { + t.Errorf("appended mount = %+v, want the durable volume bound at /home/counter", last) + } + // The prepared spec (shared with the carrier and the on-disk bundle) must + // not gain the bind. + if len(c.spec.Mounts) != len(base) { + t.Errorf("source spec now has %d mounts, want %d", len(c.spec.Mounts), len(base)) + } + }) +} diff --git a/cmd/ateom-microvm/internal/kata/overlay_linux.go b/cmd/ateom-microvm/internal/kata/overlay_linux.go index 4a7864519..abf0995df 100644 --- a/cmd/ateom-microvm/internal/kata/overlay_linux.go +++ b/cmd/ateom-microvm/internal/kata/overlay_linux.go @@ -46,8 +46,22 @@ const ( // guestSharedDir is where the agent mounts the kataShared tag in the guest; // per-container rootfs then lives at //rootfs. guestSharedDir = "/run/kata-containers/shared/containers/" + + // DurableFsTag is the virtio-fs tag for the actor's WRITABLE durable-dir + // share, served by a second virtiofsd (the kataShared share stays read-only). + DurableFsTag = "ateDurable" + // guestDurableDir is where the agent mounts DurableFsTag in the guest; each + // volume's contents live at / and are bind-mounted + // from there into the containers that declare the volume. + guestDurableDir = "/run/ateom-durable" ) +// GuestDurableVolumeDir is the in-guest path holding one durable volume's +// contents, i.e. the bind source for that volume's container mount points. +func GuestDurableVolumeDir(volumeName string) string { + return guestDurableDir + "/" + volumeName +} + // SharedDir is the host directory virtiofsd serves into the guest as the RO base. // Its layout (/rootfs) is what find-paths re-opens by path on restore. func SharedDir(id string) string { @@ -74,7 +88,32 @@ type VirtiofsdOptions struct { Binary string // virtiofsd executable; defaults to "virtiofsd" SocketPath string // vhost-user socket CH connects to (VirtiofsdSocketPath) SharedDir string // directory to serve (SharedDir(id)) - Log io.Writer + // Cache is virtiofsd's --cache mode. Empty defaults to "always", which is + // only correct for a strictly read-only share (see virtiofsdArgs). + Cache string + Log io.Writer +} + +// virtiofsdArgs builds the virtiofsd command line for o. +func virtiofsdArgs(o VirtiofsdOptions) []string { + cache := o.Cache + if cache == "" { + // The overlay RO lower is served strictly read-only (the carrier remounts it + // ro and the guest's overlayfs lowerdir is immutable), so aggressively cache + // it in the guest for read performance — there is no host<>guest write churn + // to invalidate. A WRITABLE share (the durable-dir volumes) must instead pass + // Cache: "auto", because cache=always would serve stale data once the host + // side changes underneath the guest (e.g. contents restored from a snapshot). + cache = "always" + } + return []string{ + "--socket-path=" + o.SocketPath, + "--shared-dir=" + o.SharedDir, + "--cache=" + cache, + "--thread-pool-size=1", + "--announce-submounts", + "--migration-mode", "find-paths", + } } // StartVirtiofsd launches virtiofsd in find-paths migration mode serving o.SharedDir @@ -86,22 +125,7 @@ func StartVirtiofsd(ctx context.Context, o VirtiofsdOptions) (*exec.Cmd, error) bin = "virtiofsd" } _ = os.Remove(o.SocketPath) - cmd := exec.Command(bin, - "--socket-path="+o.SocketPath, - "--shared-dir="+o.SharedDir, - // The shared dir is served strictly read-only (the overlay's RO lower; the - // carrier remounts it ro and the guest's overlayfs lowerdir is immutable), so - // aggressively cache it in the guest for read performance — there is no - // host<>guest write churn to invalidate. - // TODO: cache=always serves stale data for any writable virtio-fs mount. If we - // later need one (e.g. projected volumes), prefer keeping this mount fully - // read-only and writing such volumes via a separate mechanism (e.g. a writer - // that execs into the sandbox) rather than dropping back to cache=auto/none. - "--cache=always", - "--thread-pool-size=1", - "--announce-submounts", - "--migration-mode", "find-paths", - ) + cmd := exec.Command(bin, virtiofsdArgs(o)...) cmd.Stdout = o.Log cmd.Stderr = o.Log if err := cmd.Start(); err != nil { @@ -166,16 +190,28 @@ func ReconstructSharedDirFromImage(ctx context.Context, bundleRootfs, restoreID, // CreateSandboxForActor creates the guest sandbox with the kataShared virtio-fs mount // (the RO base backing every container's rootfs). Mirrors kata startSandbox. -func (a *AgentClient) CreateSandboxForActor(ctx context.Context, sandboxID, hostname string) error { +// +// withDurableShare additionally mounts the writable durable-dir share, whose +// per-volume subdirectories the containers bind-mount at their declared paths. +func (a *AgentClient) CreateSandboxForActor(ctx context.Context, sandboxID, hostname string, withDurableShare bool) error { + storages := []*agentpb.Storage{{ + Driver: virtioFSDriver, + Source: FsTag, + Fstype: typeVirtioFS, + MountPoint: guestSharedDir, + }} + if withDurableShare { + storages = append(storages, &agentpb.Storage{ + Driver: virtioFSDriver, + Source: DurableFsTag, + Fstype: typeVirtioFS, + MountPoint: guestDurableDir, + }) + } return a.CreateSandbox(ctx, &agentpb.CreateSandboxRequest{ Hostname: hostname, SandboxId: sandboxID, - Storages: []*agentpb.Storage{{ - Driver: virtioFSDriver, - Source: FsTag, - Fstype: typeVirtioFS, - MountPoint: guestSharedDir, - }}, + Storages: storages, }) } diff --git a/cmd/ateom-microvm/internal/kata/overlay_linux_test.go b/cmd/ateom-microvm/internal/kata/overlay_linux_test.go new file mode 100644 index 000000000..c6dfb56a8 --- /dev/null +++ b/cmd/ateom-microvm/internal/kata/overlay_linux_test.go @@ -0,0 +1,65 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kata + +import ( + "slices" + "testing" +) + +func TestVirtiofsdArgs(t *testing.T) { + tests := []struct { + name string + opts VirtiofsdOptions + wantCache string + }{ + { + name: "RO lower defaults to cache=always", + opts: VirtiofsdOptions{SocketPath: "/run/vm/virtiofsd.sock", SharedDir: "/run/shared"}, + wantCache: "--cache=always", + }, + { + name: "writable durable share overrides the cache mode", + opts: VirtiofsdOptions{ + SocketPath: "/run/vm/virtiofsd-durable.sock", + SharedDir: "/var/lib/ateom-gvisor/actors/uid/durable-dir", + Cache: "auto", + }, + wantCache: "--cache=auto", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + args := virtiofsdArgs(tc.opts) + if !slices.Contains(args, tc.wantCache) { + t.Errorf("args %v do not contain %q", args, tc.wantCache) + } + for _, want := range []string{ + "--socket-path=" + tc.opts.SocketPath, + "--shared-dir=" + tc.opts.SharedDir, + // find-paths re-opens the guest's open files by path on + // restore; both shares depend on it. + "--migration-mode", + "find-paths", + } { + if !slices.Contains(args, want) { + t.Errorf("args %v do not contain %q", args, want) + } + } + }) + } +} diff --git a/cmd/ateom-microvm/internal/kata/restore.go b/cmd/ateom-microvm/internal/kata/restore.go index 2dccbca8f..0b71bea9d 100644 --- a/cmd/ateom-microvm/internal/kata/restore.go +++ b/cmd/ateom-microvm/internal/kata/restore.go @@ -29,3 +29,9 @@ func VMDir(id string) string { return filepath.Join(vcVMDir, id) } // VsockSocketPath is the hybrid-vsock socket the CH snapshot's vsock device // references; CH recreates the listener here on restore. func VsockSocketPath(id string) string { return filepath.Join(VMDir(id), "clh.sock") } + +// DurableVirtiofsdSocketPath is the vhost-user-fs socket for the actor's writable +// durable-dir share, served by a second virtiofsd alongside the RO lower's. +func DurableVirtiofsdSocketPath(id string) string { + return filepath.Join(VMDir(id), "virtiofsd-durable.sock") +} diff --git a/cmd/ateom-microvm/internal/tarutil/fifo_linux.go b/cmd/ateom-microvm/internal/tarutil/fifo_linux.go new file mode 100644 index 000000000..c7496fc9e --- /dev/null +++ b/cmd/ateom-microvm/internal/tarutil/fifo_linux.go @@ -0,0 +1,48 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarutil + +import ( + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +// createFifo creates a FIFO at name, a path relative to root, with mode's +// permission bits. +// +// os.Root has no Mkfifo, so this opens name's parent directory THROUGH root — +// which refuses to traverse a symlink out of the tree, the same guarantee the +// rest of extraction relies on — and creates the FIFO relative to that +// directory's descriptor. The final element is a single path component, so it +// cannot walk anywhere from there. +func createFifo(root *os.Root, name string, mode os.FileMode) error { + dir, base := filepath.Split(name) + if dir == "" { + dir = "." + } + parent, err := root.Open(filepath.Clean(dir)) + if err != nil { + return fmt.Errorf("opening parent directory of %q: %w", name, err) + } + defer parent.Close() + + if err := unix.Mkfifoat(int(parent.Fd()), base, uint32(mode.Perm())); err != nil { + return fmt.Errorf("creating fifo %q: %w", name, err) + } + return nil +} diff --git a/cmd/ateom-microvm/internal/tarutil/owner_linux.go b/cmd/ateom-microvm/internal/tarutil/owner_linux.go new file mode 100644 index 000000000..9d33a0785 --- /dev/null +++ b/cmd/ateom-microvm/internal/tarutil/owner_linux.go @@ -0,0 +1,85 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarutil + +import ( + "archive/tar" + "errors" + "fmt" + "os" + "syscall" +) + +// inodeKey identifies an inode within one filesystem, so hardlinks to the same +// file can be recognized while distinct files that share an inode number across +// devices are not conflated. +type inodeKey struct { + dev uint64 + ino uint64 +} + +// inodeOf returns info's inode identity, or ok=false on a platform or file +// where it is unavailable (the caller then archives the contents rather than +// emitting a hardlink). +func inodeOf(info os.FileInfo) (inodeKey, bool) { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return inodeKey{}, false + } + return inodeKey{dev: uint64(st.Dev), ino: uint64(st.Ino)}, true +} + +// nlinkOf returns the file's link count, or 1 when unavailable. +func nlinkOf(info os.FileInfo) uint64 { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 1 + } + return uint64(st.Nlink) +} + +// setOwner records the on-disk uid/gid in the header. tar.FileInfoHeader leaves +// them zero, which would silently re-own every restored file to root. +func setOwner(hdr *tar.Header, info os.FileInfo) { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return + } + hdr.Uid = int(st.Uid) + hdr.Gid = int(st.Gid) + // Names are deliberately left empty: the guest's passwd/group namespace is + // not the host's, so only the numeric ids are meaningful. + hdr.Uname = "" + hdr.Gname = "" +} + +// lchownEntry restores an entry's ownership without following symlinks. +// +// Changing a file's owner requires privilege. The production caller (ateom) is +// root in its worker pod, so ownership is restored faithfully and any EPERM +// there signals a real problem worth failing on. An unprivileged process cannot +// chown at all, so rather than making the package unusable outside a root +// context (tests, local tooling), EPERM is tolerated there and the extracted +// files simply belong to the extracting user. +func lchownEntry(root *os.Root, name string, hdr *tar.Header) error { + err := root.Lchown(name, hdr.Uid, hdr.Gid) + if err == nil { + return nil + } + if errors.Is(err, os.ErrPermission) && os.Geteuid() != 0 { + return nil + } + return fmt.Errorf("restoring ownership of %q to %d:%d: %w", name, hdr.Uid, hdr.Gid, err) +} diff --git a/cmd/ateom-microvm/internal/tarutil/tarutil.go b/cmd/ateom-microvm/internal/tarutil/tarutil.go new file mode 100644 index 000000000..7b1a6d319 --- /dev/null +++ b/cmd/ateom-microvm/internal/tarutil/tarutil.go @@ -0,0 +1,374 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tarutil archives and restores a directory tree as a tar file, +// preserving the metadata a workload's data directory depends on: modes, +// ownership, modification times, symlinks, hardlinks, and FIFOs. +// +// It exists for snapshotting durable-dir volumes (see cmd/ateom-microvm): the +// contents are written by the sandboxed workload under arbitrary uids, shipped +// to object storage, and restored — possibly onto another node — where the +// workload must see them unchanged. +// +// Extraction is confined to the destination with os.Root, so a crafted archive +// cannot write outside it via "..", an absolute path, or a symlink. +package tarutil + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + "sort" + "strings" +) + +// Create writes a tar archive of srcDir's contents to tarPath. Entry names are +// relative to srcDir, so extracting into another directory reproduces the tree. +// srcDir itself is not an entry. +// +// Regular files, directories, symlinks, and FIFOs are archived with their mode, +// ownership, and modification time. A file with multiple links inside srcDir is +// archived once and referenced as a hardlink thereafter. Sockets are skipped +// (see writeTree). A device node is an error rather than silent data loss. +func Create(ctx context.Context, tarPath, srcDir string) error { + f, err := os.Create(tarPath) + if err != nil { + return fmt.Errorf("creating tar %q: %w", tarPath, err) + } + defer f.Close() + + tw := tar.NewWriter(f) + if err := writeTree(ctx, tw, srcDir); err != nil { + return err + } + if err := tw.Close(); err != nil { + return fmt.Errorf("closing tar %q: %w", tarPath, err) + } + // Durable-dir tars are handed to atelet for upload as soon as we return, so + // flush to disk rather than trusting the page cache to outlive us. + if err := f.Sync(); err != nil { + return fmt.Errorf("syncing tar %q: %w", tarPath, err) + } + return nil +} + +// writeTree walks srcDir in lexical order (filepath.WalkDir) and writes one +// entry per path. The deterministic order keeps archives of identical trees +// byte-comparable, which makes snapshot diffs meaningful. +func writeTree(ctx context.Context, tw *tar.Writer, srcDir string) error { + // Maps an already-archived multi-link inode to the name it was archived + // under, so later links become tar hardlink entries instead of copies. + linked := map[inodeKey]string{} + + return filepath.WalkDir(srcDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(srcDir, path) + if err != nil { + return err + } + if rel == "." { + return nil + } + + info, err := d.Info() + if err != nil { + return err + } + + // Skipped before the header is built, because archive/tar cannot + // represent a socket at all. A socket carries no data and is meaningless + // without the process listening on it, which is gone by the time this + // archive is read; workloads recreate theirs on start. GNU tar and + // containerd's archiver both skip sockets for that reason. Failing + // instead would be worse than useless here: agents leave sockets lying + // around in their data directories (ssh-agent, gpg-agent, language + // servers), and one of them would make every checkpoint fail, stranding + // the actor on its worker with no way to suspend it. + if info.Mode()&os.ModeSocket != 0 { + slog.WarnContext(ctx, "Skipping socket while archiving directory", + slog.String("path", path), slog.String("root", srcDir)) + return nil + } + + link := "" + if info.Mode()&os.ModeSymlink != 0 { + if link, err = os.Readlink(path); err != nil { + return fmt.Errorf("reading symlink %q: %w", path, err) + } + } + hdr, err := tar.FileInfoHeader(info, link) + if err != nil { + return fmt.Errorf("building tar header for %q: %w", path, err) + } + hdr.Name = filepath.ToSlash(rel) + if d.IsDir() { + hdr.Name += "/" + } + setOwner(hdr, info) + + switch { + case info.Mode().IsRegular(): + // A second link to an already-archived inode: record the link and + // skip the contents. + if key, ok := inodeOf(info); ok && info.Sys() != nil && nlinkOf(info) > 1 { + if target, seen := linked[key]; seen { + hdr.Typeflag = tar.TypeLink + hdr.Linkname = target + hdr.Size = 0 + return tw.WriteHeader(hdr) + } + linked[key] = hdr.Name + } + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("writing tar header for %q: %w", path, err) + } + return copyFileInto(tw, path) + + case d.IsDir(), info.Mode()&os.ModeSymlink != 0, info.Mode()&os.ModeNamedPipe != 0: + // FileInfoHeader already gave a FIFO Typeflag TypeFifo and size 0. + return tw.WriteHeader(hdr) + + default: + // Device nodes reach here. They need privilege to create, so one in a + // workload's data directory means something unexpected happened — + // worth failing on rather than silently dropping or re-creating it + // under a later restore. + return fmt.Errorf("unsupported file type %v at %q", info.Mode().Type(), path) + } + }) +} + +// copyFileInto streams path's contents into the archive. +func copyFileInto(tw *tar.Writer, path string) error { + in, err := os.Open(path) + if err != nil { + return fmt.Errorf("opening %q: %w", path, err) + } + defer in.Close() + if _, err := io.Copy(tw, in); err != nil { + return fmt.Errorf("archiving contents of %q: %w", path, err) + } + return nil +} + +// Extract unpacks tarPath into dstDir, which must already exist. Modes, +// ownership, modification times, symlinks, and hardlinks are restored. +// +// All writes go through an os.Root rooted at dstDir, so entries naming a path +// outside it — via "..", an absolute path, or a symlink planted earlier in the +// same archive — are refused rather than followed. An entry that collides with +// an existing path replaces it ("later entry wins", standard tar semantics), +// except that an existing directory is kept when the entry is also a directory. +func Extract(tarPath, dstDir string) error { + f, err := os.Open(tarPath) + if err != nil { + return fmt.Errorf("opening tar %q: %w", tarPath, err) + } + defer f.Close() + + root, err := os.OpenRoot(dstDir) + if err != nil { + return fmt.Errorf("opening destination %q: %w", dstDir, err) + } + defer root.Close() + + // Directories are created owner-writable so their children can be written + // even when the archive marks them read-only; the recorded modes and times + // are applied after every child exists (see restoreDirMeta). + dirs := map[string]*tar.Header{} + + tr := tar.NewReader(f) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("reading tar %q: %w", tarPath, err) + } + name, skip, err := cleanTarName(hdr.Name) + if err != nil { + return fmt.Errorf("invalid entry in tar %q: %w", tarPath, err) + } + if skip { + continue + } + if err := extractEntry(root, tr, hdr, name, dirs); err != nil { + return err + } + } + return restoreDirMeta(root, dirs) +} + +// extractEntry materializes one archive entry under root. +func extractEntry(root *os.Root, tr *tar.Reader, hdr *tar.Header, name string, dirs map[string]*tar.Header) error { + mode := hdr.FileInfo().Mode().Perm() + + switch hdr.Typeflag { + case tar.TypeDir: + if err := root.Mkdir(name, mode|0o700); err != nil && !errors.Is(err, os.ErrExist) { + return fmt.Errorf("creating directory %q: %w", name, err) + } + dirs[name] = hdr + return nil + + case tar.TypeReg: + if err := replaceExisting(root, name); err != nil { + return err + } + out, err := root.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + if err != nil { + return fmt.Errorf("creating file %q: %w", name, err) + } + _, copyErr := io.Copy(out, tr) + closeErr := out.Close() + if copyErr != nil { + return fmt.Errorf("writing contents of %q: %w", name, copyErr) + } + if closeErr != nil { + return fmt.Errorf("closing %q: %w", name, closeErr) + } + return restoreMeta(root, name, hdr) + + case tar.TypeSymlink: + if err := replaceExisting(root, name); err != nil { + return err + } + if err := root.Symlink(hdr.Linkname, name); err != nil { + return fmt.Errorf("creating symlink %q -> %q: %w", name, hdr.Linkname, err) + } + // Only ownership applies to a symlink itself; its mode is meaningless + // and its times would follow the target. + return lchownEntry(root, name, hdr) + + case tar.TypeFifo: + if err := replaceExisting(root, name); err != nil { + return err + } + if err := createFifo(root, name, mode); err != nil { + return err + } + return restoreMeta(root, name, hdr) + + case tar.TypeLink: + target, skip, err := cleanTarName(hdr.Linkname) + if err != nil || skip { + return fmt.Errorf("invalid hardlink target %q for %q", hdr.Linkname, name) + } + if err := replaceExisting(root, name); err != nil { + return err + } + if err := root.Link(target, name); err != nil { + return fmt.Errorf("creating hardlink %q -> %q: %w", name, target, err) + } + return nil + + default: + return fmt.Errorf("unsupported tar entry type %q at %q", string([]byte{hdr.Typeflag}), name) + } +} + +// replaceExisting removes whatever currently occupies name so the new entry +// does not write through a symlink, truncate a shared inode, or collide with a +// directory. +func replaceExisting(root *os.Root, name string) error { + if _, err := root.Lstat(name); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("checking existing path %q: %w", name, err) + } + if err := root.RemoveAll(name); err != nil { + return fmt.Errorf("replacing existing path %q: %w", name, err) + } + return nil +} + +// restoreDirMeta applies the archived modes, ownership, and times to extracted +// directories, deepest first: a directory's path is always longer than its +// parent's, so length-descending order restores children before the parent's +// mode can make them unreachable. +func restoreDirMeta(root *os.Root, dirs map[string]*tar.Header) error { + names := make([]string, 0, len(dirs)) + for name := range dirs { + names = append(names, name) + } + sort.Slice(names, func(i, j int) bool { return len(names[i]) > len(names[j]) }) + for _, name := range names { + if err := restoreMeta(root, name, dirs[name]); err != nil { + return err + } + } + return nil +} + +// restoredMode is the mode an extracted entry ends up with: the permission bits +// plus setuid, setgid, and sticky, which FileMode.Perm() alone would drop. +// +// The archive records them, so dropping them would make the round trip silently +// lossy. The case that bites is a setgid data directory (0o2775): the workload +// relies on group inheritance for files its different uids create, and losing +// the bit only surfaces later, when the next file lands with the wrong group. +// These bits govern behavior inside the sandbox — the host never executes a +// workload's files — and a running workload can set them on its own data +// anyway, so carrying them across a suspend/resume grants it nothing new. +func restoredMode(hdr *tar.Header) os.FileMode { + return hdr.FileInfo().Mode() & (fs.ModePerm | fs.ModeSetuid | fs.ModeSetgid | fs.ModeSticky) +} + +// restoreMeta applies ownership, mode, and modification time to an extracted +// path. Ownership is applied first because chowning a file clears its setuid +// and setgid bits, which the chmod below then puts back. +func restoreMeta(root *os.Root, name string, hdr *tar.Header) error { + if err := lchownEntry(root, name, hdr); err != nil { + return err + } + if err := root.Chmod(name, restoredMode(hdr)); err != nil { + return fmt.Errorf("restoring mode on %q: %w", name, err) + } + if !hdr.ModTime.IsZero() { + if err := root.Chtimes(name, hdr.ModTime, hdr.ModTime); err != nil { + return fmt.Errorf("restoring times on %q: %w", name, err) + } + } + return nil +} + +// cleanTarName validates an archive entry name and returns it relative and +// slash-free of surprises. skip is true for entries that name nothing ("" or +// "."), which some tar writers emit for the archive root. +func cleanTarName(name string) (cleaned string, skip bool, err error) { + if name == "" { + return "", true, nil + } + cleaned = filepath.Clean(filepath.FromSlash(name)) + cleaned = strings.TrimPrefix(cleaned, string(filepath.Separator)) + if cleaned == "" || cleaned == "." { + return "", true, nil + } + if !filepath.IsLocal(cleaned) { + return "", false, fmt.Errorf("not a local path: %q", name) + } + return cleaned, false, nil +} diff --git a/cmd/ateom-microvm/internal/tarutil/tarutil_test.go b/cmd/ateom-microvm/internal/tarutil/tarutil_test.go new file mode 100644 index 000000000..d8b72c4ae --- /dev/null +++ b/cmd/ateom-microvm/internal/tarutil/tarutil_test.go @@ -0,0 +1,499 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarutil + +import ( + "archive/tar" + "errors" + "net" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/roottest" + "golang.org/x/sys/unix" +) + +// writeTar builds a tar file at path from the given headers; a header with a +// non-empty body is written as a regular file with that content. +func writeTar(t *testing.T, path string, entries ...tar.Header) { + t.Helper() + f, err := os.Create(path) + if err != nil { + t.Fatalf("creating tar: %v", err) + } + defer f.Close() + tw := tar.NewWriter(f) + for _, hdr := range entries { + if err := tw.WriteHeader(&hdr); err != nil { + t.Fatalf("writing header %q: %v", hdr.Name, err) + } + if hdr.Size > 0 { + if _, err := tw.Write([]byte(strings.Repeat("x", int(hdr.Size)))); err != nil { + t.Fatalf("writing body %q: %v", hdr.Name, err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatalf("closing tar: %v", err) + } +} + +func TestRoundTrip(t *testing.T) { + src := t.TempDir() + mustWrite := func(rel, content string, mode os.FileMode) string { + t.Helper() + p := filepath.Join(src, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatalf("mkdir for %q: %v", rel, err) + } + if err := os.WriteFile(p, []byte(content), mode); err != nil { + t.Fatalf("writing %q: %v", rel, err) + } + if err := os.Chmod(p, mode); err != nil { + t.Fatalf("chmod %q: %v", rel, err) + } + return p + } + + mustWrite("a.txt", "hello", 0o644) + mustWrite("sub/b.txt", "nested", 0o600) + // A read-only directory: extraction must still populate it, then restore + // the mode. + mustWrite("ro/c.txt", "in-readonly", 0o444) + if err := os.Chmod(filepath.Join(src, "ro"), 0o500); err != nil { + t.Fatalf("chmod ro dir: %v", err) + } + // TempDir cleanup cannot unlink children of a non-writable directory, so + // make both copies writable again once the test is done. + restoreWritable := func(dir string) { + t.Cleanup(func() { _ = os.Chmod(filepath.Join(dir, "ro"), 0o700) }) + } + restoreWritable(src) + if err := os.Mkdir(filepath.Join(src, "empty"), 0o755); err != nil { + t.Fatalf("mkdir empty: %v", err) + } + if err := os.Symlink("a.txt", filepath.Join(src, "link")); err != nil { + t.Fatalf("symlink: %v", err) + } + if err := os.Link(filepath.Join(src, "a.txt"), filepath.Join(src, "hard.txt")); err != nil { + t.Fatalf("hardlink: %v", err) + } + mtime := time.Unix(1600000000, 0) + if err := os.Chtimes(filepath.Join(src, "a.txt"), mtime, mtime); err != nil { + t.Fatalf("chtimes: %v", err) + } + + tarPath := filepath.Join(t.TempDir(), "out.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + restoreWritable(dst) + + t.Run("contents", func(t *testing.T) { + for rel, want := range map[string]string{ + "a.txt": "hello", + "sub/b.txt": "nested", + "ro/c.txt": "in-readonly", + "hard.txt": "hello", + } { + got, err := os.ReadFile(filepath.Join(dst, rel)) + if err != nil { + t.Errorf("reading %q: %v", rel, err) + continue + } + if string(got) != want { + t.Errorf("%q = %q, want %q", rel, got, want) + } + } + }) + + t.Run("modes", func(t *testing.T) { + for rel, want := range map[string]os.FileMode{ + "a.txt": 0o644, + "sub/b.txt": 0o600, + "ro/c.txt": 0o444, + "ro": 0o500, + "empty": 0o755, + } { + st, err := os.Lstat(filepath.Join(dst, rel)) + if err != nil { + t.Errorf("lstat %q: %v", rel, err) + continue + } + if got := st.Mode().Perm(); got != want { + t.Errorf("%q mode = %v, want %v", rel, got, want) + } + } + }) + + t.Run("mtime", func(t *testing.T) { + st, err := os.Stat(filepath.Join(dst, "a.txt")) + if err != nil { + t.Fatalf("stat: %v", err) + } + if !st.ModTime().Equal(mtime) { + t.Errorf("a.txt mtime = %v, want %v", st.ModTime(), mtime) + } + }) + + t.Run("symlink", func(t *testing.T) { + got, err := os.Readlink(filepath.Join(dst, "link")) + if err != nil { + t.Fatalf("readlink: %v", err) + } + if got != "a.txt" { + t.Errorf("link target = %q, want %q", got, "a.txt") + } + }) + + t.Run("hardlink shares inode", func(t *testing.T) { + a, err := os.Stat(filepath.Join(dst, "a.txt")) + if err != nil { + t.Fatalf("stat a.txt: %v", err) + } + h, err := os.Stat(filepath.Join(dst, "hard.txt")) + if err != nil { + t.Fatalf("stat hard.txt: %v", err) + } + if !os.SameFile(a, h) { + t.Error("hard.txt is a copy, want the same inode as a.txt") + } + }) +} + +func TestCreateEmptyDir(t *testing.T) { + tarPath := filepath.Join(t.TempDir(), "empty.tar") + if err := Create(t.Context(), tarPath, t.TempDir()); err != nil { + t.Fatalf("Create: %v", err) + } + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + entries, err := os.ReadDir(dst) + if err != nil { + t.Fatalf("readdir: %v", err) + } + if len(entries) != 0 { + t.Errorf("extracted %d entries from an empty archive, want 0", len(entries)) + } +} + +func TestRoundTripFifo(t *testing.T) { + src := t.TempDir() + if err := unix.Mkfifo(filepath.Join(src, "pipe"), 0o640); err != nil { + t.Fatalf("creating fifo: %v", err) + } + + tarPath := filepath.Join(t.TempDir(), "fifo.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + + st, err := os.Lstat(filepath.Join(dst, "pipe")) + if err != nil { + t.Fatalf("lstat restored fifo: %v", err) + } + if st.Mode()&os.ModeNamedPipe == 0 { + t.Errorf("restored entry mode = %v, want a fifo", st.Mode()) + } + if got := st.Mode().Perm(); got != 0o640 { + t.Errorf("restored fifo mode = %v, want %v", got, os.FileMode(0o640)) + } +} + +// TestCreateSkipsSockets covers the availability property that matters most for +// snapshots: a socket in the archived tree — which agents leave behind routinely +// — must not fail the archive, because that would make the actor impossible to +// suspend. +func TestCreateSkipsSockets(t *testing.T) { + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "keep.txt"), []byte("data"), 0o644); err != nil { + t.Fatalf("writing file: %v", err) + } + l, err := net.Listen("unix", filepath.Join(src, "agent.sock")) + if err != nil { + t.Fatalf("creating socket: %v", err) + } + defer l.Close() + + tarPath := filepath.Join(t.TempDir(), "sock.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + + // The rest of the tree survives; the socket itself is dropped. + got, err := os.ReadFile(filepath.Join(dst, "keep.txt")) + if err != nil { + t.Fatalf("reading archived file: %v", err) + } + if string(got) != "data" { + t.Errorf("keep.txt = %q, want %q", got, "data") + } + if _, err := os.Lstat(filepath.Join(dst, "agent.sock")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("socket was archived (lstat err = %v), want it skipped", err) + } +} + +// TestRoundTripSpecialModeBits pins setuid/setgid/sticky across a round trip. +// FileMode.Perm() silently drops them, so without this the archive would record +// bits that extraction throws away — and a setgid data directory would come back +// from a suspend having quietly lost its group inheritance. +func TestRoundTripSpecialModeBits(t *testing.T) { + src := t.TempDir() + // Modes are applied after creation because MkdirAll and WriteFile mask the + // special bits against the umask. + setgidDir := filepath.Join(src, "shared") + if err := os.Mkdir(setgidDir, 0o775); err != nil { + t.Fatalf("mkdir: %v", err) + } + stickyDir := filepath.Join(src, "tmp") + if err := os.Mkdir(stickyDir, 0o777); err != nil { + t.Fatalf("mkdir: %v", err) + } + setuidFile := filepath.Join(src, "tool") + if err := os.WriteFile(setuidFile, []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatalf("writing file: %v", err) + } + for path, mode := range map[string]os.FileMode{ + setgidDir: os.ModeDir | os.ModeSetgid | 0o775, + stickyDir: os.ModeDir | os.ModeSticky | 0o777, + setuidFile: os.ModeSetuid | 0o755, + } { + if err := os.Chmod(path, mode); err != nil { + t.Fatalf("chmod %q: %v", path, err) + } + } + + tarPath := filepath.Join(t.TempDir(), "modes.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + + for name, want := range map[string]os.FileMode{ + "shared": os.ModeDir | os.ModeSetgid | 0o775, + "tmp": os.ModeDir | os.ModeSticky | 0o777, + "tool": os.ModeSetuid | 0o755, + } { + st, err := os.Lstat(filepath.Join(dst, name)) + if err != nil { + t.Errorf("lstat %q: %v", name, err) + continue + } + if got := st.Mode(); got != want { + t.Errorf("%q mode = %v, want %v", name, got, want) + } + } +} + +// TestRoundTripOwnership covers the reason this package exists rather than a +// plain tar: durable-dir contents are written by the sandboxed workload under +// its own uids, and a restore that re-owned everything to root would hand the +// workload back files it cannot write. Needs root, since only root can give a +// file away. +func TestRoundTripOwnership(t *testing.T) { + roottest.Require(t, "restoring file ownership requires CAP_CHOWN") + + const uid, gid = 1234, 5678 + src := t.TempDir() + path := filepath.Join(src, "owned.txt") + if err := os.WriteFile(path, []byte("data"), 0o644); err != nil { + t.Fatalf("writing file: %v", err) + } + if err := os.Chown(path, uid, gid); err != nil { + t.Fatalf("chown: %v", err) + } + + tarPath := filepath.Join(t.TempDir(), "owned.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + + st, err := os.Stat(filepath.Join(dst, "owned.txt")) + if err != nil { + t.Fatalf("stat restored file: %v", err) + } + sys, ok := st.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal("stat has no syscall.Stat_t") + } + if sys.Uid != uid || sys.Gid != gid { + t.Errorf("restored ownership = %d:%d, want %d:%d", sys.Uid, sys.Gid, uid, gid) + } +} + +func TestCreateRejectsDeviceNode(t *testing.T) { + roottest.Require(t, "creating a device node requires root") + + src := t.TempDir() + // /dev/null, the cheapest character device to plant. + if err := unix.Mknod(filepath.Join(src, "null"), unix.S_IFCHR|0o666, int(unix.Mkdev(1, 3))); err != nil { + t.Fatalf("creating device node: %v", err) + } + err := Create(t.Context(), filepath.Join(t.TempDir(), "dev.tar"), src) + if err == nil { + t.Fatal("Create succeeded on a device node, want an error") + } + if !strings.Contains(err.Error(), "unsupported file type") { + t.Errorf("error = %v, want it to mention an unsupported file type", err) + } +} + +func TestExtractRejectsEscapes(t *testing.T) { + tests := []struct { + name string + entries []tar.Header + }{ + { + name: "parent traversal", + entries: []tar.Header{{Name: "../escape.txt", Mode: 0o644, Size: 3, Typeflag: tar.TypeReg}}, + }, + { + name: "nested parent traversal", + entries: []tar.Header{{Name: "sub/../../escape.txt", Mode: 0o644, Size: 3, Typeflag: tar.TypeReg}}, + }, + { + name: "write through symlink out of tree", + entries: []tar.Header{ + {Name: "out", Linkname: "/tmp", Mode: 0o777, Typeflag: tar.TypeSymlink}, + {Name: "out/escape.txt", Mode: 0o644, Size: 3, Typeflag: tar.TypeReg}, + }, + }, + { + name: "hardlink to a path outside the tree", + entries: []tar.Header{{Name: "link", Linkname: "../../etc/passwd", Typeflag: tar.TypeLink}}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tarPath := filepath.Join(t.TempDir(), "bad.tar") + writeTar(t, tarPath, tc.entries...) + dst := t.TempDir() + if err := Extract(tarPath, dst); err == nil { + t.Fatal("Extract succeeded, want an error") + } + // Whatever the archive intended, nothing may exist outside dst. + if _, err := os.Lstat(filepath.Join(filepath.Dir(dst), "escape.txt")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("escape.txt exists outside the destination (err=%v)", err) + } + }) + } +} + +func TestExtractAbsoluteNameStaysInside(t *testing.T) { + tarPath := filepath.Join(t.TempDir(), "abs.tar") + writeTar(t, tarPath, tar.Header{Name: "/abs.txt", Mode: 0o644, Size: 2, Typeflag: tar.TypeReg}) + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + // The leading slash is stripped, so the entry lands inside the destination. + if _, err := os.Stat(filepath.Join(dst, "abs.txt")); err != nil { + t.Errorf("abs.txt not extracted inside the destination: %v", err) + } +} + +func TestExtractReplacesExisting(t *testing.T) { + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "a.txt"), []byte("new"), 0o644); err != nil { + t.Fatalf("writing source: %v", err) + } + tarPath := filepath.Join(t.TempDir(), "out.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + + // The destination already holds a stale file and a symlink where a regular + // file is about to land — extraction must replace both, not write through. + dst := t.TempDir() + if err := os.WriteFile(filepath.Join(dst, "a.txt"), []byte("stale"), 0o600); err != nil { + t.Fatalf("seeding destination: %v", err) + } + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + got, err := os.ReadFile(filepath.Join(dst, "a.txt")) + if err != nil { + t.Fatalf("reading result: %v", err) + } + if string(got) != "new" { + t.Errorf("a.txt = %q, want %q", got, "new") + } +} + +func TestExtractIntoPrecreatedVolumeDir(t *testing.T) { + // The durable-dir case: the destination's / subdirectory already + // exists (atelet pre-creates it) when the archive is extracted over it. + src := t.TempDir() + if err := os.MkdirAll(filepath.Join(src, "data"), 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(src, "data", "counter"), []byte("7"), 0o644); err != nil { + t.Fatalf("writing counter: %v", err) + } + tarPath := filepath.Join(t.TempDir(), "durable.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + + dst := t.TempDir() + if err := os.MkdirAll(filepath.Join(dst, "data"), 0o700); err != nil { + t.Fatalf("pre-creating volume dir: %v", err) + } + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + got, err := os.ReadFile(filepath.Join(dst, "data", "counter")) + if err != nil { + t.Fatalf("reading counter: %v", err) + } + if string(got) != "7" { + t.Errorf("counter = %q, want %q", got, "7") + } +} + +func TestExtractRejectsUnsupportedType(t *testing.T) { + tarPath := filepath.Join(t.TempDir(), "dev.tar") + writeTar(t, tarPath, tar.Header{Name: "null", Typeflag: tar.TypeChar, Devmajor: 1, Devminor: 3}) + if err := Extract(tarPath, t.TempDir()); err == nil { + t.Fatal("Extract succeeded on a device entry, want an error") + } +} diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 7c323e787..502abaf60 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -56,6 +56,10 @@ type runningActor struct { // demand-pages from it for the actor's lifetime). ateom owns it; teardownActor // kills it after the CH process. vfsdCmd *exec.Cmd + // durableVfsdCmd is the second virtiofsd, serving the actor's writable + // durable-dir volumes. nil when the actor declares none. Owned and torn down + // exactly like vfsdCmd. + durableVfsdCmd *exec.Cmd // apiSocket is the CH api-socket for this ateom-owned VMM. apiSocket string @@ -120,6 +124,10 @@ type actorContainer struct { name string bundleRootfs string spec *specs.Spec + // durableMountPaths are the in-container paths where this container mounts + // the actor's durable-dir volume (see durable.go). Empty for containers that + // declare none. + durableMountPaths []string } // resolvedRuntime holds the concrete binary/config paths for a request, taken @@ -269,6 +277,25 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload } }() + // Durable-dir volumes (if any) get their own writable virtio-fs share, served + // by a second virtiofsd from the host directory atelet prepared. + var durableVolume string + var durableVfsdCmd *exec.Cmd + if hasDurableVolumes(containers) { + if durableVolume, err = resolveDurableVolumeName(ateompath.DurableDirVolumeMountsDir(actorUID)); err != nil { + return nil, err + } + if durableVfsdCmd, err = s.stageDurableShare(ctx, rr, actorUID); err != nil { + return nil, err + } + defer func() { + if retErr != nil && durableVfsdCmd.Process != nil { + _ = durableVfsdCmd.Process.Kill() + _, _ = durableVfsdCmd.Process.Wait() + } + }() + } + // Launch a bare VMM (CH + api-socket); ateom owns this process for teardown. apiSocket := filepath.Join(kata.VMDir(actorUID), "clh-api.sock") chCmd, client, err := ch.LaunchVMM(ctx, ch.LaunchVMMOptions{ @@ -292,7 +319,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // writable upper is a guest tmpfs). serialLog is also read on a failed agent dial // below, so keep it here. serialLog := filepath.Join(kata.VMDir(actorUID), "serial.log") - vmCfg := buildVMConfig(actorUID, kernel, image, kparams, serialLog, memMiB, vcpus) + vmCfg := buildVMConfig(actorUID, kernel, image, kparams, serialLog, memMiB, vcpus, durableVolume != "") if err := client.CreateVM(ctx, vmCfg); err != nil { return nil, fmt.Errorf("while creating VM: %w", err) } @@ -348,7 +375,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload }() // Post-boot kata-agent setup: sandbox, guest networking, start each container. - if err := s.startActorContainers(ctx, ac, actorUID, vsockPath, ctrs); err != nil { + if err := s.startActorContainers(ctx, ac, actorUID, vsockPath, ctrs, durableVolume); err != nil { return nil, err } @@ -357,7 +384,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload return nil, fmt.Errorf("while waiting for container readyz: %w", err) } - ra := &runningActor{chCmd: chCmd, vfsdCmd: vfsdCmd, apiSocket: apiSocket, baseID: actorUID, logAgent: ac} + ra := &runningActor{chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, apiSocket: apiSocket, baseID: actorUID, logAgent: ac} s.running[actorUID] = ra // Forward each container's stdout/stderr into the pod logs. The overlay workload's @@ -406,7 +433,12 @@ func (s *AteomService) buildActorContainers(actorUID string, containers []*ateom if err := writeGuestResolvConf(bundleRootfs); err != nil { return nil, fmt.Errorf("while writing guest resolv.conf for %q: %w", cn, err) } - ctrs[i] = actorContainer{name: cn, bundleRootfs: bundleRootfs, spec: spec} + ctrs[i] = actorContainer{ + name: cn, + bundleRootfs: bundleRootfs, + spec: spec, + durableMountPaths: c.GetDurableDirVolumes(), + } } return ctrs, nil } @@ -461,7 +493,10 @@ func (s *AteomService) guestConfig(rr resolvedRuntime) (memMiB, vcpus int, kpara // systemd-networkd (the agent owns eth0). The console is arch-specific: ttyAMA0 on // arm64, ttyS0 on amd64. /dev/vda is the RO guest image; the actor rootfs's RO lower is // the virtio-fs device on PCI segment 1 (hence num_pci_segments=2), with no actor disks. -func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus int) ch.VmConfig { +// +// withDurable adds a second virtio-fs device for the actor's writable durable-dir +// volumes (see durable.go), served by its own virtiofsd on the same PCI segment. +func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus int, withDurable bool) ch.VmConfig { console := "ttyS0" if runtime.GOARCH == "arm64" { console = "ttyAMA0" @@ -479,10 +514,7 @@ func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus i Disks: []ch.DiskConfig{ {Path: image, Readonly: true, ImageType: "Raw", NumQueues: int32(vcpus), QueueSize: 1024}, }, - Fs: []ch.FsConfig{{ - Tag: kata.FsTag, Socket: kata.VirtiofsdSocketPath(id), - NumQueues: 1, QueueSize: 1024, PciSegment: 1, - }}, + Fs: buildFsConfigs(id, withDurable), Platform: &ch.PlatformConfig{NumPciSegments: 2}, Rng: &ch.RngConfig{Src: "/dev/urandom"}, Serial: &ch.ConsoleConfig{Mode: "File", File: serialLog}, @@ -490,16 +522,37 @@ func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus i } } +// buildFsConfigs returns the VM's virtio-fs devices: the overlay RO lower's +// share, plus the writable durable-dir share when the actor has one. Both sit on +// PCI segment 1 (the segment buildVMConfig reserves for virtio-fs). +func buildFsConfigs(id string, withDurable bool) []ch.FsConfig { + fs := []ch.FsConfig{{ + Tag: kata.FsTag, Socket: kata.VirtiofsdSocketPath(id), + NumQueues: 1, QueueSize: 1024, PciSegment: 1, + }} + if withDurable { + fs = append(fs, ch.FsConfig{ + Tag: kata.DurableFsTag, Socket: kata.DurableVirtiofsdSocketPath(id), + NumQueues: 1, QueueSize: 1024, PciSegment: 1, + }) + } + return fs +} + // startActorContainers performs the post-boot kata-agent setup the shim normally // does at boot: establish the sandbox once (mounting the kataShared virtio-fs base), // configure guest networking (eth0 IP/MAC/MTU + routes) once, then start each // container on its own overlay rootfs. On failure it dumps guest diagnostics. -func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentClient, id, vsockPath string, ctrs []actorContainer) error { +// +// durableVolume is the actor's durable-dir volume name, or "" when it has none; +// with one, the sandbox also mounts the writable durable share and each container +// binds the volume at its declared paths. +func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentClient, id, vsockPath string, ctrs []actorContainer, durableVolume string) error { // Establish the agent sandbox + the kataShared virtio-fs mount (the RO base for // every container's overlay lower). All containers share it, so use the first // container's hostname. sbCtx, sbCancel := context.WithTimeout(ctx, 20*time.Second) - err := ac.CreateSandboxForActor(sbCtx, id, ctrs[0].spec.Hostname) + err := ac.CreateSandboxForActor(sbCtx, id, ctrs[0].spec.Hostname, durableVolume != "") sbCancel() if err != nil { return fmt.Errorf("while creating agent sandbox: %w", err) @@ -517,7 +570,7 @@ func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentC } for _, c := range ctrs { - if err := startOverlayContainer(ctx, ac, vsockPath, c); err != nil { + if err := startOverlayContainer(ctx, ac, vsockPath, c, durableVolume); err != nil { return err } } @@ -528,7 +581,11 @@ func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentC // lower + guest-tmpfs upper): a carrier container (id == name) eager-binds the RO base // to /run/kata-containers//rootfs, then the workload (id == _ovl) overlays // it with a tmpfs upper. On failure it dumps the guest overlay state. -func startOverlayContainer(ctx context.Context, ac *kata.AgentClient, vsockPath string, c actorContainer) error { +// +// With a durable-dir volume, the WORKLOAD also binds it at the container's declared +// mount paths. The carrier deliberately does not: its rootfs is the read-only lower, +// where the agent could not create a missing mount point. +func startOverlayContainer(ctx context.Context, ac *kata.AgentClient, vsockPath string, c actorContainer, durableVolume string) error { carrierCtx, carrierCancel := context.WithTimeout(ctx, 30*time.Second) err := ac.CreateCarrier(carrierCtx, c.name, c.spec) carrierCancel() @@ -540,7 +597,7 @@ func startOverlayContainer(ctx context.Context, ac *kata.AgentClient, vsockPath upperBase := kata.OverlayUpperBase(c.name) wlCtx, wlCancel := context.WithTimeout(ctx, 30*time.Second) - err = ac.StartOverlayWorkload(wlCtx, c.name, overlayWorkloadID(c.name), upperBase, c.spec) + err = ac.StartOverlayWorkload(wlCtx, c.name, overlayWorkloadID(c.name), upperBase, workloadSpec(c, durableVolume)) wlCancel() if err != nil { dump := kata.DebugConsoleDump(ctx, vsockPath, diff --git a/manifests/ate-install/generated/ate.dev_actortemplates.yaml b/manifests/ate-install/generated/ate.dev_actortemplates.yaml index 0bebc1473..4fa151c4a 100644 --- a/manifests/ate-install/generated/ate.dev_actortemplates.yaml +++ b/manifests/ate-install/generated/ate.dev_actortemplates.yaml @@ -433,9 +433,6 @@ spec: rule: '!has(self.volumes) || self.volumes.all(v, has(self.containers) && self.containers.exists(c, has(c.volumeMounts) && c.volumeMounts.exists(vm, vm.name == v.name)))' - - message: DurableDir volumes are not supported when sandboxClass is 'microvm' - rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || - !has(self.volumes) || !self.volumes.exists(v, has(v.durableDir))' - message: ExternalVolumes are not supported when sandboxClass is 'microvm' rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))' diff --git a/pkg/api/v1alpha1/actortemplate_types.go b/pkg/api/v1alpha1/actortemplate_types.go index c6933c104..4835afbae 100644 --- a/pkg/api/v1alpha1/actortemplate_types.go +++ b/pkg/api/v1alpha1/actortemplate_types.go @@ -297,7 +297,6 @@ type SnapshotsConfig struct { // +kubebuilder:validation:XValidation:rule="!has(self.volumes) || self.volumes.filter(v, has(v.durableDir)).size() <= 1",message="At most one DurableDir-typed volume is supported per ActorTemplate" // +kubebuilder:validation:XValidation:rule="!has(self.containers) || self.containers.all(c, !has(c.volumeMounts) || c.volumeMounts.filter(vm, has(self.volumes) && self.volumes.exists(v, v.name == vm.name && has(v.durableDir))).size() <= 1)",message="A container may mount at most one DurableDir-typed volume" // +kubebuilder:validation:XValidation:rule="!has(self.volumes) || self.volumes.all(v, has(self.containers) && self.containers.exists(c, has(c.volumeMounts) && c.volumeMounts.exists(vm, vm.name == v.name)))",message="All volumes defined in spec.volumes must be mounted by at least one container" -// +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass != 'microvm' || !has(self.volumes) || !self.volumes.exists(v, has(v.durableDir))",message="DurableDir volumes are not supported when sandboxClass is 'microvm'" // +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass != 'microvm' || !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))",message="ExternalVolumes are not supported when sandboxClass is 'microvm'" type ActorTemplateSpec struct { // PauseImage is the container to use as the root sandbox container. diff --git a/pkg/api/v1alpha1/actortemplate_validation_test.go b/pkg/api/v1alpha1/actortemplate_validation_test.go index 89ccbc923..a8ecce1e4 100644 --- a/pkg/api/v1alpha1/actortemplate_validation_test.go +++ b/pkg/api/v1alpha1/actortemplate_validation_test.go @@ -998,7 +998,7 @@ func TestActorTemplateValidation(t *testing.T) { wantErr: true, errMsg: "Name must be a valid DNS label", }, { - name: "Volumes: DurableDir volume with SandboxClass microvm is invalid", + name: "Volumes: DurableDir volume with SandboxClass microvm is valid", mutate: func(at *ActorTemplate) { at.Spec.SandboxClass = SandboxClassMicroVM at.Spec.Volumes = []Volume{ @@ -1008,8 +1008,7 @@ func TestActorTemplateValidation(t *testing.T) { {Name: "vol1", MountPath: "/home/user"}, } }, - wantErr: true, - errMsg: "DurableDir volumes are not supported when sandboxClass is 'microvm'", + wantErr: false, }, { name: "Volumes: DurableDir volume with SandboxClass gvisor is valid", mutate: func(at *ActorTemplate) { From da5c85ca1ae37031e0582230179dc9a044a056ed Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 24 Jul 2026 16:03:26 -0700 Subject: [PATCH 2/3] ateom-microvm: capture durable-dir volumes in snapshots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The micro-VM runtime ignored the snapshot scope: every checkpoint took a whole-guest CH snapshot, and durable-dir volumes — now host-backed rather than in guest RAM — were left out of it entirely, so their contents did not survive a suspend. Checkpoint, with the guest paused either way: - Full keeps the CH snapshot (moved to snapshotVMState) and adds a tar of the volumes. - Data writes only that tar: no memory image, and no base-id, since nothing will reattach to the frozen virtio-fs lower. - Any other scope is rejected rather than silently treated as Full. Restore mirrors it. The volumes are extracted before anything can observe them — for Full before the share's virtiofsd starts, for Data before the workload cold-starts. Full then resumes the guest as before; Data cold-boots the actor, which is a Run with the volumes already populated, so RunWorkload's body becomes coldBootActor and both paths share it. Cold boot gates on readyz, so a Data-scope resume returns with the actor serving. Pausing first is what makes the tar coherent: the durable share is served write-through, so every completed guest write is already on the host and no further ones can arrive. The tar's duration is logged next to the snapshot's — it is on the suspend path and scales with the volume's contents. With two virtio-fs devices, the snapshot's fs sockets are now repointed by device tag instead of all being set to the overlay lower's socket, which would have handed the guest the wrong filesystem. --- cmd/ateom-microvm/checkpoint.go | 133 ++++++++++++++++------- cmd/ateom-microvm/restore.go | 169 ++++++++++++++++++++++-------- cmd/ateom-microvm/restore_test.go | 146 ++++++++++++++++++++++++++ cmd/ateom-microvm/run.go | 91 ++++++++++------ 4 files changed, 428 insertions(+), 111 deletions(-) create mode 100644 cmd/ateom-microvm/restore_test.go diff --git a/cmd/ateom-microvm/checkpoint.go b/cmd/ateom-microvm/checkpoint.go index f65ea4aad..b5ff7a714 100644 --- a/cmd/ateom-microvm/checkpoint.go +++ b/cmd/ateom-microvm/checkpoint.go @@ -30,19 +30,31 @@ import ( "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/proto/ateompb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -// CheckpointWorkload suspends the actor and writes a portable CH snapshot. +// CheckpointWorkload suspends the actor and writes a portable snapshot. // // Contract with atelet: after we return, atelet uploads the checkpoint dir to object // storage, then tears down bundles and resets the actor dir. // -// ateom drives the CH REST api-socket: pause -> snapshot file:// -// (config.json + state.json + sparse memory-ranges) -> tear the VMM down. Each -// container's rootfs is overlay(virtio-fs RO lower + guest-tmpfs upper), so the -// writable upper lives in guest RAM and is captured by the memory snapshot — process -// memory and rootfs writes both persist across suspend/resume. The RO lower is -// reconstructed from the OCI image at restore, so nothing rootfs-related ships here. +// What the snapshot holds depends on the requested scope: +// +// - FULL: the whole guest. ateom drives the CH REST api-socket: pause -> snapshot +// file:// (config.json + state.json + sparse memory-ranges) +// -> tear the VMM down. Each container's rootfs is overlay(virtio-fs RO lower + +// guest-tmpfs upper), so the writable upper lives in guest RAM and is captured by +// the memory snapshot — process memory and rootfs writes both persist across +// suspend/resume. The RO lower is reconstructed from the OCI image at restore, so +// nothing rootfs-related ships. Durable-dir volumes are host-backed rather than in +// guest RAM, so they ship alongside as a tar. +// - DATA: the durable-dir volumes only, as that same tar. The guest is discarded, so +// the actor cold-starts on restore with its volumes re-materialized. +// +// Either way the guest is paused first, which is what makes the tar coherent: the +// durable share is served write-through, so every completed guest write is already on +// the host and no further ones can arrive. func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.CheckpointWorkloadRequest) (*ateompb.CheckpointWorkloadResponse, error) { s.lock.Lock() defer s.lock.Unlock() @@ -55,6 +67,25 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec s.actorLogger.EmitLifecycleLog("Actor checkpointing", atespace, name, actorUID, templateNS, templateName) + // Check what the request asks for BEFORE touching the guest: these are + // properties of the request, and pausing first would leave the actor + // suspended mid-flight for a call that could never have succeeded. + // + // Durable-dir volumes are host-backed, so they are captured the same way + // under either scope — and are the ONLY thing a Data-scope snapshot captures. + durable := hasDurableVolumes(req.GetSpec().GetContainers()) + scope := req.GetScope() + switch scope { + case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: + case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: + if !durable { + return nil, status.Error(codes.FailedPrecondition, + "no durable-dir volumes found for a Data-scope snapshot") + } + default: + return nil, status.Errorf(codes.InvalidArgument, "unsupported snapshot scope: %v", scope) + } + // The actor's CH was booted by RunWorkload or relaunched by RestoreWorkload; // either way ateom owns it and tracks its api-socket. ra := s.running[actorUID] @@ -82,6 +113,61 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec return nil, fmt.Errorf("while creating checkpoint dir %q: %w", checkpointDir, err) } + // Only a Full snapshot captures the guest. A Data snapshot deliberately + // captures no VM state — no memory image, and no base-id, since nothing will + // reattach to the frozen virtio-fs lower: the actor cold-boots from the OCI + // image and gets its durable-dir volumes back from the tar below. + var dSnapshot time.Duration + if scope == ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL { + var err error + if dSnapshot, err = s.snapshotVMState(ctx, client, ra, actorUID, checkpointDir); err != nil { + return nil, err + } + } + + var dDurable time.Duration + if durable { + tDurable := time.Now() + if err := tarDurableVolumes(ctx, ateompath.DurableDirVolumeMountsDir(actorUID), checkpointDir); err != nil { + return nil, err + } + dDurable = time.Since(tDurable) + } + + // Report exactly the files we wrote so atelet ships precisely this snapshot: for + // Full, the CH snapshot (config.json + state.json + memory-ranges + base-id) plus + // any durable-dir tar; for Data, that tar alone. + snapshotFiles, err := listFiles(checkpointDir) + if err != nil { + return nil, fmt.Errorf("while listing snapshot files: %w", err) + } + + // Tear down: the actor returns to "available". Best-effort; the snapshot is + // already on disk for atelet to ship. + tTeardown := time.Now() + s.teardownActor(ctx, actorUID, ra, client) + dTeardown := time.Since(tTeardown) + delete(s.running, actorUID) + + // Tear down the per-activation actor network. + if err := s.cleanupActorNetwork(ctx); err != nil { + slog.WarnContext(ctx, "Failed to clean up actor network after checkpoint", slog.Any("err", err)) + } + + s.actorLogger.EmitLifecycleLog("Actor checkpointed", atespace, name, actorUID, templateNS, templateName) + slog.InfoContext(ctx, "Actor checkpointed", slog.String("id", actorUID), slog.Any("snapshot_files", snapshotFiles), + slog.String("scope", scope.String()), slog.Duration("pause", dPause), + slog.Duration("snapshot", dSnapshot), + // The durable-dir tar runs while the guest is paused, so its cost is part + // of the suspend latency and scales with the volume's contents. + slog.Duration("durable_dir", dDurable), slog.Duration("teardown", dTeardown)) + return &ateompb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles}, nil +} + +// snapshotVMState captures the paused guest into checkpointDir: the CH snapshot +// (config.json + state.json + memory-ranges) plus the base-id the restore side +// needs, and returns how long the snapshot itself took. +func (s *AteomService) snapshotVMState(ctx context.Context, client *ch.Client, ra *runningActor, actorUID, checkpointDir string) (time.Duration, error) { // Record the FROZEN base id (the id the guest's virtio-fs find-paths are pinned // to, /rootfs). For a cold-run actor this is its own id; for a restored // actor it is the golden id propagated via ra.baseID (set from the snapshot we @@ -94,13 +180,13 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec baseID = ra.baseID } if err := os.WriteFile(filepath.Join(checkpointDir, baseIDFile), []byte(baseID), 0o600); err != nil { - return nil, fmt.Errorf("while writing %s: %w", baseIDFile, err) + return 0, fmt.Errorf("while writing %s: %w", baseIDFile, err) } slog.InfoContext(ctx, "Snapshotting guest", slog.String("id", actorUID), slog.String("dir", checkpointDir)) tSnapshot := time.Now() if err := client.Snapshot(ctx, checkpointDir); err != nil { - return nil, fmt.Errorf("while snapshotting guest: %w", err) + return 0, fmt.Errorf("while snapshotting guest: %w", err) } dSnapshot := time.Since(tSnapshot) @@ -118,7 +204,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // CH is paused and about to be torn down, and base is discarded after. See // MergeDeltaIntoBase. (Falls back to the copying merge across filesystems.) if err := ch.MergeDeltaIntoBase(ctx, base, delta); err != nil { - return nil, fmt.Errorf("while merging OnDemand delta into restore source: %w", err) + return 0, fmt.Errorf("while merging OnDemand delta into restore source: %w", err) } slog.InfoContext(ctx, "Merged OnDemand delta into base (complete snapshot)", slog.String("id", actorUID), slog.Duration("merge", time.Since(tMerge))) @@ -127,32 +213,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // Nothing rootfs-related ships: the overlay's writable upper is a guest tmpfs, so // the actor's rootfs writes are already in the memory snapshot above, and the RO // lower is reconstructed from the OCI image at restore (it never changes). - - // Report exactly the files we wrote so atelet ships precisely the CH snapshot - // (config.json + state.json + memory-ranges + base-id). The RO base is - // reconstructed from the OCI image at restore. - snapshotFiles, err := listFiles(checkpointDir) - if err != nil { - return nil, fmt.Errorf("while listing snapshot files: %w", err) - } - - // Tear down: the actor returns to "available". Best-effort; the snapshot is - // already on disk for atelet to ship. - tTeardown := time.Now() - s.teardownActor(ctx, actorUID, ra, client) - dTeardown := time.Since(tTeardown) - delete(s.running, actorUID) - - // Tear down the per-activation actor network. - if err := s.cleanupActorNetwork(ctx); err != nil { - slog.WarnContext(ctx, "Failed to clean up actor network after checkpoint", slog.Any("err", err)) - } - - s.actorLogger.EmitLifecycleLog("Actor checkpointed", atespace, name, actorUID, templateNS, templateName) - slog.InfoContext(ctx, "Actor checkpointed", slog.String("id", actorUID), slog.Any("snapshot_files", snapshotFiles), - slog.Duration("pause", dPause), - slog.Duration("snapshot", dSnapshot), slog.Duration("teardown", dTeardown)) - return &ateompb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles}, nil + return dSnapshot, nil } // listFiles returns the (relative) names of regular files directly under dir. diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index 3a98860e1..c939af669 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -22,6 +22,7 @@ import ( "fmt" "log/slog" "os" + "os/exec" "path/filepath" "strings" "time" @@ -36,41 +37,90 @@ import ( "google.golang.org/grpc/status" ) -// RestoreWorkload restores the actor on a (possibly different) pod by relaunching -// cloud-hypervisor directly from the downloaded snapshot and resuming. +// RestoreWorkload brings the actor back from a snapshot, on a possibly different +// pod. What that means depends on the scope the snapshot was taken with: // -// Contract with atelet: the snapshot dir (config.json + state.json + memory-ranges + -// base-id) has been downloaded to RestoreStateDir. +// - FULL: relaunch cloud-hypervisor from the snapshot and resume the guest +// (restoreFullScope). +// - DATA: there is no guest to resume — re-materialize the durable-dir volumes and +// cold-boot the actor, which starts its containers afresh from the OCI image. +// +// Contract with atelet: the snapshot's files have been downloaded to RestoreStateDir, +// and the durable-dir volume directories re-created (empty). +func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.RestoreWorkloadRequest) (*ateompb.RestoreWorkloadResponse, error) { + s.lock.Lock() + defer s.lock.Unlock() + + p := actorBootParams{ + atespace: req.GetAtespace(), + actorName: req.GetActorName(), + actorUID: req.GetActorUid(), + templateNS: req.GetActorTemplateNamespace(), + templateName: req.GetActorTemplateName(), + containers: req.GetSpec().GetContainers(), + assetPaths: req.GetRuntimeAssetPaths(), + } + restoreDir := ateompath.RestoreStateDir(p.actorUID) + durableDir := ateompath.DurableDirVolumeMountsDir(p.actorUID) + tStart := time.Now() + + s.actorLogger.EmitLifecycleLog("Actor restoring", p.atespace, p.actorName, p.actorUID, p.templateNS, p.templateName) + + // Restore the durable-dir volumes before anything can observe them: for Full + // that means before the share's virtiofsd starts, for Data before the workload + // cold-starts. The snapshot must carry them — the actor declares the volume, and + // every scope captures it. + if hasDurableVolumes(p.containers) { + if err := untarDurableVolumes(durableDir, restoreDir); err != nil { + return nil, err + } + } + + switch scope := req.GetScope(); scope { + case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: + if err := s.restoreFullScope(ctx, p, restoreDir, tStart); err != nil { + return nil, err + } + case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: + // A Data snapshot holds no guest state, so this is a cold boot that + // happens to start with the volumes already populated. readyz gating comes + // with the cold-boot path, so the actor is serving when we return. + if err := s.coldBootActor(ctx, p); err != nil { + return nil, err + } + slog.InfoContext(ctx, "Actor restored (durable-dir volumes, cold boot)", + slog.String("id", p.actorUID), slog.Duration("total", time.Since(tStart))) + default: + return nil, status.Errorf(codes.InvalidArgument, "unsupported snapshot scope: %v", scope) + } + + s.actorLogger.EmitLifecycleLog("Actor restored", p.atespace, p.actorName, p.actorUID, p.templateNS, p.templateName) + return &ateompb.RestoreWorkloadResponse{}, nil +} + +// restoreFullScope restores a whole-guest snapshot: relaunch cloud-hypervisor +// directly from it and resume. // // Each container's rootfs is overlay(virtio-fs RO lower + guest-tmpfs upper). Steps: // reconstruct each RO lower from the local OCI bundle (atelet re-unpacked the golden // image) at the frozen find-paths path and start the virtiofsd serving them; rewrite -// the snapshot config's per-VMDir paths (vsock + serial + fs socket) to this actor's; +// the snapshot config's per-VMDir paths (vsock + serial + fs sockets) to this actor's; // rebuild the tap (the snapshot's virtio-net is fd-backed → fresh net_fds); relaunch // CH with --restore (OnDemand), and resume. Guest RAM — incl. the actor's in-memory // state, the tmpfs rootfs upper (so rootfs writes PERSIST), and the frozen network -// config — comes back from the memory snapshot. -func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.RestoreWorkloadRequest) (resp *ateompb.RestoreWorkloadResponse, retErr error) { - s.lock.Lock() - defer s.lock.Unlock() - - atespace := req.GetAtespace() - name := req.GetActorName() - actorUID := req.GetActorUid() - templateNS := req.GetActorTemplateNamespace() - templateName := req.GetActorTemplateName() - restoreDir := ateompath.RestoreStateDir(actorUID) - tStart := time.Now() - - s.actorLogger.EmitLifecycleLog("Actor restoring", atespace, name, actorUID, templateNS, templateName) +// config — comes back from the memory snapshot. Durable-dir volumes are host-backed +// instead, and the caller has already restored them from the snapshot's tar. +func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, restoreDir string, tStart time.Time) (retErr error) { + atespace, name, actorUID := p.atespace, p.actorName, p.actorUID + templateNS, templateName := p.templateNS, p.templateName - rr := s.resolveRuntime(req.GetRuntimeAssetPaths()) + rr := s.resolveRuntime(p.assetPaths) kata.CleanupSandboxState(ctx, actorUID) // Repoint the snapshot's vsock socket to this actor's VMDir (the disk + kernel // paths are content-addressed/per-actor and already line up on the same node). if err := rewriteSnapshotSocketPaths(restoreDir, actorUID); err != nil { - return nil, fmt.Errorf("while rewriting snapshot socket paths: %w", err) + return fmt.Errorf("while rewriting snapshot socket paths: %w", err) } srcID := actorUID if b, rerr := os.ReadFile(filepath.Join(restoreDir, baseIDFile)); rerr == nil { @@ -79,7 +129,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } } if err := os.MkdirAll(kata.VMDir(actorUID), 0o700); err != nil { - return nil, fmt.Errorf("while creating VM dir: %w", err) + return fmt.Errorf("while creating VM dir: %w", err) } // Reconstruct each container's overlay RO lower from the LOCAL OCI bundle (atelet @@ -90,20 +140,20 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore // fs socket in the snapshot config is repointed to this VMDir by // rewriteSnapshotSocketPaths above. cross-node consistency relies on a deterministic // unpack of the same image at the same /rootfs path. - containers := req.GetSpec().GetContainers() + containers := p.containers if len(containers) == 0 { - return nil, status.Error(codes.InvalidArgument, "actor spec has no containers") + return status.Error(codes.InvalidArgument, "actor spec has no containers") } if len(containers) > maxActorContainers { - return nil, status.Errorf(codes.Unimplemented, "ateom-microvm supports at most %d containers, got %d", maxActorContainers, len(containers)) + return status.Errorf(codes.Unimplemented, "ateom-microvm supports at most %d containers, got %d", maxActorContainers, len(containers)) } ctrs, err := s.buildActorContainers(actorUID, containers) if err != nil { - return nil, err + return err } vfsdCmd, err := s.stageOverlayLowers(ctx, rr, actorUID, ctrs) if err != nil { - return nil, err + return err } defer func() { if retErr != nil && vfsdCmd.Process != nil { @@ -112,10 +162,27 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } }() + // Restart the durable-dir share's virtiofsd over the contents the caller + // restored. The guest reattaches to it by the socket path rewritten into the + // snapshot config below; find-paths re-opens whatever files it still holds open + // against the same paths, which the restored tar reproduces exactly. + var durableVfsdCmd *exec.Cmd + if hasDurableVolumes(containers) { + if durableVfsdCmd, err = s.stageDurableShare(ctx, rr, actorUID); err != nil { + return err + } + defer func() { + if retErr != nil && durableVfsdCmd.Process != nil { + _ = durableVfsdCmd.Process.Kill() + _, _ = durableVfsdCmd.Process.Wait() + } + }() + } + // Networking: rebuild the per-activation veth + tap; the snapshot's virtio-net // is fd-backed, so CH needs fresh tap FDs (net_fds) on restore. if err := s.setupActorNetwork(ctx); err != nil { - return nil, fmt.Errorf("while setting up actor network: %w", err) + return fmt.Errorf("while setting up actor network: %w", err) } defer func() { if retErr != nil { @@ -131,7 +198,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore }() netDevs, err := ch.SnapshotNetDevices(restoreDir) if err != nil { - return nil, fmt.Errorf("while reading snapshot net devices: %w", err) + return fmt.Errorf("while reading snapshot net devices: %w", err) } var restoredNets []ch.RestoredNet var tapFiles []*os.File @@ -143,7 +210,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore for i, nd := range netDevs { files, terr := s.setupRestoreTap(ctx, fmt.Sprintf("tap%d_kata", i), nd.QueuePairs) if terr != nil { - return nil, fmt.Errorf("while building restore tap for %s: %w", nd.ID, terr) + return fmt.Errorf("while building restore tap for %s: %w", nd.ID, terr) } tapFiles = append(tapFiles, files...) rn := ch.RestoredNet{ID: nd.ID} @@ -160,7 +227,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore Binary: rr.chBinary, APISocket: apiSocket, Stdout: slogWriter{ctx}, Stderr: slogWriter{ctx}, }) if err != nil { - return nil, fmt.Errorf("while launching VMM for restore: %w", err) + return fmt.Errorf("while launching VMM for restore: %w", err) } defer func() { if retErr != nil && chCmd.Process != nil { @@ -175,18 +242,21 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore // rebuild a complete snapshot. CH demand-pages from restoreDir for the VM's whole // lifetime, so it must persist until teardown (atelet keeps it until reset). if err := client.RestoreWithNetFDs(ctx, restoreDir, restoredNets, "OnDemand"); err != nil { - return nil, fmt.Errorf("while restoring VM with net FDs: %w", err) + return fmt.Errorf("while restoring VM with net FDs: %w", err) } if err := client.Resume(ctx); err != nil { - return nil, fmt.Errorf("while resuming restored guest: %w", err) + return fmt.Errorf("while resuming restored guest: %w", err) } // Block until every readyz-enabled container reports 200. if err := readyz.WaitAll(ctx, containers, actorVethIP); err != nil { - return nil, fmt.Errorf("while waiting for container readyz: %w", err) + return fmt.Errorf("while waiting for container readyz: %w", err) } - ra := &runningActor{chCmd: chCmd, vfsdCmd: vfsdCmd, apiSocket: apiSocket, baseID: srcID, restoreSourceDir: restoreDir} + ra := &runningActor{ + chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, + apiSocket: apiSocket, baseID: srcID, restoreSourceDir: restoreDir, + } // Re-attach stdout/stderr forwarding for each container: the restored guest's // containers + kata-agent are alive, so a fresh dial over this actor's vsock @@ -206,18 +276,17 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } s.running[actorUID] = ra - s.actorLogger.EmitLifecycleLog("Actor restored", atespace, name, actorUID, templateNS, templateName) slog.InfoContext(ctx, "Actor restored (overlay rootfs)", slog.String("id", actorUID), slog.Duration("total", time.Since(tStart))) - return &ateompb.RestoreWorkloadResponse{}, nil + return nil } // rewriteSnapshotSocketPaths repoints the snapshot config.json's per-VMDir paths from // the source actor's VMDir to the restoring actor's: the hybrid-vsock socket, the -// File serial console, and each virtio-fs (overlay RO lower) socket, so the sockets/ -// files we create are the ones CH reopens. The kernel and /dev/vda kata image are -// content-addressed static files with identical paths on every node, so they need no -// rewrite, and the overlay has no per-actor disk to repoint. +// File serial console, and each virtio-fs socket, so the sockets/files we create are +// the ones CH reopens. The kernel and /dev/vda kata image are content-addressed static +// files with identical paths on every node, so they need no rewrite, and the overlay +// has no per-actor disk to repoint. func rewriteSnapshotSocketPaths(snapshotDir, id string) error { cfgPath := filepath.Join(snapshotDir, "config.json") b, err := os.ReadFile(cfgPath) @@ -240,12 +309,24 @@ func rewriteSnapshotSocketPaths(snapshotDir, id string) error { serial["file"] = filepath.Join(kata.VMDir(id), "serial.log") } } - // The overlay RO lower is served by a per-VMDir virtiofsd socket; the snapshot - // recorded the golden actor's, so repoint each fs device at this actor's VMDir. + // Each virtio-fs share is served by its own per-VMDir virtiofsd socket; the + // snapshot recorded the golden actor's, so repoint them at this actor's VMDir. + // Match on the device tag: the shares have separate sockets (the overlay RO + // lower's and, when the actor has durable-dir volumes, the writable share's), and + // crossing them would hand the guest the wrong filesystem. if fss, ok := cfg["fs"].([]any); ok { for _, f := range fss { - if fm, ok := f.(map[string]any); ok { + fm, ok := f.(map[string]any) + if !ok { + return fmt.Errorf("snapshot config %q has a malformed fs device", cfgPath) + } + switch tag, _ := fm["tag"].(string); tag { + case kata.FsTag: fm["socket"] = kata.VirtiofsdSocketPath(id) + case kata.DurableFsTag: + fm["socket"] = kata.DurableVirtiofsdSocketPath(id) + default: + return fmt.Errorf("snapshot config %q has fs device with unknown tag %q", cfgPath, tag) } } } diff --git a/cmd/ateom-microvm/restore_test.go b/cmd/ateom-microvm/restore_test.go new file mode 100644 index 000000000..4ea92e102 --- /dev/null +++ b/cmd/ateom-microvm/restore_test.go @@ -0,0 +1,146 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" +) + +// writeSnapshotConfig writes a config.json holding the given fs devices (plus the +// vsock and serial entries every snapshot has) into a fresh snapshot dir. +func writeSnapshotConfig(t *testing.T, fsDevices []map[string]any) string { + t.Helper() + dir := t.TempDir() + cfg := map[string]any{ + "vsock": map[string]any{"cid": 3, "socket": "/run/vc/vm/golden/clh.sock"}, + "serial": map[string]any{"mode": "File", "file": "/run/vc/vm/golden/serial.log"}, + "fs": fsDevices, + } + b, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("marshaling config: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "config.json"), b, 0o600); err != nil { + t.Fatalf("writing config.json: %v", err) + } + return dir +} + +// readFsSockets returns the rewritten config's fs sockets keyed by device tag. +func readFsSockets(t *testing.T, dir string) map[string]string { + t.Helper() + b, err := os.ReadFile(filepath.Join(dir, "config.json")) + if err != nil { + t.Fatalf("reading config.json: %v", err) + } + var cfg struct { + Vsock map[string]any `json:"vsock"` + Serial map[string]any `json:"serial"` + Fs []struct { + Tag string `json:"tag"` + Socket string `json:"socket"` + } `json:"fs"` + } + if err := json.Unmarshal(b, &cfg); err != nil { + t.Fatalf("parsing rewritten config: %v", err) + } + got := map[string]string{} + for _, f := range cfg.Fs { + got[f.Tag] = f.Socket + } + return got +} + +func TestRewriteSnapshotSocketPaths(t *testing.T) { + const id = "actor-uid" + + t.Run("overlay lower only", func(t *testing.T) { + dir := writeSnapshotConfig(t, []map[string]any{ + {"tag": kata.FsTag, "socket": "/run/vc/vm/golden/virtiofsd.sock"}, + }) + if err := rewriteSnapshotSocketPaths(dir, id); err != nil { + t.Fatalf("rewriteSnapshotSocketPaths: %v", err) + } + if got, want := readFsSockets(t, dir)[kata.FsTag], kata.VirtiofsdSocketPath(id); got != want { + t.Errorf("%s socket = %q, want %q", kata.FsTag, got, want) + } + }) + + t.Run("each share keeps its own socket", func(t *testing.T) { + // Ordered durable-first to catch a rewrite that assumes the RO lower comes + // first, which would hand the guest the wrong filesystem. + dir := writeSnapshotConfig(t, []map[string]any{ + {"tag": kata.DurableFsTag, "socket": "/run/vc/vm/golden/virtiofsd-durable.sock"}, + {"tag": kata.FsTag, "socket": "/run/vc/vm/golden/virtiofsd.sock"}, + }) + if err := rewriteSnapshotSocketPaths(dir, id); err != nil { + t.Fatalf("rewriteSnapshotSocketPaths: %v", err) + } + got := readFsSockets(t, dir) + for tag, want := range map[string]string{ + kata.FsTag: kata.VirtiofsdSocketPath(id), + kata.DurableFsTag: kata.DurableVirtiofsdSocketPath(id), + } { + if got[tag] != want { + t.Errorf("%s socket = %q, want %q", tag, got[tag], want) + } + } + if got[kata.FsTag] == got[kata.DurableFsTag] { + t.Error("both shares were pointed at the same socket") + } + }) + + t.Run("unknown tag is an error", func(t *testing.T) { + dir := writeSnapshotConfig(t, []map[string]any{ + {"tag": "somethingElse", "socket": "/run/vc/vm/golden/other.sock"}, + }) + if err := rewriteSnapshotSocketPaths(dir, id); err == nil { + t.Fatal("rewriteSnapshotSocketPaths accepted an unknown fs tag, want an error") + } + }) + + t.Run("vsock and serial are repointed", func(t *testing.T) { + dir := writeSnapshotConfig(t, []map[string]any{ + {"tag": kata.FsTag, "socket": "/run/vc/vm/golden/virtiofsd.sock"}, + }) + if err := rewriteSnapshotSocketPaths(dir, id); err != nil { + t.Fatalf("rewriteSnapshotSocketPaths: %v", err) + } + b, err := os.ReadFile(filepath.Join(dir, "config.json")) + if err != nil { + t.Fatalf("reading config.json: %v", err) + } + var cfg struct { + Vsock struct{ Socket string } `json:"vsock"` + Serial struct{ File string } `json:"serial"` + } + if err := json.Unmarshal(b, &cfg); err != nil { + t.Fatalf("parsing rewritten config: %v", err) + } + if want := kata.VsockSocketPath(id); cfg.Vsock.Socket != want { + t.Errorf("vsock socket = %q, want %q", cfg.Vsock.Socket, want) + } + if want := filepath.Join(kata.VMDir(id), "serial.log"); cfg.Serial.File != want { + t.Errorf("serial file = %q, want %q", cfg.Serial.File, want) + } + }) +} diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 502abaf60..5047f271f 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -193,43 +193,74 @@ func writeGuestResolvConf(rootfs string) error { // - The runtime assets (guest kernel, guest OS image, cloud-hypervisor, virtiofsd, // base kata config) are on disk and passed as runtime asset paths. // - The OCI bundle (config.json + populated rootfs/) is prepared per container. -func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkloadRequest) (resp *ateompb.RunWorkloadResponse, retErr error) { +func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkloadRequest) (*ateompb.RunWorkloadResponse, error) { s.lock.Lock() defer s.lock.Unlock() - atespace := req.GetAtespace() - name := req.GetActorName() - actorUID := req.GetActorUid() - templateNS := req.GetActorTemplateNamespace() - templateName := req.GetActorTemplateName() + p := actorBootParams{ + atespace: req.GetAtespace(), + actorName: req.GetActorName(), + actorUID: req.GetActorUid(), + templateNS: req.GetActorTemplateNamespace(), + templateName: req.GetActorTemplateName(), + containers: req.GetSpec().GetContainers(), + assetPaths: req.GetRuntimeAssetPaths(), + } + + s.actorLogger.EmitLifecycleLog("Actor starting", p.atespace, p.actorName, p.actorUID, p.templateNS, p.templateName) + if err := s.coldBootActor(ctx, p); err != nil { + return nil, err + } + s.actorLogger.EmitLifecycleLog("Actor started", p.atespace, p.actorName, p.actorUID, p.templateNS, p.templateName) + slog.InfoContext(ctx, "Actor started (overlay rootfs)", slog.String("id", p.actorUID)) + return &ateompb.RunWorkloadResponse{}, nil +} + +// actorBootParams is what a cold boot needs about an actor. It comes from a Run +// request, or from a Restore request whose snapshot scope covers only the +// durable-dir volumes (the workload itself cold-starts). +type actorBootParams struct { + atespace string + actorName string + actorUID string + templateNS string + templateName string + containers []*ateompb.Container + assetPaths map[string]string +} - s.actorLogger.EmitLifecycleLog("Actor starting", atespace, name, actorUID, templateNS, templateName) +// coldBootActor boots the actor's micro-VM from scratch and starts its +// containers, registering the result in s.running. The caller holds s.lock and +// owns the lifecycle logging. +func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (retErr error) { + atespace, name, actorUID := p.atespace, p.actorName, p.actorUID + templateNS, templateName := p.templateNS, p.templateName // All of the actor's containers share the one micro-VM (which is the pod // sandbox): each gets its own overlay rootfs and its own kata-agent // CreateContainer/StartContainer, driven below after the shared boot + // CreateSandbox + guest networking. - containers := req.GetSpec().GetContainers() + containers := p.containers if len(containers) == 0 { - return nil, status.Error(codes.InvalidArgument, "actor spec has no containers") + return status.Error(codes.InvalidArgument, "actor spec has no containers") } if len(containers) > maxActorContainers { - return nil, status.Errorf(codes.Unimplemented, "ateom-microvm supports at most %d containers, got %d", maxActorContainers, len(containers)) + return status.Errorf(codes.Unimplemented, "ateom-microvm supports at most %d containers, got %d", maxActorContainers, len(containers)) } // ateom builds the CH vm.create itself, so it needs the guest kernel + image // paths directly. - paths := req.GetRuntimeAssetPaths() + paths := p.assetPaths kernel, image := paths[assetKernel], paths[assetImage] if kernel == "" || image == "" { - return nil, fmt.Errorf("ateom-microvm requires %q and %q asset paths", assetKernel, assetImage) + return fmt.Errorf("ateom-microvm requires %q and %q asset paths", assetKernel, assetImage) } rr := s.resolveRuntime(paths) // Networking (host side): per-activation veth into the interior netns. The // tap + TC mirror is built below (after the VM exists) so its FDs are fresh. if err := s.setupActorNetwork(ctx); err != nil { - return nil, fmt.Errorf("while setting up actor network: %w", err) + return fmt.Errorf("while setting up actor network: %w", err) } defer func() { if retErr != nil { @@ -248,19 +279,19 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // lower). No host disk — the rootfs is overlay(virtio-fs lower + guest-tmpfs upper). ctrs, err := s.buildActorContainers(actorUID, containers) if err != nil { - return nil, err + return err } // Guest sizing + agent kernel params from the kata config. memMiB, vcpus, kparams, err := s.guestConfig(rr) if err != nil { - return nil, err + return err } // Clean stale per-sandbox state + create the runtime dir for the sockets. kata.CleanupSandboxState(ctx, actorUID) if err := os.MkdirAll(kata.VMDir(actorUID), 0o700); err != nil { - return nil, fmt.Errorf("while creating VM dir: %w", err) + return fmt.Errorf("while creating VM dir: %w", err) } // Stage the overlay RO lowers (bind each image into the shared dir) + start the @@ -268,7 +299,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // the actor's lifetime, so ateom owns the process (killed in teardownActor). vfsdCmd, err := s.stageOverlayLowers(ctx, rr, actorUID, ctrs) if err != nil { - return nil, err + return err } defer func() { if retErr != nil && vfsdCmd.Process != nil { @@ -283,10 +314,10 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload var durableVfsdCmd *exec.Cmd if hasDurableVolumes(containers) { if durableVolume, err = resolveDurableVolumeName(ateompath.DurableDirVolumeMountsDir(actorUID)); err != nil { - return nil, err + return err } if durableVfsdCmd, err = s.stageDurableShare(ctx, rr, actorUID); err != nil { - return nil, err + return err } defer func() { if retErr != nil && durableVfsdCmd.Process != nil { @@ -305,7 +336,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload Stderr: slogWriter{ctx}, }) if err != nil { - return nil, fmt.Errorf("while launching VMM: %w", err) + return fmt.Errorf("while launching VMM: %w", err) } defer func() { if retErr != nil && chCmd.Process != nil { @@ -321,14 +352,14 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload serialLog := filepath.Join(kata.VMDir(actorUID), "serial.log") vmCfg := buildVMConfig(actorUID, kernel, image, kparams, serialLog, memMiB, vcpus, durableVolume != "") if err := client.CreateVM(ctx, vmCfg); err != nil { - return nil, fmt.Errorf("while creating VM: %w", err) + return fmt.Errorf("while creating VM: %w", err) } // Network device: build the tap + TC mirror against the actor veth and add a // virtio-net to the created (pre-boot) VM with the tap FDs (SCM_RIGHTS). tapFiles, err := s.setupRestoreTap(ctx, "tap0_kata", 1) if err != nil { - return nil, fmt.Errorf("while building tap: %w", err) + return fmt.Errorf("while building tap: %w", err) } defer func() { for _, f := range tapFiles { @@ -340,12 +371,12 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload fds = append(fds, int(f.Fd())) } if err := client.AddNetWithFDs(ctx, actorGuestMAC, 2*len(tapFiles), fds); err != nil { - return nil, fmt.Errorf("while adding net device: %w", err) + return fmt.Errorf("while adding net device: %w", err) } // Boot. if err := client.BootVM(ctx); err != nil { - return nil, fmt.Errorf("while booting VM: %w", err) + return fmt.Errorf("while booting VM: %w", err) } slog.InfoContext(ctx, "Micro-VM booted", slog.String("id", actorUID), slog.String("api", apiSocket)) @@ -355,14 +386,14 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // does), rather than dialing once. vsockPath := kata.VsockSocketPath(actorUID) if !waitForFile(vsockPath, 15*time.Second) { - return nil, fmt.Errorf("kata-agent vsock socket %q did not appear", vsockPath) + return fmt.Errorf("kata-agent vsock socket %q did not appear", vsockPath) } ac, err := dialAgentRetry(ctx, vsockPath, 60*time.Second) if err != nil { if b, rerr := os.ReadFile(serialLog); rerr == nil { slog.ErrorContext(ctx, "agent dial failed; guest serial tail", slog.String("serial", tailString(string(b), 3000))) } - return nil, fmt.Errorf("while dialing kata-agent: %w", err) + return fmt.Errorf("while dialing kata-agent: %w", err) } // The agent client must stay open past this RPC: the stdout/stderr forwarding // goroutines (started below) read over it for the actor's lifetime. It is stored @@ -376,12 +407,12 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // Post-boot kata-agent setup: sandbox, guest networking, start each container. if err := s.startActorContainers(ctx, ac, actorUID, vsockPath, ctrs, durableVolume); err != nil { - return nil, err + return err } // Block until every readyz-enabled container reports 200. if err := readyz.WaitAll(ctx, containers, actorVethIP); err != nil { - return nil, fmt.Errorf("while waiting for container readyz: %w", err) + return fmt.Errorf("while waiting for container readyz: %w", err) } ra := &runningActor{chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, apiSocket: apiSocket, baseID: actorUID, logAgent: ac} @@ -395,9 +426,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload s.startActorLogForwarding(ac, atespace, name, actorUID, templateNS, templateName, overlayWorkloadID(c.name), c.name) } - s.actorLogger.EmitLifecycleLog("Actor started", atespace, name, actorUID, templateNS, templateName) - slog.InfoContext(ctx, "Actor started (overlay rootfs)", slog.String("id", actorUID)) - return &ateompb.RunWorkloadResponse{}, nil + return nil } // buildActorContainers prepares each of the actor's containers for the shared From 448400a712c8ec01f90580a24114965ca7f01b44 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 24 Jul 2026 16:01:18 -0700 Subject: [PATCH 3/3] demos,e2e,docs: exercise durable-dir volumes on the micro-VM counter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give the micro-VM counter demo the same durableDir volume the gVisor one has, and drop TestDurableDirLifecycle's micro-VM skip so the scope matrix (Full/Full, Data/Full, Data/Data) runs against both runtimes. TestActorLifecycle no longer special-cases micro-VM to expect a file counter of -1. That value was the demo failing to write: it counts in /home/counter/a.txt, which nothing created in the micro-VM rootfs. The volume mounts there now, so both runtimes assert the same counters. Also correct the glossary: an ActorTemplate may declare one DurableDir volume, not several — CEL has enforced that since the feature landed — and Full scope captures the volumes whether or not the runtime keeps them inside rootfs (the micro-VM does not). --- demos/counter/counter-microvm.yaml.tmpl | 9 +++++++ docs/architecture.md | 2 +- docs/glossary.md | 13 +++++----- internal/e2e/suites/demo/demo_test.go | 32 ++++++------------------- 4 files changed, 23 insertions(+), 33 deletions(-) diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index d53002066..ed3cc4e31 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -121,8 +121,17 @@ spec: httpGet: path: /readyz port: 80 + # The counter writes its file counter here. With the durableDir volume below + # it lands on a host-backed virtio-fs share instead of the guest-RAM rootfs + # upper, so it survives a Data-scope snapshot (which keeps no guest memory). + volumeMounts: + - name: data + mountPath: /home/counter workerSelector: matchLabels: workload: counter-microvm snapshotsConfig: location: gs://${BUCKET_NAME}/ate-demo-counter-microvm/ + volumes: + - name: data + durableDir: {} diff --git a/docs/architecture.md b/docs/architecture.md index 62d804253..9a2a512cb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -338,7 +338,7 @@ A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class * **gVisor** (`ateom-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree. - * **micro-VM** (`ateom-microvm`): Runs the workload inside a [Kata Containers](https://katacontainers.io/) guest on the [Cloud Hypervisor](https://www.cloudhypervisor.org/) VMM. Suspend and resume capture a memory-only VM snapshot and restore it on-demand using `userfaultfd` memory demand-paging, with container rootfs writes captured in guest RAM via a `tmpfs` overlay. + * **micro-VM** (`ateom-microvm`): Runs the workload inside a [Kata Containers](https://katacontainers.io/) guest on the [Cloud Hypervisor](https://www.cloudhypervisor.org/) VMM. Suspend and resume capture a memory-only VM snapshot and restore it on-demand using `userfaultfd` memory demand-paging, with container rootfs writes captured in guest RAM via a `tmpfs` overlay. `DurableDir` volumes are host-backed instead, served over a second (writable) virtio-fs share and shipped in snapshots as a tar, so a `Data`-scope snapshot can capture them without any guest memory. ### Networking Stack (`atenet` + Envoy) diff --git a/docs/glossary.md b/docs/glossary.md index 9f1fa6fe3..aadb3f383 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -76,19 +76,18 @@ because they change too frequently for etcd. - **DurableDir volume**: a directory mounted into one or more containers whose contents are preserved by the [`Data` snapshot scope](#snapshots) and therefore survive across Suspend/Resume independently of process - memory or other rootfs writes. A single `ActorTemplate` may declare - multiple `DurableDir` volumes, and the same volume may be mounted into - multiple containers (potentially at different paths). This is the - per-Actor application-data surface. + memory or other rootfs writes. An `ActorTemplate` may declare one + `DurableDir` volume, and it may be mounted into multiple containers + (potentially at different paths). This is the per-Actor + application-data surface. ## Snapshots - **Snapshot scope**: what an `ActorTemplate`'s `SnapshotsConfig` includes in a given snapshot. Two scopes exist today: - **`Full`**: process memory plus the rootfs delta on top of the OCI - image (which also includes any attached `DurableDir` volumes, - since they live inside rootfs). Used to capture everything needed - to resume hot. + image, and any attached `DurableDir` volumes. Used to capture + everything needed to resume hot. - **`Data`**: only the contents of attached volumes that support snapshots — currently `DurableDir` volumes. Process memory and the rest of rootfs are discarded; on Resume the Actor cold-boots from diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 8fb958b37..c1c404a1b 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -83,10 +83,6 @@ func TestActorLifecycle(t *testing.T) { } func TestDurableDirLifecycle(t *testing.T) { - if isMicroVMEnvironment() { - t.Skip("Skipping TestDurableDirLifecycle for microVM environment") - } - tests := []struct { name string tc actorLifecycleTestCase @@ -380,11 +376,7 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * t.Fatalf("failed to call actor: %v", err) } - if isMicroVMEnvironment() { - validateCounterResponse(t, resp, "after creation", 1, -1) - } else { - validateCounterResponse(t, resp, "after creation", 1, 1) - } + validateCounterResponse(t, resp, "after creation", 1, 1) // Pausing the actor t.Logf("Pausing Actor %q...", actorName) @@ -408,11 +400,7 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * if err != nil { t.Fatalf("failed to call actor again: %v", err) } - if isMicroVMEnvironment() { - validateCounterResponse(t, resp, "after pause", 2, -1) - } else { - validateCounterResponse(t, resp, "after pause", 2, 2) - } + validateCounterResponse(t, resp, "after pause", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -467,11 +455,7 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj if err != nil { t.Fatalf("failed to call actor: %v", err) } - if isMicroVMEnvironment() { - validateCounterResponse(t, resp, "after creation", 1, -1) - } else { - validateCounterResponse(t, resp, "after creation", 1, 1) - } + validateCounterResponse(t, resp, "after creation", 1, 1) // Suspending the actor t.Logf("Suspending Actor %q...", actorName) @@ -495,11 +479,7 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj if err != nil { t.Fatalf("failed to call actor again: %v", err) } - if isMicroVMEnvironment() { - validateCounterResponse(t, resp, "after suspend", 2, -1) - } else { - validateCounterResponse(t, resp, "after suspend", 2, 2) - } + validateCounterResponse(t, resp, "after suspend", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -826,7 +806,9 @@ func callActorOnce(t *testing.T, atespace, actorName string) (string, error) { return string(body), nil } +// isMicroVMEnvironment reports whether the suite is running against the +// micro-VM demo template, which does not support every volume type yet (see +// TestExternalVolumeLifecycle). func isMicroVMEnvironment() bool { - // TODO(BenTheElder) remove it once https://github.com/agent-substrate/substrate/pull/313 is merged. return os.Getenv("E2E_TEMPLATE_NAMESPACE") == "ate-demo-counter-microvm" }