diff --git a/.github/linters/urunc-dict.txt b/.github/linters/urunc-dict.txt index 69b63867c..f220e5a2a 100644 --- a/.github/linters/urunc-dict.txt +++ b/.github/linters/urunc-dict.txt @@ -429,3 +429,4 @@ hyperlight Hyperlight Odysseas Kalaitsidis +nowait diff --git a/docs/configuration.md b/docs/configuration.md index a2daa4ba4..8ec6b1149 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -112,11 +112,34 @@ Each monitor subsection supports the following options: | `default_vcpus` | integer | `1` | Default number of virtual CPUs | | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | +| `socket_path` | string | (empty) | Optional path for the monitor's control socket. If not set, the monitor runs without a control socket | Since Qemu is the only currently supported monitor which requires extra data to boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for Qemu's data files. +The `socket_path` option applies to the monitors that expose a control socket: +Firecracker (its API socket), Qemu (a QMP socket) and Cloud Hypervisor (its REST +API socket). It has no effect on the other monitors. The control socket is +opt-in: it exists only when `socket_path` is set. If it is not set, the +monitor runs with no control socket at all; an operator can leave it unset if +they do not need the socket, or to keep a smaller attack surface. + +When it is set, the monitor creates the socket inside its own (pivoted) rootfs. +`urunc` creates the directory of `socket_path` inside that rootfs after it drops +privileges to the monitor's user, so the path is subject to two constraints: + +- Its parent directory must be one the monitor's user can create and write. A + directory that only `root` can write does not work for a non-root monitor. +- It must not sit over an existing file or directory. `urunc` creates the + directory of `socket_path`; that fails cleanly if a file already exists at one + of the directories in the path. The monitor then binds the socket at + `socket_path`; that fails if a file already exists at `socket_path` itself. + +`urunc` removes the socket when the container is stopped (on a terminating +signal) and when it is deleted, so a restart that reuses the same `socket_path` +does not find a stale socket. + **Example:** ```toml @@ -130,6 +153,7 @@ data_path = "/usr/local/share/" default_memory_mb = 512 default_vcpus = 2 path = "/opt/firecracker/firecracker" +socket_path = "/run/urunc/fc.sock" ``` ### Extra binaries Configuration diff --git a/pkg/unikontainers/hypervisors/cloud_hypervisor.go b/pkg/unikontainers/hypervisors/cloud_hypervisor.go index 606a3c02e..2c0ab4118 100644 --- a/pkg/unikontainers/hypervisors/cloud_hypervisor.go +++ b/pkg/unikontainers/hypervisors/cloud_hypervisor.go @@ -49,6 +49,12 @@ func (ch *CloudHypervisor) UsesKVM() bool { return true } +// SupportsControlSocket reports that Cloud Hypervisor exposes a control socket +// (its REST API socket). +func (ch *CloudHypervisor) SupportsControlSocket() bool { + return true +} + // SupportsSharedfs returns true as Cloud Hypervisor supports virtiofs func (ch *CloudHypervisor) SupportsSharedfs(fsType string) bool { switch fsType { @@ -70,6 +76,12 @@ func (ch *CloudHypervisor) BuildExecCmd(args types.ExecArgs, ukernel types.Unike // Start building the command exArgs := []string{ch.binaryPath} + // Expose the REST API socket only when socket_path is set, so the runtime + // can talk to Cloud Hypervisor after boot. + if args.SocketPath != "" { + exArgs = append(exArgs, "--api-socket", "path="+args.SocketPath) + } + // Memory configuration if args.Sharedfs.Type == "virtiofs" { exArgs = append(exArgs, "--memory", fmt.Sprintf("size=%sM,shared=on", chMem)) diff --git a/pkg/unikontainers/hypervisors/cloud_hypervisor_test.go b/pkg/unikontainers/hypervisors/cloud_hypervisor_test.go new file mode 100644 index 000000000..708f97b2c --- /dev/null +++ b/pkg/unikontainers/hypervisors/cloud_hypervisor_test.go @@ -0,0 +1,78 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +const testCHBinary = "/usr/bin/cloud-hypervisor" + +// TestCloudHypervisorBuildExecCmdSocket verifies Cloud Hypervisor emits +// --api-socket only when socket_path is set. +func TestCloudHypervisorBuildExecCmdSocket(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + args types.ExecArgs + mustContain []string + mustNotContain []string + }{ + { + name: "configured SocketPath renders --api-socket on that path", + args: types.ExecArgs{ + UnikernelPath: testKernelPath, + Command: testCommand, + SocketPath: "/run/urunc/ch.sock", + }, + mustContain: []string{"--api-socket path=/run/urunc/ch.sock"}, + }, + { + name: "unset SocketPath omits --api-socket", + args: types.ExecArgs{ + UnikernelPath: testKernelPath, + Command: testCommand, + ContainerID: "abc123", + }, + mustNotContain: []string{"--api-socket"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ch := &CloudHypervisor{binary: CloudHypervisorBinary, binaryPath: testCHBinary} + out, err := ch.BuildExecCmd(tt.args, &fakeUnikernel{}) + assert.NoError(t, err) + assert.NotEmpty(t, out) + + assert.Equal(t, testCHBinary, out[0], "binary path must be the first element") + joined := strings.Join(out, " ") + + for _, want := range tt.mustContain { + assert.Contains(t, joined, want, "expected %q to be present", want) + } + for _, notWant := range tt.mustNotContain { + assert.NotContains(t, joined, notWant, "expected %q to be absent", notWant) + } + }) + } +} diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 9588a45bf..5911fb862 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -97,6 +97,12 @@ func (fc *Firecracker) SupportsSharedfs(_ string) bool { return false } +// SupportsControlSocket reports that Firecracker exposes a control socket (its API +// socket). +func (fc *Firecracker) SupportsControlSocket() bool { + return true +} + func (fc *Firecracker) Path() string { return fc.binaryPath } @@ -108,9 +114,16 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel // options in FC, since the string return value of the Monitor related // functions in the unikernel interface do not integrate well with FC's // json configuration. - cmdString := fc.Path() + " --no-api --config-file " JSONConfigFile := filepath.Join("/tmp/", FCJsonFilename) - cmdString += JSONConfigFile + cmdString := fc.Path() + // Enable the API socket when a socket_path is configured, otherwise keep the + // upstream --no-api. Either way the guest boots from the config file. + if args.SocketPath != "" { + cmdString += " --api-sock " + args.SocketPath + } else { + cmdString += " --no-api" + } + cmdString += " --config-file " + JSONConfigFile if !args.Seccomp { cmdString += " --no-seccomp" } diff --git a/pkg/unikontainers/hypervisors/firecracker_test.go b/pkg/unikontainers/hypervisors/firecracker_test.go new file mode 100644 index 000000000..e5602bc35 --- /dev/null +++ b/pkg/unikontainers/hypervisors/firecracker_test.go @@ -0,0 +1,80 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +const testFCBinary = "/usr/bin/firecracker" + +// TestFirecrackerBuildExecCmdSocket verifies Firecracker enables --api-sock +// only when socket_path is set, and restores --no-api otherwise. +func TestFirecrackerBuildExecCmdSocket(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + args types.ExecArgs + mustContain []string + mustNotContain []string + }{ + { + name: "configured SocketPath renders --api-sock and keeps --config-file", + args: types.ExecArgs{ + UnikernelPath: testKernelPath, + Command: testCommand, + SocketPath: "/run/urunc/fc.sock", + }, + mustContain: []string{"--api-sock /run/urunc/fc.sock", "--config-file"}, + mustNotContain: []string{"--no-api"}, + }, + { + name: "unset SocketPath restores --no-api and omits --api-sock", + args: types.ExecArgs{ + UnikernelPath: testKernelPath, + Command: testCommand, + ContainerID: "abc123", + }, + mustContain: []string{"--no-api", "--config-file"}, + mustNotContain: []string{"--api-sock"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + fc := &Firecracker{binary: FirecrackerBinary, binaryPath: testFCBinary} + out, err := fc.BuildExecCmd(tt.args, &fakeUnikernel{}) + assert.NoError(t, err) + assert.NotEmpty(t, out) + + assert.Equal(t, testFCBinary, out[0], "binary path must be the first element") + joined := strings.Join(out, " ") + + for _, want := range tt.mustContain { + assert.Contains(t, joined, want, "expected %q to be present", want) + } + for _, notWant := range tt.mustNotContain { + assert.NotContains(t, joined, notWant, "expected %q to be absent", notWant) + } + }) + } +} diff --git a/pkg/unikontainers/hypervisors/hedge.go b/pkg/unikontainers/hypervisors/hedge.go index 7051ba7f0..ed2c6fe6c 100644 --- a/pkg/unikontainers/hypervisors/hedge.go +++ b/pkg/unikontainers/hypervisors/hedge.go @@ -51,6 +51,11 @@ func (h *Hedge) SupportsSharedfs(_ string) bool { return false } +// SupportsControlSocket reports that Hedge exposes no control socket. +func (h *Hedge) SupportsControlSocket() bool { + return false +} + func (h *Hedge) Path() string { return "" } diff --git a/pkg/unikontainers/hypervisors/hvt.go b/pkg/unikontainers/hypervisors/hvt.go index 69cb12b91..e8f791862 100644 --- a/pkg/unikontainers/hypervisors/hvt.go +++ b/pkg/unikontainers/hypervisors/hvt.go @@ -139,6 +139,11 @@ func (h *HVT) SupportsSharedfs(_ string) bool { return false } +// SupportsControlSocket reports that HVT exposes no control socket. +func (h *HVT) SupportsControlSocket() bool { + return false +} + // Path returns the path to the hvt binary. func (h *HVT) Path() string { return h.binaryPath diff --git a/pkg/unikontainers/hypervisors/hyperlight.go b/pkg/unikontainers/hypervisors/hyperlight.go index 67d67be6d..b12981ce8 100644 --- a/pkg/unikontainers/hypervisors/hyperlight.go +++ b/pkg/unikontainers/hypervisors/hyperlight.go @@ -46,6 +46,11 @@ func (h *Hyperlight) SupportsSharedfs(_ string) bool { return false } +// SupportsControlSocket reports that Hyperlight exposes no control socket. +func (h *Hyperlight) SupportsControlSocket() bool { + return false +} + // Path returns the path to the hyperlight binary. func (h *Hyperlight) Path() string { return h.binaryPath diff --git a/pkg/unikontainers/hypervisors/qemu.go b/pkg/unikontainers/hypervisors/qemu.go index 1ac77f870..bb5e12bf3 100644 --- a/pkg/unikontainers/hypervisors/qemu.go +++ b/pkg/unikontainers/hypervisors/qemu.go @@ -56,6 +56,11 @@ func (q *Qemu) SupportsSharedfs(_ string) bool { return true } +// SupportsControlSocket reports that QEMU exposes a control socket (QMP). +func (q *Qemu) SupportsControlSocket() bool { + return true +} + func (q *Qemu) Path() string { return q.binaryPath } @@ -67,6 +72,11 @@ func (q *Qemu) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel) ([]str cmdString += " -cpu host" // Choose CPU cmdString += " -enable-kvm" // Enable KVM to use CPU virt extensions cmdString += " -display none -vga none -serial stdio -monitor null" // Disable graphic output + // Expose the QMP socket only when socket_path is set. server,nowait lets + // QEMU boot without waiting for a client. + if args.SocketPath != "" { + cmdString += " -qmp unix:" + args.SocketPath + ",server,nowait" + } if args.VCPUs > 0 { cmdString += fmt.Sprintf(" -smp %d", args.VCPUs) diff --git a/pkg/unikontainers/hypervisors/qemu_test.go b/pkg/unikontainers/hypervisors/qemu_test.go index fff45244c..e959165f4 100644 --- a/pkg/unikontainers/hypervisors/qemu_test.go +++ b/pkg/unikontainers/hypervisors/qemu_test.go @@ -92,8 +92,29 @@ func TestQemuBuildExecCmd(t *testing.T) { "vhost-user-fs-pci", "virtio-blk-pci", "vhost-vsock-pci", + "-qmp", }, }, + { + name: "configured SocketPath renders -qmp on that path", + args: types.ExecArgs{ + UnikernelPath: testKernelPath, + Command: testCommand, + SocketPath: "/run/urunc/q.sock", + }, + unikernel: &fakeUnikernel{}, + mustContain: []string{"-qmp unix:/run/urunc/q.sock,server,nowait"}, + }, + { + name: "unset SocketPath omits -qmp", + args: types.ExecArgs{ + UnikernelPath: testKernelPath, + Command: testCommand, + ContainerID: "abc123", + }, + unikernel: &fakeUnikernel{}, + mustNotContain: []string{"-qmp"}, + }, { name: "custom MemSizeB renders -m in MB", args: types.ExecArgs{ diff --git a/pkg/unikontainers/hypervisors/spt.go b/pkg/unikontainers/hypervisors/spt.go index 60b5401f6..08940a9c5 100644 --- a/pkg/unikontainers/hypervisors/spt.go +++ b/pkg/unikontainers/hypervisors/spt.go @@ -51,6 +51,11 @@ func (s *SPT) SupportsSharedfs(_ string) bool { return false } +// SupportsControlSocket reports that SPT exposes no control socket. +func (s *SPT) SupportsControlSocket() bool { + return false +} + // Path returns the path to the spt binary. func (s *SPT) Path() string { return s.binaryPath diff --git a/pkg/unikontainers/hypervisors/vmm_test.go b/pkg/unikontainers/hypervisors/vmm_test.go index 6d0f7ebdb..8e471c7f3 100644 --- a/pkg/unikontainers/hypervisors/vmm_test.go +++ b/pkg/unikontainers/hypervisors/vmm_test.go @@ -18,8 +18,36 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" ) +// TestSupportsControlSocket verifies each monitor reports whether it exposes a +// control socket (Qemu, Firecracker, Cloud Hypervisor true; the rest false). +func TestSupportsControlSocket(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + vmm types.VMM + want bool + }{ + {"qemu", &Qemu{}, true}, + {"firecracker", &Firecracker{}, true}, + {"cloud-hypervisor", &CloudHypervisor{}, true}, + {"hvt", &HVT{}, false}, + {"spt", &SPT{}, false}, + {"hedge", &Hedge{}, false}, + {"hyperlight", &Hyperlight{}, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, tt.vmm.SupportsControlSocket()) + }) + } +} + func TestVMMFactoryQemuVhostFalse(t *testing.T) { t.Parallel() factory, exists := vmmFactories[QemuVmm] diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index f5d668b0d..2285603ff 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -41,6 +41,7 @@ type VMM interface { Path() string UsesKVM() bool SupportsSharedfs(string) bool + SupportsControlSocket() bool Ok() error } @@ -111,6 +112,7 @@ type ExecArgs struct { VAccelType string // Specifies the vAccel acceleration type(e.g. vsock). When empty, vAccel is disabled VSockDevPath string // The host directory where the fc unix socket is created VSockDevID int // The guest-cid + SocketPath string // The path of the monitor's control socket (empty means no control socket) Net NetDevParams Sharedfs SharedfsParams } @@ -138,7 +140,8 @@ type ExtraBinConfig struct { type MonitorConfig struct { DefaultMemoryMB uint `toml:"default_memory_mb"` DefaultVCPUs uint `toml:"default_vcpus"` - BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary - DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) - Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary + DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) + Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket (unset means no control socket) } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index ad0bf2897..953f14268 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -29,6 +29,7 @@ import ( "sync" "syscall" + securejoin "github.com/cyphar/filepath-securejoin" "github.com/urunc-dev/urunc/pkg/network" "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" "github.com/urunc-dev/urunc/pkg/unikontainers/types" @@ -498,6 +499,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { defaultVCPUs = 1 } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB + socketPath := u.UruncCfg.Monitors[vmmType].SocketPath // ExecArgs vmmArgs := types.ExecArgs{ @@ -507,6 +509,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { Seccomp: true, // Enable Seccomp by default MemSizeB: monitorMemoryBytes(defaultMemSizeMB, u.Spec.Linux.Resources), VCPUs: uint(defaultVCPUs), + SocketPath: socketPath, Environment: os.Environ(), } @@ -717,6 +720,15 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + // Create the socket directory after setupUser, so the monitor's user owns + // it and a non-root monitor can bind there. The monitor creates the socket. + if vmm.SupportsControlSocket() && vmmArgs.SocketPath != "" { + sockDir := filepath.Dir(vmmArgs.SocketPath) + if err = os.MkdirAll(sockDir, 0o700); err != nil { + return fmt.Errorf("failed to create control socket directory %q: %w", sockDir, err) + } + } + // execute hooks // NOTE: StartContainer hooks are supposed to run right before the init of // the container. However, in the case of a Linux-based container, the init @@ -791,6 +803,41 @@ func setupUser(user specs.User) error { return nil } +// monitorRootfs returns the host path of the monitor's rootfs: the separate +// one under the bundle if it exists, else the container's own rootfs. +func (u *Unikontainer) monitorRootfs() string { + bundleDir := filepath.Clean(u.State.Bundle) + rootfsDir := filepath.Clean(u.Spec.Root.Path) + if !filepath.IsAbs(rootfsDir) { + rootfsDir = filepath.Join(bundleDir, rootfsDir) + } + monRootfs := filepath.Join(bundleDir, monitorRootfsDirName) + if _, err := os.Stat(monRootfs); !os.IsNotExist(err) { + return monRootfs + } + return rootfsDir +} + +// removeControlSocket deletes the monitor's control socket. It skips a missing +// path and never deletes a non-socket file. +func (u *Unikontainer) removeControlSocket(socketPath string) error { + sockRealPath, err := securejoin.SecureJoin(u.monitorRootfs(), socketPath) + if err != nil { + return err + } + info, err := os.Lstat(sockRealPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if info.Mode()&os.ModeSocket == 0 { + return nil + } + return os.Remove(sockRealPath) +} + // Signal sends a specified signal to container's init. func (u *Unikontainer) Signal(signal unix.Signal) error { vmmType := u.State.Annotations[annotHypervisor] @@ -799,7 +846,20 @@ func (u *Unikontainer) Signal(signal unix.Signal) error { return err } - return vmm.Signal(u.State.Pid, signal) + if err = vmm.Signal(u.State.Pid, signal); err != nil { + return err + } + + // A stop calls kill with SIGTERM and never runs Delete, so the socket is + // removed here too. Best-effort: a failure must not fail the signal. + socketPath := u.UruncCfg.Monitors[vmmType].SocketPath + if (signal == unix.SIGKILL || signal == unix.SIGTERM) && socketPath != "" && vmm.SupportsControlSocket() { + if rmErr := u.removeControlSocket(socketPath); rmErr != nil { + uniklog.Warnf("failed to remove control socket: %v", rmErr) + } + } + + return nil } // Kill stops the VMM process, first by asking the VMM struct to stop @@ -908,6 +968,14 @@ func (u *Unikontainer) Delete() error { prefPath = rootfsDir } + // Remove the control socket so a restart on the same socket_path is clean. + socketPath := u.UruncCfg.Monitors[vmmType].SocketPath + if socketPath != "" && vmm.SupportsControlSocket() { + if err = u.removeControlSocket(socketPath); err != nil { + return fmt.Errorf("failed to remove control socket: %w", err) + } + } + err = rmMultipleDirs(prefPath, dirs) if err != nil { return err diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 22573f43c..3037e933a 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -145,6 +145,7 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"binary_path"] = hvCfg.BinaryPath cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) + cfgMap[prefix+"socket_path"] = hvCfg.SocketPath } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -191,6 +192,8 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { hvCfg.BinaryPath = val case "data_path": hvCfg.DataPath = val + case "socket_path": + hvCfg.SocketPath = val case "vhost": boolVal, err := strconv.ParseBool(val) if err != nil {