Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions Guide/src/dev_guide/snapshot_format.md
Original file line number Diff line number Diff line change
Expand Up @@ -95,13 +95,17 @@ Restore likewise opens the snapshot directory once and resolves its artifacts
relative to that handle. Windows uses read-only handles with `FILE_SHARE_READ`
only, rejects reparse points, compares `FILE_ID_INFO` and EOF before and after
creating the private COW section, and keeps the directory and artifact guards
in the VM worker until teardown. Linux keeps the exact `O_NOFOLLOW` directory
and regular-file descriptors, and before handoff rejects a change of the device,
inode, length, or modification time of the opened manifest, state, or memory
file, so renaming or replacing the original path cannot substitute another
generation. A change of their links, mode, or owner alone passes, which lets a
restore open a snapshot before the capture's source, as it tears down, removes
its RAM file's temporary name, the other link of an automatic `memory.bin`.
in the VM worker until teardown. Linux opens each artifact with `O_NOFOLLOW`
and `O_NONBLOCK`, so that a FIFO without a writer, for which a blocking open
would wait, reaches the check of the opened file's type and is refused like
any other file that is not a regular file, and then makes a regular file's
descriptor blocking again. It keeps the exact directory and regular-file
descriptors, and before handoff rejects a change of the device, inode, length,
or modification time of the opened manifest, state, or memory file, so
renaming or replacing the original path cannot substitute another generation.
A change of their links, mode, or owner alone passes, which lets a restore
open a snapshot before the capture's source, as it tears down, removes its RAM
file's temporary name, the other link of an automatic `memory.bin`.
Linux file descriptors do not provide mandatory write exclusion; deployments
that need authenticated or write-proof local artifacts must add a stronger mode
such as a lease, fs-verity, or a verified artifact broker.
Expand Down
123 changes: 118 additions & 5 deletions openvmm/openvmm_helpers/src/snapshot/fs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -754,6 +754,9 @@ impl OpenedSnapshotDirectory {
})
}

/// Opens the artifact `name` of this directory for reading, and refuses it
/// without waiting unless it is a regular file, as
/// [`accept_opened_regular_file`] describes.
pub(super) fn open_regular_file(
&self,
name: &str,
Expand All @@ -767,7 +770,7 @@ impl OpenedSnapshotDirectory {
let fd = nix::fcntl::openat(
&self.file,
name,
OFlag::O_RDONLY | OFlag::O_CLOEXEC | OFlag::O_NOFOLLOW,
OFlag::O_RDONLY | OFlag::O_CLOEXEC | OFlag::O_NOFOLLOW | OFlag::O_NONBLOCK,
Mode::empty(),
)
.map_err(nix_error)
Expand All @@ -793,7 +796,7 @@ impl OpenedSnapshotDirectory {
#[cfg(not(any(target_os = "linux", windows)))]
let file = open_regular_file_impl(&self.path.join(name), description)?;

validate_opened_regular_file(&file, description, &self.display_path(name))?;
accept_opened_regular_file(&file, description, &self.display_path(name))?;
Ok(file)
}

Expand Down Expand Up @@ -951,7 +954,7 @@ fn open_regular_file_impl(path: &Path, description: &str) -> anyhow::Result<std:
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW);
options.custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK);
}
#[cfg(windows)]
{
Expand All @@ -964,10 +967,32 @@ fn open_regular_file_impl(path: &Path, description: &str) -> anyhow::Result<std:
let file = options
.open(path)
.with_context(|| format!("failed to open {description} at {}", path.display()))?;
validate_opened_regular_file(&file, description, path)?;
accept_opened_regular_file(&file, description, path)?;
Ok(file)
}

/// Refuses an opened artifact that is not a regular file and, on Unix, makes
/// the handle of one that is blocking.
///
/// Artifacts open without following a final link, or on Windows a reparse
/// point, and on Unix with `O_NONBLOCK`: an open of a FIFO without a writer,
/// or of some devices, would otherwise wait until another process acted,
/// before the type of the opened file could be checked here. Reads of a
/// regular file ignore the flag, but its handle outlives this check, as a
/// restore guard and as the descriptor that the VM worker maps as RAM, so it
/// returns to the blocking mode of an open without the flag.
fn accept_opened_regular_file(
file: &std::fs::File,
description: &str,
path: &Path,
) -> anyhow::Result<()> {
validate_opened_regular_file(file, description, path)?;
#[cfg(unix)]
pal::unix::pipe::set_nonblocking(file, false)
.with_context(|| format!("failed to make opened {description} blocking"))?;
Ok(())
}

fn validate_opened_regular_file(
file: &std::fs::File,
description: &str,
Expand Down Expand Up @@ -1154,8 +1179,96 @@ pub(super) fn sync_directory(_path: &Path) -> anyhow::Result<()> {
}

#[cfg(all(test, target_os = "linux"))]
mod tests {
pub(super) mod tests {
use super::*;
use nix::fcntl::FcntlArg;
use nix::fcntl::OFlag;

/// Runs `open` on another thread, which opens the FIFO at `fifo` while
/// nothing has the FIFO open for writing, and returns the error with which
/// `open` refused it.
///
/// Fails if `open` waits for a writer, once it has given it one, so that a
/// test fails rather than hangs.
pub(in crate::snapshot) fn refusal_of_writerless_fifo(
fifo: &Path,
open: impl FnOnce() -> anyhow::Result<()> + Send + 'static,
) -> anyhow::Error {
let (sender, receiver) = std::sync::mpsc::channel();
let opener = std::thread::spawn(move || sender.send(open()).unwrap());
let result = match receiver.recv_timeout(std::time::Duration::from_secs(10)) {
Ok(result) => result,
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
// An open for reading and writing does not wait, and gives the
// waiting open its writer.
let _writer = std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(fifo)
.unwrap();
opener.join().unwrap();
panic!("the open of {} waited for a writer", fifo.display());
}
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
std::panic::resume_unwind(opener.join().unwrap_err())
}
};
opener.join().unwrap();
result.expect_err("a FIFO was opened as a regular file")
}

#[test]
fn artifact_opens_refuse_a_writerless_fifo_without_waiting() {
type Open = fn(&Path, &str) -> anyhow::Result<std::fs::File>;
let opens: [(&str, Open); 5] = [
("open_regular_file in a directory", |dir, name| {
OpenedSnapshotDirectory::open(dir)?.open_regular_file(name, "artifact")
}),
(
"open_regular_file_for_identity in a directory",
|dir, name| {
OpenedSnapshotDirectory::open(dir)?
.open_regular_file_for_identity(name, "artifact")
},
),
("open_file_with_length in a directory", |dir, name| {
OpenedSnapshotDirectory::open(dir)?.open_file_with_length(name, 0, "artifact")
}),
("open_regular_file", |dir, name| {
open_regular_file(&dir.join(name), "artifact")
}),
("open_file_with_length", |dir, name| {
open_file_with_length(&dir.join(name), 0, "artifact")
}),
];
let dir = tempfile::tempdir().unwrap();
let fifo = dir.path().join("fifo.bin");
nix::unistd::mkfifo(
&fifo,
nix::sys::stat::Mode::S_IRUSR | nix::sys::stat::Mode::S_IWUSR,
)
.unwrap();
std::fs::write(dir.path().join("regular.bin"), b"").unwrap();

for (opener, open) in opens {
let directory = dir.path().to_owned();
let error =
refusal_of_writerless_fifo(&fifo, move || open(&directory, "fifo.bin").map(drop));
assert!(
format!("{error:#}").contains("artifact is not a regular file"),
"{opener}: {error:#}"
);

// The handle of a regular file blocks, as that of an open without
// O_NONBLOCK does.
let file = open(dir.path(), "regular.bin").unwrap();
let flags = nix::fcntl::fcntl(&file, FcntlArg::F_GETFL).unwrap();
assert!(
!OFlag::from_bits_retain(flags).contains(OFlag::O_NONBLOCK),
"{opener}"
);
}
}

#[test]
fn sparse_copy_fallbacks_preserve_extents_and_clone_independence() {
Expand Down
66 changes: 66 additions & 0 deletions openvmm/openvmm_helpers/src/snapshot/restore.rs
Original file line number Diff line number Diff line change
Expand Up @@ -982,6 +982,72 @@ mod tests {
);
}

/// Restore refuses a FIFO in place of any artifact of a paired-scratch
/// snapshot for what it is, without waiting for the writer that a
/// blocking open of the FIFO would wait for.
#[cfg(target_os = "linux")]
#[test]
fn restore_refuses_a_writerless_fifo_artifact_without_waiting() {
use super::super::fs::tests::refusal_of_writerless_fifo;

let fifo_mode = nix::sys::stat::Mode::S_IRUSR | nix::sys::stat::Mode::S_IWUSR;
let scratch = vec![0x5a_u8; 512];
let mut manifest = paired_scratch_manifest(&scratch);
manifest.snapshot_tier = SNAPSHOT_TIER_INSTANCE_CHECKPOINT.to_owned();
manifest.restore_policy = SNAPSHOT_RESTORE_POLICY_RESUME.to_owned();
manifest
.machine_contract
.as_mut()
.unwrap()
.set_effective_command_line(
"console=hvc0 nvx_snapshot_tier=instance-checkpoint".to_owned(),
);
let dir = tempfile::tempdir().unwrap();
let snap_dir = dir.path().join("snap");
let memory_source = dir.path().join("memory-source.bin");
let scratch_source = dir.path().join("scratch-source.bin");
std::fs::write(&memory_source, vec![0_u8; 1024]).unwrap();
std::fs::write(&scratch_source, &scratch).unwrap();
write_snapshot_from_memory_and_scratch_files(
&snap_dir,
&manifest,
b"state",
&std::fs::File::open(&memory_source).unwrap(),
Some(&std::fs::File::open(&scratch_source).unwrap()),
)
.unwrap();

for name in [MANIFEST_FILE_NAME, STATE_FILE_NAME, MEMORY_FILE_NAME] {
let artifact = snap_dir.join(name);
let set_aside = dir.path().join(name);
std::fs::rename(&artifact, &set_aside).unwrap();
nix::unistd::mkfifo(&artifact, fifo_mode).unwrap();
let snapshot = snap_dir.clone();
let error = refusal_of_writerless_fifo(&artifact, move || {
OpenedSnapshot::open(&snapshot).map(drop)
});
assert!(
format!("{error:#}").contains("is not a regular file"),
"{name}: {error:#}"
);
std::fs::remove_file(&artifact).unwrap();
std::fs::rename(&set_aside, &artifact).unwrap();
}

// Restore opens the paired scratch once it has opened the snapshot.
let snapshot = OpenedSnapshot::open(&snap_dir).unwrap();
let scratch_path = snap_dir.join(SCRATCH_FILE_NAME);
std::fs::remove_file(&scratch_path).unwrap();
nix::unistd::mkfifo(&scratch_path, fifo_mode).unwrap();
let error = refusal_of_writerless_fifo(&scratch_path, move || {
snapshot.open_paired_scratch_file().map(drop)
});
assert!(
format!("{error:#}").contains("scratch.img is not a regular file"),
"{error:#}"
);
}

#[cfg(unix)]
#[test]
fn opened_memory_handle_survives_path_replacement() {
Expand Down
Loading