diff --git a/README.md b/README.md index 0b4845d72..2a3201b7b 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,10 @@ check [agent-s README](agents/agent-s/README.md) - btrfs only - rootful only - amd64 only -- linux page-cache CoW currently has a memory leak that will be fixed +- Filecow layers are reclaimed with their cached inode/mapping owners rather + than eagerly when a child is deleted. Repeated unchanged forks reuse the + current layer, so retained layers are bounded by changed generations instead + of the total fork count. ## License diff --git a/criu/build.sh b/criu/build.sh index cb1d9966c..cdb1cb472 100755 --- a/criu/build.sh +++ b/criu/build.sh @@ -93,12 +93,25 @@ for mod in "${ACTIVE_MODULES[@]}"; do fi if lsmod | awk '{print $1}' | grep -qx "${mod}"; then - echo " -- ${mod}: already loaded, rmmod then insmod" - rmmod "${mod}" 2>/dev/null || true - else - echo " -- ${mod}: insmod" + echo " -- ${mod}: already loaded, trying reload" + if ! rmmod "${mod}"; then + echo "build.sh: failed to unload loaded module ${mod}; aborting to avoid stale module" >&2 + echo "build.sh: stop running tfork/podman containers that may still hold ${mod}, then retry" >&2 + exit 1 + fi + if lsmod | awk '{print $1}' | grep -qx "${mod}"; then + echo "build.sh: module ${mod} is still loaded after rmmod; aborting to avoid stale module" >&2 + echo "build.sh: stop running tfork/podman containers that may still hold ${mod}, then retry" >&2 + exit 1 + fi + fi + + echo " -- ${mod}: insmod" + if ! insmod "${ko_path}"; then + echo "build.sh: failed to insert rebuilt module ${mod} from ${ko_path}" >&2 + echo "build.sh: if the old module is still active, stop running tfork/podman containers and rerun this script" >&2 + exit 1 fi - insmod "${ko_path}" done echo " -- verify all modules loaded" diff --git a/criu/criu/action-scripts.c b/criu/criu/action-scripts.c index 6f7900186..3742d863b 100644 --- a/criu/criu/action-scripts.c +++ b/criu/criu/action-scripts.c @@ -32,6 +32,9 @@ static const char *action_names[ACT_MAX] = { [ACT_ORPHAN_PTS_MASTER] = "orphan-pts-master", [ACT_STATUS_READY] = "status-ready", [ACT_QUERY_EXT_FILES] = "query-ext-files", + [ACT_POST_TFORK_FREEZE] = "post-tfork-freeze", + [ACT_PRE_TFORK_RESTORE] = "pre-tfork-restore", + [ACT_TFORK_SOURCE_DETACHED] = "tfork-source-detached", }; struct script { diff --git a/criu/criu/clone-noasan.c b/criu/criu/clone-noasan.c index 4ba7f6f31..368f6aa1e 100644 --- a/criu/criu/clone-noasan.c +++ b/criu/criu/clone-noasan.c @@ -78,7 +78,10 @@ int clone3_with_pid_noasan(int (*fn)(void *), void *arg, int flags, int exit_sig c_args.flags = flags; c_args.set_tid = ptr_to_u64(&pid); c_args.set_tid_size = 1; + pr_info("clone3 set_tid pid=%d flags=0x%x size=1\n", pid, flags); pid = syscall(__NR_clone3, &c_args, sizeof(c_args)); + if (pid < 0) + pr_perror("clone3 set_tid failed flags=0x%x size=1", flags); if (pid == 0) exit(fn(arg)); return pid; @@ -99,6 +102,12 @@ int clone3_with_nested_pid_noasan(int (*fn)(void *), void *arg, int flags, int e BUG_ON(pid->ns_level > MAX_PID_NS_LEVEL || pid->ns_level <= 1); for (i = 0; i < pid->ns_level; i++) tids[i] = pid->ns[i].ns_pid; + pr_info("clone3 nested set_tid flags=0x%x size=%d tids=%d/%d/%d/%d\n", + flags, pid->ns_level, + tids[0], + pid->ns_level > 1 ? tids[1] : -1, + pid->ns_level > 2 ? tids[2] : -1, + pid->ns_level > 3 ? tids[3] : -1); if (!(flags & CLONE_PARENT)) { if (exit_signal != SIGCHLD) { @@ -112,6 +121,13 @@ int clone3_with_nested_pid_noasan(int (*fn)(void *), void *arg, int flags, int e c_args.set_tid = ptr_to_u64(tids); c_args.set_tid_size = pid->ns_level; pid_ret = syscall(__NR_clone3, &c_args, sizeof(c_args)); + if (pid_ret < 0) + pr_perror("clone3 nested set_tid failed flags=0x%x size=%d tids=%d/%d/%d/%d", + flags, pid->ns_level, + tids[0], + pid->ns_level > 1 ? tids[1] : -1, + pid->ns_level > 2 ? tids[2] : -1, + pid->ns_level > 3 ? tids[3] : -1); if (pid_ret == 0) exit(fn(arg)); return pid_ret; diff --git a/criu/criu/cr-dump.c b/criu/criu/cr-dump.c index d196521fc..0e9cf708e 100644 --- a/criu/criu/cr-dump.c +++ b/criu/criu/cr-dump.c @@ -6,6 +6,7 @@ #include #include #include +#include #include @@ -92,6 +93,94 @@ #include "timer.h" #include "sigact.h" +/* + * Opt-in tfork profiling. Keep the normal path to one cached branch per + * timing point and aggregate per-task intervals so profiling does not add a + * log write for every process and operation. + */ +struct tfork_task_profile { + uint64_t task_count; + uint64_t total; + uint64_t stat_identity; + uint64_t mappings; + uint64_t fds; + uint64_t proc_metadata; + uint64_t infect; + uint64_t parasite_metadata; + uint64_t image_ids; + uint64_t files; + uint64_t pages; + uint64_t signal_timers; + uint64_t core_cgroup; + uint64_t stop_threads_cure; + uint64_t mm_fs; + uint64_t cleanup; +}; + +static struct tfork_task_profile tfork_task_profile; + +static bool tfork_profile_enabled(void) +{ + static int enabled = -1; + const char *value; + + if (enabled >= 0) + return opts.tfork.active && enabled; + + value = getenv("CRIU_TFORK_PROFILE"); + enabled = value && value[0] && strcmp(value, "0"); + return opts.tfork.active && enabled; +} + +static uint64_t tfork_profile_now(void) +{ + struct timespec ts; + + if (!tfork_profile_enabled()) + return 0; + if (clock_gettime(CLOCK_MONOTONIC, &ts)) + return 0; + return (uint64_t)ts.tv_sec * 1000000000ULL + ts.tv_nsec; +} + +static void tfork_profile_add(uint64_t *total, uint64_t started) +{ + uint64_t now; + + if (!started) + return; + now = tfork_profile_now(); + if (now >= started) + *total += now - started; +} + +static void tfork_profile_dump_tasks(void) +{ + struct tfork_task_profile *p = &tfork_task_profile; + + if (!tfork_profile_enabled()) + return; + +#define TFORK_PROFILE_US(field) ((unsigned long long)(p->field / 1000ULL)) + pr_info("tfork-profile: phase=A tasks=%llu task_total_us=%llu " + "stat_identity_us=%llu mappings_us=%llu fds_us=%llu " + "proc_metadata_us=%llu\n", + (unsigned long long)p->task_count, TFORK_PROFILE_US(total), + TFORK_PROFILE_US(stat_identity), TFORK_PROFILE_US(mappings), + TFORK_PROFILE_US(fds), TFORK_PROFILE_US(proc_metadata)); + pr_info("tfork-profile: phase=A infect_us=%llu parasite_metadata_us=%llu " + "image_ids_us=%llu files_us=%llu pages_us=%llu " + "signal_timers_us=%llu\n", + TFORK_PROFILE_US(infect), TFORK_PROFILE_US(parasite_metadata), + TFORK_PROFILE_US(image_ids), TFORK_PROFILE_US(files), + TFORK_PROFILE_US(pages), TFORK_PROFILE_US(signal_timers)); + pr_info("tfork-profile: phase=A core_cgroup_us=%llu " + "stop_threads_cure_us=%llu mm_fs_us=%llu cleanup_us=%llu\n", + TFORK_PROFILE_US(core_cgroup), TFORK_PROFILE_US(stop_threads_cure), + TFORK_PROFILE_US(mm_fs), TFORK_PROFILE_US(cleanup)); +#undef TFORK_PROFILE_US +} + /* * Architectures can overwrite this function to restore register sets that * are not covered by ptrace_set/get_regs(). @@ -885,6 +974,7 @@ static int collect_pstree_ids_predump(void) * write_img_inventory(). */ + pid_init_dump(crt.i.pid, &crt.i); crt.i.pid->state = TASK_ALIVE; crt.i.pid->real = getpid(); @@ -1571,6 +1661,11 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) struct proc_posix_timers_stat proc_args; struct mem_dump_ctl mdc; unsigned long cflags; + uint64_t profile_task_started = tfork_profile_now(); + uint64_t profile_started = 0; + + if (profile_task_started) + tfork_task_profile.task_count++; vm_area_list_init(&vmas); @@ -1582,8 +1677,9 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) /* * zombies are dumped separately in dump_zombies() */ - return 0; + goto profiled_dead; + profile_started = tfork_profile_now(); pr_info("Obtaining task stat ... \n"); ret = parse_pid_stat(pid, &pps_buf); if (ret < 0) @@ -1640,13 +1736,17 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) pr_err("Dump TIME namespace (pid: %d) failed with %d\n", pid, ret); goto err; } + tfork_profile_add(&tfork_task_profile.stat_identity, profile_started); + profile_started = tfork_profile_now(); ret = collect_mappings(pid, &vmas, dump_filemap); if (ret) { pr_err("Collect mappings (pid: %d) failed with %d\n", pid, ret); goto err; } + tfork_profile_add(&tfork_task_profile.mappings, profile_started); + profile_started = tfork_profile_now(); if (!shared_fdtable(item)) { dfds = xmalloc(sizeof(*dfds)); if (!dfds) @@ -1660,7 +1760,9 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) parasite_ensure_args_size(drain_fds_size(dfds)); } + tfork_profile_add(&tfork_task_profile.fds, profile_started); + profile_started = tfork_profile_now(); ret = parse_posix_timers(pid, &proc_args); if (ret < 0) { pr_err("Can't read posix timers file (pid: %d)\n", pid); @@ -1680,12 +1782,15 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) pr_err("Dump %d rseq failed %d\n", pid, ret); goto err; } + tfork_profile_add(&tfork_task_profile.proc_metadata, profile_started); + profile_started = tfork_profile_now(); parasite_ctl = parasite_infect_seized(pid, item, &vmas); if (!parasite_ctl) { pr_err("Can't infect (pid: %d) with parasite\n", pid); goto err; } + tfork_profile_add(&tfork_task_profile.infect, profile_started); ret = fixup_thread_rseq(item, 0); if (ret) { @@ -1711,6 +1816,7 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) goto err_cure; } + profile_started = tfork_profile_now(); ret = parasite_fixup_vdso(parasite_ctl, pid, &vmas); if (ret) { pr_err("Can't fixup vdso VMAs (pid: %d)\n", pid); @@ -1728,7 +1834,9 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) pr_err("Can't dump misc (pid: %d)\n", pid); goto err_cure; } + tfork_profile_add(&tfork_task_profile.parasite_metadata, profile_started); + profile_started = tfork_profile_now(); cr_imgset = cr_task_imgset_open(uid(item), O_DUMP); if (!cr_imgset) goto err_cure; @@ -1738,7 +1846,9 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) pr_err("Dump ids (pid: %d) failed with %d\n", pid, ret); goto err_cure; } + tfork_profile_add(&tfork_task_profile.image_ids, profile_started); + profile_started = tfork_profile_now(); if (dfds) { ret = dump_task_files_seized(parasite_ctl, item, dfds); if (ret) { @@ -1751,12 +1861,14 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) goto err_cure; } } + tfork_profile_add(&tfork_task_profile.files, profile_started); mdc.pre_dump = false; mdc.lazy = opts.lazy_pages; mdc.stat = &pps_buf; mdc.parent_ie = parent_ie; + profile_started = tfork_profile_now(); if (!opts.tfork.active) { ret = parasite_dump_pages_seized(item, &vmas, &mdc, parasite_ctl); if (ret) @@ -1769,7 +1881,9 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) if (ret) goto err_cure; } + tfork_profile_add(&tfork_task_profile.pages, profile_started); + profile_started = tfork_profile_now(); ret = parasite_dump_sigacts_seized(parasite_ctl, item); if (ret) { pr_err("Can't dump sigactions (pid: %d) with parasite\n", pid); @@ -1787,7 +1901,9 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) pr_err("Can't dump posix timers (pid: %d)\n", pid); goto err_cure; } + tfork_profile_add(&tfork_task_profile.signal_timers, profile_started); + profile_started = tfork_profile_now(); ret = dump_task_core_all(parasite_ctl, item, &pps_buf, cr_imgset, &misc); if (ret) { pr_err("Dump core (pid: %d) failed with %d\n", pid, ret); @@ -1799,7 +1915,9 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) pr_err("Dump cgroup of threads in process (pid: %d) failed with %d\n", pid, ret); goto err_cure; } + tfork_profile_add(&tfork_task_profile.core_cgroup, profile_started); + profile_started = tfork_profile_now(); ret = compel_stop_daemon(parasite_ctl); if (ret) { pr_err("Can't stop daemon in parasite (pid: %d)\n", pid); @@ -1824,7 +1942,9 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) pr_err("Can't cure (pid: %d) from parasite\n", pid); goto err; } + tfork_profile_add(&tfork_task_profile.stop_threads_cure, profile_started); + profile_started = tfork_profile_now(); ret = dump_task_mm(pid, &pps_buf, &misc, &vmas, cr_imgset); if (ret) { pr_err("Dump mappings (pid: %d) failed with %d\n", pid, ret); @@ -1836,13 +1956,17 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) pr_err("Dump fs (pid: %d) failed with %d\n", pid, ret); goto err; } + tfork_profile_add(&tfork_task_profile.mm_fs, profile_started); exit_code = 0; err: + profile_started = tfork_profile_now(); close_cr_imgset(&cr_imgset); close_pid_proc(); free_mappings(&vmas); xfree(dfds); + tfork_profile_add(&tfork_task_profile.cleanup, profile_started); + tfork_profile_add(&tfork_task_profile.total, profile_task_started); return exit_code; err_cure: @@ -1850,6 +1974,10 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) if (ret) pr_err("Can't cure (pid: %d) from parasite\n", pid); goto err; + +profiled_dead: + tfork_profile_add(&tfork_task_profile.total, profile_task_started); + return 0; } static int alarm_attempts = 0; @@ -2323,9 +2451,12 @@ int cr_dump_tasks(pid_t pid) if (collect_and_suspend_lsm() < 0) goto err; + if (tfork_profile_enabled()) + memset(&tfork_task_profile, 0, sizeof(tfork_task_profile)); for_each_pstree_item(item) if (dump_one_task(item, parent_ie)) goto err; + tfork_profile_dump_tasks(); if (!opts.tfork.active) { ret = run_plugins(DUMP_DEVICES_LATE, pid); @@ -2360,6 +2491,9 @@ int cr_dump_tasks(pid_t pid) if (dump_zombies()) goto err; + if (finalize_nested_pid_ns_ids()) + goto err; + if (dump_pstree(root_item)) goto err; diff --git a/criu/criu/cr-restore.c b/criu/criu/cr-restore.c index d85279a82..f3886f505 100644 --- a/criu/criu/cr-restore.c +++ b/criu/criu/cr-restore.c @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -117,6 +118,55 @@ #define arch_export_restore_task __export_restore_task #endif +static bool tfork_restore_profile; +static uint64_t tfork_restore_profile_origin; +static uint64_t tfork_restore_profile_last; +static unsigned int tfork_restore_wait_seq; + +static uint64_t tfork_restore_profile_now(void) +{ + struct timespec ts; + + if (clock_gettime(CLOCK_MONOTONIC, &ts)) + return 0; + return (uint64_t)ts.tv_sec * 1000000000ULL + ts.tv_nsec; +} + +static void tfork_restore_profile_init(void) +{ + const char *value = getenv("CRIU_TFORK_PROFILE"); + + tfork_restore_profile = opts.tfork.active && value && value[0] && + strcmp(value, "0"); + if (!tfork_restore_profile) + return; + tfork_restore_profile_origin = tfork_restore_profile_now(); + tfork_restore_profile_last = tfork_restore_profile_origin; + tfork_restore_wait_seq = 0; + pr_info("tfork-profile: phase=B-restore mark=start pid=%d\n", getpid()); +} + +static void tfork_restore_profile_mark(const char *mark) +{ + uint64_t now; + uint64_t delta = 0; + uint64_t elapsed = 0; + + if (!tfork_restore_profile) + return; + now = tfork_restore_profile_now(); + if (now >= tfork_restore_profile_last) { + delta = now - tfork_restore_profile_last; + tfork_restore_profile_last = now; + } + if (now >= tfork_restore_profile_origin) + elapsed = now - tfork_restore_profile_origin; + pr_info("tfork-profile: phase=B-restore mark=%s pid=%d delta_us=%llu elapsed_us=%llu\n", + mark, getpid(), + (unsigned long long)(delta / 1000ULL), + (unsigned long long)(elapsed / 1000ULL)); +} + #ifndef arch_export_unmap #define arch_export_unmap __export_unmap #define arch_export_unmap_compat __export_unmap_compat @@ -185,10 +235,99 @@ static int __restore_wait_inprogress_tasks(int participants) { int ret; futex_t *np = &task_entries->nr_in_progress; + const int tfork_restore_wait_timeout_ms = 10000; + uint64_t profile_started = 0; + unsigned int profile_seq = 0; + int profile_initial = 0; + + if (tfork_restore_profile) { + profile_started = tfork_restore_profile_now(); + profile_seq = ++tfork_restore_wait_seq; + profile_initial = (int)futex_get(np); + } + + if (opts.tfork.active) { + struct timespec started, now, timeout; + int wait_ret = 0; + + /* + * All paths that decrement this barrier use + * futex_dec_and_wake(). Waiting on the observed value avoids + * paying up to one 100ms polling interval at every restore stage. + */ + if (clock_gettime(CLOCK_MONOTONIC, &started)) { + pr_perror("tfork restore wait: clock_gettime"); + return -errno; + } + while ((int)futex_get(np) > participants) { + int64_t elapsed_ns, remaining_ns; + uint32_t observed = futex_get(np); + + if (observed & FUTEX_ABORT_FLAG) + break; + if (clock_gettime(CLOCK_MONOTONIC, &now)) { + pr_perror("tfork restore wait: clock_gettime"); + return -errno; + } + elapsed_ns = + (int64_t)(now.tv_sec - started.tv_sec) * NSEC_PER_SEC + + (now.tv_nsec - started.tv_nsec); + remaining_ns = + (int64_t)tfork_restore_wait_timeout_ms * 1000000 - + elapsed_ns; + if (remaining_ns <= 0) { + wait_ret = -ETIMEDOUT; + break; + } + timeout.tv_sec = remaining_ns / NSEC_PER_SEC; + timeout.tv_nsec = remaining_ns % NSEC_PER_SEC; + wait_ret = sys_futex( + (uint32_t *)&np->raw.counter, FUTEX_WAIT, + observed, &timeout, NULL, 0); + if (wait_ret == 0 || wait_ret == -EINTR || + wait_ret == -EWOULDBLOCK) + continue; + if (wait_ret == -ETIMEDOUT) + break; + pr_err("tfork restore futex wait failed: %d\n", wait_ret); + set_cr_errno(-wait_ret); + return wait_ret; + } + + if (wait_ret == -ETIMEDOUT && + (int)futex_get(np) > participants) { + pr_err("tfork restore wait timed out after %dms: participants=%d nr_in_progress=%d start_stage=%d task_cr_err=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + tfork_restore_wait_timeout_ms, + participants, (int)futex_get(np), + (int)futex_get(&task_entries->start), + get_task_cr_err(), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + set_cr_errno(ETIMEDOUT); + return -ETIMEDOUT; + } + } else { + futex_wait_while_gt(np, participants); + } + + if (profile_started) { + uint64_t now = tfork_restore_profile_now(); + uint64_t elapsed = 0; + + if (now >= profile_started) + elapsed = now - profile_started; + pr_info("tfork-profile: phase=B-wait seq=%u pid=%d stage=%d " + "participants=%d initial=%d final=%d duration_us=%llu\n", + profile_seq, getpid(), (int)futex_get(&task_entries->start), + participants, profile_initial, (int)futex_get(np), + (unsigned long long)(elapsed / 1000ULL)); + } - futex_wait_while_gt(np, participants); ret = (int)futex_get(np); if (ret < 0) { + pr_err("restore wait aborted: participants=%d nr_in_progress=%d start_stage=%d task_cr_err=%d\n", + participants, ret, (int)futex_get(&task_entries->start), + get_task_cr_err()); set_cr_errno(get_task_cr_err()); return ret; } @@ -227,6 +366,17 @@ static inline void __restore_switch_stage(int next_stage) static int restore_switch_stage(int next_stage) { + if (opts.tfork.active) + pr_warn("tfork: restore_switch_stage %d participants=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + next_stage, stage_participants(next_stage), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + else + pr_info("restore_switch_stage %d participants=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + next_stage, stage_participants(next_stage), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + __restore_switch_stage(next_stage); return restore_wait_inprogress_tasks(); } @@ -431,7 +581,29 @@ static int populate_pid_proc(void) return 0; } -static int __collect_child_pids(struct pstree_item *p, int state, unsigned int *n) +/* + * The pid of `pi` as seen by `p`. ns[0] is innermost, so stepping in + * (child depth - p's depth) entries gives p's view. localpid() only + * matches when both live in the same namespace: a nested pidns init is 1 + * to itself and something else entirely to its parent. + */ +static pid_t pid_in_parent_ns(struct pstree_item *p, struct pstree_item *pi) +{ + int idx = pi->pid->ns_level - p->pid->ns_level; + + if (idx <= 0 || idx >= pi->pid->ns_level) + return localpid(pi); + + return pi->pid->ns[idx].ns_pid; +} + +/* + * `observer` is whoever waits on these children, which is not always p: + * children of helpers and zombies are reparented to init. Pids are + * recorded in the observer's namespace. + */ +static int __collect_child_pids(struct pstree_item *p, struct pstree_item *observer, int state, + unsigned int *n) { struct pstree_item *pi; @@ -446,7 +618,15 @@ static int __collect_child_pids(struct pstree_item *p, int state, unsigned int * return -1; (*n)++; - *child = localpid(pi); + /* + * Zombies only. A helper's pid chain is synthesised by + * get_or_create_helper_item(), so translating it through ns[] + * would yield a pid that was never real. + */ + if (state == TASK_DEAD) + *child = pid_in_parent_ns(observer, pi); + else + *child = localpid(pi); } return 0; @@ -467,12 +647,12 @@ static int collect_child_pids(int state, unsigned int *n) for_each_pstree_item(pi) { if (pi->pid->state != TASK_HELPER && pi->pid->state != TASK_DEAD) continue; - if (__collect_child_pids(pi, state, n)) + if (__collect_child_pids(pi, current, state, n)) return -1; } } - return __collect_child_pids(current, state, n); + return __collect_child_pids(current, current, state, n); } static int collect_helper_pids(struct task_restore_args *ta) @@ -1303,14 +1483,15 @@ static int set_next_pid(void *arg) return 0; } -static inline int fork_with_pid(struct pstree_item *item) +static inline int fork_with_pid_mode(struct pstree_item *item, bool parallel_sibling) { struct cr_clone_arg ca; struct ns_id *pid_ns = NULL; bool external_pidns = false; + bool last_pid_locked = false; int ret = -1; pid_t pid = localpid(item); - unsigned long strip; + unsigned long strip, syscall_clone_flags; if (item->pid->state != TASK_HELPER) { if (open_core(uid(item), &ca.core)) @@ -1395,13 +1576,38 @@ static inline int fork_with_pid(struct pstree_item *item) ca.item = item; ca.clone_flags = rsti(item)->clone_flags; + /* + * Innermost pid 1 below the restore root means a nested pidns init, + * which needs CLONE_NEWPID. A zombie has no ids image, so + * get_clone_mask() cannot derive the flag and clone3() would apply + * the set_tid chain against the parent's namespace and hit EEXIST. + * Not tfork-specific: plain dump/restore breaks the same way. + */ + if (item != root_item && + !(ca.clone_flags & CLONE_NEWPID) && + item->pid->ns_level > 1 && + item->pid->ns[0].ns_pid == INIT_PID) { + pr_info("repairing missing CLONE_NEWPID for pidns init uid=%d local=%d parent_local=%d level=%d\n", + uid(item), pid, + item->parent ? localpid(item->parent) : -1, + item->pid->ns_level); + ca.clone_flags |= CLONE_NEWPID; + } BUG_ON(ca.clone_flags & CLONE_VM); pr_info("Forking task with %d(%d) (flags 0x%lx)\n", realpid(item), pid, ca.clone_flags); if (!(ca.clone_flags & CLONE_NEWPID)) { - lock_last_pid(); + /* + * clone3(set_tid) reserves the requested PID atomically. Parallel + * sibling helpers must not serialize on the legacy ns_last_pid lock; + * the lock remains mandatory for the fallback set_next_pid path. + */ + if (!parallel_sibling || !kdat.has_clone3_set_tid) { + lock_last_pid(); + last_pid_locked = true; + } if (!kdat.has_clone3_set_tid) { if (external_pidns) { @@ -1432,14 +1638,17 @@ static inline int fork_with_pid(struct pstree_item *item) strip = CLONE_NEWNET | CLONE_NEWCGROUP | CLONE_NEWTIME; if (!(item == root_item && is_simple_userns_tree())) strip |= CLONE_NEWUSER; + syscall_clone_flags = ca.clone_flags; + if (parallel_sibling) + syscall_clone_flags |= CLONE_PARENT; if (kdat.has_clone3_set_tid) { if (item->pid->ns_level == 1) ret = clone3_with_pid_noasan(restore_task_with_children, &ca, - ca.clone_flags & ~strip, SIGCHLD, pid); + syscall_clone_flags & ~strip, SIGCHLD, pid); else ret = clone3_with_nested_pid_noasan(restore_task_with_children, &ca, - ca.clone_flags & ~strip, + syscall_clone_flags & ~strip, SIGCHLD, item->pid); } else { BUG_ON(item->pid->ns_level >= 1); @@ -1448,6 +1657,19 @@ static inline int fork_with_pid(struct pstree_item *item) (ca.clone_flags & ~strip) | SIGCHLD, &ca); } if (ret < 0) { + pr_err("fork_with_pid failed item uid=%d local=%d real=%d parent_local=%d flags=0x%lx stripped_flags=0x%lx ns_level=%d root_ns_mask=0x%lx tfork=%d\n", + uid(item), pid, realpid(item), + item->parent ? localpid(item->parent) : -1, + ca.clone_flags, ca.clone_flags & ~strip, + item->pid->ns_level, root_ns_mask, + opts.tfork.active ? 1 : 0); + if (item->pid->ns_level > 0) + pr_err("fork_with_pid pid chain uid=%d ns=%d/%d/%d/%d\n", + uid(item), + item->pid->ns[0].ns_pid, + item->pid->ns_level > 1 ? item->pid->ns[1].ns_pid : -1, + item->pid->ns_level > 2 ? item->pid->ns[2].ns_pid : -1, + item->pid->ns_level > 3 ? item->pid->ns[3].ns_pid : -1); pr_perror("Can't fork for %d", pid); if (errno == EEXIST) set_cr_errno(EEXIST); @@ -1462,7 +1684,7 @@ static inline int fork_with_pid(struct pstree_item *item) arch_shstk_unlock(item, ca.core, ret); err_unlock: - if (!(ca.clone_flags & CLONE_NEWPID)) + if (last_pid_locked) unlock_last_pid(); if (ca.core) @@ -1470,6 +1692,11 @@ static inline int fork_with_pid(struct pstree_item *item) return ret; } +static inline int fork_with_pid(struct pstree_item *item) +{ + return fork_with_pid_mode(item, false); +} + static pid_t userns_maps_helper_pid = -1; static bool item_parent_uns_is_host(struct pstree_item *item) @@ -1739,6 +1966,161 @@ static int mount_proc(void) return ret; } +#define TFORK_PARALLEL_SIBLING_MAX_WORKERS 16 +#define TFORK_PARALLEL_SIBLING_MIN_CHILDREN 32 + +struct tfork_parallel_sibling_arg { + int worker_index; + int worker_count; + bool before_setsid; +}; + +static int tfork_parallel_sibling_worker_count(void) +{ + const char *value; + char *end = NULL; + long workers; + + /* + * Restrict the prototype to one bounded helper pool for root children. + * Recursing into every wide subtree can multiply helpers without bound. + */ + if (!opts.tfork.active || current != root_item) + return 0; + value = getenv("CRIU_TFORK_PARALLEL_SIBLINGS"); + if (!value || !value[0]) + return 0; + workers = strtol(value, &end, 10); + if (!*end && workers == 0) + return 0; + if (*end || workers < 2) { + pr_warn("tfork: ignoring invalid CRIU_TFORK_PARALLEL_SIBLINGS=%s\n", value); + return 0; + } + if (workers > TFORK_PARALLEL_SIBLING_MAX_WORKERS) + workers = TFORK_PARALLEL_SIBLING_MAX_WORKERS; + return workers; +} + +static bool tfork_parallel_sibling_matches(struct pstree_item *child, bool before_setsid) +{ + return restore_before_setsid(child) == before_setsid; +} + +static int tfork_parallel_sibling_main(void *opaque) +{ + struct tfork_parallel_sibling_arg *arg = opaque; + struct pstree_item *child; + sigset_t unblock; + int ordinal = 0; + + /* The root blocks SIGCHLD while it owns/reaps the temporary helpers. */ + sigemptyset(&unblock); + sigaddset(&unblock, SIGCHLD); + if (sigprocmask(SIG_UNBLOCK, &unblock, NULL)) { + pr_perror("tfork: parallel sibling helper cannot unblock SIGCHLD"); + return 1; + } + + list_for_each_entry(child, ¤t->children, sibling) { + if (!tfork_parallel_sibling_matches(child, arg->before_setsid)) + continue; + if ((ordinal++ % arg->worker_count) != arg->worker_index) + continue; + if (arg->before_setsid) + BUG_ON(child->born_sid != -1 && getsid(0) != child->born_sid); + if (fork_with_pid_mode(child, true) < 0) + return 1; + } + return 0; +} + +/* + * Return 0 when the matching children were created, 1 when the guarded path + * is ineligible (the caller should use the serial loop), and -1 after a + * partial/failed parallel attempt. + */ +static int tfork_create_siblings_parallel(bool before_setsid, int requested_workers) +{ + struct tfork_parallel_sibling_arg args[TFORK_PARALLEL_SIBLING_MAX_WORKERS]; + pid_t helpers[TFORK_PARALLEL_SIBLING_MAX_WORKERS] = {}; + struct pstree_item *child; + sigset_t oldmask; + int child_count = 0, workers, launched = 0, i, ret = -1; + + if (!kdat.has_clone3_set_tid) + return 1; + + list_for_each_entry(child, ¤t->children, sibling) { + if (!tfork_parallel_sibling_matches(child, before_setsid)) + continue; + child_count++; + /* + * CLONE_PARENT is added only to the helper's clone3 syscall so + * that the restored child remains a child of current. Nested PID + * namespace creation and pre-existing CLONE_PARENT semantics need + * a separate dependency proof and stay on the serial path. + */ + if (rsti(child)->clone_flags & (CLONE_PARENT | CLONE_NEWPID | CLONE_VM | CLONE_THREAD)) + return 1; + } + /* Keep small process trees on the cheaper and better-tested serial path. */ + if (child_count < TFORK_PARALLEL_SIBLING_MIN_CHILDREN) + return 1; + + workers = requested_workers; + if (workers > child_count) + workers = child_count; + if (workers > TFORK_PARALLEL_SIBLING_MAX_WORKERS) + workers = TFORK_PARALLEL_SIBLING_MAX_WORKERS; + + if (block_sigmask(&oldmask, SIGCHLD)) + return -1; + + pr_info("tfork: creating %d %s-setsid siblings with %d temporary helpers\n", + child_count, before_setsid ? "pre" : "post", workers); + for (i = 0; i < workers; i++) { + pid_t helper_pid = pstree_get_free_pid(current); + + args[i].worker_index = i; + args[i].worker_count = workers; + args[i].before_setsid = before_setsid; + helpers[i] = clone3_with_pid_noasan(tfork_parallel_sibling_main, &args[i], + 0, SIGCHLD, helper_pid); + if (helpers[i] < 0) { + pr_perror("tfork: cannot create parallel sibling helper at vpid %d", helper_pid); + goto kill_helpers; + } + launched++; + } + + ret = 0; + for (i = 0; i < launched; i++) { + int status = 0; + pid_t waited; + + do { + waited = waitpid(helpers[i], &status, 0); + } while (waited < 0 && errno == EINTR); + if (waited != helpers[i] || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + pr_err("tfork: parallel sibling helper %d failed (waited=%d status=0x%x)\n", + helpers[i], waited, status); + ret = -1; + } + } + if (restore_sigmask(&oldmask)) + ret = -1; + return ret; + +kill_helpers: + for (i = 0; i < launched; i++) + kill(helpers[i], SIGKILL); + for (i = 0; i < launched; i++) + waitpid(helpers[i], NULL, 0); + restore_sigmask(&oldmask); + return -1; +} + /* * Tasks cannot change sid (session id) arbitrary, but can either * inherit one from ancestor, or create a new one with id equal to @@ -1749,30 +2131,43 @@ static int create_children_and_session(void) { int ret; struct pstree_item *child; + int parallel_workers = tfork_parallel_sibling_worker_count(); pr_info("Restoring children in alien sessions:\n"); - list_for_each_entry(child, ¤t->children, sibling) { - if (!restore_before_setsid(child)) - continue; + ret = parallel_workers > 1 ? + tfork_create_siblings_parallel(true, parallel_workers) : 1; + if (ret < 0) + return ret; + if (ret > 0) { + list_for_each_entry(child, ¤t->children, sibling) { + if (!restore_before_setsid(child)) + continue; - BUG_ON(child->born_sid != -1 && getsid(0) != child->born_sid); + BUG_ON(child->born_sid != -1 && getsid(0) != child->born_sid); - ret = fork_with_pid(child); - if (ret < 0) - return ret; + ret = fork_with_pid(child); + if (ret < 0) + return ret; + } } if (current->parent) restore_sid(); pr_info("Restoring children in our session:\n"); - list_for_each_entry(child, ¤t->children, sibling) { - if (restore_before_setsid(child)) - continue; + ret = parallel_workers > 1 ? + tfork_create_siblings_parallel(false, parallel_workers) : 1; + if (ret < 0) + return ret; + if (ret > 0) { + list_for_each_entry(child, ¤t->children, sibling) { + if (restore_before_setsid(child)) + continue; - ret = fork_with_pid(child); - if (ret < 0) - return ret; + ret = fork_with_pid(child); + if (ret < 0) + return ret; + } } return 0; @@ -2431,6 +2826,7 @@ static int restore_root_task(struct pstree_item *init) pr_err("Failed to prepare namespace before tasks\n"); return -1; } + tfork_restore_profile_mark("root-pre-restore-and-namespace-prep"); if (localpid(init) == INIT_PID) { if (!(root_ns_mask & CLONE_NEWPID)) { @@ -2480,6 +2876,7 @@ static int restore_root_task(struct pstree_item *init) pr_err("fork_with_pid failed: %d\n", ret); goto out; } + tfork_restore_profile_mark("fork-root-task"); if (is_simple_userns_tree()) { if (prepare_userns(init)) { @@ -2538,6 +2935,7 @@ static int restore_root_task(struct pstree_item *init) pr_err("restore_wait_inprogress_tasks failed: %d\n", ret); goto out_kill; } + tfork_restore_profile_mark("wait-namespaces-created"); ret = run_scripts(ACT_SETUP_NS); if (ret) { @@ -2554,6 +2952,7 @@ static int restore_root_task(struct pstree_item *init) pr_err("Root task logs above show which step failed (err_step).\n"); goto out_kill; } + tfork_restore_profile_mark("prepare-namespaces-stage"); if (root_ns_mask & CLONE_NEWNS) { mnt_ns_fd = open_proc(init->pid->real, "ns/mnt"); @@ -2598,6 +2997,7 @@ static int restore_root_task(struct pstree_item *init) pr_err("restore_wait_inprogress_tasks (post-fork) failed: %d\n", ret); goto out_kill; } + tfork_restore_profile_mark("post-fork-wait"); ret = apply_memfd_seals(); if (ret < 0) { @@ -2641,6 +3041,7 @@ static int restore_root_task(struct pstree_item *init) pr_err("restore_switch_stage RESTORE_SIGCHLD failed: %d\n", ret); goto out_kill; } + tfork_restore_profile_mark("restore-sigchld-stage"); ret = stop_usernsd(); if (ret < 0) { @@ -2696,8 +3097,10 @@ static int restore_root_task(struct pstree_item *init) pr_err("write_restored_pid failed\n"); goto out_kill; } + tfork_restore_profile_mark("post-restore-housekeeping"); network_unlock(); + tfork_restore_profile_mark("network-unlock"); /* * Stop getting sigchld, after we resume the tasks they @@ -2714,11 +3117,13 @@ static int restore_root_task(struct pstree_item *init) pr_err("attach_to_tasks failed\n"); goto out_kill_network_unlocked; } + tfork_restore_profile_mark("attach-restored-tasks"); if (restore_switch_stage(CR_STATE_RESTORE_CREDS)) { pr_err("restore_switch_stage RESTORE_CREDS failed\n"); goto out_kill_network_unlocked; } + tfork_restore_profile_mark("restore-creds-stage"); timing_stop(TIME_RESTORE); @@ -2733,6 +3138,7 @@ static int restore_root_task(struct pstree_item *init) } __restore_switch_stage(CR_STATE_COMPLETE); + tfork_restore_profile_mark("catch-lazy-and-complete"); ret = compel_stop_on_syscall(task_entries->nr_threads, __NR(rt_sigreturn, 0), __NR(rt_sigreturn, 1)); if (ret) { @@ -2745,6 +3151,7 @@ static int restore_root_task(struct pstree_item *init) /* just before releasing threads we have to restore rseq_cs */ if (restore_rseq_cs()) pr_err("Unable to restore rseq_cs state\n"); + tfork_restore_profile_mark("stop-finalize-and-rseq"); /* * Some external devices such as GPUs might need a very late @@ -2784,6 +3191,7 @@ static int restore_root_task(struct pstree_item *init) pr_err("finalize_restore_detach failed\n"); goto out_kill_network_unlocked; } + tfork_restore_profile_mark("hooks-freezer-and-detach"); pr_info("Restore finished successfully. Tasks resumed.\n"); write_stats(RESTORE_STATS); @@ -2902,6 +3310,7 @@ int cr_restore_tasks(void) if (init_service_fd()) return 1; + tfork_restore_profile_init(); if (check_async_memdump_inflight() < 0) return -1; @@ -2913,6 +3322,7 @@ int cr_restore_tasks(void) if (tfork_read_cropt()) return -1; } + tfork_restore_profile_mark("inventory-and-cropt"); if (init_stats(RESTORE_STATS)) return -1; @@ -2958,6 +3368,7 @@ int cr_restore_tasks(void) } } } + tfork_restore_profile_mark("task-entries-and-pstree"); if (fdstore_init()) return -1; @@ -2976,6 +3387,7 @@ int cr_restore_tasks(void) if (crtools_prepare_shared() < 0) goto err; + tfork_restore_profile_mark("fdstore-plugins-and-shared"); if (prepare_cgroup()) goto clean_cgroup; @@ -2985,8 +3397,10 @@ int cr_restore_tasks(void) if (prepare_lazy_pages_socket() < 0) goto clean_cgroup; + tfork_restore_profile_mark("cgroup-signals-and-lazy-pages"); ret = restore_root_task(root_item); + tfork_restore_profile_mark("restore-root-returned"); clean_cgroup: fini_cgroup(); err: @@ -4045,6 +4459,7 @@ static int sigreturn_restore(struct task_restore_args *task_args, unsigned long task_args->vdso_rt_size = vdso_rt_size; task_args->can_map_vdso = kdat.can_map_vdso; task_args->has_clone3_set_tid = kdat.has_clone3_set_tid; + task_args->tfork_active = opts.tfork.active; new_sp = restorer_stack(task_args->t->mz); diff --git a/criu/criu/cr-tfork.c b/criu/criu/cr-tfork.c index ea64d6627..f07ce9440 100644 --- a/criu/criu/cr-tfork.c +++ b/criu/criu/cr-tfork.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -44,6 +45,51 @@ #define VMA_CHERRYPICK_FD_ENV "CRIU_VMA_CHERRYPICK_FD" +static bool tfork_parent_profile; +static uint64_t tfork_parent_profile_origin; +static uint64_t tfork_parent_profile_last; + +static uint64_t tfork_parent_profile_now(void) +{ + struct timespec ts; + + if (clock_gettime(CLOCK_MONOTONIC, &ts)) + return 0; + return (uint64_t)ts.tv_sec * 1000000000ULL + ts.tv_nsec; +} + +static void tfork_parent_profile_init(void) +{ + const char *value = getenv("CRIU_TFORK_PROFILE"); + + tfork_parent_profile = value && value[0] && strcmp(value, "0"); + if (!tfork_parent_profile) + return; + tfork_parent_profile_origin = tfork_parent_profile_now(); + tfork_parent_profile_last = tfork_parent_profile_origin; +} + +static void tfork_parent_profile_mark(const char *mark) +{ + uint64_t now; + uint64_t delta = 0; + uint64_t elapsed = 0; + + if (!tfork_parent_profile) + return; + now = tfork_parent_profile_now(); + if (now >= tfork_parent_profile_last) { + delta = now - tfork_parent_profile_last; + tfork_parent_profile_last = now; + } + if (now >= tfork_parent_profile_origin) + elapsed = now - tfork_parent_profile_origin; + pr_info("tfork-profile: phase=B-parent mark=%s delta_us=%llu elapsed_us=%llu\n", + mark, + (unsigned long long)(delta / 1000ULL), + (unsigned long long)(elapsed / 1000ULL)); +} + static int tfork_dup_inherited_vma_cherrypick(int env_fd) { struct stat fst, dst; @@ -271,7 +317,7 @@ int tfork_read_cropt(void) return 0; snap_path = opts.tfork.snap_root; - if (!snap_path && opts.tfork.copies > 1 && opts.tfork.snap_roots && + if (!snap_path && opts.tfork.copies >= 1 && opts.tfork.snap_roots && opts.tfork.copy_idx < opts.tfork.snap_roots_n) snap_path = opts.tfork.snap_roots[opts.tfork.copy_idx]; @@ -573,6 +619,7 @@ static int cr_tfork_finish(int ret) { int j; + tfork_parent_profile_mark("finish-enter"); for (j = 0; j < opts.tfork.pidfd_map_nr; j++) { if (opts.tfork.pidfd_map[j].pidfd >= 0) close(opts.tfork.pidfd_map[j].pidfd); @@ -600,6 +647,7 @@ static int cr_tfork_finish(int ret) if (bfd_flush_images()) ret = -1; + tfork_parent_profile_mark("finish-close-fds-and-flush"); cgp_fini(); @@ -607,6 +655,7 @@ static int cr_tfork_finish(int ret) network_unlock(); delete_link_remaps(); clean_cr_time_mounts(); + tfork_parent_profile_mark("finish-unlock-and-clean-mounts"); cr_plugin_fini(CR_PLUGIN_STAGE__DUMP, ret); @@ -615,6 +664,9 @@ static int cr_tfork_finish(int ret) pstree_switch_state(root_item, TASK_ALIVE); timing_stop(TIME_FROZEN); + tfork_parent_profile_mark("finish-unseize-source"); + if (run_scripts(ACT_TFORK_SOURCE_DETACHED)) + pr_warn("tfork: source-detached notification failed\n"); seccomp_free_entries(); free_file_locks(); @@ -624,6 +676,7 @@ static int cr_tfork_finish(int ret) close_service_fd(CR_PROC_FD_OFF); close_image_dir(); + tfork_parent_profile_mark("finish-free-and-close"); if (ret) { pr_err("tfork FAILED.\n"); @@ -939,10 +992,12 @@ int cr_tfork_tasks(pid_t pid) } pr_info("tfork: Phase B — setting up clone restore\n"); + tfork_parent_profile_init(); opts.tfork.vma_cherrypick_fd = tfork_open_vma_cherrypick(); if (opts.tfork.vma_cherrypick_fd < 0) goto err; + tfork_parent_profile_mark("open-vma-cherrypick"); nr = 0; for_each_pstree_item(item) @@ -966,15 +1021,18 @@ int cr_tfork_tasks(pid_t pid) opts.tfork.pidfd_map[opts.tfork.pidfd_map_nr].pidfd = pidfd; opts.tfork.pidfd_map[opts.tfork.pidfd_map_nr].memfd = -1; opts.tfork.pidfd_map_nr++; - pr_info("tfork: pidfd %d for pid %d (vpid %d uid %d)\n", - pidfd, item->pid->real, localpid(item), uid(item)); + pr_info("tfork: pidfd %d for pid %d (vpid %d uid %d nsid %d level %d)\n", + pidfd, item->pid->real, localpid(item), uid(item), + item->pid->leaf_ns_id, item->pid->ns_level); } + tfork_parent_profile_mark("pidfd-map"); ret = run_scripts(ACT_PRE_TFORK_RESTORE); if (ret) { pr_err("Pre-tfork-restore script failed: %d\n", ret); goto err; } + tfork_parent_profile_mark("pre-restore-hook"); img_dir_fd = get_service_fd(IMG_FD_OFF); cropt_fd = openat(img_dir_fd, "tfork.cropt", @@ -995,6 +1053,7 @@ int cr_tfork_tasks(pid_t pid) opts.tfork.pidfd_map[j].vpid, (int)opts.tfork.pidfd_map[j].real_pid); fclose(f); + tfork_parent_profile_mark("write-cropt"); if (opts.output) { char phasea_path[PATH_MAX]; @@ -1016,6 +1075,7 @@ int cr_tfork_tasks(pid_t pid) if (dst >= 0) close(dst); } + tfork_parent_profile_mark("copy-phase-a-log"); child = fork(); if (child < 0) { @@ -1023,6 +1083,8 @@ int cr_tfork_tasks(pid_t pid) ret = -1; goto err; } + if (child > 0) + tfork_parent_profile_mark("fork-restore-child"); if (child == 0) { char img_dir_arg[PATH_MAX]; @@ -1171,7 +1233,7 @@ int cr_tfork_tasks(pid_t pid) rpc_argv[rpc_n++] = "--tfork-snap-mounts"; rpc_argv[rpc_n++] = snap_mounts_csv; } - if (opts.tfork.copies > 1) { + if (opts.tfork.copies >= 1) { snprintf(copies_arg, sizeof(copies_arg), "%d", opts.tfork.copies); rpc_argv[rpc_n++] = "--tfork-copies"; @@ -1312,6 +1374,10 @@ int cr_tfork_tasks(pid_t pid) argv_new[argc_new++] = "--pidfile"; argv_new[argc_new++] = pidfile_arg; } + if ((size_t)argc_new >= argv_max) { + pr_err("tfork restore argv overflow: used=%d max=%zu\n", argc_new, argv_max); + exit(1); + } argv_new[argc_new] = NULL; execv("/proc/self/exe", argv_new); @@ -1333,6 +1399,7 @@ int cr_tfork_tasks(pid_t pid) WTERMSIG(status)); ret = -1; } + tfork_parent_profile_mark("wait-restore-child"); err: return cr_tfork_finish(ret); diff --git a/criu/criu/crtools.c b/criu/criu/crtools.c index df40ec14f..639ad06f5 100644 --- a/criu/criu/crtools.c +++ b/criu/criu/crtools.c @@ -332,25 +332,30 @@ int main(int argc, char *argv[], char *envp[]) if (opts.tree_id) pr_warn("Using -t with criu restore is obsoleted\n"); - if (opts.tfork.copies > 1) { + if (!opts.tfork.active && opts.tfork.copies >= 1) { + pr_err("--tfork-copies requires --tfork-restore " + "(use 'criu tfork --tfork-copies=N', not " + "'criu restore --tfork-copies=N')\n"); + return 1; + } + + if (opts.tfork.active && opts.tfork.copies >= 1) { int n = opts.tfork.copies, i; pid_t *children; int (*ready_pipes)[2]; int failed = 0; const char *base_log = opts.output; - const int ns_flags = - CLONE_NEWPID | CLONE_NEWNS; - - if (!opts.tfork.active) { - pr_err("--tfork-copies>1 requires --tfork-restore " - "(use 'criu tfork --tfork-copies=N', not " - "'criu restore --tfork-copies=N')\n"); - return 1; - } + /* + * The n-copy helper must not create/occupy PID 1 in a new + * PID namespace. CRIU restores the real root task as PID 1 + * from the image; if the helper has already consumed it, + * restore fails with EEXIST ("Can't fork for 1"). + */ + const int ns_flags = CLONE_NEWNS; if (!opts.restore_detach) { - pr_err("--tfork-copies>1 requires --restore-detached\n"); + pr_err("--tfork-copies requires --restore-detached\n"); return 1; } @@ -445,6 +450,13 @@ int main(int argc, char *argv[], char *envp[]) if (opts.tfork.snap_roots_n > 0) opts.root = opts.tfork.snap_roots[i]; + /* + * The n-copy child only creates the per-copy mount namespace. PID + * namespaces must be recreated by CRIU from the image, otherwise the + * helper would occupy PID 1 before the restored root task. + */ + opts.keep_pid_hierarchy = 0; + if (tfork_load_ncopy_fabric(i)) { pr_err("tfork-ncopy: copy %d fabric load failed\n", i); diff --git a/criu/criu/image.c b/criu/criu/image.c index 2783b5797..1a93344d5 100644 --- a/criu/criu/image.c +++ b/criu/criu/image.c @@ -374,6 +374,7 @@ int prepare_inventory(InventoryEntry *he) he->has_lsmtype = true; he->lsmtype = host_lsm_type(); + pid_init_dump(crt.i.pid, &crt.i); crt.i.pid->state = TASK_ALIVE; crt.i.pid->real = getpid(); if (get_task_ids(&crt.i)) diff --git a/criu/criu/include/action-scripts.h b/criu/criu/include/action-scripts.h index 88f3e5799..d6ccc0495 100644 --- a/criu/criu/include/action-scripts.h +++ b/criu/criu/include/action-scripts.h @@ -20,6 +20,7 @@ enum script_actions { ACT_QUERY_EXT_FILES, ACT_POST_TFORK_FREEZE, ACT_PRE_TFORK_RESTORE, + ACT_TFORK_SOURCE_DETACHED, ACT_MAX }; diff --git a/criu/criu/include/namespaces.h b/criu/criu/include/namespaces.h index e442e0a39..7a5aa6437 100644 --- a/criu/criu/include/namespaces.h +++ b/criu/criu/include/namespaces.h @@ -187,6 +187,7 @@ extern int restore_mnt_ns(int rst, int *cwd_fd); extern int dump_task_ns_ids(struct pstree_item *); extern int predump_task_ns_ids(struct pstree_item *); +extern int finalize_nested_pid_ns_ids(void); extern int rst_add_ns_id(unsigned int id, struct pstree_item *, struct ns_desc *nd); extern struct ns_id *lookup_ns_by_id(unsigned int id, struct ns_desc *nd); diff --git a/criu/criu/include/pstree.h b/criu/criu/include/pstree.h index f9bdeffd1..515b6cd0e 100644 --- a/criu/criu/include/pstree.h +++ b/criu/criu/include/pstree.h @@ -20,6 +20,23 @@ extern atomic_t pid_uid_generator; #define HELPER_UID_BASE (0x40000000) +static inline void pid_init_dump(struct pid *pid, struct pstree_item *item) +{ + *pid = (struct pid){ + .item = item, + .real = -1, + .local = -1, + .uid = atomic_inc_return(&pid_uid_generator), + .state = TASK_UNDEF, + .stop_signo = -1, + .ns_level = -1, + .leaf_ns_id = ALL_PID_NS_ID, + }; + rb_init_node(&pid->leaf_ns_node); + rb_init_node(&pid->root_ns_node); + rb_init_node(&pid->uid_node); +} + struct pstree_item { struct pstree_item *parent; struct list_head children; /* list of my children */ @@ -65,6 +82,7 @@ enum { #define FDS_EVENT (1 << FDS_EVENT_BIT) extern struct pstree_item *current; +extern int pstree_get_free_pid(struct pstree_item *item); struct rst_info; /* See alloc_pstree_item() for details */ diff --git a/criu/criu/include/restorer.h b/criu/criu/include/restorer.h index 73e27caa3..40bb132d1 100644 --- a/criu/criu/include/restorer.h +++ b/criu/criu/include/restorer.h @@ -242,6 +242,7 @@ struct task_restore_args { int child_subreaper; int membarrier_registration_mask; bool has_clone3_set_tid; + bool tfork_active; /* * info about rseq from libc used to diff --git a/criu/criu/namespaces.c b/criu/criu/namespaces.c index df224db68..11909cea2 100644 --- a/criu/criu/namespaces.c +++ b/criu/criu/namespaces.c @@ -592,6 +592,93 @@ static unsigned int get_ns_id(int pid, struct ns_desc *nd, protobuf_c_boolean *s return __get_ns_id(pid, nd, supported, NULL); } +static unsigned int add_nested_pid_leaf_ns_id(struct pstree_item *item) +{ + struct ns_id *nsid; + + nsid = xzalloc(sizeof(*nsid)); + if (!nsid) + return 0; + + nsid->type = NS_OTHER; + nsid->kid = 0; + nsid->ns_populated = true; + nsid_add(nsid, &pid_ns_desc, ns_next_id++, localpid(item)); + + pr_info("Add nested pid leaf ns %d for task %d(%d), level %d\n", + nsid->id, localpid(item), realpid(item), item->pid->ns_level); + return nsid->id; +} + +static unsigned int ensure_task_leaf_pid_ns_id(struct pstree_item *item); + +static unsigned int task_leaf_pid_ns_id(struct pstree_item *item, unsigned int proc_pid_ns_id) +{ + struct pstree_item *parent = item->parent; + unsigned int selected; + + /* + * os4agent stores localpid as the innermost NSpid (pid->ns[0]). + * Keep pstree_entry.nsid at the same namespace level. Otherwise a + * nested pid namespace init such as bwrap can become (nsid=N, + * localpid=1) and collide with the container init in the same nsid. + */ + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return 0; + + if (parent && parent->pid->leaf_ns_id != ALL_PID_NS_ID) { + if (item->pid->ns_level == parent->pid->ns_level) { + selected = parent->pid->leaf_ns_id; + goto out; + } + if (item->pid->ns_level > parent->pid->ns_level && + proc_pid_ns_id != parent->pid->leaf_ns_id) { + selected = proc_pid_ns_id; + goto out; + } + if (item->pid->ns_level > parent->pid->ns_level) { + selected = add_nested_pid_leaf_ns_id(item); + goto out; + } + } + + selected = proc_pid_ns_id; + +out: + pr_info("pid leaf ns task=%d(%d) uid=%d level=%d parent_level=%d proc_nsid=%u parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent ? parent->pid->ns_level : -1, proc_pid_ns_id, + parent ? parent->pid->leaf_ns_id : -1, selected); + return selected; +} + +static unsigned int ensure_task_leaf_pid_ns_id(struct pstree_item *item) +{ + struct pstree_item *parent = item->parent; + unsigned int proc_pid_ns_id; + + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return 0; + + if (item->pid->leaf_ns_id != ALL_PID_NS_ID) { + if (!parent) + return item->pid->leaf_ns_id; + if (item->pid->ns_level == parent->pid->ns_level && + item->pid->leaf_ns_id == parent->pid->leaf_ns_id) + return item->pid->leaf_ns_id; + if (item->pid->ns_level > parent->pid->ns_level && + item->pid->leaf_ns_id != parent->pid->leaf_ns_id) + return item->pid->leaf_ns_id; + } + + proc_pid_ns_id = get_ns_id(item->pid->real, &pid_ns_desc, NULL); + if (!proc_pid_ns_id) + return 0; + + item->pid->leaf_ns_id = task_leaf_pid_ns_id(item, proc_pid_ns_id); + return item->pid->leaf_ns_id; +} + int dump_one_ns_file(int lfd, u32 id, const struct fd_parms *p) { struct cr_img *img; @@ -771,15 +858,37 @@ int dump_task_ns_ids(struct pstree_item *item) int i; int pid = item->pid->real; TaskKobjIdsEntry *ids = item->ids; + struct pstree_item *parent = item->parent; + unsigned int proc_pid_ns_id; ids->has_pid_ns_id = true; - ids->pid_ns_id = get_ns_id(pid, &pid_ns_desc, NULL); + proc_pid_ns_id = get_ns_id(pid, &pid_ns_desc, NULL); + if (!proc_pid_ns_id) { + pr_err("Can't make pidns id\n"); + return -1; + } + + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return -1; + + ids->pid_ns_id = proc_pid_ns_id; + if (parent && item->pid->ns_level == parent->pid->ns_level) + ids->pid_ns_id = parent->pid->leaf_ns_id; + else if (parent && item->pid->ns_level > parent->pid->ns_level && + ids->pid_ns_id == parent->pid->leaf_ns_id) + ids->pid_ns_id = add_nested_pid_leaf_ns_id(item); + if (!ids->pid_ns_id) { pr_err("Can't make pidns id\n"); return -1; } item->pid->leaf_ns_id = ids->pid_ns_id; + pr_info("dump pid ns task=%d(%d) uid=%d level=%d parent_level=%d proc_nsid=%u parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent ? parent->pid->ns_level : -1, proc_pid_ns_id, + parent ? parent->pid->leaf_ns_id : -1, ids->pid_ns_id); + for (i = 0; i < item->nr_threads; i++) item->threads[i].leaf_ns_id = ids->pid_ns_id; @@ -865,6 +974,40 @@ int dump_task_ns_ids(struct pstree_item *item) return 0; } +int finalize_nested_pid_ns_ids(void) +{ + struct pstree_item *item; + + for_each_pstree_item(item) { + struct pstree_item *parent = item->parent; + unsigned int nsid; + int i; + + if (!parent) + continue; + if (item->pid->ns_level <= parent->pid->ns_level) + continue; + if (item->pid->leaf_ns_id != parent->pid->leaf_ns_id) + continue; + + nsid = add_nested_pid_leaf_ns_id(item); + if (!nsid) + return -1; + + item->pid->leaf_ns_id = nsid; + for (i = 0; i < item->nr_threads; i++) + item->threads[i].leaf_ns_id = nsid; + if (item->ids && item->ids->has_pid_ns_id) + item->ids->pid_ns_id = nsid; + + pr_info("finalize nested pid ns task=%d(%d) uid=%d level=%d parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent->pid->leaf_ns_id, nsid); + } + + return 0; +} + static UsernsEntry userns_entry = USERNS_ENTRY__INIT; #define INVALID_ID (~0U) diff --git a/criu/criu/pie/restorer.c b/criu/criu/pie/restorer.c index 9aaefe502..8e49872c7 100644 --- a/criu/criu/pie/restorer.c +++ b/criu/criu/pie/restorer.c @@ -97,6 +97,7 @@ static pid_t *helpers; static int n_helpers; static pid_t *zombies; static int n_zombies; +static bool tfork_active_local; static enum faults fi_strategy; bool fault_injected(enum faults f) { @@ -162,7 +163,8 @@ static void sigchld_handler(int signal, siginfo_t *siginfo, void *data) else r = "disappeared with"; - pr_info("Task %d %s %d\n", siginfo->si_pid, r, siginfo->si_status); + pr_err("SIGCHLD during restore: task %d %s %d\n", + siginfo->si_pid, r, siginfo->si_status); futex_abort_and_wake(&task_entries_local->nr_in_progress); /* sa_restorer may be unmaped, so we can't go back to userspace*/ @@ -807,12 +809,24 @@ __visible long __export_restore_thread(struct thread_restore_args *args) } pr_info("%ld: Restored\n", sys_gettid()); - restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->ta->tfork_active) + pr_debug("tfork: thread restore stage complete pid=%d tid=%ld comm=%s ns_level=%d\n", + args->pid, sys_gettid(), args->comm, args->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->ta->tfork_active) + pr_debug("tfork: thread restore barrier returned pid=%d tid=%ld comm=%s stage=%d\n", + args->pid, sys_gettid(), args->comm, ret); if (restore_signals(args->siginfo, args->siginfo_n, false)){ goto core_restore_end; } - restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->ta->tfork_active) + pr_debug("tfork: thread sigchld stage complete pid=%d tid=%ld comm=%s ns_level=%d\n", + args->pid, sys_gettid(), args->comm, args->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->ta->tfork_active) + pr_debug("tfork: thread sigchld barrier returned pid=%d tid=%ld comm=%s stage=%d\n", + args->pid, sys_gettid(), args->comm, ret); /* * Make sure it's before creds, since it's privileged @@ -1476,6 +1490,11 @@ static int wait_zombies(struct task_restore_args *task_args) ret = sys_waitid(P_PID, task_args->zombies[i], NULL, WNOWAIT | WEXITED, NULL); if (ret == -ECHILD) { + if (task_args->tfork_active) { + pr_warn("tfork: zombie pid %d is not reparented to task %ld; skipping wait to avoid restore barrier deadlock\n", + task_args->zombies[i], sys_getpid()); + continue; + } /* A process isn't reparented to this task yet. * Let's wait when someone complete this stage * and try again. @@ -1752,6 +1771,7 @@ __visible long __export_restore_task(struct task_restore_args *args) fi_strategy = args->fault_strategy; task_entries_local = args->task_entries; + tfork_active_local = args->tfork_active; helpers = args->helpers; n_helpers = args->helpers_n; zombies = args->zombies; @@ -2495,7 +2515,14 @@ __visible long __export_restore_task(struct task_restore_args *args) args->clone_restore_fn); } if (ret != thread_args[i].pid) { - pr_err("Unable to create a thread: %ld\n", ret); + pr_err("Unable to create a thread: %ld expected=%d ns_level=%d tids=%d/%d/%d/%d tfork=%d\n", + ret, thread_args[i].pid, + thread_args[i].ns_level, + thread_args[i].tid_in_ns[0], + thread_args[i].ns_level > 1 ? thread_args[i].tid_in_ns[1] : -1, + thread_args[i].ns_level > 2 ? thread_args[i].tid_in_ns[2] : -1, + thread_args[i].ns_level > 3 ? thread_args[i].tid_in_ns[3] : -1, + args->tfork_active ? 1 : 0); sys_close(fd); mutex_unlock(&task_entries_local->last_pid_mutex); goto core_restore_end; @@ -2524,13 +2551,33 @@ __visible long __export_restore_task(struct task_restore_args *args) if (restore_membarrier_registrations(args->membarrier_registration_mask) < 0) goto core_restore_end; pr_info("%ld: Restored\n", sys_getpid()); - - restore_finish_stage(task_entries_local, CR_STATE_RESTORE); - + if (args->tfork_active) + pr_debug("tfork: leader restore stage complete pid=%d tid=%ld comm=%s threads=%d ns_level=%d\n", + args->t->pid, sys_getpid(), args->comm, args->nr_threads, + args->t->ns_level); + + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->tfork_active) + pr_debug("tfork: leader restore barrier returned pid=%d tid=%ld comm=%s stage=%ld helpers=%u zombies=%u inotify=%u\n", + args->t->pid, sys_getpid(), args->comm, ret, + args->helpers_n, args->zombies_n, args->inotify_fds_n); + + if (args->tfork_active) + pr_debug("tfork: leader wait_helpers start pid=%d tid=%ld comm=%s helpers=%u\n", + args->t->pid, sys_getpid(), args->comm, args->helpers_n); if (wait_helpers(args) < 0) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader wait_helpers done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader wait_zombies start pid=%d tid=%ld comm=%s zombies=%u\n", + args->t->pid, sys_getpid(), args->comm, args->zombies_n); if (wait_zombies(args) < 0) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader wait_zombies done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ksigfillset(&to_block); ret = sys_sigprocmask(SIG_SETMASK, &to_block, NULL, sizeof(k_rtsigset_t)); @@ -2539,9 +2586,18 @@ __visible long __export_restore_task(struct task_restore_args *args) goto core_restore_end; } + if (args->tfork_active) + pr_debug("tfork: leader cleanup_inotify start pid=%d tid=%ld comm=%s inotify=%u\n", + args->t->pid, sys_getpid(), args->comm, args->inotify_fds_n); if (cleanup_current_inotify_events(args)) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader cleanup_inotify done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader restore sigaction start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); if (!args->compatible_mode) { ret = sys_sigaction(SIGCHLD, &args->sigchld_act, NULL, sizeof(k_rtsigset_t)); } else { @@ -2558,16 +2614,38 @@ __visible long __export_restore_task(struct task_restore_args *args) pr_err("Failed to restore SIGCHLD: %ld\n", ret); goto core_restore_end; } + if (args->tfork_active) + pr_debug("tfork: leader restore sigaction done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader restore shared signals start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ret = restore_signals(args->siginfo, args->siginfo_n, true); if (ret) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader restore shared signals done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader restore private signals start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ret = restore_signals(args->t->siginfo, args->t->siginfo_n, false); if (ret) goto core_restore_end; - - restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->tfork_active) + pr_debug("tfork: leader restore private signals done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + + if (args->tfork_active) + pr_debug("tfork: leader sigchld stage complete pid=%d tid=%ld comm=%s threads=%d ns_level=%d\n", + args->t->pid, sys_getpid(), args->comm, args->nr_threads, + args->t->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->tfork_active) + pr_debug("tfork: leader sigchld barrier returned pid=%d tid=%ld comm=%s stage=%ld\n", + args->t->pid, sys_getpid(), args->comm, ret); rst_tcp_socks_all(args); diff --git a/criu/criu/pstree.c b/criu/criu/pstree.c index c448c9516..7371fcb48 100644 --- a/criu/criu/pstree.c +++ b/criu/criu/pstree.c @@ -272,26 +272,26 @@ struct pstree_item *__alloc_pstree_item(bool rst) INIT_LIST_HEAD(&item->children); INIT_LIST_HEAD(&item->sibling); - item->pid->ns_level = -1; - item->pid->leaf_ns_id = ALL_PID_NS_ID; - item->pid->real = -1; - item->pid->local = -1; - if (!rst) - item->pid->uid = atomic_inc_return(&pid_uid_generator); - else + pid_init_dump(item->pid, item); + else { + item->pid->ns_level = -1; + item->pid->leaf_ns_id = ALL_PID_NS_ID; + item->pid->real = -1; + item->pid->local = -1; item->pid->uid = -1; - item->pid->state = TASK_UNDEF; - item->pid->stop_signo = -1; + item->pid->state = TASK_UNDEF; + item->pid->stop_signo = -1; + item->pid->item = item; + rb_init_node(&item->pid->leaf_ns_node); + rb_init_node(&item->pid->root_ns_node); + rb_init_node(&item->pid->uid_node); + } item->born_sid = -1; item->tfork_pidfd = -1; item->tfork_memfd = -1; item->tfork_pagemap_fd = -1; - item->pid->item = item; futex_init(&item->task_st); - rb_init_node(&item->pid->leaf_ns_node); - rb_init_node(&item->pid->root_ns_node); - rb_init_node(&item->pid->uid_node); return item; } @@ -436,7 +436,10 @@ int dump_pstree(struct pstree_item *root_item) pstree_entry__init(e); tree_entries[nr_items++] = e; - pr_info("Process: %d(%d)\n", localpid(item), realpid(item)); + pr_info("Process: %d(%d) uid=%d nsid=%d level=%d parent=%d parent_nsid=%d\n", + localpid(item), realpid(item), uid(item), item->pid->leaf_ns_id, + item->pid->ns_level, item->parent ? realpid(item->parent) : 0, + item->parent ? item->parent->pid->leaf_ns_id : -1); e->realpid = realpid(item); e->ppid = item->parent ? realpid(item->parent) : 0; @@ -729,11 +732,37 @@ static int __pstree_insert_pid(struct pid *pid_node, struct rb_node *root_parent rb_link_and_balance(&uid_root_rb, &pid_node->uid_node, parent, link); } - return 0; + return 0; err: - rb_erase(&pid_node->root_ns_node, &pid_root_rb[ALL_PID_NS_ID]); - return -1; + rb_erase(&pid_node->root_ns_node, &pid_root_rb[ALL_PID_NS_ID]); + return -1; +} + +static void pstree_remove_pid_if_linked(struct pid *pid_node) +{ + struct pid *found; + bool valid_leaf_ns = pid_node->leaf_ns_id >= 0 && + (unsigned int)pid_node->leaf_ns_id < pid_namespace_count; + + if (pid_node->uid > 0) { + found = __lookup_pid_uid(&uid_root_rb, pid_node->uid, NULL, NULL); + if (found == pid_node) + rb_erase(&pid_node->uid_node, &uid_root_rb); + } + + if (pid_node->leaf_ns_id != ALL_PID_NS_ID && valid_leaf_ns) { + found = __lookup_pid_leaf(&pid_root_rb[pid_node->leaf_ns_id], + pid_node->local, NULL, NULL); + if (found == pid_node) + rb_erase(&pid_node->leaf_ns_node, + &pid_root_rb[pid_node->leaf_ns_id]); + } + + found = __lookup_pid_root(&pid_root_rb[ALL_PID_NS_ID], + pid_node->real, NULL, NULL); + if (found == pid_node) + rb_erase(&pid_node->root_ns_node, &pid_root_rb[ALL_PID_NS_ID]); } int pstree_insert_pid(struct pid *pid_node) @@ -741,13 +770,14 @@ int pstree_insert_pid(struct pid *pid_node) return __pstree_insert_pid(pid_node, NULL, NULL); } -static struct pstree_item *get_or_create_pstree_item(pid_t real, pid_t local, int pidns_id) +static struct pstree_item *get_or_create_pstree_item(pid_t real, pid_t local, int pidns_id, bool *created) { struct pid *found; struct pstree_item *item; - struct rb_node **root_link, *root_parent; - found = __lookup_pid_root(&pid_root_rb[ALL_PID_NS_ID], real, &root_parent, &root_link); + *created = false; + + found = __lookup_pid_root(&pid_root_rb[ALL_PID_NS_ID], real, NULL, NULL); if (found) { if (pidns_id != ALL_PID_NS_ID) { BUG_ON(found->leaf_ns_id != pidns_id || found->local != local); @@ -762,11 +792,7 @@ static struct pstree_item *get_or_create_pstree_item(pid_t real, pid_t local, in item->pid->real = real; item->pid->local = local; item->pid->leaf_ns_id = pidns_id; - - if (__pstree_insert_pid(item->pid, root_parent, root_link) < 0) { - xfree(item); - return NULL; - } + *created = true; return item; } @@ -876,13 +902,24 @@ static int read_pstree_ids(struct pstree_item *pi) */ static int read_one_pstree_item(PstreeEntry *e) { - struct pstree_item *pi; - int ret = -1, i, j; + struct pstree_item *pi = NULL; + int ret = -1, i, j, next_inserted_thread = 1; + bool linked = false, pid_inserted = false, threads_allocated = false; + bool created_item = false; - pi = get_or_create_pstree_item(e->realpid, e->localpid, e->nsid); + pi = get_or_create_pstree_item(e->realpid, e->localpid, e->nsid, &created_item); if (!pi) goto err; + /* + * get_or_create_pstree_item() can only reuse an item that is still + * TASK_UNDEF. Completed items are rejected here, so the unwind below + * cannot tear down a previously parsed pstree item. + */ BUG_ON(pi->pid->state != TASK_UNDEF); + if (!created_item && (pi->threads || pi->nr_threads)) { + pr_err("Refusing to reuse partially populated pstree item for %d\n", e->realpid); + goto err; + } /* * Populate the ns-chain on pi from the thread-leader entry before @@ -915,6 +952,27 @@ static int read_one_pstree_item(PstreeEntry *e) } pi->pid->state = TASK_ALIVE; pi->pid->uid = e->uid; + pi->nr_threads = e->n_threads; + pi->threads = xmalloc(e->n_threads * sizeof(struct pid)); + if (!pi->threads) + goto err; + threads_allocated = true; + + /* note: we don't fail if we have empty ids */ + if (read_pstree_ids(pi) < 0) + goto err; + + if (pi->ids && pi->ids->has_pid_ns_id) { + if (pi->ids->pid_ns_id != pi->pid->leaf_ns_id) { + pr_warn("PID namespace id mismatch for uid %d: pstree=%d ids=%d, keeping pstree\n", + uid(pi), pi->pid->leaf_ns_id, pi->ids->pid_ns_id); + pi->ids->pid_ns_id = pi->pid->leaf_ns_id; + } + } + + if (__pstree_insert_pid(pi->pid, NULL, NULL) < 0) + goto err; + pid_inserted = true; if (e->ppid == 0) { if (root_item) { @@ -938,13 +996,9 @@ static int read_one_pstree_item(PstreeEntry *e) parent = pid->item; pi->parent = parent; list_add(&pi->sibling, &parent->children); + linked = true; } - pi->nr_threads = e->n_threads; - pi->threads = xmalloc(e->n_threads * sizeof(struct pid)); - if (!pi->threads) - goto err; - for (i = 0; i < e->n_threads; i++) { int insert_status; pi->threads[i].uid = e->threads[i]->uid; @@ -960,7 +1014,7 @@ static int read_one_pstree_item(PstreeEntry *e) pi->threads[i].state = TASK_THREAD; pi->threads[i].item = NULL; if (i == 0) { - + /* The leader is indexed through pi->pid, not this mirror. */ pi->pid->ns_level = pi->threads[0].ns_level; pi->pid->local = pi->threads[0].ns[0].ns_pid; memcpy(pi->pid->ns, pi->threads[0].ns, e->threads[0]->n_ns * sizeof(struct pid_ns)); @@ -972,17 +1026,42 @@ static int read_one_pstree_item(PstreeEntry *e) pr_err("Unexpected task %d in a tree %d\n", e->threads[i]->ns[0]->nspid, i); goto err; } + next_inserted_thread = i + 1; } task_entries->nr_threads += e->n_threads; task_entries->nr_tasks++; - /* note: we don't fail if we have empty ids */ - if (read_pstree_ids(pi) < 0) - goto err; - ret = 1; err: + if (ret < 0 && pi) { + /* + * threads[0] is the leader mirrored by pi->pid. Only + * non-leader threads are inserted independently, and + * next_inserted_thread always points one past the last + * successfully inserted non-leader slot. + */ + for (i = 1; i < next_inserted_thread; i++) + pstree_remove_pid_if_linked(&pi->threads[i]); + if (root_item == pi) + root_item = NULL; + if (linked) + list_del_init(&pi->sibling); + if (pid_inserted) + pstree_remove_pid_if_linked(pi->pid); + pi->pid->state = TASK_UNDEF; + if (threads_allocated) { + xfree(pi->threads); + pi->threads = NULL; + pi->nr_threads = 0; + } + /* + * Restore pstree items come from the shared linear arena. The + * item may no longer be the last allocation after read_pstree_ids(), + * so it cannot be released individually. Restore teardown reclaims + * the arena after this parse failure. + */ + } return ret; } @@ -1024,7 +1103,7 @@ static int read_pstree_image(void) return ret < 0 ? -1 : 0; } -static int helper_get_free_pid(struct pstree_item *item) +int pstree_get_free_pid(struct pstree_item *item) { int pidns = item ? item->pid->leaf_ns_id : ALL_PID_NS_ID; @@ -1082,7 +1161,7 @@ static int prepare_pstree_ids(pid_t pid) } } if (leader->pid->state != TASK_UNDEF) { - helper_pid = helper_get_free_pid(item); + helper_pid = pstree_get_free_pid(item); if (helper_pid < 0) break; @@ -1262,8 +1341,12 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) unsigned int ns_level_to_truncate; clone_flags = get_clone_mask(root_item->ids, root_ids); - if (!(clone_flags & CLONE_NEWPID)) + if (!(clone_flags & CLONE_NEWPID) && + !(opts.tfork.active && root_item->pid->ns_level > 1)) return 0; + if (!(clone_flags & CLONE_NEWPID)) + pr_info("pidns: forcing tfork pid hierarchy truncation for root level=%d\n", + root_item->pid->ns_level); if (root_item->pid->ns_level <= 1) { pr_err("only 1 level of pid namespace, but CLONE_NEWPID is set, " @@ -1272,6 +1355,8 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) } ns_level_to_truncate = root_item->pid->ns_level - 1; + pr_info("pidns: truncating %u outer pid namespace level(s) for new root pid namespace\n", + ns_level_to_truncate); for (node = rb_first(&pid_root_rb[ALL_PID_NS_ID]); node; ) { next = rb_next(node); @@ -1279,9 +1364,15 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) pid_node = rb_entry(node, struct pid, root_ns_node); rb_erase(node, &pid_root_rb[ALL_PID_NS_ID]); + pr_info("pidns: truncate before uid=%d real=%d local=%d level=%d\n", + pid_node->uid, pid_node->real, pid_node->local, + pid_node->ns_level); pid_node->ns_level -= ns_level_to_truncate; BUG_ON(pid_node->ns_level <= 0); pid_node->real = pid_node->ns[pid_node->ns_level - 1].ns_pid; + pr_info("pidns: truncate after uid=%d real=%d local=%d level=%d\n", + pid_node->uid, pid_node->real, pid_node->local, + pid_node->ns_level); found = __lookup_pid_root(&new_real_rbtree, pid_node->real, &parent, &link); if (found) { diff --git a/criu/criu/seize.c b/criu/criu/seize.c index f5cde74e9..db8d95b92 100644 --- a/criu/criu/seize.c +++ b/criu/criu/seize.c @@ -379,7 +379,7 @@ static int seize_cgroup_tree(char *root_path, enum freezer_state state) */ static int freezer_wait_processes(void) { - int i; + int i, collected = 0; processes_to_wait_pids = xmalloc(sizeof(pid_t) * processes_to_wait); if (processes_to_wait_pids == NULL) @@ -388,23 +388,42 @@ static int freezer_wait_processes(void) for (i = 0; i < processes_to_wait; i++) { int status; pid_t pid; + int waited_ms = 0; /* * Here we are going to skip tasks which are already traced. * Ptraced tasks looks like children for us, so if * a task isn't ptraced yet, waitpid() will return a error. */ - pid = waitpid(-1, &status, 0); - if (pid < 0) { - pr_perror("Unable to wait processes"); - xfree(processes_to_wait_pids); - processes_to_wait_pids = NULL; - return -1; + while (1) { + pid = waitpid(-1, &status, opts.tfork.active ? WNOHANG : 0); + if (pid > 0) + break; + if (opts.tfork.active && pid < 0 && errno == EINTR && waited_ms < 500) { + usleep(10 * 1000); + waited_ms += 10; + continue; + } + if (!opts.tfork.active || (pid < 0 && errno != ECHILD && errno != EINTR)) { + pr_perror("Unable to wait processes"); + xfree(processes_to_wait_pids); + processes_to_wait_pids = NULL; + return -1; + } + if (pid < 0 || waited_ms >= 500) { + pr_warn("tfork: collected %d/%d unexpected freezer processes; continuing\n", + collected, processes_to_wait); + processes_to_wait = collected; + return 0; + } + usleep(10 * 1000); + waited_ms += 10; } pr_warn("Unexpected process %d in the freezer cgroup (status 0x%x)\n", pid, status); - processes_to_wait_pids[i] = pid; + processes_to_wait_pids[collected++] = pid; } + processes_to_wait = collected; return 0; } diff --git a/criu/criu/unittest/mock.c b/criu/criu/unittest/mock.c index b2d507278..b9601fffa 100644 --- a/criu/criu/unittest/mock.c +++ b/criu/criu/unittest/mock.c @@ -97,6 +97,10 @@ int close_service_fd(int type) return 0; } +void invalidate_proc_self_fd(void) +{ +} + void compel_log_init(int log_fn, unsigned int level) { } diff --git a/criu/criu/unittest/unit.c b/criu/criu/unittest/unit.c index 54769e6f2..cf9d79b24 100644 --- a/criu/criu/unittest/unit.c +++ b/criu/criu/unittest/unit.c @@ -3,19 +3,41 @@ #include #include "log.h" +#include "pstree.h" #include "util.h" #include "criu-log.h" int parse_statement(int i, char *line, char **configuration); +atomic_t pid_uid_generator = ATOMIC_INIT(0); + int main(int argc, char *argv[], char *envp[]) { char **configuration; + struct pid first_pid = {}; + struct pid second_pid = {}; + struct pstree_item first_item = { .pid = &first_pid }; + struct pstree_item second_item = { .pid = &second_pid }; int i; configuration = malloc(10 * sizeof(char *)); log_init(NULL); + pid_init_dump(&first_pid, &first_item); + pid_init_dump(&second_pid, &second_item); + assert(first_pid.uid > 0); + assert(second_pid.uid > first_pid.uid); + assert(first_pid.item == &first_item); + assert(first_pid.real == -1); + assert(first_pid.local == -1); + assert(first_pid.state == TASK_UNDEF); + assert(first_pid.stop_signo == -1); + assert(first_pid.ns_level == -1); + assert(first_pid.leaf_ns_id == ALL_PID_NS_ID); + assert(RB_EMPTY_NODE(&first_pid.leaf_ns_node)); + assert(RB_EMPTY_NODE(&first_pid.root_ns_node)); + assert(RB_EMPTY_NODE(&first_pid.uid_node)); + i = parse_statement(0, "", configuration); assert(i == 0); diff --git a/criu/lib/pycriu/images/images.py b/criu/lib/pycriu/images/images.py index 9db506e1e..3d2e02777 100644 --- a/criu/lib/pycriu/images/images.py +++ b/criu/lib/pycriu/images/images.py @@ -43,6 +43,8 @@ import os import array +from google.protobuf.message import DecodeError + from . import magic from . import pb from . import pb2dict @@ -190,6 +192,98 @@ def count(self, f): return entries +class pstree_handler: + """Read both legacy per-task and current file-level PSTREE images.""" + + @staticmethod + def _read_payload(f): + header = f.read(4) + if not header: + return None + if len(header) != 4: + raise ValueError("truncated PSTREE entry header") + + size, = struct.unpack('i', header) + if size < 0: + raise ValueError("negative PSTREE entry size") + + payload = f.read(size) + if len(payload) != size: + raise ValueError("truncated PSTREE entry payload") + return payload + + @staticmethod + def _parse(payload, message_type): + message = message_type() + try: + message.ParseFromString(payload) + except DecodeError: + return None + return message if message.IsInitialized() else None + + def load(self, f, pretty=False, no_payload=False): + # PSTREE has no out-of-band EXTRA data, so no_payload has no effect. + payload = self._read_payload(f) + if payload is None: + return [] + + # Format detection intentionally relies on pstree_entry being proto2 + # with required fields. A file-level payload may parse as that message, + # but it cannot be initialized because its wire fields have other types. + legacy = self._parse(payload, pb.pstree_entry) + if legacy is not None: + entries = [legacy] + while True: + payload = self._read_payload(f) + if payload is None: + break + entry = self._parse(payload, pb.pstree_entry) + if entry is None: + raise ValueError("invalid legacy PSTREE entry") + entries.append(entry) + return [pb2dict.pb2dict(entry, pretty) for entry in entries] + + entry = self._parse(payload, pb.pstree_file_entry) + if entry is None: + raise ValueError("invalid PSTREE entry") + if self._read_payload(f) is not None: + raise ValueError("file-level PSTREE image has multiple entries") + + return [pb2dict.pb2dict(entry, pretty)] + + def loads(self, data, pretty=False): + return self.load(io.BytesIO(data), pretty) + + def dump(self, entries, f): + if not entries: + return + + file_level = ('tree' in entries[0] or + 'ns_max_pids' in entries[0]) + if file_level and len(entries) != 1: + raise ValueError("file-level PSTREE image requires one entry") + + message_type = (pb.pstree_file_entry if file_level + else pb.pstree_entry) + for entry in entries: + message = message_type() + pb2dict.dict2pb(entry, message) + payload = message.SerializeToString() + f.write(struct.pack('i', len(payload))) + f.write(payload) + + def dumps(self, entries): + f = io.BytesIO() + self.dump(entries, f) + return f.getvalue() + + def count(self, f): + entries = 0 + while self._read_payload(f) is not None: + entries += 1 + return entries + + # Special handler for pagemap.img class pagemap_handler: """ @@ -502,7 +596,7 @@ def skip(self, f, pbuff): tcp_stream_extra_handler()), 'STATS': entry_handler(pb.stats_entry), 'PAGEMAP': pagemap_handler(), # Special one - 'PSTREE': entry_handler(pb.pstree_entry), + 'PSTREE': pstree_handler(), 'REG_FILES': entry_handler(pb.reg_file_entry), 'NS_FILES': entry_handler(pb.ns_file_entry), 'EVENTFD_FILE': entry_handler(pb.eventfd_file_entry), diff --git a/criu/test/others/pycriu/Makefile b/criu/test/others/pycriu/Makefile index b6e3b4814..369ac5667 100644 --- a/criu/test/others/pycriu/Makefile +++ b/criu/test/others/pycriu/Makefile @@ -13,7 +13,8 @@ CRIU_SOCKET := $(BUILD_DIR)/$(SOCKET_NAME) STATUS_FIFO := $(BUILD_DIR)/startup.status STATUS_FD := 200 -run: start +run: pstree-compat + $(MAKE) --no-print-directory start cleanup() { $(MAKE) --no-print-directory stop || true; } trap cleanup EXIT INT TERM "$(PYTHON)" test_check.py @@ -21,6 +22,9 @@ run: start "$(PYTHON)" test_check_images_dir.py "$(PYTHON)" test_check_work_dir_fd.py +pstree-compat: + "$(PYTHON)" test_pstree_compat.py + start: mkdir -p "$(BUILD_DIR)" if [ -s "$(PIDFILE)" ] && kill -0 "$$(cat "$(PIDFILE)")" 2>/dev/null; then @@ -60,4 +64,4 @@ clean: fi rm -rf "$(BUILD_DIR)" -.PHONY: start stop clean run \ No newline at end of file +.PHONY: start stop clean run pstree-compat diff --git a/criu/test/others/pycriu/test_pstree_compat.py b/criu/test/others/pycriu/test_pstree_compat.py new file mode 100644 index 000000000..5cda76e13 --- /dev/null +++ b/criu/test/others/pycriu/test_pstree_compat.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import io +import os +import struct +import sys + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +LIB_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "../../../lib")) +if LIB_DIR not in sys.path: + sys.path.insert(0, LIB_DIR) + +from pycriu.images import images, pb # noqa: E402 + + +def encode(*messages): + output = io.BytesIO() + for message in messages: + payload = message.SerializeToString() + output.write(struct.pack('i', len(payload))) + output.write(payload) + return output.getvalue() + + +def task(message, realpid, localpid, uid): + message.realpid = realpid + message.ppid = 0 + message.pgid = localpid + message.sid = localpid + message.nsid = 7 + message.localpid = localpid + message.uid = uid + + +def main(): + handler = images.handlers['PSTREE'] + + old_first = pb.pstree_entry() + task(old_first, 1001, 1, 11) + old_second = pb.pstree_entry() + task(old_second, 1002, 2, 12) + old_blob = encode(old_first, old_second) + old_entries = handler.loads(old_blob) + assert [entry['realpid'] for entry in old_entries] == [1001, 1002] + assert handler.loads(handler.dumps(old_entries)) == old_entries + assert handler.count(io.BytesIO(old_blob)) == 2 + + current = pb.pstree_file_entry() + ns_max = current.ns_max_pids.add() + ns_max.ns_id = 7 + ns_max.pid_max = 2 + task(current.tree.add(), 1001, 1, 11) + task(current.tree.add(), 1002, 2, 12) + current_blob = encode(current) + current_entries = handler.loads(current_blob) + assert len(current_entries) == 1 + assert [entry['realpid'] for entry in current_entries[0]['tree']] == [1001, 1002] + assert handler.loads(handler.dumps(current_entries)) == current_entries + assert handler.count(io.BytesIO(current_blob)) == 1 + + print("PASS") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/criu/test/others/tfork-phase-b-ab.sh b/criu/test/others/tfork-phase-b-ab.sh new file mode 100755 index 000000000..07d75a464 --- /dev/null +++ b/criu/test/others/tfork-phase-b-ab.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +set -euo pipefail + +PODMAN=${PODMAN:-podman} +PODMAN_GLOBAL_ARGS=${PODMAN_GLOBAL_ARGS:-} +OLD_CRIU_ROOT=${OLD_CRIU_ROOT:?set OLD_CRIU_ROOT} +NEW_CRIU_ROOT=${NEW_CRIU_ROOT:?set NEW_CRIU_ROOT} +SAMPLES=${SAMPLES:-20} +IMAGE=${IMAGE:-docker.io/library/alpine:3.19} +PREFIX=${PREFIX:-tfork-phase-b-$RANDOM} +OUTPUT=${OUTPUT:-/tmp/tfork-phase-b-ab.tsv} +OS4AGENT_CRUN=${OS4AGENT_CRUN:-crun} +WORKLOAD_PROCESSES=${WORKLOAD_PROCESSES:-1} +LOG_DIR=${LOG_DIR:-} +TFORK_CLONE_ARGS=${TFORK_CLONE_ARGS:-} + +read -r -a podman_global_args <<<"$PODMAN_GLOBAL_ARGS" +read -r -a tfork_clone_args <<<"$TFORK_CLONE_ARGS" +source_name=${PREFIX}-source + +podman_cmd() { + "$PODMAN" "${podman_global_args[@]}" "$@" +} + +cleanup() { + podman_cmd ps -a --format '{{.Names}}' | + awk -v prefix="$PREFIX" 'index($0, prefix) == 1' | + while read -r name; do + podman_cmd rm -f -t 0 "$name" >/dev/null 2>&1 || true + done +} +trap cleanup EXIT + +run_clone() { + local variant=$1 + local root=$2 + local index=$3 + local name=${PREFIX}-${variant}-${index} + local started ended elapsed rootfs bundle + + started=$(date +%s%N) + env \ + PATH="$root/criu:$PATH" \ + LD_LIBRARY_PATH="$root/lib/c${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + OS4AGENT_CRUN="$OS4AGENT_CRUN" \ + "$PODMAN" "${podman_global_args[@]}" container clone \ + --live --tfork-overlay-btrfs "${tfork_clone_args[@]}" \ + "$source_name" "$name" >/dev/null + ended=$(date +%s%N) + elapsed=$(( (ended - started) / 1000000 )) + + [[ $(podman_cmd exec "$name" cat /tmp/sentinel) == source-before-fork ]] + if [[ -n $LOG_DIR ]]; then + rootfs=$(podman_cmd inspect --format '{{.Rootfs}}' "$name") + bundle=$(dirname "$rootfs") + test -f "$bundle/img/tfork.log" + cp "$bundle/img/tfork.log" \ + "$LOG_DIR/${variant}-${index}.tfork.log" + if [[ -f $bundle/img/tfork-restore.log.copy0 ]]; then + cp "$bundle/img/tfork-restore.log.copy0" \ + "$LOG_DIR/${variant}-${index}.restore.log" + fi + fi + podman_cmd kill -s KILL "$name" >/dev/null + podman_cmd rm -f -t 0 "$name" >/dev/null + printf '%s\t%d\t%d\n' "$variant" "$index" "$elapsed" | tee -a "$OUTPUT" +} + +cleanup +: >"$OUTPUT" +if [[ -n $LOG_DIR ]]; then + mkdir -p "$LOG_DIR" +fi +podman_cmd run -d --name "$source_name" \ + --log-driver k8s-file \ + --security-opt seccomp=unconfined \ + --security-opt apparmor=unconfined \ + "$IMAGE" sh -c \ + 'count=$1 + i=1 + while [ "$i" -lt "$count" ]; do + sleep 86400 & + i=$((i + 1)) + done + echo source-before-fork >/tmp/sentinel + exec tail -f /dev/null' sh "$WORKLOAD_PROCESSES" >/dev/null + +for ((attempt = 0; attempt < 100; attempt++)); do + actual_processes=$(podman_cmd top "$source_name" pid | + awk 'NR > 1 { count++ } END { print count + 0 }') + if ((actual_processes == WORKLOAD_PROCESSES)); then + break + fi + sleep 0.05 +done +if ((actual_processes != WORKLOAD_PROCESSES)); then + printf 'expected %d source processes, found %d\n' \ + "$WORKLOAD_PROCESSES" "$actual_processes" >&2 + exit 1 +fi + +for ((i = 1; i <= SAMPLES; i++)); do + if ((i % 2)); then + run_clone old "$OLD_CRIU_ROOT" "$i" + run_clone new "$NEW_CRIU_ROOT" "$i" + else + run_clone new "$NEW_CRIU_ROOT" "$i" + run_clone old "$OLD_CRIU_ROOT" "$i" + fi +done + +podman_cmd rm -f -t 0 "$source_name" >/dev/null +trap - EXIT diff --git a/crun/src/libcrun/container.h b/crun/src/libcrun/container.h index 7b6649e2e..851d91b3a 100644 --- a/crun/src/libcrun/container.h +++ b/crun/src/libcrun/container.h @@ -224,6 +224,8 @@ struct libcrun_checkpoint_restore_s bool track_mem; int tfork_copies; int tfork_dumpd_parent_pid; + int tfork_pre_restore_fd; + int tfork_source_detached_fd; unsigned int tfork_ghost_limit; bool tfork_full_memcopy; diff --git a/crun/src/libcrun/criu.c b/crun/src/libcrun/criu.c index 29feeee70..e038526f0 100644 --- a/crun/src/libcrun/criu.c +++ b/crun/src/libcrun/criu.c @@ -46,6 +46,8 @@ # define DESCRIPTORS_FILENAME "descriptors.json" # define CRIU_RUNC_CONFIG_FILE "/etc/criu/runc.conf" # define CRIU_CRUN_CONFIG_FILE "/etc/criu/crun.conf" +# define CRIU_LOG_TAIL_LINES 80 +# define CRIU_LOG_LINE_SIZE 1024 # define CRIU_EXT_NETNS "extRootNetNS" # define CRIU_EXT_PIDNS "extRootPidNS" @@ -58,6 +60,31 @@ char *chroot_realpath (const char *chroot, const char *path, char resolved_path[]); static const char *console_socket = NULL; +static int tfork_pre_restore_fd = -1; +static int tfork_source_detached_fd = -1; + +static int +tfork_criu_log_level (void) +{ + const char *override = getenv ("CRIU_TFORK_LOG_LEVEL"); + + /* Keep an explicit override for controlled A/B tests and emergency + diagnostics. CRIU accepts levels from unconditional messages (0) + through debug (4). */ + if (override != NULL && override[0] >= '0' && override[0] <= '4' && override[1] == '\0') + return override[0] - '0'; + + switch (libcrun_get_verbosity ()) + { + case LIBCRUN_VERBOSITY_DEBUG: + return CRIU_LOG_DEBUG; + case LIBCRUN_VERBOSITY_WARNING: + return CRIU_LOG_WARN; + case LIBCRUN_VERBOSITY_ERROR: + default: + return CRIU_LOG_ERROR; + } +} # define LIBCRIU_MIN_VERSION 31500 @@ -239,6 +266,39 @@ load_wrapper (struct libcriu_wrapper_s **wrapper_out, libcrun_error_t *err) static int criu_notify (char *action, __attribute__ ((unused)) criu_notify_arg_t na) { + if (action == NULL) + return 0; + + if (strcmp (action, "post-tfork-freeze") == 0 && tfork_pre_restore_fd >= 0) + { + char byte; + ssize_t n; + + do + n = read (tfork_pre_restore_fd, &byte, 1); + while (n < 0 && errno == EINTR); + if (n != 1) + return -1; + close (tfork_pre_restore_fd); + tfork_pre_restore_fd = -1; + return 0; + } + + if (strcmp (action, "tfork-source-detached") == 0 && tfork_source_detached_fd >= 0) + { + char byte = 1; + ssize_t n; + + do + n = write (tfork_source_detached_fd, &byte, 1); + while (n < 0 && errno == EINTR); + if (n != 1) + return -1; + close (tfork_source_detached_fd); + tfork_source_detached_fd = -1; + return 0; + } + if (strncmp (action, "orphan-pts-master", 17) == 0) { /* CRIU sends us the master FD via the 'orphan-pts-master' @@ -543,8 +603,11 @@ static void show_criu_log (const char *work_path, const char *log) { cleanup_free char *log_path = NULL; + cleanup_free char *tail = NULL; libcrun_error_t *tmp_err = NULL; - char line[1024]; + char line[CRIU_LOG_LINE_SIZE]; + size_t tail_index = 0; + size_t tail_count = 0; FILE *f; if (UNLIKELY (append_paths (&log_path, tmp_err, work_path, log, NULL)) < 0) @@ -563,12 +626,43 @@ show_criu_log (const char *work_path, const char *log) /* Log with error verbosity as this is the default. */ libcrun_error (0, "--- excerpt from CRIU log `%s`", log_path); + tail = calloc (CRIU_LOG_TAIL_LINES, CRIU_LOG_LINE_SIZE); + if (tail == NULL) + { + fclose (f); + return; + } + while (fgets (line, sizeof (line), f) != NULL) - if (strstr (line, "Error ") != NULL) - { - line[strcspn (line, "\n")] = '\0'; - libcrun_error (0, "%s", line); - } + { + char *slot = tail + tail_index * CRIU_LOG_LINE_SIZE; + strncpy (slot, line, CRIU_LOG_LINE_SIZE - 1); + slot[CRIU_LOG_LINE_SIZE - 1] = '\0'; + tail_index = (tail_index + 1) % CRIU_LOG_TAIL_LINES; + if (tail_count < CRIU_LOG_TAIL_LINES) + tail_count++; + + if (strstr (line, "Error ") != NULL + || strstr (line, "failed") != NULL || strstr (line, "FAILED") != NULL + || strstr (line, "Unable") != NULL || strstr (line, "Can't") != NULL + || strstr (line, "No such") != NULL) + { + line[strcspn (line, "\n")] = '\0'; + libcrun_error (0, "%s", line); + } + } + + if (tail_count > 0) + { + size_t start = (tail_count == CRIU_LOG_TAIL_LINES) ? tail_index : 0; + libcrun_error (0, "--- last %zu CRIU log lines", tail_count); + for (size_t i = 0; i < tail_count; i++) + { + char *entry = tail + ((start + i) % CRIU_LOG_TAIL_LINES) * CRIU_LOG_LINE_SIZE; + entry[strcspn (entry, "\n")] = '\0'; + libcrun_error (0, "%s", entry); + } + } fclose (f); libcrun_error (0, "--- end of excerpt"); @@ -1351,6 +1445,24 @@ libcrun_container_restore_linux_criu (libcrun_container_status_t *status, libcru } # define CRIU_TFORK_LOG_FILE "tfork.log" +# define CRIU_TFORK_RESTORE_LOG_FILE "tfork-restore.log" +# define CRIU_TFORK_MAX_COPY_LOGS 16 + +static void +show_criu_tfork_restore_copy_logs (const char *image_path, int copy_count) +{ + if (copy_count < 0) + copy_count = 0; + if (copy_count > CRIU_TFORK_MAX_COPY_LOGS) + copy_count = CRIU_TFORK_MAX_COPY_LOGS; + + for (int i = 0; i < copy_count; i++) + { + char log[64]; + snprintf (log, sizeof (log), "%s.copy%d", CRIU_TFORK_RESTORE_LOG_FILE, i); + show_criu_log (image_path, log); + } +} static int read_source_state_pid_cgroup (const char *path, pid_t *pid_out, char **cgroup_path_out, libcrun_error_t *err) @@ -1474,7 +1586,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec return crun_make_error (err, 0, "--tfork-snap-root, --tfork-snap-roots, or --tfork-copy=::--tfork-snap-root=PATH is required"); - if (cr_options->tfork_copies > 1 && cr_options->tfork_snap_roots_n > 0 + if (cr_options->tfork_copies >= 1 && cr_options->tfork_snap_roots_n > 0 && (size_t) cr_options->tfork_copies != cr_options->tfork_snap_roots_n) return crun_make_error (err, 0, "--tfork-copies=%d but --tfork-snap-roots has %zu entries", @@ -1516,7 +1628,11 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec cr_options->work_path = cr_options->image_path; } - libcriu_wrapper->criu_set_log_level (4); + /* Tfork used to force CRIU debug logging even for a normal crun request. + That makes log formatting and I/O grow with every task and VMA. Match + the runtime's requested verbosity; the explicit override above retains + full tracing for Podman's custom tfork path when needed. */ + libcriu_wrapper->criu_set_log_level (tfork_criu_log_level ()); libcriu_wrapper->criu_set_log_file (CRIU_TFORK_LOG_FILE); if (cr_options->tfork_ghost_limit > 0) @@ -1527,6 +1643,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec libcriu_wrapper->criu_set_pid (source_pid); libcriu_wrapper->criu_set_leave_running (true); + libcriu_wrapper->criu_set_ext_unix_sk (cr_options->ext_unix_sk); libcriu_wrapper->criu_set_file_locks (true); cgroup_mode = libcrun_get_cgroup_mode (err); @@ -1592,7 +1709,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec } } - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) libcriu_wrapper->criu_set_tfork_copies (cr_options->tfork_copies); if (cr_options->tfork_memdump_async) @@ -1628,6 +1745,13 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec libcriu_wrapper->criu_set_tfork_full_memcopy (true); } + if (libcriu_wrapper->criu_set_network_lock && cr_options->network_lock_method > 0) + { + ret = libcriu_wrapper->criu_set_network_lock (cr_options->network_lock_method); + if (UNLIKELY (ret < 0)) + return crun_make_error (err, 0, "CRIU: failed setting tfork network lock"); + } + if (cr_options->track_mem || cr_options->tfork_memdump) libcriu_wrapper->criu_set_track_mem (true); @@ -1805,10 +1929,17 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec if (UNLIKELY (ret < 0)) return ret; + tfork_pre_restore_fd = cr_options->tfork_pre_restore_fd; + tfork_source_detached_fd = cr_options->tfork_source_detached_fd; + libcriu_wrapper->criu_set_notify_cb (criu_notify); ret = libcriu_wrapper->criu_tfork(); + tfork_pre_restore_fd = -1; + tfork_source_detached_fd = -1; if (UNLIKELY (ret != 0)) { show_criu_log (cr_options->work_path, CRIU_TFORK_LOG_FILE); + show_criu_log (cr_options->image_path, CRIU_TFORK_RESTORE_LOG_FILE); + show_criu_tfork_restore_copy_logs (cr_options->image_path, cr_options->tfork_copies); return crun_make_error (err, 0, "criu_tfork failed: %d", ret); } @@ -1819,7 +1950,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec pid_t clone_pid; const char *pidfile_name = "tfork.pid"; - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) pidfile_name = "tfork.pid.copy0"; ret = append_paths (&pidfile_path, err, cr_options->image_path, pidfile_name, NULL); @@ -1834,7 +1965,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec if (UNLIKELY (clone_pid <= 0)) return crun_make_error (err, 0, "invalid clone PID %d in `%s`", (int) clone_pid, pidfile_path); - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) { char children_path[64]; cleanup_free char *children_buf = NULL; diff --git a/crun/src/tfork.c b/crun/src/tfork.c index 31db969ef..f0524dbc3 100644 --- a/crun/src/tfork.c +++ b/crun/src/tfork.c @@ -56,7 +56,10 @@ enum OPTION_TFORK_TCP_CLOSE, OPTION_TFORK_SNAP_MOUNT, OPTION_TFORK_DUMPD_PARENT, + OPTION_TFORK_PRE_RESTORE_FD, + OPTION_TFORK_SOURCE_DETACHED_FD, OPTION_TFORK_FULL_MEMCOPY, + OPTION_NETWORK_LOCK_METHOD, OPTION_PID_FILE, OPTION_CONSOLE_SOCKET, OPTION_NO_PIVOT, @@ -83,7 +86,7 @@ static struct argp_option options[] { "parent-path", OPTION_PARENT_PATH, "DIR", 0, "previous criu images dir, for incremental memdump chains", 0 }, { "tfork-memdump", OPTION_TFORK_MEMDUMP, 0, 0, "dump pages-*.img to image-path during tfork", 0 }, { "tfork-memdump-async", OPTION_TFORK_MEMDUMP_ASYNC, 0, 0, "async page dump (implies --tfork-memdump)", 0 }, - { "tfork-copies", OPTION_TFORK_COPIES, "N", 0, "produce N parallel clones (default 1)", 0 }, + { "tfork-copies", OPTION_TFORK_COPIES, "N", 0, "produce N clones through the n-copy helper (omitted: legacy direct single-copy)", 0 }, { "track-mem", OPTION_TRACK_MEM, 0, 0, "arm soft-dirty for chained incremental dumps", 0 }, { "manage-cgroups-mode", OPTION_MANAGE_CGROUPS_MODE, "MODE", 0, "cgroups mode: 'soft' (default), 'ignore', 'full' and 'strict'", 0 }, @@ -101,8 +104,14 @@ static struct argp_option options[] "dump ESTABLISHED TCP sockets as closed; the clone wakes with sockets in closed state (apps reconnect). Required for chromium / electron / any networked workload that holds long-poll connections.", 0 }, { "tfork-dumpd-parent", OPTION_TFORK_DUMPD_PARENT, "PID", 0, "host-pidns PID for dumpd to reparent itself to (so `podman stop`/`rm` cleans dumpd up via its parent). Only meaningful with --tfork-memdump-async. If unset, defaults to getppid() at exec time (conmon when invoked from podman). Same-pidns is enforced by criu — pass a host PID. See ../criu/Documentation/CRIU_TFORK_RPC_PEER_DISCONNECT_LEAK.md.", 0 }, + { "tfork-pre-restore-fd", OPTION_TFORK_PRE_RESTORE_FD, "FD", 0, + "block at the Phase A/Phase B boundary until one byte can be read from FD", 0 }, + { "tfork-source-detached-fd", OPTION_TFORK_SOURCE_DETACHED_FD, "FD", 0, + "write one byte to FD after CRIU has safely detached from the source tree", 0 }, { "tfork-full-memcopy", OPTION_TFORK_FULL_MEMCOPY, 0, 0, "ablation knob: replace anon-private vma_cherrypick CoW with a userspace physical copy of every anon page (via pread from /proc//mem in the restorer). File-backed VMAs, memfds, and SysV IPC are unaffected. For measuring the anon-CoW signal vs full-copy baseline.", 0 }, + { "network-lock", OPTION_NETWORK_LOCK_METHOD, "METHOD", 0, + "network lock backend: 'iptables', 'nftables', or 'skip'", 0 }, { "cgroup-root", OPTION_CGROUP_ROOT, "[CTRL:]PATH", 0, "rewrite dumped cgroup paths under PATH on restore. Under tfork the rewrite synthesizes cgns_prefix so prepare_cgns() unshares CLONE_NEWCGROUP at the clone's own cgroup boundary. Required for recursive clone-of-clone (gen-1 → gen-2) so /proc/self/cgroup reads `/` inside gen-1.", 0 }, { "pid-file", OPTION_PID_FILE, "FILE", 0, "where to write the PID of the container", 0 }, @@ -189,10 +198,37 @@ parse_opt (int key, char *arg, struct argp_state *state) } break; + case OPTION_TFORK_PRE_RESTORE_FD: + { + char *endp; + const char *p = argp_mandatory_argument (arg, state); + long val = strtol (p, &endp, 10); + if (*endp != '\0' || val < 0 || val > INT_MAX) + libcrun_fail_with_error (0, "--tfork-pre-restore-fd: invalid FD `%s`", p); + cr_options.tfork_pre_restore_fd = (int) val; + } + break; + + case OPTION_TFORK_SOURCE_DETACHED_FD: + { + char *endp; + const char *p = argp_mandatory_argument (arg, state); + long val = strtol (p, &endp, 10); + if (*endp != '\0' || val < 0 || val > INT_MAX) + libcrun_fail_with_error (0, "--tfork-source-detached-fd: invalid FD `%s`", p); + cr_options.tfork_source_detached_fd = (int) val; + } + break; + case OPTION_TFORK_FULL_MEMCOPY: cr_options.tfork_full_memcopy = true; break; + case OPTION_NETWORK_LOCK_METHOD: + cr_options.network_lock_method + = crun_parse_network_lock_method (argp_mandatory_argument (arg, state)); + break; + case OPTION_TRACK_MEM: cr_options.track_mem = true; break; @@ -359,7 +395,16 @@ int crun_command_tfork (struct crun_global_arguments *global_args, int argc, char **argv, libcrun_error_t *err) { cr_options.manage_cgroups_mode = -1; - cr_options.tfork_copies = 1; + cr_options.tfork_copies = 0; + cr_options.tfork_pre_restore_fd = -1; + cr_options.tfork_source_detached_fd = -1; + /* + * External Unix stream sockets can make Codex/tmux stacks dumpable, but they + * may hide unsupported socket topology. Keep the default fail-loud and expose + * this as an explicit escape hatch for agent integrations. + */ + if (getenv ("CRUN_TFORK_EXT_UNIX_SK") != NULL) + cr_options.ext_unix_sk = true; cr_options.leave_running = true; return crun_run_create_internal (global_args, argc, argv, container_tfork, get_options, &crun_context, &run_argp, diff --git a/linux-pagecache-cow/build_kernel.sh b/linux-pagecache-cow/build_kernel.sh index c36ce0a59..94bbaeefd 100755 --- a/linux-pagecache-cow/build_kernel.sh +++ b/linux-pagecache-cow/build_kernel.sh @@ -9,6 +9,7 @@ version="7.0.1" # Append a suffix LocalVersion="-pgcachecow" num_cores=$(($(nproc --all) - 2)) +num_cores=$(( num_cores > 1 ? num_cores : 1 )) ## Functions delete_old_kernel_contents () { diff --git a/linux-pagecache-cow/mm/filecow.c b/linux-pagecache-cow/mm/filecow.c index 136201a25..5ecf4739b 100644 --- a/linux-pagecache-cow/mm/filecow.c +++ b/linux-pagecache-cow/mm/filecow.c @@ -42,6 +42,11 @@ EXPORT_SYMBOL_GPL(filecow_stat_ra_order_layer_fallback); static atomic_long_t filecow_stat_lookup_install = ATOMIC_LONG_INIT(0); static atomic_long_t filecow_stat_lookup_miss = ATOMIC_LONG_INIT(0); static atomic_long_t filecow_stat_folios_unaccounted = ATOMIC_LONG_INIT(0); +static atomic_long_t filecow_stat_layers_allocated = ATOMIC_LONG_INIT(0); +static atomic_long_t filecow_stat_layers_freed = ATOMIC_LONG_INIT(0); +static atomic_long_t filecow_stat_layers_active = ATOMIC_LONG_INIT(0); +static atomic_long_t filecow_stat_fork_no_layer = ATOMIC_LONG_INIT(0); +static atomic_long_t filecow_stat_fork_reused_layer = ATOMIC_LONG_INIT(0); static atomic_long_t filecow_diag_evict_ok = ATOMIC_LONG_INIT(0); static atomic_long_t filecow_diag_evict_refuse = ATOMIC_LONG_INIT(0); static atomic_long_t filecow_diag_evict_skipped = ATOMIC_LONG_INIT(0); @@ -90,6 +95,16 @@ static int filecow_stats_show(struct seq_file *m, void *v) atomic_long_read(&filecow_stat_ra_order_layer_fallback)); seq_printf(m, "folios_unaccounted %ld\n", atomic_long_read(&filecow_stat_folios_unaccounted)); + seq_printf(m, "layers_allocated %ld\n", + atomic_long_read(&filecow_stat_layers_allocated)); + seq_printf(m, "layers_freed %ld\n", + atomic_long_read(&filecow_stat_layers_freed)); + seq_printf(m, "layers_active %ld\n", + atomic_long_read(&filecow_stat_layers_active)); + seq_printf(m, "fork_no_layer %ld\n", + atomic_long_read(&filecow_stat_fork_no_layer)); + seq_printf(m, "fork_reused_layer %ld\n", + atomic_long_read(&filecow_stat_fork_reused_layer)); seq_printf(m, "diag_evict_ok %ld\n", atomic_long_read(&filecow_diag_evict_ok)); seq_printf(m, "diag_evict_refuse %ld\n", @@ -372,6 +387,8 @@ struct filecow_layer *filecow_layer_alloc(struct address_space *primary) INIT_LIST_HEAD(&layer->children); INIT_LIST_HEAD(&layer->sibling_link); layer->wb_err = 0; + atomic_long_inc(&filecow_stat_layers_allocated); + atomic_long_inc(&filecow_stat_layers_active); return layer; } @@ -380,6 +397,8 @@ static void __filecow_layer_free_rcu(struct rcu_head *head) struct filecow_layer *layer = container_of(head, struct filecow_layer, rcu); WARN_ON_ONCE(!list_empty(&layer->sharers)); WARN_ON_ONCE(!list_empty(&layer->children)); + atomic_long_inc(&filecow_stat_layers_freed); + atomic_long_dec(&filecow_stat_layers_active); kmem_cache_free(filecow_layer_cache, layer); } @@ -896,6 +915,44 @@ bool filecow_aggressive_evict(struct folio *folio) } EXPORT_SYMBOL_GPL(filecow_aggressive_evict); +/* + * Return true when @mapping contains state which is newer than mapping->ro + * and therefore must be captured in a new layer. Filecow folios in i_pages + * are only local lookup aliases for an existing layer and ordinary XArray + * values are reclaim metadata. A tombstone or a normal folio, however, + * changes what a descendant must observe. + * + * The caller holds the mapping invalidate lock for write. Together with the + * frozen source, this excludes fault, write, and truncate paths which could add + * private state. Reclaim can still replace an origin folio with a tombstone, + * so the population scan refreshes has_tombstone under the XArray lock before + * taking the no-new-layer fast path. + */ +static bool filecow_mapping_has_private_state(struct address_space *mapping, + bool *has_tombstone) +{ + struct folio *folio; + bool has_private = false; + XA_STATE(xas, &mapping->i_pages, 0); + + *has_tombstone = false; + xas_lock_irq(&xas); + xas_for_each(&xas, folio, ULONG_MAX) { + if (xas_retry(&xas, folio)) + continue; + if (xa_is_tombstone(folio)) { + *has_tombstone = true; + has_private = true; + continue; + } + if (!xa_is_value(folio) && !folio_test_filecow(folio)) + has_private = true; + } + xas_unlock_irq(&xas); + + return has_private; +} + int address_space_fork(struct address_space *new, struct address_space *source) { struct filecow_layer *L; @@ -904,6 +961,9 @@ int address_space_fork(struct address_space *new, struct address_space *source) unsigned long *share_bitmap = NULL; int n = 0, capacity, i, moved = 0; int ret = 0; + bool any_shareable = false; + bool has_private_state; + bool has_tombstone; XA_STATE(xas, &source->i_pages, 0); if (!READ_ONCE(sysctl_filecow_enabled)) @@ -944,10 +1004,136 @@ int address_space_fork(struct address_space *new, struct address_space *source) ret = -EBUSY; goto out_unlock; } + has_private_state = filecow_mapping_has_private_state(source, + &has_tombstone); + + /* + * A clean mapping without a filecow layer has no in-memory state to + * preserve. The filesystem snapshot already supplies the child's data, + * so creating an empty layer here only retains one unnecessary layer per + * inode in a long-lived source's generation chain. + */ + if (!source->ro && !has_private_state) { + atomic_long_inc(&filecow_stat_fork_no_layer); + goto out_unlock; + } + + /* + * When all local entries are aliases of the current immutable layer, + * attach the child to that layer directly. Repeated forks of an + * unchanged source then consume one sharer reference per live child, + * rather than permanently extending the layer chain. + */ + if (source->ro && !has_private_state) { + L = source->ro; + spin_lock(&L->sharers_lock); + refcount_inc(&L->refs); + new->ro = L; + list_add(&new->filecow_link, &L->sharers); + spin_unlock(&L->sharers_lock); + atomic_long_inc(&filecow_stat_fork_reused_layer); + goto out_unlock; + } + + capacity = source->nrpages; + if (capacity != 0) { + batch = kvmalloc_array(capacity, sizeof(*batch), + GFP_KERNEL | __GFP_NOWARN); + indices = kvmalloc_array(capacity, sizeof(*indices), + GFP_KERNEL | __GFP_NOWARN); + if (!batch || !indices) { + ret = -ENOMEM; + goto out_free_arrays; + } + + xas_lock_irq(&xas); + xas_for_each(&xas, folio, ULONG_MAX) { + if (xas_retry(&xas, folio)) + continue; + if (xa_is_tombstone(folio)) { + has_private_state = true; + has_tombstone = true; + continue; + } + if (xa_is_value(folio)) + continue; + if (folio_test_large(folio)) { + atomic_long_inc(&filecow_stat_large_seen); + atomic_long_add(folio_nr_pages(folio), + &filecow_stat_large_skipped); + continue; + } + + if (folio_test_filecow(folio)) + continue; + if (WARN_ON_ONCE(n >= capacity)) + continue; + batch[n] = folio; + indices[n] = xas.xa_index; + n++; + } + xas_unlock_irq(&xas); + + if (n > 0) { + share_bitmap = kvmalloc_array(BITS_TO_LONGS(n), + sizeof(long), + GFP_KERNEL | __GFP_ZERO); + if (!share_bitmap) { + /* + * Keep a generation under memory pressure. The + * population loop will check each folio once without + * a result bitmap. + */ + atomic_long_inc(&filecow_stat_perfolio_hook_used); + any_shareable = true; + } else if (source->a_ops && + source->a_ops->folio_extents_shared_bulk) { + source->a_ops->folio_extents_shared_bulk(source, new, + indices, n, + share_bitmap); + atomic_long_inc(&filecow_stat_bulk_hook_used); + any_shareable = !bitmap_empty(share_bitmap, n); + } else { + atomic_long_inc(&filecow_stat_perfolio_hook_used); + for (i = 0; i < n; i++) { + if (!can_share_folio(batch[i], source, new)) + continue; + any_shareable = true; + __set_bit(i, share_bitmap); + } + } + } + } + + /* + * Ordinary cached folios whose extents are not shared with the snapshot + * must remain private to the source. The snapshot already has their + * correct on-disk contents, so an empty filecow generation would retain + * memory without preserving any state. + */ + if (!has_tombstone && !any_shareable) { + kvfree(share_bitmap); + kvfree(batch); + kvfree(indices); + if (!source->ro) { + atomic_long_inc(&filecow_stat_fork_no_layer); + goto out_unlock; + } + + L = source->ro; + spin_lock(&L->sharers_lock); + refcount_inc(&L->refs); + new->ro = L; + list_add(&new->filecow_link, &L->sharers); + spin_unlock(&L->sharers_lock); + atomic_long_inc(&filecow_stat_fork_reused_layer); + goto out_unlock; + } + L = filecow_layer_alloc(source); if (!L) { ret = -ENOMEM; - goto out_unlock; + goto out_free_arrays; } L->below = source->ro; if (source->ro) { @@ -963,6 +1149,7 @@ int address_space_fork(struct address_space *new, struct address_space *source) list_for_each_entry(next_as, &L_old->sharers, filecow_link) { struct inode *cand = next_as->host; + if (cand && !(inode_state_read_once(cand) & (I_FREEING | I_WILL_FREE | I_CLEAR))) { new_primary = cand; @@ -973,54 +1160,6 @@ int address_space_fork(struct address_space *new, struct address_space *source) } spin_unlock(&L_old->sharers_lock); } - capacity = source->nrpages; - if (capacity == 0) - goto install; - - batch = kvmalloc_array(capacity, sizeof(*batch), - GFP_KERNEL | __GFP_NOWARN); - indices = kvmalloc_array(capacity, sizeof(*indices), - GFP_KERNEL | __GFP_NOWARN); - if (!batch || !indices) { - ret = -ENOMEM; - goto out_free_layer; - } - - xas_lock_irq(&xas); - xas_for_each(&xas, folio, ULONG_MAX) { - if (n >= capacity) - break; - if (xas_retry(&xas, folio)) - continue; - if (xa_is_value(folio)) - continue; - if (folio_test_large(folio)) { - atomic_long_inc(&filecow_stat_large_seen); - atomic_long_add(folio_nr_pages(folio), - &filecow_stat_large_skipped); - continue; - } - - if (folio_test_filecow(folio)) - continue; - batch[n] = folio; - indices[n] = xas.xa_index; - n++; - } - xas_unlock_irq(&xas); - - if (n > 0 && source->a_ops && - source->a_ops->folio_extents_shared_bulk) { - share_bitmap = kvmalloc(BITS_TO_LONGS(n) * sizeof(long), - GFP_KERNEL | __GFP_ZERO); - if (share_bitmap) { - source->a_ops->folio_extents_shared_bulk( - source, new, indices, n, share_bitmap); - atomic_long_inc(&filecow_stat_bulk_hook_used); - } - } - if (!share_bitmap) - atomic_long_inc(&filecow_stat_perfolio_hook_used); for (i = 0; i < n; i++) { void *prev; @@ -1059,7 +1198,6 @@ int address_space_fork(struct address_space *new, struct address_space *source) kvfree(batch); kvfree(indices); -install: spin_lock(&L->sharers_lock); refcount_set(&L->refs, 2 + moved); source->ro = L; @@ -1083,14 +1221,8 @@ int address_space_fork(struct address_space *new, struct address_space *source) } return 0; -out_free_layer: - if (L->below) { - spin_lock(&L->below->sharers_lock); - list_add(&source->filecow_link, &L->below->sharers); - spin_unlock(&L->below->sharers_lock); - } - - kmem_cache_free(filecow_layer_cache, L); +out_free_arrays: + kvfree(share_bitmap); kvfree(batch); kvfree(indices); out_unlock: diff --git a/linux-pagecache-cow/tools/testing/selftests/Makefile b/linux-pagecache-cow/tools/testing/selftests/Makefile index 450f13ba4..06b77806a 100644 --- a/linux-pagecache-cow/tools/testing/selftests/Makefile +++ b/linux-pagecache-cow/tools/testing/selftests/Makefile @@ -30,6 +30,7 @@ TARGETS += dt TARGETS += efivarfs TARGETS += exec TARGETS += fchmodat2 +TARGETS += filecow TARGETS += filesystems TARGETS += filesystems/binderfs TARGETS += filesystems/epoll diff --git a/linux-pagecache-cow/tools/testing/selftests/filecow/Makefile b/linux-pagecache-cow/tools/testing/selftests/filecow/Makefile new file mode 100644 index 000000000..1ebc86dd4 --- /dev/null +++ b/linux-pagecache-cow/tools/testing/selftests/filecow/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +TEST_PROGS := filecow_layer_lifetime.sh + +include ../lib.mk diff --git a/linux-pagecache-cow/tools/testing/selftests/filecow/filecow_layer_lifetime.sh b/linux-pagecache-cow/tools/testing/selftests/filecow/filecow_layer_lifetime.sh new file mode 100755 index 000000000..9eba6dbe4 --- /dev/null +++ b/linux-pagecache-cow/tools/testing/selftests/filecow/filecow_layer_lifetime.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +set -euo pipefail + +readonly KSFT_SKIP=4 +readonly STATS=/proc/filecow_stats +readonly LOOPS="${FILECOW_TEST_LOOPS:-100}" +readonly TEST_ROOT="${FILECOW_TEST_ROOT:-}" + +skip() +{ + echo "1..0 # SKIP $*" + exit "$KSFT_SKIP" +} + +fail() +{ + echo "not ok 1 - filecow layer lifetime" + echo "# $*" + exit 1 +} + +stat_value() +{ + awk -v key="$1" '$1 == key { print $2; found = 1 } + END { if (!found) exit 1 }' "$STATS" +} + +[[ $EUID -eq 0 ]] || skip "must be run as root" +[[ -n "$TEST_ROOT" ]] || + skip "set FILECOW_TEST_ROOT to an existing btrfs directory" +[[ "$LOOPS" =~ ^[1-9][0-9]*$ ]] || skip "FILECOW_TEST_LOOPS must be positive" +[[ -r "$STATS" ]] || skip "$STATS is unavailable" +command -v btrfs >/dev/null || skip "btrfs-progs is unavailable" +[[ "$(findmnt -n -o FSTYPE -T "$TEST_ROOT")" == btrfs ]] || + skip "$TEST_ROOT is not on btrfs" + +for key in layers_allocated layers_active fork_no_layer fork_reused_layer; do + stat_value "$key" >/dev/null || + skip "running kernel does not expose the $key counter" +done + +workdir="$(mktemp -d "${TEST_ROOT%/}/filecow-layer-lifetime.XXXXXX")" +source_subvol="$workdir/source" +child_subvol="$workdir/child" + +cleanup() +{ + exec 8<&- 9<&- + if btrfs subvolume show "$child_subvol" >/dev/null 2>&1; then + btrfs subvolume delete "$child_subvol" >/dev/null + fi + if btrfs subvolume show "$source_subvol" >/dev/null 2>&1; then + btrfs subvolume delete "$source_subvol" >/dev/null + fi + rmdir "$workdir" 2>/dev/null || true +} +trap cleanup EXIT + +btrfs subvolume create "$source_subvol" >/dev/null +printf 'filecow-layer-lifetime\n' >"$source_subvol/cached" +dd if=/dev/zero bs=4096 count=16 status=none >>"$source_subvol/cached" +: >"$source_subvol/empty" +sync -f "$source_subvol/cached" + +# Populate one source folio. The first snapshot needs a layer for it; later +# unchanged snapshots must share that layer rather than extend the chain. +cat "$source_subvol/cached" >/dev/null +exec 8<"$source_subvol/cached" +exec 9<"$source_subvol/empty" + +allocated_before="$(stat_value layers_allocated)" +active_before="$(stat_value layers_active)" +no_layer_before="$(stat_value fork_no_layer)" +reused_before="$(stat_value fork_reused_layer)" + +for ((i = 0; i < LOOPS; i++)); do + btrfs subvolume snapshot "$source_subvol" "$child_subvol" >/dev/null + [[ "$(head -n 1 "$child_subvol/cached")" == filecow-layer-lifetime ]] || + fail "snapshot data mismatch in iteration $i" + stat "$child_subvol/empty" >/dev/null + btrfs subvolume delete "$child_subvol" >/dev/null +done + +allocated_after="$(stat_value layers_allocated)" +active_after="$(stat_value layers_active)" +no_layer_after="$(stat_value fork_no_layer)" +reused_after="$(stat_value fork_reused_layer)" + +allocated_delta=$((allocated_after - allocated_before)) +active_delta=$((active_after - active_before)) +no_layer_delta=$((no_layer_after - no_layer_before)) +reused_delta=$((reused_after - reused_before)) + +((allocated_delta <= 1)) || + fail "unchanged forks allocated $allocated_delta layers; expected at most 1" +((active_delta <= 1)) || + fail "unchanged forks retained $active_delta layers; expected at most 1" +((no_layer_delta >= LOOPS)) || + fail "empty inode skipped only $no_layer_delta layers; expected at least $LOOPS" +((reused_delta >= LOOPS - 1)) || + fail "cached inode reused only $reused_delta layers; expected at least $((LOOPS - 1))" + +echo "ok 1 - filecow layer lifetime" +echo "# loops=$LOOPS allocated_delta=$allocated_delta active_delta=$active_delta" +echo "# fork_no_layer_delta=$no_layer_delta fork_reused_layer_delta=$reused_delta" +echo "1..1" diff --git a/podman/cmd/podman/containers/clone.go b/podman/cmd/podman/containers/clone.go index e0264ade5..90f8e3d16 100644 --- a/podman/cmd/podman/containers/clone.go +++ b/podman/cmd/podman/containers/clone.go @@ -1,7 +1,9 @@ package containers import ( + jsonencoding "encoding/json" "fmt" + "os" "github.com/containers/podman/v5/cmd/podman/common" "github.com/containers/podman/v5/cmd/podman/registry" @@ -59,6 +61,15 @@ func cloneFlags(cmd *cobra.Command) { tforkOverlayBtrfsFlagName := "tfork-overlay-btrfs" flags.BoolVar(&ctrClone.TforkOverlayBtrfs, tforkOverlayBtrfsFlagName, false, "use overlay-on-btrfs for per-clone rootfs (default: per-clone btrfs subvolume snapshot; overlay shares lower's page cache across siblings; requires --live)") + tforkMetadataFlagName := "tfork-metadata" + flags.BoolVar(&ctrClone.TforkMetadata, tforkMetadataFlagName, false, "print live-clone identity, PID, and rootfs metadata as JSON (only with --live)") + + tforkInjectFileFlagName := "tfork-inject-file" + flags.StringSliceVar(&ctrClone.TforkInjectFiles, tforkInjectFileFlagName, nil, "inject COPY_INDEX:HOST_PATH:CONTAINER_PATH after live restore and before publication (new files use Podman's UID/GID; existing ownership is preserved; mode is forced to 0600)") + + tforkInjectSourceFileFlagName := "tfork-inject-source-file" + flags.StringSliceVar(&ctrClone.TforkInjectSourceFiles, tforkInjectSourceFileFlagName, nil, "atomically rotate HOST_PATH:CONTAINER_PATH in the frozen source after clone restore; a committed rotation is not rolled back by later publication failure") + tforkGhostLimitFlagName := "tfork-ghost-limit" flags.UintVar(&ctrClone.TforkGhostLimit, tforkGhostLimitFlagName, 256<<20, "raise CRIU's ghost-file size cap (bytes); GUI apps need >1MiB default (only with --live)") @@ -68,6 +79,9 @@ func cloneFlags(cmd *cobra.Command) { tforkFullMemcopyFlagName := "tfork-full-memcopy" flags.BoolVar(&ctrClone.TforkFullMemcopy, tforkFullMemcopyFlagName, false, "ablation: physical-copy anon-private VMAs instead of CoW (for measuring the anon-CoW signal; only with --live)") + tforkNetworkLockFlagName := "tfork-network-lock" + flags.StringVar(&ctrClone.TforkNetworkLock, tforkNetworkLockFlagName, "nftables", "network lock backend: iptables or nftables (only with --live)") + common.DefineCreateDefaults(&ctrClone.CreateOpts) common.DefineCreateFlags(cmd, &ctrClone.CreateOpts, entities.CloneMode) } @@ -140,6 +154,15 @@ func clone(cmd *cobra.Command, args []string) error { if ctrClone.TforkOverlayBtrfs { return fmt.Errorf("--tfork-overlay-btrfs requires --live: %w", define.ErrInvalidArg) } + if ctrClone.TforkMetadata { + return fmt.Errorf("--tfork-metadata requires --live: %w", define.ErrInvalidArg) + } + if len(ctrClone.TforkInjectFiles) > 0 { + return fmt.Errorf("--tfork-inject-file requires --live: %w", define.ErrInvalidArg) + } + if len(ctrClone.TforkInjectSourceFiles) > 0 { + return fmt.Errorf("--tfork-inject-source-file requires --live: %w", define.ErrInvalidArg) + } } ctrClone.ID = args[0] @@ -148,6 +171,9 @@ func clone(cmd *cobra.Command, args []string) error { if err != nil { return err } + if ctrClone.TforkMetadata { + return jsonencoding.NewEncoder(os.Stdout).Encode(rep.TforkClones) + } fmt.Println(rep.Id) return nil } diff --git a/podman/hack/tfork-network-lock-ab.sh b/podman/hack/tfork-network-lock-ab.sh new file mode 100755 index 000000000..208415bc7 --- /dev/null +++ b/podman/hack/tfork-network-lock-ab.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +set -euo pipefail + +PODMAN=${PODMAN:-podman} +PODMAN_GLOBAL_ARGS=${PODMAN_GLOBAL_ARGS:-} +CRIU_ROOT=${CRIU_ROOT:?set CRIU_ROOT} +OS4AGENT_CRUN=${OS4AGENT_CRUN:-crun} +SAMPLES=${SAMPLES:-10} +WORKLOAD_PROCESSES=${WORKLOAD_PROCESSES:-100} +IMAGE=${IMAGE:-docker.io/library/alpine:3.19} +PREFIX=${PREFIX:-tfork-network-lock-$RANDOM} +OUTPUT=${OUTPUT:-/tmp/tfork-network-lock-ab.tsv} +LOG_DIR=${LOG_DIR:-} + +read -r -a podman_global_args <<<"$PODMAN_GLOBAL_ARGS" +source_name=${PREFIX}-source + +podman_cmd() { + "$PODMAN" "${podman_global_args[@]}" "$@" +} + +cleanup() { + podman_cmd ps -a --format '{{.Names}}' | + awk -v prefix="$PREFIX" 'index($0, prefix) == 1' | + while read -r name; do + podman_cmd rm -f -t 0 "$name" >/dev/null 2>&1 || true + done +} +trap cleanup EXIT + +run_clone() { + local backend=$1 + local index=$2 + local name=${PREFIX}-${backend}-${index} + local started ended elapsed rootfs bundle + + started=$(date +%s%N) + env \ + PATH="$CRIU_ROOT/criu:$PATH" \ + LD_LIBRARY_PATH="$CRIU_ROOT/lib/c${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + OS4AGENT_CRUN="$OS4AGENT_CRUN" \ + "$PODMAN" "${podman_global_args[@]}" container clone \ + --live --tfork-overlay-btrfs \ + --tfork-network-lock="$backend" \ + "$source_name" "$name" >/dev/null + ended=$(date +%s%N) + elapsed=$(( (ended - started) / 1000000 )) + + [[ $(podman_cmd exec "$name" cat /tmp/sentinel) == source-before-fork ]] + if [[ -n $LOG_DIR ]]; then + rootfs=$(podman_cmd inspect --format '{{.Rootfs}}' "$name") + bundle=$(dirname "$rootfs") + test -f "$bundle/img/tfork.log" + cp "$bundle/img/tfork.log" \ + "$LOG_DIR/${backend}-${index}.tfork.log" + fi + podman_cmd kill -s KILL "$name" >/dev/null + podman_cmd rm -f -t 0 "$name" >/dev/null + printf '%s\t%d\t%d\n' "$backend" "$index" "$elapsed" | + tee -a "$OUTPUT" +} + +cleanup +: >"$OUTPUT" +if [[ -n $LOG_DIR ]]; then + mkdir -p "$LOG_DIR" +fi +podman_cmd run -d --name "$source_name" \ + --log-driver k8s-file \ + --security-opt seccomp=unconfined \ + --security-opt apparmor=unconfined \ + "$IMAGE" sh -c \ + 'count=$1 + i=1 + while [ "$i" -lt "$count" ]; do + sleep 86400 & + i=$((i + 1)) + done + echo source-before-fork >/tmp/sentinel + exec tail -f /dev/null' sh "$WORKLOAD_PROCESSES" >/dev/null + +for ((attempt = 0; attempt < 100; attempt++)); do + actual_processes=$(podman_cmd top "$source_name" pid | + awk 'NR > 1 { count++ } END { print count + 0 }') + if ((actual_processes == WORKLOAD_PROCESSES)); then + break + fi + sleep 0.05 +done +if ((actual_processes != WORKLOAD_PROCESSES)); then + printf 'expected %d source processes, found %d\n' \ + "$WORKLOAD_PROCESSES" "$actual_processes" >&2 + exit 1 +fi + +for ((i = 1; i <= SAMPLES; i++)); do + if ((i % 2)); then + run_clone iptables "$i" + run_clone nftables "$i" + else + run_clone nftables "$i" + run_clone iptables "$i" + fi +done + +podman_cmd rm -f -t 0 "$source_name" >/dev/null +trap - EXIT diff --git a/podman/hack/tfork-network-lock-smoke.sh b/podman/hack/tfork-network-lock-smoke.sh new file mode 100755 index 000000000..83abf5f88 --- /dev/null +++ b/podman/hack/tfork-network-lock-smoke.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +set -euo pipefail + +PODMAN=${PODMAN:-podman} +PODMAN_GLOBAL_ARGS=${PODMAN_GLOBAL_ARGS:-} +CRIU_ROOT=${CRIU_ROOT:?set CRIU_ROOT} +OS4AGENT_CRUN=${OS4AGENT_CRUN:-crun} +IMAGE=${IMAGE:-docker.io/library/alpine:3.19} +PREFIX=${PREFIX:-tfork-network-smoke-$RANDOM} + +read -r -a podman_global_args <<<"$PODMAN_GLOBAL_ARGS" +source_name=${PREFIX}-source +clone_name=${PREFIX}-clone +unsafe_name=${PREFIX}-unsafe + +podman_cmd() { + "$PODMAN" "${podman_global_args[@]}" "$@" +} + +cleanup() { + podman_cmd rm -f -t 0 "$clone_name" >/dev/null 2>&1 || true + podman_cmd rm -f -t 0 "$unsafe_name" >/dev/null 2>&1 || true + podman_cmd rm -f -t 0 "$source_name" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +assert_no_criu_table() { + local name=$1 + local pid + + pid=$(podman_cmd inspect --format '{{.State.Pid}}' "$name") + if nsenter -t "$pid" -n nft list tables 2>/dev/null | + grep -qi criu; then + printf 'stale CRIU nftables table in %s\n' "$name" >&2 + return 1 + fi +} + +cleanup +podman_cmd run -d --name "$source_name" \ + --log-driver k8s-file \ + --security-opt seccomp=unconfined \ + --security-opt apparmor=unconfined \ + "$IMAGE" sh -c \ + 'echo source-before-fork >/tmp/sentinel + while :; do + printf "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok" | + nc -l -p 18080 + done' >/dev/null + +for ((attempt = 0; attempt < 100; attempt++)); do + if [[ $(podman_cmd exec "$source_name" \ + wget -qO- http://127.0.0.1:18080 2>/dev/null || true) == ok ]]; then + break + fi + sleep 0.05 +done +if ((attempt == 100)); then + printf 'source listener did not become ready\n' >&2 + exit 1 +fi +assert_no_criu_table "$source_name" + +if env \ + PATH="$CRIU_ROOT/criu:$PATH" \ + LD_LIBRARY_PATH="$CRIU_ROOT/lib/c${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + OS4AGENT_CRUN="$OS4AGENT_CRUN" \ + "$PODMAN" "${podman_global_args[@]}" container clone \ + --live --tfork-overlay-btrfs \ + --tfork-network-lock=skip \ + "$source_name" "$unsafe_name" >/dev/null 2>&1; then + printf 'unsafe skip network backend was accepted\n' >&2 + exit 1 +fi +[[ $(podman_cmd exec "$source_name" cat /tmp/sentinel) == source-before-fork ]] + +env \ + PATH="$CRIU_ROOT/criu:$PATH" \ + LD_LIBRARY_PATH="$CRIU_ROOT/lib/c${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + OS4AGENT_CRUN="$OS4AGENT_CRUN" \ + "$PODMAN" "${podman_global_args[@]}" container clone \ + --live --tfork-overlay-btrfs \ + --tfork-network-lock=nftables \ + "$source_name" "$clone_name" >/dev/null + +[[ $(podman_cmd exec "$clone_name" cat /tmp/sentinel) == source-before-fork ]] +[[ $(podman_cmd exec "$source_name" \ + wget -qO- http://127.0.0.1:18080) == ok ]] +[[ $(podman_cmd exec "$clone_name" \ + wget -qO- http://127.0.0.1:18080) == ok ]] +assert_no_criu_table "$source_name" +assert_no_criu_table "$clone_name" + +podman_cmd exec "$clone_name" sh -c 'echo clone-only >/tmp/divergence' +test "$(podman_cmd exec "$clone_name" cat /tmp/divergence)" = clone-only +if podman_cmd exec "$source_name" test -e /tmp/divergence; then + printf 'clone filesystem write leaked into source\n' >&2 + exit 1 +fi + +cleanup +trap - EXIT +printf 'tfork nftables network-lock smoke: PASS\n' diff --git a/podman/hack/tfork-transaction-smoke.sh b/podman/hack/tfork-transaction-smoke.sh new file mode 100755 index 000000000..8008fb68f --- /dev/null +++ b/podman/hack/tfork-transaction-smoke.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +PODMAN=${PODMAN:-podman} +PODMAN_GLOBAL_ARGS=${PODMAN_GLOBAL_ARGS:-} +IMAGE=${IMAGE:-docker.io/library/alpine:3.19} +PREFIX=${PREFIX:-tfork-txn-$RANDOM} +TFORK_RUNTIME_ROOT=${TFORK_RUNTIME_ROOT:-/run/libpod/tfork} + +source_name=${PREFIX}-source +clone_name=${PREFIX}-clone +read -r -a podman_global_args <<<"$PODMAN_GLOBAL_ARGS" + +podman_cmd() { + "$PODMAN" "${podman_global_args[@]}" "$@" +} + +count_dirs() { + local path=$1 + if [[ ! -d $path ]]; then + echo 0 + return + fi + find "$path" -mindepth 1 -maxdepth 1 -type d -print | wc -l +} + +cleanup() { + podman_cmd kill -s KILL "$clone_name" >/dev/null 2>&1 || true + podman_cmd rm -f -t 0 "$clone_name" >/dev/null 2>&1 || true + podman_cmd kill -s KILL "$source_name" >/dev/null 2>&1 || true + podman_cmd rm -f -t 0 "$source_name" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +runtime_before=$(count_dirs "$TFORK_RUNTIME_ROOT") +bundle_root=$(podman_cmd info --format '{{.Store.GraphRoot}}')/tfork-bundles +bundles_before=$(count_dirs "$bundle_root") + +podman_cmd run -d --name "$source_name" \ + --log-driver k8s-file \ + --security-opt seccomp=unconfined \ + --security-opt apparmor=unconfined \ + "$IMAGE" sh -c 'echo source-before-fork >/tmp/sentinel; exec tail -f /dev/null' >/dev/null +cgroups_before=$(find /sys/fs/cgroup -type d -name 'libpod-*' -print 2>/dev/null | wc -l) + +for stage in \ + before_freeze \ + after_freeze \ + after_filesystem \ + before_restore \ + after_restore \ + after_thaw \ + after_register_0 \ + before_commit +do + if env PODMAN_TFORK_FAULT_INJECT="$stage" \ + "$PODMAN" "${podman_global_args[@]}" container clone --live --tfork-overlay-btrfs \ + "$source_name" "$clone_name" >/tmp/tfork-fault.stdout 2>/tmp/tfork-fault.stderr + then + echo "fault stage $stage unexpectedly succeeded" >&2 + exit 1 + fi + if ! grep -Fq "injected tfork fault at $stage" /tmp/tfork-fault.stderr; then + echo "fault stage $stage was not reached; clone failed for another reason" >&2 + sed -n '1,120p' /tmp/tfork-fault.stderr >&2 + exit 1 + fi + podman_cmd exec "$source_name" sh -c \ + 'test "$(cat /tmp/sentinel)" = source-before-fork' + if podman_cmd container exists "$clone_name"; then + echo "fault stage $stage published clone $clone_name" >&2 + exit 1 + fi + if [[ $(count_dirs "$bundle_root") != "$bundles_before" ]]; then + echo "fault stage $stage leaked a graphroot bundle" >&2 + exit 1 + fi + if [[ $(count_dirs "$TFORK_RUNTIME_ROOT") != "$runtime_before" ]]; then + echo "fault stage $stage leaked a runtime publication" >&2 + exit 1 + fi + if [[ $(find /sys/fs/cgroup -type d -name 'libpod-*' -print 2>/dev/null | wc -l) != "$cgroups_before" ]]; then + echo "fault stage $stage leaked a cgroup" >&2 + exit 1 + fi + echo "PASS fault=$stage" +done + +podman_cmd container clone --live --tfork-overlay-btrfs \ + "$source_name" "$clone_name" >/dev/null +[[ $(podman_cmd exec "$clone_name" cat /tmp/sentinel) == source-before-fork ]] +podman_cmd exec "$clone_name" sh -c 'echo child-only >/tmp/sentinel' +[[ $(podman_cmd exec "$source_name" cat /tmp/sentinel) == source-before-fork ]] + +podman_cmd kill -s KILL "$clone_name" >/dev/null +podman_cmd rm -f -t 0 "$clone_name" >/dev/null +if [[ $(count_dirs "$TFORK_RUNTIME_ROOT") != "$runtime_before" ]]; then + echo "successful clone cleanup leaked a runtime publication" >&2 + exit 1 +fi + +echo "PASS normal-fork" diff --git a/podman/libpod/container_config.go b/podman/libpod/container_config.go index dc5e02efe..f13a8a4f8 100644 --- a/podman/libpod/container_config.go +++ b/podman/libpod/container_config.go @@ -472,6 +472,7 @@ type ContainerMiscConfig struct { TforkPersistent string `json:"tforkPersistent,omitempty"` TforkSourceID string `json:"tforkSourceID,omitempty"` TforkParentClone string `json:"tforkParentClone,omitempty"` + TforkInitPIDStartTime uint64 `json:"tforkInitPIDStartTime,omitempty"` TforkDumpdHolderPid int `json:"tforkDumpdHolderPid,omitempty"` TforkDumpdHolderStartTime uint64 `json:"tforkDumpdHolderStartTime,omitempty"` } diff --git a/podman/libpod/container_internal_linux.go b/podman/libpod/container_internal_linux.go index 29159584d..d87256fc7 100644 --- a/podman/libpod/container_internal_linux.go +++ b/podman/libpod/container_internal_linux.go @@ -528,6 +528,22 @@ func (c *Container) CleanupExternalCloneStorage() { return } + // Direct n-copy tfork uses a small runtime-only publication directory for + // attach/log plumbing. It is outside the graphroot bundle and therefore + // was not covered by the storage teardown below. + runtimeBundle := filepath.Clean(c.config.ExternalBundlePath) + runtimeRoot := filepath.Clean("/run/libpod/tfork") + if runtimeBundle != "" && runtimeBundle != "." && + strings.HasPrefix(runtimeBundle, runtimeRoot+string(os.PathSeparator)) { + if err := os.RemoveAll(runtimeBundle); err != nil && !errors.Is(err, fs.ErrNotExist) { + logrus.Debugf("tfork: clone %s rm runtime bundle %s: %v", c.ID(), runtimeBundle, err) + } + parent := filepath.Dir(runtimeBundle) + if entries, err := os.ReadDir(parent); err == nil && len(entries) == 0 { + _ = os.Remove(parent) + } + } + if hpid := c.config.TforkDumpdHolderPid; hpid > 0 { expectedStart := c.config.TforkDumpdHolderStartTime curStart, stErr := ReadProcStartTime(hpid) diff --git a/podman/libpod/oci_conmon_common.go b/podman/libpod/oci_conmon_common.go index 0b521a180..f289e328e 100644 --- a/podman/libpod/oci_conmon_common.go +++ b/podman/libpod/oci_conmon_common.go @@ -828,6 +828,30 @@ func (r *ConmonOCIRuntime) CheckpointContainer(ctr *Container, options Container func (r *ConmonOCIRuntime) CheckConmonRunning(ctr *Container) (bool, error) { if ctr.state.ConmonPID == 0 { + // Direct tfork clones deliberately use a lightweight exit watcher + // instead of conmon. Their init PID, not a missing conmon PID, is the + // authoritative liveness signal. + if ctr.config.ExternalSetup && ctr.state.PID > 0 { + if expected := ctr.config.TforkInitPIDStartTime; expected != 0 { + current, err := ReadProcStartTime(ctr.state.PID) + if errors.Is(err, os.ErrNotExist) || errors.Is(err, unix.ESRCH) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("reading external clone pid %d start time: %w", ctr.state.PID, err) + } + if current != expected { + return false, nil + } + } + if err := unix.Kill(ctr.state.PID, 0); err != nil { + if errors.Is(err, unix.ESRCH) { + return false, nil + } + return false, fmt.Errorf("pinging external clone pid %d: %w", ctr.state.PID, err) + } + return true, nil + } // If the container is running or paused, assume Conmon is // running. We didn't record Conmon PID on some old versions, so // that is likely what's going on... diff --git a/podman/libpod/oci_conmon_tfork_test.go b/podman/libpod/oci_conmon_tfork_test.go new file mode 100644 index 000000000..63d080f60 --- /dev/null +++ b/podman/libpod/oci_conmon_tfork_test.go @@ -0,0 +1,44 @@ +//go:build !remote && linux + +package libpod + +import ( + "os" + "testing" +) + +func TestCheckConmonRunningExternalClonePIDIdentity(t *testing.T) { + pid := os.Getpid() + startTime, err := ReadProcStartTime(pid) + if err != nil { + t.Fatalf("reading test process start time: %v", err) + } + + ctr := &Container{ + config: &ContainerConfig{ + ContainerMiscConfig: ContainerMiscConfig{ + ExternalSetup: true, + TforkInitPIDStartTime: startTime, + }, + }, + state: &ContainerState{PID: pid}, + } + runtime := &ConmonOCIRuntime{} + + alive, err := runtime.CheckConmonRunning(ctr) + if err != nil { + t.Fatalf("checking matching process identity: %v", err) + } + if !alive { + t.Fatal("matching process identity reported dead") + } + + ctr.config.TforkInitPIDStartTime++ + alive, err = runtime.CheckConmonRunning(ctr) + if err != nil { + t.Fatalf("checking recycled process identity: %v", err) + } + if alive { + t.Fatal("mismatched process identity reported alive") + } +} diff --git a/podman/libpod/runtime_ctr.go b/podman/libpod/runtime_ctr.go index fafaac15a..0cb13dbfd 100644 --- a/podman/libpod/runtime_ctr.go +++ b/podman/libpod/runtime_ctr.go @@ -150,6 +150,17 @@ func (r *Runtime) RegisterExternalContainer(ctx context.Context, rSpec *spec.Spe } ctr.valid = true + createdPaths := make([]string, 0, 2) + defer func() { + if retErr == nil { + return + } + for i := len(createdPaths) - 1; i >= 0; i-- { + if err := os.RemoveAll(createdPaths[i]); err != nil { + logrus.Errorf("Removing path for failed external container registration %s: %v", createdPaths[i], err) + } + } + }() if ctr.config.StaticDir == "" { sd := filepath.Join(r.config.Engine.StaticDir, "containers", ctr.ID(), "userdata") @@ -157,11 +168,13 @@ func (r *Runtime) RegisterExternalContainer(ctx context.Context, rSpec *spec.Spe return nil, fmt.Errorf("creating static dir: %w", err) } ctr.config.StaticDir = sd + createdPaths = append(createdPaths, filepath.Dir(sd)) } rd := filepath.Join(r.storageConfig.RunRoot, "containers", ctr.ID(), "userdata") if err := os.MkdirAll(rd, 0o700); err != nil { return nil, fmt.Errorf("creating run dir: %w", err) } + createdPaths = append(createdPaths, filepath.Dir(rd)) ctr.state.RunDir = rd if ctr.config.ConmonPidFile == "" { diff --git a/podman/pkg/api/handlers/libpod/containers_tfork.go b/podman/pkg/api/handlers/libpod/containers_tfork.go new file mode 100644 index 000000000..cb1884636 --- /dev/null +++ b/podman/pkg/api/handlers/libpod/containers_tfork.go @@ -0,0 +1,128 @@ +//go:build !remote + +package libpod + +import ( + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "os" + + "github.com/containers/podman/v5/libpod" + "github.com/containers/podman/v5/libpod/define" + "github.com/containers/podman/v5/pkg/api/handlers/utils" + api "github.com/containers/podman/v5/pkg/api/types" + "github.com/containers/podman/v5/pkg/domain/entities" + "github.com/containers/podman/v5/pkg/domain/infra/abi" +) + +const tforkLocalAPIMaxBody = 1 << 20 + +type tforkLocalAPIRequest struct { + Name string `json:"name"` + Copies int `json:"copies,omitempty"` + Persistent string `json:"persistent,omitempty"` + WithPrevious bool `json:"withPrevious,omitempty"` + SharedUsr bool `json:"sharedUsr,omitempty"` + OverlayBtrfs *bool `json:"overlayBtrfs,omitempty"` + GhostLimit uint `json:"ghostLimit,omitempty"` + TCPClose *bool `json:"tcpClose,omitempty"` + FullMemcopy bool `json:"fullMemcopy,omitempty"` + NetworkLock string `json:"networkLock,omitempty"` + InjectFiles []string `json:"injectFiles,omitempty"` + InjectSourceFiles []string `json:"injectSourceFiles,omitempty"` +} + +// TforkCloneLocal exposes the existing live-clone ABI to a root-owned local +// Podman service. It is deliberately opt-in and Unix-socket-only: the normal +// Podman API has no live-clone contract, and this experimental endpoint must +// not silently become available on a TCP listener. +func TforkCloneLocal(w http.ResponseWriter, r *http.Request) { + if os.Getenv("PODMAN_TFORK_LOCAL_API") != "1" { + utils.Error(w, http.StatusNotFound, fmt.Errorf("local tfork API is disabled")) + return + } + localAddr, ok := r.Context().Value(http.LocalAddrContextKey).(net.Addr) + if !ok || localAddr.Network() != "unix" { + utils.Error(w, http.StatusForbidden, fmt.Errorf("local tfork API requires a Unix-domain listener")) + return + } + + r.Body = http.MaxBytesReader(w, r.Body, tforkLocalAPIMaxBody) + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + request := tforkLocalAPIRequest{} + if err := decoder.Decode(&request); err != nil { + utils.Error(w, http.StatusBadRequest, fmt.Errorf("decode local tfork request: %w", err)) + return + } + if request.Name == "" { + utils.Error(w, http.StatusBadRequest, fmt.Errorf("clone name is required")) + return + } + if request.Copies < 0 { + utils.Error(w, http.StatusBadRequest, fmt.Errorf("copies must be non-negative")) + return + } + if request.Persistent == "" { + request.Persistent = "async" + } + if request.Persistent != "async" && request.Persistent != "sync" { + utils.Error(w, http.StatusBadRequest, fmt.Errorf("persistent must be async or sync")) + return + } + if request.WithPrevious && request.Persistent == "" { + utils.Error(w, http.StatusBadRequest, fmt.Errorf("withPrevious requires persistent state")) + return + } + + overlayBtrfs := true + if request.OverlayBtrfs != nil { + overlayBtrfs = *request.OverlayBtrfs + } + tcpClose := true + if request.TCPClose != nil { + tcpClose = *request.TCPClose + } + if request.GhostLimit == 0 { + request.GhostLimit = 256 << 20 + } + if request.NetworkLock == "" { + request.NetworkLock = "nftables" + } + + runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) + containerEngine := abi.ContainerEngine{Libpod: runtime} + options := entities.ContainerCloneOptions{ + ID: utils.GetName(r), + Live: true, + Run: true, + Copies: request.Copies, + Persistent: request.Persistent, + WithPrevious: request.WithPrevious, + SharedUsr: request.SharedUsr, + TforkOverlayBtrfs: overlayBtrfs, + TforkMetadata: true, + TforkGhostLimit: request.GhostLimit, + TforkTCPClose: tcpClose, + TforkFullMemcopy: request.FullMemcopy, + TforkNetworkLock: request.NetworkLock, + TforkInjectFiles: request.InjectFiles, + TforkInjectSourceFiles: request.InjectSourceFiles, + } + options.CreateOpts.Name = request.Name + options.CreateOpts.IsClone = true + + report, err := containerEngine.ContainerClone(r.Context(), options) + if err != nil { + if errors.Is(err, define.ErrNoSuchCtr) { + utils.ContainerNotFound(w, options.ID, err) + return + } + utils.InternalServerError(w, err) + return + } + utils.WriteResponse(w, http.StatusCreated, report) +} diff --git a/podman/pkg/api/server/register_containers.go b/podman/pkg/api/server/register_containers.go index fee3a8aca..191f5c0b0 100644 --- a/podman/pkg/api/server/register_containers.go +++ b/podman/pkg/api/server/register_containers.go @@ -1817,5 +1817,8 @@ func (s *APIServer) registerContainersHandlers(r *mux.Router) error { // 500: // $ref: "#/responses/internalError" r.HandleFunc(VersionedPath("/libpod/containers/{name}/update"), s.APIHandler(libpod.UpdateContainer)).Methods(http.MethodPost) + // Experimental root-owned local live-clone endpoint. The handler rejects + // TCP listeners and remains disabled unless PODMAN_TFORK_LOCAL_API=1. + r.HandleFunc(VersionedPath("/libpod/containers/{name}/tfork"), s.APIHandler(libpod.TforkCloneLocal)).Methods(http.MethodPost) return nil } diff --git a/podman/pkg/domain/entities/containers.go b/podman/pkg/domain/entities/containers.go index 8295ffe98..89b83eb50 100644 --- a/podman/pkg/domain/entities/containers.go +++ b/podman/pkg/domain/entities/containers.go @@ -238,7 +238,15 @@ type RestoreOptions struct { type RestoreReport = types.RestoreReport type ContainerCreateReport struct { - Id string + Id string `json:"Id"` + TforkClones []TforkCloneMetadata `json:"TforkClones,omitempty"` +} + +type TforkCloneMetadata struct { + ID string `json:"id"` + Name string `json:"name"` + PID int `json:"pid"` + Rootfs string `json:"rootfs"` } // AttachOptions describes the cli and other values @@ -495,15 +503,19 @@ type ContainerCloneOptions struct { Run bool Force bool - Live bool - Copies int - Persistent string - WithPrevious bool - SharedUsr bool - TforkGhostLimit uint - TforkTCPClose bool - TforkFullMemcopy bool - TforkOverlayBtrfs bool + Live bool + Copies int + Persistent string + WithPrevious bool + SharedUsr bool + TforkGhostLimit uint + TforkTCPClose bool + TforkFullMemcopy bool + TforkNetworkLock string + TforkOverlayBtrfs bool + TforkMetadata bool + TforkInjectFiles []string + TforkInjectSourceFiles []string } // ContainerUpdateOptions containers options for updating an existing containers cgroup configuration diff --git a/podman/pkg/domain/infra/abi/container_tfork.go b/podman/pkg/domain/infra/abi/container_tfork.go index 734a3fcec..58e5881f5 100644 --- a/podman/pkg/domain/infra/abi/container_tfork.go +++ b/podman/pkg/domain/infra/abi/container_tfork.go @@ -4,11 +4,14 @@ import ( "bufio" "context" "encoding/json" + "errors" "fmt" + "io" "net" "os" "os/exec" "path/filepath" + "strconv" "strings" "syscall" "time" @@ -16,13 +19,14 @@ import ( "github.com/containers/podman/v5/libpod" "github.com/containers/podman/v5/libpod/define" - "strconv" + "github.com/cyphar/filepath-securejoin/pathrs-lite" "github.com/containers/podman/v5/pkg/domain/entities" "github.com/containers/podman/v5/utils" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" "go.podman.io/common/libnetwork/types" + commonconfig "go.podman.io/common/pkg/config" "go.podman.io/storage/pkg/stringid" "golang.org/x/sys/unix" ) @@ -31,11 +35,75 @@ const ( tforkSourceFreezeTimeout = 10 * time.Second tforkSourceThawTimeout = 10 * time.Second tforkCloneReadyTimeout = 60 * time.Second - tforkCrunFinishTimeout = tforkCloneReadyTimeout tforkCgroupPollInterval = 50 * time.Millisecond tforkClonePollInterval = 200 * time.Millisecond ) +type tforkSourceSyncMode string + +const ( + tforkSourceSyncFS tforkSourceSyncMode = "syncfs" + tforkSourceSyncGlobal tforkSourceSyncMode = "global" + tforkSourceSyncNone tforkSourceSyncMode = "none" +) + +func tforkSourceSyncModeFromEnv() (tforkSourceSyncMode, error) { + value := strings.ToLower(strings.TrimSpace(os.Getenv("PODMAN_TFORK_SYNC_MODE"))) + if value == "" { + return tforkSourceSyncFS, nil + } + mode := tforkSourceSyncMode(value) + switch mode { + case tforkSourceSyncFS, tforkSourceSyncGlobal, tforkSourceSyncNone: + return mode, nil + default: + return "", fmt.Errorf("invalid PODMAN_TFORK_SYNC_MODE=%q (must be syncfs, global, or none)", value) + } +} + +func tforkSyncSource(rootfs string) error { + mode, err := tforkSourceSyncModeFromEnv() + if err != nil { + return err + } + + switch mode { + case tforkSourceSyncNone: + logrus.Infof("tfork: source sync disabled by PODMAN_TFORK_SYNC_MODE=none") + return nil + case tforkSourceSyncGlobal: + if out, err := exec.Command("sync").CombinedOutput(); err != nil { + return fmt.Errorf("global sync: %s: %w", strings.TrimSpace(string(out)), err) + } + return nil + case tforkSourceSyncFS: + root, err := os.Open(rootfs) + if err != nil { + return fmt.Errorf("open source rootfs %s for syncfs: %w", rootfs, err) + } + defer root.Close() + if err := unix.Syncfs(int(root.Fd())); err != nil { + return fmt.Errorf("syncfs source rootfs %s: %w", rootfs, err) + } + return nil + default: + return fmt.Errorf("unsupported source sync mode %q", mode) + } +} + +func tforkCloneReadyTimeoutFromEnv() time.Duration { + value := strings.TrimSpace(os.Getenv("PODMAN_TFORK_CLONE_READY_TIMEOUT_SECS")) + if value == "" { + return tforkCloneReadyTimeout + } + seconds, err := strconv.Atoi(value) + if err != nil || seconds <= 0 { + logrus.Warnf("tfork: ignoring invalid PODMAN_TFORK_CLONE_READY_TIMEOUT_SECS=%q", value) + return tforkCloneReadyTimeout + } + return time.Duration(seconds) * time.Second +} + func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities.ContainerCloneOptions) (rep *entities.ContainerCreateReport, retErr error) { src, err := ic.Libpod.LookupContainer(opts.ID) if err != nil { @@ -49,11 +117,34 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if state != define.ContainerStateRunning { return nil, fmt.Errorf("source %q is not running (state=%s); tfork requires a live source", src.ID(), state.String()) } + if manager := src.CgroupManager(); manager != commonconfig.CgroupfsCgroupsManager { + return nil, fmt.Errorf("tfork currently requires the cgroupfs cgroup manager (source uses %q); retry Podman with --cgroup-manager=cgroupfs", manager) + } copies := opts.Copies if copies <= 0 { copies = 1 } + requestedCopies := copies + fileInjections, err := tforkParseFileInjections(opts.TforkInjectFiles, requestedCopies) + if err != nil { + return nil, err + } + sourceFileInjections, err := tforkParseSourceFileInjections(opts.TforkInjectSourceFiles) + if err != nil { + return nil, err + } + preparedSourceFileInjections, err := tforkPrepareFileInjections(sourceFileInjections) + if err != nil { + return nil, fmt.Errorf("prepare source file injection: %w", err) + } + useSingleCopyConmon := requestedCopies == 1 && os.Getenv("PODMAN_TFORK_SINGLE_COPY_CONMON") == "1" + // PODMAN_TFORK_SINGLE_COPY_DIRECT is a debugging escape hatch that skips + // the n-copy restore helper for single-copy experiments. Production paths + // keep the n-copy helper even for copies=1 so attach/status handling is + // consistent with multi-copy forks. + useSingleCopyDirect := requestedCopies == 1 && os.Getenv("PODMAN_TFORK_SINGLE_COPY_DIRECT") == "1" + useNcopyRestore := copies > 1 || (requestedCopies == 1 && !useSingleCopyConmon && !useSingleCopyDirect) var srcRootfs string if cfg := src.Config(); cfg != nil && cfg.ExternalSetup && cfg.Rootfs != "" { @@ -75,25 +166,38 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if err := os.MkdirAll(bundleDir, 0o700); err != nil { return nil, fmt.Errorf("mkdir bundle: %w", err) } + txn := newTforkCloneTransaction(ctx, ic.Libpod, src, bundleDir, copies) + defer func() { + if retErr != nil { + txn.rollback(retErr) + } + }() + // A completed source rotation is the irreversible commit point for that + // side effect. Clone rollback still removes unpublished children, but must + // not overwrite a credential the live source may have observed after thaw. + sourceInjectionCommitted := false + defer func() { + if retErr != nil && sourceInjectionCommitted { + retErr = fmt.Errorf("%w; source file rotation was committed and was not rolled back", retErr) + } + }() snapRO := filepath.Join(bundleDir, "snap-ro") + if err := tforkInjectFault("before_freeze"); err != nil { + return nil, err + } thawSource, err := tforkFreezeSourceCgroup(src, tforkSourceFreezeTimeout) if err != nil { return nil, fmt.Errorf("freeze source cgroup before tfork snapshot: %w", err) } - sourceThawed := false - defer func() { - if sourceThawed { - return - } - if err := thawSource(); err != nil { - logrus.Warnf("tfork: thaw source cgroup after clone setup: %v", err) - } - }() + txn.setSourceRestore(thawSource) + if err := tforkInjectFault("after_freeze"); err != nil { + return nil, err + } - if out, err := exec.Command("sync").CombinedOutput(); err != nil { - return nil, fmt.Errorf("sync: %s: %w", out, err) + if err := tforkSyncSource(srcRootfs); err != nil { + return nil, err } recursive := false @@ -139,6 +243,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities cloneIDs := make([]string, copies) cloneRootfsList := make([]string, copies) cloneRootfsRel := make([]string, copies) + overlapSocketPurge := os.Getenv("PODMAN_TFORK_OVERLAP_SOCKET_PURGE") == "1" for i := 0; i < copies; i++ { cloneIDs[i] = stringid.GenerateRandomID() var rel string @@ -162,8 +267,10 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities return nil, fmt.Errorf("setup snap-subvol rootfs for copy %d: %w", i, err) } } - if err := tforkPurgeSockets(cloneRootfsList[i]); err != nil { - logrus.Warnf("tfork: purge sockets in %s: %v (proceeding)", cloneRootfsList[i], err) + if !overlapSocketPurge { + if err := tforkPurgeSockets(cloneRootfsList[i]); err != nil { + logrus.Warnf("tfork: purge sockets in %s: %v (proceeding)", cloneRootfsList[i], err) + } } if srcPID, perr := src.PID(); perr == nil && srcPID > 0 { @@ -180,6 +287,43 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } } } + var socketPurgeRead *os.File + var socketPurgeFinished chan struct{} + var socketPurgeErr error + waitSocketPurge := func() error { + if socketPurgeFinished == nil { + return nil + } + <-socketPurgeFinished + return socketPurgeErr + } + if overlapSocketPurge { + readEnd, writeEnd, err := os.Pipe() + if err != nil { + return nil, fmt.Errorf("create socket-purge barrier: %w", err) + } + socketPurgeRead = readEnd + socketPurgeFinished = make(chan struct{}) + manifestPath := filepath.Join(bundleDir, "socket-paths.manifest0") + go func() { + defer close(socketPurgeFinished) + defer writeEnd.Close() + socketPurgeErr = tforkPurgeSocketsAndSignal(cloneRootfsList, manifestPath, writeEnd) + if socketPurgeErr != nil { + logrus.Errorf("tfork: overlapped socket purge failed; restore remains blocked: %v", socketPurgeErr) + } + }() + defer func() { + _ = waitSocketPurge() + if socketPurgeRead != nil { + _ = socketPurgeRead.Close() + } + }() + } + txn.setCloneIDs(cloneIDs) + if err := tforkInjectFault("after_filesystem"); err != nil { + return nil, err + } if recursive && parentUpperFrozen != "" { if err := os.RemoveAll(parentUpperFrozen); err != nil { @@ -283,6 +427,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if err := os.MkdirAll(imgDir, 0o700); err != nil { return nil, fmt.Errorf("mkdir img: %w", err) } + txn.setImageDir(imgDir) cloneCgroupPaths := make([]string, copies) if cfg := src.Config(); cfg != nil { @@ -295,6 +440,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities cloneCgroupPaths[i] = cgRel } } + txn.setCgroupPaths(cloneCgroupPaths) srcStatePath := fmt.Sprintf("/run/crun/%s/status", src.ID()) crunArgs := []string{ @@ -303,7 +449,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities "--source-state", srcStatePath, "--image-path", imgDir, } - if copies == 1 { + if !useNcopyRestore { crunArgs = append(crunArgs, "--tfork-snap-root", cloneRootfsList[0]) crunArgs = append(crunArgs, "--tfork-snap-mount", "/") if cloneCgroupPaths[0] != "" { @@ -361,6 +507,15 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities default: return nil, fmt.Errorf("invalid --persistent value %q (must be \"async\" or \"sync\")", opts.Persistent) } + if opts.TforkNetworkLock == "" { + opts.TforkNetworkLock = "nftables" + } + switch opts.TforkNetworkLock { + case "iptables", "nftables": + crunArgs = append(crunArgs, "--network-lock", opts.TforkNetworkLock) + default: + return nil, fmt.Errorf("invalid --tfork-network-lock value %q (must be \"iptables\" or \"nftables\")", opts.TforkNetworkLock) + } dumpdHolderPid := 0 var dumpdHolderStartTime uint64 if opts.Persistent == "async" { @@ -407,6 +562,9 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities crunArgs = append(crunArgs, "--parent-path", parentImgDir) logrus.Infof("tfork: --with-previous chains off clone %s (imgDir=%s)", parentCloneID, parentImgDir) } + if err := tforkInjectFault("before_restore"); err != nil { + return nil, err + } cloneLogPaths := make([]string, copies) cloneConmonPids := make([]int, copies) @@ -423,9 +581,19 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } }() hasTTY := len(ttySrcFds) > 0 - useConmon := copies == 1 + // Keep the legacy single-copy conmon bootstrap opt-in because it can fail + // before crun starts and report only `conmon reported pid=-1`. + useConmon := copies == 1 && useSingleCopyConmon if useConmon { + if socketPurgeRead != nil { + purgeErr := waitSocketPurge() + _ = socketPurgeRead.Close() + socketPurgeRead = nil + if purgeErr != nil { + return nil, fmt.Errorf("purge clone sockets before restore: %w", purgeErr) + } + } conmonInheritFds := inheritFds if hasTTY { conmonInheritFds = filterOutTtyInheritFds(inheritFds) @@ -448,6 +616,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities ghostLimit: opts.TforkGhostLimit, tcpClose: opts.TforkTCPClose, fullMemcopy: opts.TforkFullMemcopy, + networkLock: opts.TforkNetworkLock, cgroupRoot: cloneCgroupPaths[0], dumpdHolderPid: dumpdHolderPid, }) @@ -456,9 +625,29 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } cloneLogPaths[0] = logPath cloneConmonPids[0] = conmonPid + txn.trackPID(conmonPid, "conmon") logrus.Infof("tfork: clone %s up via conmon pid=%d; log=%s", cloneIDs[0], conmonPid, logPath) } else { directArgs := append([]string{}, crunArgs...) + eventReadiness := os.Getenv("PODMAN_TFORK_EVENT_READINESS") == "1" + var sourceDetachedRead *os.File + var sourceDetachedWrite *os.File + var sourceDetachedDone chan error + if eventReadiness { + var err error + sourceDetachedRead, sourceDetachedWrite, err = os.Pipe() + if err != nil { + return nil, fmt.Errorf("create source-detached event pipe: %w", err) + } + defer func() { + if sourceDetachedRead != nil { + _ = sourceDetachedRead.Close() + } + if sourceDetachedWrite != nil { + _ = sourceDetachedWrite.Close() + } + }() + } if dumpdHolderPid > 0 { directArgs = append(directArgs, fmt.Sprintf("--tfork-dumpd-parent=%d", dumpdHolderPid)) @@ -466,8 +655,8 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities var perCopyExtraFiles []*os.File var perCopyReadEnds []*os.File var perCopyArgs []string - skipTtySrcFds := copies > 1 && hasTTY - if copies > 1 { + skipTtySrcFds := useNcopyRestore && hasTTY + if useNcopyRestore { ifdsForPerCopy, stdioKeys := splitStdioInheritFds(inheritFds) inheritFds = ifdsForPerCopy extraFDBase := 3 @@ -500,6 +689,17 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities for _, pcArg := range perCopyArgs { directArgs = append(directArgs, pcArg) } + nextExtraFD := 3 + len(perCopyExtraFiles) + if hasTTY && !skipTtySrcFds { + nextExtraFD += len(ttySrcFds) + } + if socketPurgeRead != nil { + directArgs = append(directArgs, "--tfork-pre-restore-fd", strconv.Itoa(nextExtraFD)) + nextExtraFD++ + } + if sourceDetachedWrite != nil { + directArgs = append(directArgs, "--tfork-source-detached-fd", strconv.Itoa(nextExtraFD)) + } directArgs = append(directArgs, cloneIDs[0]) crunCmd := exec.Command(defaultCrunPath, directArgs...) crunCmd.Stdin = nil @@ -509,6 +709,12 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if len(perCopyExtraFiles) > 0 { crunCmd.ExtraFiles = append(crunCmd.ExtraFiles, perCopyExtraFiles...) } + if socketPurgeRead != nil { + crunCmd.ExtraFiles = append(crunCmd.ExtraFiles, socketPurgeRead) + } + if sourceDetachedWrite != nil { + crunCmd.ExtraFiles = append(crunCmd.ExtraFiles, sourceDetachedWrite) + } logPath := filepath.Join(bundleDir, "crun-tfork.log") logF, err := os.Create(logPath) if err != nil { @@ -520,6 +726,24 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities logF.Close() return nil, fmt.Errorf("start crun tfork: %w", err) } + if socketPurgeRead != nil { + _ = socketPurgeRead.Close() + socketPurgeRead = nil + } + if sourceDetachedWrite != nil { + _ = sourceDetachedWrite.Close() + sourceDetachedWrite = nil + sourceDetachedDone = make(chan error, 1) + go func() { + var byte [1]byte + n, err := sourceDetachedRead.Read(byte[:]) + if err == nil && n != 1 { + err = fmt.Errorf("short source-detached event read: %d bytes", n) + } + sourceDetachedDone <- err + }() + } + txn.trackPID(crunCmd.Process.Pid, "crun-tfork") crunDone := make(chan error, 1) var crunAborted bool defer func() { @@ -540,6 +764,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities shortBatch = shortBatch[:12] } runtimeAttachBase := filepath.Join("/run/libpod/tfork", shortBatch) + txn.setRuntimeBatchDir(runtimeAttachBase) for i := 0; i < copies; i++ { perCopyBundle := filepath.Join(runtimeAttachBase, fmt.Sprintf("%d", i)) if err := os.MkdirAll(perCopyBundle, 0o700); err != nil { @@ -562,8 +787,11 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } else { cloneLogPath = filepath.Join(bundleDir, fmt.Sprintf("clone.%d.log", i)) } - if err := spawnTforkStdioHelper(readEnd, perCopyAttachSocks[i], cloneLogPath, hasTTY); err != nil { + helperPID, err := spawnTforkStdioHelper(readEnd, perCopyAttachSocks[i], cloneLogPath, hasTTY) + if err != nil { logrus.Warnf("tfork: per-copy %d stdio-helper spawn: %v", i, err) + } else { + txn.trackPID(helperPID, fmt.Sprintf("stdio-helper-%d", i)) } if perCopyAttachSocks[i] != nil { _ = perCopyAttachSocks[i].Close() @@ -573,64 +801,127 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } pidFileFor := func(i int) string { - if copies == 1 { + if !useNcopyRestore { return filepath.Join(imgDir, "tfork.pid") } return filepath.Join(imgDir, fmt.Sprintf("tfork.pid.copy%d", i)) } statePath := fmt.Sprintf("/run/crun/%s/status", cloneIDs[0]) - needState := copies == 1 - deadline := time.Now().Add(tforkCloneReadyTimeout) + needState := !useNcopyRestore + cloneReadyTimeout := tforkCloneReadyTimeoutFromEnv() + deadline := time.Now().Add(cloneReadyTimeout) readyCopies := 0 stateReady := !needState crunExited := false var crunErr error - for readyCopies < copies || !stateReady { - if !crunExited { + if eventReadiness { + timer := time.NewTimer(cloneReadyTimeout) + defer timer.Stop() + sourceDetached := false + for !crunExited || !sourceDetached { select { + case detachErr := <-sourceDetachedDone: + if detachErr != nil { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("wait for tfork source-detached event: %w; see %s", detachErr, logPath) + } + sourceDetached = true + if len(preparedSourceFileInjections) == 0 { + if err := txn.restoreSourceOnce(); err != nil { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("early thaw at tfork source-detached event: %w", err) + } + logrus.Infof("tfork: source thawed at CRIU source-detached event") + } else { + logrus.Infof("tfork: source detached; deferring thaw until post-restore source file rotation") + } case crunErr = <-crunDone: crunExited = true - default: + if crunErr != nil { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("crun tfork failed before event readiness: %w; see %s", crunErr, logPath) + } + case <-timer.C: + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("timeout waiting %s for tfork source-detached and runtime-exit events; see %s", + cloneReadyTimeout, logPath) } } + readyCopies = 0 for i := 0; i < copies; i++ { if _, err := os.Stat(pidFileFor(i)); err == nil { readyCopies++ } } - if needState && !stateReady { - if _, err := os.Stat(statePath); err == nil { - stateReady = true - } + if needState { + _, err := os.Stat(statePath) + stateReady = err == nil } - if readyCopies >= copies && stateReady { - break + if readyCopies < copies || !stateReady { + return nil, fmt.Errorf("crun exited successfully but clone readiness artifacts are incomplete: pidfiles=%d/%d stateReady=%v; see %s", + readyCopies, copies, stateReady, logPath) } - if crunExited && readyCopies < copies { - tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) - crunAborted = true - return nil, fmt.Errorf("crun tfork exited before %d clones came up (got %d); see %s", - copies, readyCopies, logPath) + } else { + for readyCopies < copies || !stateReady { + if !crunExited { + select { + case crunErr = <-crunDone: + crunExited = true + default: + } + } + readyCopies = 0 + for i := 0; i < copies; i++ { + if _, err := os.Stat(pidFileFor(i)); err == nil { + readyCopies++ + } + } + if needState && !stateReady { + if _, err := os.Stat(statePath); err == nil { + stateReady = true + } + } + if readyCopies >= copies && stateReady { + break + } + if crunExited && readyCopies < copies { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + if purgeErr := waitSocketPurge(); purgeErr != nil { + return nil, fmt.Errorf("purge clone sockets before restore: %w", purgeErr) + } + return nil, fmt.Errorf("crun tfork exited before %d clones came up (got %d); see %s", + copies, readyCopies, logPath) + } + if time.Now().After(deadline) { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("timeout waiting for %d tfork.pid* files in %s (got %d, stateReady=%v); see %s", + copies, imgDir, readyCopies, stateReady, logPath) + } + time.Sleep(tforkClonePollInterval) } - if time.Now().After(deadline) { - tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) - crunAborted = true - return nil, fmt.Errorf("timeout waiting for %d tfork.pid* files in %s (got %d, stateReady=%v); see %s", - copies, imgDir, readyCopies, stateReady, logPath) + if !crunExited { + select { + case crunErr = <-crunDone: + crunExited = true + case <-time.After(cloneReadyTimeout): + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("timeout waiting %s for crun tfork to finish after %d clones came up; see %s", + cloneReadyTimeout, copies, logPath) + } } - time.Sleep(tforkClonePollInterval) } - if !crunExited { - select { - case crunErr = <-crunDone: - crunExited = true - case <-time.After(tforkCrunFinishTimeout): - tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) - crunAborted = true - return nil, fmt.Errorf("timeout waiting %s for crun tfork to finish after %d clones came up; see %s", - tforkCrunFinishTimeout, copies, logPath) - } + if purgeErr := waitSocketPurge(); purgeErr != nil { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("purge clone sockets before restore: %w", purgeErr) } if crunErr != nil { tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) @@ -640,26 +931,66 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } logrus.Infof("tfork: batch %s up (N=%d); crun-tfork.log at %s", batchID, copies, logPath) } + if err := tforkInjectFault("after_restore"); err != nil { + return nil, err + } + if len(preparedSourceFileInjections) > 0 { + srcPID, err := src.PID() + if err != nil { + return nil, fmt.Errorf("read frozen source PID for file rotation: %w", err) + } + if err := tforkInstallPreparedFileInjections(srcPID, preparedSourceFileInjections, nil); err != nil { + return nil, fmt.Errorf("rotate files in frozen source: %w", err) + } + sourceInjectionCommitted = true + logrus.Infof("tfork: committed %d source file rotation(s); later clone rollback will retain them", + len(preparedSourceFileInjections)) + } - if err := thawSource(); err != nil { + if err := txn.restoreSourceOnce(); err != nil { logrus.Warnf("tfork: clones are up, but thawing source cgroup after restore failed: %v", err) return nil, fmt.Errorf("thaw source cgroup after tfork restore: %w", err) } - sourceThawed = true + if err := tforkInjectFault("after_thaw"); err != nil { + return nil, err + } + srcPID, err := src.PID() + if err != nil { + return nil, fmt.Errorf("read source PID after restore: %w", err) + } + if err := tforkPIDRunning(srcPID); err != nil { + return nil, fmt.Errorf("source is not usable after tfork restore: %w", err) + } srcCfg := src.Config() if srcCfg == nil { return nil, fmt.Errorf("source %q: could not read libpod config", src.ID()) } + visibleCloneIDs := make([]string, 0, requestedCopies) + visibleClones := make([]entities.TforkCloneMetadata, 0, requestedCopies) for i, cloneID := range cloneIDs { - clonePID, err := readTforkClonePID(cloneID, imgDir, i, copies) + clonePID, err := readTforkClonePID(cloneID, imgDir, i, copies, useNcopyRestore) if err != nil { return nil, fmt.Errorf("read clone %d PID: %w", i, err) } + txn.trackPID(clonePID, fmt.Sprintf("clone-init-%d", i)) + if err := tforkPIDRunning(clonePID); err != nil { + return nil, fmt.Errorf("clone %d is not running before publication: %w", i, err) + } + clonePIDStartTime, err := libpod.ReadProcStartTime(clonePID) + if err != nil { + return nil, fmt.Errorf("read clone %d PID start time: %w", i, err) + } + for _, injection := range fileInjections[i] { + if err := tforkInjectFileIntoProcessRoot(clonePID, injection); err != nil { + return nil, fmt.Errorf("inject clone %d file %s: %w", i, injection.destination, err) + } + } cloneCfg, err := buildCloneContainerConfig(srcCfg, cloneID, cloneNames[i], cloneRootfsList[i], cloneSpecs[i]) if err != nil { return nil, fmt.Errorf("build clone %d config: %w", i, err) } + cloneCfg.TforkInitPIDStartTime = clonePIDStartTime if len(srcCfg.PortMappings) > 0 { clonePorts, err := buildClonePortMappings(srcCfg.PortMappings) if err != nil { @@ -689,16 +1020,17 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if err != nil { return nil, fmt.Errorf("register clone %d (%s) in libpod state: %w", i, cloneID, err) } + txn.addRegistered(ctr) if opts.TforkOverlayBtrfs { tforkFreezeUpperForRollback(bundleDir, i, copies) } if cloneConmonPids[i] > 0 { if err := ctr.SetConmonPID(cloneConmonPids[i]); err != nil { - logrus.Warnf("tfork: clone %s SetConmonPID(%d): %v", cloneID, cloneConmonPids[i], err) + return nil, fmt.Errorf("clone %s SetConmonPID(%d): %w", cloneID, cloneConmonPids[i], err) } } if err := ic.Libpod.SetupExternalCloneNetwork(src, ctr, clonePID); err != nil { - logrus.Warnf("tfork: clone %s network setup failed (clone has no network): %v", cloneID, err) + return nil, fmt.Errorf("clone %s network setup: %w", cloneID, err) } if cmd := exec.Command("nsenter", "-t", strconv.Itoa(clonePID), "-n", "--", @@ -709,17 +1041,385 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } } if err := ctr.MoveExternalCloneToOwnCgroup(clonePID); err != nil { - logrus.Warnf("tfork: clone %s F8 cgroup migration failed: %v (clone shares source's cgroup)", cloneID, err) + return nil, fmt.Errorf("clone %s cgroup migration: %w", cloneID, err) } if cloneConmonPids[i] == 0 { - if err := spawnTforkExitWatcher(ctx, ic.Libpod, ctr, clonePID); err != nil { - logrus.Warnf("tfork: clone %s exit-watcher spawn: %v (podman will fall back to F4.2 /proc liveness check)", cloneID, err) + watcherPID, err := spawnTforkExitWatcher(ctx, ic.Libpod, ctr, clonePID) + if err != nil { + return nil, fmt.Errorf("tfork: clone %s exit-watcher spawn: %w", cloneID, err) } + txn.trackPID(watcherPID, fmt.Sprintf("exit-watcher-%d", i)) + } + if err := tforkPIDRunning(clonePID); err != nil { + return nil, fmt.Errorf("clone %d died during publication: %w", i, err) + } + if err := tforkInjectFault(fmt.Sprintf("after_register_%d", i)); err != nil { + return nil, err } logrus.Infof("tfork: clone %s (%s) registered in libpod state, pid=%d", cloneID, cloneNames[i], clonePID) + visibleCloneIDs = append(visibleCloneIDs, cloneID) + visibleClones = append(visibleClones, entities.TforkCloneMetadata{ + ID: cloneID, + Name: cloneNames[i], + PID: clonePID, + Rootfs: cloneRootfsList[i], + }) + } + + if err := tforkInjectFault("before_commit"); err != nil { + return nil, err + } + txn.commit() + return &entities.ContainerCreateReport{ + Id: strings.Join(visibleCloneIDs, "\n"), + TforkClones: visibleClones, + }, nil +} + +type tforkFileInjection struct { + source string + destination string +} + +const tforkMaximumInjectionSize = 1 << 20 + +type tforkPreparedFileInjection struct { + source string + destination string + payload []byte +} + +type tforkStagedFileInjection struct { + parent *os.File + destination string + temporary string + backup string + existed bool + originalMoved bool + replacementInstalled bool +} + +func tforkParseFileInjections(specs []string, copies int) (map[int][]tforkFileInjection, error) { + parsed := make(map[int][]tforkFileInjection) + for _, spec := range specs { + parts := strings.SplitN(spec, ":", 3) + if len(parts) != 3 { + return nil, fmt.Errorf("invalid --tfork-inject-file %q (expected COPY_INDEX:HOST_PATH:CONTAINER_PATH)", spec) + } + copyIndex, err := strconv.Atoi(parts[0]) + if err != nil || copyIndex < 0 || copyIndex >= copies { + return nil, fmt.Errorf("invalid --tfork-inject-file copy index %q for %d copies", parts[0], copies) + } + source := filepath.Clean(parts[1]) + destination := filepath.Clean(parts[2]) + if !filepath.IsAbs(source) || source == string(os.PathSeparator) { + return nil, fmt.Errorf("tfork injection source must be a non-root absolute path: %q", parts[1]) + } + if !filepath.IsAbs(destination) || destination == string(os.PathSeparator) { + return nil, fmt.Errorf("tfork injection destination must be a non-root absolute path: %q", parts[2]) + } + parsed[copyIndex] = append(parsed[copyIndex], tforkFileInjection{ + source: source, + destination: destination, + }) + } + return parsed, nil +} + +func tforkParseSourceFileInjections(specs []string) ([]tforkFileInjection, error) { + parsed := make([]tforkFileInjection, 0, len(specs)) + seenDestinations := make(map[string]struct{}, len(specs)) + for _, spec := range specs { + parts := strings.SplitN(spec, ":", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid --tfork-inject-source-file %q (expected HOST_PATH:CONTAINER_PATH)", spec) + } + source := filepath.Clean(parts[0]) + destination := filepath.Clean(parts[1]) + if !filepath.IsAbs(source) || source == string(os.PathSeparator) { + return nil, fmt.Errorf("tfork source injection source must be a non-root absolute path: %q", parts[0]) + } + if !filepath.IsAbs(destination) || destination == string(os.PathSeparator) { + return nil, fmt.Errorf("tfork source injection destination must be a non-root absolute path: %q", parts[1]) + } + if _, exists := seenDestinations[destination]; exists { + return nil, fmt.Errorf("duplicate tfork source injection destination %q", destination) + } + seenDestinations[destination] = struct{}{} + parsed = append(parsed, tforkFileInjection{source: source, destination: destination}) + } + return parsed, nil +} + +func tforkReadInjectionSource(path string) ([]byte, error) { + sourceFD, err := unix.Open(path, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if err != nil { + return nil, fmt.Errorf("open source %s: %w", path, err) + } + source := os.NewFile(uintptr(sourceFD), path) + defer source.Close() + info, err := source.Stat() + if err != nil { + return nil, fmt.Errorf("stat source %s: %w", path, err) + } + if !info.Mode().IsRegular() || info.Size() > tforkMaximumInjectionSize { + return nil, fmt.Errorf("source must be a regular file no larger than %d bytes", tforkMaximumInjectionSize) + } + payload, err := io.ReadAll(io.LimitReader(source, tforkMaximumInjectionSize+1)) + if err != nil { + return nil, fmt.Errorf("read source %s: %w", path, err) + } + if len(payload) > tforkMaximumInjectionSize { + return nil, fmt.Errorf("source grew beyond %d bytes while being read", tforkMaximumInjectionSize) + } + return payload, nil +} + +func tforkPrepareFileInjections(injections []tforkFileInjection) ([]tforkPreparedFileInjection, error) { + prepared := make([]tforkPreparedFileInjection, 0, len(injections)) + for _, injection := range injections { + payload, err := tforkReadInjectionSource(injection.source) + if err != nil { + return nil, err + } + prepared = append(prepared, tforkPreparedFileInjection{ + source: injection.source, + destination: injection.destination, + payload: payload, + }) + } + return prepared, nil +} + +func tforkInjectionTemporaryNames(parentFD int) (string, string, error) { + for attempt := 0; attempt < 16; attempt++ { + id := stringid.GenerateRandomID() + if len(id) > 16 { + id = id[:16] + } + temporary := ".tfork-inject-new-" + id + backup := ".tfork-inject-old-" + id + var stat unix.Stat_t + if err := unix.Fstatat(parentFD, temporary, &stat, unix.AT_SYMLINK_NOFOLLOW); !errors.Is(err, unix.ENOENT) { + if err != nil { + return "", "", fmt.Errorf("check temporary injection path: %w", err) + } + continue + } + if err := unix.Fstatat(parentFD, backup, &stat, unix.AT_SYMLINK_NOFOLLOW); !errors.Is(err, unix.ENOENT) { + if err != nil { + return "", "", fmt.Errorf("check backup injection path: %w", err) + } + continue + } + return temporary, backup, nil + } + return "", "", fmt.Errorf("cannot allocate unique source injection paths") +} + +func tforkStagePreparedFileInjection(root string, injection tforkPreparedFileInjection) (*tforkStagedFileInjection, error) { + parent, err := pathrs.OpenInRoot(root, filepath.Dir(injection.destination)) + if err != nil { + return nil, fmt.Errorf("open destination parent for %s: %w", injection.destination, err) + } + parentFD := int(parent.Fd()) + stage := &tforkStagedFileInjection{ + parent: parent, + destination: filepath.Base(injection.destination), + } + var existing unix.Stat_t + err = unix.Fstatat(parentFD, stage.destination, &existing, unix.AT_SYMLINK_NOFOLLOW) + if err == nil { + if existing.Mode&unix.S_IFMT != unix.S_IFREG { + parent.Close() + return nil, fmt.Errorf("destination %s exists and is not a regular file", injection.destination) + } + stage.existed = true + } else if !errors.Is(err, unix.ENOENT) { + parent.Close() + return nil, fmt.Errorf("stat destination %s: %w", injection.destination, err) + } + + stage.temporary, stage.backup, err = tforkInjectionTemporaryNames(parentFD) + if err != nil { + parent.Close() + return nil, err + } + temporaryFD, err := unix.Openat(parentFD, stage.temporary, + unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0o600) + if err != nil { + parent.Close() + return nil, fmt.Errorf("create temporary injection file for %s: %w", injection.destination, err) + } + temporary := os.NewFile(uintptr(temporaryFD), stage.temporary) + failed := true + defer func() { + if failed { + temporary.Close() + _ = unix.Unlinkat(parentFD, stage.temporary, 0) + parent.Close() + } + }() + if n, err := temporary.Write(injection.payload); err != nil { + return nil, fmt.Errorf("write temporary injection file for %s: %w", injection.destination, err) + } else if n != len(injection.payload) { + return nil, fmt.Errorf("write temporary injection file for %s: %w", injection.destination, io.ErrShortWrite) + } + if stage.existed { + var temporaryStat unix.Stat_t + if err := unix.Fstat(temporaryFD, &temporaryStat); err != nil { + return nil, fmt.Errorf("stat temporary injection file for %s: %w", injection.destination, err) + } + if temporaryStat.Uid != existing.Uid || temporaryStat.Gid != existing.Gid { + if err := unix.Fchown(temporaryFD, int(existing.Uid), int(existing.Gid)); err != nil { + return nil, fmt.Errorf("preserve ownership for %s: %w", injection.destination, err) + } + } + } + if err := unix.Fchmod(temporaryFD, 0o600); err != nil { + return nil, fmt.Errorf("chmod temporary injection file for %s: %w", injection.destination, err) + } + if err := temporary.Close(); err != nil { + return nil, fmt.Errorf("close temporary injection file for %s: %w", injection.destination, err) + } + failed = false + return stage, nil +} + +func tforkCloseStagedFileInjections(staged []*tforkStagedFileInjection) { + for _, stage := range staged { + if stage.parent != nil { + stage.parent.Close() + stage.parent = nil + } + } +} + +func tforkRemoveStagedTemporaryFiles(staged []*tforkStagedFileInjection) { + for _, stage := range staged { + if stage.parent != nil && stage.temporary != "" { + _ = unix.Unlinkat(int(stage.parent.Fd()), stage.temporary, 0) + } + } +} + +func tforkRollbackInstalledFileInjections(staged []*tforkStagedFileInjection) error { + var rollbackErrors []error + for i := len(staged) - 1; i >= 0; i-- { + stage := staged[i] + parentFD := int(stage.parent.Fd()) + if stage.originalMoved { + if err := unix.Renameat(parentFD, stage.backup, parentFD, stage.destination); err != nil { + rollbackErrors = append(rollbackErrors, + fmt.Errorf("restore original %s: %w", stage.destination, err)) + } else { + stage.originalMoved = false + stage.replacementInstalled = false + } + } else if stage.replacementInstalled { + if err := unix.Unlinkat(parentFD, stage.destination, 0); err != nil && !errors.Is(err, unix.ENOENT) { + rollbackErrors = append(rollbackErrors, + fmt.Errorf("remove new destination %s: %w", stage.destination, err)) + } else { + stage.replacementInstalled = false + } + } + } + return errors.Join(rollbackErrors...) +} + +func tforkInstallPreparedFileInjections(pid int, injections []tforkPreparedFileInjection, afterCommit func(int) error) error { + root := filepath.Join("/proc", strconv.Itoa(pid), "root") + staged := make([]*tforkStagedFileInjection, 0, len(injections)) + for _, injection := range injections { + stage, err := tforkStagePreparedFileInjection(root, injection) + if err != nil { + tforkRemoveStagedTemporaryFiles(staged) + tforkCloseStagedFileInjections(staged) + return err + } + staged = append(staged, stage) + } + defer tforkCloseStagedFileInjections(staged) + + for i, stage := range staged { + parentFD := int(stage.parent.Fd()) + if stage.existed { + if err := unix.Renameat(parentFD, stage.destination, parentFD, stage.backup); err != nil { + rollbackErr := tforkRollbackInstalledFileInjections(staged) + tforkRemoveStagedTemporaryFiles(staged) + return errors.Join(fmt.Errorf("backup destination %s: %w", stage.destination, err), rollbackErr) + } + stage.originalMoved = true + } + if err := unix.Renameat(parentFD, stage.temporary, parentFD, stage.destination); err != nil { + rollbackErr := tforkRollbackInstalledFileInjections(staged) + tforkRemoveStagedTemporaryFiles(staged) + return errors.Join(fmt.Errorf("install destination %s: %w", stage.destination, err), rollbackErr) + } + stage.replacementInstalled = true + if afterCommit != nil { + if err := afterCommit(i); err != nil { + rollbackErr := tforkRollbackInstalledFileInjections(staged) + tforkRemoveStagedTemporaryFiles(staged) + return errors.Join(err, rollbackErr) + } + } } - return &entities.ContainerCreateReport{Id: strings.Join(cloneIDs, "\n")}, nil + for _, stage := range staged { + if stage.originalMoved { + if err := unix.Unlinkat(int(stage.parent.Fd()), stage.backup, 0); err != nil { + logrus.Warnf("tfork: remove committed source injection backup %s: %v", stage.backup, err) + } + stage.originalMoved = false + } + } + return nil +} + +// New destination files inherit Podman's filesystem UID/GID, while an existing +// destination retains its ownership. All destinations are forced to mode 0600; +// this interface intentionally does not provide ownership or mode overrides. +func tforkInjectFileIntoProcessRoot(pid int, injection tforkFileInjection) error { + sourceFD, err := unix.Open(injection.source, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if err != nil { + return fmt.Errorf("open source %s: %w", injection.source, err) + } + source := os.NewFile(uintptr(sourceFD), injection.source) + defer source.Close() + info, err := source.Stat() + if err != nil { + return fmt.Errorf("stat source %s: %w", injection.source, err) + } + if !info.Mode().IsRegular() || info.Size() > tforkMaximumInjectionSize { + return fmt.Errorf("source must be a regular file no larger than %d bytes", tforkMaximumInjectionSize) + } + + root := filepath.Join("/proc", strconv.Itoa(pid), "root") + parent, err := pathrs.OpenInRoot(root, filepath.Dir(injection.destination)) + if err != nil { + return fmt.Errorf("open destination parent: %w", err) + } + defer parent.Close() + destinationFD, err := unix.Openat( + int(parent.Fd()), + filepath.Base(injection.destination), + unix.O_WRONLY|unix.O_CREAT|unix.O_TRUNC|unix.O_CLOEXEC|unix.O_NOFOLLOW, + 0o600, + ) + if err != nil { + return fmt.Errorf("open destination: %w", err) + } + destination := os.NewFile(uintptr(destinationFD), injection.destination) + defer destination.Close() + if err := destination.Chmod(0o600); err != nil { + return fmt.Errorf("chmod destination: %w", err) + } + if _, err := io.Copy(destination, source); err != nil { + return fmt.Errorf("copy payload: %w", err) + } + return nil } type tforkCgroupFreezer struct { @@ -919,23 +1619,40 @@ func spawnTforkDumpdHolder() (int, uint64, error) { func tforkAbortCrunCmd(crunCmd *exec.Cmd, src *libpod.Container, bundleDir string, copies int) { if crunCmd != nil && crunCmd.Process != nil { + protected := make(map[int]bool) + if src != nil { + if srcPID, err := src.PID(); err == nil && srcPID > 0 { + protected[srcPID] = true + for _, pid := range tforkCollectDescendants(srcPID) { + protected[pid] = true + } + } + } toKill := tforkCollectDescendants(crunCmd.Process.Pid) - toKill = append(toKill, crunCmd.Process.Pid) - for _, pid := range toKill { - _ = syscall.Kill(pid, syscall.SIGKILL) + // Give CRIU's service and restore helpers a chance to unwind ptrace, + // parasite, namespace, and cgyard state before killing their parent. + // Never signal a source-tree PID even if transient reparenting makes it + // appear below the runtime. + for i := len(toKill) - 1; i >= 0; i-- { + if !protected[toKill[i]] { + _ = syscall.Kill(toKill[i], syscall.SIGTERM) + } } - done := make(chan struct{}) - go func() { - _, _ = crunCmd.Process.Wait() - close(done) - }() - select { - case <-done: - case <-time.After(3 * time.Second): - logrus.Warnf("tfork: crun-tfork didn't exit in 3s after SIGKILL — source may still be ptraced") + if !tforkWaitPIDGone(crunCmd.Process.Pid, time.Second) { + for i := len(toKill) - 1; i >= 0; i-- { + if !protected[toKill[i]] { + _ = syscall.Kill(toKill[i], syscall.SIGKILL) + } + } + _ = syscall.Kill(crunCmd.Process.Pid, syscall.SIGKILL) + if !tforkWaitPIDGone(crunCmd.Process.Pid, 3*time.Second) { + logrus.Warnf("tfork: crun-tfork didn't exit after TERM/KILL escalation — source may still be ptraced") + } } for _, pid := range tforkCollectDescendants(crunCmd.Process.Pid) { - _ = syscall.Kill(pid, syscall.SIGKILL) + if !protected[pid] { + _ = syscall.Kill(pid, syscall.SIGKILL) + } } } if src != nil { @@ -955,6 +1672,19 @@ func tforkAbortCrunCmd(crunCmd *exec.Cmd, src *libpod.Container, bundleDir strin } } +func tforkWaitPIDGone(pid int, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for { + if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) { + return true + } + if time.Now().After(deadline) { + return false + } + time.Sleep(20 * time.Millisecond) + } +} + func tforkCollectDescendants(root int) []int { entries, err := os.ReadDir("/proc") if err != nil { @@ -1093,9 +1823,9 @@ while True: return nil } -func spawnTforkStdioHelper(readEnd *os.File, attachSock *os.File, logPath string, tty bool) error { +func spawnTforkStdioHelper(readEnd *os.File, attachSock *os.File, logPath string, tty bool) (int, error) { if err := os.MkdirAll(filepath.Dir(logPath), 0o755); err != nil { - return fmt.Errorf("mkdir %s: %w", filepath.Dir(logPath), err) + return 0, fmt.Errorf("mkdir %s: %w", filepath.Dir(logPath), err) } pyCode := `import os, sys, asyncio, datetime, socket, io, traceback LOG_PATH = os.environ["LOG_PATH"] @@ -1215,13 +1945,15 @@ except Exception: env = append(env, fmt.Sprintf("ATTACH_FD=%d", attachFD)) cmd.Env = env if err := cmd.Start(); err != nil { - return fmt.Errorf("start stdio-helper: %w", err) + return 0, fmt.Errorf("start stdio-helper: %w", err) } + pid := 0 if cmd.Process != nil { + pid = cmd.Process.Pid _ = cmd.Process.Release() } logrus.Debugf("tfork: spawned stdio-helper for read-fd → %s (attach=%v, tty=%v)", logPath, attachSock != nil, tty) - return nil + return pid, nil } func boolToInt(b bool) int { @@ -1255,10 +1987,10 @@ func allocPerCopyAttachSocket(path string) (*os.File, error) { return f, nil } -func spawnTforkExitWatcher(ctx context.Context, rt *libpod.Runtime, ctr *libpod.Container, clonePID int) error { +func spawnTforkExitWatcher(ctx context.Context, rt *libpod.Runtime, ctr *libpod.Container, clonePID int) (int, error) { exitDir := "/run/libpod/exits" if err := os.MkdirAll(exitDir, 0o755); err != nil { - return fmt.Errorf("mkdir %s: %w", exitDir, err) + return 0, fmt.Errorf("mkdir %s: %w", exitDir, err) } exitFile := filepath.Join(exitDir, ctr.ID()) script := fmt.Sprintf(`while [ -d /proc/%d ]; do sleep 0.2; done; tmp=%s.tmp; printf 137 > "$tmp" && mv "$tmp" %s`, clonePID, exitFile, exitFile) @@ -1267,16 +1999,18 @@ func spawnTforkExitWatcher(ctx context.Context, rt *libpod.Runtime, ctr *libpod. cmd.Stdout = nil cmd.Stderr = nil if err := cmd.Start(); err != nil { - return fmt.Errorf("start exit-watcher: %w", err) + return 0, fmt.Errorf("start exit-watcher: %w", err) } + pid := 0 if cmd.Process != nil { + pid = cmd.Process.Pid _ = cmd.Process.Release() } logrus.Debugf("tfork: spawned exit-watcher for clone %s (PID %d) → %s", ctr.ID(), clonePID, exitFile) - return nil + return pid, nil } -func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, error) { +func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int, useNcopyRestore bool) (int, error) { statePath := fmt.Sprintf("/run/crun/%s/status", cloneID) if data, err := os.ReadFile(statePath); err == nil { var st struct { @@ -1287,7 +2021,7 @@ func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, } } var pidFile string - if copies == 1 { + if !useNcopyRestore { pidFile = filepath.Join(imgDir, "tfork.pid") } else { pidFile = filepath.Join(imgDir, fmt.Sprintf("tfork.pid.copy%d", copyIdx)) @@ -1300,7 +2034,7 @@ func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, if err != nil { return 0, err } - if copies == 1 { + if !useNcopyRestore { return rcPID, nil } initPID, err := readFirstChildPID(rcPID) @@ -1423,6 +2157,7 @@ type conmonForTforkOpts struct { ghostLimit uint tcpClose bool fullMemcopy bool + networkLock string cgroupRoot string dumpdHolderPid int } @@ -1648,6 +2383,10 @@ func spawnConmonForTfork(ctx context.Context, opts conmonForTforkOpts) (int, err if opts.fullMemcopy { tforkRuntimeOpts = append(tforkRuntimeOpts, "--tfork-full-memcopy") } + if opts.networkLock != "" { + tforkRuntimeOpts = append(tforkRuntimeOpts, + fmt.Sprintf("--network-lock=%s", opts.networkLock)) + } if opts.cgroupRoot != "" { tforkRuntimeOpts = append(tforkRuntimeOpts, fmt.Sprintf("--cgroup-root=%s", opts.cgroupRoot)) @@ -1980,6 +2719,57 @@ func tforkPurgeSockets(rootfs string) error { return nil } +func tforkPurgeSocketsWithManifest(rootfsList []string, manifestPath string) error { + if len(rootfsList) == 0 { + return fmt.Errorf("socket purge requires at least one clone rootfs") + } + started := time.Now() + cmd := exec.Command("find", rootfsList[0], "-mindepth", "1", "-type", "s", "-printf", "%P\\0") + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("find socket manifest in %s: %s: %w", rootfsList[0], strings.TrimSpace(string(out)), err) + } + if err := os.WriteFile(manifestPath, out, 0o600); err != nil { + return fmt.Errorf("write socket manifest %s: %w", manifestPath, err) + } + + count := 0 + for _, rel := range strings.Split(string(out), "\x00") { + if rel == "" { + continue + } + clean := filepath.Clean(rel) + if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) { + return fmt.Errorf("unsafe socket path %q in %s", rel, manifestPath) + } + count++ + for _, rootfs := range rootfsList { + if err := os.Remove(filepath.Join(rootfs, clean)); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove socket %s from %s: %w", clean, rootfs, err) + } + } + } + logrus.Infof("tfork: socket manifest found %d path(s), applied to %d clone rootfs(es) in %s", + count, len(rootfsList), time.Since(started)) + return nil +} + +// tforkPurgeSocketsAndSignal opens the restore barrier only after every clone +// rootfs has been purged. On any error it writes no byte, so closing barrier +// produces EOF in crun and makes the clone transaction roll back. +func tforkPurgeSocketsAndSignal(rootfsList []string, manifestPath string, barrier *os.File) error { + if barrier == nil { + return fmt.Errorf("socket-purge barrier is nil") + } + if err := tforkPurgeSocketsWithManifest(rootfsList, manifestPath); err != nil { + return err + } + if _, err := barrier.Write([]byte{1}); err != nil { + return fmt.Errorf("signal socket-purge completion: %w", err) + } + return nil +} + func tforkBuildSkipMnts(srcPID int) []string { out := make([]string, 0, len(tforkPodmanSkipMnts)+8) seen := make(map[string]struct{}, len(tforkPodmanSkipMnts)) diff --git a/podman/pkg/domain/infra/abi/container_tfork_socket_purge_test.go b/podman/pkg/domain/infra/abi/container_tfork_socket_purge_test.go new file mode 100644 index 000000000..34a736e30 --- /dev/null +++ b/podman/pkg/domain/infra/abi/container_tfork_socket_purge_test.go @@ -0,0 +1,82 @@ +//go:build !remote + +package abi + +import ( + "errors" + "io" + "net" + "os" + "path/filepath" + "testing" +) + +func TestTforkPurgeSocketsAndSignalSuccess(t *testing.T) { + rootfsList := []string{t.TempDir(), t.TempDir()} + const socketRel = "run/test.sock" + for _, rootfs := range rootfsList { + socketPath := filepath.Join(rootfs, socketRel) + if err := os.MkdirAll(filepath.Dir(socketPath), 0o755); err != nil { + t.Fatal(err) + } + listener, err := net.ListenUnix("unix", &net.UnixAddr{Name: socketPath, Net: "unix"}) + if err != nil { + t.Fatal(err) + } + listener.SetUnlinkOnClose(false) + if err := listener.Close(); err != nil { + t.Fatal(err) + } + } + manifestPath := filepath.Join(t.TempDir(), "sockets.manifest") + readEnd, writeEnd, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer readEnd.Close() + + if err := tforkPurgeSocketsAndSignal(rootfsList, manifestPath, writeEnd); err != nil { + t.Fatalf("purge and signal failed: %v", err) + } + if err := writeEnd.Close(); err != nil { + t.Fatal(err) + } + + var got [1]byte + if n, err := readEnd.Read(got[:]); err != nil || n != 1 || got[0] != 1 { + t.Fatalf("barrier read = (%d, %v, %v); want (1, nil, [1])", n, err, got[:n]) + } + for _, rootfs := range rootfsList { + if _, err := os.Lstat(filepath.Join(rootfs, socketRel)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("socket remains in %s: %v", rootfs, err) + } + } + manifest, err := os.ReadFile(manifestPath) + if err != nil { + t.Fatal(err) + } + if got, want := string(manifest), socketRel+"\x00"; got != want { + t.Fatalf("manifest = %q; want %q", got, want) + } +} + +func TestTforkPurgeSocketsAndSignalFailureKeepsBarrierClosed(t *testing.T) { + missingRootfs := filepath.Join(t.TempDir(), "missing") + readEnd, writeEnd, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer readEnd.Close() + + if err := tforkPurgeSocketsAndSignal([]string{missingRootfs}, filepath.Join(t.TempDir(), "sockets.manifest"), writeEnd); err == nil { + t.Fatal("purge unexpectedly succeeded") + } + if err := writeEnd.Close(); err != nil { + t.Fatal(err) + } + + var got [1]byte + if n, err := readEnd.Read(got[:]); n != 0 || !errors.Is(err, io.EOF) { + t.Fatalf("barrier read = (%d, %v); want (0, EOF)", n, err) + } +} diff --git a/podman/pkg/domain/infra/abi/container_tfork_sync_test.go b/podman/pkg/domain/infra/abi/container_tfork_sync_test.go new file mode 100644 index 000000000..60cff2728 --- /dev/null +++ b/podman/pkg/domain/infra/abi/container_tfork_sync_test.go @@ -0,0 +1,37 @@ +package abi + +import "testing" + +func TestTforkSourceSyncModeFromEnv(t *testing.T) { + tests := []struct { + name string + value string + want tforkSourceSyncMode + wantErr bool + }{ + {name: "default", value: "", want: tforkSourceSyncFS}, + {name: "syncfs", value: "syncfs", want: tforkSourceSyncFS}, + {name: "case and whitespace", value: " Global\n", want: tforkSourceSyncGlobal}, + {name: "none", value: "none", want: tforkSourceSyncNone}, + {name: "invalid", value: "source", wantErr: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Setenv("PODMAN_TFORK_SYNC_MODE", test.value) + got, err := tforkSourceSyncModeFromEnv() + if test.wantErr { + if err == nil { + t.Fatalf("expected an error, got mode %q", got) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != test.want { + t.Fatalf("mode = %q, want %q", got, test.want) + } + }) + } +} diff --git a/podman/pkg/domain/infra/abi/container_tfork_transaction.go b/podman/pkg/domain/infra/abi/container_tfork_transaction.go new file mode 100644 index 000000000..c8314ec8c --- /dev/null +++ b/podman/pkg/domain/infra/abi/container_tfork_transaction.go @@ -0,0 +1,291 @@ +//go:build !remote + +package abi + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/containers/podman/v5/libpod" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" +) + +const tforkRollbackWait = 3 * time.Second + +type tforkTrackedPID struct { + pid int + startTime uint64 + role string +} + +// tforkCloneTransaction owns every host-side object created before a clone is +// published. Ownership is transferred to libpod only by commit. Until then, +// every return path converges on rollback. +type tforkCloneTransaction struct { + ctx context.Context + rt *libpod.Runtime + src *libpod.Container + + bundleDir string + runtimeBatchDir string + imgDir string + copies int + cloneIDs []string + cgroupPaths []string + + restoreSource func() error + sourceRestored bool + registered []*libpod.Container + pids []tforkTrackedPID + committed bool + rolledBack bool +} + +func newTforkCloneTransaction(ctx context.Context, rt *libpod.Runtime, src *libpod.Container, bundleDir string, copies int) *tforkCloneTransaction { + return &tforkCloneTransaction{ + ctx: ctx, + rt: rt, + src: src, + bundleDir: bundleDir, + copies: copies, + } +} + +func (t *tforkCloneTransaction) setCloneIDs(ids []string) { + t.cloneIDs = append([]string(nil), ids...) +} + +func (t *tforkCloneTransaction) setCgroupPaths(paths []string) { + t.cgroupPaths = append([]string(nil), paths...) +} + +func (t *tforkCloneTransaction) setRuntimeBatchDir(path string) { + t.runtimeBatchDir = path +} + +func (t *tforkCloneTransaction) setImageDir(path string) { + t.imgDir = path +} + +func (t *tforkCloneTransaction) setSourceRestore(restore func() error) { + t.restoreSource = restore +} + +func (t *tforkCloneTransaction) restoreSourceOnce() error { + if t.sourceRestored || t.restoreSource == nil { + return nil + } + if err := t.restoreSource(); err != nil { + return err + } + t.sourceRestored = true + return nil +} + +func (t *tforkCloneTransaction) trackPID(pid int, role string) { + if pid <= 0 { + return + } + for _, tracked := range t.pids { + if tracked.pid == pid { + return + } + } + startTime, err := libpod.ReadProcStartTime(pid) + if err != nil { + logrus.Debugf("tfork: transaction cannot record %s pid=%d start time: %v", role, pid, err) + return + } + t.pids = append(t.pids, tforkTrackedPID{pid: pid, startTime: startTime, role: role}) +} + +func (t *tforkCloneTransaction) addRegistered(ctr *libpod.Container) { + if ctr != nil { + t.registered = append(t.registered, ctr) + } +} + +func (t *tforkCloneTransaction) commit() { + t.committed = true +} + +func (t *tforkCloneTransaction) rollback(cause error) { + if t == nil || t.committed || t.rolledBack { + return + } + t.rolledBack = true + logrus.Warnf("tfork: rolling back unpublished clone transaction: %v", cause) + + // A registered external clone knows how to tear down its network, state, + // cgroup, and per-copy storage. Remove in reverse publication order. + zero := uint(0) + for i := len(t.registered) - 1; i >= 0; i-- { + ctr := t.registered[i] + if err := t.rt.RemoveContainer(context.WithoutCancel(t.ctx), ctr, true, false, &zero); err != nil { + logrus.Warnf("tfork: rollback remove registered clone %s: %v", ctr.ID(), err) + } + } + + t.trackRestorePIDs() + t.killCloneCgroups() + for i := len(t.pids) - 1; i >= 0; i-- { + t.killTrackedPID(t.pids[i]) + } + + if err := t.restoreSourceOnce(); err != nil { + logrus.Warnf("tfork: rollback restore source cgroup: %v", err) + } + + t.removeCloneCgroups() + for _, id := range t.cloneIDs { + _ = os.Remove(filepath.Join("/run/libpod/exits", id)) + _ = os.RemoveAll(filepath.Join("/run/crun", id)) + } + if t.runtimeBatchDir != "" { + if err := os.RemoveAll(t.runtimeBatchDir); err != nil { + logrus.Warnf("tfork: rollback remove runtime publication %s: %v", t.runtimeBatchDir, err) + } + tforkRemoveEmptyParent(filepath.Dir(t.runtimeBatchDir)) + } + if t.bundleDir != "" { + tforkBestEffortBundleReap(t.bundleDir, t.copies) + } +} + +func (t *tforkCloneTransaction) trackRestorePIDs() { + if t.imgDir == "" { + return + } + for i := 0; i < t.copies; i++ { + for _, path := range []string{ + filepath.Join(t.imgDir, "tfork.pid"), + filepath.Join(t.imgDir, fmt.Sprintf("tfork.pid.copy%d", i)), + } { + data, err := os.ReadFile(path) + if err != nil { + continue + } + pid, err := parsePIDBytes(data, path) + if err == nil { + t.trackPID(pid, "restore-child") + if child, err := readFirstChildPID(pid); err == nil { + t.trackPID(child, "clone-init") + } + } + } + } +} + +func (t *tforkCloneTransaction) killCloneCgroups() { + for _, rel := range t.cgroupPaths { + root := tforkCgroupFSPath(rel) + if root == "" { + continue + } + killPath := filepath.Join(root, "cgroup.kill") + if err := os.WriteFile(killPath, []byte("1"), 0o644); err != nil && !errors.Is(err, os.ErrNotExist) { + logrus.Debugf("tfork: rollback write %s: %v", killPath, err) + } + } +} + +func (t *tforkCloneTransaction) removeCloneCgroups() { + deadline := time.Now().Add(tforkRollbackWait) + for _, rel := range t.cgroupPaths { + root := tforkCgroupFSPath(rel) + if root == "" { + continue + } + for { + err := os.Remove(root) + if err == nil || errors.Is(err, os.ErrNotExist) { + break + } + if time.Now().After(deadline) { + logrus.Warnf("tfork: rollback cgroup remains at %s: %v", root, err) + break + } + time.Sleep(25 * time.Millisecond) + } + } +} + +func tforkCgroupFSPath(rel string) string { + clean := strings.TrimPrefix(filepath.Clean(rel), string(os.PathSeparator)) + if clean == "" || clean == "." || strings.HasPrefix(clean, "..") { + return "" + } + return filepath.Join("/sys/fs/cgroup", clean) +} + +func (t *tforkCloneTransaction) killTrackedPID(tracked tforkTrackedPID) { + current, err := libpod.ReadProcStartTime(tracked.pid) + if errors.Is(err, os.ErrNotExist) || errors.Is(err, unix.ESRCH) { + return + } + if err != nil { + logrus.Warnf("tfork: rollback cannot verify %s pid=%d: %v; refusing unsafe kill", tracked.role, tracked.pid, err) + return + } + if current != tracked.startTime { + logrus.Warnf("tfork: rollback %s pid=%d was recycled; refusing kill", tracked.role, tracked.pid) + return + } + descendants := tforkCollectDescendants(tracked.pid) + for i := len(descendants) - 1; i >= 0; i-- { + _ = syscall.Kill(descendants[i], syscall.SIGKILL) + } + _ = syscall.Kill(tracked.pid, syscall.SIGKILL) +} + +func tforkRemoveEmptyParent(path string) { + if path == "" { + return + } + entries, err := os.ReadDir(path) + if err == nil && len(entries) == 0 { + _ = os.Remove(path) + } +} + +func tforkPIDRunning(pid int) error { + if pid <= 0 { + return fmt.Errorf("invalid pid %d", pid) + } + data, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat")) + if err != nil { + return err + } + rp := strings.LastIndexByte(string(data), ')') + if rp < 0 || rp+2 >= len(data) { + return fmt.Errorf("malformed /proc/%d/stat", pid) + } + if data[rp+2] == 'Z' { + return fmt.Errorf("pid %d is a zombie", pid) + } + return nil +} + +func tforkInjectFault(stage string) error { + // PODMAN_TFORK_FAULT_INJECT is an opt-in integration-test hook. Keeping + // the hook at transaction boundaries exercises the real rollback path; + // production calls take the unset fast path below without changing state. + want := strings.TrimSpace(os.Getenv("PODMAN_TFORK_FAULT_INJECT")) + if want == "" { + return nil + } + for _, candidate := range strings.Split(want, ",") { + if strings.TrimSpace(candidate) == stage { + return fmt.Errorf("injected tfork fault at %s", stage) + } + } + return nil +} diff --git a/podman/pkg/domain/infra/abi/container_tfork_transaction_test.go b/podman/pkg/domain/infra/abi/container_tfork_transaction_test.go new file mode 100644 index 000000000..8f86fedd1 --- /dev/null +++ b/podman/pkg/domain/infra/abi/container_tfork_transaction_test.go @@ -0,0 +1,201 @@ +//go:build !remote + +package abi + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestTforkInjectFault(t *testing.T) { + t.Setenv("PODMAN_TFORK_FAULT_INJECT", "after_freeze, before_commit") + for _, stage := range []string{"after_freeze", "before_commit"} { + if err := tforkInjectFault(stage); err == nil { + t.Fatalf("expected injected fault at %s", stage) + } + } + if err := tforkInjectFault("after_restore"); err != nil { + t.Fatalf("unexpected fault at unselected stage: %v", err) + } +} + +func TestTforkCgroupFSPath(t *testing.T) { + for _, unsafe := range []string{"", ".", "/", "..", "../escape"} { + if got := tforkCgroupFSPath(unsafe); got != "" { + t.Errorf("tforkCgroupFSPath(%q) = %q; want empty", unsafe, got) + } + } + const rel = "machine.slice/libpod-test" + if got, want := tforkCgroupFSPath(rel), "/sys/fs/cgroup/"+rel; got != want { + t.Fatalf("tforkCgroupFSPath(%q) = %q; want %q", rel, got, want) + } +} + +func TestTforkPIDRunning(t *testing.T) { + if err := tforkPIDRunning(os.Getpid()); err != nil { + t.Fatalf("current process should be running: %v", err) + } + if err := tforkPIDRunning(-1); err == nil { + t.Fatal("negative pid unexpectedly reported running") + } +} + +func TestTforkParseFileInjections(t *testing.T) { + parsed, err := tforkParseFileInjections([]string{ + "0:/tmp/source-0:/tmp/context.json", + "1:/tmp/source-1:/run/context.json", + }, 2) + if err != nil { + t.Fatal(err) + } + if got := parsed[1][0].destination; got != "/run/context.json" { + t.Fatalf("destination = %q", got) + } + for _, spec := range []string{ + "missing-fields", + "2:/tmp/source:/tmp/context", + "0:relative:/tmp/context", + "0:/tmp/source:relative", + } { + if _, err := tforkParseFileInjections([]string{spec}, 2); err == nil { + t.Fatalf("expected %q to fail", spec) + } + } +} + +func TestTforkParseSourceFileInjections(t *testing.T) { + parsed, err := tforkParseSourceFileInjections([]string{ + "/tmp/source.json:/tmp/gensee-run-context.json", + }) + if err != nil { + t.Fatal(err) + } + if len(parsed) != 1 || parsed[0].source != "/tmp/source.json" || parsed[0].destination != "/tmp/gensee-run-context.json" { + t.Fatalf("unexpected source file injections: %#v", parsed) + } + if _, err := tforkParseSourceFileInjections([]string{"relative:/tmp/context"}); err == nil { + t.Fatal("expected relative source path to fail") + } + if _, err := tforkParseSourceFileInjections([]string{ + "/tmp/source-0:/tmp/context", + "/tmp/source-1:/tmp/context", + }); err == nil { + t.Fatal("expected duplicate destination to fail") + } +} + +func TestTforkInstallPreparedFileInjectionsRollsBackBatch(t *testing.T) { + dir := t.TempDir() + existing := filepath.Join(dir, "existing") + created := filepath.Join(dir, "created") + if err := os.WriteFile(existing, []byte("original"), 0o640); err != nil { + t.Fatal(err) + } + injections := []tforkPreparedFileInjection{ + {source: "/host/first", destination: existing, payload: []byte("replacement")}, + {source: "/host/second", destination: created, payload: []byte("new")}, + } + wantFailure := errors.New("fail after second commit") + err := tforkInstallPreparedFileInjections(os.Getpid(), injections, func(index int) error { + if index == 1 { + return wantFailure + } + return nil + }) + if !errors.Is(err, wantFailure) { + t.Fatalf("install error = %v; want %v", err, wantFailure) + } + data, err := os.ReadFile(existing) + if err != nil { + t.Fatal(err) + } + if got, want := string(data), "original"; got != want { + t.Fatalf("existing content = %q; want %q", got, want) + } + if info, err := os.Stat(existing); err != nil { + t.Fatal(err) + } else if got, want := info.Mode().Perm(), os.FileMode(0o640); got != want { + t.Fatalf("existing mode = %o; want %o", got, want) + } + if _, err := os.Stat(created); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("new destination survived rollback: %v", err) + } + assertNoTforkInjectionTemporaryFiles(t, dir) +} + +func TestTforkInstallPreparedFileInjectionsCommitsBatch(t *testing.T) { + dir := t.TempDir() + existing := filepath.Join(dir, "existing") + created := filepath.Join(dir, "created") + if err := os.WriteFile(existing, []byte("original"), 0o640); err != nil { + t.Fatal(err) + } + injections := []tforkPreparedFileInjection{ + {source: "/host/first", destination: existing, payload: []byte("replacement")}, + {source: "/host/second", destination: created, payload: []byte("new")}, + } + if err := tforkInstallPreparedFileInjections(os.Getpid(), injections, nil); err != nil { + t.Fatal(err) + } + for path, want := range map[string]string{existing: "replacement", created: "new"} { + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if got := string(data); got != want { + t.Fatalf("%s content = %q; want %q", path, got, want) + } + if info, err := os.Stat(path); err != nil { + t.Fatal(err) + } else if got, want := info.Mode().Perm(), os.FileMode(0o600); got != want { + t.Fatalf("%s mode = %o; want %o", path, got, want) + } + } + assertNoTforkInjectionTemporaryFiles(t, dir) +} + +func assertNoTforkInjectionTemporaryFiles(t *testing.T, dir string) { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".tfork-inject-") { + t.Fatalf("source injection left temporary file %s", entry.Name()) + } + } +} + +func TestTforkTransactionRollbackIsIdempotent(t *testing.T) { + temp := t.TempDir() + bundle := filepath.Join(temp, "graph", "tfork-bundles", "batch") + runtimeBatch := filepath.Join(temp, "run", "tfork", "batch") + for _, path := range []string{bundle, runtimeBatch} { + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } + } + + restoreCalls := 0 + txn := newTforkCloneTransaction(t.Context(), nil, nil, bundle, 1) + txn.setRuntimeBatchDir(runtimeBatch) + txn.setSourceRestore(func() error { + restoreCalls++ + return nil + }) + txn.rollback(errors.New("test")) + txn.rollback(errors.New("test again")) + + if restoreCalls != 1 { + t.Fatalf("source restore called %d times; want exactly once", restoreCalls) + } + for _, path := range []string{bundle, runtimeBatch} { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("rollback left %s: %v", path, err) + } + } +} diff --git a/ubuntu-img/Dockerfile b/ubuntu-img/Dockerfile index 235528ac9..782c9193c 100644 --- a/ubuntu-img/Dockerfile +++ b/ubuntu-img/Dockerfile @@ -119,6 +119,7 @@ RUN \ libreoffice-style-breeze \ libreoffice-writer \ thunderbird \ + tmux \ ubuntu-wallpapers \ ubuntu-wallpapers-jammy \ vlc && \ diff --git a/ubuntu-img/docker-compose.yml b/ubuntu-img/docker-compose.yml index cf5768548..3dc4c708a 100644 --- a/ubuntu-img/docker-compose.yml +++ b/ubuntu-img/docker-compose.yml @@ -1,6 +1,6 @@ services: webtop: - image: ${IMAGE:-ghcr.io/wuklab/webtop:ubuntu-kde} + image: ${IMAGE:-localhost/gensee-tclone-webtop:tmux} container_name: webtop security_opt: - seccomp=unconfined