diff --git a/README.md b/README.md index 95ca25692..0b4845d72 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Tclone +# Tclone: Low-Latency Full-Workspace Forking for AI Agents Tclone is a workspace-versioning substrate built for computer-use agents. Tclone provides a versioned personal workspace that can be quickly forked, snapshotted, and rolledback. It forks a live, running container in milliseconds: clones share diff --git a/criu/build.sh b/criu/build.sh index cb1d9966c..58cb6d15c 100755 --- a/criu/build.sh +++ b/criu/build.sh @@ -93,12 +93,22 @@ for mod in "${ACTIVE_MODULES[@]}"; do fi if lsmod | awk '{print $1}' | grep -qx "${mod}"; then - echo " -- ${mod}: already loaded, rmmod then insmod" + echo " -- ${mod}: already loaded, trying reload" rmmod "${mod}" 2>/dev/null || true - else - echo " -- ${mod}: insmod" + if lsmod | awk '{print $1}' | grep -qx "${mod}"; then + echo " -- ${mod}: still loaded; keeping existing module" + continue + fi + fi + + echo " -- ${mod}: insmod" + if ! insmod "${ko_path}"; then + if lsmod | awk '{print $1}' | grep -qx "${mod}"; then + echo " -- ${mod}: insmod reported already loaded; keeping existing module" + continue + fi + exit 1 fi - insmod "${ko_path}" done echo " -- verify all modules loaded" diff --git a/criu/criu/clone-noasan.c b/criu/criu/clone-noasan.c index 4ba7f6f31..368f6aa1e 100644 --- a/criu/criu/clone-noasan.c +++ b/criu/criu/clone-noasan.c @@ -78,7 +78,10 @@ int clone3_with_pid_noasan(int (*fn)(void *), void *arg, int flags, int exit_sig c_args.flags = flags; c_args.set_tid = ptr_to_u64(&pid); c_args.set_tid_size = 1; + pr_info("clone3 set_tid pid=%d flags=0x%x size=1\n", pid, flags); pid = syscall(__NR_clone3, &c_args, sizeof(c_args)); + if (pid < 0) + pr_perror("clone3 set_tid failed flags=0x%x size=1", flags); if (pid == 0) exit(fn(arg)); return pid; @@ -99,6 +102,12 @@ int clone3_with_nested_pid_noasan(int (*fn)(void *), void *arg, int flags, int e BUG_ON(pid->ns_level > MAX_PID_NS_LEVEL || pid->ns_level <= 1); for (i = 0; i < pid->ns_level; i++) tids[i] = pid->ns[i].ns_pid; + pr_info("clone3 nested set_tid flags=0x%x size=%d tids=%d/%d/%d/%d\n", + flags, pid->ns_level, + tids[0], + pid->ns_level > 1 ? tids[1] : -1, + pid->ns_level > 2 ? tids[2] : -1, + pid->ns_level > 3 ? tids[3] : -1); if (!(flags & CLONE_PARENT)) { if (exit_signal != SIGCHLD) { @@ -112,6 +121,13 @@ int clone3_with_nested_pid_noasan(int (*fn)(void *), void *arg, int flags, int e c_args.set_tid = ptr_to_u64(tids); c_args.set_tid_size = pid->ns_level; pid_ret = syscall(__NR_clone3, &c_args, sizeof(c_args)); + if (pid_ret < 0) + pr_perror("clone3 nested set_tid failed flags=0x%x size=%d tids=%d/%d/%d/%d", + flags, pid->ns_level, + tids[0], + pid->ns_level > 1 ? tids[1] : -1, + pid->ns_level > 2 ? tids[2] : -1, + pid->ns_level > 3 ? tids[3] : -1); if (pid_ret == 0) exit(fn(arg)); return pid_ret; diff --git a/criu/criu/cr-dump.c b/criu/criu/cr-dump.c index d196521fc..6728b8625 100644 --- a/criu/criu/cr-dump.c +++ b/criu/criu/cr-dump.c @@ -2360,6 +2360,9 @@ int cr_dump_tasks(pid_t pid) if (dump_zombies()) goto err; + if (finalize_nested_pid_ns_ids()) + goto err; + if (dump_pstree(root_item)) goto err; diff --git a/criu/criu/cr-restore.c b/criu/criu/cr-restore.c index d85279a82..fc1f089c7 100644 --- a/criu/criu/cr-restore.c +++ b/criu/criu/cr-restore.c @@ -186,9 +186,32 @@ static int __restore_wait_inprogress_tasks(int participants) int ret; futex_t *np = &task_entries->nr_in_progress; - futex_wait_while_gt(np, participants); + if (opts.tfork.active) { + int waited; + + for (waited = 0; waited < 100; waited++) { + if ((int)futex_get(np) <= participants) + break; + usleep(100000); + } + + if ((int)futex_get(np) > participants) { + pr_err("tfork restore wait timed out: participants=%d nr_in_progress=%d start_stage=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + participants, (int)futex_get(np), + (int)futex_get(&task_entries->start), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + return -ETIMEDOUT; + } + } else { + futex_wait_while_gt(np, participants); + } + ret = (int)futex_get(np); if (ret < 0) { + pr_err("restore wait aborted: participants=%d nr_in_progress=%d start_stage=%d task_cr_err=%d\n", + participants, ret, (int)futex_get(&task_entries->start), + get_task_cr_err()); set_cr_errno(get_task_cr_err()); return ret; } @@ -227,6 +250,17 @@ static inline void __restore_switch_stage(int next_stage) static int restore_switch_stage(int next_stage) { + if (opts.tfork.active) + pr_warn("tfork: restore_switch_stage %d participants=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + next_stage, stage_participants(next_stage), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + else + pr_info("restore_switch_stage %d participants=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + next_stage, stage_participants(next_stage), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + __restore_switch_stage(next_stage); return restore_wait_inprogress_tasks(); } @@ -1395,6 +1429,16 @@ static inline int fork_with_pid(struct pstree_item *item) ca.item = item; ca.clone_flags = rsti(item)->clone_flags; + if (opts.tfork.active && item != root_item && + !(ca.clone_flags & CLONE_NEWPID) && + item->pid->ns_level > 1 && + item->pid->ns[0].ns_pid == INIT_PID) { + pr_info("tfork: repairing missing CLONE_NEWPID for pidns init uid=%d local=%d parent_local=%d level=%d\n", + uid(item), pid, + item->parent ? localpid(item->parent) : -1, + item->pid->ns_level); + ca.clone_flags |= CLONE_NEWPID; + } BUG_ON(ca.clone_flags & CLONE_VM); @@ -1434,13 +1478,32 @@ static inline int fork_with_pid(struct pstree_item *item) strip |= CLONE_NEWUSER; if (kdat.has_clone3_set_tid) { - if (item->pid->ns_level == 1) + if (opts.tfork.active && (ca.clone_flags & CLONE_NEWPID)) { + pr_info("tfork: restore pidns init uid=%d local pid %d with fresh parent pid, dumped chain level=%d\n", + uid(item), pid, item->pid->ns_level); ret = clone3_with_pid_noasan(restore_task_with_children, &ca, ca.clone_flags & ~strip, SIGCHLD, pid); - else + } else if (item->pid->ns_level == 1) + ret = clone3_with_pid_noasan(restore_task_with_children, &ca, + ca.clone_flags & ~strip, SIGCHLD, pid); + else { + struct pid tfork_pid = {}; + struct pid *restore_pid = item->pid; + + if (opts.tfork.active && (root_ns_mask & CLONE_NEWPID) && + root_item && root_item->pid->ns_level > 1 && + item->pid->ns_level > 1) { + tfork_pid = *item->pid; + tfork_pid.ns_level--; + restore_pid = &tfork_pid; + pr_info("tfork: restore pid uid=%d local=%d with rebased pid chain level %d -> %d\n", + uid(item), pid, item->pid->ns_level, + restore_pid->ns_level); + } ret = clone3_with_nested_pid_noasan(restore_task_with_children, &ca, ca.clone_flags & ~strip, - SIGCHLD, item->pid); + SIGCHLD, restore_pid); + } } else { BUG_ON(item->pid->ns_level >= 1); close_pid_proc(); @@ -1448,13 +1511,26 @@ static inline int fork_with_pid(struct pstree_item *item) (ca.clone_flags & ~strip) | SIGCHLD, &ca); } if (ret < 0) { + pr_err("fork_with_pid failed item uid=%d local=%d real=%d parent_local=%d flags=0x%lx stripped_flags=0x%lx ns_level=%d root_ns_mask=0x%lx tfork=%d\n", + uid(item), pid, realpid(item), + item->parent ? localpid(item->parent) : -1, + ca.clone_flags, ca.clone_flags & ~strip, + item->pid->ns_level, root_ns_mask, + opts.tfork.active ? 1 : 0); + if (item->pid->ns_level > 0) + pr_err("fork_with_pid pid chain uid=%d ns=%d/%d/%d/%d\n", + uid(item), + item->pid->ns[0].ns_pid, + item->pid->ns_level > 1 ? item->pid->ns[1].ns_pid : -1, + item->pid->ns_level > 2 ? item->pid->ns[2].ns_pid : -1, + item->pid->ns_level > 3 ? item->pid->ns[3].ns_pid : -1); pr_perror("Can't fork for %d", pid); if (errno == EEXIST) set_cr_errno(EEXIST); goto err_unlock; } - if (item == root_item) { + if (opts.tfork.active || item == root_item) { item->pid->real = ret; pr_debug("PID: real %d virt %d\n", item->pid->real, localpid(item)); } @@ -4045,6 +4121,7 @@ static int sigreturn_restore(struct task_restore_args *task_args, unsigned long task_args->vdso_rt_size = vdso_rt_size; task_args->can_map_vdso = kdat.can_map_vdso; task_args->has_clone3_set_tid = kdat.has_clone3_set_tid; + task_args->tfork_active = opts.tfork.active; new_sp = restorer_stack(task_args->t->mz); diff --git a/criu/criu/cr-tfork.c b/criu/criu/cr-tfork.c index ea64d6627..aa33b0a04 100644 --- a/criu/criu/cr-tfork.c +++ b/criu/criu/cr-tfork.c @@ -271,7 +271,7 @@ int tfork_read_cropt(void) return 0; snap_path = opts.tfork.snap_root; - if (!snap_path && opts.tfork.copies > 1 && opts.tfork.snap_roots && + if (!snap_path && opts.tfork.copies >= 1 && opts.tfork.snap_roots && opts.tfork.copy_idx < opts.tfork.snap_roots_n) snap_path = opts.tfork.snap_roots[opts.tfork.copy_idx]; @@ -966,8 +966,9 @@ int cr_tfork_tasks(pid_t pid) opts.tfork.pidfd_map[opts.tfork.pidfd_map_nr].pidfd = pidfd; opts.tfork.pidfd_map[opts.tfork.pidfd_map_nr].memfd = -1; opts.tfork.pidfd_map_nr++; - pr_info("tfork: pidfd %d for pid %d (vpid %d uid %d)\n", - pidfd, item->pid->real, localpid(item), uid(item)); + pr_info("tfork: pidfd %d for pid %d (vpid %d uid %d nsid %d level %d)\n", + pidfd, item->pid->real, localpid(item), uid(item), + item->pid->leaf_ns_id, item->pid->ns_level); } ret = run_scripts(ACT_PRE_TFORK_RESTORE); @@ -1078,7 +1079,7 @@ int cr_tfork_tasks(pid_t pid) list_for_each_entry(cgo_iter, &opts.new_cgroup_roots, node) rpc_n_cg_root++; - rpc_max = 32 + 2 * (rpc_n_ifd + rpc_n_ext + rpc_n_cg_root + rpc_max = 33 + 2 * (rpc_n_ifd + rpc_n_ext + rpc_n_cg_root + opts.tfork.snap_mount_n) + rpc_n_copy_args + 2; rpc_argv = calloc(rpc_max, sizeof(*rpc_argv)); @@ -1098,6 +1099,7 @@ int cr_tfork_tasks(pid_t pid) rpc_argv[rpc_n++] = "-o"; rpc_argv[rpc_n++] = restore_log_arg; rpc_argv[rpc_n++] = "-v2"; + rpc_argv[rpc_n++] = "--keep-pid-hierarchy"; if (opts.root) { rpc_argv[rpc_n++] = "--root"; @@ -1171,7 +1173,7 @@ int cr_tfork_tasks(pid_t pid) rpc_argv[rpc_n++] = "--tfork-snap-mounts"; rpc_argv[rpc_n++] = snap_mounts_csv; } - if (opts.tfork.copies > 1) { + if (opts.tfork.copies >= 1) { snprintf(copies_arg, sizeof(copies_arg), "%d", opts.tfork.copies); rpc_argv[rpc_n++] = "--tfork-copies"; @@ -1267,7 +1269,7 @@ int cr_tfork_tasks(pid_t pid) buf[off] = '\0'; end = buf + off; - argv_max = 8 + 2; + argv_max = 8 + 3; for (p = buf; p < end; p++) if (*p == '\0') argv_max++; @@ -1312,6 +1314,7 @@ int cr_tfork_tasks(pid_t pid) argv_new[argc_new++] = "--pidfile"; argv_new[argc_new++] = pidfile_arg; } + argv_new[argc_new++] = "--keep-pid-hierarchy"; argv_new[argc_new] = NULL; execv("/proc/self/exe", argv_new); diff --git a/criu/criu/crtools.c b/criu/criu/crtools.c index df40ec14f..1243c8087 100644 --- a/criu/criu/crtools.c +++ b/criu/criu/crtools.c @@ -332,25 +332,30 @@ int main(int argc, char *argv[], char *envp[]) if (opts.tree_id) pr_warn("Using -t with criu restore is obsoleted\n"); - if (opts.tfork.copies > 1) { + if (opts.tfork.active && opts.tfork.copies >= 1) { int n = opts.tfork.copies, i; pid_t *children; int (*ready_pipes)[2]; int failed = 0; const char *base_log = opts.output; - const int ns_flags = - CLONE_NEWPID | CLONE_NEWNS; + /* + * The n-copy helper must not create/occupy PID 1 in a new + * PID namespace. CRIU restores the real root task as PID 1 + * from the image; if the helper has already consumed it, + * restore fails with EEXIST ("Can't fork for 1"). + */ + const int ns_flags = CLONE_NEWNS; if (!opts.tfork.active) { - pr_err("--tfork-copies>1 requires --tfork-restore " + pr_err("--tfork-copies requires --tfork-restore " "(use 'criu tfork --tfork-copies=N', not " "'criu restore --tfork-copies=N')\n"); return 1; } if (!opts.restore_detach) { - pr_err("--tfork-copies>1 requires --restore-detached\n"); + pr_err("--tfork-copies requires --restore-detached\n"); return 1; } @@ -445,6 +450,8 @@ int main(int argc, char *argv[], char *envp[]) if (opts.tfork.snap_roots_n > 0) opts.root = opts.tfork.snap_roots[i]; + opts.keep_pid_hierarchy = 0; + if (tfork_load_ncopy_fabric(i)) { pr_err("tfork-ncopy: copy %d fabric load failed\n", i); diff --git a/criu/criu/fsnotify.c b/criu/criu/fsnotify.c index ab8a29dd6..2aac52d0b 100644 --- a/criu/criu/fsnotify.c +++ b/criu/criu/fsnotify.c @@ -1037,6 +1037,23 @@ static int open_inotify_fd(struct file_desc *d, int *new_fd) return -1; } + if (opts.tfork.active) { + unsigned int skipped = 0; + + list_for_each_entry(wd_info, &info->marks, list) + skipped++; + + if (skipped) + pr_warn("tfork: restored inotify fd %#08x without %u watch mark(s)\n", + info->ife->id, skipped); + + if (restore_fown(tmp, info->ife->fown)) + close_safe(&tmp); + + *new_fd = tmp; + return tmp < 0 ? -1 : 0; + } + list_for_each_entry(wd_info, &info->marks, list) { pr_info("\tRestore 0x%x wd for %#08x\n", wd_info->iwe->wd, wd_info->iwe->id); if (restore_one_inotify(tmp, wd_info)) { diff --git a/criu/criu/include/namespaces.h b/criu/criu/include/namespaces.h index e442e0a39..7a5aa6437 100644 --- a/criu/criu/include/namespaces.h +++ b/criu/criu/include/namespaces.h @@ -187,6 +187,7 @@ extern int restore_mnt_ns(int rst, int *cwd_fd); extern int dump_task_ns_ids(struct pstree_item *); extern int predump_task_ns_ids(struct pstree_item *); +extern int finalize_nested_pid_ns_ids(void); extern int rst_add_ns_id(unsigned int id, struct pstree_item *, struct ns_desc *nd); extern struct ns_id *lookup_ns_by_id(unsigned int id, struct ns_desc *nd); diff --git a/criu/criu/include/restorer.h b/criu/criu/include/restorer.h index 73e27caa3..40bb132d1 100644 --- a/criu/criu/include/restorer.h +++ b/criu/criu/include/restorer.h @@ -242,6 +242,7 @@ struct task_restore_args { int child_subreaper; int membarrier_registration_mask; bool has_clone3_set_tid; + bool tfork_active; /* * info about rseq from libc used to diff --git a/criu/criu/namespaces.c b/criu/criu/namespaces.c index df224db68..11909cea2 100644 --- a/criu/criu/namespaces.c +++ b/criu/criu/namespaces.c @@ -592,6 +592,93 @@ static unsigned int get_ns_id(int pid, struct ns_desc *nd, protobuf_c_boolean *s return __get_ns_id(pid, nd, supported, NULL); } +static unsigned int add_nested_pid_leaf_ns_id(struct pstree_item *item) +{ + struct ns_id *nsid; + + nsid = xzalloc(sizeof(*nsid)); + if (!nsid) + return 0; + + nsid->type = NS_OTHER; + nsid->kid = 0; + nsid->ns_populated = true; + nsid_add(nsid, &pid_ns_desc, ns_next_id++, localpid(item)); + + pr_info("Add nested pid leaf ns %d for task %d(%d), level %d\n", + nsid->id, localpid(item), realpid(item), item->pid->ns_level); + return nsid->id; +} + +static unsigned int ensure_task_leaf_pid_ns_id(struct pstree_item *item); + +static unsigned int task_leaf_pid_ns_id(struct pstree_item *item, unsigned int proc_pid_ns_id) +{ + struct pstree_item *parent = item->parent; + unsigned int selected; + + /* + * os4agent stores localpid as the innermost NSpid (pid->ns[0]). + * Keep pstree_entry.nsid at the same namespace level. Otherwise a + * nested pid namespace init such as bwrap can become (nsid=N, + * localpid=1) and collide with the container init in the same nsid. + */ + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return 0; + + if (parent && parent->pid->leaf_ns_id != ALL_PID_NS_ID) { + if (item->pid->ns_level == parent->pid->ns_level) { + selected = parent->pid->leaf_ns_id; + goto out; + } + if (item->pid->ns_level > parent->pid->ns_level && + proc_pid_ns_id != parent->pid->leaf_ns_id) { + selected = proc_pid_ns_id; + goto out; + } + if (item->pid->ns_level > parent->pid->ns_level) { + selected = add_nested_pid_leaf_ns_id(item); + goto out; + } + } + + selected = proc_pid_ns_id; + +out: + pr_info("pid leaf ns task=%d(%d) uid=%d level=%d parent_level=%d proc_nsid=%u parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent ? parent->pid->ns_level : -1, proc_pid_ns_id, + parent ? parent->pid->leaf_ns_id : -1, selected); + return selected; +} + +static unsigned int ensure_task_leaf_pid_ns_id(struct pstree_item *item) +{ + struct pstree_item *parent = item->parent; + unsigned int proc_pid_ns_id; + + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return 0; + + if (item->pid->leaf_ns_id != ALL_PID_NS_ID) { + if (!parent) + return item->pid->leaf_ns_id; + if (item->pid->ns_level == parent->pid->ns_level && + item->pid->leaf_ns_id == parent->pid->leaf_ns_id) + return item->pid->leaf_ns_id; + if (item->pid->ns_level > parent->pid->ns_level && + item->pid->leaf_ns_id != parent->pid->leaf_ns_id) + return item->pid->leaf_ns_id; + } + + proc_pid_ns_id = get_ns_id(item->pid->real, &pid_ns_desc, NULL); + if (!proc_pid_ns_id) + return 0; + + item->pid->leaf_ns_id = task_leaf_pid_ns_id(item, proc_pid_ns_id); + return item->pid->leaf_ns_id; +} + int dump_one_ns_file(int lfd, u32 id, const struct fd_parms *p) { struct cr_img *img; @@ -771,15 +858,37 @@ int dump_task_ns_ids(struct pstree_item *item) int i; int pid = item->pid->real; TaskKobjIdsEntry *ids = item->ids; + struct pstree_item *parent = item->parent; + unsigned int proc_pid_ns_id; ids->has_pid_ns_id = true; - ids->pid_ns_id = get_ns_id(pid, &pid_ns_desc, NULL); + proc_pid_ns_id = get_ns_id(pid, &pid_ns_desc, NULL); + if (!proc_pid_ns_id) { + pr_err("Can't make pidns id\n"); + return -1; + } + + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return -1; + + ids->pid_ns_id = proc_pid_ns_id; + if (parent && item->pid->ns_level == parent->pid->ns_level) + ids->pid_ns_id = parent->pid->leaf_ns_id; + else if (parent && item->pid->ns_level > parent->pid->ns_level && + ids->pid_ns_id == parent->pid->leaf_ns_id) + ids->pid_ns_id = add_nested_pid_leaf_ns_id(item); + if (!ids->pid_ns_id) { pr_err("Can't make pidns id\n"); return -1; } item->pid->leaf_ns_id = ids->pid_ns_id; + pr_info("dump pid ns task=%d(%d) uid=%d level=%d parent_level=%d proc_nsid=%u parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent ? parent->pid->ns_level : -1, proc_pid_ns_id, + parent ? parent->pid->leaf_ns_id : -1, ids->pid_ns_id); + for (i = 0; i < item->nr_threads; i++) item->threads[i].leaf_ns_id = ids->pid_ns_id; @@ -865,6 +974,40 @@ int dump_task_ns_ids(struct pstree_item *item) return 0; } +int finalize_nested_pid_ns_ids(void) +{ + struct pstree_item *item; + + for_each_pstree_item(item) { + struct pstree_item *parent = item->parent; + unsigned int nsid; + int i; + + if (!parent) + continue; + if (item->pid->ns_level <= parent->pid->ns_level) + continue; + if (item->pid->leaf_ns_id != parent->pid->leaf_ns_id) + continue; + + nsid = add_nested_pid_leaf_ns_id(item); + if (!nsid) + return -1; + + item->pid->leaf_ns_id = nsid; + for (i = 0; i < item->nr_threads; i++) + item->threads[i].leaf_ns_id = nsid; + if (item->ids && item->ids->has_pid_ns_id) + item->ids->pid_ns_id = nsid; + + pr_info("finalize nested pid ns task=%d(%d) uid=%d level=%d parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent->pid->leaf_ns_id, nsid); + } + + return 0; +} + static UsernsEntry userns_entry = USERNS_ENTRY__INIT; #define INVALID_ID (~0U) diff --git a/criu/criu/pie/restorer.c b/criu/criu/pie/restorer.c index 9aaefe502..a58ae5ed5 100644 --- a/criu/criu/pie/restorer.c +++ b/criu/criu/pie/restorer.c @@ -97,6 +97,7 @@ static pid_t *helpers; static int n_helpers; static pid_t *zombies; static int n_zombies; +static bool tfork_active_local; static enum faults fi_strategy; bool fault_injected(enum faults f) { @@ -149,6 +150,14 @@ static void sigchld_handler(int signal, siginfo_t *siginfo, void *data) if (siginfo->si_pid == zombies[i]) return; + if (tfork_active_local && + siginfo->si_code == CLD_EXITED && + siginfo->si_status == 0) { + pr_info("tfork: ignoring clean child exit during restore: task %d\n", + siginfo->si_pid); + return; + } + if (siginfo->si_code == CLD_EXITED) r = "exited, status="; else if (siginfo->si_code == CLD_KILLED) @@ -162,7 +171,8 @@ static void sigchld_handler(int signal, siginfo_t *siginfo, void *data) else r = "disappeared with"; - pr_info("Task %d %s %d\n", siginfo->si_pid, r, siginfo->si_status); + pr_err("SIGCHLD during restore: task %d %s %d\n", + siginfo->si_pid, r, siginfo->si_status); futex_abort_and_wake(&task_entries_local->nr_in_progress); /* sa_restorer may be unmaped, so we can't go back to userspace*/ @@ -761,8 +771,16 @@ __visible long __export_restore_thread(struct thread_restore_args *args) int ret; if (my_pid != args->pid) { - pr_err("Thread pid mismatch %d/%d\n", my_pid, args->pid); - goto core_restore_end; + if (args->ta && args->ta->tfork_active) { + pr_info("tfork: accepting fresh thread tid %d instead of dumped tid %d\n", + my_pid, args->pid); + args->pid = my_pid; + if (args->ns_level > 0) + args->tid_in_ns[args->ns_level - 1] = my_pid; + } else { + pr_err("Thread pid mismatch %d/%d\n", my_pid, args->pid); + goto core_restore_end; + } } /* restore original shadow stack */ @@ -807,12 +825,24 @@ __visible long __export_restore_thread(struct thread_restore_args *args) } pr_info("%ld: Restored\n", sys_gettid()); - restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->ta->tfork_active) + pr_warn("tfork: thread restore stage complete pid=%d tid=%ld comm=%s ns_level=%d\n", + args->pid, sys_gettid(), args->comm, args->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->ta->tfork_active) + pr_warn("tfork: thread restore barrier returned pid=%d tid=%ld comm=%s stage=%d\n", + args->pid, sys_gettid(), args->comm, ret); if (restore_signals(args->siginfo, args->siginfo_n, false)){ goto core_restore_end; } - restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->ta->tfork_active) + pr_warn("tfork: thread sigchld stage complete pid=%d tid=%ld comm=%s ns_level=%d\n", + args->pid, sys_gettid(), args->comm, args->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->ta->tfork_active) + pr_warn("tfork: thread sigchld barrier returned pid=%d tid=%ld comm=%s stage=%d\n", + args->pid, sys_gettid(), args->comm, ret); /* * Make sure it's before creds, since it's privileged @@ -1476,6 +1506,11 @@ static int wait_zombies(struct task_restore_args *task_args) ret = sys_waitid(P_PID, task_args->zombies[i], NULL, WNOWAIT | WEXITED, NULL); if (ret == -ECHILD) { + if (task_args->tfork_active) { + pr_warn("tfork: zombie pid %d is not reparented to task %ld; skipping wait to avoid restore barrier deadlock\n", + task_args->zombies[i], sys_getpid()); + continue; + } /* A process isn't reparented to this task yet. * Let's wait when someone complete this stage * and try again. @@ -1752,6 +1787,7 @@ __visible long __export_restore_task(struct task_restore_args *args) fi_strategy = args->fault_strategy; task_entries_local = args->task_entries; + tfork_active_local = args->tfork_active; helpers = args->helpers; n_helpers = args->helpers_n; zombies = args->zombies; @@ -2465,6 +2501,15 @@ __visible long __export_restore_task(struct task_restore_args *args) c_args.set_tid = ptr_to_u64(thread_args[i].tid_in_ns); c_args.flags = clone_flags; c_args.set_tid_size = thread_args[i].ns_level; + if (args->tfork_active) { + pr_info("tfork: restore thread pid=%d with fresh tid, set_tid_size %d -> 0 tids=%d/%d\n", + thread_args[i].pid, + thread_args[i].ns_level, + thread_args[i].tid_in_ns[0], + thread_args[i].ns_level > 1 ? thread_args[i].tid_in_ns[1] : -1); + c_args.set_tid = 0; + c_args.set_tid_size = 0; + } /* The kernel does stack + stack_size. */ c_args.stack = new_sp - RESTORE_STACK_SIZE; c_args.stack_size = RESTORE_STACK_SIZE; @@ -2494,8 +2539,22 @@ __visible long __export_restore_task(struct task_restore_args *args) RUN_CLONE_RESTORE_FN(ret, clone_flags, new_sp, parent_tid, thread_args, args->clone_restore_fn); } + if (args->tfork_active && ret > 0 && ret != thread_args[i].pid) { + pr_info("tfork: thread tid remapped %d -> %ld\n", + thread_args[i].pid, ret); + thread_args[i].pid = ret; + if (thread_args[i].ns_level > 0) + thread_args[i].tid_in_ns[thread_args[i].ns_level - 1] = ret; + } if (ret != thread_args[i].pid) { - pr_err("Unable to create a thread: %ld\n", ret); + pr_err("Unable to create a thread: %ld expected=%d ns_level=%d tids=%d/%d/%d/%d tfork=%d\n", + ret, thread_args[i].pid, + thread_args[i].ns_level, + thread_args[i].tid_in_ns[0], + thread_args[i].ns_level > 1 ? thread_args[i].tid_in_ns[1] : -1, + thread_args[i].ns_level > 2 ? thread_args[i].tid_in_ns[2] : -1, + thread_args[i].ns_level > 3 ? thread_args[i].tid_in_ns[3] : -1, + args->tfork_active ? 1 : 0); sys_close(fd); mutex_unlock(&task_entries_local->last_pid_mutex); goto core_restore_end; @@ -2524,13 +2583,33 @@ __visible long __export_restore_task(struct task_restore_args *args) if (restore_membarrier_registrations(args->membarrier_registration_mask) < 0) goto core_restore_end; pr_info("%ld: Restored\n", sys_getpid()); - - restore_finish_stage(task_entries_local, CR_STATE_RESTORE); - + if (args->tfork_active) + pr_warn("tfork: leader restore stage complete pid=%d tid=%ld comm=%s threads=%d ns_level=%d\n", + args->t->pid, sys_getpid(), args->comm, args->nr_threads, + args->t->ns_level); + + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->tfork_active) + pr_warn("tfork: leader restore barrier returned pid=%d tid=%ld comm=%s stage=%ld helpers=%u zombies=%u inotify=%u\n", + args->t->pid, sys_getpid(), args->comm, ret, + args->helpers_n, args->zombies_n, args->inotify_fds_n); + + if (args->tfork_active) + pr_warn("tfork: leader wait_helpers start pid=%d tid=%ld comm=%s helpers=%u\n", + args->t->pid, sys_getpid(), args->comm, args->helpers_n); if (wait_helpers(args) < 0) goto core_restore_end; + if (args->tfork_active) + pr_warn("tfork: leader wait_helpers done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_warn("tfork: leader wait_zombies start pid=%d tid=%ld comm=%s zombies=%u\n", + args->t->pid, sys_getpid(), args->comm, args->zombies_n); if (wait_zombies(args) < 0) goto core_restore_end; + if (args->tfork_active) + pr_warn("tfork: leader wait_zombies done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ksigfillset(&to_block); ret = sys_sigprocmask(SIG_SETMASK, &to_block, NULL, sizeof(k_rtsigset_t)); @@ -2539,9 +2618,18 @@ __visible long __export_restore_task(struct task_restore_args *args) goto core_restore_end; } + if (args->tfork_active) + pr_warn("tfork: leader cleanup_inotify start pid=%d tid=%ld comm=%s inotify=%u\n", + args->t->pid, sys_getpid(), args->comm, args->inotify_fds_n); if (cleanup_current_inotify_events(args)) goto core_restore_end; + if (args->tfork_active) + pr_warn("tfork: leader cleanup_inotify done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_warn("tfork: leader restore sigaction start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); if (!args->compatible_mode) { ret = sys_sigaction(SIGCHLD, &args->sigchld_act, NULL, sizeof(k_rtsigset_t)); } else { @@ -2558,16 +2646,38 @@ __visible long __export_restore_task(struct task_restore_args *args) pr_err("Failed to restore SIGCHLD: %ld\n", ret); goto core_restore_end; } + if (args->tfork_active) + pr_warn("tfork: leader restore sigaction done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_warn("tfork: leader restore shared signals start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ret = restore_signals(args->siginfo, args->siginfo_n, true); if (ret) goto core_restore_end; + if (args->tfork_active) + pr_warn("tfork: leader restore shared signals done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_warn("tfork: leader restore private signals start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ret = restore_signals(args->t->siginfo, args->t->siginfo_n, false); if (ret) goto core_restore_end; - - restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->tfork_active) + pr_warn("tfork: leader restore private signals done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + + if (args->tfork_active) + pr_warn("tfork: leader sigchld stage complete pid=%d tid=%ld comm=%s threads=%d ns_level=%d\n", + args->t->pid, sys_getpid(), args->comm, args->nr_threads, + args->t->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->tfork_active) + pr_warn("tfork: leader sigchld barrier returned pid=%d tid=%ld comm=%s stage=%ld\n", + args->t->pid, sys_getpid(), args->comm, ret); rst_tcp_socks_all(args); diff --git a/criu/criu/pstree.c b/criu/criu/pstree.c index c448c9516..b773b6d8c 100644 --- a/criu/criu/pstree.c +++ b/criu/criu/pstree.c @@ -436,7 +436,10 @@ int dump_pstree(struct pstree_item *root_item) pstree_entry__init(e); tree_entries[nr_items++] = e; - pr_info("Process: %d(%d)\n", localpid(item), realpid(item)); + pr_info("Process: %d(%d) uid=%d nsid=%d level=%d parent=%d parent_nsid=%d\n", + localpid(item), realpid(item), uid(item), item->pid->leaf_ns_id, + item->pid->ns_level, item->parent ? realpid(item->parent) : 0, + item->parent ? item->parent->pid->leaf_ns_id : -1); e->realpid = realpid(item); e->ppid = item->parent ? realpid(item->parent) : 0; @@ -745,9 +748,8 @@ static struct pstree_item *get_or_create_pstree_item(pid_t real, pid_t local, in { struct pid *found; struct pstree_item *item; - struct rb_node **root_link, *root_parent; - found = __lookup_pid_root(&pid_root_rb[ALL_PID_NS_ID], real, &root_parent, &root_link); + found = __lookup_pid_root(&pid_root_rb[ALL_PID_NS_ID], real, NULL, NULL); if (found) { if (pidns_id != ALL_PID_NS_ID) { BUG_ON(found->leaf_ns_id != pidns_id || found->local != local); @@ -763,11 +765,6 @@ static struct pstree_item *get_or_create_pstree_item(pid_t real, pid_t local, in item->pid->local = local; item->pid->leaf_ns_id = pidns_id; - if (__pstree_insert_pid(item->pid, root_parent, root_link) < 0) { - xfree(item); - return NULL; - } - return item; } @@ -916,6 +913,21 @@ static int read_one_pstree_item(PstreeEntry *e) pi->pid->state = TASK_ALIVE; pi->pid->uid = e->uid; + /* note: we don't fail if we have empty ids */ + if (read_pstree_ids(pi) < 0) + goto err; + + if (pi->ids && pi->ids->has_pid_ns_id) { + if (pi->ids->pid_ns_id != pi->pid->leaf_ns_id) { + pr_warn("PID namespace id mismatch for uid %d: pstree=%d ids=%d, keeping pstree\n", + uid(pi), pi->pid->leaf_ns_id, pi->ids->pid_ns_id); + pi->ids->pid_ns_id = pi->pid->leaf_ns_id; + } + } + + if (__pstree_insert_pid(pi->pid, NULL, NULL) < 0) + goto err; + if (e->ppid == 0) { if (root_item) { pr_err("Parent missed on non-root task " @@ -977,10 +989,6 @@ static int read_one_pstree_item(PstreeEntry *e) task_entries->nr_threads += e->n_threads; task_entries->nr_tasks++; - /* note: we don't fail if we have empty ids */ - if (read_pstree_ids(pi) < 0) - goto err; - ret = 1; err: return ret; @@ -1262,8 +1270,12 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) unsigned int ns_level_to_truncate; clone_flags = get_clone_mask(root_item->ids, root_ids); - if (!(clone_flags & CLONE_NEWPID)) + if (!(clone_flags & CLONE_NEWPID) && + !(opts.tfork.active && root_item->pid->ns_level > 1)) return 0; + if (!(clone_flags & CLONE_NEWPID)) + pr_info("pidns: forcing tfork pid hierarchy truncation for root level=%d\n", + root_item->pid->ns_level); if (root_item->pid->ns_level <= 1) { pr_err("only 1 level of pid namespace, but CLONE_NEWPID is set, " @@ -1272,6 +1284,8 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) } ns_level_to_truncate = root_item->pid->ns_level - 1; + pr_info("pidns: truncating %u outer pid namespace level(s) for new root pid namespace\n", + ns_level_to_truncate); for (node = rb_first(&pid_root_rb[ALL_PID_NS_ID]); node; ) { next = rb_next(node); @@ -1279,9 +1293,15 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) pid_node = rb_entry(node, struct pid, root_ns_node); rb_erase(node, &pid_root_rb[ALL_PID_NS_ID]); + pr_info("pidns: truncate before uid=%d real=%d local=%d level=%d\n", + pid_node->uid, pid_node->real, pid_node->local, + pid_node->ns_level); pid_node->ns_level -= ns_level_to_truncate; BUG_ON(pid_node->ns_level <= 0); pid_node->real = pid_node->ns[pid_node->ns_level - 1].ns_pid; + pr_info("pidns: truncate after uid=%d real=%d local=%d level=%d\n", + pid_node->uid, pid_node->real, pid_node->local, + pid_node->ns_level); found = __lookup_pid_root(&new_real_rbtree, pid_node->real, &parent, &link); if (found) { diff --git a/criu/criu/seize.c b/criu/criu/seize.c index f5cde74e9..5fc7c850e 100644 --- a/criu/criu/seize.c +++ b/criu/criu/seize.c @@ -379,7 +379,7 @@ static int seize_cgroup_tree(char *root_path, enum freezer_state state) */ static int freezer_wait_processes(void) { - int i; + int i, collected = 0; processes_to_wait_pids = xmalloc(sizeof(pid_t) * processes_to_wait); if (processes_to_wait_pids == NULL) @@ -388,23 +388,37 @@ static int freezer_wait_processes(void) for (i = 0; i < processes_to_wait; i++) { int status; pid_t pid; + int waited_ms = 0; /* * Here we are going to skip tasks which are already traced. * Ptraced tasks looks like children for us, so if * a task isn't ptraced yet, waitpid() will return a error. */ - pid = waitpid(-1, &status, 0); - if (pid < 0) { - pr_perror("Unable to wait processes"); - xfree(processes_to_wait_pids); - processes_to_wait_pids = NULL; - return -1; + while (1) { + pid = waitpid(-1, &status, opts.tfork.active ? WNOHANG : 0); + if (pid > 0) + break; + if (!opts.tfork.active || (pid < 0 && errno != ECHILD && errno != EINTR)) { + pr_perror("Unable to wait processes"); + xfree(processes_to_wait_pids); + processes_to_wait_pids = NULL; + return -1; + } + if (pid < 0 || waited_ms >= 500) { + pr_warn("tfork: collected %d/%d unexpected freezer processes; continuing\n", + collected, processes_to_wait); + processes_to_wait = collected; + return 0; + } + usleep(10 * 1000); + waited_ms += 10; } pr_warn("Unexpected process %d in the freezer cgroup (status 0x%x)\n", pid, status); - processes_to_wait_pids[i] = pid; + processes_to_wait_pids[collected++] = pid; } + processes_to_wait = collected; return 0; } diff --git a/criu/lib/pycriu/images/images.py b/criu/lib/pycriu/images/images.py index 9db506e1e..927eee972 100644 --- a/criu/lib/pycriu/images/images.py +++ b/criu/lib/pycriu/images/images.py @@ -502,7 +502,7 @@ def skip(self, f, pbuff): tcp_stream_extra_handler()), 'STATS': entry_handler(pb.stats_entry), 'PAGEMAP': pagemap_handler(), # Special one - 'PSTREE': entry_handler(pb.pstree_entry), + 'PSTREE': entry_handler(pb.pstree_file_entry), 'REG_FILES': entry_handler(pb.reg_file_entry), 'NS_FILES': entry_handler(pb.ns_file_entry), 'EVENTFD_FILE': entry_handler(pb.eventfd_file_entry), diff --git a/crun/src/libcrun/criu.c b/crun/src/libcrun/criu.c index 8bf068d03..1e1dbbed2 100644 --- a/crun/src/libcrun/criu.c +++ b/crun/src/libcrun/criu.c @@ -545,6 +545,9 @@ show_criu_log (const char *work_path, const char *log) cleanup_free char *log_path = NULL; libcrun_error_t *tmp_err = NULL; char line[1024]; + char tail[200][1024]; + size_t tail_index = 0; + size_t tail_count = 0; FILE *f; if (UNLIKELY (append_paths (&log_path, tmp_err, work_path, log, NULL)) < 0) @@ -564,11 +567,34 @@ show_criu_log (const char *work_path, const char *log) /* Log with error verbosity as this is the default. */ libcrun_error (0, "--- excerpt from CRIU log `%s`", log_path); while (fgets (line, sizeof (line), f) != NULL) - if (strstr (line, "Error ") != NULL) - { - line[strcspn (line, "\n")] = '\0'; - libcrun_error (0, "%s", line); - } + { + strncpy (tail[tail_index], line, sizeof (tail[tail_index]) - 1); + tail[tail_index][sizeof (tail[tail_index]) - 1] = '\0'; + tail_index = (tail_index + 1) % 200; + if (tail_count < 200) + tail_count++; + + if (strstr (line, "Error ") != NULL || strstr (line, "Warn ") != NULL + || strstr (line, "failed") != NULL || strstr (line, "FAILED") != NULL + || strstr (line, "Unable") != NULL || strstr (line, "Can't") != NULL + || strstr (line, "No such") != NULL) + { + line[strcspn (line, "\n")] = '\0'; + libcrun_error (0, "%s", line); + } + } + + if (tail_count > 0) + { + size_t start = (tail_count == 200) ? tail_index : 0; + libcrun_error (0, "--- last %zu CRIU log lines", tail_count); + for (size_t i = 0; i < tail_count; i++) + { + char *entry = tail[(start + i) % 200]; + entry[strcspn (entry, "\n")] = '\0'; + libcrun_error (0, "%s", entry); + } + } fclose (f); libcrun_error (0, "--- end of excerpt"); @@ -1351,14 +1377,28 @@ libcrun_container_restore_linux_criu (libcrun_container_status_t *status, libcru } # define CRIU_TFORK_LOG_FILE "tfork.log" +# define CRIU_TFORK_RESTORE_LOG_FILE "tfork-restore.log" +# define CRIU_TFORK_MAX_COPY_LOGS 16 + +static void +show_criu_tfork_restore_copy_logs (const char *image_path) +{ + for (int i = 0; i < CRIU_TFORK_MAX_COPY_LOGS; i++) + { + char log[64]; + snprintf (log, sizeof (log), "%s.copy%d", CRIU_TFORK_RESTORE_LOG_FILE, i); + show_criu_log (image_path, log); + } +} static int -read_source_state_pid (const char *path, pid_t *pid_out, libcrun_error_t *err) +read_source_state_pid_cgroup (const char *path, pid_t *pid_out, char **cgroup_path_out, libcrun_error_t *err) { cleanup_free char *buffer = NULL; char err_buffer[256]; yajl_val tree, tmp; const char *pid_path[] = { "pid", NULL }; + const char *cgroup_path[] = { "cgroup-path", NULL }; int ret; ret = read_all_file (path, &buffer, NULL, err); @@ -1377,6 +1417,15 @@ read_source_state_pid (const char *path, pid_t *pid_out, libcrun_error_t *err) } *pid_out = (pid_t) strtoull (YAJL_GET_NUMBER (tmp), NULL, 10); + + tmp = yajl_tree_get (tree, cgroup_path, yajl_t_string); + if (UNLIKELY (tmp == NULL)) + { + yajl_tree_free (tree); + return crun_make_error (err, 0, "`cgroup-path` missing in source state.json `%s`", path); + } + + *cgroup_path_out = xstrdup (YAJL_GET_STRING (tmp)); yajl_tree_free (tree); return 0; } @@ -1439,10 +1488,13 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec { runtime_spec_schema_config_schema *def = container->container_def; cleanup_wrapper struct libcriu_wrapper_s *wrapper = NULL; + cleanup_free char *freezer_path = NULL; cleanup_free char *rootfs_path = NULL; + cleanup_free char *source_cgroup_path = NULL; cleanup_close int image_fd = -1; cleanup_close int work_fd = -1; pid_t source_pid = 0; + int cgroup_mode; int ret; ret = load_wrapper (&wrapper, err); @@ -1461,7 +1513,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec return crun_make_error (err, 0, "--tfork-snap-root, --tfork-snap-roots, or --tfork-copy=::--tfork-snap-root=PATH is required"); - if (cr_options->tfork_copies > 1 && cr_options->tfork_snap_roots_n > 0 + if (cr_options->tfork_copies >= 1 && cr_options->tfork_snap_roots_n > 0 && (size_t) cr_options->tfork_copies != cr_options->tfork_snap_roots_n) return crun_make_error (err, 0, "--tfork-copies=%d but --tfork-snap-roots has %zu entries", @@ -1470,7 +1522,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec if (UNLIKELY (cr_options->image_path == NULL)) return crun_make_error (err, 0, "--image-path is required"); - ret = read_source_state_pid (cr_options->source_state, &source_pid, err); + ret = read_source_state_pid_cgroup (cr_options->source_state, &source_pid, &source_cgroup_path, err); if (UNLIKELY (ret < 0)) return ret; if (UNLIKELY (source_pid <= 0)) @@ -1514,8 +1566,24 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec libcriu_wrapper->criu_set_pid (source_pid); libcriu_wrapper->criu_set_leave_running (true); + libcriu_wrapper->criu_set_ext_unix_sk (cr_options->ext_unix_sk); libcriu_wrapper->criu_set_file_locks (true); + cgroup_mode = libcrun_get_cgroup_mode (err); + if (UNLIKELY (cgroup_mode < 0)) + return cgroup_mode; + + if (cgroup_mode == CGROUP_MODE_UNIFIED) + ret = append_paths (&freezer_path, err, CGROUP_ROOT, source_cgroup_path, NULL); + else + ret = append_paths (&freezer_path, err, CGROUP_ROOT "/freezer", source_cgroup_path, NULL); + if (UNLIKELY (ret < 0)) + return ret; + + ret = libcriu_wrapper->criu_set_freeze_cgroup (freezer_path); + if (UNLIKELY (ret < 0)) + return crun_make_error (err, -ret, "CRIU: failed setting tfork freezer %d", ret); + if (def->root != NULL && def->root->path != NULL) { ret = append_paths (&rootfs_path, err, container->context ? container->context->bundle : ".", @@ -1564,7 +1632,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec } } - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) libcriu_wrapper->criu_set_tfork_copies (cr_options->tfork_copies); if (cr_options->tfork_memdump_async) @@ -1781,6 +1849,8 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec if (UNLIKELY (ret != 0)) { show_criu_log (cr_options->work_path, CRIU_TFORK_LOG_FILE); + show_criu_log (cr_options->image_path, CRIU_TFORK_RESTORE_LOG_FILE); + show_criu_tfork_restore_copy_logs (cr_options->image_path); return crun_make_error (err, 0, "criu_tfork failed: %d", ret); } @@ -1791,7 +1861,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec pid_t clone_pid; const char *pidfile_name = "tfork.pid"; - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) pidfile_name = "tfork.pid.copy0"; ret = append_paths (&pidfile_path, err, cr_options->image_path, pidfile_name, NULL); @@ -1806,7 +1876,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec if (UNLIKELY (clone_pid <= 0)) return crun_make_error (err, 0, "invalid clone PID %d in `%s`", (int) clone_pid, pidfile_path); - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) { char children_path[64]; cleanup_free char *children_buf = NULL; diff --git a/crun/src/tfork.c b/crun/src/tfork.c index 31db969ef..2c1d3438e 100644 --- a/crun/src/tfork.c +++ b/crun/src/tfork.c @@ -83,7 +83,7 @@ static struct argp_option options[] { "parent-path", OPTION_PARENT_PATH, "DIR", 0, "previous criu images dir, for incremental memdump chains", 0 }, { "tfork-memdump", OPTION_TFORK_MEMDUMP, 0, 0, "dump pages-*.img to image-path during tfork", 0 }, { "tfork-memdump-async", OPTION_TFORK_MEMDUMP_ASYNC, 0, 0, "async page dump (implies --tfork-memdump)", 0 }, - { "tfork-copies", OPTION_TFORK_COPIES, "N", 0, "produce N parallel clones (default 1)", 0 }, + { "tfork-copies", OPTION_TFORK_COPIES, "N", 0, "produce N clones through the n-copy helper (omitted: legacy direct single-copy)", 0 }, { "track-mem", OPTION_TRACK_MEM, 0, 0, "arm soft-dirty for chained incremental dumps", 0 }, { "manage-cgroups-mode", OPTION_MANAGE_CGROUPS_MODE, "MODE", 0, "cgroups mode: 'soft' (default), 'ignore', 'full' and 'strict'", 0 }, @@ -359,7 +359,13 @@ int crun_command_tfork (struct crun_global_arguments *global_args, int argc, char **argv, libcrun_error_t *err) { cr_options.manage_cgroups_mode = -1; - cr_options.tfork_copies = 1; + cr_options.tfork_copies = 0; + /* + * Agent runtimes commonly keep Unix sockets connected to helpers outside the + * dumped process subtree (tmux/Codex hooks, host-control bridges, etc.). + * Without this CRIU rejects the dump before restore begins. + */ + cr_options.ext_unix_sk = true; cr_options.leave_running = true; return crun_run_create_internal (global_args, argc, argv, container_tfork, get_options, &crun_context, &run_argp, diff --git a/podman/pkg/domain/infra/abi/container_tfork.go b/podman/pkg/domain/infra/abi/container_tfork.go index 8aa6926d5..2a14b9d0c 100644 --- a/podman/pkg/domain/infra/abi/container_tfork.go +++ b/podman/pkg/domain/infra/abi/container_tfork.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "syscall" "time" @@ -16,7 +17,6 @@ import ( "github.com/containers/podman/v5/libpod" "github.com/containers/podman/v5/libpod/define" - "strconv" "github.com/containers/podman/v5/pkg/domain/entities" "github.com/containers/podman/v5/utils" @@ -27,6 +27,27 @@ import ( "golang.org/x/sys/unix" ) +const ( + tforkSourceFreezeTimeout = 10 * time.Second + tforkSourceThawTimeout = 10 * time.Second + tforkCloneReadyTimeout = 60 * time.Second + tforkCgroupPollInterval = 50 * time.Millisecond + tforkClonePollInterval = 200 * time.Millisecond +) + +func tforkCloneReadyTimeoutFromEnv() time.Duration { + value := strings.TrimSpace(os.Getenv("PODMAN_TFORK_CLONE_READY_TIMEOUT_SECS")) + if value == "" { + return tforkCloneReadyTimeout + } + seconds, err := strconv.Atoi(value) + if err != nil || seconds <= 0 { + logrus.Warnf("tfork: ignoring invalid PODMAN_TFORK_CLONE_READY_TIMEOUT_SECS=%q", value) + return tforkCloneReadyTimeout + } + return time.Duration(seconds) * time.Second +} + func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities.ContainerCloneOptions) (rep *entities.ContainerCreateReport, retErr error) { src, err := ic.Libpod.LookupContainer(opts.ID) if err != nil { @@ -45,6 +66,10 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if copies <= 0 { copies = 1 } + requestedCopies := copies + useSingleCopyConmon := requestedCopies == 1 && os.Getenv("PODMAN_TFORK_SINGLE_COPY_CONMON") == "1" + useSingleCopyDirect := requestedCopies == 1 && os.Getenv("PODMAN_TFORK_SINGLE_COPY_DIRECT") == "1" + useNcopyRestore := copies > 1 || (requestedCopies == 1 && !useSingleCopyConmon && !useSingleCopyDirect) var srcRootfs string if cfg := src.Config(); cfg != nil && cfg.ExternalSetup && cfg.Rootfs != "" { @@ -69,6 +94,20 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities snapRO := filepath.Join(bundleDir, "snap-ro") + thawSource, err := tforkFreezeSourceCgroup(src, tforkSourceFreezeTimeout) + if err != nil { + return nil, fmt.Errorf("freeze source cgroup before tfork snapshot: %w", err) + } + sourceThawed := false + defer func() { + if sourceThawed { + return + } + if err := thawSource(); err != nil { + logrus.Warnf("tfork: thaw source cgroup after clone setup: %v", err) + } + }() + if out, err := exec.Command("sync").CombinedOutput(); err != nil { return nil, fmt.Errorf("sync: %s: %w", out, err) } @@ -280,7 +319,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities "--source-state", srcStatePath, "--image-path", imgDir, } - if copies == 1 { + if !useNcopyRestore { crunArgs = append(crunArgs, "--tfork-snap-root", cloneRootfsList[0]) crunArgs = append(crunArgs, "--tfork-snap-mount", "/") if cloneCgroupPaths[0] != "" { @@ -400,7 +439,9 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } }() hasTTY := len(ttySrcFds) > 0 - useConmon := copies == 1 + // Keep the legacy single-copy conmon bootstrap opt-in because it can fail + // before crun starts and report only `conmon reported pid=-1`. + useConmon := copies == 1 && useSingleCopyConmon if useConmon { conmonInheritFds := inheritFds @@ -443,8 +484,8 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities var perCopyExtraFiles []*os.File var perCopyReadEnds []*os.File var perCopyArgs []string - skipTtySrcFds := copies > 1 && hasTTY - if copies > 1 { + skipTtySrcFds := useNcopyRestore && hasTTY + if useNcopyRestore { ifdsForPerCopy, stdioKeys := splitStdioInheritFds(inheritFds) inheritFds = ifdsForPerCopy extraFDBase := 3 @@ -497,6 +538,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities logF.Close() return nil, fmt.Errorf("start crun tfork: %w", err) } + crunDone := make(chan error, 1) var crunAborted bool defer func() { if !crunAborted && retErr != nil { @@ -507,7 +549,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities _ = f.Close() } go func() { - _ = crunCmd.Wait() + crunDone <- crunCmd.Wait() logF.Close() }() perCopyAttachSocks := make([]*os.File, copies) @@ -549,17 +591,27 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } pidFileFor := func(i int) string { - if copies == 1 { + if !useNcopyRestore { return filepath.Join(imgDir, "tfork.pid") } return filepath.Join(imgDir, fmt.Sprintf("tfork.pid.copy%d", i)) } statePath := fmt.Sprintf("/run/crun/%s/status", cloneIDs[0]) - needState := copies == 1 - deadline := time.Now().Add(60 * time.Second) + needState := !useNcopyRestore && copies == 1 + cloneReadyTimeout := tforkCloneReadyTimeoutFromEnv() + deadline := time.Now().Add(cloneReadyTimeout) readyCopies := 0 stateReady := !needState + crunExited := false + var crunErr error for readyCopies < copies || !stateReady { + if !crunExited { + select { + case crunErr = <-crunDone: + crunExited = true + default: + } + } readyCopies = 0 for i := 0; i < copies; i++ { if _, err := os.Stat(pidFileFor(i)); err == nil { @@ -574,7 +626,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if readyCopies >= copies && stateReady { break } - if crunCmd.ProcessState != nil && crunCmd.ProcessState.Exited() && readyCopies < copies { + if crunExited && readyCopies < copies { tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) crunAborted = true return nil, fmt.Errorf("crun tfork exited before %d clones came up (got %d); see %s", @@ -586,17 +638,41 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities return nil, fmt.Errorf("timeout waiting for %d tfork.pid* files in %s (got %d, stateReady=%v); see %s", copies, imgDir, readyCopies, stateReady, logPath) } - time.Sleep(200 * time.Millisecond) + time.Sleep(tforkClonePollInterval) + } + if !crunExited { + select { + case crunErr = <-crunDone: + crunExited = true + case <-time.After(cloneReadyTimeout): + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("timeout waiting %s for crun tfork to finish after %d clones came up; see %s", + cloneReadyTimeout, copies, logPath) + } + } + if crunErr != nil { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("crun tfork failed after %d clones came up: %w; see %s", + copies, crunErr, logPath) } logrus.Infof("tfork: batch %s up (N=%d); crun-tfork.log at %s", batchID, copies, logPath) } + if err := thawSource(); err != nil { + logrus.Warnf("tfork: clones are up, but thawing source cgroup after restore failed: %v", err) + return nil, fmt.Errorf("thaw source cgroup after tfork restore: %w", err) + } + sourceThawed = true + srcCfg := src.Config() if srcCfg == nil { return nil, fmt.Errorf("source %q: could not read libpod config", src.ID()) } + visibleCloneIDs := make([]string, 0, requestedCopies) for i, cloneID := range cloneIDs { - clonePID, err := readTforkClonePID(cloneID, imgDir, i, copies) + clonePID, err := readTforkClonePID(cloneID, imgDir, i, copies, useNcopyRestore) if err != nil { return nil, fmt.Errorf("read clone %d PID: %w", i, err) } @@ -661,9 +737,184 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } } logrus.Infof("tfork: clone %s (%s) registered in libpod state, pid=%d", cloneID, cloneNames[i], clonePID) + visibleCloneIDs = append(visibleCloneIDs, cloneID) } - return &entities.ContainerCreateReport{Id: strings.Join(cloneIDs, "\n")}, nil + return &entities.ContainerCreateReport{Id: strings.Join(visibleCloneIDs, "\n")}, nil +} + +type tforkCgroupFreezer struct { + root string + statePath string + eventsPath string + version string +} + +func tforkFreezeSourceCgroup(src *libpod.Container, timeout time.Duration) (func() error, error) { + if src == nil { + return nil, fmt.Errorf("source container is nil") + } + cgPath, err := src.CgroupPath() + if err != nil { + return nil, fmt.Errorf("read source cgroup path: %w", err) + } + cgPath = strings.TrimPrefix(filepath.Clean(cgPath), string(os.PathSeparator)) + if cgPath == "" || cgPath == "." { + return nil, fmt.Errorf("source cgroup path is empty") + } + freezer, err := tforkSourceCgroupFreezer(cgPath) + if err != nil { + return nil, err + } + + originalFrozen, err := freezer.frozen() + if err != nil { + return nil, err + } + if err := freezer.freeze(); err != nil { + return nil, err + } + if err := freezer.waitFrozen(true, timeout); err != nil { + if !originalFrozen { + _ = freezer.thaw() + } + return nil, err + } + logrus.Infof("tfork: froze source cgroup for snapshot consistency: %s (%s)", freezer.root, freezer.version) + + thaw := func() error { + if originalFrozen { + return freezer.freeze() + } + if err := freezer.thaw(); err != nil { + return err + } + if err := freezer.waitFrozen(false, tforkSourceThawTimeout); err != nil { + return err + } + logrus.Infof("tfork: thawed source cgroup after clone restore: %s (%s)", freezer.root, freezer.version) + return nil + } + return thaw, nil +} + +func tforkSourceCgroupFreezer(cgPath string) (*tforkCgroupFreezer, error) { + v2Root := filepath.Join("/sys/fs/cgroup", cgPath) + v2FreezePath := filepath.Join(v2Root, "cgroup.freeze") + v2EventsPath := filepath.Join(v2Root, "cgroup.events") + if _, err := os.Stat(v2FreezePath); err == nil { + return &tforkCgroupFreezer{ + root: v2Root, + statePath: v2FreezePath, + eventsPath: v2EventsPath, + version: "cgroup v2", + }, nil + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("stat %s: %w", v2FreezePath, err) + } + + v1Root := filepath.Join("/sys/fs/cgroup/freezer", cgPath) + v1StatePath := filepath.Join(v1Root, "freezer.state") + if _, err := os.Stat(v1StatePath); err == nil { + return &tforkCgroupFreezer{ + root: v1Root, + statePath: v1StatePath, + version: "cgroup v1 freezer", + }, nil + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("stat %s: %w", v1StatePath, err) + } + + return nil, fmt.Errorf("source cgroup freezer not found for %q; expected cgroup v2 %s or cgroup v1 %s", + cgPath, v2FreezePath, v1StatePath) +} + +func (f *tforkCgroupFreezer) frozen() (bool, error) { + data, err := os.ReadFile(f.statePath) + if err != nil { + return false, fmt.Errorf("read %s: %w", f.statePath, err) + } + frozen, ok := f.parseFrozen(data) + if !ok { + return false, fmt.Errorf("could not parse frozen state from %s", f.statePath) + } + return frozen, nil +} + +func (f *tforkCgroupFreezer) freeze() error { + value := []byte("1") + if f.version == "cgroup v1 freezer" { + value = []byte("FROZEN") + } + if err := os.WriteFile(f.statePath, value, 0o644); err != nil { + return fmt.Errorf("write %s=%s: %w", f.statePath, value, err) + } + return nil +} + +func (f *tforkCgroupFreezer) thaw() error { + value := []byte("0") + if f.version == "cgroup v1 freezer" { + value = []byte("THAWED") + } + if err := os.WriteFile(f.statePath, value, 0o644); err != nil { + return fmt.Errorf("write %s=%s: %w", f.statePath, value, err) + } + return nil +} + +func (f *tforkCgroupFreezer) waitFrozen(wantFrozen bool, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + path := f.statePath + if f.eventsPath != "" { + path = f.eventsPath + } + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + frozen, ok := f.parseFrozen(data) + if ok && frozen == wantFrozen { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("timeout waiting %s for %s to report frozen=%t", timeout, path, wantFrozen) + } + time.Sleep(tforkCgroupPollInterval) + } +} + +func (f *tforkCgroupFreezer) parseFrozen(data []byte) (bool, bool) { + if f.version == "cgroup v1 freezer" { + switch strings.TrimSpace(string(data)) { + case "FROZEN": + return true, true + case "THAWED": + return false, true + } + return false, false + } + + switch strings.TrimSpace(string(data)) { + case "0": + return false, true + case "1": + return true, true + } + + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && fields[0] == "frozen" { + switch fields[1] { + case "0": + return false, true + case "1": + return true, true + } + } + } + return false, false } func spawnTforkDumpdHolder() (int, uint64, error) { @@ -1046,7 +1297,7 @@ func spawnTforkExitWatcher(ctx context.Context, rt *libpod.Runtime, ctr *libpod. return nil } -func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, error) { +func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int, useNcopyRestore bool) (int, error) { statePath := fmt.Sprintf("/run/crun/%s/status", cloneID) if data, err := os.ReadFile(statePath); err == nil { var st struct { @@ -1057,7 +1308,7 @@ func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, } } var pidFile string - if copies == 1 { + if !useNcopyRestore { pidFile = filepath.Join(imgDir, "tfork.pid") } else { pidFile = filepath.Join(imgDir, fmt.Sprintf("tfork.pid.copy%d", copyIdx)) @@ -1070,7 +1321,7 @@ func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, if err != nil { return 0, err } - if copies == 1 { + if !useNcopyRestore { return rcPID, nil } initPID, err := readFirstChildPID(rcPID)