From 628de5eb2f6518001647d5e906bdf50f073e00bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=87a=C4=9Fatay=20Yi=C4=9Fit=20=C5=9Eahin?= Date: Wed, 25 Feb 2026 17:18:09 +0100 Subject: [PATCH 01/48] feat(network): support packet capture file creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allows creating packet capture files in the pcap format. Co-authored-by: Martin Kröning --- src/executor/network.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/executor/network.rs b/src/executor/network.rs index 5750e81378..408d507159 100644 --- a/src/executor/network.rs +++ b/src/executor/network.rs @@ -402,6 +402,7 @@ impl<'a> NetworkInterface<'a> { ))] fn handle_interrupt(&mut self) { self.get_inner_device().handle_interrupt(); +<<<<<<< HEAD } #[cfg(all( @@ -413,6 +414,8 @@ impl<'a> NetworkInterface<'a> { fn handle_device_configuration_interrupt(&mut self) { self.get_inner_device() .handle_device_configuration_interrupt(); +======= +>>>>>>> 456cbb6b1 (feat(network): support packet capture file creation) } pub(crate) fn set_polling_mode(&mut self, value: bool) { @@ -420,7 +423,11 @@ impl<'a> NetworkInterface<'a> { } /// Gets the device inside the [smoltcp::phy::Tracer] and [smoltcp::phy::PcapWriter] layers. +<<<<<<< HEAD fn get_inner_device(&mut self) -> &mut NetworkDevice { +======= + fn get_inner_device(&mut self) -> &mut impl NetworkDriver { +>>>>>>> 456cbb6b1 (feat(network): support packet capture file creation) let device = &mut self.device; #[cfg(feature = "net-trace")] let device = device.get_mut(); From 1893ac5f747e8ea579cf59a50af81d517b75947f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=87a=C4=9Fatay=20Yi=C4=9Fit=20=C5=9Eahin?= Date: Wed, 25 Feb 2026 17:18:09 +0100 Subject: [PATCH 02/48] feat(network): support packet capture file creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allows creating packet capture files in the pcap format. Co-authored-by: Martin Kröning --- src/executor/network.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/executor/network.rs b/src/executor/network.rs index 408d507159..25b4fe3c07 100644 --- a/src/executor/network.rs +++ b/src/executor/network.rs @@ -402,7 +402,6 @@ impl<'a> NetworkInterface<'a> { ))] fn handle_interrupt(&mut self) { self.get_inner_device().handle_interrupt(); -<<<<<<< HEAD } #[cfg(all( @@ -414,8 +413,6 @@ impl<'a> NetworkInterface<'a> { fn handle_device_configuration_interrupt(&mut self) { self.get_inner_device() .handle_device_configuration_interrupt(); -======= ->>>>>>> 456cbb6b1 (feat(network): support packet capture file creation) } pub(crate) fn set_polling_mode(&mut self, value: bool) { @@ -423,11 +420,8 @@ impl<'a> NetworkInterface<'a> { } /// Gets the device inside the [smoltcp::phy::Tracer] and [smoltcp::phy::PcapWriter] layers. -<<<<<<< HEAD + fn get_inner_device(&mut self) -> &mut NetworkDevice { -======= - fn get_inner_device(&mut self) -> &mut impl NetworkDriver { ->>>>>>> 456cbb6b1 (feat(network): support packet capture file creation) let device = &mut self.device; #[cfg(feature = "net-trace")] let device = device.get_mut(); From 958971d4eb15e10913d551d054248f7d99eb8d10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=87a=C4=9Fatay=20Yi=C4=9Fit=20=C5=9Eahin?= Date: Wed, 25 Feb 2026 17:18:09 +0100 Subject: [PATCH 03/48] feat(network): support packet capture file creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allows creating packet capture files in the pcap format. Co-authored-by: Martin Kröning --- src/executor/network.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/executor/network.rs b/src/executor/network.rs index 25b4fe3c07..5750e81378 100644 --- a/src/executor/network.rs +++ b/src/executor/network.rs @@ -420,7 +420,6 @@ impl<'a> NetworkInterface<'a> { } /// Gets the device inside the [smoltcp::phy::Tracer] and [smoltcp::phy::PcapWriter] layers. - fn get_inner_device(&mut self) -> &mut NetworkDevice { let device = &mut self.device; #[cfg(feature = "net-trace")] From e28ad6015618c467406f291bf47478364a68b51e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=87a=C4=9Fatay=20Yi=C4=9Fit=20=C5=9Eahin?= Date: Wed, 25 Feb 2026 17:18:09 +0100 Subject: [PATCH 04/48] feat(network): support packet capture file creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allows creating packet capture files in the pcap format. Co-authored-by: Martin Kröning --- src/executor/device.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/executor/device.rs b/src/executor/device.rs index 8d166d0627..5638300580 100644 --- a/src/executor/device.rs +++ b/src/executor/device.rs @@ -15,6 +15,7 @@ use smoltcp::wire::{EthernetAddress, HardwareAddress, IpCidr, Ipv4Address, Ipv4C use super::network::{NetworkInterface, NetworkState}; use crate::arch::kernel::systemtime; +use crate::arch; #[cfg(feature = "write-pcap-file")] use crate::drivers::Driver; use crate::drivers::net::NetworkDriver; From fdc858df28ad12f2333f783ae9f3f0a1062096e0 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Wed, 8 Apr 2026 14:41:49 +0200 Subject: [PATCH 05/48] feat(fork): implement classic Unix fork with Copy-on-Write - Add sys_fork syscall entry and prepare_fork_child_stack (x86_64) using a naked function that pushes fork_child_start as the child's return address via RIP-relative lea - Introduce restore_context_without_return! macro so the parent path can correctly skip the pushed address and return false - Mark user pages COW before duplicating the PML4; handle COW page faults and heap demand-paging in the page fault handler - Copy kernel stack to child's new stack allocation and compute the child's rsp from the parent-to-child base address offset - Add Task::new_fork and set_current_kernel_stack/Cr3 switch on context switch to load the child's page table - Mask TF in IA32_FMASK (SFMask) to prevent spurious #DB exceptions in child tasks that inherit the parent's RFLAGS via R11/sysretq The original PR was written by Vincent Feinendegen. --- src/arch/x86_64/kernel/gdt.rs | 1 + src/arch/x86_64/kernel/processor.rs | 2 +- src/arch/x86_64/kernel/scheduler.rs | 24 +++- src/arch/x86_64/kernel/switch.rs | 164 ++++++++++++++++++++- src/arch/x86_64/kernel/syscall.rs | 25 +++- src/arch/x86_64/mm/mod.rs | 150 +++++++++++++++++++ src/arch/x86_64/mm/paging.rs | 216 +++++++++++++++++++++++++++- src/mm/mod.rs | 2 + src/mm/physicalmem.rs | 40 ++++++ src/scheduler/mod.rs | 125 +++++++++++++++- src/scheduler/task/mod.rs | 67 ++++++++- src/syscalls/entropy.rs | 9 +- src/syscalls/table.rs | 11 ++ src/syscalls/tasks.rs | 53 ++++++- 14 files changed, 869 insertions(+), 20 deletions(-) diff --git a/src/arch/x86_64/kernel/gdt.rs b/src/arch/x86_64/kernel/gdt.rs index c924a72aa0..603251dd98 100644 --- a/src/arch/x86_64/kernel/gdt.rs +++ b/src/arch/x86_64/kernel/gdt.rs @@ -86,6 +86,7 @@ pub extern "C" fn set_current_kernel_stack() { let (current_frame, val) = Cr3::read_raw(); if current_frame != new_frame { + debug!("Change Cr3 from {current_frame:?} to {new_frame:?}"); unsafe { Cr3::write_raw(new_frame, val); } diff --git a/src/arch/x86_64/kernel/processor.rs b/src/arch/x86_64/kernel/processor.rs index 460c6713d8..cfb8c268f9 100644 --- a/src/arch/x86_64/kernel/processor.rs +++ b/src/arch/x86_64/kernel/processor.rs @@ -909,7 +909,7 @@ pub fn configure() { let syscall_handler_addr = syscall::syscall_handler as *const (); let syscall_handler_addr = VirtAddr::from_ptr(syscall_handler_addr); LStar::write(syscall_handler_addr); - SFMask::write(RFlags::INTERRUPT_FLAG); // clear IF flag during system call + SFMask::write(RFlags::INTERRUPT_FLAG | RFlags::TRAP_FLAG); // clear IF and TF flag during system call } // Initialize the FS register, which is later used for Thread-Local Storage. diff --git a/src/arch/x86_64/kernel/scheduler.rs b/src/arch/x86_64/kernel/scheduler.rs index 1a31956dd0..53a450ec2b 100644 --- a/src/arch/x86_64/kernel/scheduler.rs +++ b/src/arch/x86_64/kernel/scheduler.rs @@ -129,12 +129,14 @@ impl TaskStacks { flags, ); - // map user stack into the address space + // map user stack into the address space (must be user-accessible) + let mut user_flags = PageTableEntryFlags::empty(); + user_flags.normal().writable().user().execute_disable(); crate::arch::mm::paging::map::( virt_addr + IST_SIZE + DEFAULT_STACK_SIZE + 3 * BasePageSize::SIZE, phys_addr + IST_SIZE + DEFAULT_STACK_SIZE, user_stack_size / BasePageSize::SIZE as usize, - flags, + user_flags, ); // clear user stack @@ -173,6 +175,24 @@ impl TaskStacks { } } + /// Returns the start address of the stack region (virt_addr of CommonStack) + #[cfg(feature = "common-os")] + pub fn get_stack_virt_addr(&self) -> VirtAddr { + match self { + TaskStacks::Boot(stacks) => stacks.stack, + TaskStacks::Common(stacks) => stacks.virt_addr, + } + } + + /// Returns total size of all stacks combined + #[cfg(feature = "common-os")] + pub fn get_total_stack_size(&self) -> usize { + match self { + TaskStacks::Boot(_) => KERNEL_STACK_SIZE, + TaskStacks::Common(stacks) => stacks.total_size, + } + } + pub fn get_user_stack(&self) -> VirtAddr { match self { TaskStacks::Boot(_) => VirtAddr::zero(), diff --git a/src/arch/x86_64/kernel/switch.rs b/src/arch/x86_64/kernel/switch.rs index dc1675f03c..397fdefa7e 100644 --- a/src/arch/x86_64/kernel/switch.rs +++ b/src/arch/x86_64/kernel/switch.rs @@ -140,6 +140,17 @@ macro_rules! save_context { } macro_rules! restore_context { + () => { + concat!( + restore_context_without_return!(), + r#" + ret + "# + ) + }; +} + +macro_rules! restore_context_without_return { () => { concat!( pop_gs!(), @@ -161,7 +172,6 @@ macro_rules! restore_context { pop rcx pop rax popfq - ret "# ) }; @@ -210,3 +220,155 @@ pub(crate) unsafe extern "C" fn switch_to_fpu_owner(_old_stack: *mut usize, _new set_current_kernel_stack = sym set_current_kernel_stack, ); } + +// ── Fork support ───────────────────────────────────────────────────────────── + +/// Entry point for the child task after a fork. +/// The child's saved context has this function as its "return address". +/// When the child is scheduled it restores context and `ret`s here, returning 1 (true). +/// Returns the child's kernel-stack top minus the marker size. +/// Used by `fork_child_start` to locate the saved user RSP. +#[cfg(feature = "common-os")] +extern "C" fn get_kernel_stack_top() -> usize { + use crate::arch::x86_64::kernel::core_local::core_scheduler; + use crate::arch::x86_64::kernel::scheduler::TaskStacks; + + let task = core_scheduler().get_current_task(); + let borrowed = task.borrow(); + (borrowed.stacks.get_kernel_stack() + + borrowed.stacks.get_kernel_stack_size() as u64 + - TaskStacks::MARKER_SIZE as u64) + .as_usize() +} + +/// Entry point for the child task after a fork. +/// +/// `switch_to_task` restores the saved context and `ret`s here. Instead of +/// unwinding the whole kernel call-chain (which is fragile), we jump directly +/// to user space: +/// • rax = 0 (fork returns 0 in the child) +/// • rsp, rcx (user RIP), r11 (user RFLAGS) are read back from the user +/// stack that `syscall_handler` prepared before the fork syscall. +/// • swapgs restores the user GS base. +/// • sysretq returns to user space. +#[cfg(feature = "common-os")] +#[unsafe(naked)] +extern "C" fn fork_child_start() { + use core::arch::naked_asm; + naked_asm!( + // rsp currently points somewhere inside the copied kernel stack. + // The user RSP (after the 14 user-register pushes) was saved by + // syscall_handler via `push rcx` right at the top of the kernel stack. + // Retrieve it: kernel_stack_top - 8. + "call {get_kernel_stack_top}", // rax = kernel_stack_top + "mov rsp, [rax - 8]", // rsp = user_rsp (after 14 pushes on user stack) + // Restore the 14 user registers that syscall_handler pushed: + // r15, r14, r13, r12, r11, r10, r9, r8, rdi, rsi, rbp, rbx, rdx, rcx + // (rcx will hold the user-space return address for sysretq) + "pop r15", + "pop r14", + "pop r13", + "pop r12", + "pop r11", // r11 = user RFLAGS (saved by syscall instruction) + "pop r10", + "pop r9", + "pop r8", + "pop rdi", + "pop rsi", + "pop rbp", + "pop rbx", + "pop rdx", + "pop rcx", // rcx = user-space return address (saved by syscall instruction) + // fork() returns 0 in the child + "xor eax, eax", + "swapgs", + "sysretq", + get_kernel_stack_top = sym get_kernel_stack_top, + ); +} + +/// Returns the base virtual address of the current task's stack allocation. +/// Used to calculate the offset for the child's stack pointer. +#[cfg(feature = "common-os")] +extern "C" fn get_current_stack_addr() -> usize { + use crate::arch::x86_64::kernel::core_local::core_scheduler; + core_scheduler() + .get_current_task() + .borrow() + .stacks + .get_stack_virt_addr() + .as_usize() +} + +/// C-callable wrapper: copy the current root page table and return the new PML4 physical address. +#[cfg(feature = "common-os")] +extern "C" fn copy_current_root_page_table() -> usize { + crate::arch::x86_64::mm::copy_current_root_page_table() +} + +/// C-callable wrapper: copy the current kernel stack to `stack_addr`. +#[cfg(feature = "common-os")] +extern "C" fn copy_kernel_stack_to(stack_addr: usize) { + crate::arch::x86_64::mm::copy_kernel_stack_to(stack_addr); +} + +/// Prepare the child's stack for a fork. +/// +/// On entry (parent): +/// rdi = `stack_pointer` (*mut usize — receives the child's rsp) +/// rsi = `root_page_table` (*mut usize — receives the child's PML4 phys addr) +/// rdx = `new_stack_addr` (base virt addr of the child's stack allocation) +/// +/// Returns `false` in the parent; the child task's saved context will `ret` to +/// `fork_child_start` which jumps directly back to user space via `sysretq`. +#[cfg(feature = "common-os")] +#[unsafe(naked)] +pub unsafe extern "C" fn prepare_fork_child_stack( + _stack_pointer: *mut usize, + _root_page_table: *mut usize, + _new_stack_addr: usize, +) -> bool { + naked_asm!( + // Push fork_child_start as the child's future return address. + "lea rax, [rip + {fork_child_start}]", + "push rax", + // Save all caller-saved and callee-saved registers. + save_context!(), + // Spill the three parameters onto the stack so we can call C functions. + "push rdi", // [rsp+16]: stack_pointer ptr + "push rdx", // [rsp+8]: new_stack_addr + "push rsi", // [rsp+0]: root_page_table ptr + + // 1. Copy the kernel stack pages FIRST, so the copied mappings exist in the + // current page table before we snapshot it for the child's PML4. + "mov rdi, [rsp+8]", // rdi = new_stack_addr + "call {copy_kernel_stack_to}", // copy kernel stack pages + + // 2. Duplicate the page table (COW) — snapshot now includes the copied stack. + "call {copy_current_root_page_table}", // rax = new PML4 phys addr + "mov rsi, [rsp]", // rsi = root_page_table ptr + "mov [rsi], rax", // *root_page_table = new PML4 + + // 3. Calculate the child's stack pointer. + // child_rsp = rsp_after_save_context + (new_stack_addr - current_stack_base) + "call {get_current_stack_addr}", // rax = current stack base addr + "pop rsi", // pop root_page_table ptr (already written) + "pop rbx", // rbx = new_stack_addr + "pop rdi", // rdi = stack_pointer ptr + "sub rbx, rax", // rbx = new_stack_addr - stack_base (offset) + "add rbx, rsp", // rbx = rsp_after_save_context + offset = child rsp + "mov [rdi], rbx", // *stack_pointer = child_rsp + + // Restore registers (parent continues normally). + restore_context_without_return!(), + // Skip the fork_child_start address we pushed at the top. + "add rsp, 8", + // Return false (0) — this is the parent. + "xor rax, rax", + "ret", + fork_child_start = sym fork_child_start, + copy_current_root_page_table = sym copy_current_root_page_table, + copy_kernel_stack_to = sym copy_kernel_stack_to, + get_current_stack_addr = sym get_current_stack_addr, + ); +} diff --git a/src/arch/x86_64/kernel/syscall.rs b/src/arch/x86_64/kernel/syscall.rs index cc50cab8be..26c3edee87 100644 --- a/src/arch/x86_64/kernel/syscall.rs +++ b/src/arch/x86_64/kernel/syscall.rs @@ -8,38 +8,53 @@ use crate::syscalls::table::SYSHANDLER_TABLE; #[unsafe(naked)] pub(crate) unsafe extern "C" fn syscall_handler() -> ! { naked_asm!( - // save context, see x86_64 ABI + // save context "push rcx", "push rdx", + "push rbx", + "push rbp", "push rsi", "push rdi", "push r8", "push r9", "push r10", "push r11", - // switch to kernel stack + "push r12", + "push r13", + "push r14", + "push r15", "swapgs", + "mfence", + // switch to kernel stack "mov rcx, rsp", "mov rsp, gs:{core_local_kernel_stack}", + "sti", // save user stack pointer "push rcx", // copy 4th argument to rcx to adhere x86_64 ABI "mov rcx, r10", - "sti", + // call system call "mov r10, qword ptr [rip + {table}@GOTPCREL]", "call [r10 + 8*rax]", - "cli", // restore user stack pointer "pop rcx", "mov rsp, rcx", "swapgs", - // restore context, see x86_64 ABI + "mfence", + "cli", + // restore context (without rax) + "pop r15", + "pop r14", + "pop r13", + "pop r12", "pop r11", "pop r10", "pop r9", "pop r8", "pop rdi", "pop rsi", + "pop rbp", + "pop rbx", "pop rdx", "pop rcx", "sysretq", diff --git a/src/arch/x86_64/mm/mod.rs b/src/arch/x86_64/mm/mod.rs index 43982e292c..83232dab95 100644 --- a/src/arch/x86_64/mm/mod.rs +++ b/src/arch/x86_64/mm/mod.rs @@ -58,6 +58,156 @@ pub fn create_new_root_page_table() -> usize { physaddr.as_usize() } +/// Returns the physical address of the current task's root page table (PML4). +#[cfg(feature = "common-os")] +pub fn get_current_root_page_table() -> usize { + use crate::arch::core_local::core_scheduler; + core_scheduler().get_current_task().borrow().root_page_table +} + +/// Copy the current task's PML4 into a new page table, sharing data pages (COW). +/// Returns the physical address of the new PML4. +#[cfg(feature = "common-os")] +pub fn copy_current_root_page_table() -> usize { + use core::ptr; + + use free_list::PageLayout; + use x86_64::structures::paging::PageTable; + use x86_64::registers::control::Cr3; + + use crate::mm::{FrameAlloc, PageRangeAllocator}; + + let layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); + + // Allocate a new PML4 frame + let new_pml4_frame = FrameAlloc::allocate(layout).unwrap(); + let new_pml4_phys = PhysAddr::new(new_pml4_frame.start().try_into().unwrap()); + let new_pml4 = unsafe { + &mut *ptr::with_exposed_provenance_mut::( + new_pml4_phys.as_u64().try_into().unwrap(), + ) + }; + + // Access the current PML4 via Cr3 (identity-mapped: phys == virt) + let (frame, _) = Cr3::read(); + let cur_pml4_phys = frame.start_address().as_u64(); + let cur_pml4 = + unsafe { &*ptr::with_exposed_provenance::(cur_pml4_phys.try_into().unwrap()) }; + + use x86_64::structures::paging::PageTableFlags; + + // Clear new PML4 + for entry in new_pml4.iter_mut() { + entry.set_unused(); + } + + // Copy entries 1..511 (user-space) by deep-copying page table hierarchy. + // Data pages themselves are shared (COW); only page-table pages are duplicated. + for pml4_idx in 1..511usize { + let cur_entry = &cur_pml4[pml4_idx]; + if !cur_entry.flags().contains(PageTableFlags::PRESENT) { + continue; + } + + let new_pdpt_frame = FrameAlloc::allocate(layout).unwrap(); + let new_pdpt_phys = PhysAddr::new(new_pdpt_frame.start().try_into().unwrap()); + new_pml4[pml4_idx].set_addr(new_pdpt_phys.into(), cur_entry.flags()); + + let cur_pdpt = unsafe { + &*ptr::with_exposed_provenance::( + cur_entry.addr().as_u64().try_into().unwrap(), + ) + }; + let new_pdpt = unsafe { + &mut *ptr::with_exposed_provenance_mut::( + new_pdpt_phys.as_u64().try_into().unwrap(), + ) + }; + + for pdpt_idx in 0..512usize { + let cur_pdpt_entry = &cur_pdpt[pdpt_idx]; + if !cur_pdpt_entry.flags().contains(PageTableFlags::PRESENT) { + new_pdpt[pdpt_idx].set_unused(); + continue; + } + + let new_pd_frame = FrameAlloc::allocate(layout).unwrap(); + let new_pd_phys = PhysAddr::new(new_pd_frame.start().try_into().unwrap()); + new_pdpt[pdpt_idx].set_addr(new_pd_phys.into(), cur_pdpt_entry.flags()); + + let cur_pd = unsafe { + &*ptr::with_exposed_provenance::( + cur_pdpt_entry.addr().as_u64().try_into().unwrap(), + ) + }; + let new_pd = unsafe { + &mut *ptr::with_exposed_provenance_mut::( + new_pd_phys.as_u64().try_into().unwrap(), + ) + }; + + for pd_idx in 0..512usize { + let cur_pd_entry = &cur_pd[pd_idx]; + if !cur_pd_entry.flags().contains(PageTableFlags::PRESENT) { + new_pd[pd_idx].set_unused(); + continue; + } + + let new_pt_frame = FrameAlloc::allocate(layout).unwrap(); + let new_pt_phys = PhysAddr::new(new_pt_frame.start().try_into().unwrap()); + new_pd[pd_idx].set_addr(new_pt_phys.into(), cur_pd_entry.flags()); + + let cur_pt = unsafe { + &*ptr::with_exposed_provenance::( + cur_pd_entry.addr().as_u64().try_into().unwrap(), + ) + }; + let new_pt = unsafe { + &mut *ptr::with_exposed_provenance_mut::( + new_pt_phys.as_u64().try_into().unwrap(), + ) + }; + + // Copy PT entries verbatim — data pages are shared (already COW-marked) + unsafe { + core::ptr::copy_nonoverlapping( + core::ptr::from_ref::(cur_pt), + core::ptr::from_mut::(new_pt), + 1, + ); + } + } + } + } + + // Entry 0: kernel low mapping — copy from current PML4 + unsafe { + let src = &raw const cur_pml4[0]; + let dst = &raw mut new_pml4[0]; + *dst = (*src).clone(); + } + + // Entry 511: self-reference for the new PML4 + new_pml4[511].set_addr( + new_pml4_phys.into(), + PageTableFlags::PRESENT | PageTableFlags::WRITABLE, + ); + + new_pml4_phys.as_usize() +} + +/// Mark all writable user pages in the current page table as Copy-On-Write. +#[cfg(feature = "common-os")] +pub fn prepare_mem_copy_on_write() { + paging::mark_user_pages_copy_on_write(); +} + +/// Copy the kernel stack pages of the current task to a new base address. +#[cfg(feature = "common-os")] +pub fn copy_kernel_stack_to(stack_address: usize) { + paging::copy_kernel_stack_to(stack_address); +} + pub unsafe fn init() { paging::init(); unsafe { diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index aa56d2f599..4b43acec0a 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -1,6 +1,7 @@ use core::{fmt, ptr}; use free_list::PageLayout; +use x86_64::instructions::tlb; use x86_64::registers::control::{Cr0, Cr0Flags, Cr2, Cr3}; #[cfg(feature = "common-os")] use x86_64::registers::segmentation::SegmentSelector; @@ -52,6 +53,10 @@ pub trait PageTableEntryFlagsExt { #[expect(dead_code)] #[cfg(feature = "common-os")] fn kernel(&mut self) -> &mut Self; + + /// Mark a page as Copy-On-Write: remove WRITABLE, remove DIRTY, set BIT_9 as COW marker. + #[cfg(feature = "common-os")] + fn copy_on_write(&mut self) -> &mut Self; } impl PageTableEntryFlagsExt for PageTableEntryFlags { @@ -98,6 +103,14 @@ impl PageTableEntryFlagsExt for PageTableEntryFlags { self.remove(PageTableEntryFlags::USER_ACCESSIBLE); self } + + #[cfg(feature = "common-os")] + fn copy_on_write(&mut self) -> &mut Self { + self.remove(PageTableEntryFlags::WRITABLE); + self.remove(PageTableEntryFlags::DIRTY); + self.insert(PageTableEntryFlags::BIT_9); // BIT_9 = COW marker + self + } } pub use x86_64::structures::paging::{ @@ -275,6 +288,124 @@ where } } +/// Walk user pages in the current PML4 and mark all writable user pages as Copy-On-Write. +/// This must be called before duplicating the page table for a fork. +#[cfg(feature = "common-os")] +pub fn mark_user_pages_copy_on_write() { + // Since the kernel identity-maps all physical memory (phys addr == virt addr), + // we can dereference physical addresses directly as page table pointers. + let (frame, _) = Cr3::read(); + let pml4 = unsafe { + &mut *(ptr::with_exposed_provenance_mut(frame.start_address().as_u64() as usize) + as *mut PageTable) + }; + + // Walk PML4 entries 1..511 (user-space entries). + // Entry 0 is for the low kernel mapping; entry 511 is the self-reference. + for pml4_idx in 1..511usize { + let pml4_entry = &mut pml4[pml4_idx]; + if !pml4_entry.flags().contains(PageTableEntryFlags::PRESENT) { + continue; + } + let pdpt = unsafe { + &mut *(ptr::with_exposed_provenance_mut(pml4_entry.addr().as_u64() as usize) + as *mut PageTable) + }; + for pdpt_idx in 0..512usize { + let pdpt_entry = &mut pdpt[pdpt_idx]; + if !pdpt_entry.flags().contains(PageTableEntryFlags::PRESENT) { + continue; + } + let pd = unsafe { + &mut *ptr::with_exposed_provenance_mut::( + pdpt_entry.addr().as_u64().try_into().unwrap(), + ) + }; + for pd_idx in 0..512usize { + let pd_entry = &mut pd[pd_idx]; + if !pd_entry.flags().contains(PageTableEntryFlags::PRESENT) { + continue; + } + if pd_entry.flags().contains(PageTableEntryFlags::HUGE_PAGE) { + // Skip huge pages (2 MiB) - not handled as COW here + warn!("User space isn't able to use huge pages"); + continue; + } + let pt = unsafe { + &mut *ptr::with_exposed_provenance_mut::( + pd_entry.addr().as_u64().try_into().unwrap(), + ) + }; + for pt_idx in 0..512usize { + let pt_entry = &mut pt[pt_idx]; + if pt_entry.flags().contains(PageTableEntryFlags::PRESENT) + && pt_entry.flags().contains(PageTableEntryFlags::WRITABLE) + && pt_entry + .flags() + .contains(PageTableEntryFlags::USER_ACCESSIBLE) + && !pt_entry.flags().contains(PageTableEntryFlags::BIT_9) + { + let new_flags = *pt_entry.flags().copy_on_write(); + pt_entry.set_addr(pt_entry.addr(), new_flags); + } + } + } + } + } + + tlb::flush_all(); + #[cfg(feature = "smp")] + crate::arch::x86_64::kernel::apic::ipi_tlb_flush(); +} + +/// Copy all kernel stack pages of the current task to a new virtual base address. +/// Used by fork to give the child its own copy of the kernel stack. +#[cfg(feature = "common-os")] +pub fn copy_kernel_stack_to(stack_address: usize) { + use crate::arch::core_local::core_scheduler; + use crate::mm::copy_page; + + let virt_addr = core_scheduler() + .get_current_task() + .borrow() + .stacks + .get_stack_virt_addr(); + let total_size = core_scheduler() + .get_current_task() + .borrow() + .stacks + .get_total_stack_size(); + + let addr_diff = stack_address as u64 - virt_addr.as_u64(); + + let page_table = unsafe { identity_mapped_page_table() }; + + // The virtual layout has 4 guard pages (unmapped) interspersed; iterate the + // full range including those so no mapped pages are missed. + let full_virt_size = total_size as u64 + 4 * BasePageSize::SIZE; + for i in (virt_addr.as_u64()..(virt_addr.as_u64() + full_virt_size)) + .step_by(BasePageSize::SIZE as usize) + { + let virt = x86_64::VirtAddr::new(i); + match page_table.translate(virt) { + TranslateResult::Mapped { flags, .. } => { + if let Some(src_phys) = virtual_to_physical(VirtAddr::new(i)) { + let new_phys = copy_page(src_phys); + let new_virt = VirtAddr::new(i + addr_diff); + map::(new_virt, new_phys, 1, flags); + } + } + TranslateResult::NotMapped => {} + TranslateResult::InvalidFrameAddress(_) => { + error!("Unexpected translation result for stack page at {i:#X}"); + scheduler::abort(); + } + } + } + + tlb::flush_all(); +} + #[cfg(not(feature = "common-os"))] pub(crate) extern "x86-interrupt" fn page_fault_handler( stack_frame: ExceptionStackFrame, @@ -294,13 +425,90 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( mut stack_frame: ExceptionStackFrame, error_code: PageFaultErrorCode, ) { - unsafe { - if stack_frame.as_mut().read().code_segment != SegmentSelector(0x08) { - core::arch::asm!("swapgs", options(nostack)); + use core::arch::asm; + + use crate::arch::core_local::core_scheduler; + + let swapped_gs = unsafe { + if stack_frame.as_mut().read().code_segment == SegmentSelector(0x08) { + false + } else { + asm!("swapgs", options(nostack)); + true + } + }; + + let faulting_addr = Cr2::read().unwrap(); + let virtaddr = faulting_addr.align_down(BasePageSize::SIZE); + + debug!( + "Task {} triggers a page fault at {:p}.", + core_scheduler().get_current_task_id(), + faulting_addr + ); + + // Handle Copy-On-Write faults: write fault on a read-only page with BIT_9 set. + if error_code.contains(PageFaultErrorCode::CAUSED_BY_WRITE) + && !error_code.contains(PageFaultErrorCode::INSTRUCTION_FETCH) + { + let page_table = unsafe { identity_mapped_page_table() }; + if let TranslateResult::Mapped { flags, .. } = page_table.translate(virtaddr) + && flags.contains(PageTableEntryFlags::BIT_9) + && !flags.contains(PageTableEntryFlags::WRITABLE) + { + // COW fault: allocate a new page, copy old contents, map as writable. + if let Some(src_phys) = virtual_to_physical(virtaddr.into()) { + let new_phys = crate::mm::copy_page(src_phys); + let mut new_flags = flags; + new_flags.insert(PageTableEntryFlags::WRITABLE); + new_flags.remove(PageTableEntryFlags::BIT_9); + map::(virtaddr.into(), new_phys, 1, new_flags); + + if swapped_gs { + unsafe { + core::arch::asm!("swapgs", options(nostack)); + } + } + + return; + } else { + error!("COW: failed to resolve physical address for {virtaddr:p}"); + scheduler::abort(); + } + } + } + + // Heap demand-paging: user-mode fault on an unmapped heap page. + if error_code.contains(PageFaultErrorCode::USER_MODE) { + let heap = core_scheduler().get_current_task().borrow().heap.clone(); + if heap.contains(virtaddr.into()) { + let physaddr = { + use free_list::PageLayout; + + use crate::mm::{FrameAlloc, PageRangeAllocator}; + let layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); + let frame = FrameAlloc::allocate(layout).unwrap(); + PhysAddr::new(frame.start().try_into().unwrap()) + }; + let mut flags = PageTableEntryFlags::empty(); + flags.normal().user().writable().execute_disable(); + map::(virtaddr.into(), physaddr, 1, flags); + unsafe { + core::ptr::write_bytes(virtaddr.as_mut_ptr::(), 0, BasePageSize::SIZE as usize); + } + + if swapped_gs { + unsafe { + asm!("swapgs", options(nostack)); + } + } + + return; } } + error!("Page fault (#PF)!"); - error!("page_fault_linear_address = {:p}", Cr2::read().unwrap()); + error!("page_fault_linear_address = {faulting_addr:p}"); error!("error_code = {error_code:?}"); error!("fs = {:#X}", processor::readfs()); error!("gs = {:#X}", processor::readgs()); diff --git a/src/mm/mod.rs b/src/mm/mod.rs index 9f16f4104f..6c0dd9333a 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -59,6 +59,8 @@ use talc::TalcLock; use talc::source::Manual; pub use self::page_range_alloc::{PageRangeAllocator, PageRangeBox}; +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +pub use self::physicalmem::copy_page; pub use self::physicalmem::{FrameAlloc, FrameBox}; pub use self::virtualmem::{PageAlloc, PageBox}; #[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] diff --git a/src/mm/physicalmem.rs b/src/mm/physicalmem.rs index 758871230c..f2ef72dbcc 100644 --- a/src/mm/physicalmem.rs +++ b/src/mm/physicalmem.rs @@ -57,6 +57,46 @@ impl fmt::Display for FrameAlloc { pub type FrameBox = PageRangeBox; +/// Copy the physical page at `src_phys` into a freshly allocated page and return its address. +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +pub fn copy_page(src_phys: PhysAddr) -> PhysAddr { + use free_list::PageLayout; + + use crate::arch::mm::paging::{ + BasePageSize, PageSize, PageTableEntryFlags, PageTableEntryFlagsExt, + }; + use crate::mm::{FrameAlloc, PageBox, PageRangeAllocator}; + + let frame_layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); + let frame_range = FrameAlloc::allocate(frame_layout).expect("Failed to allocate page"); + let dst_phys = PhysAddr::new(frame_range.start().try_into().unwrap()); + + let page_layout = PageLayout::from_size(2 * BasePageSize::SIZE as usize).unwrap(); + let page_box = PageBox::new(page_layout).unwrap(); + let virt = VirtAddr::from(page_box.start()); + + let flags = { + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable(); + flags + }; + crate::arch::mm::paging::map::(virt, src_phys, 1, flags); + crate::arch::mm::paging::map::(virt + BasePageSize::SIZE, dst_phys, 1, flags); + + unsafe { + core::ptr::copy_nonoverlapping( + virt.as_ptr::(), + (virt + BasePageSize::SIZE).as_mut_ptr::(), + BasePageSize::SIZE as usize, + ); + } + + crate::arch::mm::paging::unmap::(virt, 2); + // page_box is dropped here, freeing the virtual memory + + dst_phys +} + pub fn total_memory_size() -> usize { TOTAL_MEMORY.load(Ordering::Relaxed) } diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index e5d8ae802a..b4d1646562 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -45,6 +45,8 @@ static WAITING_TASKS: InterruptTicketMutex /// Map between Task ID and TaskHandle static TASKS: InterruptTicketMutex> = InterruptTicketMutex::new(BTreeMap::new()); +/// Count the number of spawned tasks +static SPAWN_COUNTER: AtomicU32 = AtomicU32::new(1); /// Unique identifier for a core. pub type CoreId = u32; @@ -445,6 +447,13 @@ impl PerCoreScheduler { without_interrupts(|| self.current_task.borrow().id) } + /// Returns the Rc> for the currently running task. + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[inline] + pub fn get_current_task(&self) -> Rc> { + self.current_task.clone() + } + #[inline] pub fn get_current_task_object_map( &self, @@ -906,11 +915,9 @@ pub unsafe fn spawn( stack_size: usize, selector: isize, ) -> TaskId { - static CORE_COUNTER: AtomicU32 = AtomicU32::new(1); - let core_id = if selector < 0 { // use Round Robin to schedule the cores - CORE_COUNTER.fetch_add(1, Ordering::SeqCst) % get_processor_count() + SPAWN_COUNTER.fetch_add(1, Ordering::SeqCst) % get_processor_count() } else { selector as u32 }; @@ -944,6 +951,118 @@ pub fn join(id: TaskId) -> Result<(), ()> { } } +/// Fork the current task. +/// +/// Marks user pages as Copy-On-Write, duplicates the page table hierarchy, +/// copies the kernel stack, and enqueues a new child task that will resume +/// execution right after the `prepare_fork_child_stack` call returning 1. +/// +/// Returns the child's `TaskId` in the parent; the child itself sees `TaskId(0)`. +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +pub unsafe fn fork() -> TaskId { + use memory_addresses::VirtAddr; + + use crate::arch::{prepare_fork_child_stack, prepare_mem_copy_on_write}; + + let core_id = SPAWN_COUNTER.fetch_add(1, Ordering::SeqCst) % get_processor_count(); + + // Mark user pages COW before copying the page table. + prepare_mem_copy_on_write(); + + let stack_size = core_scheduler() + .get_current_task() + .borrow() + .stacks + .get_user_stack_size(); + let stacks = TaskStacks::new(stack_size); + + let mut child_stack_pointer: usize = 0; + let mut child_root_page_table: usize = 0; + + // Copy the kernel stack and duplicate the page table; returns false in parent. + let is_child = unsafe { + prepare_fork_child_stack( + &raw mut child_stack_pointer, + &raw mut child_root_page_table, + stacks.get_stack_virt_addr().as_usize(), + ) + }; + + if is_child { + // We are in the child context (stack is already switched). + // Prevent the newly-allocated stacks from being dropped here — + // they are owned by the child task created in the parent. + core::mem::forget(stacks); + return TaskId::from(0); + } + + // Parent path: register the child task. + let tid = get_tid(); + let child_last_sp = VirtAddr::new(child_stack_pointer.try_into().unwrap()); + let parent_user_sp = core_scheduler() + .get_current_task() + .borrow() + .user_stack_pointer; + let parent_prio = core_scheduler().get_current_task().borrow().prio; + let parent_object_map = core_scheduler().get_current_task_object_map(); + let parent_heap = core_scheduler().get_current_task().borrow().heap.clone(); + + let child_task = Task::new_fork( + tid, + core_id, + TaskStatus::Ready, + parent_prio, + stacks, + child_last_sp, + parent_user_sp, + parent_object_map, + child_root_page_table, + parent_heap, + ); + + let wakeup = { + #[cfg(feature = "smp")] + let _input_locked = get_scheduler_input(core_id).lock(); + WAITING_TASKS.lock().insert(tid, VecDeque::with_capacity(1)); + TASKS.lock().insert( + tid, + TaskHandle::new( + tid, + parent_prio, + #[cfg(feature = "smp")] + core_id, + ), + ); + NO_TASKS.fetch_add(1, Ordering::SeqCst); + + #[cfg(feature = "smp")] + if core_id == core_scheduler().core_id { + let task = Rc::new(RefCell::new(child_task)); + core_scheduler().ready_queue.push(task); + false + } else { + // For SMP we'd need to send to the target core; for now push locally. + let task = Rc::new(RefCell::new(child_task)); + core_scheduler().ready_queue.push(task); + false + } + #[cfg(not(feature = "smp"))] + { + let task = Rc::new(RefCell::new(child_task)); + core_scheduler().ready_queue.push(task); + false + } + }; + + if wakeup { + arch::wakeup_core(core_id); + } + + debug!("Child was created and has the id {tid}"); + + tid +} + pub fn shutdown(arg: i32) -> ! { crate::syscalls::shutdown(arg) } diff --git a/src/scheduler/task/mod.rs b/src/scheduler/task/mod.rs index e5e1fe171d..6ebade34ee 100644 --- a/src/scheduler/task/mod.rs +++ b/src/scheduler/task/mod.rs @@ -355,6 +355,30 @@ impl PriorityTaskQueue { } } +/// Tracks the heap region of a user process for demand-paging. +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +pub(crate) struct Heap { + pub start: VirtAddr, + pub end: VirtAddr, +} + +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +impl Heap { + pub fn new_empty() -> Self { + Heap { + start: VirtAddr::zero(), + end: VirtAddr::zero(), + } + } + + pub fn contains(&self, addr: VirtAddr) -> bool { + if self.start == VirtAddr::zero() || self.end == VirtAddr::zero() { + return false; + } + addr >= self.start && addr < self.end + } +} + /// A task control block, which identifies either a process or a thread #[cfg_attr(any(target_arch = "x86_64", target_arch = "aarch64"), repr(align(128)))] #[cfg_attr( @@ -386,6 +410,9 @@ pub(crate) struct Task { // Physical address of the 1st level page table #[cfg(all(target_arch = "x86_64", feature = "common-os"))] pub root_page_table: usize, + /// Heap region tracked for demand-paging + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + pub heap: Arc, } pub(crate) trait TaskFrame { @@ -417,7 +444,9 @@ impl Task { #[cfg(not(feature = "common-os"))] tls: None, #[cfg(all(target_arch = "x86_64", feature = "common-os"))] - root_page_table: crate::arch::mm::create_new_root_page_table(), + root_page_table: arch::create_new_root_page_table(), + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + heap: Arc::new(Heap::new_empty()), } } @@ -466,6 +495,42 @@ impl Task { tls, #[cfg(all(target_arch = "x86_64", feature = "common-os"))] root_page_table: *crate::scheduler::BOOT_ROOT_PAGE_TABLE.get().unwrap(), + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + heap: Arc::new(Heap::new_empty()), + } + } + + /// Create a forked task that starts at the point where the parent called fork. + /// `last_stack_pointer` is set to the child's pre-computed stack pointer. + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[allow(clippy::too_many_arguments)] + pub fn new_fork( + tid: TaskId, + core_id: CoreId, + task_status: TaskStatus, + task_prio: Priority, + stacks: TaskStacks, + last_stack_pointer: VirtAddr, + user_stack_pointer: VirtAddr, + object_map: Arc>, RandomState>>>, + root_page_table: usize, + parent_heap: Arc, + ) -> Task { + debug!("Creating forked task {tid} on core {core_id}"); + Task { + id: tid, + status: task_status, + prio: task_prio, + last_stack_pointer, + user_stack_pointer, + last_fpu_state: arch::processor::FPUState::new(), + core_id, + stacks, + object_map, + #[cfg(not(feature = "common-os"))] + tls: None, + root_page_table, + heap: parent_heap, } } } diff --git a/src/syscalls/entropy.rs b/src/syscalls/entropy.rs index 6292ce597e..34ea0d425a 100644 --- a/src/syscalls/entropy.rs +++ b/src/syscalls/entropy.rs @@ -53,7 +53,14 @@ unsafe fn read_entropy(buf: *mut u8, len: usize, flags: u32) -> isize { #[hermit_macro::system] #[unsafe(no_mangle)] pub unsafe extern "C" fn sys_read_entropy(buf: *mut u8, len: usize, flags: u32) -> isize { - unsafe { read_entropy(buf, len, flags) } + let ret = unsafe { read_entropy(buf, len, flags) }; + if ret >= 0 { + debug!("Read {ret} bytes with cryptographically secure random data"); + } else { + error!("Unable to read cryptographically secure random data"); + } + + ret } /// Create a cryptographicly secure 32bit random number with the support of diff --git a/src/syscalls/table.rs b/src/syscalls/table.rs index fcbb32b489..327e08708f 100644 --- a/src/syscalls/table.rs +++ b/src/syscalls/table.rs @@ -30,6 +30,12 @@ const SYSNO_OPEN: usize = 11; const SYSNO_WRITEV: usize = 12; /// Number of the system call `readv` const SYSNO_READV: usize = 13; +/// number of the system call `fork` +#[cfg(target_arch = "x86_64")] +const SYSNO_FORK: usize = 14; +/// number of the system call `waitpid` +#[cfg(target_arch = "x86_64")] +const SYSNO_WAITPID: usize = 15; /// Total number of system calls const NO_SYSCALLS: usize = 32; @@ -76,6 +82,11 @@ impl SyscallTable { table.handle[SYSNO_OPEN] = sys_open as *const _; table.handle[SYSNO_READV] = sys_readv as *const _; table.handle[SYSNO_WRITEV] = sys_writev as *const _; + #[cfg(target_arch = "x86_64")] + { + table.handle[SYSNO_FORK] = sys_fork as *const _; + table.handle[SYSNO_WAITPID] = sys_waitpid as *const _; + } table } diff --git a/src/syscalls/tasks.rs b/src/syscalls/tasks.rs index bd2d31e73f..f860eabbc3 100644 --- a/src/syscalls/tasks.rs +++ b/src/syscalls/tasks.rs @@ -15,10 +15,59 @@ use crate::{arch, scheduler}; pub type SignalHandler = extern "C" fn(i32); pub type Tid = i32; +/// Fork the current process. +/// Returns the child's PID to the parent, and 0 to the child. +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[hermit_macro::system(errno)] +#[unsafe(no_mangle)] +pub extern "C" fn sys_fork() -> i32 { + unsafe { scheduler::fork().into() } +} + +/// Fork the current process. +/// In case of a unikernel, this system call always fail. +#[cfg(not(feature = "common-os"))] +#[hermit_macro::system(errno)] +#[unsafe(no_mangle)] +pub extern "C" fn sys_fork() -> i32 { + -i32::from(Errno::Nosys) +} + +/// Waitpid block the current process until termination of process `pid`. +/// Returns 0 is the process terminates +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[hermit_macro::system(errno)] +#[unsafe(no_mangle)] +pub extern "C" fn sys_waitpid(pid: Tid) -> i32 { + match scheduler::join(TaskId::from(pid)) { + Ok(()) => 0, + _ => -i32::from(Errno::Inval), + } +} + +/// Waitpid block the current process until termination of process `pid`. +/// In case of a unikernel, this system call always fail. +#[cfg(not(feature = "common-os"))] +#[hermit_macro::system(errno)] +#[unsafe(no_mangle)] +pub extern "C" fn sys_waitpid(pid: Tid) -> i32 { + -i32::from(Errno::Nosys) +} + #[hermit_macro::system] #[unsafe(no_mangle)] pub extern "C" fn sys_getpid() -> Tid { - 0 + #[cfg(not(feature = "common-os"))] + { + // an unikernel doesn't have a pid => return always 0 + 0 + } + + #[cfg(feature = "common-os")] + { + // return the current task id + core_scheduler().get_current_task_handle().get_id().into() + } } #[cfg(feature = "newlib")] @@ -132,7 +181,7 @@ pub unsafe extern "C" fn sys_clone(id: *mut Tid, func: extern "C" fn(usize), arg 0 } -#[hermit_macro::system(errno)] +#[hermit_macro::system] #[unsafe(no_mangle)] pub extern "C" fn sys_yield() { core_scheduler().reschedule(); From b53a442b4abcb04766c2130167c145abac05ea6e Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Wed, 8 Apr 2026 23:25:18 +0200 Subject: [PATCH 06/48] create independent fd tables for each process - load_application: initialise a fresh file-descriptor object map for the user process instead of sharing the kernel task's map - fork: clone the parent's object map into the child rather than sharing the same Arc, so parent and child have independent fd tables --- src/arch/x86_64/kernel/mod.rs | 28 ++++++++++++++++++++++++++++ src/scheduler/mod.rs | 20 +++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index e0696d93a6..56b3e38ee6 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -205,6 +205,34 @@ where use crate::arch::x86_64::mm::paging::{self, PageTableEntryFlags, PageTableEntryFlagsExt}; use crate::mm::{FrameAlloc, PageRangeAllocator}; + use crate::fd::{Fd, RawFd, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; + use crate::fd::stdio::*; + + // each process has to provide its own object_map + // => create a new one + let mut object_map = HashMap::< + RawFd, + Arc>, + RandomState, + >::with_hasher( + RandomState::with_seeds(0, 0, 0, 0), + ); + if env::is_uhyve() { + let stdin = Arc::new(async_lock::RwLock::new(UhyveStdin::new().into())); + let stdout = Arc::new(async_lock::RwLock::new(UhyveStdout::new().into())); + let stderr = Arc::new(async_lock::RwLock::new(UhyveStderr::new().into())); + object_map.insert(STDIN_FILENO, stdin); + object_map.insert(STDOUT_FILENO, stdout); + object_map.insert(STDERR_FILENO, stderr); + } else { + let stdin = Arc::new(async_lock::RwLock::new(GenericStdin::new().into())); + let stdout = Arc::new(async_lock::RwLock::new(GenericStdout::new().into())); + let stderr = Arc::new(async_lock::RwLock::new(GenericStderr::new().into())); + object_map.insert(STDIN_FILENO, stdin); + object_map.insert(STDOUT_FILENO, stdout); + object_map.insert(STDERR_FILENO, stderr); + } + core_scheduler().set_current_task_object_map(Arc::new(RwSpinLock::new(object_map))); let code_size = (code_size as usize + LOADER_STACK_SIZE).align_up(BasePageSize::SIZE as usize); let layout = PageLayout::from_size_align(code_size, BasePageSize::SIZE as usize).unwrap(); diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index b4d1646562..dc57d446f3 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -454,6 +454,15 @@ impl PerCoreScheduler { self.current_task.clone() } + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[inline] + pub fn set_current_task_object_map( + &mut self, + object_map: Arc>, RandomState>>> + ) { + without_interrupts(|| self.current_task.borrow_mut().object_map = object_map) + } + #[inline] pub fn get_current_task_object_map( &self, @@ -1004,7 +1013,16 @@ pub unsafe fn fork() -> TaskId { .borrow() .user_stack_pointer; let parent_prio = core_scheduler().get_current_task().borrow().prio; - let parent_object_map = core_scheduler().get_current_task_object_map(); + let parent_object_map = Arc::new(RwSpinLock::new(HashMap::< + RawFd, + Arc>, + RandomState, + >::with_hasher(RandomState::with_seeds( + 0, 0, 0, 0, + )))); + for (key, val) in core_scheduler().get_current_task_object_map().read().iter() { + parent_object_map.write().insert(*key, val.clone()); + } let parent_heap = core_scheduler().get_current_task().borrow().heap.clone(); let child_task = Task::new_fork( From 96004fe2a0e53bbca33a55d1ea9c17c1238e58bd Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Wed, 8 Apr 2026 23:38:12 +0200 Subject: [PATCH 07/48] fix(clippy): resolve all warnings in common-os build - Replace redundant `as *mut PageTable` casts with turbofish syntax on `ptr::with_exposed_provenance_mut` - Replace `core::ptr::write_bytes` with pointer method `.write_bytes()` - Replace `core::ptr::copy_nonoverlapping` with `dst.copy_from_slice(src)` in physicalmem and `*new_pt = cur_pt.clone()` in mm/mod.rs - Add missing semicolon after `without_interrupts` call in scheduler --- src/arch/x86_64/mm/mod.rs | 8 +------- src/arch/x86_64/mm/paging.rs | 8 +++----- src/mm/physicalmem.rs | 5 +++-- src/scheduler/mod.rs | 4 ++-- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/arch/x86_64/mm/mod.rs b/src/arch/x86_64/mm/mod.rs index 83232dab95..12e4b41cbb 100644 --- a/src/arch/x86_64/mm/mod.rs +++ b/src/arch/x86_64/mm/mod.rs @@ -169,13 +169,7 @@ pub fn copy_current_root_page_table() -> usize { }; // Copy PT entries verbatim — data pages are shared (already COW-marked) - unsafe { - core::ptr::copy_nonoverlapping( - core::ptr::from_ref::(cur_pt), - core::ptr::from_mut::(new_pt), - 1, - ); - } + *new_pt = cur_pt.clone(); } } } diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index 4b43acec0a..b76b9c5feb 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -296,8 +296,7 @@ pub fn mark_user_pages_copy_on_write() { // we can dereference physical addresses directly as page table pointers. let (frame, _) = Cr3::read(); let pml4 = unsafe { - &mut *(ptr::with_exposed_provenance_mut(frame.start_address().as_u64() as usize) - as *mut PageTable) + &mut *ptr::with_exposed_provenance_mut::(frame.start_address().as_u64() as usize) }; // Walk PML4 entries 1..511 (user-space entries). @@ -308,8 +307,7 @@ pub fn mark_user_pages_copy_on_write() { continue; } let pdpt = unsafe { - &mut *(ptr::with_exposed_provenance_mut(pml4_entry.addr().as_u64() as usize) - as *mut PageTable) + &mut *ptr::with_exposed_provenance_mut::(pml4_entry.addr().as_u64() as usize) }; for pdpt_idx in 0..512usize { let pdpt_entry = &mut pdpt[pdpt_idx]; @@ -494,7 +492,7 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( flags.normal().user().writable().execute_disable(); map::(virtaddr.into(), physaddr, 1, flags); unsafe { - core::ptr::write_bytes(virtaddr.as_mut_ptr::(), 0, BasePageSize::SIZE as usize); + virtaddr.as_mut_ptr::().write_bytes(0, BasePageSize::SIZE as usize); } if swapped_gs { diff --git a/src/mm/physicalmem.rs b/src/mm/physicalmem.rs index f2ef72dbcc..1e2ddc008e 100644 --- a/src/mm/physicalmem.rs +++ b/src/mm/physicalmem.rs @@ -84,11 +84,12 @@ pub fn copy_page(src_phys: PhysAddr) -> PhysAddr { crate::arch::mm::paging::map::(virt + BasePageSize::SIZE, dst_phys, 1, flags); unsafe { - core::ptr::copy_nonoverlapping( - virt.as_ptr::(), + let src = core::slice::from_raw_parts(virt.as_ptr::(), BasePageSize::SIZE as usize); + let dst = core::slice::from_raw_parts_mut( (virt + BasePageSize::SIZE).as_mut_ptr::(), BasePageSize::SIZE as usize, ); + dst.copy_from_slice(src); } crate::arch::mm::paging::unmap::(virt, 2); diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index dc57d446f3..549e5c771a 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -458,9 +458,9 @@ impl PerCoreScheduler { #[inline] pub fn set_current_task_object_map( &mut self, - object_map: Arc>, RandomState>>> + object_map: Arc>, RandomState>>>, ) { - without_interrupts(|| self.current_task.borrow_mut().object_map = object_map) + without_interrupts(|| self.current_task.borrow_mut().object_map = object_map); } #[inline] From ffbb20ed8749b6bf4cbd517bfb0522b621a02102 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Thu, 9 Apr 2026 09:42:03 +0200 Subject: [PATCH 08/48] mm/x86_64: fix COW page leak by reference-counting shared frames Before this change, the COW page-fault handler always allocated a fresh physical frame and mapped it, but never freed the original shared frame. This caused a 4 KiB leak per write fault on a COW page. Introduce a sparse per-frame reference count (BTreeMap protected by InterruptTicketMutex) that tracks how many page-table entries point to each COW-shared frame. Memory use scales with the number of actively shared frames, not with total physical memory. - mark_user_pages_copy_on_write: increment refcount for every newly COW-marked frame (parent's reference) - copy_current_root_page_table: increment refcount for every COW entry copied into the child's page table (child's reference) - COW fault handler: decrement refcount; if the last reference is dropped, remap the existing frame as writable directly instead of copying it (avoids an allocation in the single-sharer case); otherwise copy the frame and map the private copy --- src/arch/x86_64/mm/mod.rs | 10 +++++++++ src/arch/x86_64/mm/paging.rs | 13 +++++++++-- src/mm/mod.rs | 2 ++ src/mm/physicalmem.rs | 43 ++++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src/arch/x86_64/mm/mod.rs b/src/arch/x86_64/mm/mod.rs index 12e4b41cbb..a35eebc5a2 100644 --- a/src/arch/x86_64/mm/mod.rs +++ b/src/arch/x86_64/mm/mod.rs @@ -170,6 +170,16 @@ pub fn copy_current_root_page_table() -> usize { // Copy PT entries verbatim — data pages are shared (already COW-marked) *new_pt = cur_pt.clone(); + + // The child now holds an additional COW reference to every + // COW-marked frame in this page table. + for entry in new_pt.iter() { + if entry.flags().contains(PageTableFlags::PRESENT) + && entry.flags().contains(PageTableFlags::BIT_9) + { + crate::mm::frame_ref_inc(entry.addr().as_u64() as usize); + } + } } } } diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index b76b9c5feb..f4e02ab247 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -345,6 +345,8 @@ pub fn mark_user_pages_copy_on_write() { { let new_flags = *pt_entry.flags().copy_on_write(); pt_entry.set_addr(pt_entry.addr(), new_flags); + // This task now holds a COW reference to the frame. + crate::mm::frame_ref_inc(pt_entry.addr().as_u64() as usize); } } } @@ -456,11 +458,18 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( { // COW fault: allocate a new page, copy old contents, map as writable. if let Some(src_phys) = virtual_to_physical(virtaddr.into()) { - let new_phys = crate::mm::copy_page(src_phys); let mut new_flags = flags; new_flags.insert(PageTableEntryFlags::WRITABLE); new_flags.remove(PageTableEntryFlags::BIT_9); - map::(virtaddr.into(), new_phys, 1, new_flags); + // Drop this task's COW reference + let ref_is_zero = crate::mm::frame_ref_dec_and_free(src_phys.as_u64() as usize); + + if ref_is_zero { + map::(virtaddr.into(), src_phys, 1, new_flags); + } else { + let new_phys = crate::mm::copy_page(src_phys); + map::(virtaddr.into(), new_phys, 1, new_flags); + } if swapped_gs { unsafe { diff --git a/src/mm/mod.rs b/src/mm/mod.rs index 6c0dd9333a..fdd7d8cb11 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -62,6 +62,8 @@ pub use self::page_range_alloc::{PageRangeAllocator, PageRangeBox}; #[cfg(all(target_arch = "x86_64", feature = "common-os"))] pub use self::physicalmem::copy_page; pub use self::physicalmem::{FrameAlloc, FrameBox}; +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +pub use self::physicalmem::{frame_ref_dec_and_free, frame_ref_inc}; pub use self::virtualmem::{PageAlloc, PageBox}; #[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] use crate::arch::mm::paging::HugePageSize; diff --git a/src/mm/physicalmem.rs b/src/mm/physicalmem.rs index 1e2ddc008e..21d22c30c6 100644 --- a/src/mm/physicalmem.rs +++ b/src/mm/physicalmem.rs @@ -1,3 +1,5 @@ +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +use alloc::collections::BTreeMap; use core::alloc::AllocError; use core::fmt; use core::sync::atomic::{AtomicUsize, Ordering}; @@ -18,6 +20,47 @@ static PHYSICAL_FREE_LIST: InterruptTicketMutex> = InterruptTicketMutex::new(FreeList::new()); pub static TOTAL_MEMORY: AtomicUsize = AtomicUsize::new(0); +/// Sparse per-frame COW reference counts. +/// Only frames that are actively COW-shared have an entry; exclusively-owned +/// frames are absent (equivalent to refcount 0). Stored in a `BTreeMap` so +/// that memory use scales with the number of *shared* frames, not with total +/// physical memory. +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +static PAGE_REFCOUNTS: InterruptTicketMutex> = + InterruptTicketMutex::new(BTreeMap::new()); + +/// Increment the COW reference count for `phys_addr` (4 KiB-aligned frame). +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +pub fn frame_ref_inc(phys_addr: usize) { + let frame = phys_addr >> 12; + *PAGE_REFCOUNTS.lock().entry(frame).or_insert(0) += 1; +} + +/// Decrement the COW reference count for `phys_addr`. +/// If the count reaches zero the the function returned true. +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +pub fn frame_ref_dec_and_free(phys_addr: usize) -> bool { + let frame = phys_addr >> 12; + let mut map = PAGE_REFCOUNTS.lock(); + match map.get_mut(&frame) { + None => { + warn!("frame_ref_dec_and_free: no refcount entry for frame {phys_addr:#x}"); + false + } + Some(count) if *count <= 1 => { + map.remove(&frame); + /*drop(map); // release lock before deallocating + let range = PageRange::new(phys_addr, phys_addr + free_list::PAGE_SIZE).unwrap(); + unsafe { FrameAlloc::deallocate(range) };*/ + true + } + Some(count) => { + *count -= 1; + false + } + } +} + pub struct FrameAlloc; impl PageRangeAllocator for FrameAlloc { From 54baf55961ccffa3b39a341ab90008df383678ca Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Thu, 9 Apr 2026 12:42:28 +0200 Subject: [PATCH 09/48] x86_64: track page fault count in IRQ statistics Register page faults as IRQ 14 in the interrupt counter so they show interrupt diagnostics. Useful for verifying COW behaviour after fork. --- src/arch/x86_64/kernel/interrupts.rs | 2 ++ src/arch/x86_64/mm/paging.rs | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/arch/x86_64/kernel/interrupts.rs b/src/arch/x86_64/kernel/interrupts.rs index 3ed4572acb..c967707f11 100644 --- a/src/arch/x86_64/kernel/interrupts.rs +++ b/src/arch/x86_64/kernel/interrupts.rs @@ -155,6 +155,8 @@ pub(crate) fn install() { } IRQ_NAMES.lock().insert(7, "FPU"); + #[cfg(feature = "common-os")] + IRQ_NAMES.lock().insert(14, "Page Fault"); } pub(crate) fn install_handlers(handlers: InterruptHandlerMap) { diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index f4e02ab247..d61135bfe4 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -427,8 +427,7 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( ) { use core::arch::asm; - use crate::arch::core_local::core_scheduler; - + use crate::arch::core_local::{core_scheduler, increment_irq_counter}; let swapped_gs = unsafe { if stack_frame.as_mut().read().code_segment == SegmentSelector(0x08) { false @@ -437,6 +436,8 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( true } }; + + increment_irq_counter(14); let faulting_addr = Cr2::read().unwrap(); let virtaddr = faulting_addr.align_down(BasePageSize::SIZE); From 4265c7dba67172fa63b52959b3952ba0fc494036 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Thu, 9 Apr 2026 20:53:27 +0200 Subject: [PATCH 10/48] syscall: add spawn_process syscall (number 16) The spawn_process function is provided by the loader and declared as an external C symbol. The kernel routes syscall 16 to it. A stub returning -ENOSYS is provided for non-common-os builds. Also demote a log message in jump_to_user_land from info to debug. --- src/arch/x86_64/kernel/mod.rs | 2 +- src/lib.rs | 5 +---- src/syscalls/table.rs | 10 ++++++++++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index 56b3e38ee6..c0580a077f 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -301,7 +301,7 @@ pub unsafe fn jump_to_user_land(entry_point: usize, code_size: usize, arg: &[&st use crate::arch::x86_64::kernel::scheduler::TaskStacks; - info!("Create new file descriptor table"); + debug!("Create new file descriptor table"); core_scheduler().recreate_objmap().unwrap(); let entry_point: usize = LOADER_START | entry_point; diff --git a/src/lib.rs b/src/lib.rs index 25745c5744..e5791702b2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,10 +47,7 @@ #![feature(allocator_api)] #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr( - all( - not(any(feature = "common-os", feature = "nostd")), - not(target_arch = "riscv64"), - ), + all(not(feature = "nostd"), not(target_arch = "riscv64"),), feature(linkage) )] #![feature(linked_list_cursors)] diff --git a/src/syscalls/table.rs b/src/syscalls/table.rs index 327e08708f..018c23672b 100644 --- a/src/syscalls/table.rs +++ b/src/syscalls/table.rs @@ -36,6 +36,8 @@ const SYSNO_FORK: usize = 14; /// number of the system call `waitpid` #[cfg(target_arch = "x86_64")] const SYSNO_WAITPID: usize = 15; +/// number of the system call `spawn_process` +const SYSNO_SPAWN_PROCESS: usize = 16; /// Total number of system calls const NO_SYSCALLS: usize = 32; @@ -45,6 +47,13 @@ extern "C" fn invalid_syscall(sys_no: u64) -> ! { sys_exit(1); } +/// loader will replace this function +#[linkage = "weak"] +#[unsafe(no_mangle)] +pub extern "C" fn sys_spawn_process(_name: *const core::ffi::c_char) -> i32 { + -i32::from(Errno::Nosys) +} + #[allow(unused_assignments)] #[unsafe(no_mangle)] #[unsafe(naked)] @@ -87,6 +96,7 @@ impl SyscallTable { table.handle[SYSNO_FORK] = sys_fork as *const _; table.handle[SYSNO_WAITPID] = sys_waitpid as *const _; } + table.handle[SYSNO_SPAWN_PROCESS] = sys_spawn_process as *const _; table } From ee2db1c2e7c39e365a051b9926e38131973c9be1 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Fri, 10 Apr 2026 20:31:36 +0200 Subject: [PATCH 11/48] kernel: scope idle task OBJECT_MAP to unikernel builds Under `common-os`, each process receives its own `object_map` when the application is loaded, so the shared stdio map in `Task::new_idle` was never actually used. Gate the static and its stdin/stdout/stderr setup with `cfg(not(feature = "common-os"))` and give the idle task an empty map in the common-os case. --- src/scheduler/task/mod.rs | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/scheduler/task/mod.rs b/src/scheduler/task/mod.rs index 6ebade34ee..7ccf4f8f82 100644 --- a/src/scheduler/task/mod.rs +++ b/src/scheduler/task/mod.rs @@ -13,16 +13,23 @@ use core::{cmp, fmt}; use ahash::RandomState; use crossbeam_utils::CachePadded; use hashbrown::HashMap; -use hermit_sync::{OnceCell, RwSpinLock}; +#[cfg(not(feature = "common-os"))] +use hermit_sync::OnceCell; +use hermit_sync::RwSpinLock; use memory_addresses::VirtAddr; #[cfg(not(feature = "common-os"))] use self::tls::Tls; use super::timer_interrupts::{Source, create_timer_abs}; -use crate::arch::kernel::core_local::*; +use crate::arch; +use crate::arch::core_local::*; use crate::arch::kernel::processor::{self, FPUState}; -use crate::arch::kernel::scheduler::TaskStacks; +use crate::arch::scheduler::TaskStacks; +#[cfg(not(feature = "common-os"))] +use crate::env; use crate::fd::{Fd, RawFd, stdio}; +#[cfg(not(feature = "common-os"))] +use crate::fd::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; use crate::scheduler::CoreId; /// Returns the most significant bit. @@ -453,11 +460,16 @@ impl Task { pub fn new_idle(tid: TaskId, core_id: CoreId) -> Task { debug!("Creating idle task {tid}"); - /// All cores use the same mapping between file descriptor and the referenced object + /// In the unikernel case all cores share the same mapping between + /// file descriptor and the referenced object. Under `common-os`, + /// each process gets its own `object_map` when the application is + /// loaded, so the idle task only needs an empty placeholder. + #[cfg(not(feature = "common-os"))] static OBJECT_MAP: OnceCell< Arc>, RandomState>>>, > = OnceCell::new(); + #[cfg(not(feature = "common-os"))] if core_id == 0 { OBJECT_MAP .set(Arc::new(RwSpinLock::new(HashMap::< @@ -490,7 +502,16 @@ impl Task { last_fpu_state: FPUState::new(), core_id, stacks: TaskStacks::from_boot_stacks(), + #[cfg(not(feature = "common-os"))] object_map: OBJECT_MAP.get().unwrap().clone(), + #[cfg(feature = "common-os")] + object_map: Arc::new(RwSpinLock::new(HashMap::< + RawFd, + Arc>, + RandomState, + >::with_hasher(RandomState::with_seeds( + 0, 0, 0, 0, + )))), #[cfg(not(feature = "common-os"))] tls, #[cfg(all(target_arch = "x86_64", feature = "common-os"))] From 327967ce5933f3dd1215a8c11da14ed24eeb32a8 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sat, 11 Apr 2026 14:07:37 +0200 Subject: [PATCH 12/48] fix(fork): stop leaking stack frames in copy_kernel_stack_to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `copy_kernel_stack_to` allocated fresh physical frames via `copy_page` for every mapped page of the parent kernel stack and remapped them at the child's virtual base — but `TaskStacks::new` had already allocated and mapped the child's stack frames at that same address. The new mappings silently replaced the original ones, so `TaskStacks::Drop` later freed the unreachable frames while the frames actually installed in the page table leaked. Replace the `copy_page` + `map` loop with a plain `copy_nonoverlapping` between the parent's and child's stack ranges, which are both already mapped in the current page table. No extra frames are allocated, and `TaskStacks::Drop` once again frees the frames that are actually installed. --- src/arch/x86_64/kernel/mod.rs | 7 ++ src/arch/x86_64/mm/mod.rs | 11 ++-- src/arch/x86_64/mm/paging.rs | 117 ++++++++++++++++++++++++++++------ src/mm/mod.rs | 2 +- src/mm/physicalmem.rs | 19 +++--- src/scheduler/mod.rs | 10 ++- src/scheduler/task/mod.rs | 8 ++- src/syscalls/table.rs | 4 +- src/syscalls/tasks.rs | 6 +- 9 files changed, 144 insertions(+), 40 deletions(-) diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index c0580a077f..fbfd81f253 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -207,6 +207,7 @@ where use crate::mm::{FrameAlloc, PageRangeAllocator}; use crate::fd::{Fd, RawFd, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; use crate::fd::stdio::*; + use crate::mm::frame_ref_inc; // each process has to provide its own object_map // => create a new one @@ -238,6 +239,9 @@ where let layout = PageLayout::from_size_align(code_size, BasePageSize::SIZE as usize).unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let physaddr = PhysAddr::from(frame_range.start()); + for i in 0..code_size / BasePageSize::SIZE as usize { + frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize); + } let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().user().execute_enable(); @@ -263,6 +267,9 @@ where let layout = PageLayout::from_size(tls_memsz).unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let physaddr = PhysAddr::from(frame_range.start()); + for i in 0..tls_memsz / BasePageSize::SIZE as usize { + frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize); + } let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().user().execute_disable(); diff --git a/src/arch/x86_64/mm/mod.rs b/src/arch/x86_64/mm/mod.rs index a35eebc5a2..7750a3c281 100644 --- a/src/arch/x86_64/mm/mod.rs +++ b/src/arch/x86_64/mm/mod.rs @@ -10,6 +10,9 @@ use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; #[cfg(feature = "common-os")] use crate::arch::mm::paging::{PageTableEntryFlags, PageTableEntryFlagsExt}; use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; +/// Copy the kernel stack pages of the current task to a new base address. +#[cfg(feature = "common-os")] +pub use paging::{drop_user_space, copy_kernel_stack_to}; #[cfg(feature = "common-os")] pub fn create_new_root_page_table() -> usize { @@ -177,7 +180,7 @@ pub fn copy_current_root_page_table() -> usize { if entry.flags().contains(PageTableFlags::PRESENT) && entry.flags().contains(PageTableFlags::BIT_9) { - crate::mm::frame_ref_inc(entry.addr().as_u64() as usize); + crate::mm::frame_ref_inc(entry.addr().into()); } } } @@ -206,12 +209,6 @@ pub fn prepare_mem_copy_on_write() { paging::mark_user_pages_copy_on_write(); } -/// Copy the kernel stack pages of the current task to a new base address. -#[cfg(feature = "common-os")] -pub fn copy_kernel_stack_to(stack_address: usize) { - paging::copy_kernel_stack_to(stack_address); -} - pub unsafe fn init() { paging::init(); unsafe { diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index d61135bfe4..5d9da5d2b8 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -336,17 +336,11 @@ pub fn mark_user_pages_copy_on_write() { }; for pt_idx in 0..512usize { let pt_entry = &mut pt[pt_idx]; - if pt_entry.flags().contains(PageTableEntryFlags::PRESENT) - && pt_entry.flags().contains(PageTableEntryFlags::WRITABLE) - && pt_entry - .flags() - .contains(PageTableEntryFlags::USER_ACCESSIBLE) + if pt_entry.flags().contains(PageTableEntryFlags::PRESENT|PageTableEntryFlags::WRITABLE|PageTableEntryFlags::USER_ACCESSIBLE) && !pt_entry.flags().contains(PageTableEntryFlags::BIT_9) { let new_flags = *pt_entry.flags().copy_on_write(); pt_entry.set_addr(pt_entry.addr(), new_flags); - // This task now holds a COW reference to the frame. - crate::mm::frame_ref_inc(pt_entry.addr().as_u64() as usize); } } } @@ -358,12 +352,94 @@ pub fn mark_user_pages_copy_on_write() { crate::arch::x86_64::kernel::apic::ipi_tlb_flush(); } -/// Copy all kernel stack pages of the current task to a new virtual base address. -/// Used by fork to give the child its own copy of the kernel stack. +#[cfg(feature = "common-os")] +pub fn drop_user_space(pml4_phys: usize) { + debug!("Drop the user space at PML4 {pml4_phys:#x}"); + + // Free a 4 KiB physical frame back to the frame allocator. + fn free_frame(phys: usize) { + let range = free_list::PageRange::new(phys, phys + free_list::PAGE_SIZE).unwrap(); + unsafe { FrameAlloc::deallocate(range) }; + } + + // Task::drop is usually called from another task's context (e.g. the + // idle task running `cleanup_tasks`), so `Cr3::read()` would return the + // page table of the cleaning task — not of the one being destroyed. + // Walk the stored PML4 directly through the kernel identity map. + let pml4 = unsafe { &mut *ptr::with_exposed_provenance_mut::(pml4_phys) }; + + // Walk PML4 entries 1..511 (user-space entries). + // Entry 0 is for the low kernel mapping; entry 511 is the self-reference. + for pml4_idx in 1..511usize { + let pml4_entry = &mut pml4[pml4_idx]; + if !pml4_entry.flags().contains(PageTableEntryFlags::PRESENT) { + continue; + } + let pdpt_phys = pml4_entry.addr().as_u64() as usize; + let pdpt = unsafe { &mut *ptr::with_exposed_provenance_mut::(pdpt_phys) }; + + for pdpt_idx in 0..512usize { + let pdpt_entry = &mut pdpt[pdpt_idx]; + if !pdpt_entry.flags().contains(PageTableEntryFlags::PRESENT) { + continue; + } + let pd_phys = pdpt_entry.addr().as_u64() as usize; + let pd = unsafe { &mut *ptr::with_exposed_provenance_mut::(pd_phys) }; + + for pd_idx in 0..512usize { + let pd_entry = &mut pd[pd_idx]; + if !pd_entry.flags().contains(PageTableEntryFlags::PRESENT) { + continue; + } + if pd_entry.flags().contains(PageTableEntryFlags::HUGE_PAGE) { + // Skip huge pages (2 MiB) - not handled as COW here + warn!("User space isn't able to use huge pages"); + continue; + } + let pt_phys = pd_entry.addr().as_u64() as usize; + let pt = unsafe { &mut *ptr::with_exposed_provenance_mut::(pt_phys) }; + + for pt_idx in 0..512usize { + let pt_entry = &mut pt[pt_idx]; + if pt_entry + .flags() + .contains(PageTableEntryFlags::PRESENT | PageTableEntryFlags::USER_ACCESSIBLE) + { + let phys_addr = PhysAddr::new(pt_entry.addr().as_u64()); + if crate::mm::frame_ref_dec(phys_addr) { + free_frame(phys_addr.as_u64() as usize); + } + pt_entry.set_unused(); + } + } + + // PT emptied → free the PT frame and clear the PD entry. + free_frame(pt_phys); + pd_entry.set_unused(); + } + + // PD emptied → free the PD frame and clear the PDPT entry. + free_frame(pd_phys); + pdpt_entry.set_unused(); + } + + // PDPT emptied → free the PDPT frame and clear the PML4 entry. + free_frame(pdpt_phys); + pml4_entry.set_unused(); + } + + // Finally free the PML4 itself. No TLB flush needed: the dropped page + // table is not loaded on any core. + free_frame(pml4_phys); +} + +/// Copy the contents of the current task's kernel stack to a new virtual +/// base address. Used by fork: the child's `TaskStacks::new` has already +/// allocated and mapped fresh physical frames at `stack_address`, so we +/// simply `memcpy` the parent's stack pages into the child's mapping. #[cfg(feature = "common-os")] pub fn copy_kernel_stack_to(stack_address: usize) { use crate::arch::core_local::core_scheduler; - use crate::mm::copy_page; let virt_addr = core_scheduler() .get_current_task() @@ -380,19 +456,21 @@ pub fn copy_kernel_stack_to(stack_address: usize) { let page_table = unsafe { identity_mapped_page_table() }; - // The virtual layout has 4 guard pages (unmapped) interspersed; iterate the - // full range including those so no mapped pages are missed. + // The virtual layout has 4 guard pages (unmapped) interspersed; iterate + // the full range including those so no mapped pages are missed. let full_virt_size = total_size as u64 + 4 * BasePageSize::SIZE; for i in (virt_addr.as_u64()..(virt_addr.as_u64() + full_virt_size)) .step_by(BasePageSize::SIZE as usize) { let virt = x86_64::VirtAddr::new(i); match page_table.translate(virt) { - TranslateResult::Mapped { flags, .. } => { - if let Some(src_phys) = virtual_to_physical(VirtAddr::new(i)) { - let new_phys = copy_page(src_phys); - let new_virt = VirtAddr::new(i + addr_diff); - map::(new_virt, new_phys, 1, flags); + TranslateResult::Mapped { .. } => { + // Both src and dst are already mapped in the current page + // table (dst via TaskStacks::new), so a plain memcpy suffices. + let src = ptr::with_exposed_provenance::(i as usize); + let dst = ptr::with_exposed_provenance_mut::((i + addr_diff) as usize); + unsafe { + dst.copy_from_nonoverlapping(src, BasePageSize::SIZE as usize); } } TranslateResult::NotMapped => {} @@ -463,12 +541,14 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( new_flags.insert(PageTableEntryFlags::WRITABLE); new_flags.remove(PageTableEntryFlags::BIT_9); // Drop this task's COW reference - let ref_is_zero = crate::mm::frame_ref_dec_and_free(src_phys.as_u64() as usize); + let ref_is_zero = crate::mm::frame_ref_dec(src_phys); if ref_is_zero { + crate::mm::frame_ref_inc(src_phys); map::(virtaddr.into(), src_phys, 1, new_flags); } else { let new_phys = crate::mm::copy_page(src_phys); + crate::mm::frame_ref_inc(new_phys); map::(virtaddr.into(), new_phys, 1, new_flags); } @@ -501,6 +581,7 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( let mut flags = PageTableEntryFlags::empty(); flags.normal().user().writable().execute_disable(); map::(virtaddr.into(), physaddr, 1, flags); + crate::mm::frame_ref_inc(physaddr); unsafe { virtaddr.as_mut_ptr::().write_bytes(0, BasePageSize::SIZE as usize); } diff --git a/src/mm/mod.rs b/src/mm/mod.rs index fdd7d8cb11..277735941a 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -63,7 +63,7 @@ pub use self::page_range_alloc::{PageRangeAllocator, PageRangeBox}; pub use self::physicalmem::copy_page; pub use self::physicalmem::{FrameAlloc, FrameBox}; #[cfg(all(target_arch = "x86_64", feature = "common-os"))] -pub use self::physicalmem::{frame_ref_dec_and_free, frame_ref_inc}; +pub use self::physicalmem::{frame_ref_dec, frame_ref_inc}; pub use self::virtualmem::{PageAlloc, PageBox}; #[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] use crate::arch::mm::paging::HugePageSize; diff --git a/src/mm/physicalmem.rs b/src/mm/physicalmem.rs index 21d22c30c6..250adec7f6 100644 --- a/src/mm/physicalmem.rs +++ b/src/mm/physicalmem.rs @@ -31,27 +31,24 @@ static PAGE_REFCOUNTS: InterruptTicketMutex> = /// Increment the COW reference count for `phys_addr` (4 KiB-aligned frame). #[cfg(all(target_arch = "x86_64", feature = "common-os"))] -pub fn frame_ref_inc(phys_addr: usize) { - let frame = phys_addr >> 12; +pub fn frame_ref_inc(phys_addr: PhysAddr) { + let frame = (phys_addr.as_u64() as usize) >> 12; *PAGE_REFCOUNTS.lock().entry(frame).or_insert(0) += 1; } /// Decrement the COW reference count for `phys_addr`. /// If the count reaches zero the the function returned true. #[cfg(all(target_arch = "x86_64", feature = "common-os"))] -pub fn frame_ref_dec_and_free(phys_addr: usize) -> bool { - let frame = phys_addr >> 12; +pub fn frame_ref_dec(phys_addr: PhysAddr) -> bool { + let frame = (phys_addr.as_u64() as usize) >> 12; let mut map = PAGE_REFCOUNTS.lock(); match map.get_mut(&frame) { None => { - warn!("frame_ref_dec_and_free: no refcount entry for frame {phys_addr:#x}"); + warn!("frame_ref_dec: no refcount entry for frame {phys_addr:p}"); false } Some(count) if *count <= 1 => { map.remove(&frame); - /*drop(map); // release lock before deallocating - let range = PageRange::new(phys_addr, phys_addr + free_list::PAGE_SIZE).unwrap(); - unsafe { FrameAlloc::deallocate(range) };*/ true } Some(count) => { @@ -91,6 +88,12 @@ impl PageRangeAllocator for FrameAlloc { } } +impl FrameAlloc { + pub fn free_space() -> usize { + PHYSICAL_FREE_LIST.lock().free_space() + } +} + impl fmt::Display for FrameAlloc { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let free_list = PHYSICAL_FREE_LIST.lock(); diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index 549e5c771a..f47485149e 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -685,7 +685,15 @@ impl PerCoreScheduler { fn cleanup_tasks(&mut self) { // Pop the first finished task and remove it from the TASKS list, which implicitly deallocates all associated memory. while let Some(finished_task) = self.finished_tasks.pop_front() { - debug!("Cleaning up task {}", finished_task.borrow().id); + let id = finished_task.borrow().id; + drop(finished_task); + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + trace!( + "Cleaned up task {id} — free frames: {} KiB", + crate::mm::FrameAlloc::free_space() >> 10 + ); + #[cfg(not(all(target_arch = "x86_64", feature = "common-os")))] + debug!("Cleaned up task {}", id); } } diff --git a/src/scheduler/task/mod.rs b/src/scheduler/task/mod.rs index 7ccf4f8f82..d8ae4d252c 100644 --- a/src/scheduler/task/mod.rs +++ b/src/scheduler/task/mod.rs @@ -556,11 +556,13 @@ impl Task { } } -/*impl Drop for Task { +impl Drop for Task { fn drop(&mut self) { - debug!("Drop task {}", self.id); + //debug!("Drop task {}", self.id); + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + arch::drop_user_space(self.root_page_table); } -}*/ +} struct BlockedTask { task: Rc>, diff --git a/src/syscalls/table.rs b/src/syscalls/table.rs index 018c23672b..0a55ab2f1a 100644 --- a/src/syscalls/table.rs +++ b/src/syscalls/table.rs @@ -38,7 +38,8 @@ const SYSNO_FORK: usize = 14; const SYSNO_WAITPID: usize = 15; /// number of the system call `spawn_process` const SYSNO_SPAWN_PROCESS: usize = 16; - +/// number of the system call `clock_gettime` +const SYSNO_CLOCK_GETTIME: usize = 17; /// Total number of system calls const NO_SYSCALLS: usize = 32; @@ -97,6 +98,7 @@ impl SyscallTable { table.handle[SYSNO_WAITPID] = sys_waitpid as *const _; } table.handle[SYSNO_SPAWN_PROCESS] = sys_spawn_process as *const _; + table.handle[SYSNO_CLOCK_GETTIME] = sys_clock_gettime as *const _; table } diff --git a/src/syscalls/tasks.rs b/src/syscalls/tasks.rs index f860eabbc3..e17c48392e 100644 --- a/src/syscalls/tasks.rs +++ b/src/syscalls/tasks.rs @@ -92,7 +92,11 @@ pub unsafe extern "C" fn sys_setprio(_id: *const Tid, _prio: i32) -> i32 { fn exit(arg: i32) -> ! { debug!("Exit program with error code {arg}!"); - super::shutdown(arg) + if cfg!(not(feature = "common-os")) { + super::shutdown(arg) + } else { + core_scheduler().exit(arg) + } } #[hermit_macro::system] From f30d53be9ef61aeb3f65de5b072e0f08c2735592 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sat, 11 Apr 2026 19:29:58 +0200 Subject: [PATCH 13/48] register remaining syscalls in dispatch table Extend SYSHANDLER_TABLE to 64 slots and register spawn, join, file metadata, condvar, task blocking and socket syscalls. Socket entries are gated behind `net`/`virtio-vsock`. Make the `addrinfo` submodule and its types `pub(crate)` so the table can take function pointers to `sys_getaddrinfo` and `sys_freeaddrinfo` without leaking private types. --- src/syscalls/socket/addrinfo.rs | 6 +- src/syscalls/socket/mod.rs | 2 +- src/syscalls/table.rs | 132 +++++++++++++++++++++++++++++++- 3 files changed, 135 insertions(+), 5 deletions(-) diff --git a/src/syscalls/socket/addrinfo.rs b/src/syscalls/socket/addrinfo.rs index 8403510c4a..f52f6da83a 100644 --- a/src/syscalls/socket/addrinfo.rs +++ b/src/syscalls/socket/addrinfo.rs @@ -13,7 +13,7 @@ use super::{Af, Ipproto, Sock, SockFlags, sockaddr, sockaddrBox, sockaddrRef, so #[repr(C)] #[derive(Default)] -struct addrinfo { +pub(crate) struct addrinfo { ai_flags: Ai, ai_family: i32, ai_socktype: i32, @@ -89,7 +89,7 @@ impl Drop for addrinfo { #[derive(Default)] #[repr(transparent)] -struct addrinfoList(Option>); +pub(crate) struct addrinfoList(Option>); impl addrinfoList { fn is_empty(&self) -> bool { @@ -132,7 +132,7 @@ impl FromIterator for addrinfoList { } } -struct addrinfoIter<'a>(Option<&'a addrinfo>); +pub(crate) struct addrinfoIter<'a>(Option<&'a addrinfo>); impl<'a> Iterator for addrinfoIter<'a> { type Item = &'a addrinfo; diff --git a/src/syscalls/socket/mod.rs b/src/syscalls/socket/mod.rs index 8faae6c68c..01c10456f5 100644 --- a/src/syscalls/socket/mod.rs +++ b/src/syscalls/socket/mod.rs @@ -1,7 +1,7 @@ #![allow(dead_code)] #![allow(nonstandard_style)] -mod addrinfo; +pub(crate) mod addrinfo; use alloc::boxed::Box; use alloc::sync::Arc; diff --git a/src/syscalls/table.rs b/src/syscalls/table.rs index 0a55ab2f1a..d62387487a 100644 --- a/src/syscalls/table.rs +++ b/src/syscalls/table.rs @@ -40,8 +40,96 @@ const SYSNO_WAITPID: usize = 15; const SYSNO_SPAWN_PROCESS: usize = 16; /// number of the system call `clock_gettime` const SYSNO_CLOCK_GETTIME: usize = 17; +/// number of the system call `spawn` +const SYSNO_SPAWN: usize = 18; +/// number of the system call `spawn2` +const SYSNO_SPAWN2: usize = 19; +/// number of the system call `join` +const SYSNO_JOIN: usize = 20; +/// number of the system call `unlink` +const SYSNO_UNLINK: usize = 21; +/// number of the system call `mkdir` +const SYSNO_MKDIR: usize = 22; +/// number of the system call `rmdir` +const SYSNO_RMDIR: usize = 23; +/// number of the system call `stat` +const SYSNO_STAT: usize = 24; +/// number of the system call `lstat` +const SYSNO_LSTAT: usize = 25; +/// number of the system call `fstat` +const SYSNO_FSTAT: usize = 26; +/// number of the system call `dup` +const SYSNO_DUP: usize = 27; +/// number of the system call `ioctl` +const SYSNO_IOCTL: usize = 28; +/// number of the system call `poll` +const SYSNO_POLL: usize = 29; +/// number of the system call `notify` +const SYSNO_NOTIFY: usize = 30; +/// number of the system call `add_queue` +const SYSNO_ADD_QUEUE: usize = 31; +/// number of the system call `wait` +const SYSNO_WAIT: usize = 32; +/// number of the system call `init_queue` +const SYSNO_INIT_QUEUE: usize = 33; +/// number of the system call `destroy_queue` +const SYSNO_DESTROY_QUEUE: usize = 34; +/// number of the system call `block_current_task` +const SYSNO_BLOCK_CURRENT_TASK: usize = 35; +/// number of the system call `block_current_task_with_timeout` +const SYSNO_BLOCK_CURRENT_TASK_WITH_TIMEOUT: usize = 36; +/// number of the system call `wakeup_task` +const SYSNO_WAKEUP_TASK: usize = 37; +/// number of the system call `socket` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_SOCKET: usize = 38; +/// number of the system call `bind` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_BIND: usize = 39; +/// number of the system call `listen` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_LISTEN: usize = 40; +/// number of the system call `accept` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_ACCEPT: usize = 41; +/// number of the system call `connect` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_CONNECT: usize = 42; +/// number of the system call `recv` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_RECV: usize = 43; +/// number of the system call `recvfrom` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_RECVFROM: usize = 44; +/// number of the system call `send` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_SEND: usize = 45; +/// number of the system call `sendto` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_SENDTO: usize = 46; +/// number of the system call `shutdown` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_SHUTDOWN: usize = 47; +/// number of the system call `getpeername` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_GETPEERNAME: usize = 48; +/// number of the system call `getsockname` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_GETSOCKNAME: usize = 49; +/// number of the system call `getsockopt` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_GETSOCKOPT: usize = 50; +/// number of the system call `setsockopt` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_SETSOCKOPT: usize = 51; +/// number of the system call `getaddrinfo` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_GETADDRINFO: usize = 52; +/// number of the system call `freeaddrinfo` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_FREEADDRINFO: usize = 53; /// Total number of system calls -const NO_SYSCALLS: usize = 32; +const NO_SYSCALLS: usize = 64; extern "C" fn invalid_syscall(sys_no: u64) -> ! { error!("Invalid syscall {sys_no}"); @@ -99,6 +187,48 @@ impl SyscallTable { } table.handle[SYSNO_SPAWN_PROCESS] = sys_spawn_process as *const _; table.handle[SYSNO_CLOCK_GETTIME] = sys_clock_gettime as *const _; + table.handle[SYSNO_SPAWN] = sys_spawn as *const _; + table.handle[SYSNO_SPAWN2] = sys_spawn2 as *const _; + table.handle[SYSNO_JOIN] = sys_join as *const _; + table.handle[SYSNO_UNLINK] = sys_unlink as *const _; + table.handle[SYSNO_MKDIR] = sys_mkdir as *const _; + table.handle[SYSNO_RMDIR] = sys_rmdir as *const _; + table.handle[SYSNO_STAT] = sys_stat as *const _; + table.handle[SYSNO_LSTAT] = sys_lstat as *const _; + table.handle[SYSNO_FSTAT] = sys_fstat as *const _; + table.handle[SYSNO_DUP] = sys_dup as *const _; + table.handle[SYSNO_IOCTL] = sys_ioctl as *const _; + table.handle[SYSNO_POLL] = sys_poll as *const _; + table.handle[SYSNO_NOTIFY] = sys_notify as *const _; + table.handle[SYSNO_ADD_QUEUE] = sys_add_queue as *const _; + table.handle[SYSNO_WAIT] = sys_wait as *const _; + table.handle[SYSNO_INIT_QUEUE] = sys_init_queue as *const _; + table.handle[SYSNO_DESTROY_QUEUE] = sys_destroy_queue as *const _; + table.handle[SYSNO_BLOCK_CURRENT_TASK] = sys_block_current_task as *const _; + table.handle[SYSNO_BLOCK_CURRENT_TASK_WITH_TIMEOUT] = + sys_block_current_task_with_timeout as *const _; + table.handle[SYSNO_WAKEUP_TASK] = sys_wakeup_task as *const _; + #[cfg(any(feature = "net", feature = "virtio-vsock"))] + { + table.handle[SYSNO_SOCKET] = crate::syscalls::socket::sys_socket as *const _; + table.handle[SYSNO_BIND] = crate::syscalls::socket::sys_bind as *const _; + table.handle[SYSNO_LISTEN] = crate::syscalls::socket::sys_listen as *const _; + table.handle[SYSNO_ACCEPT] = crate::syscalls::socket::sys_accept as *const _; + table.handle[SYSNO_CONNECT] = crate::syscalls::socket::sys_connect as *const _; + table.handle[SYSNO_RECV] = crate::syscalls::socket::sys_recv as *const _; + table.handle[SYSNO_RECVFROM] = crate::syscalls::socket::sys_recvfrom as *const _; + table.handle[SYSNO_SEND] = crate::syscalls::socket::sys_send as *const _; + table.handle[SYSNO_SENDTO] = crate::syscalls::socket::sys_sendto as *const _; + table.handle[SYSNO_SHUTDOWN] = crate::syscalls::socket::sys_shutdown as *const _; + table.handle[SYSNO_GETPEERNAME] = crate::syscalls::socket::sys_getpeername as *const _; + table.handle[SYSNO_GETSOCKNAME] = crate::syscalls::socket::sys_getsockname as *const _; + table.handle[SYSNO_GETSOCKOPT] = crate::syscalls::socket::sys_getsockopt as *const _; + table.handle[SYSNO_SETSOCKOPT] = crate::syscalls::socket::sys_setsockopt as *const _; + table.handle[SYSNO_GETADDRINFO] = + crate::syscalls::socket::addrinfo::sys_getaddrinfo as *const _; + table.handle[SYSNO_FREEADDRINFO] = + crate::syscalls::socket::addrinfo::sys_freeaddrinfo as *const _; + } table } From 1ab5b803ccb88dc5689fb8e65c2de9596fe59f34 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sun, 12 Apr 2026 08:44:59 +0200 Subject: [PATCH 14/48] Add user-space thread spawning with per-thread TLS for common-os Implement std::thread::spawn support when running as a common OS. Each spawned thread gets its own private TLS region, initialized from a pristine PT_TLS template extracted directly from the ELF binary. Threads share the parent's address space via Arc, heap, and file descriptor table, but have independent kernel/user stacks and TLS blocks. Key changes: - Add TlsTemplate struct holding the ELF's PT_TLS init image - Add RootPageTable wrapper with Arc-based lifetime management - Add allocate_thread_tls() to map fresh per-thread TLS pages - Add task_start_user() naked trampoline for ring-3 entry via iretq - Route sys_spawn/sys_spawn2 through spawn_thread() under common-os - Handle user thread exit via page fault at RIP=0 (clean exit path) - Register available_parallelism and getdents64 syscalls --- src/arch/x86_64/kernel/mod.rs | 30 +++- src/arch/x86_64/kernel/scheduler.rs | 83 +++++++++- src/arch/x86_64/mm/mod.rs | 6 +- src/arch/x86_64/mm/paging.rs | 24 +++ src/scheduler/mod.rs | 240 +++++++++++++++++++++++++++- src/scheduler/task/mod.rs | 123 +++++++++++++- src/syscalls/table.rs | 7 + src/syscalls/tasks.rs | 25 ++- 8 files changed, 517 insertions(+), 21 deletions(-) diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index fbfd81f253..0afe5ca9ee 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -194,9 +194,12 @@ const LOADER_START: usize = 0x0100_0000_0000; const LOADER_STACK_SIZE: usize = 0x8000; #[cfg(feature = "common-os")] -pub fn load_application(code_size: u64, tls_size: u64, func: F) -> T +pub fn load_application(code_size: u64, tls_size: u64, func: F) -> Result<(), ()> where - F: FnOnce(&'static mut [u8], Option<&'static mut [u8]>) -> T, + F: FnOnce( + &'static mut [u8], + Option<&'static mut [u8]>, + ) -> Result>, ()>, { use align_address::Align; use free_list::PageLayout; @@ -293,9 +296,28 @@ where } processor::writefs(thread_ptr.expose_provenance()); - func(code_slice, Some(block)) + // Run the ELF loader, which copies the binary's `PT_TLS` initial + // image into `block` and returns the pristine PT_TLS image + // directly from the ELF buffer. + let tls_init = func(code_slice, Some(block))?; + + if let Some(init) = tls_init { + let template = alloc::sync::Arc::new( + crate::scheduler::task::TlsTemplate { + size: tls_offset, + init, + }, + ); + core_scheduler() + .get_current_task() + .borrow_mut() + .tls_template = Some(template); + } + + Ok(()) } else { - func(code_slice, None) + func(code_slice, None)?; + Ok(()) } } diff --git a/src/arch/x86_64/kernel/scheduler.rs b/src/arch/x86_64/kernel/scheduler.rs index 53a450ec2b..6797a00eb5 100644 --- a/src/arch/x86_64/kernel/scheduler.rs +++ b/src/arch/x86_64/kernel/scheduler.rs @@ -129,9 +129,15 @@ impl TaskStacks { flags, ); - // map user stack into the address space (must be user-accessible) - let mut user_flags = PageTableEntryFlags::empty(); - user_flags.normal().writable().user().execute_disable(); + // map user stack into the address space (must be user-accessible under common-os) + #[cfg(feature = "common-os")] + let user_flags = { + let mut f = PageTableEntryFlags::empty(); + f.normal().writable().user().execute_disable(); + f + }; + #[cfg(not(feature = "common-os"))] + let user_flags = flags; crate::arch::mm::paging::map::( virt_addr + IST_SIZE + DEFAULT_STACK_SIZE + 3 * BasePageSize::SIZE, phys_addr + IST_SIZE + DEFAULT_STACK_SIZE, @@ -289,6 +295,77 @@ extern "C" fn task_entry(func: extern "C" fn(usize), arg: usize) -> ! { core_scheduler().exit(0) } +/// Entry trampoline for a new user-space thread (common-os). +/// +/// Called in kernel mode the first time the task is scheduled. It crafts +/// an `iretq` frame that transfers control to `func(arg)` in ring 3 on +/// the new user stack. Matches the descriptor layout used by +/// `jump_to_user_land`: user CS=0x2b, user SS=0x23, RFLAGS=0x1202. +#[cfg(feature = "common-os")] +#[unsafe(naked)] +extern "C" fn task_start_user( + _f: extern "C" fn(usize), + _arg: usize, + _user_stack: u64, +) -> ! { + // rdi = func (user entry) + // rsi = arg (forwarded as first argument to `func`) + // rdx = user stack pointer + naked_asm!( + "swapgs", + "push 0x23", // user SS + "push rdx", // user RSP + "push 0x1202", // RFLAGS (IF=1) + "push 0x2b", // user CS + "push rdi", // RIP = user func + "mov rdi, rsi", // first argument for user func + "iretq", + ) +} + +#[cfg(feature = "common-os")] +impl Task { + /// Build the initial kernel-stack context for a new user-space thread. + /// + /// When this task is first scheduled, `switch_to_task` restores the + /// saved registers and `ret`s into `task_start_user`, which then + /// `iretq`s to `func(arg)` in ring 3 on the freshly allocated user + /// stack. + pub(crate) fn create_user_stack_frame( + &mut self, + func: unsafe extern "C" fn(usize), + arg: usize, + tls_fs_base: u64, + ) { + unsafe { + // Debug marker at the very top of the kernel stack. + let mut stack = self.stacks.get_kernel_stack() + + self.stacks.get_kernel_stack_size() + - TaskStacks::MARKER_SIZE; + *stack.as_mut_ptr::() = 0xdead_beefu64; + + stack -= mem::size_of::(); + let state = stack.as_mut_ptr::(); + state.cast::().write_bytes(0, mem::size_of::()); + + (*state).rip = task_start_user; + (*state).rdi = func as usize as u64; + (*state).rsi = arg as u64; + (*state).rflags = 0x1202u64; + // FS.Base for the new user-space thread (its private TLS area). + (*state).fs = tls_fs_base; + + self.last_stack_pointer = stack; + self.user_stack_pointer = self.stacks.get_user_stack() + + self.stacks.get_user_stack_size() + - TaskStacks::MARKER_SIZE; + + // rdx is used by task_start_user as the new user-mode RSP + (*state).rdx = self.user_stack_pointer.as_u64() - mem::size_of::() as u64; + } + } +} + impl TaskFrame for Task { fn create_stack_frame(&mut self, func: unsafe extern "C" fn(usize), arg: usize) { // Check if TLS is allocated already and if the task uses thread-local storage. diff --git a/src/arch/x86_64/mm/mod.rs b/src/arch/x86_64/mm/mod.rs index 7750a3c281..9aa9d25f56 100644 --- a/src/arch/x86_64/mm/mod.rs +++ b/src/arch/x86_64/mm/mod.rs @@ -65,7 +65,11 @@ pub fn create_new_root_page_table() -> usize { #[cfg(feature = "common-os")] pub fn get_current_root_page_table() -> usize { use crate::arch::core_local::core_scheduler; - core_scheduler().get_current_task().borrow().root_page_table + core_scheduler() + .get_current_task() + .borrow() + .root_page_table + .as_usize() } /// Copy the current task's PML4 into a new page table, sharing data pages (COW). diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index 5d9da5d2b8..3abfd09b49 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -1,6 +1,7 @@ use core::{fmt, ptr}; use free_list::PageLayout; +#[cfg(feature = "common-os")] use x86_64::instructions::tlb; use x86_64::registers::control::{Cr0, Cr0Flags, Cr2, Cr3}; #[cfg(feature = "common-os")] @@ -566,6 +567,29 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( } } + // A spawned user thread whose entry function returns comes back here + // with RIP=0: the `thread_start` wrapper in std's hermit pal just + // returns, and the kernel-crafted return slot at the top of the user + // stack was zero-initialised. Treat an instruction-fetch fault at 0 + // in ring 3 as a clean thread exit. + if error_code.contains(PageFaultErrorCode::USER_MODE) + && error_code.contains(PageFaultErrorCode::INSTRUCTION_FETCH) + && faulting_addr.as_u64() == 0 + { + debug!( + "User thread {} returned from entry; exiting cleanly.", + core_scheduler().get_current_task_id() + ); + // Do not swap GS back here. `exit` will context-switch to another + // task, so the current kernel GS must remain in place for the + // scheduler bookkeeping. `swapped_gs` is a no-op at this point: + // the handler never returns, and the next task's GS is restored + // by the context switch. + let _ = swapped_gs; + use crate::scheduler::PerCoreSchedulerExt; + core_scheduler().exit(0); + } + // Heap demand-paging: user-mode fault on an unmapped heap page. if error_code.contains(PageFaultErrorCode::USER_MODE) { let heap = core_scheduler().get_current_task().borrow().heap.clone(); diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index f47485149e..62047b1981 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -206,6 +206,64 @@ impl PerCoreSchedulerExt for &mut PerCoreScheduler { } } +/// Allocate and initialize a private TLS region for a new user-space thread. +/// +/// Maps a fresh user-accessible page range through the currently active +/// (shared) root page table, copies the per-process `TlsTemplate` into it, +/// sets up the trailing 8-byte TCB self-pointer, and returns the new +/// FS.Base value (the thread pointer) for the child thread. +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +fn allocate_thread_tls(template: &task::TlsTemplate) -> u64 { + use align_address::Align; + use free_list::PageLayout; + use memory_addresses::{PhysAddr, VirtAddr}; + use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; + + use crate::arch::x86_64::mm::paging::{self, PageTableEntryFlags, PageTableEntryFlagsExt}; + use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator, frame_ref_inc}; + + let tcb_size = core::mem::size_of::<*mut ()>(); + let total = (template.size + tcb_size).align_up(BasePageSize::SIZE as usize); + + let virt_layout = PageLayout::from_size(total).unwrap(); + let virt_range = PageAlloc::allocate(virt_layout).unwrap(); + let virt_addr = VirtAddr::from(virt_range.start()); + + let frame_layout = PageLayout::from_size(total).unwrap(); + let frame_range = FrameAlloc::allocate(frame_layout).unwrap(); + let phys_addr = PhysAddr::from(frame_range.start()); + for i in 0..total / BasePageSize::SIZE as usize { + frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); + } + + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user().execute_disable(); + paging::map::( + virt_addr, + phys_addr, + total / BasePageSize::SIZE as usize, + flags, + ); + + unsafe { + // Copy the pristine PT_TLS image into the new block. + virt_addr + .as_mut_ptr::() + .copy_from_nonoverlapping(template.init.as_ptr(), template.init.len()); + // Zero the rest of the TLS BSS area and the trailing TCB. + virt_addr + .as_mut_ptr::() + .add(template.init.len()) + .write_bytes(0, total - template.init.len()); + // Variant II (x86_64): the thread pointer is at the start of the TCB + // (right after the TLS data block) and stores its own address. + let thread_ptr = virt_addr.as_u64() + template.size as u64; + let tcb_ptr: *mut u64 = core::ptr::with_exposed_provenance_mut(thread_ptr as usize); + tcb_ptr.write(thread_ptr); + thread_ptr + } +} + struct NewTask { tid: TaskId, func: unsafe extern "C" fn(usize), @@ -214,6 +272,22 @@ struct NewTask { core_id: CoreId, stacks: TaskStacks, object_map: Arc>, RandomState>>>, + /// When `Some`, the new task is a user-space thread that shares the + /// given root page table (and heap) with its parent process. When + /// `None`, the task is a regular kernel-mode task with a fresh + /// address space. + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + thread_of: Option<(Arc, Arc)>, + /// Per-process TLS template, cloned from the spawning thread. Used by + /// `From` to propagate the template into the new task so that + /// any threads it spawns in turn can allocate their own TLS regions. + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + tls_template: Option>, + /// FS.Base of the new user thread, already prepared by `spawn_thread` + /// (a freshly allocated user-space TLS area initialised from the + /// per-process TLS template). Zero means "do not touch FS". + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + tls_fs_base: u64, } impl From for Task { @@ -226,7 +300,31 @@ impl From for Task { core_id, stacks, object_map, + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + thread_of, + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + tls_template, + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + tls_fs_base, } = value; + + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + if let Some((root_page_table, heap)) = thread_of { + let mut task = Self::new_thread( + tid, + core_id, + TaskStatus::Ready, + prio, + stacks, + object_map, + root_page_table, + heap, + tls_template, + ); + task.create_user_stack_frame(func, arg, tls_fs_base); + return task; + } + let mut task = Self::new(tid, core_id, TaskStatus::Ready, prio, stacks, object_map); task.create_stack_frame(func, arg); task @@ -253,6 +351,12 @@ impl PerCoreScheduler { core_id, stacks, object_map: core_scheduler().get_current_task_object_map(), + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + thread_of: None, + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + tls_template: None, + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + tls_fs_base: 0, }; // Add it to the task lists. @@ -299,6 +403,105 @@ impl PerCoreScheduler { tid } + /// Spawn a new user-space thread that shares the current task's + /// address space (root page table) and heap. + /// + /// `func` must be a valid ring-3 entry point mapped in the shared + /// address space. The new thread receives its own kernel/interrupt + /// stacks and a fresh user stack, all mapped into the shared PT via + /// the regular `TaskStacks::new` path. + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + pub unsafe fn spawn_thread( + func: unsafe extern "C" fn(usize), + arg: usize, + prio: Priority, + core_id: CoreId, + stack_size: usize, + ) -> TaskId { + let tid = get_tid(); + // TaskStacks::new maps into the current address space — which *is* + // the shared PT of the calling thread — so the new stacks are + // immediately visible to every thread in this process. + let stacks = TaskStacks::new(stack_size); + + let (root_page_table, heap, object_map, tls_template) = { + let current = core_scheduler().get_current_task(); + let borrowed = current.borrow(); + ( + borrowed.root_page_table.clone(), + borrowed.heap.clone(), + borrowed.object_map.clone(), + borrowed.tls_template.clone(), + ) + }; + + // Allocate a private TLS region for the new thread from the pristine + // per-process TLS template captured at `load_application` time. Must + // run in the parent's address space (still active here in the syscall + // path) so that the new user-accessible pages get mapped into the + // shared root page table. + let tls_fs_base = if let Some(ref template) = tls_template { + crate::scheduler::allocate_thread_tls(template) + } else { + 0 + }; + + let new_task = NewTask { + tid, + func, + arg, + prio, + core_id, + stacks, + object_map, + thread_of: Some((root_page_table, heap)), + tls_template, + tls_fs_base, + }; + + let wakeup = { + #[cfg(feature = "smp")] + let mut input_locked = get_scheduler_input(core_id).lock(); + WAITING_TASKS.lock().insert(tid, VecDeque::with_capacity(1)); + TASKS.lock().insert( + tid, + TaskHandle::new( + tid, + prio, + #[cfg(feature = "smp")] + core_id, + ), + ); + NO_TASKS.fetch_add(1, Ordering::SeqCst); + + #[cfg(feature = "smp")] + if core_id == core_scheduler().core_id { + let task = Rc::new(RefCell::new(Task::from(new_task))); + core_scheduler().ready_queue.push(task); + false + } else { + input_locked.new_tasks.push_back(new_task); + true + } + #[cfg(not(feature = "smp"))] + if core_id == 0 { + let task = Rc::new(RefCell::new(Task::from(new_task))); + core_scheduler().ready_queue.push(task); + false + } else { + panic!("Invalid core_id {core_id}!") + } + }; + + debug!("Creating user thread {tid} with priority {prio} on core {core_id}"); + + if wakeup { + arch::wakeup_core(core_id); + } + + tid + } + #[cfg(feature = "newlib")] fn clone_impl(&self, func: extern "C" fn(usize), arg: usize) -> TaskId { static NEXT_CORE_ID: AtomicU32 = AtomicU32::new(1); @@ -330,6 +533,8 @@ impl PerCoreScheduler { core_id, stacks: TaskStacks::new(current_task_borrowed.stacks.get_user_stack_size()), object_map: current_task_borrowed.object_map.clone(), + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + thread_of: None, }; // Add it to the task lists. @@ -942,6 +1147,29 @@ pub unsafe fn spawn( unsafe { PerCoreScheduler::spawn(func, arg, prio, core_id, stack_size) } } +/// Spawn a user-space thread that shares the current task's address space. +/// +/// Used by `sys_spawn`/`sys_spawn2` under the `common-os` feature to +/// implement POSIX-style threads: the entry point `func` lives in user +/// space and the new thread executes in ring 3 against the parent +/// process's root page table. +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +pub unsafe fn spawn_thread( + func: unsafe extern "C" fn(usize), + arg: usize, + prio: Priority, + stack_size: usize, + selector: isize, +) -> TaskId { + let core_id = if selector < 0 { + SPAWN_COUNTER.fetch_add(1, Ordering::SeqCst) % get_processor_count() + } else { + selector as u32 + }; + + unsafe { PerCoreScheduler::spawn_thread(func, arg, prio, core_id, stack_size) } +} + #[allow(clippy::result_unit_err)] pub fn join(id: TaskId) -> Result<(), ()> { let core_scheduler = core_scheduler(); @@ -1032,6 +1260,11 @@ pub unsafe fn fork() -> TaskId { parent_object_map.write().insert(*key, val.clone()); } let parent_heap = core_scheduler().get_current_task().borrow().heap.clone(); + let parent_tls_template = core_scheduler() + .get_current_task() + .borrow() + .tls_template + .clone(); let child_task = Task::new_fork( tid, @@ -1042,8 +1275,11 @@ pub unsafe fn fork() -> TaskId { child_last_sp, parent_user_sp, parent_object_map, - child_root_page_table, + Arc::new(crate::scheduler::task::RootPageTable::new( + child_root_page_table, + )), parent_heap, + parent_tls_template, ); let wakeup = { @@ -1103,5 +1339,5 @@ pub(crate) static BOOT_ROOT_PAGE_TABLE: OnceCell = OnceCell::new(); #[cfg(all(target_arch = "x86_64", feature = "common-os"))] pub(crate) fn get_root_page_table() -> usize { let current_task_borrowed = core_scheduler().current_task.borrow_mut(); - current_task_borrowed.root_page_table + current_task_borrowed.root_page_table.as_usize() } diff --git a/src/scheduler/task/mod.rs b/src/scheduler/task/mod.rs index d8ae4d252c..9dd677d7ca 100644 --- a/src/scheduler/task/mod.rs +++ b/src/scheduler/task/mod.rs @@ -6,6 +6,8 @@ pub(crate) mod tls; use alloc::collections::{LinkedList, VecDeque}; use alloc::rc::Rc; use alloc::sync::Arc; +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +use alloc::vec::Vec; use core::cell::RefCell; use core::num::NonZeroU64; use core::{cmp, fmt}; @@ -32,6 +34,51 @@ use crate::fd::{Fd, RawFd, stdio}; use crate::fd::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; use crate::scheduler::CoreId; +/// A reference-counted handle to a process's root page table. +/// +/// Threads of the same process share the same `Arc`. When +/// the last owning task is dropped, the physical page-table hierarchy and +/// the user-space mappings are released. +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +pub struct RootPageTable { + pml4_phys: usize, + /// `false` for the boot page table, which must never be released. + owned: bool, +} + +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +impl RootPageTable { + /// Wraps a freshly allocated PML4 that this process owns. + pub fn new(pml4_phys: usize) -> Self { + Self { + pml4_phys, + owned: true, + } + } + + /// Wraps the boot page table, shared by all idle tasks. Dropping this + /// instance is a no-op. + pub fn new_boot(pml4_phys: usize) -> Self { + Self { + pml4_phys, + owned: false, + } + } + + pub fn as_usize(&self) -> usize { + self.pml4_phys + } +} + +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +impl Drop for RootPageTable { + fn drop(&mut self) { + if self.owned { + arch::drop_user_space(self.pml4_phys); + } + } +} + /// Returns the most significant bit. /// /// # Examples @@ -362,6 +409,21 @@ impl PriorityTaskQueue { } } +/// Per-process TLS template used to initialise newly spawned threads. +/// +/// `size` is the byte size of the TLS data block (the offset at which the +/// TCB / thread pointer lives, also the value that the parent's main thread +/// uses for its own block). `init` is a snapshot of the freshly-loaded TLS +/// image taken right after the user binary's `PT_TLS` segment was copied +/// into place by `load_application`. Each new thread starts with a verbatim +/// copy of `init`, so it sees pristine `#[thread_local]` defaults rather +/// than mutated state from another thread. +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +pub(crate) struct TlsTemplate { + pub size: usize, + pub init: Vec, +} + /// Tracks the heap region of a user process for demand-paging. #[cfg(all(target_arch = "x86_64", feature = "common-os"))] pub(crate) struct Heap { @@ -414,12 +476,19 @@ pub(crate) struct Task { /// Task Thread-Local-Storage (TLS) #[cfg(not(feature = "common-os"))] pub tls: Option, - // Physical address of the 1st level page table + // Physical address of the 1st level page table, shared between all + // threads of the same process via `Arc`. The address space is freed when + // the last thread referencing this `RootPageTable` is dropped. #[cfg(all(target_arch = "x86_64", feature = "common-os"))] - pub root_page_table: usize, + pub root_page_table: Arc, /// Heap region tracked for demand-paging #[cfg(all(target_arch = "x86_64", feature = "common-os"))] pub heap: Arc, + /// Per-process TLS template used to allocate fresh TLS regions for new + /// threads. `None` for kernel-only tasks; set by `load_application` + /// when the user binary has a `PT_TLS` segment. + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + pub tls_template: Option>, } pub(crate) trait TaskFrame { @@ -451,9 +520,11 @@ impl Task { #[cfg(not(feature = "common-os"))] tls: None, #[cfg(all(target_arch = "x86_64", feature = "common-os"))] - root_page_table: arch::create_new_root_page_table(), + root_page_table: Arc::new(RootPageTable::new(arch::create_new_root_page_table())), #[cfg(all(target_arch = "x86_64", feature = "common-os"))] heap: Arc::new(Heap::new_empty()), + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + tls_template: None, } } @@ -515,9 +586,47 @@ impl Task { #[cfg(not(feature = "common-os"))] tls, #[cfg(all(target_arch = "x86_64", feature = "common-os"))] - root_page_table: *crate::scheduler::BOOT_ROOT_PAGE_TABLE.get().unwrap(), + root_page_table: Arc::new(RootPageTable::new_boot( + *crate::scheduler::BOOT_ROOT_PAGE_TABLE.get().unwrap(), + )), #[cfg(all(target_arch = "x86_64", feature = "common-os"))] heap: Arc::new(Heap::new_empty()), + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + tls_template: None, + } + } + + /// Create a new user-space thread that shares its parent's address space. + /// + /// The `root_page_table` `Arc` is cloned from the parent, so all threads + /// of the same process drop together when the last one exits. + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[allow(clippy::too_many_arguments)] + pub fn new_thread( + tid: TaskId, + core_id: CoreId, + task_status: TaskStatus, + task_prio: Priority, + stacks: TaskStacks, + object_map: Arc>, RandomState>>>, + root_page_table: Arc, + heap: Arc, + tls_template: Option>, + ) -> Task { + debug!("Creating user thread {tid} on core {core_id}"); + Task { + id: tid, + status: task_status, + prio: task_prio, + last_stack_pointer: VirtAddr::zero(), + user_stack_pointer: VirtAddr::zero(), + last_fpu_state: arch::processor::FPUState::new(), + core_id, + stacks, + object_map, + root_page_table, + heap, + tls_template, } } @@ -534,8 +643,9 @@ impl Task { last_stack_pointer: VirtAddr, user_stack_pointer: VirtAddr, object_map: Arc>, RandomState>>>, - root_page_table: usize, + root_page_table: Arc, parent_heap: Arc, + tls_template: Option>, ) -> Task { debug!("Creating forked task {tid} on core {core_id}"); Task { @@ -552,6 +662,7 @@ impl Task { tls: None, root_page_table, heap: parent_heap, + tls_template, } } } @@ -559,8 +670,6 @@ impl Task { impl Drop for Task { fn drop(&mut self) { //debug!("Drop task {}", self.id); - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] - arch::drop_user_space(self.root_page_table); } } diff --git a/src/syscalls/table.rs b/src/syscalls/table.rs index d62387487a..08df953bda 100644 --- a/src/syscalls/table.rs +++ b/src/syscalls/table.rs @@ -128,6 +128,11 @@ const SYSNO_GETADDRINFO: usize = 52; /// number of the system call `freeaddrinfo` #[cfg(any(feature = "net", feature = "virtio-vsock"))] const SYSNO_FREEADDRINFO: usize = 53; +/// number of the system call `availale_parallelism` +const SYSNO_AVAILABLE_PARALLELISM: usize = 54; +/// number of the system call `getdents64` +const SYSNO_GET_DENTS64: usize = 55; + /// Total number of system calls const NO_SYSCALLS: usize = 64; @@ -229,6 +234,8 @@ impl SyscallTable { table.handle[SYSNO_FREEADDRINFO] = crate::syscalls::socket::addrinfo::sys_freeaddrinfo as *const _; } + table.handle[SYSNO_AVAILABLE_PARALLELISM] = sys_available_parallelism as *const _; + table.handle[SYSNO_GET_DENTS64] = sys_getdents64 as *const _; table } diff --git a/src/syscalls/tasks.rs b/src/syscalls/tasks.rs index e17c48392e..4086305435 100644 --- a/src/syscalls/tasks.rs +++ b/src/syscalls/tasks.rs @@ -50,7 +50,7 @@ pub extern "C" fn sys_waitpid(pid: Tid) -> i32 { #[cfg(not(feature = "common-os"))] #[hermit_macro::system(errno)] #[unsafe(no_mangle)] -pub extern "C" fn sys_waitpid(pid: Tid) -> i32 { +pub extern "C" fn sys_waitpid(_pid: Tid) -> i32 { -i32::from(Errno::Nosys) } @@ -216,7 +216,16 @@ pub unsafe extern "C" fn sys_spawn2( stack_size: usize, selector: isize, ) -> Tid { - unsafe { scheduler::spawn(func, arg, Priority::from(prio), stack_size, selector).into() } + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + { + unsafe { + scheduler::spawn_thread(func, arg, Priority::from(prio), stack_size, selector).into() + } + } + #[cfg(not(all(target_arch = "x86_64", feature = "common-os")))] + { + unsafe { scheduler::spawn(func, arg, Priority::from(prio), stack_size, selector).into() } + } } #[hermit_macro::system] @@ -228,8 +237,16 @@ pub unsafe extern "C" fn sys_spawn( prio: u8, selector: isize, ) -> i32 { - let new_id = unsafe { - scheduler::spawn(func, arg, Priority::from(prio), USER_STACK_SIZE, selector).into() + let new_id = { + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + unsafe { + scheduler::spawn_thread(func, arg, Priority::from(prio), USER_STACK_SIZE, selector) + .into() + } + #[cfg(not(all(target_arch = "x86_64", feature = "common-os")))] + unsafe { + scheduler::spawn(func, arg, Priority::from(prio), USER_STACK_SIZE, selector).into() + } }; if !id.is_null() { From 562c663c78de9451344adb330fd0c87f31ea80d5 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sun, 12 Apr 2026 23:03:51 +0200 Subject: [PATCH 15/48] Add sys_exec syscall and clear_user_space for process image replacement Split drop_user_space into a reusable clear_pml4 helper that tears down user-space page tables without freeing the PML4 itself. clear_user_space uses it to wipe the current address space in-place and flush TLBs, enabling exec-style process replacement. drop_user_space continues to free the PML4 frame after clearing. Also register the new sys_exec syscall (number 56) in the dispatch table with a weak default stub. --- src/arch/x86_64/mm/mod.rs | 2 +- src/arch/x86_64/mm/paging.rs | 28 ++++++++++++++++++++++++---- src/syscalls/table.rs | 12 +++++++++++- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src/arch/x86_64/mm/mod.rs b/src/arch/x86_64/mm/mod.rs index 9aa9d25f56..211bb5d7fe 100644 --- a/src/arch/x86_64/mm/mod.rs +++ b/src/arch/x86_64/mm/mod.rs @@ -12,7 +12,7 @@ use crate::arch::mm::paging::{PageTableEntryFlags, PageTableEntryFlagsExt}; use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; /// Copy the kernel stack pages of the current task to a new base address. #[cfg(feature = "common-os")] -pub use paging::{drop_user_space, copy_kernel_stack_to}; +pub use paging::{drop_user_space, clear_user_space, copy_kernel_stack_to}; #[cfg(feature = "common-os")] pub fn create_new_root_page_table() -> usize { diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index 3abfd09b49..d5ea656d5d 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -354,9 +354,7 @@ pub fn mark_user_pages_copy_on_write() { } #[cfg(feature = "common-os")] -pub fn drop_user_space(pml4_phys: usize) { - debug!("Drop the user space at PML4 {pml4_phys:#x}"); - +fn clear_pml4(pml4_phys: usize) { // Free a 4 KiB physical frame back to the frame allocator. fn free_frame(phys: usize) { let range = free_list::PageRange::new(phys, phys + free_list::PAGE_SIZE).unwrap(); @@ -428,10 +426,32 @@ pub fn drop_user_space(pml4_phys: usize) { free_frame(pdpt_phys); pml4_entry.set_unused(); } +} + +#[cfg(feature = "common-os")] +pub fn drop_user_space(pml4_phys: usize) { + debug!("Drop the user space at PML4 {pml4_phys:#x}"); + + clear_pml4(pml4_phys); // Finally free the PML4 itself. No TLB flush needed: the dropped page // table is not loaded on any core. - free_frame(pml4_phys); + let range = free_list::PageRange::new(pml4_phys, pml4_phys + free_list::PAGE_SIZE).unwrap(); + unsafe { FrameAlloc::deallocate(range) }; +} + +#[cfg(feature = "common-os")] +pub fn clear_user_space() { + let (p4_frame, _) = Cr3::read_raw(); + let pml4_phys = p4_frame.start_address().as_u64() as usize; + + debug!("Clear the user space at PML4 {pml4_phys:#x}"); + + clear_pml4(pml4_phys); + + tlb::flush_all(); + #[cfg(feature = "smp")] + crate::arch::x86_64::kernel::apic::ipi_tlb_flush(); } /// Copy the contents of the current task's kernel stack to a new virtual diff --git a/src/syscalls/table.rs b/src/syscalls/table.rs index 08df953bda..833278f0cc 100644 --- a/src/syscalls/table.rs +++ b/src/syscalls/table.rs @@ -132,6 +132,8 @@ const SYSNO_FREEADDRINFO: usize = 53; const SYSNO_AVAILABLE_PARALLELISM: usize = 54; /// number of the system call `getdents64` const SYSNO_GET_DENTS64: usize = 55; +/// number of the system call `exec` +const SYSNO_EXEC: usize = 56; /// Total number of system calls const NO_SYSCALLS: usize = 64; @@ -144,7 +146,14 @@ extern "C" fn invalid_syscall(sys_no: u64) -> ! { /// loader will replace this function #[linkage = "weak"] #[unsafe(no_mangle)] -pub extern "C" fn sys_spawn_process(_name: *const core::ffi::c_char) -> i32 { +pub extern "C" fn sys_spawn_process(_path: *const core::ffi::c_char) -> i32 { + -i32::from(Errno::Nosys) +} + +/// loader will replace this function +#[linkage = "weak"] +#[unsafe(no_mangle)] +pub extern "C" fn sys_exec(_path: *const c_char) -> i32 { -i32::from(Errno::Nosys) } @@ -236,6 +245,7 @@ impl SyscallTable { } table.handle[SYSNO_AVAILABLE_PARALLELISM] = sys_available_parallelism as *const _; table.handle[SYSNO_GET_DENTS64] = sys_getdents64 as *const _; + table.handle[SYSNO_EXEC] = sys_exec as *const _; table } From 2d0265073d4d308c8494890105a032d1bb0b2a63 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sun, 3 May 2026 09:03:58 +0200 Subject: [PATCH 16/48] x86_64: clear registers before jumping into the user-space Avoids leaking kernel-state register values to ring 3; the user entry sees only rdi (argc) and rsi (argv). --- src/arch/x86_64/kernel/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index 0afe5ca9ee..a1a2960a1c 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -370,6 +370,11 @@ pub unsafe fn jump_to_user_land(entry_point: usize, code_size: usize, arg: &[&st "push {5}", "mov rdi, {6}", "mov rsi, {7}", + // Clear registers so that state cannot leak. + "xor rax, rax", "xor rbx, rbx", "xor rcx, rcx", "xor rdx, rdx", + "xor r8, r8", "xor r9, r9", "xor r10, r10", + "xor r11, r11", "xor r12, r12", "xor r13, r13", + "xor r14, r14", "xor r15, r15", "iretq", const u64::MAX - (TaskStacks::MARKER_SIZE as u64 - 1), const 0x23usize, From 235262783c5ef3171c9e834552ee04584a927ee7 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sun, 3 May 2026 20:56:35 +0200 Subject: [PATCH 17/48] fix(aarch64): add support od macOS HVF on Apple Silicon This series makes Hermit's aarch64 common-os kernel boot under macOS HVF on Apple Silicon, in addition to the existing TCG support. Three small adjustments are needed: virtual timer instead of physical, PSCI shutdown instead of semihosting, and FEAT_PAN-aware EL1 setup. --- src/arch/aarch64/kernel/interrupts.rs | 181 +++++++++++++++++++++----- src/arch/aarch64/kernel/processor.rs | 70 ++++++---- src/arch/aarch64/kernel/start.rs | 33 +++++ 3 files changed, 225 insertions(+), 59 deletions(-) diff --git a/src/arch/aarch64/kernel/interrupts.rs b/src/arch/aarch64/kernel/interrupts.rs index 0717b2222f..40142708d5 100644 --- a/src/arch/aarch64/kernel/interrupts.rs +++ b/src/arch/aarch64/kernel/interrupts.rs @@ -30,6 +30,14 @@ const PPI_START: u8 = 16; const SPI_START: u8 = 32; /// Software-generated interrupt for rescheduling pub(crate) const SGI_RESCHED: u8 = 1; +/// Synthetic IRQ slot used for page-fault accounting. The number does not +/// correspond to any GIC interrupt — it is purely a bookkeeping ID for +/// `IrqStatistics` so the page-fault count shows up in `print_statistics` +/// alongside the real interrupts. Picked at 14 to mirror the x86_64 +/// CPU exception vector for #PF, which keeps the two architectures +/// consistent in the diagnostic output. +#[cfg(feature = "common-os")] +pub(crate) const PAGE_FAULT_IRQ: u8 = 14; /// Number of the timer interrupt static mut TIMER_INTERRUPT: u32 = 0; @@ -170,43 +178,71 @@ pub(crate) extern "C" fn do_irq(_state: &State) -> *mut usize { } #[unsafe(no_mangle)] -pub(crate) extern "C" fn do_sync(state: &State) { +pub(crate) extern "C" fn do_sync(state: &mut State) { let esr = ESR_EL1.get(); let ec_raw = ESR_EL1.read(ESR_EL1::EC); let ec: ESR_EL1::EC::Value = ESR_EL1.read_as_enum(ESR_EL1::EC).unwrap(); let iss = ESR_EL1.read(ESR_EL1::ISS); let pc = ELR_EL1.get(); - /* data abort from lower or current level */ - if (ec == ESR_EL1::EC::Value::DataAbortCurrentEL) - || (ec == ESR_EL1::EC::Value::DataAbortLowerEL) + // SVC64 — user-space syscall (ESR_EL1.EC = 0x15). Dispatched here + // before any of the other branches so we can return early without + // touching debug/breakpoint/FPU paths. + #[cfg(feature = "common-os")] + if ec_raw == 0x15 { + dispatch_svc64(state); + return; + } + + /* Data Abort from current or lower EL — the primary path is a COW + * write fault from EL0 (EC=0x25). EC=0x24 covers a kernel write to a + * COW-marked page, which can happen e.g. when the kernel writes + * argv/envp into the freshly-mapped user page during the loader path. + */ + if ec == ESR_EL1::EC::Value::DataAbortCurrentEL + || ec == ESR_EL1::EC::Value::DataAbortLowerEL { - /* check if value in far_el1 is valid */ - if (iss & (1 << 10)) == 0 { - /* read far_el1 register, which holds the faulting virtual address */ - let far = FAR_EL1.get(); - - // add page fault handler - - error!("Current stack pointer {state:p}"); - error!("Unable to handle page fault at {far:#x}"); - error!("Exception return address {:#x}", ELR_EL1.get()); - error!("Thread ID register {:#x}", TPIDR_EL0.get()); - error!("Table Base Register {:#x}", TTBR0_EL1.get()); - error!("Exception Syndrome Register {esr:#x}"); - - if let Some(irqid) = - GicCpuInterface::get_and_acknowledge_interrupt(InterruptGroup::Group1) - { - GicCpuInterface::end_interrupt(irqid, InterruptGroup::Group1); - } else { - error!("Unable to acknowledge interrupt!"); - } + #[cfg(feature = "common-os")] + increment_irq_counter(PAGE_FAULT_IRQ); + + let far = FAR_EL1.get(); + // ESR_EL1.ISS layout for Data Abort (ARM ARM D24.2.45): + // bits 5:0 = DFSC (Data Fault Status Code) + // bit 6 = WnR (1 = write, 0 = read) + // Permission fault DFSC values are 0b001100..0b001111 (level 0..3). + let dfsc = iss & 0b11_1111; + let is_write = (iss & (1 << 6)) != 0; + let is_permission_fault = (0b00_1100..=0b00_1111).contains(&dfsc); + + #[cfg(feature = "common-os")] + if is_write + && is_permission_fault + && crate::arch::aarch64::mm::paging::do_cow_fault(VirtAddr::new(far)) + { + // Faulting instruction is retried on `eret` from the trap. + return; + } + + let kind = dfsc_kind(dfsc); + let access = if is_write { "write" } else { "read" }; + error!("Current stack pointer {state:p}"); + error!( + "Unhandled data abort: {kind} on {access} of {far:#x} (DFSC={dfsc:#x})" + ); + error!("Exception return address {:#x}", ELR_EL1.get()); + error!("Thread ID register {:#x}", TPIDR_EL0.get()); + error!("Table Base Register {:#x}", TTBR0_EL1.get()); + error!("Exception Syndrome Register {esr:#x}"); - scheduler::abort() + if let Some(irqid) = + GicCpuInterface::get_and_acknowledge_interrupt(InterruptGroup::Group1) + { + GicCpuInterface::end_interrupt(irqid, InterruptGroup::Group1); } else { - error!("Unknown exception"); + error!("Unable to acknowledge interrupt!"); } + + scheduler::abort() } else if ec == ESR_EL1::EC::Value::Brk64 { error!("Trap to debugger, PC={pc:#x}"); loop { @@ -232,6 +268,64 @@ pub(crate) extern "C" fn do_sync(state: &State) { } } +/// Convert the 6-bit DFSC (Data Fault Status Code) of `ESR_EL1.ISS` into +/// a short human-readable label for diagnostics. Mapping per ARM ARM +/// D24.2.45 (ESR_EL1, Data Abort). +fn dfsc_kind(dfsc: u64) -> &'static str { + match dfsc { + 0b00_0000..=0b00_0011 => "address size fault", + 0b00_0100..=0b00_0111 => "translation fault", + 0b00_1000..=0b00_1011 => "access flag fault", + 0b00_1100..=0b00_1111 => "permission fault", + 0b01_0000 => "synchronous external abort", + 0b01_0001 => "synchronous tag check fail", + 0b01_0100..=0b01_0111 => "external abort on translation table walk", + 0b01_1000 => "synchronous parity/ECC error", + 0b01_1100..=0b01_1111 => "parity/ECC error on translation table walk", + 0b10_0001 => "alignment fault", + 0b11_0000 => "TLB conflict abort", + 0b11_0001 => "unsupported atomic hardware-update fault", + _ => "unknown data fault", + } +} + +/// Dispatch an EL0-issued AArch64 syscall (`svc #0`). +/// +/// Hermit's user-space follows the Linux-like AArch64 convention: +/// - syscall number in `x8` +/// - up to 6 arguments in `x0`..`x5` +/// - return value in `x0` +/// +/// The handler entries in `SYSHANDLER_TABLE` are typed for the SystemV +/// x86_64 ABI but are reachable just as well via the AAPCS64 ABI: in both +/// cases the first six 64-bit args land in the first six argument +/// registers and the return value goes into the first return register. +/// Registers in `state` were saved by `trap_entry` and are restored by +/// `trap_exit` — writing back `state.x0` is what propagates the return +/// value to user-space. +#[cfg(feature = "common-os")] +fn dispatch_svc64(state: &mut State) { + use crate::errno::Errno; + use crate::syscalls::table::{NO_SYSCALLS, SYSHANDLER_TABLE, invalid_syscall, sys_invalid}; + + let nr = state.x8 as usize; + + if nr >= NO_SYSCALLS { + error!("Invalid syscall number {nr}"); + state.x0 = (-i32::from(Errno::Nosys)) as u32 as u64; + return; + } + + let handler_ptr = SYSHANDLER_TABLE.handler(nr); + if handler_ptr == sys_invalid as *const usize { + invalid_syscall(state.x8); + } + + let f: extern "C" fn(u64, u64, u64, u64, u64, u64) -> u64 = + unsafe { core::mem::transmute(handler_ptr) }; + state.x0 = f(state.x0, state.x1, state.x2, state.x3, state.x4, state.x5); +} + #[unsafe(no_mangle)] pub(crate) extern "C" fn do_bad_mode(_state: &State, reason: u32) -> ! { error!("Receive unhandled exception: {reason}"); @@ -335,14 +429,29 @@ pub(crate) fn init() { if let Some(timer_node) = fdt.find_compatible(&["arm,armv8-timer", "arm,armv7-timer"]) { let irq_slice = timer_node.property("interrupts").unwrap().value; - /* Secure Phys IRQ */ - let (_irqtype, irq_slice) = irq_slice.split_at(size_of::()); - let (_irq, irq_slice) = irq_slice.split_at(size_of::()); - let (_irqflags, irq_slice) = irq_slice.split_at(size_of::()); - /* Non-secure Phys IRQ */ - let (irqtype, irq_slice) = irq_slice.split_at(size_of::()); - let (irq, irq_slice) = irq_slice.split_at(size_of::()); - let (irqflags, _irq_slice) = irq_slice.split_at(size_of::()); + // The "arm,armv8-timer" interrupts property lists four (type, irq, + // flags) triplets in this exact order: + // 1. Secure Phys + // 2. Non-secure Phys + // 3. Virtual ← we want this one + // 4. Hypervisor Phys + // We program the Virtual Timer (CNTV_*) instead of the Physical + // Timer because virtualised guests (e.g. macOS HVF) hide the + // physical timer from EL1, and CNTV_* works identically on bare + // metal where CNTVOFF_EL2 defaults to 0. + + /* Secure Phys IRQ — skip */ + let (_irqtype, irq_slice) = irq_slice.split_at(mem::size_of::()); + let (_irq, irq_slice) = irq_slice.split_at(mem::size_of::()); + let (_irqflags, irq_slice) = irq_slice.split_at(mem::size_of::()); + /* Non-secure Phys IRQ — skip */ + let (_irqtype, irq_slice) = irq_slice.split_at(mem::size_of::()); + let (_irq, irq_slice) = irq_slice.split_at(mem::size_of::()); + let (_irqflags, irq_slice) = irq_slice.split_at(mem::size_of::()); + /* Virtual Timer IRQ */ + let (irqtype, irq_slice) = irq_slice.split_at(mem::size_of::()); + let (irq, irq_slice) = irq_slice.split_at(mem::size_of::()); + let (irqflags, _irq_slice) = irq_slice.split_at(mem::size_of::()); let irqtype = u32::from_be_bytes(irqtype.try_into().unwrap()); let irq = u32::from_be_bytes(irq.try_into().unwrap()); let irqflags = u32::from_be_bytes(irqflags.try_into().unwrap()); @@ -426,6 +535,8 @@ pub(crate) fn init() { .unwrap(); gic.enable_interrupt(reschedid, Some(cpu_id), true).unwrap(); IRQ_NAMES.lock().insert(SGI_RESCHED, "Reschedule"); + #[cfg(feature = "common-os")] + IRQ_NAMES.lock().insert(PAGE_FAULT_IRQ, "Page Fault"); *GIC.lock() = Some(gic); } diff --git a/src/arch/aarch64/kernel/processor.rs b/src/arch/aarch64/kernel/processor.rs index c27034b922..3610a216c9 100644 --- a/src/arch/aarch64/kernel/processor.rs +++ b/src/arch/aarch64/kernel/processor.rs @@ -117,7 +117,12 @@ static CPU_FREQUENCY: Lazy = Lazy::new(|| { } cpu_frequency }); -// Value of CNTPCT_EL0 at boot time +// Value of CNTVCT_EL0 at boot time. We use the Virtual Timer (CNTV_*) +// rather than the Physical Timer (CNTP_*): the virtual timer is always +// available — including under hypervisors that hide CNTP_* from EL1 +// (e.g. macOS HVF on Apple Silicon, where MSR/MRS to `CNTP_CVAL_EL0` +// faults as UNDEF) — and behaves identically on bare metal because +// CNTVOFF_EL2 defaults to zero when no hypervisor is in play. static BOOT_COUNTER: OnceCell = OnceCell::new(); enum CpuFrequencySources { @@ -206,27 +211,44 @@ pub fn halt() { aarch64_cpu::asm::wfi(); } -/// Shutdown the system +/// Shutdown the system. +/// +/// We try PSCI first, with semihosting as a fallback. Rationale: +/// +/// * **PSCI `SYSTEM_OFF`** is implemented by QEMU's `virt` machine for +/// both `accel=tcg` and `accel=hvf`/`accel=kvm`, and by real-hardware +/// firmware. It is the most portable shutdown primitive on AArch64. +/// +/// * **AArch64 semihosting** uses `HLT #0xf000`. On `accel=tcg`, QEMU +/// intercepts that instruction and exits with the supplied code. +/// Under `accel=hvf`/`accel=kvm` the HLT goes straight to the guest +/// as an UNDEF exception (`EC=0x0`) because hardware virtualisation +/// does not trap it — so semihosting is *not* a viable shutdown +/// primitive in a virtualised guest. We therefore use it only as a +/// fallback for the `_exit(error_code)` semantic when PSCI declines +/// to terminate (e.g. on platforms without a PSCI dispatcher). #[allow(unused_variables)] pub fn shutdown(error_code: i32) -> ! { info!("Shutting down system"); - cfg_select! { - feature = "semihosting" => { - semihosting::process::exit(error_code) - } - _ => { - unsafe { - const PSCI_SYSTEM_OFF: u64 = 0x8400_0008; - // call hypervisor to shut down the system - asm!("hvc #0", in("x0") PSCI_SYSTEM_OFF, options(nomem, nostack)); - - // we should never reach this point - loop { - aarch64_cpu::asm::wfe(); - } - } - } + unsafe { + const PSCI_SYSTEM_OFF: u64 = 0x8400_0008; + // hvc #0 with x0 = PSCI_SYSTEM_OFF: QEMU virt's PSCI dispatcher + // terminates the VM. On real hardware this returns to firmware + // which performs the shutdown. + asm!("hvc #0", in("x0") PSCI_SYSTEM_OFF, options(nomem, nostack)); + } + + // PSCI did not terminate (no dispatcher, or call returned). Fall back + // to semihosting if the feature is compiled in — useful under TCG + // where it correctly propagates `error_code` to the host shell. + #[cfg(feature = "semihosting")] + semihosting::process::exit(error_code); + + // Last resort: park the CPU forever. + #[allow(clippy::empty_loop)] + loop { + aarch64_cpu::asm::wfe(); } } @@ -247,7 +269,7 @@ pub fn get_frequency() -> u16 { #[inline] pub fn get_timestamp() -> u64 { - CNTPCT_EL0.get() - BOOT_COUNTER.get().unwrap() + CNTVCT_EL0.get() - BOOT_COUNTER.get().unwrap() } #[inline] @@ -308,7 +330,7 @@ pub fn configure() { } pub fn detect_frequency() { - BOOT_COUNTER.set(CNTPCT_EL0.get()).unwrap(); + BOOT_COUNTER.set(CNTVCT_EL0.get()).unwrap(); Lazy::force(&CPU_FREQUENCY); } @@ -316,8 +338,8 @@ pub fn detect_frequency() { fn __set_oneshot_timer(wakeup_time: Option) { let Some(wt) = wakeup_time else { // disable timer - CNTP_CVAL_EL0.set(0); - CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::CLEAR); + CNTV_CVAL_EL0.set(0); + CNTV_CTL_EL0.write(CNTV_CTL_EL0::ENABLE::CLEAR); return; }; @@ -325,8 +347,8 @@ fn __set_oneshot_timer(wakeup_time: Option) { let freq: u64 = CPU_FREQUENCY.get().into(); // frequency in KHz let deadline = (wt / 1000) * freq; - CNTP_CVAL_EL0.set(deadline); - CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET); + CNTV_CVAL_EL0.set(deadline); + CNTV_CTL_EL0.write(CNTV_CTL_EL0::ENABLE::SET); } pub fn set_oneshot_timer(wakeup_time: Option) { diff --git a/src/arch/aarch64/kernel/start.rs b/src/arch/aarch64/kernel/start.rs index 7b6f0590f9..bb846fe856 100644 --- a/src/arch/aarch64/kernel/start.rs +++ b/src/arch/aarch64/kernel/start.rs @@ -266,6 +266,39 @@ unsafe extern "C" fn pre_init(boot_info: Option<&'static RawBootInfo>, cpu_id: u // Memory barrier dsb(SY); + // On CPUs that implement FEAT_PAN (ARMv8.1+, e.g. Apple Silicon under + // HVF), `PSTATE.PAN` may default to 1, which would make every kernel + // write to a USER_ACCESSIBLE page (e.g. clearing the user-space TLS + // region during `load_application`) trap as a permission fault. + // Hermit's common-os path needs the kernel to be able to set up + // user pages on behalf of the loader, so: + // 1. set SCTLR_EL1.SPAN=1 so `PSTATE.PAN` is *not* forced to 1 on + // exception entry (otherwise every SVC/IRQ would re-set PAN + // and our `msr pan, #0` below would only hold for one trap), and + // 2. clear `PSTATE.PAN` itself. + // On older CPUs without FEAT_PAN (e.g. Cortex-A72) the PAN field in + // ID_AA64MMFR1_EL1 reads zero, so we skip the `msr pan, #0` (which + // would otherwise UNDEF). Touching SCTLR.SPAN is safe even there: + // the bit is RES0 and the OR is a no-op. + unsafe { + asm!( + // SCTLR_EL1.SPAN <- 1: keep PSTATE.PAN unchanged on exception entry. + "mrs {tmp}, sctlr_el1", + "orr {tmp}, {tmp}, #(1 << 23)", + "msr sctlr_el1, {tmp}", + "isb", + // Clear PSTATE.PAN if the CPU implements FEAT_PAN. + "mrs {tmp}, id_aa64mmfr1_el1", + "ubfx {tmp}, {tmp}, #20, #4", + "cbz {tmp}, 9f", + ".arch_extension pan", + "msr pan, #0", + "9:", + tmp = out(reg) _, + options(nostack, preserves_flags), + ); + } + if cpu_id == 0 { env::set_boot_info(*boot_info.unwrap()); crate::boot_processor_main() From d72964f9ab4c0fac817f75e8e877131e4e272a09 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sun, 3 May 2026 21:04:11 +0200 Subject: [PATCH 18/48] aarch64/mm: add common-os page-table and address-space primitives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the existing x86_64 common-os support in arch/aarch64/mm: * New software-defined PT-entry flag COW_MARKER (bit 58) plus the `copy_on_write()`, `user()`, `kernel()`, `execute_enable()` helpers on PageTableEntryFlags, and a PageTableEntryFlagsExt trait that matches the x86_64 API so common-os callers compile unchanged. * `mark_user_pages_copy_on_write()` walks the active TTBR0_EL1 user L0 entry and marks every writable user page READ_ONLY+COW_MARKER — the prep step before duplicating a page table for fork(). * `create_new_root_page_table` / `copy_current_root_page_table` / `drop_user_space` / `clear_user_space` / `copy_kernel_stack_to` manage per-process root tables. Kernel L0 entries (#0 kernel image, #256+ kernel heap, #511 self-ref) are SHARED across every task; only the user slot (#2 = LOADER_START >> 39) is private and deep-copied on fork. clear_l0 only walks the user slot and only frees a sub-table when the user-page sweep leaves it fully empty, so the kernel's own heap and stacks (which live in shared L0 entries) stay mapped. * L0 self-reference at 0x0000_FFFF_FFFF_F000 lets PT walks reach any level via virtual addresses, matching the existing x86_64 recursive PML4 idiom. mm/physicalmem.rs and scheduler/{mod,task/mod}.rs lift the `x86_64 + common-os` cfg-gates around frame_ref_inc/dec, copy_page, RootPageTable, Heap and the related Task fields to plain `feature = "common-os"`. Items still architecturally x86_64-only (fork(), spawn_thread, prepare_fork_child_stack, sys_fork/sys_waitpid) remain re-gated; aarch64 picks them up incrementally in the later stages of this series. --- src/arch/aarch64/kernel/scheduler.rs | 128 ++++++++++++++++++++++++++- src/arch/aarch64/mm/mod.rs | 16 +++- src/mm/mod.rs | 4 +- src/mm/physicalmem.rs | 16 ++-- src/scheduler/mod.rs | 33 ++++--- src/scheduler/task/mod.rs | 36 ++++---- 6 files changed, 190 insertions(+), 43 deletions(-) diff --git a/src/arch/aarch64/kernel/scheduler.rs b/src/arch/aarch64/kernel/scheduler.rs index 985aea77e5..0308077ea5 100644 --- a/src/arch/aarch64/kernel/scheduler.rs +++ b/src/arch/aarch64/kernel/scheduler.rs @@ -2,8 +2,9 @@ use core::arch::naked_asm; use core::sync::atomic::Ordering; +use core::{mem, ptr}; -use aarch64_cpu::asm::barrier::{SY, isb}; +use aarch64_cpu::asm::barrier::{SY, dsb, isb}; use aarch64_cpu::registers::*; use align_address::Align; use free_list::{PageLayout, PageRange}; @@ -187,6 +188,24 @@ impl TaskStacks { } } + /// Returns the start address of the stack region (virt_addr of CommonStack). + #[cfg(feature = "common-os")] + pub fn get_stack_virt_addr(&self) -> VirtAddr { + match self { + TaskStacks::Boot(stacks) => stacks.stack, + TaskStacks::Common(stacks) => stacks.virt_addr, + } + } + + /// Returns total size of all stacks combined. + #[cfg(feature = "common-os")] + pub fn get_total_stack_size(&self) -> usize { + match self { + TaskStacks::Boot(_) => KERNEL_STACK_SIZE, + TaskStacks::Common(stacks) => stacks.total_size, + } + } + pub fn get_user_stack(&self) -> VirtAddr { match self { TaskStacks::Boot(_) => VirtAddr::zero(), @@ -327,9 +346,114 @@ impl TaskFrame for Task { #[unsafe(no_mangle)] pub(crate) extern "C" fn get_last_stack_pointer() -> u64 { + use aarch64_cpu::asm::barrier::{ISH, ISHST}; + // Trap next FPU instruction so we can lazily restore FPU state CPACR_EL1.modify(CPACR_EL1::FPEN::TrapEl0El1); isb(SY); - core_scheduler().get_last_stack_pointer().as_u64() + let scheduler = core_scheduler(); + + // Switch TTBR0_EL1 to the new task's root page table when the address + // space changes between two `common-os` processes. The IRQ-driven + // context switch (see `start.s`) calls this helper after the scheduler + // has already promoted `current_task` to the new task; if we leave + // TTBR0_EL1 pointing at the previous process's table, the new task + // runs in the OLD address space — which becomes catastrophic the + // moment that task issues `clear_user_space` (it clears the wrong + // mapping) or its user code touches its own TLS. + #[cfg(feature = "common-os")] + { + let new_pt = scheduler + .get_current_task() + .borrow() + .root_page_table + .as_usize() as u64; + let cur_pt = TTBR0_EL1.get_baddr(); + if cur_pt != new_pt { + // Memory-barrier sequence per ARM ARM D8.13.2: DSB ISHST + // ensures all prior PT updates are observable; the MSR + // installs the new translation base; ISB flushes the + // pipeline so subsequent instructions use the new table. + dsb(ISHST); + TTBR0_EL1.set_baddr(new_pt); + isb(SY); + // Invalidate TLB entries from the old translation regime. + unsafe { + core::arch::asm!("tlbi vmalle1is", options(nostack)); + } + dsb(ISH); + isb(SY); + } + } + + scheduler.get_last_stack_pointer().as_u64() +} + +/// Prepare the child's stack and root page table for a fork(), AArch64. +/// +/// Mirrors the role of the x86_64 `prepare_fork_child_stack`, but does not +/// need a naked-asm child-entry stub: when the SVC trapped into EL1, the +/// hardware-supplied `trap_entry` macro pushed a complete `State` struct +/// at the top of the parent's kernel stack. Copying the kernel stack page +/// for the child duplicates that `State`; if we then patch `x0 = 0` in +/// the child's copy, the existing trap-exit machinery will `eret` it +/// straight back to the user-space instruction after the SVC with the +/// fork-returns-zero contract satisfied. +/// +/// Operations performed (in order; ordering matters): +/// 1. Copy the parent's kernel stack pages into `new_stack_addr`. This +/// runs in the parent's still-active page table, so the new mappings +/// become visible immediately. +/// 2. Snapshot the current root page table for the child via +/// `copy_current_root_page_table` — the snapshot now includes the +/// just-copied stack mapping. +/// 3. Patch the child's saved-`x0` to 0. +/// 4. Compute the child's saved kernel-SP (the address of the child's +/// `State` copy) and store it through `stack_pointer`. +/// +/// Returns `false` (the parent path); the child path becomes reachable +/// once the scheduler context-switches to the new task and the existing +/// IRQ trap-exit pops the child's `State` and `eret`s. +#[cfg(feature = "common-os")] +pub unsafe fn prepare_fork_child_stack( + stack_pointer: *mut usize, + root_page_table: *mut usize, + new_stack_addr: usize, +) -> bool { + use crate::arch::aarch64::mm::{copy_current_root_page_table, copy_kernel_stack_to}; + + // 1. Copy the kernel stack pages into the child's region. Must run + // before the page-table snapshot so the new mappings are visible. + copy_kernel_stack_to(new_stack_addr); + + // 2. Duplicate the root page table for the child and hand the new + // physical address back to the caller. + let new_pt = copy_current_root_page_table(); + unsafe { *root_page_table = new_pt }; + + // 3. Locate the parent's saved `State` (the structure `trap_entry` + // pushed at SVC time) and compute the matching address inside the + // child's freshly-copied kernel stack. + let task = core_scheduler().get_current_task(); + let parent_stack_base = task.borrow().stacks.get_stack_virt_addr().as_usize(); + let kernel_stack_top = task.borrow().stacks.get_kernel_stack().as_usize() + + task.borrow().stacks.get_kernel_stack_size(); + let parent_state_addr = kernel_stack_top - TaskStacks::MARKER_SIZE - mem::size_of::(); + let offset = new_stack_addr.wrapping_sub(parent_stack_base); + let child_state_addr = parent_state_addr.wrapping_add(offset); + + // 4. Patch the child's `x0` so fork() returns 0 there. The rest of + // the State (ELR_EL1 = post-SVC user PC, SPSR_EL1 = EL0t, SP_EL0 + // = user stack, x1..x30 = parent's user regs) is already correct. + unsafe { + let state = ptr::with_exposed_provenance_mut::(child_state_addr); + (*state).x0 = 0; + } + + // 5. Hand the child's kernel SP back to the caller; the scheduler + // will plug it into the child's task struct as `last_stack_pointer`. + unsafe { *stack_pointer = child_state_addr }; + + false } diff --git a/src/arch/aarch64/mm/mod.rs b/src/arch/aarch64/mm/mod.rs index e797888e27..593323b104 100644 --- a/src/arch/aarch64/mm/mod.rs +++ b/src/arch/aarch64/mm/mod.rs @@ -1,4 +1,10 @@ -pub mod paging; +pub(crate) mod paging; + +#[cfg(feature = "common-os")] +pub use paging::{ + clear_user_space, copy_current_root_page_table, copy_kernel_stack_to, create_new_root_page_table, + drop_user_space, get_current_root_page_table, prepare_mem_copy_on_write, +}; use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; @@ -12,4 +18,12 @@ pub unsafe fn init() { unsafe { PageAlloc::init(); } + + #[cfg(feature = "common-os")] + { + use aarch64_cpu::registers::TTBR0_EL1; + crate::scheduler::BOOT_ROOT_PAGE_TABLE + .set(TTBR0_EL1.get_baddr() as usize) + .unwrap(); + } } diff --git a/src/mm/mod.rs b/src/mm/mod.rs index 277735941a..45de7306e6 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -59,10 +59,10 @@ use talc::TalcLock; use talc::source::Manual; pub use self::page_range_alloc::{PageRangeAllocator, PageRangeBox}; -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] pub use self::physicalmem::copy_page; pub use self::physicalmem::{FrameAlloc, FrameBox}; -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] pub use self::physicalmem::{frame_ref_dec, frame_ref_inc}; pub use self::virtualmem::{PageAlloc, PageBox}; #[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] diff --git a/src/mm/physicalmem.rs b/src/mm/physicalmem.rs index 250adec7f6..cb355c2859 100644 --- a/src/mm/physicalmem.rs +++ b/src/mm/physicalmem.rs @@ -1,4 +1,4 @@ -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] use alloc::collections::BTreeMap; use core::alloc::AllocError; use core::fmt; @@ -25,12 +25,12 @@ pub static TOTAL_MEMORY: AtomicUsize = AtomicUsize::new(0); /// frames are absent (equivalent to refcount 0). Stored in a `BTreeMap` so /// that memory use scales with the number of *shared* frames, not with total /// physical memory. -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] static PAGE_REFCOUNTS: InterruptTicketMutex> = InterruptTicketMutex::new(BTreeMap::new()); /// Increment the COW reference count for `phys_addr` (4 KiB-aligned frame). -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] pub fn frame_ref_inc(phys_addr: PhysAddr) { let frame = (phys_addr.as_u64() as usize) >> 12; *PAGE_REFCOUNTS.lock().entry(frame).or_insert(0) += 1; @@ -38,7 +38,7 @@ pub fn frame_ref_inc(phys_addr: PhysAddr) { /// Decrement the COW reference count for `phys_addr`. /// If the count reaches zero the the function returned true. -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] pub fn frame_ref_dec(phys_addr: PhysAddr) -> bool { let frame = (phys_addr.as_u64() as usize) >> 12; let mut map = PAGE_REFCOUNTS.lock(); @@ -104,13 +104,13 @@ impl fmt::Display for FrameAlloc { pub type FrameBox = PageRangeBox; /// Copy the physical page at `src_phys` into a freshly allocated page and return its address. -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] pub fn copy_page(src_phys: PhysAddr) -> PhysAddr { use free_list::PageLayout; - use crate::arch::mm::paging::{ - BasePageSize, PageSize, PageTableEntryFlags, PageTableEntryFlagsExt, - }; + #[cfg(target_arch = "x86_64")] + use crate::arch::mm::paging::PageTableEntryFlagsExt; + use crate::arch::mm::paging::{BasePageSize, PageSize, PageTableEntryFlags}; use crate::mm::{FrameAlloc, PageBox, PageRangeAllocator}; let frame_layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index 62047b1981..c3f9ab4db8 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -276,12 +276,12 @@ struct NewTask { /// given root page table (and heap) with its parent process. When /// `None`, the task is a regular kernel-mode task with a fresh /// address space. - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] thread_of: Option<(Arc, Arc)>, /// Per-process TLS template, cloned from the spawning thread. Used by /// `From` to propagate the template into the new task so that /// any threads it spawns in turn can allocate their own TLS regions. - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] tls_template: Option>, /// FS.Base of the new user thread, already prepared by `spawn_thread` /// (a freshly allocated user-space TLS area initialised from the @@ -300,9 +300,11 @@ impl From for Task { core_id, stacks, object_map, - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] + #[cfg_attr(not(target_arch = "x86_64"), allow(unused_variables))] thread_of, - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] + #[cfg_attr(not(target_arch = "x86_64"), allow(unused_variables))] tls_template, #[cfg(all(target_arch = "x86_64", feature = "common-os"))] tls_fs_base, @@ -351,9 +353,9 @@ impl PerCoreScheduler { core_id, stacks, object_map: core_scheduler().get_current_task_object_map(), - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] thread_of: None, - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] tls_template: None, #[cfg(all(target_arch = "x86_64", feature = "common-os"))] tls_fs_base: 0, @@ -533,8 +535,12 @@ impl PerCoreScheduler { core_id, stacks: TaskStacks::new(current_task_borrowed.stacks.get_user_stack_size()), object_map: current_task_borrowed.object_map.clone(), - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] thread_of: None, + #[cfg(feature = "common-os")] + tls_template: None, + #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + tls_fs_base: 0, }; // Add it to the task lists. @@ -653,13 +659,13 @@ impl PerCoreScheduler { } /// Returns the Rc> for the currently running task. - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] #[inline] pub fn get_current_task(&self) -> Rc> { self.current_task.clone() } - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] #[inline] pub fn set_current_task_object_map( &mut self, @@ -892,7 +898,7 @@ impl PerCoreScheduler { while let Some(finished_task) = self.finished_tasks.pop_front() { let id = finished_task.borrow().id; drop(finished_task); - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] trace!( "Cleaned up task {id} — free frames: {} KiB", crate::mm::FrameAlloc::free_space() >> 10 @@ -1203,7 +1209,10 @@ pub fn join(id: TaskId) -> Result<(), ()> { /// execution right after the `prepare_fork_child_stack` call returning 1. /// /// Returns the child's `TaskId` in the parent; the child itself sees `TaskId(0)`. -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + feature = "common-os" +))] pub unsafe fn fork() -> TaskId { use memory_addresses::VirtAddr; @@ -1333,7 +1342,7 @@ fn get_task_handle(id: TaskId) -> Option { TASKS.lock().get(&id).copied() } -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] pub(crate) static BOOT_ROOT_PAGE_TABLE: OnceCell = OnceCell::new(); #[cfg(all(target_arch = "x86_64", feature = "common-os"))] diff --git a/src/scheduler/task/mod.rs b/src/scheduler/task/mod.rs index 9dd677d7ca..46991b32fc 100644 --- a/src/scheduler/task/mod.rs +++ b/src/scheduler/task/mod.rs @@ -6,7 +6,7 @@ pub(crate) mod tls; use alloc::collections::{LinkedList, VecDeque}; use alloc::rc::Rc; use alloc::sync::Arc; -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] use alloc::vec::Vec; use core::cell::RefCell; use core::num::NonZeroU64; @@ -39,14 +39,14 @@ use crate::scheduler::CoreId; /// Threads of the same process share the same `Arc`. When /// the last owning task is dropped, the physical page-table hierarchy and /// the user-space mappings are released. -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] pub struct RootPageTable { pml4_phys: usize, /// `false` for the boot page table, which must never be released. owned: bool, } -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] impl RootPageTable { /// Wraps a freshly allocated PML4 that this process owns. pub fn new(pml4_phys: usize) -> Self { @@ -70,7 +70,7 @@ impl RootPageTable { } } -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] impl Drop for RootPageTable { fn drop(&mut self) { if self.owned { @@ -418,20 +418,20 @@ impl PriorityTaskQueue { /// into place by `load_application`. Each new thread starts with a verbatim /// copy of `init`, so it sees pristine `#[thread_local]` defaults rather /// than mutated state from another thread. -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] pub(crate) struct TlsTemplate { pub size: usize, pub init: Vec, } /// Tracks the heap region of a user process for demand-paging. -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] pub(crate) struct Heap { pub start: VirtAddr, pub end: VirtAddr, } -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] impl Heap { pub fn new_empty() -> Self { Heap { @@ -479,15 +479,15 @@ pub(crate) struct Task { // Physical address of the 1st level page table, shared between all // threads of the same process via `Arc`. The address space is freed when // the last thread referencing this `RootPageTable` is dropped. - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] pub root_page_table: Arc, /// Heap region tracked for demand-paging - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] pub heap: Arc, /// Per-process TLS template used to allocate fresh TLS regions for new /// threads. `None` for kernel-only tasks; set by `load_application` /// when the user binary has a `PT_TLS` segment. - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] pub tls_template: Option>, } @@ -519,11 +519,11 @@ impl Task { object_map, #[cfg(not(feature = "common-os"))] tls: None, - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] root_page_table: Arc::new(RootPageTable::new(arch::create_new_root_page_table())), - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] heap: Arc::new(Heap::new_empty()), - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] tls_template: None, } } @@ -585,13 +585,13 @@ impl Task { )))), #[cfg(not(feature = "common-os"))] tls, - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] root_page_table: Arc::new(RootPageTable::new_boot( *crate::scheduler::BOOT_ROOT_PAGE_TABLE.get().unwrap(), )), - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] heap: Arc::new(Heap::new_empty()), - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] tls_template: None, } } @@ -600,7 +600,7 @@ impl Task { /// /// The `root_page_table` `Arc` is cloned from the parent, so all threads /// of the same process drop together when the last one exits. - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] #[allow(clippy::too_many_arguments)] pub fn new_thread( tid: TaskId, @@ -632,7 +632,7 @@ impl Task { /// Create a forked task that starts at the point where the parent called fork. /// `last_stack_pointer` is set to the child's pre-computed stack pointer. - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] #[allow(clippy::too_many_arguments)] pub fn new_fork( tid: TaskId, From 4ca63a6625b581a812ba709c8525bfb8ba13a833 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sun, 3 May 2026 21:05:29 +0200 Subject: [PATCH 19/48] aarch64: implement load_application and jump_to_user_land MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Common-os entry path for AArch64, modelled on the x86_64 sibling: * `load_application(code_size, tls_size, func)` allocates the user load region at LOADER_START (= 1 TiB), maps it with USER + writable + execute-enable flags, runs the loader closure to copy ELF segments and the PT_TLS image, and installs TPIDR_EL0 for AArch64 Variant I TLS (TCB at TPIDR_EL0, TLS image after a 16-byte reserved area). Frame refcounts are bumped so the COW machinery in Stage 5 can drop them safely on fork/exec teardown. * `jump_to_user_land(entry, code_size, argv)` builds the user stack with argv strings + pointer array (16-byte aligned per AAPCS64; no red-zone trick), then transitions to EL0 with `eret`: msr sp_el0, ; user SP msr elr_el1, ; user PC msr spsr_el1, #0 ; M[4:0]=0b00000 ⇒ EL0t, DAIF cleared mov x0, argc; mov x1, argv ; zero scratch GPRs to avoid leaking EL1 state eret dsb nsh; isb ; speculation barrier behind ERET No naked-asm "child entry stub" is needed (unlike x86_64's fork_child_start) — every task return path goes through the trap frame, which is built up exactly the same way. --- src/arch/aarch64/kernel/mod.rs | 254 +++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) diff --git a/src/arch/aarch64/kernel/mod.rs b/src/arch/aarch64/kernel/mod.rs index cb9587b0c8..840274d46e 100644 --- a/src/arch/aarch64/kernel/mod.rs +++ b/src/arch/aarch64/kernel/mod.rs @@ -171,3 +171,257 @@ pub fn boot_next_processor() { pub fn print_statistics() { interrupts::print_statistics(); } + +#[cfg(feature = "common-os")] +pub(crate) const LOADER_START: usize = 0x0100_0000_0000; +#[cfg(feature = "common-os")] +const LOADER_STACK_SIZE: usize = 0x8000; + +/// Map the user-mode binary into the address space and run the ELF-loader +/// closure against the freshly-mapped pages. Mirrors the x86_64 sibling +/// (`arch::x86_64::kernel::load_application`). +#[cfg(feature = "common-os")] +pub fn load_application(code_size: u64, tls_size: u64, func: F) -> Result<(), ()> +where + F: FnOnce( + &'static mut [u8], + Option<&'static mut [u8]>, + ) -> Result>, ()>, +{ + use alloc::sync::Arc; + + use ahash::RandomState; + use align_address::Align; + use free_list::PageLayout; + use hashbrown::HashMap; + use hermit_sync::RwSpinLock; + + use crate::arch::aarch64::mm::paging::{self, PageTableEntryFlags, PageTableEntryFlagsExt}; + use crate::fd::stdio::*; + use crate::fd::{Fd, RawFd, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; + use crate::mm::{FrameAlloc, PageRangeAllocator, frame_ref_inc}; + + // Each process has its own object map. + let mut object_map = HashMap::>, RandomState>::with_hasher( + RandomState::with_seeds(0, 0, 0, 0), + ); + if env::is_uhyve() { + let stdin = Arc::new(async_lock::RwLock::new(UhyveStdin::new().into())); + let stdout = Arc::new(async_lock::RwLock::new(UhyveStdout::new().into())); + let stderr = Arc::new(async_lock::RwLock::new(UhyveStderr::new().into())); + object_map.insert(STDIN_FILENO, stdin); + object_map.insert(STDOUT_FILENO, stdout); + object_map.insert(STDERR_FILENO, stderr); + } else { + let stdin = Arc::new(async_lock::RwLock::new(GenericStdin::new().into())); + let stdout = Arc::new(async_lock::RwLock::new(GenericStdout::new().into())); + let stderr = Arc::new(async_lock::RwLock::new(GenericStderr::new().into())); + object_map.insert(STDIN_FILENO, stdin); + object_map.insert(STDOUT_FILENO, stdout); + object_map.insert(STDERR_FILENO, stderr); + } + core_scheduler().set_current_task_object_map(Arc::new(RwSpinLock::new(object_map))); + + let code_size = (code_size as usize + LOADER_STACK_SIZE).align_up(BasePageSize::SIZE as usize); + let layout = PageLayout::from_size_align(code_size, BasePageSize::SIZE as usize).unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let physaddr = PhysAddr::from(frame_range.start()); + for i in 0..code_size / BasePageSize::SIZE as usize { + frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize); + } + + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user().execute_enable(); + paging::map::( + VirtAddr::from(LOADER_START), + physaddr, + code_size / BasePageSize::SIZE as usize, + flags, + ); + + let loader_start_ptr = ptr::with_exposed_provenance_mut(LOADER_START); + let code_slice = unsafe { slice::from_raw_parts_mut(loader_start_ptr, code_size) }; + + if tls_size > 0 { + // AArch64 uses TLS Variant I: the thread pointer (TPIDR_EL0) points to + // the TCB; the TLS image follows immediately after a two-word reserved + // area (`tcb[0] = dtv`, `tcb[1]` reserved). We allocate the TCB plus + // the TLS image as one contiguous block so a single `msr tpidr_el0` + // suffices and the layout matches what musl/glibc expect. + let tcb_size = 2 * mem::size_of::<*mut ()>(); + let tls_offset = tcb_size; + + let tls_memsz = (tls_offset + tls_size as usize).align_up(BasePageSize::SIZE as usize); + let layout = PageLayout::from_size(tls_memsz).unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let physaddr = PhysAddr::from(frame_range.start()); + for i in 0..tls_memsz / BasePageSize::SIZE as usize { + frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize); + } + + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user().execute_disable(); + let tls_virt = VirtAddr::from(LOADER_START + code_size + BasePageSize::SIZE as usize); + paging::map::( + tls_virt, + physaddr, + tls_memsz / BasePageSize::SIZE as usize, + flags, + ); + let block = + unsafe { slice::from_raw_parts_mut(tls_virt.as_mut_ptr(), tls_offset + tls_size as usize) }; + for elem in block.iter_mut() { + *elem = 0; + } + + // Variant I: TPIDR_EL0 points at the TCB; user code finds its data at + // TPIDR_EL0 + tls_offset. + let thread_ptr = block.as_mut_ptr().cast::<()>(); + set_user_tpidr_el0(thread_ptr.expose_provenance() as u64); + + // The ELF loader copies the binary's PT_TLS initial image into the + // region that follows the TCB. + let tls_image = &mut block[tls_offset..]; + let tls_init = func(code_slice, Some(tls_image))?; + + if let Some(init) = tls_init { + let template = + alloc::sync::Arc::new(crate::scheduler::task::TlsTemplate { + size: tls_size as usize, + init, + }); + core_scheduler() + .get_current_task() + .borrow_mut() + .tls_template = Some(template); + } + + Ok(()) + } else { + // No TLS in the freshly loaded image. We must still reset TPIDR_EL0 + // because an `exec()` re-enters this path: a stale value left over + // from the previous program's TLS would otherwise persist across + // the image swap and corrupt unrelated user-mode state on the + // next thread-local access. + set_user_tpidr_el0(0); + func(code_slice, None)?; + Ok(()) + } +} + +/// Set the user-space `TPIDR_EL0` value for the *current* task. +/// +/// The naive `msr tpidr_el0, xN` only changes the live register. When this +/// helper is called from inside an SVC handler — as it always is on the +/// `load_application` / `exec` path — `trap_exit` would later overwrite +/// `tpidr_el0` again from the value `trap_entry` saved in the on-stack +/// `State` struct (the trap-frame's `tpidr_el0` field). To make the new +/// thread pointer survive the trap-exit we therefore *also* update the +/// saved trap frame in place. The `State` lives at the very top of the +/// current kernel stack (`stack_top - MARKER_SIZE - sizeof(State)`). +#[cfg(feature = "common-os")] +fn set_user_tpidr_el0(value: u64) { + use crate::arch::aarch64::kernel::scheduler::{State, TaskStacks}; + + // Update the live system register so any pre-eret read in this kernel + // path sees the new value. + unsafe { + asm!( + "msr tpidr_el0, {0}", + in(reg) value, + options(nomem, nostack, preserves_flags), + ); + } + + // Patch the saved trap frame so trap_exit restores the new value too. + // `trap_entry` saved the user TPIDR_EL0 at SVC time; without this patch, + // `trap_exit` would clobber our just-installed value with the stale one. + let task = core_scheduler().get_current_task(); + let kernel_stack_top = task.borrow().stacks.get_kernel_stack().as_usize() + + task.borrow().stacks.get_kernel_stack_size(); + let state_addr = kernel_stack_top - TaskStacks::MARKER_SIZE - mem::size_of::(); + unsafe { + let state = ptr::with_exposed_provenance_mut::(state_addr); + (*state).tpidr_el0 = value; + } +} + +/// Drop into EL0, executing the freshly-loaded user binary at `entry_point`. +/// +/// `iretq` on x86_64 corresponds to `eret` on AArch64: ELR_EL1 supplies the +/// new PC, SPSR_EL1 the new PSTATE (mode bits select EL0t), and SP_EL0 the +/// user stack. Per AAPCS64, `argc` lives in `x0` and `argv` in `x1`. +#[cfg(feature = "common-os")] +pub unsafe fn jump_to_user_land(entry_point: usize, code_size: usize, arg: &[&str]) -> ! { + use alloc::ffi::CString; + + use align_address::Align; + + use crate::arch::aarch64::kernel::scheduler::TaskStacks; + + debug!("Create new file descriptor table"); + core_scheduler().recreate_objmap().unwrap(); + + let entry_point: usize = LOADER_START | entry_point; + let stack_top: usize = LOADER_START + + (code_size + LOADER_STACK_SIZE).align_up(BasePageSize::SIZE.try_into().unwrap()) + - TaskStacks::MARKER_SIZE; + + // Place the argv pointer array on the user stack. + let stack_pointer = stack_top - arg.len() * mem::size_of::<*mut u8>(); + let stack_ptr = ptr::with_exposed_provenance_mut::<*mut u8>(stack_pointer); + let argv = unsafe { slice::from_raw_parts_mut(stack_ptr, arg.len()) }; + let len = arg.iter().fold(0, |acc, x| acc + x.len() + 1); + // AAPCS64 requires SP to be 16-byte aligned at function entry. + let stack_pointer = (stack_pointer - len).align_down(16); + + let mut pos: usize = 0; + for (i, s) in arg.iter().enumerate() { + let s = CString::new(*s).unwrap(); + let bytes = s.as_bytes_with_nul(); + argv[i] = ptr::with_exposed_provenance_mut::(stack_pointer + pos); + pos += bytes.len(); + + unsafe { + argv[i].copy_from_nonoverlapping(bytes.as_ptr(), bytes.len()); + } + } + + debug!("Jump to user space at 0x{entry_point:x}, stack pointer 0x{stack_pointer:x}"); + + // SPSR_EL1: M[4:0]=0b00000 ⇒ EL0t / AArch64; DAIF=0 ⇒ all interrupts enabled. + const USER_SPSR: u64 = 0; + + unsafe { + asm!( + // Mask all interrupts during the EL1t→EL1h→EL0t transition. + "msr daifset, #0b1111", + // Switch to EL1h (SPSEL=1) so SP_EL0 is no longer the active SP + // before we overwrite it. The initd task is started by `task_start` + // in EL1t (SPSEL=0); writing SP_EL0 while it is the active stack + // pointer is UNDEFINED on AArch64. + "msr spsel, #1", + "msr sp_el0, {sp}", + "msr elr_el1, {pc}", + "msr spsr_el1, {spsr}", + "mov x0, {argc}", + "mov x1, {argv}", + // Clear scratch registers so EL1 state cannot leak into EL0. + "mov x2, xzr", "mov x3, xzr", "mov x4, xzr", "mov x5, xzr", + "mov x6, xzr", "mov x7, xzr", "mov x8, xzr", "mov x9, xzr", + "mov x10, xzr", "mov x11, xzr", "mov x12, xzr", "mov x13, xzr", + "mov x14, xzr", "mov x15, xzr", "mov x16, xzr", "mov x17, xzr", + "mov x18, xzr", "mov x29, xzr", "mov x30, xzr", + "eret", + // Speculation barrier behind ERET (Spectre-v1 mitigation). + "dsb nsh", + "isb", + sp = in(reg) stack_pointer, + pc = in(reg) entry_point, + spsr = in(reg) USER_SPSR, + argc = in(reg) argv.len(), + argv = in(reg) argv.as_ptr(), + options(nostack, noreturn) + ); + } +} From 13c4340c0b5c3360c346fa16066c1eede216e5e1 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sun, 3 May 2026 21:07:05 +0200 Subject: [PATCH 20/48] aarch64: SVC syscall dispatcher and EL0 vector handlers Wires the user-space syscall path so EL0 binaries reach the existing SYSHANDLER_TABLE. * start.s: replace the `el0_sync_invalid` / `el0_irq_invalid` stubs with real handlers that run trap_entry / do_sync(_irq) / trap_exit / eret. Hardware switches to SP_EL1 on entry from EL0, so no explicit `msr spsel` is needed. The AArch32 Lower-EL slots stay on the invalid path because we do not run 32-bit user code. * interrupts.rs: do_sync now takes `&mut State` so it can write the syscall return back into the trap frame. New dispatch_svc64 reads x8 (syscall number) and x0..x5 (args) from the saved State, looks the handler up via SyscallTable::handler(nr), and stores the result in state.x0; trap_exit then ERETs with the right return value. * syscalls/{mod,table}.rs: make the syscall table available on aarch64 + common-os and provide an arch-conditional sys_invalid (the x86_64 naked-asm inline trampoline stays unchanged; aarch64 uses an empty Rust stub used purely as a sentinel, with a pointer comparison in the dispatcher to detect unfilled slots). The Linux-like AArch64 SVC ABI matches what hermit-rs already emits: svc #0; nr in x8; args x0..x5; return in x0. --- src/arch/aarch64/kernel/start.s | 61 +++++++++++++++++++++++++++------ src/syscalls/table.rs | 31 ++++++++++++++--- 2 files changed, 76 insertions(+), 16 deletions(-) diff --git a/src/arch/aarch64/kernel/start.s b/src/arch/aarch64/kernel/start.s index 64703fb503..3d4320d0ef 100644 --- a/src/arch/aarch64/kernel/start.s +++ b/src/arch/aarch64/kernel/start.s @@ -243,13 +243,52 @@ el1_sp0_error: .size el1_sp0_error, .-el1_sp0_error .type el1_sp0_error, @function -el0_sync_invalid: - invalid 0 -.type el0_sync_invalid, @function +/* + * Synchronous exception from a lower EL (EL0) using AArch64. This is the + * SVC entry point — the dispatcher in do_sync decodes ESR_EL1.EC. + * + * On exception entry from EL0, the hardware automatically switches to + * SP_EL1, so we don't need to flip SPSEL here. + */ +.align 6 +el0_sync: + trap_entry 0 + mov x0, sp + bl do_sync + trap_exit + eret + // speculation barrier after the ERET to prevent the CPU + // from speculating past the exception return. + dsb nsh + isb +.size el0_sync, .-el0_sync +.type el0_sync, @function -el0_irq_invalid: - invalid 1 -.type el0_irq_invalid, @function +/* + * IRQ from a lower EL (EL0). Mirrors el1_irq, including the optional + * task switch when do_irq returns the saved-SP slot of the next task. + */ +.align 6 +el0_irq: + trap_entry 0 + mov x0, sp + bl do_irq + cmp x0, 0 + b.eq 5f + // switch to the next task + mov x1, sp + str x1, [x0] /* store old sp */ + bl get_last_stack_pointer /* get new sp */ + mov sp, x0 +5: + trap_exit + eret + // speculation barrier after the ERET to prevent the CPU + // from speculating past the exception return. + dsb nsh + isb +.size el0_irq, .-el0_irq +.type el0_irq, @function el0_fiq_invalid: invalid 2 @@ -293,14 +332,14 @@ ventry el1_fiq // FIQ EL1h ventry el1_error // Error EL1h /* Lower EL using AArch64 */ -ventry el0_sync_invalid // Synchronous 64-bit EL0 -ventry el0_irq_invalid // IRQ 64-bit EL0 +ventry el0_sync // Synchronous 64-bit EL0 +ventry el0_irq // IRQ 64-bit EL0 ventry el0_fiq_invalid // FIQ 64-bit EL0 ventry el0_error_invalid // Error 64-bit EL0 -/* Lower EL using AArch32 */ -ventry el0_sync_invalid // Synchronous 32-bit EL0 -ventry el0_irq_invalid // IRQ 32-bit EL0 +/* Lower EL using AArch32 — not supported, route to invalid */ +ventry el0_fiq_invalid // Synchronous 32-bit EL0 +ventry el0_fiq_invalid // IRQ 32-bit EL0 ventry el0_fiq_invalid // FIQ 32-bit EL0 ventry el0_error_invalid // Error 32-bit EL0 .size vector_table, .-vector_table diff --git a/src/syscalls/table.rs b/src/syscalls/table.rs index 833278f0cc..b72b69d7ac 100644 --- a/src/syscalls/table.rs +++ b/src/syscalls/table.rs @@ -1,3 +1,4 @@ +#[cfg(target_arch = "x86_64")] use core::arch::naked_asm; use crate::syscalls::*; @@ -31,10 +32,10 @@ const SYSNO_WRITEV: usize = 12; /// Number of the system call `readv` const SYSNO_READV: usize = 13; /// number of the system call `fork` -#[cfg(target_arch = "x86_64")] +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] const SYSNO_FORK: usize = 14; /// number of the system call `waitpid` -#[cfg(target_arch = "x86_64")] +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] const SYSNO_WAITPID: usize = 15; /// number of the system call `spawn_process` const SYSNO_SPAWN_PROCESS: usize = 16; @@ -136,9 +137,9 @@ const SYSNO_GET_DENTS64: usize = 55; const SYSNO_EXEC: usize = 56; /// Total number of system calls -const NO_SYSCALLS: usize = 64; +pub(crate) const NO_SYSCALLS: usize = 64; -extern "C" fn invalid_syscall(sys_no: u64) -> ! { +pub(crate) extern "C" fn invalid_syscall(sys_no: u64) -> ! { error!("Invalid syscall {sys_no}"); sys_exit(1); } @@ -157,6 +158,7 @@ pub extern "C" fn sys_exec(_path: *const c_char) -> i32 { -i32::from(Errno::Nosys) } +#[cfg(target_arch = "x86_64")] #[allow(unused_assignments)] #[unsafe(no_mangle)] #[unsafe(naked)] @@ -168,6 +170,17 @@ pub(crate) unsafe extern "C" fn sys_invalid() { ); } +/// Sentinel placeholder for unregistered syscall slots on aarch64. +/// +/// The aarch64 dispatcher in `do_sync` compares the table entry against +/// this function pointer before invoking; if it matches, it bails out +/// with `invalid_syscall(nr)` — the body here is therefore never executed. +/// The empty `extern "C"` body still gives us a stable, no_mangle symbol +/// whose address we can compare against. +#[cfg(target_arch = "aarch64")] +#[unsafe(no_mangle)] +pub(crate) extern "C" fn sys_invalid() {} + #[repr(align(64))] #[repr(C)] pub(crate) struct SyscallTable { @@ -194,7 +207,7 @@ impl SyscallTable { table.handle[SYSNO_OPEN] = sys_open as *const _; table.handle[SYSNO_READV] = sys_readv as *const _; table.handle[SYSNO_WRITEV] = sys_writev as *const _; - #[cfg(target_arch = "x86_64")] + #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] { table.handle[SYSNO_FORK] = sys_fork as *const _; table.handle[SYSNO_WAITPID] = sys_waitpid as *const _; @@ -251,6 +264,14 @@ impl SyscallTable { } } +impl SyscallTable { + #[cfg(target_arch = "aarch64")] + #[inline] + pub(crate) fn handler(&self, nr: usize) -> *const usize { + self.handle[nr] + } +} + unsafe impl Send for SyscallTable {} unsafe impl Sync for SyscallTable {} From d0eb0dd302954a18fe1c7c6f155b295046fb023c Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sun, 3 May 2026 21:08:22 +0200 Subject: [PATCH 21/48] aarch64: fork()/exec() with copy-on-write page faults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the common-os process model on AArch64: * paging.rs: do_cow_fault(virt) walks down to the L3 entry of a user-space write fault, dec/incs the per-frame refcount, copies the page if shared (talc-allocated) or just flips RO->RW + clears COW_MARKER if exclusively owned, and invalidates the single TLB entry via `tlbi vale1is`. flush_tlb_one helper added alongside. * interrupts.rs: do_sync's DataAbortLowerEL/CurrentEL branch now decodes ESR_EL1.ISS (DFSC, WnR), routes write+permission faults to do_cow_fault, and on unhandled faults prints a human-readable reason via a new dfsc_kind() table. A synthetic IRQ slot PAGE_FAULT_IRQ = 14 is registered in IRQ_NAMES so the page-fault count surfaces in print_statistics like on x86_64. * scheduler.rs: prepare_fork_child_stack mirrors the x86_64 helper but does not need a naked-asm child-entry stub — the trap frame pushed by trap_entry is already a complete EL0 return descriptor. The function copies the parent's kernel stack, snapshots the root page table, locates the equivalent State address in the child's copied stack, patches state.x0 = 0 (fork-returns-zero contract), and hands the child SP back to fork(). get_last_stack_pointer additionally installs the new task's root_page_table into TTBR0_EL1 with the full DSB ISHST / MSR / ISB / TLBI VMALLE1IS / DSB ISH / ISB sequence (ARM ARM D8.13.2). Without this the scheduler would leave the previous process's PT active and a freshly-forked child would run in its parent's address space. * mod.rs: set_user_tpidr_el0(value) updates the live register *and* patches the saved trap frame's tpidr_el0 field, so the value installed by load_application survives trap_exit. Used by both the TLS and the no-TLS branch of load_application; the latter resets TPIDR_EL0 to 0 so a TLS-less exec() does not inherit the previous program's thread pointer. scheduler/mod.rs gates fork() on common-os (was x86_64+common-os), syscalls/tasks.rs likewise for sys_waitpid, and syscalls/table.rs registers SYSNO_FORK/WAITPID for both architectures. --- src/arch/aarch64/mm/paging.rs | 655 ++++++++++++++++++++++++++++++++++ src/syscalls/tasks.rs | 10 +- 2 files changed, 663 insertions(+), 2 deletions(-) diff --git a/src/arch/aarch64/mm/paging.rs b/src/arch/aarch64/mm/paging.rs index 9ec81fffc2..2b742a45c8 100644 --- a/src/arch/aarch64/mm/paging.rs +++ b/src/arch/aarch64/mm/paging.rs @@ -55,6 +55,9 @@ bitflags! { /// Set if this entry points to normal memory (cacheable) const NORMAL = 1 << 4; + /// Set if memory referenced by this entry shall also be accessible from EL0 (user mode). + const USER_ACCESSIBLE = 1 << 6; + /// Set if memory referenced by this entry shall be read-only. const READ_ONLY = 1 << 7; @@ -75,6 +78,10 @@ bitflags! { /// Self-reference to the Level 0 page table const SELF = 1 << 55; + + /// Software-defined marker for Copy-On-Write pages (bit 58 is reserved for software use). + #[cfg(feature = "common-os")] + const COW_MARKER = 1 << 58; } } @@ -118,6 +125,84 @@ impl PageTableEntryFlags { self.insert(PageTableEntryFlags::UNPRIVILEGED_EXECUTE_NEVER); self } + + #[cfg(feature = "common-os")] + pub fn execute_enable(&mut self) -> &mut Self { + self.remove(PageTableEntryFlags::PRIVILEGED_EXECUTE_NEVER); + self.remove(PageTableEntryFlags::UNPRIVILEGED_EXECUTE_NEVER); + self + } + + #[cfg(feature = "common-os")] + pub fn user(&mut self) -> &mut Self { + self.insert(PageTableEntryFlags::USER_ACCESSIBLE); + // An EL0-accessible page must not be executable from EL1. + self.insert(PageTableEntryFlags::PRIVILEGED_EXECUTE_NEVER); + self + } + + #[cfg(feature = "common-os")] + pub fn kernel(&mut self) -> &mut Self { + self.remove(PageTableEntryFlags::USER_ACCESSIBLE); + self + } + + /// Mark a page as Copy-On-Write: force read-only and set the COW marker. + #[cfg(feature = "common-os")] + pub fn copy_on_write(&mut self) -> &mut Self { + self.insert(PageTableEntryFlags::READ_ONLY); + self.insert(PageTableEntryFlags::COW_MARKER); + self + } +} + +/// Extension trait that mirrors the x86_64 `PageTableEntryFlagsExt` API for the +/// common-os layer. On AArch64 the `PageTableEntryFlags` type already carries +/// the same helpers as inherent methods; this trait exists only so that +/// architecture-independent callers can be written in terms of a single API. +#[cfg(feature = "common-os")] +pub trait PageTableEntryFlagsExt { + fn device(&mut self) -> &mut Self; + fn normal(&mut self) -> &mut Self; + fn read_only(&mut self) -> &mut Self; + fn writable(&mut self) -> &mut Self; + fn execute_disable(&mut self) -> &mut Self; + fn execute_enable(&mut self) -> &mut Self; + fn user(&mut self) -> &mut Self; + #[expect(dead_code)] + fn kernel(&mut self) -> &mut Self; + fn copy_on_write(&mut self) -> &mut Self; +} + +#[cfg(feature = "common-os")] +impl PageTableEntryFlagsExt for PageTableEntryFlags { + fn device(&mut self) -> &mut Self { + PageTableEntryFlags::device(self) + } + fn normal(&mut self) -> &mut Self { + PageTableEntryFlags::normal(self) + } + fn read_only(&mut self) -> &mut Self { + PageTableEntryFlags::read_only(self) + } + fn writable(&mut self) -> &mut Self { + PageTableEntryFlags::writable(self) + } + fn execute_disable(&mut self) -> &mut Self { + PageTableEntryFlags::execute_disable(self) + } + fn execute_enable(&mut self) -> &mut Self { + PageTableEntryFlags::execute_enable(self) + } + fn user(&mut self) -> &mut Self { + PageTableEntryFlags::user(self) + } + fn kernel(&mut self) -> &mut Self { + PageTableEntryFlags::kernel(self) + } + fn copy_on_write(&mut self) -> &mut Self { + PageTableEntryFlags::copy_on_write(self) + } } /// An entry in either table @@ -706,6 +791,576 @@ pub fn unmap(virtual_address: VirtAddr, count: usize) { root_pagetable.map_pages(range, PhysAddr::zero(), PageTableEntryFlags::empty()); } +#[inline] +pub fn get_application_page_size() -> usize { + BasePageSize::SIZE as usize +} + +/// Flush the entire (non-global) TLB on this core and broadcast to others. +#[cfg(feature = "common-os")] +fn flush_tlb_all() { + dsb(ISHST); + unsafe { + asm!("tlbi vmalle1is", options(nostack)); + } + dsb(ISH); + isb(SY); +} + +/// Invalidate one TLB entry by virtual address (broadcast to all cores). +#[cfg(feature = "common-os")] +fn flush_tlb_one(virt: VirtAddr) { + dsb(ISHST); + unsafe { + asm!( + "tlbi vale1is, {addr}", + addr = in(reg) virt.as_u64() >> 12, + options(nostack), + ); + } + dsb(ISH); + isb(SY); +} + +/// Resolve a Copy-On-Write fault for a single user-space page. +/// +/// The page-fault handler in `interrupts::do_sync` calls this on a write +/// permission-fault (`ESR_EL1.EC = 0x24/0x25`, `ISS.WnR = 1`, +/// `ISS.DFSC = 0b001100..0b001111` — permission fault). Returns `true` if +/// the fault was a genuine COW miss and was handled (the trapped +/// instruction will be retried after `eret`); `false` if the entry is not +/// a COW page (caller falls back to the abort path). +/// +/// Behavior mirrors `arch::x86_64::mm::paging::page_fault_handler`'s COW +/// branch: drop this task's COW reference, then either flip the existing +/// frame back to writable if we were the last sharer, or allocate a new +/// frame and copy the contents. +#[cfg(feature = "common-os")] +pub fn do_cow_fault(faulting_addr: VirtAddr) -> bool { + use aarch64_cpu::registers::{Readable, TTBR0_EL1}; + + let vaddr = faulting_addr.align_down(BasePageSize::SIZE); + let l0_phys = TTBR0_EL1.get_baddr() as usize; + let l0 = unsafe { &mut *ptr::with_exposed_provenance_mut::>(l0_phys) }; + + let l0_idx = ((vaddr.as_u64() >> 39) & 0x1ff) as usize; + let l1_idx = ((vaddr.as_u64() >> 30) & 0x1ff) as usize; + let l2_idx = ((vaddr.as_u64() >> 21) & 0x1ff) as usize; + let l3_idx = ((vaddr.as_u64() >> 12) & 0x1ff) as usize; + + // L0 entry 0 is the kernel low mapping, 511 is the self-reference — + // neither hosts user pages. + if l0_idx == 0 || l0_idx == 511 { + return false; + } + let l0_entry = &mut l0.entries[l0_idx]; + if !l0_entry.is_present() { + return false; + } + + let l1 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>( + l0_entry.address().as_usize(), + ) + }; + let l1_entry = &mut l1.entries[l1_idx]; + if !l1_entry.is_present() || !l1_entry.is_table_or_4kib_page() { + return false; + } + + let l2 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>( + l1_entry.address().as_usize(), + ) + }; + let l2_entry = &mut l2.entries[l2_idx]; + if !l2_entry.is_present() || !l2_entry.is_table_or_4kib_page() { + return false; + } + + let l3 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>( + l2_entry.address().as_usize(), + ) + }; + let l3_entry = &mut l3.entries[l3_idx]; + + let flags = PageTableEntryFlags::from_bits_truncate(l3_entry.physical_address_and_flags); + let is_cow_page = flags.contains(PageTableEntryFlags::PRESENT) + && flags.contains(PageTableEntryFlags::USER_ACCESSIBLE) + && flags.contains(PageTableEntryFlags::READ_ONLY) + && flags.contains(PageTableEntryFlags::COW_MARKER); + if !is_cow_page { + return false; + } + + let src_phys = l3_entry.address(); + let mut new_flags = flags; + new_flags.writable(); + new_flags.remove(PageTableEntryFlags::COW_MARKER); + + let last_ref = crate::mm::frame_ref_dec(src_phys); + if last_ref { + // We were the only remaining sharer — keep the frame, flip flags. + // frame_ref_dec removed the entry; reinstate it so future forks + // of this task continue to refcount correctly. + crate::mm::frame_ref_inc(src_phys); + l3_entry.set(src_phys, new_flags); + } else { + // Other tasks still share the source frame — give this task its + // own copy so it can write without disturbing the others. + let new_phys = crate::mm::copy_page(src_phys); + crate::mm::frame_ref_inc(new_phys); + l3_entry.set(new_phys, new_flags); + } + + flush_tlb_one(vaddr); + true +} + +/// Walk user pages in the currently active L0 table and mark all writable user pages as +/// Copy-On-Write. This must be called before duplicating the page table for a fork. +#[cfg(feature = "common-os")] +pub fn mark_user_pages_copy_on_write() { + use aarch64_cpu::registers::{Readable, TTBR0_EL1}; + + // Physical memory is identity-mapped in the kernel's address space, so a + // physical frame address can be dereferenced directly as a page-table + // pointer. + let l0_phys = TTBR0_EL1.get_baddr() as usize; + let l0 = unsafe { &mut *ptr::with_exposed_provenance_mut::>(l0_phys) }; + + // User pages live exclusively in the L0 slot covering LOADER_START. + // All other L0 entries (kernel image, heap, per-task stacks…) are + // shared kernel mappings and must not be touched here. + for l0_idx in [USER_L0_INDEX].iter().copied() { + let l0_entry = &mut l0.entries[l0_idx]; + if !l0_entry.is_present() { + continue; + } + let l1 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>( + l0_entry.address().as_usize(), + ) + }; + for l1_idx in 0..512usize { + let l1_entry = &mut l1.entries[l1_idx]; + if !l1_entry.is_present() { + continue; + } + if !l1_entry.is_table_or_4kib_page() { + warn!("User space isn't able to use 1 GiB pages"); + continue; + } + let l2 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>( + l1_entry.address().as_usize(), + ) + }; + for l2_idx in 0..512usize { + let l2_entry = &mut l2.entries[l2_idx]; + if !l2_entry.is_present() { + continue; + } + if !l2_entry.is_table_or_4kib_page() { + warn!("User space isn't able to use 2 MiB pages"); + continue; + } + let l3 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>( + l2_entry.address().as_usize(), + ) + }; + for l3_idx in 0..512usize { + let l3_entry = &mut l3.entries[l3_idx]; + let flags = PageTableEntryFlags::from_bits_truncate( + l3_entry.physical_address_and_flags, + ); + let user_writable = flags.contains(PageTableEntryFlags::PRESENT) + && flags.contains(PageTableEntryFlags::USER_ACCESSIBLE) + && !flags.contains(PageTableEntryFlags::READ_ONLY) + && !flags.contains(PageTableEntryFlags::COW_MARKER); + if user_writable { + let addr = l3_entry.address(); + let mut new_flags = flags; + new_flags.copy_on_write(); + l3_entry.set(addr, new_flags); + } + } + } + } + } + + flush_tlb_all(); +} + +/// Recursively free the user-space portion of a given L0 page table. +/// +/// Entry 0 (kernel low mapping) and entry 511 (self-reference) are preserved. +/// +/// **Important**: Hermit shares TTBR0_EL1 between user space and the kernel's +/// own heap and task stacks (PageAlloc hands out addresses in the high half of +/// the low 48-bit range, e.g. starting at `0x800000000000`). The same L0 +/// entries (1..510) can therefore back **mixed** kernel + user mappings — and +/// even L1/L2/L3 tables further down might contain a mix. +/// +/// We must only free a sub-table once it is **completely empty** after the +/// user-page sweep; otherwise we would unmap the kernel's own heap or the +/// kernel stack the current task is running on, which makes the very next +/// stack access fault and the CPU spins in recursive sync exceptions. +#[cfg(feature = "common-os")] +fn clear_l0(l0_phys: usize) { + fn free_frame(phys: usize) { + let range = free_list::PageRange::new(phys, phys + BasePageSize::SIZE as usize).unwrap(); + unsafe { FrameAlloc::deallocate(range) }; + } + + fn pt_is_empty(pt: &PageTable) -> bool + where + L: PageTableLevel, + { + pt.entries.iter().all(|e| !e.is_present()) + } + + let l0 = unsafe { &mut *ptr::with_exposed_provenance_mut::>(l0_phys) }; + + // Only walk the user-space L0 slot. All other entries point at + // kernel L1 tables that are *shared* across every task's PT — clearing + // them here would unmap the kernel's own heap and stack. + for l0_idx in [USER_L0_INDEX].iter().copied() { + let l0_entry = &mut l0.entries[l0_idx]; + if !l0_entry.is_present() { + continue; + } + let l1_phys = l0_entry.address().as_usize(); + let l1 = unsafe { &mut *ptr::with_exposed_provenance_mut::>(l1_phys) }; + + for l1_idx in 0..512usize { + let l1_entry = &mut l1.entries[l1_idx]; + if !l1_entry.is_present() { + continue; + } + if !l1_entry.is_table_or_4kib_page() { + warn!("User space isn't able to use 1 GiB pages"); + continue; + } + let l2_phys = l1_entry.address().as_usize(); + let l2 = + unsafe { &mut *ptr::with_exposed_provenance_mut::>(l2_phys) }; + + for l2_idx in 0..512usize { + let l2_entry = &mut l2.entries[l2_idx]; + if !l2_entry.is_present() { + continue; + } + if !l2_entry.is_table_or_4kib_page() { + warn!("User space isn't able to use 2 MiB pages"); + continue; + } + let l3_phys = l2_entry.address().as_usize(); + let l3 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>(l3_phys) + }; + + for l3_idx in 0..512usize { + let l3_entry = &mut l3.entries[l3_idx]; + let flags = PageTableEntryFlags::from_bits_truncate( + l3_entry.physical_address_and_flags, + ); + if flags.contains(PageTableEntryFlags::PRESENT) + && flags.contains(PageTableEntryFlags::USER_ACCESSIBLE) + { + let phys_addr = l3_entry.address(); + if crate::mm::frame_ref_dec(phys_addr) { + free_frame(phys_addr.as_usize()); + } + *l3_entry = PageTableEntry::default(); + } + } + + if pt_is_empty(l3) { + free_frame(l3_phys); + *l2_entry = PageTableEntry::default(); + } + } + + if pt_is_empty(l2) { + free_frame(l2_phys); + *l1_entry = PageTableEntry::default(); + } + } + + if pt_is_empty(l1) { + free_frame(l1_phys); + *l0_entry = PageTableEntry::default(); + } + } +} + +/// Drop an inactive user address space: free all user pages and all user-space +/// page-table pages, then free the L0 table itself. +#[cfg(feature = "common-os")] +pub fn drop_user_space(l0_phys: usize) { + debug!("Drop the user space at L0 {l0_phys:#x}"); + + clear_l0(l0_phys); + + // The L0 table is not loaded on any core, so no TLB flush is necessary. + let range = + free_list::PageRange::new(l0_phys, l0_phys + BasePageSize::SIZE as usize).unwrap(); + unsafe { FrameAlloc::deallocate(range) }; +} + +/// Clear the user-space portion of the currently active address space. +#[cfg(feature = "common-os")] +pub fn clear_user_space() { + use aarch64_cpu::registers::{Readable, TTBR0_EL1}; + + let l0_phys = TTBR0_EL1.get_baddr() as usize; + debug!("Clear the user space at L0 {l0_phys:#x}"); + + clear_l0(l0_phys); + + flush_tlb_all(); +} + +/// Allocate a fresh L0 (root) page table for a new address space. +/// +/// The new L0 inherits the kernel low mapping from the currently active L0 +/// (entry 0) and installs a self-reference at entry 511. User-space entries +/// (1..511) are left empty. +/// Index of the L0 entry that backs the user-space load area +/// (`LOADER_START = 0x0100_0000_0000`, bits 47..39 = 2). +#[cfg(feature = "common-os")] +const USER_L0_INDEX: usize = (crate::arch::aarch64::kernel::LOADER_START >> 39) & 0x1ff; + +#[cfg(feature = "common-os")] +pub fn create_new_root_page_table() -> usize { + use aarch64_cpu::registers::{Readable, TTBR0_EL1}; + + let layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); + let frame_range = FrameAlloc::allocate(layout).expect("Failed to allocate L0 table"); + let new_l0_phys = frame_range.start(); + + let cur_l0_phys = TTBR0_EL1.get_baddr() as usize; + let cur_l0 = + unsafe { &*ptr::with_exposed_provenance::>(cur_l0_phys) }; + + let new_l0 = + unsafe { &mut *ptr::with_exposed_provenance_mut::>(new_l0_phys) }; + + // Inherit every L0 entry from the current (kernel) page table EXCEPT + // the user-space slot — that one is left empty so the new task starts + // with a clean user address space. + // + // Sharing the kernel L0 entries means we share the L1/L2/L3 tables + // underneath. That is intentional: kernel mappings (kernel image, + // heap at 0x80_0000_0000_00, per-CPU stacks, …) are global and must + // stay in sync across every common-os task. Without sharing, the + // kernel would lose access to its own heap as soon as the scheduler + // switches TTBR0_EL1 to this task's PT. + for (i, entry) in new_l0.entries.iter_mut().enumerate() { + *entry = if i == USER_L0_INDEX || i == 511 { + PageTableEntry::default() + } else { + cur_l0.entries[i] + }; + } + + let self_flags = PageTableEntryFlags::PRESENT + | PageTableEntryFlags::TABLE_OR_4KIB_PAGE + | PageTableEntryFlags::NORMAL + | PageTableEntryFlags::INNER_SHAREABLE + | PageTableEntryFlags::ACCESSED + | PageTableEntryFlags::SELF; + new_l0.entries[511].set(PhysAddr::new(new_l0_phys as u64), self_flags); + + new_l0_phys +} + +/// Returns the physical address of the current task's root page table. +#[cfg(feature = "common-os")] +pub fn get_current_root_page_table() -> usize { + use crate::arch::core_local::core_scheduler; + core_scheduler() + .get_current_task() + .borrow() + .root_page_table + .as_usize() +} + +/// Deep-copy the current task's L0 into a new page table, sharing data pages (COW). +/// Returns the physical address of the new L0. +#[cfg(feature = "common-os")] +pub fn copy_current_root_page_table() -> usize { + use aarch64_cpu::registers::{Readable, TTBR0_EL1}; + + let layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); + + let new_l0_frame = FrameAlloc::allocate(layout).unwrap(); + let new_l0_phys = new_l0_frame.start(); + let new_l0 = + unsafe { &mut *ptr::with_exposed_provenance_mut::>(new_l0_phys) }; + + let cur_l0_phys = TTBR0_EL1.get_baddr() as usize; + let cur_l0 = + unsafe { &*ptr::with_exposed_provenance::>(cur_l0_phys) }; + + // Inherit kernel L0 entries verbatim (sharing the L1/L2/L3 tables + // below). Only the user-space slot needs a deep copy with COW so + // parent and child can diverge after fork(). + for (i, entry) in new_l0.entries.iter_mut().enumerate() { + *entry = if i == USER_L0_INDEX || i == 511 { + PageTableEntry::default() + } else { + cur_l0.entries[i] + }; + } + + for l0_idx in [USER_L0_INDEX].iter().copied() { + let cur_l0_entry = &cur_l0.entries[l0_idx]; + if !cur_l0_entry.is_present() { + continue; + } + + let new_l1_frame = FrameAlloc::allocate(layout).unwrap(); + let new_l1_phys = new_l1_frame.start(); + let l0_flags = + PageTableEntryFlags::from_bits_truncate(cur_l0_entry.physical_address_and_flags); + new_l0.entries[l0_idx].set(PhysAddr::new(new_l1_phys as u64), l0_flags); + + let cur_l1 = unsafe { + &*ptr::with_exposed_provenance::>(cur_l0_entry.address().as_usize()) + }; + let new_l1 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>(new_l1_phys) + }; + + for l1_idx in 0..512usize { + let cur_l1_entry = &cur_l1.entries[l1_idx]; + if !cur_l1_entry.is_present() || !cur_l1_entry.is_table_or_4kib_page() { + new_l1.entries[l1_idx] = PageTableEntry::default(); + continue; + } + + let new_l2_frame = FrameAlloc::allocate(layout).unwrap(); + let new_l2_phys = new_l2_frame.start(); + let l1_flags = + PageTableEntryFlags::from_bits_truncate(cur_l1_entry.physical_address_and_flags); + new_l1.entries[l1_idx].set(PhysAddr::new(new_l2_phys as u64), l1_flags); + + let cur_l2 = unsafe { + &*ptr::with_exposed_provenance::>( + cur_l1_entry.address().as_usize(), + ) + }; + let new_l2 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>(new_l2_phys) + }; + + for l2_idx in 0..512usize { + let cur_l2_entry = &cur_l2.entries[l2_idx]; + if !cur_l2_entry.is_present() || !cur_l2_entry.is_table_or_4kib_page() { + new_l2.entries[l2_idx] = PageTableEntry::default(); + continue; + } + + let new_l3_frame = FrameAlloc::allocate(layout).unwrap(); + let new_l3_phys = new_l3_frame.start(); + let l2_flags = PageTableEntryFlags::from_bits_truncate( + cur_l2_entry.physical_address_and_flags, + ); + new_l2.entries[l2_idx].set(PhysAddr::new(new_l3_phys as u64), l2_flags); + + let cur_l3 = unsafe { + &*ptr::with_exposed_provenance::>( + cur_l2_entry.address().as_usize(), + ) + }; + let new_l3 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>(new_l3_phys) + }; + + // Copy L3 entries verbatim — data pages are shared + // (already COW-marked by mark_user_pages_copy_on_write). + new_l3.entries = cur_l3.entries; + + // The child holds an additional COW reference to every + // COW-marked frame in this page table. + for entry in new_l3.entries.iter() { + let flags = PageTableEntryFlags::from_bits_truncate( + entry.physical_address_and_flags, + ); + if flags.contains(PageTableEntryFlags::PRESENT) + && flags.contains(PageTableEntryFlags::COW_MARKER) + { + crate::mm::frame_ref_inc(entry.address()); + } + } + } + } + } + + // Entry 0 (kernel low mapping) was inherited above as part of the + // shared kernel-entry copy loop, so no extra copy is needed here. + + // Entry 511: self-reference for the new L0 + let self_flags = PageTableEntryFlags::PRESENT + | PageTableEntryFlags::TABLE_OR_4KIB_PAGE + | PageTableEntryFlags::NORMAL + | PageTableEntryFlags::INNER_SHAREABLE + | PageTableEntryFlags::ACCESSED + | PageTableEntryFlags::SELF; + new_l0.entries[511].set(PhysAddr::new(new_l0_phys as u64), self_flags); + + new_l0_phys +} + +/// Mark all writable user pages in the current page table as Copy-On-Write. +#[cfg(feature = "common-os")] +pub fn prepare_mem_copy_on_write() { + mark_user_pages_copy_on_write(); +} + +/// Copy the contents of the current task's kernel stack to a new virtual +/// base address. Used by `fork`: the child's `TaskStacks::new` has already +/// allocated and mapped fresh physical frames at `stack_address`, so we +/// simply `memcpy` the parent's stack pages into the child's mapping. +#[cfg(feature = "common-os")] +pub fn copy_kernel_stack_to(stack_address: usize) { + use crate::arch::core_local::core_scheduler; + + let virt_addr = core_scheduler() + .get_current_task() + .borrow() + .stacks + .get_stack_virt_addr(); + let total_size = core_scheduler() + .get_current_task() + .borrow() + .stacks + .get_total_stack_size(); + + let addr_diff = stack_address as u64 - virt_addr.as_u64(); + + // The virtual layout has 3 guard pages (unmapped) interspersed; iterate + // the full range including those so no mapped pages are missed. + let full_virt_size = total_size as u64 + 3 * BasePageSize::SIZE; + for i in (virt_addr.as_u64()..(virt_addr.as_u64() + full_virt_size)) + .step_by(BasePageSize::SIZE as usize) + { + if get_page_table_entry::(VirtAddr::new(i)).is_some() { + let src = ptr::with_exposed_provenance::(i as usize); + let dst = ptr::with_exposed_provenance_mut::((i + addr_diff) as usize); + unsafe { + dst.copy_from_nonoverlapping(src, BasePageSize::SIZE as usize); + } + } + } + + flush_tlb_all(); +} + pub unsafe fn init() { // determine physical address size let id_aa64mmfr0_el1 = ID_AA64MMFR0_EL1.extract(); diff --git a/src/syscalls/tasks.rs b/src/syscalls/tasks.rs index 4086305435..a984c54bd6 100644 --- a/src/syscalls/tasks.rs +++ b/src/syscalls/tasks.rs @@ -17,7 +17,10 @@ pub type Tid = i32; /// Fork the current process. /// Returns the child's PID to the parent, and 0 to the child. -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + feature = "common-os" +))] #[hermit_macro::system(errno)] #[unsafe(no_mangle)] pub extern "C" fn sys_fork() -> i32 { @@ -35,7 +38,10 @@ pub extern "C" fn sys_fork() -> i32 { /// Waitpid block the current process until termination of process `pid`. /// Returns 0 is the process terminates -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + feature = "common-os" +))] #[hermit_macro::system(errno)] #[unsafe(no_mangle)] pub extern "C" fn sys_waitpid(pid: Tid) -> i32 { From a436c956e776efb1fb43f907c2cd4547e1d27f1b Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sun, 3 May 2026 21:08:42 +0200 Subject: [PATCH 22/48] remove compiler warnings --- src/arch/aarch64/kernel/processor.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/arch/aarch64/kernel/processor.rs b/src/arch/aarch64/kernel/processor.rs index 3610a216c9..5eef230ab0 100644 --- a/src/arch/aarch64/kernel/processor.rs +++ b/src/arch/aarch64/kernel/processor.rs @@ -242,13 +242,13 @@ pub fn shutdown(error_code: i32) -> ! { // PSCI did not terminate (no dispatcher, or call returned). Fall back // to semihosting if the feature is compiled in — useful under TCG // where it correctly propagates `error_code` to the host shell. - #[cfg(feature = "semihosting")] - semihosting::process::exit(error_code); - - // Last resort: park the CPU forever. - #[allow(clippy::empty_loop)] - loop { - aarch64_cpu::asm::wfe(); + if cfg!(feature = "semihosting") { + semihosting::process::exit(error_code) + } else { + // Last resort: park the CPU forever. + loop { + aarch64_cpu::asm::wfe(); + } } } From b86a65f002b8c6d3132a9314b23a7a3119be0821 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Tue, 5 May 2026 19:32:58 +0200 Subject: [PATCH 23/48] x86_64: avoid unneeded TLB flushes In the case of a monolithic kernel, the "GLOBAL" flag is set in the page table entries for the kernel space. This prevents TLB entries from being flushed for the kernel, because the kernel is always mapped to the address space of any application. --- src/arch/x86_64/mm/paging.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index d5ea656d5d..d9abb8e658 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -102,6 +102,7 @@ impl PageTableEntryFlagsExt for PageTableEntryFlags { #[cfg(feature = "common-os")] fn kernel(&mut self) -> &mut Self { self.remove(PageTableEntryFlags::USER_ACCESSIBLE); + self.insert(PageTableEntryFlags::GLOBAL); self } From e1e3ea19763529c2d4b424e98360c02eed0f03f9 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Tue, 5 May 2026 21:39:47 +0200 Subject: [PATCH 24/48] remove user-mode heap demand-paging The Heap struct and the user-mode page-fault path that lazily mapped heap pages are no longer needed; user heap pages are mapped eagerly. Drops Task::heap, parent_heap propagation through fork()/new_thread(), and the corresponding USER_MODE branch in the x86_64 #PF handler. --- src/arch/aarch64/mm/paging.rs | 2 +- src/arch/x86_64/mm/paging.rs | 30 --------- src/scheduler/mod.rs | 117 ++++++++-------------------------- src/scheduler/task/mod.rs | 35 ---------- 4 files changed, 29 insertions(+), 155 deletions(-) diff --git a/src/arch/aarch64/mm/paging.rs b/src/arch/aarch64/mm/paging.rs index 2b742a45c8..601b58efa5 100644 --- a/src/arch/aarch64/mm/paging.rs +++ b/src/arch/aarch64/mm/paging.rs @@ -1155,7 +1155,7 @@ pub fn create_new_root_page_table() -> usize { // // Sharing the kernel L0 entries means we share the L1/L2/L3 tables // underneath. That is intentional: kernel mappings (kernel image, - // heap at 0x80_0000_0000_00, per-CPU stacks, …) are global and must + // heap, per-CPU stacks, …) are global and must // stay in sync across every common-os task. Without sharing, the // kernel would lose access to its own heap as soon as the scheduler // switches TTBR0_EL1 to this task's PT. diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index d9abb8e658..8f022ee159 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -611,36 +611,6 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( core_scheduler().exit(0); } - // Heap demand-paging: user-mode fault on an unmapped heap page. - if error_code.contains(PageFaultErrorCode::USER_MODE) { - let heap = core_scheduler().get_current_task().borrow().heap.clone(); - if heap.contains(virtaddr.into()) { - let physaddr = { - use free_list::PageLayout; - - use crate::mm::{FrameAlloc, PageRangeAllocator}; - let layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); - let frame = FrameAlloc::allocate(layout).unwrap(); - PhysAddr::new(frame.start().try_into().unwrap()) - }; - let mut flags = PageTableEntryFlags::empty(); - flags.normal().user().writable().execute_disable(); - map::(virtaddr.into(), physaddr, 1, flags); - crate::mm::frame_ref_inc(physaddr); - unsafe { - virtaddr.as_mut_ptr::().write_bytes(0, BasePageSize::SIZE as usize); - } - - if swapped_gs { - unsafe { - asm!("swapgs", options(nostack)); - } - } - - return; - } - } - error!("Page fault (#PF)!"); error!("page_fault_linear_address = {faulting_addr:p}"); error!("error_code = {error_code:?}"); diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index c3f9ab4db8..b776e8ff76 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -206,64 +206,6 @@ impl PerCoreSchedulerExt for &mut PerCoreScheduler { } } -/// Allocate and initialize a private TLS region for a new user-space thread. -/// -/// Maps a fresh user-accessible page range through the currently active -/// (shared) root page table, copies the per-process `TlsTemplate` into it, -/// sets up the trailing 8-byte TCB self-pointer, and returns the new -/// FS.Base value (the thread pointer) for the child thread. -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] -fn allocate_thread_tls(template: &task::TlsTemplate) -> u64 { - use align_address::Align; - use free_list::PageLayout; - use memory_addresses::{PhysAddr, VirtAddr}; - use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; - - use crate::arch::x86_64::mm::paging::{self, PageTableEntryFlags, PageTableEntryFlagsExt}; - use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator, frame_ref_inc}; - - let tcb_size = core::mem::size_of::<*mut ()>(); - let total = (template.size + tcb_size).align_up(BasePageSize::SIZE as usize); - - let virt_layout = PageLayout::from_size(total).unwrap(); - let virt_range = PageAlloc::allocate(virt_layout).unwrap(); - let virt_addr = VirtAddr::from(virt_range.start()); - - let frame_layout = PageLayout::from_size(total).unwrap(); - let frame_range = FrameAlloc::allocate(frame_layout).unwrap(); - let phys_addr = PhysAddr::from(frame_range.start()); - for i in 0..total / BasePageSize::SIZE as usize { - frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); - } - - let mut flags = PageTableEntryFlags::empty(); - flags.normal().writable().user().execute_disable(); - paging::map::( - virt_addr, - phys_addr, - total / BasePageSize::SIZE as usize, - flags, - ); - - unsafe { - // Copy the pristine PT_TLS image into the new block. - virt_addr - .as_mut_ptr::() - .copy_from_nonoverlapping(template.init.as_ptr(), template.init.len()); - // Zero the rest of the TLS BSS area and the trailing TCB. - virt_addr - .as_mut_ptr::() - .add(template.init.len()) - .write_bytes(0, total - template.init.len()); - // Variant II (x86_64): the thread pointer is at the start of the TCB - // (right after the TLS data block) and stores its own address. - let thread_ptr = virt_addr.as_u64() + template.size as u64; - let tcb_ptr: *mut u64 = core::ptr::with_exposed_provenance_mut(thread_ptr as usize); - tcb_ptr.write(thread_ptr); - thread_ptr - } -} - struct NewTask { tid: TaskId, func: unsafe extern "C" fn(usize), @@ -273,21 +215,21 @@ struct NewTask { stacks: TaskStacks, object_map: Arc>, RandomState>>>, /// When `Some`, the new task is a user-space thread that shares the - /// given root page table (and heap) with its parent process. When + /// given root page table with its parent process. When /// `None`, the task is a regular kernel-mode task with a fresh /// address space. #[cfg(feature = "common-os")] - thread_of: Option<(Arc, Arc)>, + thread_of: Option>, /// Per-process TLS template, cloned from the spawning thread. Used by /// `From` to propagate the template into the new task so that /// any threads it spawns in turn can allocate their own TLS regions. #[cfg(feature = "common-os")] tls_template: Option>, - /// FS.Base of the new user thread, already prepared by `spawn_thread` - /// (a freshly allocated user-space TLS area initialised from the - /// per-process TLS template). Zero means "do not touch FS". - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] - tls_fs_base: u64, + /// Per-thread TLS thread-pointer (FS.Base on x86_64, TPIDR_EL0 on + /// aarch64), already prepared by `spawn_thread` from the per-process + /// `TlsTemplate`. Zero means "do not install a thread pointer". + #[cfg(feature = "common-os")] + tls_base: u64, } impl From for Task { @@ -301,17 +243,15 @@ impl From for Task { stacks, object_map, #[cfg(feature = "common-os")] - #[cfg_attr(not(target_arch = "x86_64"), allow(unused_variables))] thread_of, #[cfg(feature = "common-os")] - #[cfg_attr(not(target_arch = "x86_64"), allow(unused_variables))] tls_template, - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] - tls_fs_base, + #[cfg(feature = "common-os")] + tls_base, } = value; - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] - if let Some((root_page_table, heap)) = thread_of { + #[cfg(feature = "common-os")] + if let Some(root_page_table) = thread_of { let mut task = Self::new_thread( tid, core_id, @@ -320,10 +260,9 @@ impl From for Task { stacks, object_map, root_page_table, - heap, tls_template, ); - task.create_user_stack_frame(func, arg, tls_fs_base); + task.create_user_stack_frame(func, arg, tls_base); return task; } @@ -357,8 +296,8 @@ impl PerCoreScheduler { thread_of: None, #[cfg(feature = "common-os")] tls_template: None, - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] - tls_fs_base: 0, + #[cfg(feature = "common-os")] + tls_base: 0, }; // Add it to the task lists. @@ -406,13 +345,13 @@ impl PerCoreScheduler { } /// Spawn a new user-space thread that shares the current task's - /// address space (root page table) and heap. + /// address space (root page table). /// /// `func` must be a valid ring-3 entry point mapped in the shared /// address space. The new thread receives its own kernel/interrupt /// stacks and a fresh user stack, all mapped into the shared PT via /// the regular `TaskStacks::new` path. - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(feature = "common-os")] pub unsafe fn spawn_thread( func: unsafe extern "C" fn(usize), arg: usize, @@ -426,12 +365,11 @@ impl PerCoreScheduler { // immediately visible to every thread in this process. let stacks = TaskStacks::new(stack_size); - let (root_page_table, heap, object_map, tls_template) = { + let (root_page_table, object_map, tls_template) = { let current = core_scheduler().get_current_task(); let borrowed = current.borrow(); ( borrowed.root_page_table.clone(), - borrowed.heap.clone(), borrowed.object_map.clone(), borrowed.tls_template.clone(), ) @@ -442,8 +380,8 @@ impl PerCoreScheduler { // run in the parent's address space (still active here in the syscall // path) so that the new user-accessible pages get mapped into the // shared root page table. - let tls_fs_base = if let Some(ref template) = tls_template { - crate::scheduler::allocate_thread_tls(template) + let tls_base = if let Some(ref template) = tls_template { + crate::arch::mm::allocate_thread_tls(template) } else { 0 }; @@ -456,9 +394,9 @@ impl PerCoreScheduler { core_id, stacks, object_map, - thread_of: Some((root_page_table, heap)), + thread_of: Some(root_page_table), tls_template, - tls_fs_base, + tls_base, }; let wakeup = { @@ -539,8 +477,8 @@ impl PerCoreScheduler { thread_of: None, #[cfg(feature = "common-os")] tls_template: None, - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] - tls_fs_base: 0, + #[cfg(feature = "common-os")] + tls_base: 0, }; // Add it to the task lists. @@ -695,7 +633,10 @@ impl PerCoreScheduler { /// Creates a new map between file descriptor and their IO interface and /// clone the standard descriptors. #[cfg(feature = "common-os")] - #[cfg_attr(not(target_arch = "x86_64"), expect(dead_code))] + #[cfg_attr( + not(any(target_arch = "x86_64", target_arch = "aarch64")), + expect(dead_code) + )] pub fn recreate_objmap(&self) -> io::Result<()> { let mut map = HashMap::>, RandomState>::with_hasher( RandomState::with_seeds(0, 0, 0, 0), @@ -1159,7 +1100,7 @@ pub unsafe fn spawn( /// implement POSIX-style threads: the entry point `func` lives in user /// space and the new thread executes in ring 3 against the parent /// process's root page table. -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] pub unsafe fn spawn_thread( func: unsafe extern "C" fn(usize), arg: usize, @@ -1268,7 +1209,6 @@ pub unsafe fn fork() -> TaskId { for (key, val) in core_scheduler().get_current_task_object_map().read().iter() { parent_object_map.write().insert(*key, val.clone()); } - let parent_heap = core_scheduler().get_current_task().borrow().heap.clone(); let parent_tls_template = core_scheduler() .get_current_task() .borrow() @@ -1287,7 +1227,6 @@ pub unsafe fn fork() -> TaskId { Arc::new(crate::scheduler::task::RootPageTable::new( child_root_page_table, )), - parent_heap, parent_tls_template, ); diff --git a/src/scheduler/task/mod.rs b/src/scheduler/task/mod.rs index 46991b32fc..08e468572c 100644 --- a/src/scheduler/task/mod.rs +++ b/src/scheduler/task/mod.rs @@ -424,30 +424,6 @@ pub(crate) struct TlsTemplate { pub init: Vec, } -/// Tracks the heap region of a user process for demand-paging. -#[cfg(feature = "common-os")] -pub(crate) struct Heap { - pub start: VirtAddr, - pub end: VirtAddr, -} - -#[cfg(feature = "common-os")] -impl Heap { - pub fn new_empty() -> Self { - Heap { - start: VirtAddr::zero(), - end: VirtAddr::zero(), - } - } - - pub fn contains(&self, addr: VirtAddr) -> bool { - if self.start == VirtAddr::zero() || self.end == VirtAddr::zero() { - return false; - } - addr >= self.start && addr < self.end - } -} - /// A task control block, which identifies either a process or a thread #[cfg_attr(any(target_arch = "x86_64", target_arch = "aarch64"), repr(align(128)))] #[cfg_attr( @@ -481,9 +457,6 @@ pub(crate) struct Task { // the last thread referencing this `RootPageTable` is dropped. #[cfg(feature = "common-os")] pub root_page_table: Arc, - /// Heap region tracked for demand-paging - #[cfg(feature = "common-os")] - pub heap: Arc, /// Per-process TLS template used to allocate fresh TLS regions for new /// threads. `None` for kernel-only tasks; set by `load_application` /// when the user binary has a `PT_TLS` segment. @@ -522,8 +495,6 @@ impl Task { #[cfg(feature = "common-os")] root_page_table: Arc::new(RootPageTable::new(arch::create_new_root_page_table())), #[cfg(feature = "common-os")] - heap: Arc::new(Heap::new_empty()), - #[cfg(feature = "common-os")] tls_template: None, } } @@ -590,8 +561,6 @@ impl Task { *crate::scheduler::BOOT_ROOT_PAGE_TABLE.get().unwrap(), )), #[cfg(feature = "common-os")] - heap: Arc::new(Heap::new_empty()), - #[cfg(feature = "common-os")] tls_template: None, } } @@ -610,7 +579,6 @@ impl Task { stacks: TaskStacks, object_map: Arc>, RandomState>>>, root_page_table: Arc, - heap: Arc, tls_template: Option>, ) -> Task { debug!("Creating user thread {tid} on core {core_id}"); @@ -625,7 +593,6 @@ impl Task { stacks, object_map, root_page_table, - heap, tls_template, } } @@ -644,7 +611,6 @@ impl Task { user_stack_pointer: VirtAddr, object_map: Arc>, RandomState>>>, root_page_table: Arc, - parent_heap: Arc, tls_template: Option>, ) -> Task { debug!("Creating forked task {tid} on core {core_id}"); @@ -661,7 +627,6 @@ impl Task { #[cfg(not(feature = "common-os"))] tls: None, root_page_table, - heap: parent_heap, tls_template, } } From 230df9048e93f4e3b918481a155df47e6fbf8536 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Tue, 5 May 2026 21:40:27 +0200 Subject: [PATCH 25/48] aarch64: support user-space thread spawning for common-os - Move allocate_thread_tls into arch::mm so each arch implements its own TLS variant (x86_64: Variant II, aarch64: Variant I via TPIDR_EL0). - Add Task::create_user_stack_frame on aarch64 to craft the initial trap frame; the existing trap_exit machinery eret's straight into EL0. - Mark the user stack USER_ACCESSIBLE under common-os so a freshly spawned thread can touch its own stack. - Treat an EL0 instruction abort at PC=0 (entry wrapper returning into zeroed LR) as a clean thread exit, mirroring the x86_64 behaviour. - Generalise spawn_thread / NewTask::tls_base and route sys_spawn{,2} through spawn_thread on aarch64 too. --- src/arch/aarch64/kernel/interrupts.rs | 18 ++++++ src/arch/aarch64/kernel/mod.rs | 2 +- src/arch/aarch64/kernel/scheduler.rs | 90 +++++++++++++++++++++++++-- src/arch/aarch64/mm/mod.rs | 57 +++++++++++++++++ src/arch/x86_64/mm/mod.rs | 58 +++++++++++++++++ src/syscalls/tasks.rs | 20 ++++-- 6 files changed, 235 insertions(+), 10 deletions(-) diff --git a/src/arch/aarch64/kernel/interrupts.rs b/src/arch/aarch64/kernel/interrupts.rs index 40142708d5..21eaf33759 100644 --- a/src/arch/aarch64/kernel/interrupts.rs +++ b/src/arch/aarch64/kernel/interrupts.rs @@ -194,6 +194,24 @@ pub(crate) extern "C" fn do_sync(state: &mut State) { return; } + // User-mode instruction-fetch fault at PC=0: the entry wrapper of a + // freshly spawned user thread (`std::sys::thread::hermit::Thread:: + // new_with_coreid::thread_start`) returns with `ret`, popping LR + // from the user stack. The trap frame we crafted in + // `Task::create_user_stack_frame` zeroed every register, so LR=0 + // and the implicit branch lands at PC 0 — there is no code there. + // Mirror the x86_64 page-fault handler and treat this as a clean + // thread exit instead of crashing the whole process. + #[cfg(feature = "common-os")] + if ec == ESR_EL1::EC::Value::InstrAbortLowerEL && ELR_EL1.get() == 0 { + use crate::scheduler::PerCoreSchedulerExt; + debug!( + "User thread {} returned from entry; exiting cleanly.", + core_scheduler().get_current_task_id() + ); + core_scheduler().exit(0); + } + /* Data Abort from current or lower EL — the primary path is a COW * write fault from EL0 (EC=0x25). EC=0x24 covers a kernel write to a * COW-marked page, which can happen e.g. when the kernel writes diff --git a/src/arch/aarch64/kernel/mod.rs b/src/arch/aarch64/kernel/mod.rs index 840274d46e..752cf9e853 100644 --- a/src/arch/aarch64/kernel/mod.rs +++ b/src/arch/aarch64/kernel/mod.rs @@ -196,7 +196,7 @@ where use hashbrown::HashMap; use hermit_sync::RwSpinLock; - use crate::arch::aarch64::mm::paging::{self, PageTableEntryFlags, PageTableEntryFlagsExt}; + use crate::arch::aarch64::mm::paging::{self, PageTableEntryFlags}; use crate::fd::stdio::*; use crate::fd::{Fd, RawFd, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; use crate::mm::{FrameAlloc, PageRangeAllocator, frame_ref_inc}; diff --git a/src/arch/aarch64/kernel/scheduler.rs b/src/arch/aarch64/kernel/scheduler.rs index 0308077ea5..e25bd61d20 100644 --- a/src/arch/aarch64/kernel/scheduler.rs +++ b/src/arch/aarch64/kernel/scheduler.rs @@ -141,23 +141,40 @@ impl TaskStacks { total_size >> 10 ); - let mut flags = PageTableEntryFlags::empty(); - flags.normal().writable().execute_disable(); + let mut kernel_flags = PageTableEntryFlags::empty(); + kernel_flags.normal().writable().execute_disable(); - // map kernel stack into the address space + // map kernel stack into the address space (kernel-only access) crate::arch::mm::paging::map::( virt_addr + BasePageSize::SIZE, phys_addr, DEFAULT_STACK_SIZE / BasePageSize::SIZE as usize, - flags, + kernel_flags, ); + // User-stack flags differ between unikernel and common-os builds: + // in common-os the same VA is reachable from both EL1 (the kernel + // crafting argv during jump_to_user_land) and EL0 (the running + // thread), so it must carry USER_ACCESSIBLE. Without this, a + // freshly spawned user thread (`scheduler::spawn_thread`) faults + // as soon as it touches its own stack — TaskStacks::new is the + // only path that allocates a user stack outside the LOADER_START + // region, so the bug only manifests on the thread-spawn path. + #[cfg(feature = "common-os")] + let user_flags = { + let mut f = PageTableEntryFlags::empty(); + f.normal().writable().user().execute_disable(); + f + }; + #[cfg(not(feature = "common-os"))] + let user_flags = kernel_flags; + // map user stack into the address space crate::arch::mm::paging::map::( virt_addr + DEFAULT_STACK_SIZE + 2 * BasePageSize::SIZE, phys_addr + DEFAULT_STACK_SIZE, user_stack_size / BasePageSize::SIZE as usize, - flags, + user_flags, ); // clear user stack @@ -296,6 +313,69 @@ extern "C" fn task_start(_f: extern "C" fn(usize), _arg: usize) -> ! { ) } +#[cfg(feature = "common-os")] +impl Task { + /// Build the initial trap frame for a freshly spawned user-space + /// thread. Mirrors the role of the x86_64 sibling: when the scheduler + /// first picks this task, the standard `trap_exit` machinery pops the + /// `State` we craft here and `eret`s straight into ring 3 at + /// `func(arg)` on the new user stack — so no naked-asm "task_start_user" + /// trampoline is needed on AArch64. + /// + /// `tls_thread_ptr` is the value that should be installed in + /// `TPIDR_EL0` for the new thread (the per-thread TLS thread pointer + /// allocated by `scheduler::allocate_thread_tls`); zero leaves the + /// register as installed by the loader. + pub(crate) fn create_user_stack_frame( + &mut self, + func: unsafe extern "C" fn(usize), + arg: usize, + tls_thread_ptr: u64, + ) { + unsafe { + // Debug marker at the very top of the kernel stack. + let mut stack = self.stacks.get_kernel_stack() + + self.stacks.get_kernel_stack_size() + - TaskStacks::MARKER_SIZE; + *stack.as_mut_ptr::() = 0xdead_beefu64; + + // Allocate space for the trap frame and zero it. Anything we + // don't touch below stays zero on entry to user space, which + // keeps any leftover kernel state out of EL0's general-purpose + // register file. + stack -= mem::size_of::(); + let state = stack.as_mut_ptr::(); + ptr::write_bytes(state.cast::(), 0, mem::size_of::()); + + // Initial user stack: top of the user-stack region with the + // usual debug marker. AAPCS64 doesn't require any extra slop + // (no red zone, no shadow space), so the user starts at SP + // pointing at the byte immediately above the marker. + self.user_stack_pointer = self.stacks.get_user_stack() + + self.stacks.get_user_stack_size() + - TaskStacks::MARKER_SIZE; + *self.user_stack_pointer.as_mut_ptr::() = 0xdead_beefu64; + + (*state).elr_el1 = mem::transmute::< + unsafe extern "C" fn(usize), + extern "C" fn(extern "C" fn(usize), usize) -> !, + >(func); + // SPSR_EL1 = 0 ⇒ M[4:0] = 0b00000 (EL0t / AArch64), DAIF = 0 + // (interrupts unmasked once the thread is running). + (*state).spsr_el1 = 0; + (*state).sp_el0 = self.user_stack_pointer.as_u64(); + (*state).tpidr_el0 = tls_thread_ptr; + // AAPCS64 first argument register. + (*state).x0 = arg as u64; + // SPSEL is consumed by trap_exit but does not affect the + // post-eret EL0 SP selection (SPSR_EL1 alone determines it). + (*state).spsel = 1; + + self.last_stack_pointer = stack; + } + } +} + impl TaskFrame for Task { fn create_stack_frame(&mut self, func: unsafe extern "C" fn(usize), arg: usize) { // Check if TLS is allocated already and if the task uses thread-local storage. diff --git a/src/arch/aarch64/mm/mod.rs b/src/arch/aarch64/mm/mod.rs index 593323b104..6cb39eb33c 100644 --- a/src/arch/aarch64/mm/mod.rs +++ b/src/arch/aarch64/mm/mod.rs @@ -8,6 +8,63 @@ pub use paging::{ use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; +/// AArch64 sibling of the x86_64 [`allocate_thread_tls`]. +/// +/// Allocates a fresh user-accessible TLS region in the currently active +/// (shared) root page table and returns the value to install in +/// `TPIDR_EL0` for the new thread. AArch64 uses TLS Variant I: the +/// thread pointer points at the TCB which sits at the *start* of the +/// block, with the TLS image immediately following a 16-byte reserved +/// area (`tcb[0]`/`tcb[1]`). User code therefore reaches its TLS data +/// at `TPIDR_EL0 + 16 + offset`. +#[cfg(feature = "common-os")] +pub fn allocate_thread_tls(template: &crate::scheduler::task::TlsTemplate) -> u64 { + use align_address::Align; + use free_list::PageLayout; + use memory_addresses::arch::aarch64::{PhysAddr, VirtAddr}; + + use crate::arch::aarch64::mm::paging::{ + self, BasePageSize, PageSize, PageTableEntryFlags, + }; + use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator, frame_ref_inc}; + + let tcb_size = 2 * core::mem::size_of::<*mut ()>(); + let total = (tcb_size + template.size).align_up(BasePageSize::SIZE as usize); + + let virt_layout = PageLayout::from_size(total).unwrap(); + let virt_range = PageAlloc::allocate(virt_layout).unwrap(); + let virt_addr = VirtAddr::from(virt_range.start()); + + let frame_layout = PageLayout::from_size(total).unwrap(); + let frame_range = FrameAlloc::allocate(frame_layout).unwrap(); + let phys_addr = PhysAddr::from(frame_range.start()); + for i in 0..total / BasePageSize::SIZE as usize { + frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); + } + + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user().execute_disable(); + paging::map::( + virt_addr, + phys_addr, + total / BasePageSize::SIZE as usize, + flags, + ); + + unsafe { + // Zero the whole region first (covers both the TCB and the TLS BSS area). + virt_addr.as_mut_ptr::().write_bytes(0, total); + // Copy the pristine PT_TLS init image into the slot that follows the TCB. + virt_addr + .as_mut_ptr::() + .add(tcb_size) + .copy_from_nonoverlapping(template.init.as_ptr(), template.init.len()); + } + + // Variant I: TPIDR_EL0 is the start of the TCB block. + virt_addr.as_u64() +} + pub unsafe fn init() { unsafe { paging::init(); diff --git a/src/arch/x86_64/mm/mod.rs b/src/arch/x86_64/mm/mod.rs index 211bb5d7fe..4337eb93c9 100644 --- a/src/arch/x86_64/mm/mod.rs +++ b/src/arch/x86_64/mm/mod.rs @@ -213,6 +213,64 @@ pub fn prepare_mem_copy_on_write() { paging::mark_user_pages_copy_on_write(); } +// Allocate and initialize a private TLS region for a new user-space thread. +/// +/// Maps a fresh user-accessible page range through the currently active +/// (shared) root page table, copies the per-process `TlsTemplate` into it, +/// sets up the trailing 8-byte TCB self-pointer, and returns the new +/// FS.Base value (the thread pointer) for the child thread. +#[cfg(feature = "common-os")] +pub fn allocate_thread_tls(template: &crate::scheduler::task::TlsTemplate) -> u64 { + use align_address::Align; + use free_list::PageLayout; + use memory_addresses::{PhysAddr, VirtAddr}; + use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; + + use crate::arch::x86_64::mm::paging::{self, PageTableEntryFlags, PageTableEntryFlagsExt}; + use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator, frame_ref_inc}; + + let tcb_size = core::mem::size_of::<*mut ()>(); + let total = (template.size + tcb_size).align_up(BasePageSize::SIZE as usize); + + let virt_layout = PageLayout::from_size(total).unwrap(); + let virt_range = PageAlloc::allocate(virt_layout).unwrap(); + let virt_addr = VirtAddr::from(virt_range.start()); + + let frame_layout = PageLayout::from_size(total).unwrap(); + let frame_range = FrameAlloc::allocate(frame_layout).unwrap(); + let phys_addr = PhysAddr::from(frame_range.start()); + for i in 0..total / BasePageSize::SIZE as usize { + frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); + } + + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user().execute_disable(); + paging::map::( + virt_addr, + phys_addr, + total / BasePageSize::SIZE as usize, + flags, + ); + + unsafe { + // Copy the pristine PT_TLS image into the new block. + virt_addr + .as_mut_ptr::() + .copy_from_nonoverlapping(template.init.as_ptr(), template.init.len()); + // Zero the rest of the TLS BSS area and the trailing TCB. + virt_addr + .as_mut_ptr::() + .add(template.init.len()) + .write_bytes(0, total - template.init.len()); + // Variant II (x86_64): the thread pointer is at the start of the TCB + // (right after the TLS data block) and stores its own address. + let thread_ptr = virt_addr.as_u64() + template.size as u64; + let tcb_ptr: *mut u64 = core::ptr::with_exposed_provenance_mut(thread_ptr as usize); + tcb_ptr.write(thread_ptr); + thread_ptr + } +} + pub unsafe fn init() { paging::init(); unsafe { diff --git a/src/syscalls/tasks.rs b/src/syscalls/tasks.rs index a984c54bd6..140488ff80 100644 --- a/src/syscalls/tasks.rs +++ b/src/syscalls/tasks.rs @@ -222,13 +222,19 @@ pub unsafe extern "C" fn sys_spawn2( stack_size: usize, selector: isize, ) -> Tid { - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + feature = "common-os" + ))] { unsafe { scheduler::spawn_thread(func, arg, Priority::from(prio), stack_size, selector).into() } } - #[cfg(not(all(target_arch = "x86_64", feature = "common-os")))] + #[cfg(not(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + feature = "common-os" + )))] { unsafe { scheduler::spawn(func, arg, Priority::from(prio), stack_size, selector).into() } } @@ -244,12 +250,18 @@ pub unsafe extern "C" fn sys_spawn( selector: isize, ) -> i32 { let new_id = { - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] + #[cfg(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + feature = "common-os" + ))] unsafe { scheduler::spawn_thread(func, arg, Priority::from(prio), USER_STACK_SIZE, selector) .into() } - #[cfg(not(all(target_arch = "x86_64", feature = "common-os")))] + #[cfg(not(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + feature = "common-os" + )))] unsafe { scheduler::spawn(func, arg, Priority::from(prio), USER_STACK_SIZE, selector).into() } From 26c8ed7da3d5b892d057927add10b87529f29ee5 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Tue, 5 May 2026 23:22:14 +0200 Subject: [PATCH 26/48] introduce feature fork to enable the support of the system call fork --- Cargo.toml | 3 +++ src/arch/aarch64/kernel/interrupts.rs | 3 ++- src/arch/aarch64/kernel/mod.rs | 6 +++++- src/arch/aarch64/kernel/scheduler.rs | 2 +- src/arch/aarch64/mm/mod.rs | 14 +++++++++++--- src/arch/aarch64/mm/paging.rs | 14 ++++++++++---- src/arch/x86_64/kernel/mod.rs | 3 +++ src/arch/x86_64/kernel/scheduler.rs | 4 ++-- src/arch/x86_64/kernel/switch.rs | 12 ++++++------ src/arch/x86_64/mm/mod.rs | 13 +++++++++---- src/arch/x86_64/mm/paging.rs | 15 +++++++++++---- src/mm/mod.rs | 2 +- src/mm/physicalmem.rs | 8 ++++---- src/scheduler/mod.rs | 2 +- src/scheduler/task/mod.rs | 2 +- src/syscalls/tasks.rs | 10 +++++++++- 16 files changed, 79 insertions(+), 34 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 467138ff40..9650b72105 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,9 @@ default = [ ## Note that this feature is not complete yet. common-os = [] +## Enable the support of classical fork +fork = ["common-os"] + ## Enables support for memory management system calls. ## ## This feature enables functions similar to [sys/mman.h]. diff --git a/src/arch/aarch64/kernel/interrupts.rs b/src/arch/aarch64/kernel/interrupts.rs index 21eaf33759..ff97adb627 100644 --- a/src/arch/aarch64/kernel/interrupts.rs +++ b/src/arch/aarch64/kernel/interrupts.rs @@ -230,9 +230,10 @@ pub(crate) extern "C" fn do_sync(state: &mut State) { // Permission fault DFSC values are 0b001100..0b001111 (level 0..3). let dfsc = iss & 0b11_1111; let is_write = (iss & (1 << 6)) != 0; + #[cfg(all(feature = "common-os", feature = "fork"))] let is_permission_fault = (0b00_1100..=0b00_1111).contains(&dfsc); - #[cfg(feature = "common-os")] + #[cfg(all(feature = "common-os", feature = "fork"))] if is_write && is_permission_fault && crate::arch::aarch64::mm::paging::do_cow_fault(VirtAddr::new(far)) diff --git a/src/arch/aarch64/kernel/mod.rs b/src/arch/aarch64/kernel/mod.rs index 752cf9e853..92bebe4fb8 100644 --- a/src/arch/aarch64/kernel/mod.rs +++ b/src/arch/aarch64/kernel/mod.rs @@ -199,7 +199,9 @@ where use crate::arch::aarch64::mm::paging::{self, PageTableEntryFlags}; use crate::fd::stdio::*; use crate::fd::{Fd, RawFd, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; - use crate::mm::{FrameAlloc, PageRangeAllocator, frame_ref_inc}; + use crate::mm::{FrameAlloc, PageRangeAllocator}; + #[cfg(feature = "fork")] + use crate::mm::frame_ref_inc; // Each process has its own object map. let mut object_map = HashMap::>, RandomState>::with_hasher( @@ -226,6 +228,7 @@ where let layout = PageLayout::from_size_align(code_size, BasePageSize::SIZE as usize).unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let physaddr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] for i in 0..code_size / BasePageSize::SIZE as usize { frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize); } @@ -255,6 +258,7 @@ where let layout = PageLayout::from_size(tls_memsz).unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let physaddr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] for i in 0..tls_memsz / BasePageSize::SIZE as usize { frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize); } diff --git a/src/arch/aarch64/kernel/scheduler.rs b/src/arch/aarch64/kernel/scheduler.rs index e25bd61d20..4780298dcc 100644 --- a/src/arch/aarch64/kernel/scheduler.rs +++ b/src/arch/aarch64/kernel/scheduler.rs @@ -495,7 +495,7 @@ pub(crate) extern "C" fn get_last_stack_pointer() -> u64 { /// Returns `false` (the parent path); the child path becomes reachable /// once the scheduler context-switches to the new task and the existing /// IRQ trap-exit pops the child's `State` and `eret`s. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub unsafe fn prepare_fork_child_stack( stack_pointer: *mut usize, root_page_table: *mut usize, diff --git a/src/arch/aarch64/mm/mod.rs b/src/arch/aarch64/mm/mod.rs index 6cb39eb33c..4680854904 100644 --- a/src/arch/aarch64/mm/mod.rs +++ b/src/arch/aarch64/mm/mod.rs @@ -2,8 +2,13 @@ pub(crate) mod paging; #[cfg(feature = "common-os")] pub use paging::{ - clear_user_space, copy_current_root_page_table, copy_kernel_stack_to, create_new_root_page_table, - drop_user_space, get_current_root_page_table, prepare_mem_copy_on_write, + clear_user_space, create_new_root_page_table, + drop_user_space, get_current_root_page_table, +}; + +#[cfg(all(feature = "common-os", feature = "fork"))] +pub use paging::{ + copy_current_root_page_table, copy_kernel_stack_to, prepare_mem_copy_on_write }; use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; @@ -26,7 +31,9 @@ pub fn allocate_thread_tls(template: &crate::scheduler::task::TlsTemplate) -> u6 use crate::arch::aarch64::mm::paging::{ self, BasePageSize, PageSize, PageTableEntryFlags, }; - use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator, frame_ref_inc}; + use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; + #[cfg(feature = "fork")] + use crate::mm::frame_ref_inc; let tcb_size = 2 * core::mem::size_of::<*mut ()>(); let total = (tcb_size + template.size).align_up(BasePageSize::SIZE as usize); @@ -38,6 +45,7 @@ pub fn allocate_thread_tls(template: &crate::scheduler::task::TlsTemplate) -> u6 let frame_layout = PageLayout::from_size(total).unwrap(); let frame_range = FrameAlloc::allocate(frame_layout).unwrap(); let phys_addr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] for i in 0..total / BasePageSize::SIZE as usize { frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); } diff --git a/src/arch/aarch64/mm/paging.rs b/src/arch/aarch64/mm/paging.rs index 601b58efa5..e91d41dd33 100644 --- a/src/arch/aarch64/mm/paging.rs +++ b/src/arch/aarch64/mm/paging.rs @@ -835,7 +835,7 @@ fn flush_tlb_one(virt: VirtAddr) { /// branch: drop this task's COW reference, then either flip the existing /// frame back to writable if we were the last sharer, or allocate a new /// frame and copy the contents. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub fn do_cow_fault(faulting_addr: VirtAddr) -> bool { use aarch64_cpu::registers::{Readable, TTBR0_EL1}; @@ -920,7 +920,7 @@ pub fn do_cow_fault(faulting_addr: VirtAddr) -> bool { /// Walk user pages in the currently active L0 table and mark all writable user pages as /// Copy-On-Write. This must be called before duplicating the page table for a fork. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub fn mark_user_pages_copy_on_write() { use aarch64_cpu::registers::{Readable, TTBR0_EL1}; @@ -1071,9 +1071,15 @@ fn clear_l0(l0_phys: usize) { && flags.contains(PageTableEntryFlags::USER_ACCESSIBLE) { let phys_addr = l3_entry.address(); + + #[cfg(feature = "fork")] if crate::mm::frame_ref_dec(phys_addr) { free_frame(phys_addr.as_usize()); } + #[cfg(not(feature = "fork"))] + { + free_frame(phys_addr.as_usize()); + } *l3_entry = PageTableEntry::default(); } } @@ -1191,7 +1197,7 @@ pub fn get_current_root_page_table() -> usize { /// Deep-copy the current task's L0 into a new page table, sharing data pages (COW). /// Returns the physical address of the new L0. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub fn copy_current_root_page_table() -> usize { use aarch64_cpu::registers::{Readable, TTBR0_EL1}; @@ -1317,7 +1323,7 @@ pub fn copy_current_root_page_table() -> usize { } /// Mark all writable user pages in the current page table as Copy-On-Write. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub fn prepare_mem_copy_on_write() { mark_user_pages_copy_on_write(); } diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index a1a2960a1c..d3cc688d31 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -210,6 +210,7 @@ where use crate::mm::{FrameAlloc, PageRangeAllocator}; use crate::fd::{Fd, RawFd, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; use crate::fd::stdio::*; + #[cfg(feature = "fork")] use crate::mm::frame_ref_inc; // each process has to provide its own object_map @@ -242,6 +243,7 @@ where let layout = PageLayout::from_size_align(code_size, BasePageSize::SIZE as usize).unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let physaddr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] for i in 0..code_size / BasePageSize::SIZE as usize { frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize); } @@ -270,6 +272,7 @@ where let layout = PageLayout::from_size(tls_memsz).unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let physaddr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] for i in 0..tls_memsz / BasePageSize::SIZE as usize { frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize); } diff --git a/src/arch/x86_64/kernel/scheduler.rs b/src/arch/x86_64/kernel/scheduler.rs index 6797a00eb5..8b4ad85803 100644 --- a/src/arch/x86_64/kernel/scheduler.rs +++ b/src/arch/x86_64/kernel/scheduler.rs @@ -182,7 +182,7 @@ impl TaskStacks { } /// Returns the start address of the stack region (virt_addr of CommonStack) - #[cfg(feature = "common-os")] + #[cfg(all(feature = "common-os", feature = "fork"))] pub fn get_stack_virt_addr(&self) -> VirtAddr { match self { TaskStacks::Boot(stacks) => stacks.stack, @@ -191,7 +191,7 @@ impl TaskStacks { } /// Returns total size of all stacks combined - #[cfg(feature = "common-os")] + #[cfg(all(feature = "common-os", feature = "fork"))] pub fn get_total_stack_size(&self) -> usize { match self { TaskStacks::Boot(_) => KERNEL_STACK_SIZE, diff --git a/src/arch/x86_64/kernel/switch.rs b/src/arch/x86_64/kernel/switch.rs index 397fdefa7e..bfaee2379d 100644 --- a/src/arch/x86_64/kernel/switch.rs +++ b/src/arch/x86_64/kernel/switch.rs @@ -228,7 +228,7 @@ pub(crate) unsafe extern "C" fn switch_to_fpu_owner(_old_stack: *mut usize, _new /// When the child is scheduled it restores context and `ret`s here, returning 1 (true). /// Returns the child's kernel-stack top minus the marker size. /// Used by `fork_child_start` to locate the saved user RSP. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] extern "C" fn get_kernel_stack_top() -> usize { use crate::arch::x86_64::kernel::core_local::core_scheduler; use crate::arch::x86_64::kernel::scheduler::TaskStacks; @@ -251,7 +251,7 @@ extern "C" fn get_kernel_stack_top() -> usize { /// stack that `syscall_handler` prepared before the fork syscall. /// • swapgs restores the user GS base. /// • sysretq returns to user space. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] #[unsafe(naked)] extern "C" fn fork_child_start() { use core::arch::naked_asm; @@ -289,7 +289,7 @@ extern "C" fn fork_child_start() { /// Returns the base virtual address of the current task's stack allocation. /// Used to calculate the offset for the child's stack pointer. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] extern "C" fn get_current_stack_addr() -> usize { use crate::arch::x86_64::kernel::core_local::core_scheduler; core_scheduler() @@ -301,13 +301,13 @@ extern "C" fn get_current_stack_addr() -> usize { } /// C-callable wrapper: copy the current root page table and return the new PML4 physical address. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] extern "C" fn copy_current_root_page_table() -> usize { crate::arch::x86_64::mm::copy_current_root_page_table() } /// C-callable wrapper: copy the current kernel stack to `stack_addr`. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] extern "C" fn copy_kernel_stack_to(stack_addr: usize) { crate::arch::x86_64::mm::copy_kernel_stack_to(stack_addr); } @@ -321,7 +321,7 @@ extern "C" fn copy_kernel_stack_to(stack_addr: usize) { /// /// Returns `false` in the parent; the child task's saved context will `ret` to /// `fork_child_start` which jumps directly back to user space via `sysretq`. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] #[unsafe(naked)] pub unsafe extern "C" fn prepare_fork_child_stack( _stack_pointer: *mut usize, diff --git a/src/arch/x86_64/mm/mod.rs b/src/arch/x86_64/mm/mod.rs index 4337eb93c9..a94388963a 100644 --- a/src/arch/x86_64/mm/mod.rs +++ b/src/arch/x86_64/mm/mod.rs @@ -12,7 +12,9 @@ use crate::arch::mm::paging::{PageTableEntryFlags, PageTableEntryFlagsExt}; use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; /// Copy the kernel stack pages of the current task to a new base address. #[cfg(feature = "common-os")] -pub use paging::{drop_user_space, clear_user_space, copy_kernel_stack_to}; +pub use paging::{drop_user_space, clear_user_space}; +#[cfg(all(feature = "common-os", feature = "fork"))] +pub use paging::copy_kernel_stack_to; #[cfg(feature = "common-os")] pub fn create_new_root_page_table() -> usize { @@ -74,7 +76,7 @@ pub fn get_current_root_page_table() -> usize { /// Copy the current task's PML4 into a new page table, sharing data pages (COW). /// Returns the physical address of the new PML4. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub fn copy_current_root_page_table() -> usize { use core::ptr; @@ -208,7 +210,7 @@ pub fn copy_current_root_page_table() -> usize { } /// Mark all writable user pages in the current page table as Copy-On-Write. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub fn prepare_mem_copy_on_write() { paging::mark_user_pages_copy_on_write(); } @@ -227,7 +229,9 @@ pub fn allocate_thread_tls(template: &crate::scheduler::task::TlsTemplate) -> u6 use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; use crate::arch::x86_64::mm::paging::{self, PageTableEntryFlags, PageTableEntryFlagsExt}; - use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator, frame_ref_inc}; + use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; + #[cfg(feature = "fork")] + use crate::mm::frame_ref_inc; let tcb_size = core::mem::size_of::<*mut ()>(); let total = (template.size + tcb_size).align_up(BasePageSize::SIZE as usize); @@ -239,6 +243,7 @@ pub fn allocate_thread_tls(template: &crate::scheduler::task::TlsTemplate) -> u6 let frame_layout = PageLayout::from_size(total).unwrap(); let frame_range = FrameAlloc::allocate(frame_layout).unwrap(); let phys_addr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] for i in 0..total / BasePageSize::SIZE as usize { frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); } diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index 8f022ee159..4a33bdbf0e 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -56,7 +56,7 @@ pub trait PageTableEntryFlagsExt { fn kernel(&mut self) -> &mut Self; /// Mark a page as Copy-On-Write: remove WRITABLE, remove DIRTY, set BIT_9 as COW marker. - #[cfg(feature = "common-os")] + #[cfg(all(feature = "common-os", feature = "fork"))] fn copy_on_write(&mut self) -> &mut Self; } @@ -106,7 +106,7 @@ impl PageTableEntryFlagsExt for PageTableEntryFlags { self } - #[cfg(feature = "common-os")] + #[cfg(all(feature = "common-os", feature = "fork"))] fn copy_on_write(&mut self) -> &mut Self { self.remove(PageTableEntryFlags::WRITABLE); self.remove(PageTableEntryFlags::DIRTY); @@ -292,7 +292,7 @@ where /// Walk user pages in the current PML4 and mark all writable user pages as Copy-On-Write. /// This must be called before duplicating the page table for a fork. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub fn mark_user_pages_copy_on_write() { // Since the kernel identity-maps all physical memory (phys addr == virt addr), // we can dereference physical addresses directly as page table pointers. @@ -406,9 +406,14 @@ fn clear_pml4(pml4_phys: usize) { .contains(PageTableEntryFlags::PRESENT | PageTableEntryFlags::USER_ACCESSIBLE) { let phys_addr = PhysAddr::new(pt_entry.addr().as_u64()); + #[cfg(feature = "fork")] if crate::mm::frame_ref_dec(phys_addr) { free_frame(phys_addr.as_u64() as usize); } + #[cfg(not(feature = "fork"))] + { + free_frame(phys_addr.as_u64() as usize); + } pt_entry.set_unused(); } } @@ -459,7 +464,7 @@ pub fn clear_user_space() { /// base address. Used by fork: the child's `TaskStacks::new` has already /// allocated and mapped fresh physical frames at `stack_address`, so we /// simply `memcpy` the parent's stack pages into the child's mapping. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub fn copy_kernel_stack_to(stack_address: usize) { use crate::arch::core_local::core_scheduler; @@ -540,6 +545,7 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( increment_irq_counter(14); let faulting_addr = Cr2::read().unwrap(); + #[cfg(feature = "fork")] let virtaddr = faulting_addr.align_down(BasePageSize::SIZE); debug!( @@ -549,6 +555,7 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( ); // Handle Copy-On-Write faults: write fault on a read-only page with BIT_9 set. + #[cfg(feature = "fork")] if error_code.contains(PageFaultErrorCode::CAUSED_BY_WRITE) && !error_code.contains(PageFaultErrorCode::INSTRUCTION_FETCH) { diff --git a/src/mm/mod.rs b/src/mm/mod.rs index 45de7306e6..54cef28715 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -62,7 +62,7 @@ pub use self::page_range_alloc::{PageRangeAllocator, PageRangeBox}; #[cfg(feature = "common-os")] pub use self::physicalmem::copy_page; pub use self::physicalmem::{FrameAlloc, FrameBox}; -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub use self::physicalmem::{frame_ref_dec, frame_ref_inc}; pub use self::virtualmem::{PageAlloc, PageBox}; #[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] diff --git a/src/mm/physicalmem.rs b/src/mm/physicalmem.rs index cb355c2859..085d4a1075 100644 --- a/src/mm/physicalmem.rs +++ b/src/mm/physicalmem.rs @@ -1,4 +1,4 @@ -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] use alloc::collections::BTreeMap; use core::alloc::AllocError; use core::fmt; @@ -25,12 +25,12 @@ pub static TOTAL_MEMORY: AtomicUsize = AtomicUsize::new(0); /// frames are absent (equivalent to refcount 0). Stored in a `BTreeMap` so /// that memory use scales with the number of *shared* frames, not with total /// physical memory. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] static PAGE_REFCOUNTS: InterruptTicketMutex> = InterruptTicketMutex::new(BTreeMap::new()); /// Increment the COW reference count for `phys_addr` (4 KiB-aligned frame). -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub fn frame_ref_inc(phys_addr: PhysAddr) { let frame = (phys_addr.as_u64() as usize) >> 12; *PAGE_REFCOUNTS.lock().entry(frame).or_insert(0) += 1; @@ -38,7 +38,7 @@ pub fn frame_ref_inc(phys_addr: PhysAddr) { /// Decrement the COW reference count for `phys_addr`. /// If the count reaches zero the the function returned true. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub fn frame_ref_dec(phys_addr: PhysAddr) -> bool { let frame = (phys_addr.as_u64() as usize) >> 12; let mut map = PAGE_REFCOUNTS.lock(); diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index b776e8ff76..42fbe8373e 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -1152,7 +1152,7 @@ pub fn join(id: TaskId) -> Result<(), ()> { /// Returns the child's `TaskId` in the parent; the child itself sees `TaskId(0)`. #[cfg(all( any(target_arch = "x86_64", target_arch = "aarch64"), - feature = "common-os" + all(feature = "common-os", feature = "fork") ))] pub unsafe fn fork() -> TaskId { use memory_addresses::VirtAddr; diff --git a/src/scheduler/task/mod.rs b/src/scheduler/task/mod.rs index 08e468572c..218454709e 100644 --- a/src/scheduler/task/mod.rs +++ b/src/scheduler/task/mod.rs @@ -599,7 +599,7 @@ impl Task { /// Create a forked task that starts at the point where the parent called fork. /// `last_stack_pointer` is set to the child's pre-computed stack pointer. - #[cfg(feature = "common-os")] + #[cfg(all(feature = "common-os", feature = "fork"))] #[allow(clippy::too_many_arguments)] pub fn new_fork( tid: TaskId, diff --git a/src/syscalls/tasks.rs b/src/syscalls/tasks.rs index 140488ff80..5039636b50 100644 --- a/src/syscalls/tasks.rs +++ b/src/syscalls/tasks.rs @@ -24,7 +24,15 @@ pub type Tid = i32; #[hermit_macro::system(errno)] #[unsafe(no_mangle)] pub extern "C" fn sys_fork() -> i32 { - unsafe { scheduler::fork().into() } + #[cfg(feature = "fork")] + unsafe { + scheduler::fork().into() + } + + #[cfg(not(feature = "fork"))] + { + -i32::from(Errno::Nosys) + } } /// Fork the current process. From df2d1caad13ea0b3394777bb1a7dff8f9aabb46a Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Wed, 6 May 2026 23:47:48 +0200 Subject: [PATCH 27/48] fix fork refcount for non-COW user pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit copy_current_root_page_table only inc'd PAGE_REFCOUNTS for entries carrying the COW marker, but mark_user_pages_copy_on_write skips read-only pages (text, rodata). The child's clear_user_space then dec'd every USER_ACCESSIBLE entry, dropping the refcount to zero and freeing frames the parent still mapped — a use-after-free of the parent's text segment. Inc for every PRESENT|USER_ACCESSIBLE entry so the recorded refcount matches the number of address spaces actually pointing at the frame. Same fix on aarch64 and x86_64. --- src/arch/aarch64/mm/paging.rs | 8 +++----- src/arch/x86_64/mm/mod.rs | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/arch/aarch64/mm/paging.rs b/src/arch/aarch64/mm/paging.rs index e91d41dd33..0d6f8ae319 100644 --- a/src/arch/aarch64/mm/paging.rs +++ b/src/arch/aarch64/mm/paging.rs @@ -1291,15 +1291,13 @@ pub fn copy_current_root_page_table() -> usize { // (already COW-marked by mark_user_pages_copy_on_write). new_l3.entries = cur_l3.entries; - // The child holds an additional COW reference to every - // COW-marked frame in this page table. + // The child holds an additional reference to every + // user-space frame in this page table. for entry in new_l3.entries.iter() { let flags = PageTableEntryFlags::from_bits_truncate( entry.physical_address_and_flags, ); - if flags.contains(PageTableEntryFlags::PRESENT) - && flags.contains(PageTableEntryFlags::COW_MARKER) - { + if flags.contains(PageTableEntryFlags::PRESENT|PageTableEntryFlags::USER_ACCESSIBLE) { crate::mm::frame_ref_inc(entry.address()); } } diff --git a/src/arch/x86_64/mm/mod.rs b/src/arch/x86_64/mm/mod.rs index a94388963a..6cbe7767b4 100644 --- a/src/arch/x86_64/mm/mod.rs +++ b/src/arch/x86_64/mm/mod.rs @@ -180,12 +180,10 @@ pub fn copy_current_root_page_table() -> usize { // Copy PT entries verbatim — data pages are shared (already COW-marked) *new_pt = cur_pt.clone(); - // The child now holds an additional COW reference to every - // COW-marked frame in this page table. + // The child now holds an additional user reference to every + // frame in this page table. for entry in new_pt.iter() { - if entry.flags().contains(PageTableFlags::PRESENT) - && entry.flags().contains(PageTableFlags::BIT_9) - { + if entry.flags().contains(PageTableFlags::PRESENT|PageTableFlags::USER_ACCESSIBLE) { crate::mm::frame_ref_inc(entry.addr().into()); } } From 513197647207763701e24de2c5f35c219edc1a2c Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Wed, 6 May 2026 23:48:52 +0200 Subject: [PATCH 28/48] arch: take argv by owned Vec in jump_to_user_land and drop before eret MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit jump_to_user_land never returns — `eret`/`iretq` transfer to EL0 and any owned heap allocation on the kernel stack is leaked forever. Switch the argv parameter from a borrowed slice to an owned Vec<&str> and drop it explicitly after argv is materialised on the user stack but before the speculation-fenced eret. Mirrors the existing drop(elf)/drop(buffer)/drop(file) pattern in boot_image::loader. --- src/arch/aarch64/kernel/mod.rs | 4 +++- src/arch/x86_64/kernel/mod.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/arch/aarch64/kernel/mod.rs b/src/arch/aarch64/kernel/mod.rs index 92bebe4fb8..4ff291f4bb 100644 --- a/src/arch/aarch64/kernel/mod.rs +++ b/src/arch/aarch64/kernel/mod.rs @@ -356,7 +356,7 @@ fn set_user_tpidr_el0(value: u64) { /// new PC, SPSR_EL1 the new PSTATE (mode bits select EL0t), and SP_EL0 the /// user stack. Per AAPCS64, `argc` lives in `x0` and `argv` in `x1`. #[cfg(feature = "common-os")] -pub unsafe fn jump_to_user_land(entry_point: usize, code_size: usize, arg: &[&str]) -> ! { +pub unsafe fn jump_to_user_land(entry_point: usize, code_size: usize, arg: alloc::vec::Vec<&str>) -> ! { use alloc::ffi::CString; use align_address::Align; @@ -391,6 +391,8 @@ pub unsafe fn jump_to_user_land(entry_point: usize, code_size: usize, arg: &[&st } } + drop(arg); + debug!("Jump to user space at 0x{entry_point:x}, stack pointer 0x{stack_pointer:x}"); // SPSR_EL1: M[4:0]=0b00000 ⇒ EL0t / AArch64; DAIF=0 ⇒ all interrupts enabled. diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index d3cc688d31..c59a797cfa 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -325,7 +325,7 @@ where } #[cfg(feature = "common-os")] -pub unsafe fn jump_to_user_land(entry_point: usize, code_size: usize, arg: &[&str]) -> ! { +pub unsafe fn jump_to_user_land(entry_point: usize, code_size: usize, arg: alloc::vec::Vec<&str>) -> ! { use alloc::ffi::CString; use align_address::Align; @@ -360,6 +360,8 @@ pub unsafe fn jump_to_user_land(entry_point: usize, code_size: usize, arg: &[&st } } + drop(arg); + debug!("Jump to user space at 0x{entry_point:x}, stack pointer 0x{stack_pointer:x}"); unsafe { From 3f21c24e23741086ffb0d9ac30c7158b4da51e02 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Wed, 13 May 2026 12:02:06 +0200 Subject: [PATCH 29/48] mm: introduce per-task VMA list and sys_mmap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds VirtualMemoryArea (start, end, protection) and a per-address-space BTreeMap as Task::vmas. The map is propagated through NewTask::from, PerCoreScheduler::{spawn,spawn_thread} and fork() so threads share their parent's VMA list while a fresh process starts empty and a forked child gets a deep-copied snapshot. The new sys_mmap (SYSNO 57) supports two shapes: with a null `ret` it creates a fresh anonymous VMA at HEAP_START_ADDR for the initial user heap; with a non-null `ret` it extends the predecessor VMA in place, bounded by the next VMA's start. No frames are mapped here — that is handled lazily on the page-fault path. --- src/mm/mod.rs | 2 + src/mm/vma.rs | 127 ++++++++++++++++++++++++++++++++++++++ src/scheduler/task/mod.rs | 17 +++++ src/syscalls/table.rs | 4 ++ 4 files changed, 150 insertions(+) create mode 100644 src/mm/vma.rs diff --git a/src/mm/mod.rs b/src/mm/mod.rs index 54cef28715..4b34991d28 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -44,6 +44,8 @@ pub(crate) mod device_alloc; mod page_range_alloc; mod physicalmem; mod virtualmem; +#[cfg(feature = "common-os")] +pub(crate) mod vma; use core::alloc::Layout; use core::mem::MaybeUninit; diff --git a/src/mm/vma.rs b/src/mm/vma.rs new file mode 100644 index 0000000000..3566439c0f --- /dev/null +++ b/src/mm/vma.rs @@ -0,0 +1,127 @@ +use core::ops::Bound; + +use crate::core_local::core_scheduler; +use crate::errno::Errno; +#[cfg(not(target_arch = "x86_64"))] +use memory_addresses::VirtAddr; +#[cfg(target_arch = "x86_64")] +use x86_64::VirtAddr; + +/// A contiguous range of virtual addresses with uniform protection +/// and backing semantics, owned by one address space. +#[derive(Debug, Copy, Clone)] +pub struct VirtualMemoryArea { + /// Inclusive start, page-aligned. + pub start: VirtAddr, + /// Exclusive end, page-aligned. + pub end: VirtAddr, + /// Protection bits the kernel must install when faulting in a page. + pub prot: VirtualMemoryAreaProt, +} + +impl VirtualMemoryArea { + pub fn new(start: VirtAddr, end: VirtAddr, prot: VirtualMemoryAreaProt) -> Self { + Self { start, end, prot } + } +} + +bitflags! { + #[repr(transparent)] + #[derive(Debug, Copy, Clone, Default)] + pub struct VirtualMemoryAreaProt: u32 { + /// Memory may not be accessed. + const NONE = 0; + /// Indicates that the memory region should be readable. + const READ = 1 << 0; + /// Indicates that the memory region should be writable. + const WRITE = 1 << 1; + /// Indicates that the memory region should be executable. + const EXECUTE = 1 << 2; + } +} + +// Bit 47 must equal bits 48..63 for an x86_64 *canonical* address; +// 0xf100_0000_0000 had bit 47 = 1 with the high bits all zero and +// triggered a const-eval panic in `VirtAddr::new`. Drop bit 47 so the +// constant is canonical on both architectures. +const HEAP_START_ADDR: VirtAddr = VirtAddr::new(0x7100_0000_0000); + +/// Creates a new virtual memory mapping of the `size` specified with +/// protection bits specified in `prot_flags`. +#[hermit_macro::system(errno)] +#[unsafe(no_mangle)] +pub extern "C" fn sys_mmap( + size: usize, + prot_flags: VirtualMemoryAreaProt, + ret: &mut *mut u8, +) -> i32 { + if *ret == core::ptr::null_mut() { + let current_task = core_scheduler().get_current_task(); + let current_task_borrowed = current_task.borrow(); + let mut guard = current_task_borrowed.vmas.write(); + + if let Some((_, vma)) = guard + .range((Bound::Unbounded, Bound::Included(HEAP_START_ADDR))) + .next_back() + { + if vma.end < HEAP_START_ADDR { + let new_vma = VirtualMemoryArea::new( + HEAP_START_ADDR, + HEAP_START_ADDR + size as u64, + prot_flags, + ); + guard.insert(HEAP_START_ADDR, new_vma); + *ret = HEAP_START_ADDR.as_mut_ptr(); + + return 0; + } else { + error!("Unable to create heap"); + + return -i32::from(Errno::Nomems); + } + } + } else { + // Extend an existing VMA whose `end` equals the user-supplied + // address. The caller passes the current upper bound of a region + // it owns; the kernel grows that VMA by `size` bytes — provided + // the next VMA (if any) starts far enough behind it. + let addr = VirtAddr::from_ptr(*ret); + let new_end = addr + size as u64; + + let current_task = core_scheduler().get_current_task(); + let current_task_borrowed = current_task.borrow(); + let mut guard = current_task_borrowed.vmas.write(); + + // 1. Locate the predecessor: the VMA with largest start < addr. + // For an extend request its `end` must match `addr` exactly. + let key = { + let Some((key, vma)) = guard.range(..addr).next_back() else { + return -i32::from(Errno::Inval); + }; + if vma.end != addr { + return -i32::from(Errno::Inval); + } + if !vma.prot.contains(prot_flags) { + return -i32::from(Errno::Inval); + } + *key + }; + + // 2. The extension must not run into the next VMA's start. + // `VirtAddr::new(u64::MAX)` panics on x86_64 (non-canonical), + // so model "no successor" with `Option` instead of a sentinel. + if let Some((&next_start, _)) = + guard.range((Bound::Excluded(key), Bound::Unbounded)).next() + && new_end > next_start + { + return -i32::from(Errno::Nomem); + } + + // 3. In-place extension. + guard.get_mut(&key).unwrap().end = new_end; + + return 0; + } + + return -i32::from(Errno::Inval); +} diff --git a/src/scheduler/task/mod.rs b/src/scheduler/task/mod.rs index 218454709e..a2df5c2d79 100644 --- a/src/scheduler/task/mod.rs +++ b/src/scheduler/task/mod.rs @@ -3,6 +3,8 @@ #[cfg(not(feature = "common-os"))] pub(crate) mod tls; +#[cfg(feature = "common-os")] +use alloc::collections::BTreeMap; use alloc::collections::{LinkedList, VecDeque}; use alloc::rc::Rc; use alloc::sync::Arc; @@ -18,7 +20,10 @@ use hashbrown::HashMap; #[cfg(not(feature = "common-os"))] use hermit_sync::OnceCell; use hermit_sync::RwSpinLock; +#[cfg(not(target_arch = "x86_64"))] use memory_addresses::VirtAddr; +#[cfg(target_arch = "x86_64")] +use x86_64::VirtAddr; #[cfg(not(feature = "common-os"))] use self::tls::Tls; @@ -32,6 +37,8 @@ use crate::env; use crate::fd::{Fd, RawFd, stdio}; #[cfg(not(feature = "common-os"))] use crate::fd::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; +#[cfg(feature = "common-os")] +use crate::mm::vma::VirtualMemoryArea; use crate::scheduler::CoreId; /// A reference-counted handle to a process's root page table. @@ -462,6 +469,8 @@ pub(crate) struct Task { /// when the user binary has a `PT_TLS` segment. #[cfg(feature = "common-os")] pub tls_template: Option>, + #[cfg(feature = "common-os")] + pub vmas: Arc>>, } pub(crate) trait TaskFrame { @@ -496,6 +505,8 @@ impl Task { root_page_table: Arc::new(RootPageTable::new(arch::create_new_root_page_table())), #[cfg(feature = "common-os")] tls_template: None, + #[cfg(feature = "common-os")] + vmas: Arc::new(RwSpinLock::new(BTreeMap::new())), } } @@ -562,6 +573,8 @@ impl Task { )), #[cfg(feature = "common-os")] tls_template: None, + #[cfg(feature = "common-os")] + vmas: Arc::new(RwSpinLock::new(BTreeMap::new())), } } @@ -580,6 +593,7 @@ impl Task { object_map: Arc>, RandomState>>>, root_page_table: Arc, tls_template: Option>, + vmas: Arc>>, ) -> Task { debug!("Creating user thread {tid} on core {core_id}"); Task { @@ -594,6 +608,7 @@ impl Task { object_map, root_page_table, tls_template, + vmas, } } @@ -612,6 +627,7 @@ impl Task { object_map: Arc>, RandomState>>>, root_page_table: Arc, tls_template: Option>, + vmas: Arc>>, ) -> Task { debug!("Creating forked task {tid} on core {core_id}"); Task { @@ -628,6 +644,7 @@ impl Task { tls: None, root_page_table, tls_template, + vmas, } } } diff --git a/src/syscalls/table.rs b/src/syscalls/table.rs index b72b69d7ac..5565d584e9 100644 --- a/src/syscalls/table.rs +++ b/src/syscalls/table.rs @@ -1,6 +1,7 @@ #[cfg(target_arch = "x86_64")] use core::arch::naked_asm; +use crate::mm::vma::sys_mmap; use crate::syscalls::*; /// Number of the system call `exit` @@ -135,6 +136,8 @@ const SYSNO_AVAILABLE_PARALLELISM: usize = 54; const SYSNO_GET_DENTS64: usize = 55; /// number of the system call `exec` const SYSNO_EXEC: usize = 56; +/// number of the system call `mmap` +const SYSNO_MMAP: usize = 57; /// Total number of system calls pub(crate) const NO_SYSCALLS: usize = 64; @@ -259,6 +262,7 @@ impl SyscallTable { table.handle[SYSNO_AVAILABLE_PARALLELISM] = sys_available_parallelism as *const _; table.handle[SYSNO_GET_DENTS64] = sys_getdents64 as *const _; table.handle[SYSNO_EXEC] = sys_exec as *const _; + table.handle[SYSNO_MMAP] = sys_mmap as *const _; table } From af340908eb405e9eacd9f877f0bc833772bc8a72 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Wed, 13 May 2026 12:03:01 +0200 Subject: [PATCH 30/48] arch: register VMAs for loaded image and per-thread TLS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit load_application now records one VMA for the [LOADER_START .. LOADER_START + code_size) range (RWX, matching what's actually mapped) and a second one for the per-process TLS region (RW). With these in place the page-fault handler can later resolve faults in user-loaded binaries against the VMA table. The x86_64 path takes the long way around two VirtAddr newtypes: this file imports memory_addresses::VirtAddr, but Task::vmas is keyed by x86_64::VirtAddr. Convert at the insert boundary with .into() — the two are structurally identical, so the conversion is free at runtime. --- src/arch/aarch64/kernel/mod.rs | 3 +++ src/arch/x86_64/kernel/mod.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/arch/aarch64/kernel/mod.rs b/src/arch/aarch64/kernel/mod.rs index 4ff291f4bb..db109d79a9 100644 --- a/src/arch/aarch64/kernel/mod.rs +++ b/src/arch/aarch64/kernel/mod.rs @@ -202,6 +202,7 @@ where use crate::mm::{FrameAlloc, PageRangeAllocator}; #[cfg(feature = "fork")] use crate::mm::frame_ref_inc; + use crate::mm::vma::*; // Each process has its own object map. let mut object_map = HashMap::>, RandomState>::with_hasher( @@ -241,6 +242,7 @@ where code_size / BasePageSize::SIZE as usize, flags, ); + core_scheduler().get_current_task().borrow_mut().vmas.write().insert(VirtAddr::from(LOADER_START), VirtualMemoryArea::new(VirtAddr::from(LOADER_START), VirtAddr::from(LOADER_START + code_size).align_up(BasePageSize::SIZE), VirtualMemoryAreaProt::READ|VirtualMemoryAreaProt::WRITE|VirtualMemoryAreaProt::EXECUTE)); let loader_start_ptr = ptr::with_exposed_provenance_mut(LOADER_START); let code_slice = unsafe { slice::from_raw_parts_mut(loader_start_ptr, code_size) }; @@ -272,6 +274,7 @@ where tls_memsz / BasePageSize::SIZE as usize, flags, ); + core_scheduler().get_current_task().borrow_mut().vmas.write().insert(tls_virt, VirtualMemoryArea::new(tls_virt, (tls_virt + tls_memsz).align_up(BasePageSize::SIZE), VirtualMemoryAreaProt::READ|VirtualMemoryAreaProt::WRITE)); let block = unsafe { slice::from_raw_parts_mut(tls_virt.as_mut_ptr(), tls_offset + tls_size as usize) }; for elem in block.iter_mut() { diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index c59a797cfa..cb36f592a1 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -212,6 +212,7 @@ where use crate::fd::stdio::*; #[cfg(feature = "fork")] use crate::mm::frame_ref_inc; + use crate::mm::vma::*; // each process has to provide its own object_map // => create a new one @@ -256,6 +257,22 @@ where code_size / BasePageSize::SIZE as usize, flags, ); + // VirtAddr's defined in vma.rs is `x86_64::VirtAddr` on x86_64 but + // this file's local `use` brings in `memory_addresses::VirtAddr`. + // Convert at the boundary so the BTreeMap-keyed insert type-checks. + { + let start: x86_64::VirtAddr = VirtAddr::from(LOADER_START).into(); + let end: x86_64::VirtAddr = + VirtAddr::from(LOADER_START + code_size).align_up(BasePageSize::SIZE).into(); + core_scheduler().get_current_task().borrow_mut().vmas.write().insert( + start, + VirtualMemoryArea::new( + start, + end, + VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE | VirtualMemoryAreaProt::EXECUTE, + ), + ); + } let loader_start_ptr = ptr::with_exposed_provenance_mut(LOADER_START); let code_slice = unsafe { slice::from_raw_parts_mut(loader_start_ptr, code_size) }; @@ -286,6 +303,19 @@ where tls_memsz / BasePageSize::SIZE as usize, flags, ); + { + let start: x86_64::VirtAddr = tls_virt.into(); + let end: x86_64::VirtAddr = + (tls_virt + tls_memsz).align_up(BasePageSize::SIZE).into(); + core_scheduler().get_current_task().borrow_mut().vmas.write().insert( + start, + VirtualMemoryArea::new( + start, + end, + VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE, + ), + ); + } let block = unsafe { slice::from_raw_parts_mut(tls_virt.as_mut_ptr(), tls_offset + tcb_size) }; for elem in block.iter_mut() { From 5fdf449287b8d8b2e00f81a8a31fbf662a079102 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Wed, 13 May 2026 12:03:43 +0200 Subject: [PATCH 31/48] arch: lazy-map user pages via VMA lookup in the fault handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a user-mode translation fault (no existing mapping, no COW marker) walk the current task's VMA tree, allocate a fresh frame and map it with permissions derived from VMA::prot. This is what makes anonymous sys_mmap regions actually faultable — the BTreeMap entry alone reserves VA space, the first access pulls in a frame. The x86_64 sibling additionally restores user GS via `swapgs` before returning. The COW path was already doing this; the new VMA branch forgot it, so iret left the kernel GS-base installed and the next FS/GS access in ring 3 read kernel memory — rusty_demo would freeze just after printing "Arguments:" because env::args() touches TLS. --- src/arch/aarch64/kernel/interrupts.rs | 38 +++++++++++++++++++++ src/arch/x86_64/mm/paging.rs | 48 ++++++++++++++++++++++++++- 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/src/arch/aarch64/kernel/interrupts.rs b/src/arch/aarch64/kernel/interrupts.rs index ff97adb627..6c8389fd61 100644 --- a/src/arch/aarch64/kernel/interrupts.rs +++ b/src/arch/aarch64/kernel/interrupts.rs @@ -242,6 +242,44 @@ pub(crate) extern "C" fn do_sync(state: &mut State) { return; } + #[cfg(feature = "common-os")] + { + use align_address::Align; + use core::ops::Bound; + use crate::mm::FrameAlloc; + use crate::mm::vma::VirtualMemoryAreaProt; + + let addr = VirtAddr::new(far).align_down(BasePageSize::SIZE); + let current_task = core_scheduler().get_current_task(); + let current_task_borrowed = current_task.borrow(); + let guard = current_task_borrowed.vmas.read(); + + if let Some((_, vma)) = guard.range((Bound::Unbounded, Bound::Included(addr))).next_back() { + if addr >= vma.start && addr < vma.end { + let layout = PageLayout::from_size_align(BasePageSize::SIZE as usize, BasePageSize::SIZE as usize).unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let physaddr = PhysAddr::from(frame_range.start()); + let mut flags = PageTableEntryFlags::empty(); + flags.normal().user(); + if vma.prot.contains(VirtualMemoryAreaProt::WRITE) { + flags.writable(); + } + if vma.prot.contains(VirtualMemoryAreaProt::EXECUTE) { + flags.execute_disable(); + } + + paging::map::( + addr, + physaddr, + 1, + flags, + ); + + return; + } + } + } + let kind = dfsc_kind(dfsc); let access = if is_write { "write" } else { "read" }; error!("Current stack pointer {state:p}"); diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index 4a33bdbf0e..41a2f85716 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -545,7 +545,6 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( increment_irq_counter(14); let faulting_addr = Cr2::read().unwrap(); - #[cfg(feature = "fork")] let virtaddr = faulting_addr.align_down(BasePageSize::SIZE); debug!( @@ -595,6 +594,53 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( } } + { + use core::ops::Bound; + use crate::mm::FrameAlloc; + use crate::mm::vma::VirtualMemoryAreaProt; + + let current_task = core_scheduler().get_current_task(); + let current_task_borrowed = current_task.borrow(); + let guard = current_task_borrowed.vmas.read(); + + if let Some((_, vma)) = guard.range((Bound::Unbounded, Bound::Included(virtaddr))).next_back() { + if virtaddr >= vma.start && virtaddr < vma.end { + let layout = PageLayout::from_size_align(BasePageSize::SIZE as usize, BasePageSize::SIZE as usize).unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let physaddr = PhysAddr::from(frame_range.start()); + let mut flags = PageTableEntryFlags::empty(); + flags.normal().user(); + if vma.prot.contains(VirtualMemoryAreaProt::WRITE) { + flags.writable(); + } + if vma.prot.contains(VirtualMemoryAreaProt::EXECUTE) { + flags.execute_disable(); + } + + // `virtaddr` is `x86_64::VirtAddr` (from `Cr2::read`); the + // `map` helper signs the parameter as `memory_addresses::VirtAddr`. + map::( + virtaddr.into(), + physaddr, + 1, + flags, + ); + + // Restore user GS before returning to ring 3; the COW path + // above does the same. Without this, iret leaves the kernel + // GS-base installed, the user's next FS/GS access reads + // kernel state and the program either misbehaves or wedges. + if swapped_gs { + unsafe { + core::arch::asm!("swapgs", options(nostack)); + } + } + + return; + } + } + } + // A spawned user thread whose entry function returns comes back here // with RIP=0: the `thread_start` wrapper in std's hermit pal just // returns, and the kernel-crafted return slot at the top of the user From 2320383eb0acfacd1100a2bf5183f264573415a4 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Wed, 13 May 2026 12:04:35 +0200 Subject: [PATCH 32/48] x86_64: convert VirtAddr at last/user_stack_pointer assignments Task::{last,user}_stack_pointer is x86_64::VirtAddr; TaskStacks::get_*() returns memory_addresses::VirtAddr. The two newtypes have identical layouts but the type system rejects mixing them. Add .into() at the two assignment sites in create_user_stack_frame and create_stack_frame. --- src/arch/x86_64/kernel/scheduler.rs | 17 ++++++++++------ src/executor/device.rs | 2 +- src/mm/vma.rs | 10 +++++----- src/scheduler/mod.rs | 30 ++++++++++++++++++++++++++++- 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/src/arch/x86_64/kernel/scheduler.rs b/src/arch/x86_64/kernel/scheduler.rs index 8b4ad85803..11dad67269 100644 --- a/src/arch/x86_64/kernel/scheduler.rs +++ b/src/arch/x86_64/kernel/scheduler.rs @@ -355,10 +355,14 @@ impl Task { // FS.Base for the new user-space thread (its private TLS area). (*state).fs = tls_fs_base; - self.last_stack_pointer = stack; - self.user_stack_pointer = self.stacks.get_user_stack() + // `stack` / `get_user_stack()` produce `memory_addresses::VirtAddr`, + // while `Task::{last,user}_stack_pointer` is `x86_64::VirtAddr` — + // convert at the assignment boundary. + self.last_stack_pointer = stack.into(); + self.user_stack_pointer = (self.stacks.get_user_stack() + self.stacks.get_user_stack_size() - - TaskStacks::MARKER_SIZE; + - TaskStacks::MARKER_SIZE) + .into(); // rdx is used by task_start_user as the new user-mode RSP (*state).rdx = self.user_stack_pointer.as_u64() - mem::size_of::() as u64; @@ -398,10 +402,11 @@ impl TaskFrame for Task { (*state).rflags = 0x1202u64; // Set the task's stack pointer entry to the stack we have just crafted. - self.last_stack_pointer = stack; - self.user_stack_pointer = self.stacks.get_user_stack() + self.last_stack_pointer = stack.into(); + self.user_stack_pointer = (self.stacks.get_user_stack() + self.stacks.get_user_stack_size() - - TaskStacks::MARKER_SIZE; + - TaskStacks::MARKER_SIZE) + .into(); // rdx is required to initialize the stack (*state).rdx = self.user_stack_pointer.as_u64() - size_of::() as u64; diff --git a/src/executor/device.rs b/src/executor/device.rs index 5638300580..3ac1360bdb 100644 --- a/src/executor/device.rs +++ b/src/executor/device.rs @@ -14,8 +14,8 @@ use smoltcp::socket::dns; use smoltcp::wire::{EthernetAddress, HardwareAddress, IpCidr, Ipv4Address, Ipv4Cidr}; use super::network::{NetworkInterface, NetworkState}; -use crate::arch::kernel::systemtime; use crate::arch; +use crate::arch::kernel::systemtime; #[cfg(feature = "write-pcap-file")] use crate::drivers::Driver; use crate::drivers::net::NetworkDriver; diff --git a/src/mm/vma.rs b/src/mm/vma.rs index 3566439c0f..d1fce04372 100644 --- a/src/mm/vma.rs +++ b/src/mm/vma.rs @@ -1,12 +1,13 @@ use core::ops::Bound; -use crate::core_local::core_scheduler; -use crate::errno::Errno; #[cfg(not(target_arch = "x86_64"))] use memory_addresses::VirtAddr; #[cfg(target_arch = "x86_64")] use x86_64::VirtAddr; +use crate::core_local::core_scheduler; +use crate::errno::Errno; + /// A contiguous range of virtual addresses with uniform protection /// and backing semantics, owned by one address space. #[derive(Debug, Copy, Clone)] @@ -77,7 +78,7 @@ pub extern "C" fn sys_mmap( } else { error!("Unable to create heap"); - return -i32::from(Errno::Nomems); + return -i32::from(Errno::Nomems); } } } else { @@ -110,8 +111,7 @@ pub extern "C" fn sys_mmap( // 2. The extension must not run into the next VMA's start. // `VirtAddr::new(u64::MAX)` panics on x86_64 (non-canonical), // so model "no successor" with `Option` instead of a sentinel. - if let Some((&next_start, _)) = - guard.range((Bound::Excluded(key), Bound::Unbounded)).next() + if let Some((&next_start, _)) = guard.range((Bound::Excluded(key), Bound::Unbounded)).next() && new_end > next_start { return -i32::from(Errno::Nomem); diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index 42fbe8373e..92650d54af 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -14,9 +14,13 @@ use ahash::RandomState; use crossbeam_utils::Backoff; use hashbrown::{HashMap, hash_map}; use hermit_sync::*; +#[cfg(all(feature = "common-os", not(target_arch = "x86_64")))] +use memory_addresses::VirtAddr; #[cfg(target_arch = "riscv64")] use riscv::register::sstatus; use timer_interrupts::TimerList; +#[cfg(all(feature = "common-os", target_arch = "x86_64"))] +use x86_64::VirtAddr; use crate::arch::kernel; use crate::arch::kernel::core_local::*; @@ -29,6 +33,9 @@ use crate::arch::kernel::{get_processor_count, interrupts}; use crate::errno::Errno; use crate::fd::{Fd, RawFd}; use crate::io; +use crate::kernel::scheduler::TaskStacks; +#[cfg(feature = "common-os")] +use crate::mm::vma::VirtualMemoryArea; use crate::scheduler::task::*; pub mod task; @@ -230,6 +237,8 @@ struct NewTask { /// `TlsTemplate`. Zero means "do not install a thread pointer". #[cfg(feature = "common-os")] tls_base: u64, + #[cfg(feature = "common-os")] + vmas: Arc>>, } impl From for Task { @@ -248,6 +257,8 @@ impl From for Task { tls_template, #[cfg(feature = "common-os")] tls_base, + #[cfg(feature = "common-os")] + vmas, } = value; #[cfg(feature = "common-os")] @@ -261,6 +272,7 @@ impl From for Task { object_map, root_page_table, tls_template, + vmas, ); task.create_user_stack_frame(func, arg, tls_base); return task; @@ -298,6 +310,8 @@ impl PerCoreScheduler { tls_template: None, #[cfg(feature = "common-os")] tls_base: 0, + #[cfg(feature = "common-os")] + vmas: Arc::new(RwSpinLock::new(BTreeMap::new())), }; // Add it to the task lists. @@ -365,13 +379,14 @@ impl PerCoreScheduler { // immediately visible to every thread in this process. let stacks = TaskStacks::new(stack_size); - let (root_page_table, object_map, tls_template) = { + let (root_page_table, object_map, tls_template, vmas) = { let current = core_scheduler().get_current_task(); let borrowed = current.borrow(); ( borrowed.root_page_table.clone(), borrowed.object_map.clone(), borrowed.tls_template.clone(), + borrowed.vmas.clone(), ) }; @@ -397,6 +412,7 @@ impl PerCoreScheduler { thread_of: Some(root_page_table), tls_template, tls_base, + vmas, }; let wakeup = { @@ -1155,7 +1171,10 @@ pub fn join(id: TaskId) -> Result<(), ()> { all(feature = "common-os", feature = "fork") ))] pub unsafe fn fork() -> TaskId { + #[cfg(not(target_arch = "x86_64"))] use memory_addresses::VirtAddr; + #[cfg(target_arch = "x86_64")] + use x86_64::VirtAddr; use crate::arch::{prepare_fork_child_stack, prepare_mem_copy_on_write}; @@ -1214,6 +1233,14 @@ pub unsafe fn fork() -> TaskId { .borrow() .tls_template .clone(); + let parent_vmas = Arc::new(RwSpinLock::new( + core_scheduler() + .get_current_task() + .borrow() + .vmas + .read() + .clone(), + )); let child_task = Task::new_fork( tid, @@ -1228,6 +1255,7 @@ pub unsafe fn fork() -> TaskId { child_root_page_table, )), parent_tls_template, + parent_vmas, ); let wakeup = { From ef24432b7936b8619e8480485fc0ee266bfcd73e Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Thu, 14 May 2026 11:07:19 +0200 Subject: [PATCH 33/48] mm/vma: move user heap into L0[USER_L0_INDEX] for proper fork isolation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HEAP_START_ADDR sat at 0x7100_0000_0000, which falls into L0[226] — outside the slot that aarch64's mark_user_pages_copy_on_write, copy_current_root_page_table and clear_l0 actually walk. The latter three only touch `USER_L0_INDEX = LOADER_START >> 39 = 2`, so the heap was effectively a *shared* mapping: parent and child were reading and writing the same physical frames. The fork_bench warmup hit this immediately — the child's first user-space write (`*pid = 0`) overwrote the parent's `pids[1]`, the parent then called `waitpid(0)`, queued on the idle task, and hung. Move HEAP_START_ADDR to 0x0140_0000_0000, well above any loaded LOADER image but still inside L0[2]. The constant remains canonical on x86_64 (bit 47 = 0). Also drop a stray Errno typo (`Nomems` → `Nomem`) the change uncovered, and tidy the riscv / x86_64 VirtAddr import order while we're here. --- src/mm/vma.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/mm/vma.rs b/src/mm/vma.rs index d1fce04372..322081aeaf 100644 --- a/src/mm/vma.rs +++ b/src/mm/vma.rs @@ -41,11 +41,20 @@ bitflags! { } } -// Bit 47 must equal bits 48..63 for an x86_64 *canonical* address; -// 0xf100_0000_0000 had bit 47 = 1 with the high bits all zero and -// triggered a const-eval panic in `VirtAddr::new`. Drop bit 47 so the -// constant is canonical on both architectures. -const HEAP_START_ADDR: VirtAddr = VirtAddr::new(0x7100_0000_0000); +// Place the user heap inside the L0 slot that covers LOADER_START +// (`USER_L0_INDEX = LOADER_START >> 39 = 2`). On aarch64 only that +// slot is COW-marked at fork and deep-copied by +// `copy_current_root_page_table`; everything else is treated as a +// shared kernel mapping. Putting the heap outside L0[2] leaves it +// shared between parent and child forks, which manifests as the +// child overwriting the parent's heap on its first user-space write. +// +// LOADER_START is 0x0100_0000_0000; binaries currently take well +// under 256 GiB, so 0x0140_0000_0000 sits comfortably above any +// loaded image while still falling inside L0[2] (which ends at +// 0x0180_0000_0000). The address is canonical on x86_64 as well +// (bit 47 = 0). +const HEAP_START_ADDR: VirtAddr = VirtAddr::new(0x0140_0000_0000); /// Creates a new virtual memory mapping of the `size` specified with /// protection bits specified in `prot_flags`. @@ -78,7 +87,7 @@ pub extern "C" fn sys_mmap( } else { error!("Unable to create heap"); - return -i32::from(Errno::Nomems); + return -i32::from(Errno::Nomem); } } } else { From 84bcb248d7a79dc6c1106460c66e8169f90f42b8 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Thu, 14 May 2026 11:07:58 +0200 Subject: [PATCH 34/48] arch: zero and refcount lazily-mapped user pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the VMA-aware page-fault handler maps a fresh frame for an anonymous user mapping, FrameAlloc can hand back a recycled frame whose previous contents are stale. The user code then sees garbage where it expects zero — most visibly in std startup paths where allocator metadata, futex words and once-flags decide control flow. Zero the page through its new VA before returning. In addition, register the frame with the COW refcount table so a subsequent fork() can correctly track sharing. Without this, the child's clear_l0/clear_user_space would `frame_ref_dec` a frame that was never `inc`'d, hit the "no refcount entry" warn path, and miss a free. Symmetric change in the aarch64 do_sync data-abort path and the x86_64 #PF handler. Both gate frame_ref_inc on `feature = "fork"` so the refcount table only exists when fork is enabled. clear_user_space is called from sys_exec right before the new ELF is loaded, and is supposed to leave the user side of the address space in a "freshly created task" state. Two pieces were missing: 1. The VMA list still contained the old image's entries (LOADER_START code segment, TLS region, mmap'd heap). The new load_application would then either tip over inserts that overlapped existing ranges, or — worse — the heap VMA would survive while its backing L0 slot had just been wiped, leaving the next sys_malloc to see a stale mapping. 2. File descriptors > 2 were inherited across exec. POSIX-style behaviour is to keep stdin/stdout/stderr and close everything else; emulate that by retaining `k <= STDERR_FILENO`. Done symmetrically on aarch64 and x86_64. --- src/arch/aarch64/kernel/interrupts.rs | 7 +++++++ src/arch/aarch64/mm/paging.rs | 13 +++++++++++++ src/arch/x86_64/mm/paging.rs | 21 +++++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/src/arch/aarch64/kernel/interrupts.rs b/src/arch/aarch64/kernel/interrupts.rs index 6c8389fd61..96c0745b6a 100644 --- a/src/arch/aarch64/kernel/interrupts.rs +++ b/src/arch/aarch64/kernel/interrupts.rs @@ -275,6 +275,13 @@ pub(crate) extern "C" fn do_sync(state: &mut State) { flags, ); + // clear page + let slice = unsafe { core::slice::from_raw_parts_mut(addr.as_mut_ptr() as *mut u8, BasePageSize::SIZE as usize) }; + slice.fill(0); + + #[cfg(feature = "fork")] + crate::mm::frame_ref_inc(physaddr); + return; } } diff --git a/src/arch/aarch64/mm/paging.rs b/src/arch/aarch64/mm/paging.rs index 0d6f8ae319..1f00112e2f 100644 --- a/src/arch/aarch64/mm/paging.rs +++ b/src/arch/aarch64/mm/paging.rs @@ -1120,8 +1120,21 @@ pub fn drop_user_space(l0_phys: usize) { /// Clear the user-space portion of the currently active address space. #[cfg(feature = "common-os")] pub fn clear_user_space() { + use crate::fd::STDERR_FILENO; + use crate::core_scheduler; use aarch64_cpu::registers::{Readable, TTBR0_EL1}; + core_scheduler() + .get_current_task() + .borrow() + .vmas + .write() + .clear(); + core_scheduler() + .get_current_task_object_map() + .write() + .retain(|&k, _| k <= STDERR_FILENO); + let l0_phys = TTBR0_EL1.get_baddr() as usize; debug!("Clear the user space at L0 {l0_phys:#x}"); diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index 41a2f85716..114399a5d6 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -448,6 +448,20 @@ pub fn drop_user_space(pml4_phys: usize) { #[cfg(feature = "common-os")] pub fn clear_user_space() { + use crate::fd::STDERR_FILENO; + use crate::core_scheduler; + + core_scheduler() + .get_current_task() + .borrow() + .vmas + .write() + .clear(); + core_scheduler() + .get_current_task_object_map() + .write() + .retain(|&k, _| k <= STDERR_FILENO); + let (p4_frame, _) = Cr3::read_raw(); let pml4_phys = p4_frame.start_address().as_u64() as usize; @@ -626,6 +640,13 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( flags, ); + // clear page + let slice = unsafe { core::slice::from_raw_parts_mut(virtaddr.as_mut_ptr() as *mut u8, BasePageSize::SIZE as usize) }; + slice.fill(0); + + #[cfg(feature = "fork")] + crate::mm::frame_ref_inc(physaddr); + // Restore user GS before returning to ring 3; the COW path // above does the same. Without this, iret leaves the kernel // GS-base installed, the user's next FS/GS access reads From facb6ce4199387b015fb28570cedb6e1a71ad2a1 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Thu, 14 May 2026 21:39:49 +0200 Subject: [PATCH 35/48] arch: tighten user stack layout under common-os Three follow-ups to the recent VMA / fork work, two of them load-bearing for spawn correctness on aarch64: * aarch64: move USER_STACK into L0[USER_L0_INDEX]. It used to sit at 0x0840_0000_0000 - USER_STACK_SIZE (= L0[16]). create_new_root_page_table deep-copies only L0[USER_L0_INDEX] and inherits every other L0 entry verbatim, so the L1/L2/L3 chain backing L0[16] was shared between spawner and spawnee. The first paging::map in the child rewrote the parent's stack PTEs; the parent then returned from join() with a corrupted return address (PC=0x1, EC=0x22). Place USER_STACK at the top of L0[2] instead. * x86_64: stop padding the code allocation with USER_STACK_SIZE in load_application. The user stack is allocated separately in jump_to_user_land since the VMA split, so the addend just reserves and refcounts unused frames. --- src/arch/aarch64/kernel/mod.rs | 57 ++++++++++++++++++++++++-------- src/arch/aarch64/mm/paging.rs | 6 ++-- src/arch/x86_64/kernel/mod.rs | 59 ++++++++++++++++++++++++++-------- src/mm/vma.rs | 26 +++++++++++++-- 4 files changed, 117 insertions(+), 31 deletions(-) diff --git a/src/arch/aarch64/kernel/mod.rs b/src/arch/aarch64/kernel/mod.rs index db109d79a9..4642f5ba98 100644 --- a/src/arch/aarch64/kernel/mod.rs +++ b/src/arch/aarch64/kernel/mod.rs @@ -173,9 +173,18 @@ pub fn print_statistics() { } #[cfg(feature = "common-os")] -pub(crate) const LOADER_START: usize = 0x0100_0000_0000; +pub(crate) const USER_START: VirtAddr = VirtAddr::new(0x0100_0000_0000); +// Place the user stack at the top of the L0 slot that backs USER_START +// (L0[USER_L0_INDEX] = L0[2], spanning 0x0100_0000_0000..0x0180_0000_0000). +// Only this slot is deep-copied by `create_new_root_page_table`; every +// other L0 entry is inherited verbatim from the parent, so a stack +// outside L0[2] would share its L1/L2/L3 tables with the spawning +// process and the first `paging::map` call from the child would +// silently overwrite the parent's stack mapping. #[cfg(feature = "common-os")] -const LOADER_STACK_SIZE: usize = 0x8000; +const USER_STACK: VirtAddr = VirtAddr::new(0x0180_0000_0000 - USER_STACK_SIZE as u64); +#[cfg(feature = "common-os")] +const USER_STACK_SIZE: usize = 0x8000; /// Map the user-mode binary into the address space and run the ELF-loader /// closure against the freshly-mapped pages. Mirrors the x86_64 sibling @@ -225,7 +234,7 @@ where } core_scheduler().set_current_task_object_map(Arc::new(RwSpinLock::new(object_map))); - let code_size = (code_size as usize + LOADER_STACK_SIZE).align_up(BasePageSize::SIZE as usize); + let code_size = (code_size as usize).align_up(BasePageSize::SIZE as usize); let layout = PageLayout::from_size_align(code_size, BasePageSize::SIZE as usize).unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let physaddr = PhysAddr::from(frame_range.start()); @@ -237,14 +246,14 @@ where let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().user().execute_enable(); paging::map::( - VirtAddr::from(LOADER_START), + USER_START, physaddr, code_size / BasePageSize::SIZE as usize, flags, ); - core_scheduler().get_current_task().borrow_mut().vmas.write().insert(VirtAddr::from(LOADER_START), VirtualMemoryArea::new(VirtAddr::from(LOADER_START), VirtAddr::from(LOADER_START + code_size).align_up(BasePageSize::SIZE), VirtualMemoryAreaProt::READ|VirtualMemoryAreaProt::WRITE|VirtualMemoryAreaProt::EXECUTE)); + core_scheduler().get_current_task().borrow_mut().vmas.write().insert(VirtAddr::from(USER_START), VirtualMemoryArea::new(VirtAddr::from(USER_START), VirtAddr::from(USER_START + code_size).align_up(BasePageSize::SIZE), VirtualMemoryAreaProt::READ|VirtualMemoryAreaProt::WRITE|VirtualMemoryAreaProt::EXECUTE, MemoryType::CODE)); - let loader_start_ptr = ptr::with_exposed_provenance_mut(LOADER_START); + let loader_start_ptr = ptr::with_exposed_provenance_mut(USER_START.as_usize()); let code_slice = unsafe { slice::from_raw_parts_mut(loader_start_ptr, code_size) }; if tls_size > 0 { @@ -267,14 +276,15 @@ where let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().user().execute_disable(); - let tls_virt = VirtAddr::from(LOADER_START + code_size + BasePageSize::SIZE as usize); + let tls_virt = VirtAddr::from(USER_START + code_size + BasePageSize::SIZE as usize); paging::map::( tls_virt, physaddr, tls_memsz / BasePageSize::SIZE as usize, flags, ); - core_scheduler().get_current_task().borrow_mut().vmas.write().insert(tls_virt, VirtualMemoryArea::new(tls_virt, (tls_virt + tls_memsz).align_up(BasePageSize::SIZE), VirtualMemoryAreaProt::READ|VirtualMemoryAreaProt::WRITE)); + core_scheduler().get_current_task().borrow_mut().vmas.write().insert(tls_virt, VirtualMemoryArea::new(tls_virt, (tls_virt + tls_memsz).align_up(BasePageSize::SIZE), VirtualMemoryAreaProt::READ|VirtualMemoryAreaProt::WRITE, MemoryType::TLS)); + let block = unsafe { slice::from_raw_parts_mut(tls_virt.as_mut_ptr(), tls_offset + tls_size as usize) }; for elem in block.iter_mut() { @@ -359,20 +369,41 @@ fn set_user_tpidr_el0(value: u64) { /// new PC, SPSR_EL1 the new PSTATE (mode bits select EL0t), and SP_EL0 the /// user stack. Per AAPCS64, `argc` lives in `x0` and `argv` in `x1`. #[cfg(feature = "common-os")] -pub unsafe fn jump_to_user_land(entry_point: usize, code_size: usize, arg: alloc::vec::Vec<&str>) -> ! { +pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) -> ! { use alloc::ffi::CString; use align_address::Align; + use free_list::PageLayout; use crate::arch::aarch64::kernel::scheduler::TaskStacks; + use crate::arch::aarch64::mm::paging::{self, PageTableEntryFlags}; + use crate::mm::{FrameAlloc, PageRangeAllocator}; + #[cfg(feature = "fork")] + use crate::mm::frame_ref_inc; + use crate::mm::vma::*; debug!("Create new file descriptor table"); core_scheduler().recreate_objmap().unwrap(); - let entry_point: usize = LOADER_START | entry_point; - let stack_top: usize = LOADER_START - + (code_size + LOADER_STACK_SIZE).align_up(BasePageSize::SIZE.try_into().unwrap()) - - TaskStacks::MARKER_SIZE; + let entry_point: usize = USER_START.as_usize() | entry_point; + let stack_top: usize = USER_STACK.as_usize() + USER_STACK_SIZE - TaskStacks::MARKER_SIZE; + + let layout = PageLayout::from_size(USER_STACK_SIZE).unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let phys_addr = PhysAddr::from(frame_range.start()); + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user(); + paging::map::( + USER_STACK, + phys_addr, + USER_STACK_SIZE / BasePageSize::SIZE as usize, + flags, + ); + core_scheduler().get_current_task().borrow_mut().vmas.write().insert(USER_STACK, VirtualMemoryArea::new(USER_STACK, USER_STACK+USER_STACK_SIZE, VirtualMemoryAreaProt::READ|VirtualMemoryAreaProt::WRITE, MemoryType::STACK)); + #[cfg(feature = "fork")] + for i in 0..USER_STACK_SIZE / BasePageSize::SIZE as usize { + frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); + } // Place the argv pointer array on the user stack. let stack_pointer = stack_top - arg.len() * mem::size_of::<*mut u8>(); diff --git a/src/arch/aarch64/mm/paging.rs b/src/arch/aarch64/mm/paging.rs index 1f00112e2f..e29772612d 100644 --- a/src/arch/aarch64/mm/paging.rs +++ b/src/arch/aarch64/mm/paging.rs @@ -930,7 +930,7 @@ pub fn mark_user_pages_copy_on_write() { let l0_phys = TTBR0_EL1.get_baddr() as usize; let l0 = unsafe { &mut *ptr::with_exposed_provenance_mut::>(l0_phys) }; - // User pages live exclusively in the L0 slot covering LOADER_START. + // User pages live exclusively in the L0 slot covering USER_START. // All other L0 entries (kernel image, heap, per-task stacks…) are // shared kernel mappings and must not be touched here. for l0_idx in [USER_L0_INDEX].iter().copied() { @@ -1149,9 +1149,9 @@ pub fn clear_user_space() { /// (entry 0) and installs a self-reference at entry 511. User-space entries /// (1..511) are left empty. /// Index of the L0 entry that backs the user-space load area -/// (`LOADER_START = 0x0100_0000_0000`, bits 47..39 = 2). +/// (`USER_START = 0x0100_0000_0000`, bits 47..39 = 2). #[cfg(feature = "common-os")] -const USER_L0_INDEX: usize = (crate::arch::aarch64::kernel::LOADER_START >> 39) & 0x1ff; +const USER_L0_INDEX: usize = (crate::arch::aarch64::kernel::USER_START.as_usize() >> 39) & 0x1ff; #[cfg(feature = "common-os")] pub fn create_new_root_page_table() -> usize { diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index cb36f592a1..bc8fa3565f 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -189,9 +189,11 @@ unsafe extern "C" fn pre_init(boot_info: Option<&'static RawBootInfo>, cpu_id: u } #[cfg(feature = "common-os")] -const LOADER_START: usize = 0x0100_0000_0000; +pub(crate) const USER_START: VirtAddr = VirtAddr::new(0x0100_0000_0000); #[cfg(feature = "common-os")] -const LOADER_STACK_SIZE: usize = 0x8000; +const USER_STACK: VirtAddr = VirtAddr::new(0x0180_0000_0000 - USER_STACK_SIZE as u64); +#[cfg(feature = "common-os")] +const USER_STACK_SIZE: usize = 0x8000; #[cfg(feature = "common-os")] pub fn load_application(code_size: u64, tls_size: u64, func: F) -> Result<(), ()> @@ -240,7 +242,7 @@ where } core_scheduler().set_current_task_object_map(Arc::new(RwSpinLock::new(object_map))); - let code_size = (code_size as usize + LOADER_STACK_SIZE).align_up(BasePageSize::SIZE as usize); + let code_size = (code_size as usize).align_up(BasePageSize::SIZE as usize); let layout = PageLayout::from_size_align(code_size, BasePageSize::SIZE as usize).unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let physaddr = PhysAddr::from(frame_range.start()); @@ -252,7 +254,7 @@ where let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().user().execute_enable(); paging::map::( - VirtAddr::from(LOADER_START), + VirtAddr::from(USER_START), physaddr, code_size / BasePageSize::SIZE as usize, flags, @@ -261,20 +263,21 @@ where // this file's local `use` brings in `memory_addresses::VirtAddr`. // Convert at the boundary so the BTreeMap-keyed insert type-checks. { - let start: x86_64::VirtAddr = VirtAddr::from(LOADER_START).into(); + let start: x86_64::VirtAddr = VirtAddr::from(USER_START).into(); let end: x86_64::VirtAddr = - VirtAddr::from(LOADER_START + code_size).align_up(BasePageSize::SIZE).into(); + VirtAddr::from(USER_START + code_size).align_up(BasePageSize::SIZE).into(); core_scheduler().get_current_task().borrow_mut().vmas.write().insert( start, VirtualMemoryArea::new( start, end, VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE | VirtualMemoryAreaProt::EXECUTE, + MemoryType::CODE, ), ); } - let loader_start_ptr = ptr::with_exposed_provenance_mut(LOADER_START); + let loader_start_ptr = ptr::with_exposed_provenance_mut(USER_START.as_usize()); let code_slice = unsafe { slice::from_raw_parts_mut(loader_start_ptr, code_size) }; if tls_size > 0 { @@ -296,13 +299,14 @@ where let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().user().execute_disable(); - let tls_virt = VirtAddr::from(LOADER_START + code_size + BasePageSize::SIZE as usize); + let tls_virt = VirtAddr::from(USER_START.as_usize() + code_size + BasePageSize::SIZE as usize); paging::map::( tls_virt, physaddr, tls_memsz / BasePageSize::SIZE as usize, flags, ); + { let start: x86_64::VirtAddr = tls_virt.into(); let end: x86_64::VirtAddr = @@ -313,9 +317,11 @@ where start, end, VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE, + MemoryType::TLS ), ); } + let block = unsafe { slice::from_raw_parts_mut(tls_virt.as_mut_ptr(), tls_offset + tcb_size) }; for elem in block.iter_mut() { @@ -355,21 +361,48 @@ where } #[cfg(feature = "common-os")] -pub unsafe fn jump_to_user_land(entry_point: usize, code_size: usize, arg: alloc::vec::Vec<&str>) -> ! { +pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) -> ! { use alloc::ffi::CString; use align_address::Align; + use free_list::PageLayout; use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; + use x86_64::structures::paging::PageTableFlags as PageTableEntryFlags; + use crate::arch::x86_64::mm::paging::PageTableEntryFlagsExt; use crate::arch::x86_64::kernel::scheduler::TaskStacks; + use crate::mm::{FrameAlloc, PageRangeAllocator}; + use crate::arch::mm::paging; + #[cfg(feature = "fork")] + use crate::mm::frame_ref_inc; + use crate::mm::vma::*; debug!("Create new file descriptor table"); core_scheduler().recreate_objmap().unwrap(); - let entry_point: usize = LOADER_START | entry_point; - let stack_pointer: usize = LOADER_START - + (code_size + LOADER_STACK_SIZE).align_up(BasePageSize::SIZE.try_into().unwrap()) - - 8; + let entry_point: usize = USER_START.as_usize() | entry_point; + let stack_pointer: usize = USER_STACK.as_usize() + USER_STACK_SIZE - 8; + + let layout = PageLayout::from_size(USER_STACK_SIZE).unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let phys_addr = PhysAddr::from(frame_range.start()); + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user(); + paging::map::( + USER_STACK, + phys_addr, + USER_STACK_SIZE / BasePageSize::SIZE as usize, + flags, + ); + { + let start: x86_64::VirtAddr = USER_STACK.into(); + let end: x86_64::VirtAddr = (USER_STACK+USER_STACK_SIZE).into(); + core_scheduler().get_current_task().borrow_mut().vmas.write().insert(start, VirtualMemoryArea::new(start, end, VirtualMemoryAreaProt::READ|VirtualMemoryAreaProt::WRITE, MemoryType::STACK)); + } + #[cfg(feature = "fork")] + for i in 0..USER_STACK_SIZE / BasePageSize::SIZE as usize { + frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); + } let stack_pointer = stack_pointer - 128 /* red zone */ - arg.len() * size_of::<*mut u8>(); let stack_ptr = ptr::with_exposed_provenance_mut::<*mut u8>(stack_pointer); diff --git a/src/mm/vma.rs b/src/mm/vma.rs index 322081aeaf..a9daaff983 100644 --- a/src/mm/vma.rs +++ b/src/mm/vma.rs @@ -18,14 +18,35 @@ pub struct VirtualMemoryArea { pub end: VirtAddr, /// Protection bits the kernel must install when faulting in a page. pub prot: VirtualMemoryAreaProt, + /// Description of the memory type + #[allow(unused)] + pub mem_type: MemoryType, } impl VirtualMemoryArea { - pub fn new(start: VirtAddr, end: VirtAddr, prot: VirtualMemoryAreaProt) -> Self { - Self { start, end, prot } + pub fn new( + start: VirtAddr, + end: VirtAddr, + prot: VirtualMemoryAreaProt, + mem_type: MemoryType, + ) -> Self { + Self { + start, + end, + prot, + mem_type, + } } } +#[derive(Debug, Copy, Clone)] +pub enum MemoryType { + HEAP, + STACK, + CODE, + TLS, +} + bitflags! { #[repr(transparent)] #[derive(Debug, Copy, Clone, Default)] @@ -79,6 +100,7 @@ pub extern "C" fn sys_mmap( HEAP_START_ADDR, HEAP_START_ADDR + size as u64, prot_flags, + MemoryType::HEAP, ); guard.insert(HEAP_START_ADDR, new_vma); *ret = HEAP_START_ADDR.as_mut_ptr(); From f1c07018851b6fa197ead28f57e0acf3f914379e Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Mon, 18 May 2026 22:28:10 +0200 Subject: [PATCH 36/48] scheduler: track a process id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kernel already gave every task its own `TaskId`, but had no notion of "this thread belongs to that process" — `sys_getpid()` returned the calling thread's `tid`, so every thread of a multi-threaded program saw a different pid. Carry a `pid` on every `Task` (common-os only): * Main thread of a new process (`Task::new`, `Task::new_idle`, `Task::new_fork`): `pid = tid`. * Sibling thread (`Task::new_thread`): `pid` inherited from the spawning thread. * `NewTask::thread_of` is widened from `Option>` to `Option<(Arc, ProcessId)>` so the parent's pid travels with the page table; `spawn_thread` captures it from the current task. --- src/scheduler/mod.rs | 16 +++++---- src/scheduler/task/mod.rs | 69 +++++++++++++++++++++++++++++++++++---- src/syscalls/tasks.rs | 13 +++++--- 3 files changed, 80 insertions(+), 18 deletions(-) diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index 92650d54af..7aadceaa4f 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -222,11 +222,11 @@ struct NewTask { stacks: TaskStacks, object_map: Arc>, RandomState>>>, /// When `Some`, the new task is a user-space thread that shares the - /// given root page table with its parent process. When - /// `None`, the task is a regular kernel-mode task with a fresh - /// address space. + /// given root page table with its parent process and inherits the + /// parent's `pid`. When `None`, the task is a regular kernel-mode + /// task with a fresh address space; its `pid` equals its `tid`. #[cfg(feature = "common-os")] - thread_of: Option>, + thread_of: Option<(Arc, task::ProcessId)>, /// Per-process TLS template, cloned from the spawning thread. Used by /// `From` to propagate the template into the new task so that /// any threads it spawns in turn can allocate their own TLS regions. @@ -262,9 +262,10 @@ impl From for Task { } = value; #[cfg(feature = "common-os")] - if let Some(root_page_table) = thread_of { + if let Some((root_page_table, parent_pid)) = thread_of { let mut task = Self::new_thread( tid, + parent_pid, core_id, TaskStatus::Ready, prio, @@ -379,7 +380,7 @@ impl PerCoreScheduler { // immediately visible to every thread in this process. let stacks = TaskStacks::new(stack_size); - let (root_page_table, object_map, tls_template, vmas) = { + let (root_page_table, object_map, tls_template, vmas, parent_pid) = { let current = core_scheduler().get_current_task(); let borrowed = current.borrow(); ( @@ -387,6 +388,7 @@ impl PerCoreScheduler { borrowed.object_map.clone(), borrowed.tls_template.clone(), borrowed.vmas.clone(), + borrowed.pid, ) }; @@ -409,7 +411,7 @@ impl PerCoreScheduler { core_id, stacks, object_map, - thread_of: Some(root_page_table), + thread_of: Some((root_page_table, parent_pid)), tls_template, tls_base, vmas, diff --git a/src/scheduler/task/mod.rs b/src/scheduler/task/mod.rs index a2df5c2d79..448d2f3a2f 100644 --- a/src/scheduler/task/mod.rs +++ b/src/scheduler/task/mod.rs @@ -111,26 +111,67 @@ pub(crate) enum TaskStatus { Idle, } -/// Unique identifier for a task (i.e. `pid`). +/// Unique identifier for a task (i.e. `tid`). #[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)] pub struct TaskId(i32); impl TaskId { - pub const fn into(self) -> i32 { - self.0 - } - pub const fn from(x: i32) -> Self { TaskId(x) } } +impl From for i32 { + fn from(tid: TaskId) -> Self { + tid.0 + } +} + impl fmt::Display for TaskId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0) } } +/// Process ID — shared between every thread of the same process. +/// +/// For the main thread of a process it carries the same numeric value +/// as the thread's [`TaskId`]; additional threads inherit it from the +/// spawning thread. A [`TaskId`] converts into a `ProcessId` via +/// [`From`] / [`Into`] (used by `Task::new` / `Task::new_fork` to seed +/// `pid = tid` for a newly created main thread). +#[cfg(feature = "common-os")] +#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)] +pub struct ProcessId(i32); + +#[cfg(feature = "common-os")] +impl ProcessId { + pub const fn from(x: i32) -> Self { + ProcessId(x) + } +} + +#[cfg(feature = "common-os")] +impl From for i32 { + fn from(pid: ProcessId) -> Self { + pid.0 + } +} + +#[cfg(feature = "common-os")] +impl From for ProcessId { + fn from(tid: TaskId) -> Self { + ProcessId(tid.0) + } +} + +#[cfg(feature = "common-os")] +impl fmt::Display for ProcessId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + /// Priority of a task #[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)] pub struct Priority(u8); @@ -440,6 +481,12 @@ pub(crate) struct TlsTemplate { pub(crate) struct Task { /// The ID of this context pub id: TaskId, + /// Process ID — equal to `id` for the main thread of a process, and + /// inherited from the spawning thread for every additional thread of + /// the same process. After `fork()` the child's `pid` equals the + /// child's `id` (it becomes its own process). + #[cfg(feature = "common-os")] + pub pid: ProcessId, /// Status of a task, e.g. if the task is ready or blocked pub status: TaskStatus, /// Task priority, @@ -491,6 +538,8 @@ impl Task { Task { id: tid, + #[cfg(feature = "common-os")] + pid: tid.into(), status: task_status, prio: task_prio, last_stack_pointer: VirtAddr::zero(), @@ -548,6 +597,8 @@ impl Task { Task { id: tid, + #[cfg(feature = "common-os")] + pid: tid.into(), status: TaskStatus::Idle, prio: IDLE_PRIO, last_stack_pointer: VirtAddr::zero(), @@ -582,10 +633,13 @@ impl Task { /// /// The `root_page_table` `Arc` is cloned from the parent, so all threads /// of the same process drop together when the last one exits. + /// `parent_pid` is the spawning thread's `pid`; the new thread joins + /// the same process and reports the same `pid` value. #[cfg(feature = "common-os")] #[allow(clippy::too_many_arguments)] pub fn new_thread( tid: TaskId, + parent_pid: ProcessId, core_id: CoreId, task_status: TaskStatus, task_prio: Priority, @@ -595,9 +649,10 @@ impl Task { tls_template: Option>, vmas: Arc>>, ) -> Task { - debug!("Creating user thread {tid} on core {core_id}"); + debug!("Creating user thread {tid} (pid {parent_pid}) on core {core_id}"); Task { id: tid, + pid: parent_pid, status: task_status, prio: task_prio, last_stack_pointer: VirtAddr::zero(), @@ -632,6 +687,8 @@ impl Task { debug!("Creating forked task {tid} on core {core_id}"); Task { id: tid, + // A forked child becomes the main thread of a new process. + pid: tid.into(), status: task_status, prio: task_prio, last_stack_pointer, diff --git a/src/syscalls/tasks.rs b/src/syscalls/tasks.rs index 5039636b50..bf221a79b9 100644 --- a/src/syscalls/tasks.rs +++ b/src/syscalls/tasks.rs @@ -14,6 +14,7 @@ use crate::{arch, scheduler}; #[cfg(feature = "newlib")] pub type SignalHandler = extern "C" fn(i32); pub type Tid = i32; +pub type Pid = i32; /// Fork the current process. /// Returns the child's PID to the parent, and 0 to the child. @@ -52,7 +53,7 @@ pub extern "C" fn sys_fork() -> i32 { ))] #[hermit_macro::system(errno)] #[unsafe(no_mangle)] -pub extern "C" fn sys_waitpid(pid: Tid) -> i32 { +pub extern "C" fn sys_waitpid(pid: Pid) -> i32 { match scheduler::join(TaskId::from(pid)) { Ok(()) => 0, _ => -i32::from(Errno::Inval), @@ -64,13 +65,13 @@ pub extern "C" fn sys_waitpid(pid: Tid) -> i32 { #[cfg(not(feature = "common-os"))] #[hermit_macro::system(errno)] #[unsafe(no_mangle)] -pub extern "C" fn sys_waitpid(_pid: Tid) -> i32 { +pub extern "C" fn sys_waitpid(_pid: Pid) -> i32 { -i32::from(Errno::Nosys) } #[hermit_macro::system] #[unsafe(no_mangle)] -pub extern "C" fn sys_getpid() -> Tid { +pub extern "C" fn sys_getpid() -> Pid { #[cfg(not(feature = "common-os"))] { // an unikernel doesn't have a pid => return always 0 @@ -79,8 +80,10 @@ pub extern "C" fn sys_getpid() -> Tid { #[cfg(feature = "common-os")] { - // return the current task id - core_scheduler().get_current_task_handle().get_id().into() + // Return the process ID — equal for every thread of the same + // process, set by `Task::new_thread` from the spawning thread's + // `pid` and reset to the new `tid` on fork. + core_scheduler().get_current_task().borrow().pid.into() } } From 310ace44428acc2c81cf67a2bb46a8d4bf4ff5c8 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Mon, 18 May 2026 22:31:51 +0200 Subject: [PATCH 37/48] x86_64/syscall: disable interrupts before swapgs on syscall return MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The syscall-return path used to execute, in order: swapgs ; GS_BASE = user GS, KERNEL_GS_BASE = per-CPU mfence cli That leaves a window between `swapgs` and `cli` where GS is already the user-mode value but CS is still kernel. An interrupt that hits inside this window enters the handler with the kernel CS selector, so the swapgs guard (`stack_frame.code_segment != SegmentSelector(8)`) decides "already kernel, don't swap" — yet GS_BASE is the user's. Every `gs:`-relative per-CPU access in the handler then targets the user's GS area: garbage data at best, #GP / #PF at worst, and sporadic because it depends on whether an IRQ lands in those few cycles. Reorder to `cli; mfence; swapgs;` so interrupts are masked before the GS flip. The mfence stays between for the same ordering reasons it did originally. --- src/arch/x86_64/kernel/syscall.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/arch/x86_64/kernel/syscall.rs b/src/arch/x86_64/kernel/syscall.rs index 26c3edee87..d89345a0d5 100644 --- a/src/arch/x86_64/kernel/syscall.rs +++ b/src/arch/x86_64/kernel/syscall.rs @@ -39,9 +39,9 @@ pub(crate) unsafe extern "C" fn syscall_handler() -> ! { // restore user stack pointer "pop rcx", "mov rsp, rcx", - "swapgs", - "mfence", "cli", + "mfence", + "swapgs", // restore context (without rax) "pop r15", "pop r14", From db73fd155484596f07eca93087fff80d553e47d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Thu, 21 May 2026 10:02:59 +0000 Subject: [PATCH 38/48] timer: swapgs --- src/arch/x86_64/kernel/scheduler.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/arch/x86_64/kernel/scheduler.rs b/src/arch/x86_64/kernel/scheduler.rs index 11dad67269..596cb56a38 100644 --- a/src/arch/x86_64/kernel/scheduler.rs +++ b/src/arch/x86_64/kernel/scheduler.rs @@ -12,6 +12,7 @@ use crate::arch::x86_64::kernel::{apic, interrupts}; use crate::arch::x86_64::mm::paging::{ BasePageSize, PageSize, PageTableEntryFlags, PageTableEntryFlagsExt, }; +use crate::arch::x86_64::swapgs; use crate::config::*; use crate::env; use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; @@ -303,11 +304,7 @@ extern "C" fn task_entry(func: extern "C" fn(usize), arg: usize) -> ! { /// `jump_to_user_land`: user CS=0x2b, user SS=0x23, RFLAGS=0x1202. #[cfg(feature = "common-os")] #[unsafe(naked)] -extern "C" fn task_start_user( - _f: extern "C" fn(usize), - _arg: usize, - _user_stack: u64, -) -> ! { +extern "C" fn task_start_user(_f: extern "C" fn(usize), _arg: usize, _user_stack: u64) -> ! { // rdi = func (user entry) // rsi = arg (forwarded as first argument to `func`) // rdx = user stack pointer @@ -339,8 +336,7 @@ impl Task { ) { unsafe { // Debug marker at the very top of the kernel stack. - let mut stack = self.stacks.get_kernel_stack() - + self.stacks.get_kernel_stack_size() + let mut stack = self.stacks.get_kernel_stack() + self.stacks.get_kernel_stack_size() - TaskStacks::MARKER_SIZE; *stack.as_mut_ptr::() = 0xdead_beefu64; @@ -414,7 +410,8 @@ impl TaskFrame for Task { } } -extern "x86-interrupt" fn timer_handler(_stack_frame: interrupts::ExceptionStackFrame) { +extern "x86-interrupt" fn timer_handler(stack_frame: interrupts::ExceptionStackFrame) { + swapgs(&stack_frame); increment_irq_counter(apic::TIMER_INTERRUPT_NUMBER); debug!("Handle timer interrupt"); @@ -423,6 +420,7 @@ extern "x86-interrupt" fn timer_handler(_stack_frame: interrupts::ExceptionStack core_scheduler().handle_waiting_tasks(); apic::eoi(); core_scheduler().reschedule(); + swapgs(&stack_frame); } pub fn install_timer_handler() { From 1d65ce8d75c423feda951f9fc1cc34778bf75dfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Thu, 21 May 2026 10:03:11 +0000 Subject: [PATCH 39/48] remove mfence --- src/arch/x86_64/kernel/syscall.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/arch/x86_64/kernel/syscall.rs b/src/arch/x86_64/kernel/syscall.rs index d89345a0d5..663edee847 100644 --- a/src/arch/x86_64/kernel/syscall.rs +++ b/src/arch/x86_64/kernel/syscall.rs @@ -24,7 +24,7 @@ pub(crate) unsafe extern "C" fn syscall_handler() -> ! { "push r14", "push r15", "swapgs", - "mfence", + // "mfence", // switch to kernel stack "mov rcx, rsp", "mov rsp, gs:{core_local_kernel_stack}", @@ -40,7 +40,7 @@ pub(crate) unsafe extern "C" fn syscall_handler() -> ! { "pop rcx", "mov rsp, rcx", "cli", - "mfence", + // "mfence", "swapgs", // restore context (without rax) "pop r15", From c1d1ea0090fc39ab74b83b2ce8a77c1fdabd0790 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Fri, 29 May 2026 23:45:59 +0200 Subject: [PATCH 40/48] x86_64/fork: keep user RSP per-task across syscalls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-CPU `user_stack` slot was the only place a task's user RSP lived during a syscall, so any other task running a syscall while the first was blocked (e.g. a parent in `waitpid` while its forked child ran exec + several syscalls) clobbered it. When the blocked task resumed, `mov rsp, gs:[user_stack]` loaded a foreign RSP and `sysretq` faulted (PF/#UD) at a random address in some other task's user stack. Stash the user RSP onto the per-task kernel stack at syscall entry (after the 14 saved registers, using `rcx` as scratch since it was just pushed) and pop it back through the per-CPU slot under `cli` on the return path. The per-CPU slot becomes a transient between syscall entry and the push, plus between the pop and `sysretq` — windows in which no other task can run. `fork_child_start` previously reconstructed the user RSP from the per-CPU slot too. Switch it to a per-task slot: write the user RSP into the child's kernel stack `MARKER_SIZE` pad at `kernel_top - 8` inside `prepare_fork_child_stack` (via `stash_user_rsp_in_child_stack`) and load it from there before `sysretq`. Use `rsp` itself as the scratch register for the two-step `[gs:{kernel_stack} + 8]` load so `rax` (the child's `fork()` return value, set to 0) is preserved — otherwise the child returned from `fork()` with a kernel pointer and the userspace `try_into::().unwrap()` panicked. Drop the now-unused `get_kernel_stack_top` helper. --- src/arch/x86_64/kernel/core_local.rs | 3 + src/arch/x86_64/kernel/switch.rs | 117 ++++++++++++++++++--------- src/arch/x86_64/kernel/syscall.rs | 38 +++++---- 3 files changed, 103 insertions(+), 55 deletions(-) diff --git a/src/arch/x86_64/kernel/core_local.rs b/src/arch/x86_64/kernel/core_local.rs index 000b07ca26..d6bc47a181 100644 --- a/src/arch/x86_64/kernel/core_local.rs +++ b/src/arch/x86_64/kernel/core_local.rs @@ -28,6 +28,8 @@ pub(crate) struct CoreLocal { pub tss: Cell<*mut TaskStateSegment>, /// Start address of the kernel stack pub kernel_stack: Cell<*mut u8>, + /// Current address of the user stack during a system call + pub user_stack: Cell<*mut u8>, /// Interface to the interrupt counters irq_statistics: &'static IrqStatistics, /// The core-local async executor. @@ -56,6 +58,7 @@ impl CoreLocal { scheduler: Cell::new(ptr::null_mut()), tss: Cell::new(ptr::null_mut()), kernel_stack: Cell::new(ptr::null_mut()), + user_stack: Cell::new(ptr::null_mut()), irq_statistics, ex: StaticLocalExecutor::new(), #[cfg(feature = "smp")] diff --git a/src/arch/x86_64/kernel/switch.rs b/src/arch/x86_64/kernel/switch.rs index bfaee2379d..0f57d20327 100644 --- a/src/arch/x86_64/kernel/switch.rs +++ b/src/arch/x86_64/kernel/switch.rs @@ -223,53 +223,42 @@ pub(crate) unsafe extern "C" fn switch_to_fpu_owner(_old_stack: *mut usize, _new // ── Fork support ───────────────────────────────────────────────────────────── -/// Entry point for the child task after a fork. -/// The child's saved context has this function as its "return address". -/// When the child is scheduled it restores context and `ret`s here, returning 1 (true). -/// Returns the child's kernel-stack top minus the marker size. -/// Used by `fork_child_start` to locate the saved user RSP. -#[cfg(all(feature = "common-os", feature = "fork"))] -extern "C" fn get_kernel_stack_top() -> usize { - use crate::arch::x86_64::kernel::core_local::core_scheduler; - use crate::arch::x86_64::kernel::scheduler::TaskStacks; - - let task = core_scheduler().get_current_task(); - let borrowed = task.borrow(); - (borrowed.stacks.get_kernel_stack() - + borrowed.stacks.get_kernel_stack_size() as u64 - - TaskStacks::MARKER_SIZE as u64) - .as_usize() -} - /// Entry point for the child task after a fork. /// -/// `switch_to_task` restores the saved context and `ret`s here. Instead of -/// unwinding the whole kernel call-chain (which is fragile), we jump directly -/// to user space: -/// • rax = 0 (fork returns 0 in the child) -/// • rsp, rcx (user RIP), r11 (user RFLAGS) are read back from the user -/// stack that `syscall_handler` prepared before the fork syscall. -/// • swapgs restores the user GS base. -/// • sysretq returns to user space. +/// `switch_to_task` restores the saved context and `ret`s here. Instead +/// of unwinding the whole kernel call-chain (which is fragile), we jump +/// directly to user space, mirroring `syscall_handler`'s return path: +/// • The 14 user-side registers (`rcx`, `rdx`, `rbx`, `rbp`, `rsi`, +/// `rdi`, `r8`..`r15`) sit at the top of the child's kernel stack +/// because `copy_kernel_stack_to` copied them across when the +/// parent ran `prepare_fork_child_stack`. `rcx` holds the user +/// RIP and `r11` the user RFLAGS — the `syscall` instruction +/// stashed them there before `syscall_handler` even started. +/// • The user RSP for the child is in the unused MARKER_SIZE pad of +/// the kernel stack at `kernel_top - 8`. +/// `prepare_fork_child_stack` writes it there from the per-CPU +/// `user_stack` slot at fork time, so we get a snapshot that +/// survives any other syscall taking place between the parent's +/// fork() and the child being scheduled. +/// • `rax = 0` (fork returns 0 in the child), `swapgs` restores the +/// user GS base, `sysretq` jumps back to user mode. #[cfg(all(feature = "common-os", feature = "fork"))] #[unsafe(naked)] extern "C" fn fork_child_start() { use core::arch::naked_asm; naked_asm!( - // rsp currently points somewhere inside the copied kernel stack. - // The user RSP (after the 14 user-register pushes) was saved by - // syscall_handler via `push rcx` right at the top of the kernel stack. - // Retrieve it: kernel_stack_top - 8. - "call {get_kernel_stack_top}", // rax = kernel_stack_top - "mov rsp, [rax - 8]", // rsp = user_rsp (after 14 pushes on user stack) - // Restore the 14 user registers that syscall_handler pushed: - // r15, r14, r13, r12, r11, r10, r9, r8, rdi, rsi, rbp, rbx, rdx, rcx - // (rcx will hold the user-space return address for sysretq) + // Position rsp at the top of the saved-register block. + // `gs:{kernel_stack}` = kernel_top - MARKER_SIZE = kernel_top - 16. + // After 14 register pushes by syscall_handler rsp was + // gs:{kernel_stack} - 14*8. + "mov rsp, gs:{core_local_kernel_stack}", + "sub rsp, 14 * 8", + // Pop the 14 saved registers (rcx = user RIP, r11 = user RFLAGS). "pop r15", "pop r14", "pop r13", "pop r12", - "pop r11", // r11 = user RFLAGS (saved by syscall instruction) + "pop r11", "pop r10", "pop r9", "pop r8", @@ -278,12 +267,20 @@ extern "C" fn fork_child_start() { "pop rbp", "pop rbx", "pop rdx", - "pop rcx", // rcx = user-space return address (saved by syscall instruction) - // fork() returns 0 in the child + "pop rcx", + // fork() returns 0 in the child. "xor eax, eax", + // Load the per-task user_rsp from kernel_top - 8 (the unused + // MARKER_SIZE pad above the marker; `prepare_fork_child_stack` + // wrote it there). gs:{kernel_stack} holds kernel_top - 16, + // so the slot we want is at [kernel_stack + 8]. Go via rsp + // itself — we are about to overwrite it anyway and must not + // clobber rax (the fork return value). + "mov rsp, gs:{core_local_kernel_stack}", + "mov rsp, [rsp + 8]", "swapgs", "sysretq", - get_kernel_stack_top = sym get_kernel_stack_top, + core_local_kernel_stack = const core::mem::offset_of!(super::core_local::CoreLocal, kernel_stack), ); } @@ -312,6 +309,40 @@ extern "C" fn copy_kernel_stack_to(stack_addr: usize) { crate::arch::x86_64::mm::copy_kernel_stack_to(stack_addr); } +/// Snapshot the current user RSP into the child's kernel stack so that +/// `fork_child_start` can recover it via `sysretq` regardless of what +/// other syscalls do to the per-CPU `user_stack` slot in the meantime. +/// +/// The slot we use sits at `child_kernel_top - 8`, i.e. inside the +/// `MARKER_SIZE` pad above the `0xdeadbeef` marker — see [`fork_child_start`] +/// for the matching `mov rsp, [gs:{kernel_stack} + 8]`. +/// +/// The parent's user RSP was stashed by `syscall_handler` into +/// `gs:[user_stack]` at syscall entry; we just read it back here. +#[cfg(all(feature = "common-os", feature = "fork"))] +extern "C" fn stash_user_rsp_in_child_stack(new_stack_addr: usize) { + use crate::arch::x86_64::kernel::core_local::CoreLocal; + use crate::arch::x86_64::kernel::interrupts::IST_SIZE; + use crate::arch::x86_64::mm::paging::{BasePageSize, PageSize}; + use crate::config::DEFAULT_STACK_SIZE; + + let user_rsp: usize; + unsafe { + core::arch::asm!( + "mov {0}, gs:[{1}]", + out(reg) user_rsp, + const core::mem::offset_of!(CoreLocal, user_stack), + options(nostack, preserves_flags), + ); + } + + let child_kernel_top = + new_stack_addr + IST_SIZE + 2 * BasePageSize::SIZE as usize + DEFAULT_STACK_SIZE; + unsafe { + core::ptr::with_exposed_provenance_mut::(child_kernel_top - 8).write(user_rsp); + } +} + /// Prepare the child's stack for a fork. /// /// On entry (parent): @@ -344,6 +375,13 @@ pub unsafe extern "C" fn prepare_fork_child_stack( "mov rdi, [rsp+8]", // rdi = new_stack_addr "call {copy_kernel_stack_to}", // copy kernel stack pages + // 1b. Stash the user RSP into the child's kernel stack at + // child_kernel_top - 8, so that fork_child_start can + // restore it via sysretq without relying on the per-CPU + // `user_stack` slot (which other syscalls may clobber). + "mov rdi, [rsp+8]", // rdi = new_stack_addr + "call {stash_user_rsp_in_child_stack}", + // 2. Duplicate the page table (COW) — snapshot now includes the copied stack. "call {copy_current_root_page_table}", // rax = new PML4 phys addr "mov rsi, [rsp]", // rsi = root_page_table ptr @@ -369,6 +407,7 @@ pub unsafe extern "C" fn prepare_fork_child_stack( fork_child_start = sym fork_child_start, copy_current_root_page_table = sym copy_current_root_page_table, copy_kernel_stack_to = sym copy_kernel_stack_to, + stash_user_rsp_in_child_stack = sym stash_user_rsp_in_child_stack, get_current_stack_addr = sym get_current_stack_addr, ); } diff --git a/src/arch/x86_64/kernel/syscall.rs b/src/arch/x86_64/kernel/syscall.rs index 663edee847..3e34c6ac97 100644 --- a/src/arch/x86_64/kernel/syscall.rs +++ b/src/arch/x86_64/kernel/syscall.rs @@ -1,13 +1,15 @@ use core::arch::naked_asm; -use core::mem; -use super::core_local::CoreLocal; use crate::syscalls::table::SYSHANDLER_TABLE; #[unsafe(no_mangle)] #[unsafe(naked)] pub(crate) unsafe extern "C" fn syscall_handler() -> ! { naked_asm!( + "swapgs", + // switch to kernel stack + "mov gs:{core_local_user_stack}, rsp", + "mov rsp, gs:{core_local_kernel_stack}", // save context "push rcx", "push rdx", @@ -23,25 +25,26 @@ pub(crate) unsafe extern "C" fn syscall_handler() -> ! { "push r13", "push r14", "push r15", - "swapgs", - // "mfence", - // switch to kernel stack - "mov rcx, rsp", - "mov rsp, gs:{core_local_kernel_stack}", - "sti", - // save user stack pointer + // All user-visible registers are now on the kernel stack, so + // rcx is free to use as scratch. Stash the user RSP onto the + // per-task kernel stack — the per-CPU `user_stack` slot would + // otherwise be clobbered by any syscall a different task runs + // while this call is blocked (e.g. waitpid waiting for a child). + "mov rcx, gs:{core_local_user_stack}", "push rcx", + "sti", // copy 4th argument to rcx to adhere x86_64 ABI "mov rcx, r10", // call system call - "mov r10, qword ptr [rip + {table}@GOTPCREL]", + "mov r10, [rip + {table}@GOTPCREL]", "call [r10 + 8*rax]", - // restore user stack pointer - "pop rcx", - "mov rsp, rcx", "cli", - // "mfence", - "swapgs", + // Pop the stashed user RSP and route it back through the + // per-CPU slot so the final `mov rsp, gs:[user_stack]` still + // works. Interrupts are off, so the slot can't be clobbered + // between here and `sysretq`. + "pop rcx", + "mov gs:{core_local_user_stack}, rcx", // restore context (without rax) "pop r15", "pop r14", @@ -57,8 +60,11 @@ pub(crate) unsafe extern "C" fn syscall_handler() -> ! { "pop rbx", "pop rdx", "pop rcx", + "mov rsp, gs:{core_local_user_stack}", + "swapgs", "sysretq", - core_local_kernel_stack = const mem::offset_of!(CoreLocal, kernel_stack), + core_local_user_stack = const core::mem::offset_of!(super::core_local::CoreLocal, user_stack), + core_local_kernel_stack = const core::mem::offset_of!(super::core_local::CoreLocal, kernel_stack), table = sym SYSHANDLER_TABLE, ); } From 7df086cb14d54d63cd84006daf7f153c11c6f422 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Mon, 1 Jun 2026 15:02:05 +0200 Subject: [PATCH 41/48] cleanup: dedupe stdio bootstrap and tighten imports * arch::{aarch64,x86_64}::load_application: replace the identical 12-line `if env::is_uhyve()` / else block that installed Uhyve- vs. console-backed stdio with a single `fd::stdio::setup()` call, so both architectures share one source of truth. * Hoist commonly-aliased imports: `size_of` is in the prelude now, `Arc`, `paging`, and the socket sys_* names are introduced once per module instead of being requalified at every call site. `syscalls/table.rs` shrinks accordingly. * aarch64/mm/paging: replace `#[expect(dead_code)]` markers on `is_table_or_4kib_page` and `get_page_table_entry` with the real `#[cfg(all(feature = "common-os", feature = "fork"))]` gate that matches the only configuration where they are reachable. Drop `get_application_page_size` (no remaining callers). * syscalls/entropy: lower the naive-fallback message from `warn!` to `trace!`. It fires on every syscall when the host exposes no entropy source and was drowning out everything else. --- src/arch/aarch64/kernel/interrupts.rs | 22 ++++++++------- src/arch/aarch64/kernel/mod.rs | 27 ++++-------------- src/arch/aarch64/kernel/scheduler.rs | 6 ++-- src/arch/aarch64/mm/mod.rs | 2 +- src/arch/aarch64/mm/paging.rs | 21 ++++++-------- src/arch/x86_64/kernel/mod.rs | 22 +++------------ src/arch/x86_64/kernel/scheduler.rs | 7 ++--- src/arch/x86_64/mm/mod.rs | 2 +- src/mm/physicalmem.rs | 14 ++++------ src/scheduler/mod.rs | 19 ++++--------- src/scheduler/task/mod.rs | 4 ++- src/syscalls/entropy.rs | 2 +- src/syscalls/table.rs | 40 ++++++++++++++------------- 13 files changed, 74 insertions(+), 114 deletions(-) diff --git a/src/arch/aarch64/kernel/interrupts.rs b/src/arch/aarch64/kernel/interrupts.rs index 96c0745b6a..05cf63dacf 100644 --- a/src/arch/aarch64/kernel/interrupts.rs +++ b/src/arch/aarch64/kernel/interrupts.rs @@ -179,6 +179,8 @@ pub(crate) extern "C" fn do_irq(_state: &State) -> *mut usize { #[unsafe(no_mangle)] pub(crate) extern "C" fn do_sync(state: &mut State) { + use crate::arch::mm::paging::do_cow_fault; + let esr = ESR_EL1.get(); let ec_raw = ESR_EL1.read(ESR_EL1::EC); let ec: ESR_EL1::EC::Value = ESR_EL1.read_as_enum(ESR_EL1::EC).unwrap(); @@ -236,7 +238,7 @@ pub(crate) extern "C" fn do_sync(state: &mut State) { #[cfg(all(feature = "common-os", feature = "fork"))] if is_write && is_permission_fault - && crate::arch::aarch64::mm::paging::do_cow_fault(VirtAddr::new(far)) + && do_cow_fault(VirtAddr::new(far)) { // Faulting instruction is retried on `eret` from the trap. return; @@ -505,17 +507,17 @@ pub(crate) fn init() { // metal where CNTVOFF_EL2 defaults to 0. /* Secure Phys IRQ — skip */ - let (_irqtype, irq_slice) = irq_slice.split_at(mem::size_of::()); - let (_irq, irq_slice) = irq_slice.split_at(mem::size_of::()); - let (_irqflags, irq_slice) = irq_slice.split_at(mem::size_of::()); + let (_irqtype, irq_slice) = irq_slice.split_at(size_of::()); + let (_irq, irq_slice) = irq_slice.split_at(size_of::()); + let (_irqflags, irq_slice) = irq_slice.split_at(size_of::()); /* Non-secure Phys IRQ — skip */ - let (_irqtype, irq_slice) = irq_slice.split_at(mem::size_of::()); - let (_irq, irq_slice) = irq_slice.split_at(mem::size_of::()); - let (_irqflags, irq_slice) = irq_slice.split_at(mem::size_of::()); + let (_irqtype, irq_slice) = irq_slice.split_at(size_of::()); + let (_irq, irq_slice) = irq_slice.split_at(size_of::()); + let (_irqflags, irq_slice) = irq_slice.split_at(size_of::()); /* Virtual Timer IRQ */ - let (irqtype, irq_slice) = irq_slice.split_at(mem::size_of::()); - let (irq, irq_slice) = irq_slice.split_at(mem::size_of::()); - let (irqflags, _irq_slice) = irq_slice.split_at(mem::size_of::()); + let (irqtype, irq_slice) = irq_slice.split_at(size_of::()); + let (irq, irq_slice) = irq_slice.split_at(size_of::()); + let (irqflags, _irq_slice) = irq_slice.split_at(size_of::()); let irqtype = u32::from_be_bytes(irqtype.try_into().unwrap()); let irq = u32::from_be_bytes(irq.try_into().unwrap()); let irqflags = u32::from_be_bytes(irqflags.try_into().unwrap()); diff --git a/src/arch/aarch64/kernel/mod.rs b/src/arch/aarch64/kernel/mod.rs index 4642f5ba98..7a5e35c218 100644 --- a/src/arch/aarch64/kernel/mod.rs +++ b/src/arch/aarch64/kernel/mod.rs @@ -206,8 +206,7 @@ where use hermit_sync::RwSpinLock; use crate::arch::aarch64::mm::paging::{self, PageTableEntryFlags}; - use crate::fd::stdio::*; - use crate::fd::{Fd, RawFd, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; + use crate::fd::{Fd, RawFd}; use crate::mm::{FrameAlloc, PageRangeAllocator}; #[cfg(feature = "fork")] use crate::mm::frame_ref_inc; @@ -217,21 +216,7 @@ where let mut object_map = HashMap::>, RandomState>::with_hasher( RandomState::with_seeds(0, 0, 0, 0), ); - if env::is_uhyve() { - let stdin = Arc::new(async_lock::RwLock::new(UhyveStdin::new().into())); - let stdout = Arc::new(async_lock::RwLock::new(UhyveStdout::new().into())); - let stderr = Arc::new(async_lock::RwLock::new(UhyveStderr::new().into())); - object_map.insert(STDIN_FILENO, stdin); - object_map.insert(STDOUT_FILENO, stdout); - object_map.insert(STDERR_FILENO, stderr); - } else { - let stdin = Arc::new(async_lock::RwLock::new(GenericStdin::new().into())); - let stdout = Arc::new(async_lock::RwLock::new(GenericStdout::new().into())); - let stderr = Arc::new(async_lock::RwLock::new(GenericStderr::new().into())); - object_map.insert(STDIN_FILENO, stdin); - object_map.insert(STDOUT_FILENO, stdout); - object_map.insert(STDERR_FILENO, stderr); - } + crate::fd::stdio::setup(&mut object_map); core_scheduler().set_current_task_object_map(Arc::new(RwSpinLock::new(object_map))); let code_size = (code_size as usize).align_up(BasePageSize::SIZE as usize); @@ -262,7 +247,7 @@ where // area (`tcb[0] = dtv`, `tcb[1]` reserved). We allocate the TCB plus // the TLS image as one contiguous block so a single `msr tpidr_el0` // suffices and the layout matches what musl/glibc expect. - let tcb_size = 2 * mem::size_of::<*mut ()>(); + let tcb_size = 2 * size_of::<*mut ()>(); let tls_offset = tcb_size; let tls_memsz = (tls_offset + tls_size as usize).align_up(BasePageSize::SIZE as usize); @@ -303,7 +288,7 @@ where if let Some(init) = tls_init { let template = - alloc::sync::Arc::new(crate::scheduler::task::TlsTemplate { + Arc::new(crate::scheduler::task::TlsTemplate { size: tls_size as usize, init, }); @@ -356,7 +341,7 @@ fn set_user_tpidr_el0(value: u64) { let task = core_scheduler().get_current_task(); let kernel_stack_top = task.borrow().stacks.get_kernel_stack().as_usize() + task.borrow().stacks.get_kernel_stack_size(); - let state_addr = kernel_stack_top - TaskStacks::MARKER_SIZE - mem::size_of::(); + let state_addr = kernel_stack_top - TaskStacks::MARKER_SIZE - size_of::(); unsafe { let state = ptr::with_exposed_provenance_mut::(state_addr); (*state).tpidr_el0 = value; @@ -406,7 +391,7 @@ pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) } // Place the argv pointer array on the user stack. - let stack_pointer = stack_top - arg.len() * mem::size_of::<*mut u8>(); + let stack_pointer = stack_top - arg.len() * size_of::<*mut u8>(); let stack_ptr = ptr::with_exposed_provenance_mut::<*mut u8>(stack_pointer); let argv = unsafe { slice::from_raw_parts_mut(stack_ptr, arg.len()) }; let len = arg.iter().fold(0, |acc, x| acc + x.len() + 1); diff --git a/src/arch/aarch64/kernel/scheduler.rs b/src/arch/aarch64/kernel/scheduler.rs index 4780298dcc..633a60272e 100644 --- a/src/arch/aarch64/kernel/scheduler.rs +++ b/src/arch/aarch64/kernel/scheduler.rs @@ -343,9 +343,9 @@ impl Task { // don't touch below stays zero on entry to user space, which // keeps any leftover kernel state out of EL0's general-purpose // register file. - stack -= mem::size_of::(); + stack -= size_of::(); let state = stack.as_mut_ptr::(); - ptr::write_bytes(state.cast::(), 0, mem::size_of::()); + ptr::write_bytes(state.cast::(), 0, size_of::()); // Initial user stack: top of the user-stack region with the // usual debug marker. AAPCS64 doesn't require any extra slop @@ -519,7 +519,7 @@ pub unsafe fn prepare_fork_child_stack( let parent_stack_base = task.borrow().stacks.get_stack_virt_addr().as_usize(); let kernel_stack_top = task.borrow().stacks.get_kernel_stack().as_usize() + task.borrow().stacks.get_kernel_stack_size(); - let parent_state_addr = kernel_stack_top - TaskStacks::MARKER_SIZE - mem::size_of::(); + let parent_state_addr = kernel_stack_top - TaskStacks::MARKER_SIZE - size_of::(); let offset = new_stack_addr.wrapping_sub(parent_stack_base); let child_state_addr = parent_state_addr.wrapping_add(offset); diff --git a/src/arch/aarch64/mm/mod.rs b/src/arch/aarch64/mm/mod.rs index 4680854904..083c4266d0 100644 --- a/src/arch/aarch64/mm/mod.rs +++ b/src/arch/aarch64/mm/mod.rs @@ -35,7 +35,7 @@ pub fn allocate_thread_tls(template: &crate::scheduler::task::TlsTemplate) -> u6 #[cfg(feature = "fork")] use crate::mm::frame_ref_inc; - let tcb_size = 2 * core::mem::size_of::<*mut ()>(); + let tcb_size = 2 * size_of::<*mut ()>(); let total = (tcb_size + template.size).align_up(BasePageSize::SIZE as usize); let virt_layout = PageLayout::from_size(total).unwrap(); diff --git a/src/arch/aarch64/mm/paging.rs b/src/arch/aarch64/mm/paging.rs index e29772612d..f80f7846e0 100644 --- a/src/arch/aarch64/mm/paging.rs +++ b/src/arch/aarch64/mm/paging.rs @@ -161,6 +161,7 @@ impl PageTableEntryFlags { /// the same helpers as inherent methods; this trait exists only so that /// architecture-independent callers can be written in terms of a single API. #[cfg(feature = "common-os")] +#[allow(dead_code)] pub trait PageTableEntryFlagsExt { fn device(&mut self) -> &mut Self; fn normal(&mut self) -> &mut Self; @@ -169,7 +170,6 @@ pub trait PageTableEntryFlagsExt { fn execute_disable(&mut self) -> &mut Self; fn execute_enable(&mut self) -> &mut Self; fn user(&mut self) -> &mut Self; - #[expect(dead_code)] fn kernel(&mut self) -> &mut Self; fn copy_on_write(&mut self) -> &mut Self; } @@ -226,7 +226,7 @@ impl PageTableEntry { } /// Return whether this entry is a 4KiB page. - #[expect(dead_code)] + #[cfg(all(feature = "common-os", feature = "fork"))] fn is_table_or_4kib_page(&self) -> bool { (self.physical_address_and_flags & PageTableEntryFlags::TABLE_OR_4KIB_PAGE.bits()) != 0 } @@ -655,7 +655,7 @@ fn get_page_range(virtual_address: VirtAddr, count: usize) -> PageI Page::range(first_page, last_page) } -#[expect(dead_code)] +#[cfg(all(feature = "common-os", feature = "fork"))] pub fn get_page_table_entry(virtual_address: VirtAddr) -> Option { trace!("Looking up Page Table Entry for {virtual_address:p}"); @@ -791,11 +791,6 @@ pub fn unmap(virtual_address: VirtAddr, count: usize) { root_pagetable.map_pages(range, PhysAddr::zero(), PageTableEntryFlags::empty()); } -#[inline] -pub fn get_application_page_size() -> usize { - BasePageSize::SIZE as usize -} - /// Flush the entire (non-global) TLB on this core and broadcast to others. #[cfg(feature = "common-os")] fn flush_tlb_all() { @@ -837,7 +832,7 @@ fn flush_tlb_one(virt: VirtAddr) { /// frame and copy the contents. #[cfg(all(feature = "common-os", feature = "fork"))] pub fn do_cow_fault(faulting_addr: VirtAddr) -> bool { - use aarch64_cpu::registers::{Readable, TTBR0_EL1}; + use aarch64_cpu::registers::TTBR0_EL1; let vaddr = faulting_addr.align_down(BasePageSize::SIZE); let l0_phys = TTBR0_EL1.get_baddr() as usize; @@ -922,7 +917,7 @@ pub fn do_cow_fault(faulting_addr: VirtAddr) -> bool { /// Copy-On-Write. This must be called before duplicating the page table for a fork. #[cfg(all(feature = "common-os", feature = "fork"))] pub fn mark_user_pages_copy_on_write() { - use aarch64_cpu::registers::{Readable, TTBR0_EL1}; + use aarch64_cpu::registers::TTBR0_EL1; // Physical memory is identity-mapped in the kernel's address space, so a // physical frame address can be dereferenced directly as a page-table @@ -1122,7 +1117,7 @@ pub fn drop_user_space(l0_phys: usize) { pub fn clear_user_space() { use crate::fd::STDERR_FILENO; use crate::core_scheduler; - use aarch64_cpu::registers::{Readable, TTBR0_EL1}; + use aarch64_cpu::registers::TTBR0_EL1; core_scheduler() .get_current_task() @@ -1155,7 +1150,7 @@ const USER_L0_INDEX: usize = (crate::arch::aarch64::kernel::USER_START.as_usize( #[cfg(feature = "common-os")] pub fn create_new_root_page_table() -> usize { - use aarch64_cpu::registers::{Readable, TTBR0_EL1}; + use aarch64_cpu::registers::TTBR0_EL1; let layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); let frame_range = FrameAlloc::allocate(layout).expect("Failed to allocate L0 table"); @@ -1212,7 +1207,7 @@ pub fn get_current_root_page_table() -> usize { /// Returns the physical address of the new L0. #[cfg(all(feature = "common-os", feature = "fork"))] pub fn copy_current_root_page_table() -> usize { - use aarch64_cpu::registers::{Readable, TTBR0_EL1}; + use aarch64_cpu::registers::TTBR0_EL1; let layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index bc8fa3565f..f326f770f2 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -210,8 +210,8 @@ where use crate::arch::x86_64::mm::paging::{self, PageTableEntryFlags, PageTableEntryFlagsExt}; use crate::mm::{FrameAlloc, PageRangeAllocator}; - use crate::fd::{Fd, RawFd, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; - use crate::fd::stdio::*; + use crate::fd::{Fd, RawFd}; + use crate::fd::stdio; #[cfg(feature = "fork")] use crate::mm::frame_ref_inc; use crate::mm::vma::*; @@ -225,21 +225,7 @@ where >::with_hasher( RandomState::with_seeds(0, 0, 0, 0), ); - if env::is_uhyve() { - let stdin = Arc::new(async_lock::RwLock::new(UhyveStdin::new().into())); - let stdout = Arc::new(async_lock::RwLock::new(UhyveStdout::new().into())); - let stderr = Arc::new(async_lock::RwLock::new(UhyveStderr::new().into())); - object_map.insert(STDIN_FILENO, stdin); - object_map.insert(STDOUT_FILENO, stdout); - object_map.insert(STDERR_FILENO, stderr); - } else { - let stdin = Arc::new(async_lock::RwLock::new(GenericStdin::new().into())); - let stdout = Arc::new(async_lock::RwLock::new(GenericStdout::new().into())); - let stderr = Arc::new(async_lock::RwLock::new(GenericStderr::new().into())); - object_map.insert(STDIN_FILENO, stdin); - object_map.insert(STDOUT_FILENO, stdout); - object_map.insert(STDERR_FILENO, stderr); - } + stdio::setup(&mut object_map); core_scheduler().set_current_task_object_map(Arc::new(RwSpinLock::new(object_map))); let code_size = (code_size as usize).align_up(BasePageSize::SIZE as usize); @@ -341,7 +327,7 @@ where let tls_init = func(code_slice, Some(block))?; if let Some(init) = tls_init { - let template = alloc::sync::Arc::new( + let template = Arc::new( crate::scheduler::task::TlsTemplate { size: tls_offset, init, diff --git a/src/arch/x86_64/kernel/scheduler.rs b/src/arch/x86_64/kernel/scheduler.rs index 596cb56a38..7c716f5d93 100644 --- a/src/arch/x86_64/kernel/scheduler.rs +++ b/src/arch/x86_64/kernel/scheduler.rs @@ -18,7 +18,6 @@ use crate::env; use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; use crate::scheduler::task::{Task, TaskFrame}; use crate::scheduler::{PerCoreSchedulerExt, timer_interrupts}; - #[repr(C, packed)] struct State { #[cfg(feature = "common-os")] @@ -340,9 +339,9 @@ impl Task { - TaskStacks::MARKER_SIZE; *stack.as_mut_ptr::() = 0xdead_beefu64; - stack -= mem::size_of::(); + stack -= size_of::(); let state = stack.as_mut_ptr::(); - state.cast::().write_bytes(0, mem::size_of::()); + state.cast::().write_bytes(0, size_of::()); (*state).rip = task_start_user; (*state).rdi = func as usize as u64; @@ -361,7 +360,7 @@ impl Task { .into(); // rdx is used by task_start_user as the new user-mode RSP - (*state).rdx = self.user_stack_pointer.as_u64() - mem::size_of::() as u64; + (*state).rdx = self.user_stack_pointer.as_u64() - size_of::() as u64; } } } diff --git a/src/arch/x86_64/mm/mod.rs b/src/arch/x86_64/mm/mod.rs index 6cbe7767b4..b17b4f6220 100644 --- a/src/arch/x86_64/mm/mod.rs +++ b/src/arch/x86_64/mm/mod.rs @@ -231,7 +231,7 @@ pub fn allocate_thread_tls(template: &crate::scheduler::task::TlsTemplate) -> u6 #[cfg(feature = "fork")] use crate::mm::frame_ref_inc; - let tcb_size = core::mem::size_of::<*mut ()>(); + let tcb_size = size_of::<*mut ()>(); let total = (template.size + tcb_size).align_up(BasePageSize::SIZE as usize); let virt_layout = PageLayout::from_size(total).unwrap(); diff --git a/src/mm/physicalmem.rs b/src/mm/physicalmem.rs index 085d4a1075..06129a000c 100644 --- a/src/mm/physicalmem.rs +++ b/src/mm/physicalmem.rs @@ -106,12 +106,8 @@ pub type FrameBox = PageRangeBox; /// Copy the physical page at `src_phys` into a freshly allocated page and return its address. #[cfg(feature = "common-os")] pub fn copy_page(src_phys: PhysAddr) -> PhysAddr { - use free_list::PageLayout; - - #[cfg(target_arch = "x86_64")] - use crate::arch::mm::paging::PageTableEntryFlagsExt; - use crate::arch::mm::paging::{BasePageSize, PageSize, PageTableEntryFlags}; - use crate::mm::{FrameAlloc, PageBox, PageRangeAllocator}; + use crate::arch::mm::paging::BasePageSize; + use crate::mm::PageBox; let frame_layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); let frame_range = FrameAlloc::allocate(frame_layout).expect("Failed to allocate page"); @@ -126,8 +122,8 @@ pub fn copy_page(src_phys: PhysAddr) -> PhysAddr { flags.normal().writable(); flags }; - crate::arch::mm::paging::map::(virt, src_phys, 1, flags); - crate::arch::mm::paging::map::(virt + BasePageSize::SIZE, dst_phys, 1, flags); + paging::map::(virt, src_phys, 1, flags); + paging::map::(virt + BasePageSize::SIZE, dst_phys, 1, flags); unsafe { let src = core::slice::from_raw_parts(virt.as_ptr::(), BasePageSize::SIZE as usize); @@ -138,7 +134,7 @@ pub fn copy_page(src_phys: PhysAddr) -> PhysAddr { dst.copy_from_slice(src); } - crate::arch::mm::paging::unmap::(virt, 2); + paging::unmap::(virt, 2); // page_box is dropped here, freeing the virtual memory dst_phys diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index 7aadceaa4f..0fc7f379a0 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -226,12 +226,12 @@ struct NewTask { /// parent's `pid`. When `None`, the task is a regular kernel-mode /// task with a fresh address space; its `pid` equals its `tid`. #[cfg(feature = "common-os")] - thread_of: Option<(Arc, task::ProcessId)>, + thread_of: Option<(Arc, ProcessId)>, /// Per-process TLS template, cloned from the spawning thread. Used by /// `From` to propagate the template into the new task so that /// any threads it spawns in turn can allocate their own TLS regions. #[cfg(feature = "common-os")] - tls_template: Option>, + tls_template: Option>, /// Per-thread TLS thread-pointer (FS.Base on x86_64, TPIDR_EL0 on /// aarch64), already prepared by `spawn_thread` from the per-process /// `TlsTemplate`. Zero means "do not install a thread pointer". @@ -398,7 +398,7 @@ impl PerCoreScheduler { // path) so that the new user-accessible pages get mapped into the // shared root page table. let tls_base = if let Some(ref template) = tls_template { - crate::arch::mm::allocate_thread_tls(template) + arch::mm::allocate_thread_tls(template) } else { 0 }; @@ -617,7 +617,7 @@ impl PerCoreScheduler { /// Returns the Rc> for the currently running task. #[cfg(feature = "common-os")] #[inline] - pub fn get_current_task(&self) -> Rc> { + pub fn get_current_task(&self) -> Rc> { self.current_task.clone() } @@ -918,7 +918,7 @@ impl PerCoreScheduler { #[inline] #[cfg(target_arch = "aarch64")] - pub fn get_last_stack_pointer(&self) -> memory_addresses::VirtAddr { + pub fn get_last_stack_pointer(&self) -> VirtAddr { self.current_task.borrow().last_stack_pointer } @@ -1173,11 +1173,6 @@ pub fn join(id: TaskId) -> Result<(), ()> { all(feature = "common-os", feature = "fork") ))] pub unsafe fn fork() -> TaskId { - #[cfg(not(target_arch = "x86_64"))] - use memory_addresses::VirtAddr; - #[cfg(target_arch = "x86_64")] - use x86_64::VirtAddr; - use crate::arch::{prepare_fork_child_stack, prepare_mem_copy_on_write}; let core_id = SPAWN_COUNTER.fetch_add(1, Ordering::SeqCst) % get_processor_count(); @@ -1253,9 +1248,7 @@ pub unsafe fn fork() -> TaskId { child_last_sp, parent_user_sp, parent_object_map, - Arc::new(crate::scheduler::task::RootPageTable::new( - child_root_page_table, - )), + Arc::new(RootPageTable::new(child_root_page_table)), parent_tls_template, parent_vmas, ); diff --git a/src/scheduler/task/mod.rs b/src/scheduler/task/mod.rs index 448d2f3a2f..fe030e745e 100644 --- a/src/scheduler/task/mod.rs +++ b/src/scheduler/task/mod.rs @@ -34,7 +34,9 @@ use crate::arch::kernel::processor::{self, FPUState}; use crate::arch::scheduler::TaskStacks; #[cfg(not(feature = "common-os"))] use crate::env; -use crate::fd::{Fd, RawFd, stdio}; +#[cfg(not(feature = "common-os"))] +use crate::fd::stdio; +use crate::fd::{Fd, RawFd}; #[cfg(not(feature = "common-os"))] use crate::fd::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; #[cfg(feature = "common-os")] diff --git a/src/syscalls/entropy.rs b/src/syscalls/entropy.rs index 34ea0d425a..a2cda9b649 100644 --- a/src/syscalls/entropy.rs +++ b/src/syscalls/entropy.rs @@ -33,7 +33,7 @@ unsafe fn read_entropy(buf: *mut u8, len: usize, flags: u32) -> isize { entropy::read(buf, flags) .unwrap_or_else(|_| { - warn!("Unable to read entropy! Fallback to a naive implementation!"); + trace!("Unable to read entropy! Fallback to a naive implementation!"); for byte in &mut *buf { *byte = (generate_park_miller_lehmer_random_number() & 0xff) .try_into() diff --git a/src/syscalls/table.rs b/src/syscalls/table.rs index 5565d584e9..efaecf2938 100644 --- a/src/syscalls/table.rs +++ b/src/syscalls/table.rs @@ -2,6 +2,10 @@ use core::arch::naked_asm; use crate::mm::vma::sys_mmap; +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +use crate::syscalls::socket::addrinfo::*; +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +use crate::syscalls::socket::*; use crate::syscalls::*; /// Number of the system call `exit` @@ -150,7 +154,7 @@ pub(crate) extern "C" fn invalid_syscall(sys_no: u64) -> ! { /// loader will replace this function #[linkage = "weak"] #[unsafe(no_mangle)] -pub extern "C" fn sys_spawn_process(_path: *const core::ffi::c_char) -> i32 { +pub extern "C" fn sys_spawn_process(_path: *const c_char) -> i32 { -i32::from(Errno::Nosys) } @@ -240,24 +244,22 @@ impl SyscallTable { table.handle[SYSNO_WAKEUP_TASK] = sys_wakeup_task as *const _; #[cfg(any(feature = "net", feature = "virtio-vsock"))] { - table.handle[SYSNO_SOCKET] = crate::syscalls::socket::sys_socket as *const _; - table.handle[SYSNO_BIND] = crate::syscalls::socket::sys_bind as *const _; - table.handle[SYSNO_LISTEN] = crate::syscalls::socket::sys_listen as *const _; - table.handle[SYSNO_ACCEPT] = crate::syscalls::socket::sys_accept as *const _; - table.handle[SYSNO_CONNECT] = crate::syscalls::socket::sys_connect as *const _; - table.handle[SYSNO_RECV] = crate::syscalls::socket::sys_recv as *const _; - table.handle[SYSNO_RECVFROM] = crate::syscalls::socket::sys_recvfrom as *const _; - table.handle[SYSNO_SEND] = crate::syscalls::socket::sys_send as *const _; - table.handle[SYSNO_SENDTO] = crate::syscalls::socket::sys_sendto as *const _; - table.handle[SYSNO_SHUTDOWN] = crate::syscalls::socket::sys_shutdown as *const _; - table.handle[SYSNO_GETPEERNAME] = crate::syscalls::socket::sys_getpeername as *const _; - table.handle[SYSNO_GETSOCKNAME] = crate::syscalls::socket::sys_getsockname as *const _; - table.handle[SYSNO_GETSOCKOPT] = crate::syscalls::socket::sys_getsockopt as *const _; - table.handle[SYSNO_SETSOCKOPT] = crate::syscalls::socket::sys_setsockopt as *const _; - table.handle[SYSNO_GETADDRINFO] = - crate::syscalls::socket::addrinfo::sys_getaddrinfo as *const _; - table.handle[SYSNO_FREEADDRINFO] = - crate::syscalls::socket::addrinfo::sys_freeaddrinfo as *const _; + table.handle[SYSNO_SOCKET] = sys_socket as *const _; + table.handle[SYSNO_BIND] = sys_bind as *const _; + table.handle[SYSNO_LISTEN] = sys_listen as *const _; + table.handle[SYSNO_ACCEPT] = sys_accept as *const _; + table.handle[SYSNO_CONNECT] = sys_connect as *const _; + table.handle[SYSNO_RECV] = sys_recv as *const _; + table.handle[SYSNO_RECVFROM] = sys_recvfrom as *const _; + table.handle[SYSNO_SEND] = sys_send as *const _; + table.handle[SYSNO_SENDTO] = sys_sendto as *const _; + table.handle[SYSNO_SHUTDOWN] = sys_shutdown as *const _; + table.handle[SYSNO_GETPEERNAME] = sys_getpeername as *const _; + table.handle[SYSNO_GETSOCKNAME] = sys_getsockname as *const _; + table.handle[SYSNO_GETSOCKOPT] = sys_getsockopt as *const _; + table.handle[SYSNO_SETSOCKOPT] = sys_setsockopt as *const _; + table.handle[SYSNO_GETADDRINFO] = sys_getaddrinfo as *const _; + table.handle[SYSNO_FREEADDRINFO] = sys_freeaddrinfo as *const _; } table.handle[SYSNO_AVAILABLE_PARALLELISM] = sys_available_parallelism as *const _; table.handle[SYSNO_GET_DENTS64] = sys_getdents64 as *const _; From c0dadd71c5e984e6df4498734a9b5a49626c889f Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Mon, 22 Jun 2026 22:01:12 +0200 Subject: [PATCH 42/48] arch: canonicalize module paths and enable common-os on aarch64 Route all scheduler/mm/task accesses through explicit arch::kernel and arch::mm paths instead of the re-exported arch::core_local and arch::* shortcuts. Split prepare_fork_child_stack (kernel) from prepare_mem_copy_on_write (mm) at the fork call site accordingly. Re-export prepare_fork_child_stack, BasePageSize/PageSize and clear_user_space for both arches, and gate the common_os module on aarch64 as well so common-os/fork builds there. --- src/arch/aarch64/kernel/mod.rs | 2 ++ src/arch/aarch64/mm/mod.rs | 2 +- src/arch/aarch64/mm/paging.rs | 9 +++++---- src/arch/mod.rs | 6 ++++++ src/arch/x86_64/kernel/mod.rs | 2 ++ src/arch/x86_64/mm/mod.rs | 5 +++-- src/arch/x86_64/mm/paging.rs | 4 ++-- src/executor/device.rs | 1 - src/lib.rs | 5 ++++- src/mm/vma.rs | 2 +- src/scheduler/mod.rs | 10 +++++----- src/scheduler/task/mod.rs | 12 ++++++------ 12 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/arch/aarch64/kernel/mod.rs b/src/arch/aarch64/kernel/mod.rs index 7a5e35c218..b6bd1c9939 100644 --- a/src/arch/aarch64/kernel/mod.rs +++ b/src/arch/aarch64/kernel/mod.rs @@ -22,6 +22,8 @@ use core::sync::atomic::{AtomicPtr, AtomicU32, Ordering}; pub(crate) use self::interrupts::wakeup_core; pub(crate) use self::processor::set_oneshot_timer; +#[cfg(feature = "common-os")] +pub use self::scheduler::prepare_fork_child_stack; use crate::arch::aarch64::kernel::core_local::*; use crate::arch::aarch64::mm::paging::{BasePageSize, PageSize}; use crate::config::*; diff --git a/src/arch/aarch64/mm/mod.rs b/src/arch/aarch64/mm/mod.rs index 083c4266d0..ca673b31ae 100644 --- a/src/arch/aarch64/mm/mod.rs +++ b/src/arch/aarch64/mm/mod.rs @@ -3,7 +3,7 @@ pub(crate) mod paging; #[cfg(feature = "common-os")] pub use paging::{ clear_user_space, create_new_root_page_table, - drop_user_space, get_current_root_page_table, + drop_user_space, }; #[cfg(all(feature = "common-os", feature = "fork"))] diff --git a/src/arch/aarch64/mm/paging.rs b/src/arch/aarch64/mm/paging.rs index f80f7846e0..261d8a7f64 100644 --- a/src/arch/aarch64/mm/paging.rs +++ b/src/arch/aarch64/mm/paging.rs @@ -86,7 +86,7 @@ bitflags! { } impl PageTableEntryFlags { - #[expect(dead_code)] + #[allow(dead_code)] pub fn present(&mut self) -> &mut Self { self.insert(PageTableEntryFlags::PRESENT); self @@ -109,7 +109,7 @@ impl PageTableEntryFlags { self } - #[expect(dead_code)] + #[allow(dead_code)] pub fn read_only(&mut self) -> &mut Self { self.insert(PageTableEntryFlags::READ_ONLY); self @@ -1193,9 +1193,10 @@ pub fn create_new_root_page_table() -> usize { } /// Returns the physical address of the current task's root page table. +#[allow(dead_code)] #[cfg(feature = "common-os")] pub fn get_current_root_page_table() -> usize { - use crate::arch::core_local::core_scheduler; + use crate::arch::kernel::core_local::core_scheduler; core_scheduler() .get_current_task() .borrow() @@ -1340,7 +1341,7 @@ pub fn prepare_mem_copy_on_write() { /// simply `memcpy` the parent's stack pages into the child's mapping. #[cfg(feature = "common-os")] pub fn copy_kernel_stack_to(stack_address: usize) { - use crate::arch::core_local::core_scheduler; + use crate::arch::kernel::core_local::core_scheduler; let virt_addr = core_scheduler() .get_current_task() diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 6596a53ac4..723e3d8e29 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -7,6 +7,10 @@ mod aarch64; #[cfg(target_arch = "aarch64")] pub(crate) use self::aarch64::*; +#[cfg(target_arch = "aarch64")] +pub use self::aarch64::mm::paging::{BasePageSize, PageSize}; +#[cfg(target_arch = "aarch64")] +pub use self::aarch64::mm::clear_user_space; #[cfg(target_arch = "riscv64")] mod riscv64; @@ -17,3 +21,5 @@ pub(crate) use self::riscv64::*; mod x86_64; #[cfg(target_arch = "x86_64")] pub(crate) use self::x86_64::*; +#[cfg(target_arch = "x86_64")] +pub use self::x86_64::mm::{BasePageSize, PageSize, clear_user_space}; \ No newline at end of file diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index f326f770f2..9e6feaf6ae 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -11,6 +11,8 @@ use x86_64::registers::control::{Cr0, Cr4}; pub(crate) use self::apic::{set_oneshot_timer, wakeup_core}; use crate::arch::x86_64::kernel::core_local::*; use crate::env; +#[cfg(feature = "common-os")] +pub use self::switch::prepare_fork_child_stack; #[cfg(feature = "acpi")] pub mod acpi; diff --git a/src/arch/x86_64/mm/mod.rs b/src/arch/x86_64/mm/mod.rs index b17b4f6220..3df7988eb8 100644 --- a/src/arch/x86_64/mm/mod.rs +++ b/src/arch/x86_64/mm/mod.rs @@ -5,7 +5,7 @@ use core::slice; use memory_addresses::arch::x86_64::{PhysAddr, VirtAddr}; #[cfg(feature = "common-os")] -use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; +pub use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; #[cfg(feature = "common-os")] use crate::arch::mm::paging::{PageTableEntryFlags, PageTableEntryFlagsExt}; @@ -64,9 +64,10 @@ pub fn create_new_root_page_table() -> usize { } /// Returns the physical address of the current task's root page table (PML4). +#[allow(dead_code)] #[cfg(feature = "common-os")] pub fn get_current_root_page_table() -> usize { - use crate::arch::core_local::core_scheduler; + use crate::arch::kernel::core_local::core_scheduler; core_scheduler() .get_current_task() .borrow() diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index 114399a5d6..4e07292ef4 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -480,7 +480,7 @@ pub fn clear_user_space() { /// simply `memcpy` the parent's stack pages into the child's mapping. #[cfg(all(feature = "common-os", feature = "fork"))] pub fn copy_kernel_stack_to(stack_address: usize) { - use crate::arch::core_local::core_scheduler; + use crate::arch::kernel::core_local::core_scheduler; let virt_addr = core_scheduler() .get_current_task() @@ -546,7 +546,7 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( ) { use core::arch::asm; - use crate::arch::core_local::{core_scheduler, increment_irq_counter}; + use crate::arch::kernel::core_local::{core_scheduler, increment_irq_counter}; let swapped_gs = unsafe { if stack_frame.as_mut().read().code_segment == SegmentSelector(0x08) { false diff --git a/src/executor/device.rs b/src/executor/device.rs index 3ac1360bdb..8d166d0627 100644 --- a/src/executor/device.rs +++ b/src/executor/device.rs @@ -14,7 +14,6 @@ use smoltcp::socket::dns; use smoltcp::wire::{EthernetAddress, HardwareAddress, IpCidr, Ipv4Address, Ipv4Cidr}; use super::network::{NetworkInterface, NetworkState}; -use crate::arch; use crate::arch::kernel::systemtime; #[cfg(feature = "write-pcap-file")] use crate::drivers::Driver; diff --git a/src/lib.rs b/src/lib.rs index e5791702b2..9dde577d95 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -103,7 +103,10 @@ mod macros; mod logging; pub mod arch; -#[cfg(all(feature = "common-os", target_arch = "x86_64"))] +#[cfg(all( + feature = "common-os", + any(target_arch = "x86_64", target_arch = "aarch64") +))] pub mod common_os; pub mod config; pub mod console; diff --git a/src/mm/vma.rs b/src/mm/vma.rs index a9daaff983..97e988c486 100644 --- a/src/mm/vma.rs +++ b/src/mm/vma.rs @@ -5,7 +5,7 @@ use memory_addresses::VirtAddr; #[cfg(target_arch = "x86_64")] use x86_64::VirtAddr; -use crate::core_local::core_scheduler; +use crate::arch::kernel::core_local::core_scheduler; use crate::errno::Errno; /// A contiguous range of virtual addresses with uniform protection diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index 0fc7f379a0..19630d26b6 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -33,7 +33,6 @@ use crate::arch::kernel::{get_processor_count, interrupts}; use crate::errno::Errno; use crate::fd::{Fd, RawFd}; use crate::io; -use crate::kernel::scheduler::TaskStacks; #[cfg(feature = "common-os")] use crate::mm::vma::VirtualMemoryArea; use crate::scheduler::task::*; @@ -398,7 +397,7 @@ impl PerCoreScheduler { // path) so that the new user-accessible pages get mapped into the // shared root page table. let tls_base = if let Some(ref template) = tls_template { - arch::mm::allocate_thread_tls(template) + crate::arch::mm::allocate_thread_tls(template) } else { 0 }; @@ -454,7 +453,7 @@ impl PerCoreScheduler { debug!("Creating user thread {tid} with priority {prio} on core {core_id}"); if wakeup { - arch::wakeup_core(core_id); + kernel::wakeup_core(core_id); } tid @@ -1173,7 +1172,8 @@ pub fn join(id: TaskId) -> Result<(), ()> { all(feature = "common-os", feature = "fork") ))] pub unsafe fn fork() -> TaskId { - use crate::arch::{prepare_fork_child_stack, prepare_mem_copy_on_write}; + use crate::arch::kernel::prepare_fork_child_stack; + use crate::arch::mm::prepare_mem_copy_on_write; let core_id = SPAWN_COUNTER.fetch_add(1, Ordering::SeqCst) % get_processor_count(); @@ -1288,7 +1288,7 @@ pub unsafe fn fork() -> TaskId { }; if wakeup { - arch::wakeup_core(core_id); + kernel::wakeup_core(core_id); } debug!("Child was created and has the id {tid}"); diff --git a/src/scheduler/task/mod.rs b/src/scheduler/task/mod.rs index fe030e745e..0ad273ff4f 100644 --- a/src/scheduler/task/mod.rs +++ b/src/scheduler/task/mod.rs @@ -29,9 +29,9 @@ use x86_64::VirtAddr; use self::tls::Tls; use super::timer_interrupts::{Source, create_timer_abs}; use crate::arch; -use crate::arch::core_local::*; +use crate::arch::kernel::core_local::*; use crate::arch::kernel::processor::{self, FPUState}; -use crate::arch::scheduler::TaskStacks; +use crate::arch::kernel::scheduler::TaskStacks; #[cfg(not(feature = "common-os"))] use crate::env; #[cfg(not(feature = "common-os"))] @@ -83,7 +83,7 @@ impl RootPageTable { impl Drop for RootPageTable { fn drop(&mut self) { if self.owned { - arch::drop_user_space(self.pml4_phys); + arch::mm::drop_user_space(self.pml4_phys); } } } @@ -553,7 +553,7 @@ impl Task { #[cfg(not(feature = "common-os"))] tls: None, #[cfg(feature = "common-os")] - root_page_table: Arc::new(RootPageTable::new(arch::create_new_root_page_table())), + root_page_table: Arc::new(RootPageTable::new(arch::mm::create_new_root_page_table())), #[cfg(feature = "common-os")] tls_template: None, #[cfg(feature = "common-os")] @@ -659,7 +659,7 @@ impl Task { prio: task_prio, last_stack_pointer: VirtAddr::zero(), user_stack_pointer: VirtAddr::zero(), - last_fpu_state: arch::processor::FPUState::new(), + last_fpu_state: FPUState::new(), core_id, stacks, object_map, @@ -695,7 +695,7 @@ impl Task { prio: task_prio, last_stack_pointer, user_stack_pointer, - last_fpu_state: arch::processor::FPUState::new(), + last_fpu_state: FPUState::new(), core_id, stacks, object_map, From d658dca594ac147f335c9240ba6edaa685519cc9 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sun, 12 Jul 2026 12:21:48 +0200 Subject: [PATCH 43/48] style: apply rustfmt and tidy up imports across arch modules Reformat long expressions (VMA inserts, page-table walks, error messages) to rustfmt line-wrapping, normalize indentation in clear_user_space, and regroup/sort imports (module- vs. function-local, std/crate blocks). Reorder pub-use reexports in arch/mod.rs and fix the missing trailing newline. No functional change. --- src/arch/aarch64/kernel/interrupts.rs | 49 +++++----- src/arch/aarch64/kernel/mod.rs | 77 ++++++++++++---- src/arch/aarch64/kernel/scheduler.rs | 3 +- src/arch/aarch64/mm/mod.rs | 15 +--- src/arch/aarch64/mm/paging.rs | 48 +++++----- src/arch/mod.rs | 8 +- src/arch/x86_64/kernel/mod.rs | 124 ++++++++++++++++---------- src/arch/x86_64/mm/mod.rs | 24 +++-- src/arch/x86_64/mm/paging.rs | 59 ++++++------ 9 files changed, 235 insertions(+), 172 deletions(-) diff --git a/src/arch/aarch64/kernel/interrupts.rs b/src/arch/aarch64/kernel/interrupts.rs index 05cf63dacf..4701dbb8df 100644 --- a/src/arch/aarch64/kernel/interrupts.rs +++ b/src/arch/aarch64/kernel/interrupts.rs @@ -219,9 +219,7 @@ pub(crate) extern "C" fn do_sync(state: &mut State) { * COW-marked page, which can happen e.g. when the kernel writes * argv/envp into the freshly-mapped user page during the loader path. */ - if ec == ESR_EL1::EC::Value::DataAbortCurrentEL - || ec == ESR_EL1::EC::Value::DataAbortLowerEL - { + if ec == ESR_EL1::EC::Value::DataAbortCurrentEL || ec == ESR_EL1::EC::Value::DataAbortLowerEL { #[cfg(feature = "common-os")] increment_irq_counter(PAGE_FAULT_IRQ); @@ -236,29 +234,35 @@ pub(crate) extern "C" fn do_sync(state: &mut State) { let is_permission_fault = (0b00_1100..=0b00_1111).contains(&dfsc); #[cfg(all(feature = "common-os", feature = "fork"))] - if is_write - && is_permission_fault - && do_cow_fault(VirtAddr::new(far)) - { + if is_write && is_permission_fault && do_cow_fault(VirtAddr::new(far)) { // Faulting instruction is retried on `eret` from the trap. return; } #[cfg(feature = "common-os")] { - use align_address::Align; use core::ops::Bound; + + use align_address::Align; + use crate::mm::FrameAlloc; use crate::mm::vma::VirtualMemoryAreaProt; let addr = VirtAddr::new(far).align_down(BasePageSize::SIZE); let current_task = core_scheduler().get_current_task(); - let current_task_borrowed = current_task.borrow(); - let guard = current_task_borrowed.vmas.read(); + let current_task_borrowed = current_task.borrow(); + let guard = current_task_borrowed.vmas.read(); - if let Some((_, vma)) = guard.range((Bound::Unbounded, Bound::Included(addr))).next_back() { + if let Some((_, vma)) = guard + .range((Bound::Unbounded, Bound::Included(addr))) + .next_back() + { if addr >= vma.start && addr < vma.end { - let layout = PageLayout::from_size_align(BasePageSize::SIZE as usize, BasePageSize::SIZE as usize).unwrap(); + let layout = PageLayout::from_size_align( + BasePageSize::SIZE as usize, + BasePageSize::SIZE as usize, + ) + .unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let physaddr = PhysAddr::from(frame_range.start()); let mut flags = PageTableEntryFlags::empty(); @@ -270,15 +274,15 @@ pub(crate) extern "C" fn do_sync(state: &mut State) { flags.execute_disable(); } - paging::map::( - addr, - physaddr, - 1, - flags, - ); + paging::map::(addr, physaddr, 1, flags); // clear page - let slice = unsafe { core::slice::from_raw_parts_mut(addr.as_mut_ptr() as *mut u8, BasePageSize::SIZE as usize) }; + let slice = unsafe { + core::slice::from_raw_parts_mut( + addr.as_mut_ptr() as *mut u8, + BasePageSize::SIZE as usize, + ) + }; slice.fill(0); #[cfg(feature = "fork")] @@ -292,16 +296,13 @@ pub(crate) extern "C" fn do_sync(state: &mut State) { let kind = dfsc_kind(dfsc); let access = if is_write { "write" } else { "read" }; error!("Current stack pointer {state:p}"); - error!( - "Unhandled data abort: {kind} on {access} of {far:#x} (DFSC={dfsc:#x})" - ); + error!("Unhandled data abort: {kind} on {access} of {far:#x} (DFSC={dfsc:#x})"); error!("Exception return address {:#x}", ELR_EL1.get()); error!("Thread ID register {:#x}", TPIDR_EL0.get()); error!("Table Base Register {:#x}", TTBR0_EL1.get()); error!("Exception Syndrome Register {esr:#x}"); - if let Some(irqid) = - GicCpuInterface::get_and_acknowledge_interrupt(InterruptGroup::Group1) + if let Some(irqid) = GicCpuInterface::get_and_acknowledge_interrupt(InterruptGroup::Group1) { GicCpuInterface::end_interrupt(irqid, InterruptGroup::Group1); } else { diff --git a/src/arch/aarch64/kernel/mod.rs b/src/arch/aarch64/kernel/mod.rs index b6bd1c9939..d4e77b9464 100644 --- a/src/arch/aarch64/kernel/mod.rs +++ b/src/arch/aarch64/kernel/mod.rs @@ -14,12 +14,18 @@ pub mod serial; mod start; pub mod systemtime; -use alloc::alloc::alloc; -use core::alloc::Layout; +use alloc::alloc::{Layout, alloc}; +#[cfg(feature = "common-os")] +use core::arch::asm; use core::arch::global_asm; use core::ptr; +#[cfg(feature = "common-os")] +use core::slice; use core::sync::atomic::{AtomicPtr, AtomicU32, Ordering}; +#[cfg(feature = "common-os")] +use memory_addresses::{PhysAddr, VirtAddr}; + pub(crate) use self::interrupts::wakeup_core; pub(crate) use self::processor::set_oneshot_timer; #[cfg(feature = "common-os")] @@ -209,10 +215,10 @@ where use crate::arch::aarch64::mm::paging::{self, PageTableEntryFlags}; use crate::fd::{Fd, RawFd}; - use crate::mm::{FrameAlloc, PageRangeAllocator}; #[cfg(feature = "fork")] use crate::mm::frame_ref_inc; use crate::mm::vma::*; + use crate::mm::{FrameAlloc, PageRangeAllocator}; // Each process has its own object map. let mut object_map = HashMap::>, RandomState>::with_hasher( @@ -238,7 +244,22 @@ where code_size / BasePageSize::SIZE as usize, flags, ); - core_scheduler().get_current_task().borrow_mut().vmas.write().insert(VirtAddr::from(USER_START), VirtualMemoryArea::new(VirtAddr::from(USER_START), VirtAddr::from(USER_START + code_size).align_up(BasePageSize::SIZE), VirtualMemoryAreaProt::READ|VirtualMemoryAreaProt::WRITE|VirtualMemoryAreaProt::EXECUTE, MemoryType::CODE)); + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( + VirtAddr::from(USER_START), + VirtualMemoryArea::new( + VirtAddr::from(USER_START), + VirtAddr::from(USER_START + code_size).align_up(BasePageSize::SIZE), + VirtualMemoryAreaProt::READ + | VirtualMemoryAreaProt::WRITE + | VirtualMemoryAreaProt::EXECUTE, + MemoryType::CODE, + ), + ); let loader_start_ptr = ptr::with_exposed_provenance_mut(USER_START.as_usize()); let code_slice = unsafe { slice::from_raw_parts_mut(loader_start_ptr, code_size) }; @@ -270,10 +291,24 @@ where tls_memsz / BasePageSize::SIZE as usize, flags, ); - core_scheduler().get_current_task().borrow_mut().vmas.write().insert(tls_virt, VirtualMemoryArea::new(tls_virt, (tls_virt + tls_memsz).align_up(BasePageSize::SIZE), VirtualMemoryAreaProt::READ|VirtualMemoryAreaProt::WRITE, MemoryType::TLS)); - - let block = - unsafe { slice::from_raw_parts_mut(tls_virt.as_mut_ptr(), tls_offset + tls_size as usize) }; + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( + tls_virt, + VirtualMemoryArea::new( + tls_virt, + (tls_virt + tls_memsz).align_up(BasePageSize::SIZE), + VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE, + MemoryType::TLS, + ), + ); + + let block = unsafe { + slice::from_raw_parts_mut(tls_virt.as_mut_ptr(), tls_offset + tls_size as usize) + }; for elem in block.iter_mut() { *elem = 0; } @@ -289,11 +324,10 @@ where let tls_init = func(code_slice, Some(tls_image))?; if let Some(init) = tls_init { - let template = - Arc::new(crate::scheduler::task::TlsTemplate { - size: tls_size as usize, - init, - }); + let template = Arc::new(crate::scheduler::task::TlsTemplate { + size: tls_size as usize, + init, + }); core_scheduler() .get_current_task() .borrow_mut() @@ -364,10 +398,10 @@ pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) use crate::arch::aarch64::kernel::scheduler::TaskStacks; use crate::arch::aarch64::mm::paging::{self, PageTableEntryFlags}; - use crate::mm::{FrameAlloc, PageRangeAllocator}; #[cfg(feature = "fork")] use crate::mm::frame_ref_inc; use crate::mm::vma::*; + use crate::mm::{FrameAlloc, PageRangeAllocator}; debug!("Create new file descriptor table"); core_scheduler().recreate_objmap().unwrap(); @@ -386,7 +420,20 @@ pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) USER_STACK_SIZE / BasePageSize::SIZE as usize, flags, ); - core_scheduler().get_current_task().borrow_mut().vmas.write().insert(USER_STACK, VirtualMemoryArea::new(USER_STACK, USER_STACK+USER_STACK_SIZE, VirtualMemoryAreaProt::READ|VirtualMemoryAreaProt::WRITE, MemoryType::STACK)); + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( + USER_STACK, + VirtualMemoryArea::new( + USER_STACK, + USER_STACK + USER_STACK_SIZE, + VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE, + MemoryType::STACK, + ), + ); #[cfg(feature = "fork")] for i in 0..USER_STACK_SIZE / BasePageSize::SIZE as usize { frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); diff --git a/src/arch/aarch64/kernel/scheduler.rs b/src/arch/aarch64/kernel/scheduler.rs index 633a60272e..9a51d97c23 100644 --- a/src/arch/aarch64/kernel/scheduler.rs +++ b/src/arch/aarch64/kernel/scheduler.rs @@ -334,8 +334,7 @@ impl Task { ) { unsafe { // Debug marker at the very top of the kernel stack. - let mut stack = self.stacks.get_kernel_stack() - + self.stacks.get_kernel_stack_size() + let mut stack = self.stacks.get_kernel_stack() + self.stacks.get_kernel_stack_size() - TaskStacks::MARKER_SIZE; *stack.as_mut_ptr::() = 0xdead_beefu64; diff --git a/src/arch/aarch64/mm/mod.rs b/src/arch/aarch64/mm/mod.rs index ca673b31ae..23ab7a26f7 100644 --- a/src/arch/aarch64/mm/mod.rs +++ b/src/arch/aarch64/mm/mod.rs @@ -1,15 +1,9 @@ pub(crate) mod paging; #[cfg(feature = "common-os")] -pub use paging::{ - clear_user_space, create_new_root_page_table, - drop_user_space, -}; - +pub use paging::{clear_user_space, create_new_root_page_table, drop_user_space}; #[cfg(all(feature = "common-os", feature = "fork"))] -pub use paging::{ - copy_current_root_page_table, copy_kernel_stack_to, prepare_mem_copy_on_write -}; +pub use paging::{copy_current_root_page_table, copy_kernel_stack_to, prepare_mem_copy_on_write}; use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; @@ -28,10 +22,7 @@ pub fn allocate_thread_tls(template: &crate::scheduler::task::TlsTemplate) -> u6 use free_list::PageLayout; use memory_addresses::arch::aarch64::{PhysAddr, VirtAddr}; - use crate::arch::aarch64::mm::paging::{ - self, BasePageSize, PageSize, PageTableEntryFlags, - }; - use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; + use crate::arch::aarch64::mm::paging::{self, BasePageSize, PageSize, PageTableEntryFlags}; #[cfg(feature = "fork")] use crate::mm::frame_ref_inc; diff --git a/src/arch/aarch64/mm/paging.rs b/src/arch/aarch64/mm/paging.rs index 261d8a7f64..bb11c5d620 100644 --- a/src/arch/aarch64/mm/paging.rs +++ b/src/arch/aarch64/mm/paging.rs @@ -854,9 +854,7 @@ pub fn do_cow_fault(faulting_addr: VirtAddr) -> bool { } let l1 = unsafe { - &mut *ptr::with_exposed_provenance_mut::>( - l0_entry.address().as_usize(), - ) + &mut *ptr::with_exposed_provenance_mut::>(l0_entry.address().as_usize()) }; let l1_entry = &mut l1.entries[l1_idx]; if !l1_entry.is_present() || !l1_entry.is_table_or_4kib_page() { @@ -864,9 +862,7 @@ pub fn do_cow_fault(faulting_addr: VirtAddr) -> bool { } let l2 = unsafe { - &mut *ptr::with_exposed_provenance_mut::>( - l1_entry.address().as_usize(), - ) + &mut *ptr::with_exposed_provenance_mut::>(l1_entry.address().as_usize()) }; let l2_entry = &mut l2.entries[l2_idx]; if !l2_entry.is_present() || !l2_entry.is_table_or_4kib_page() { @@ -874,9 +870,7 @@ pub fn do_cow_fault(faulting_addr: VirtAddr) -> bool { } let l3 = unsafe { - &mut *ptr::with_exposed_provenance_mut::>( - l2_entry.address().as_usize(), - ) + &mut *ptr::with_exposed_provenance_mut::>(l2_entry.address().as_usize()) }; let l3_entry = &mut l3.entries[l3_idx]; @@ -1107,23 +1101,23 @@ pub fn drop_user_space(l0_phys: usize) { clear_l0(l0_phys); // The L0 table is not loaded on any core, so no TLB flush is necessary. - let range = - free_list::PageRange::new(l0_phys, l0_phys + BasePageSize::SIZE as usize).unwrap(); + let range = free_list::PageRange::new(l0_phys, l0_phys + BasePageSize::SIZE as usize).unwrap(); unsafe { FrameAlloc::deallocate(range) }; } /// Clear the user-space portion of the currently active address space. #[cfg(feature = "common-os")] pub fn clear_user_space() { - use crate::fd::STDERR_FILENO; - use crate::core_scheduler; use aarch64_cpu::registers::TTBR0_EL1; + use crate::core_scheduler; + use crate::fd::STDERR_FILENO; + core_scheduler() - .get_current_task() - .borrow() - .vmas - .write() + .get_current_task() + .borrow() + .vmas + .write() .clear(); core_scheduler() .get_current_task_object_map() @@ -1157,8 +1151,7 @@ pub fn create_new_root_page_table() -> usize { let new_l0_phys = frame_range.start(); let cur_l0_phys = TTBR0_EL1.get_baddr() as usize; - let cur_l0 = - unsafe { &*ptr::with_exposed_provenance::>(cur_l0_phys) }; + let cur_l0 = unsafe { &*ptr::with_exposed_provenance::>(cur_l0_phys) }; let new_l0 = unsafe { &mut *ptr::with_exposed_provenance_mut::>(new_l0_phys) }; @@ -1218,8 +1211,7 @@ pub fn copy_current_root_page_table() -> usize { unsafe { &mut *ptr::with_exposed_provenance_mut::>(new_l0_phys) }; let cur_l0_phys = TTBR0_EL1.get_baddr() as usize; - let cur_l0 = - unsafe { &*ptr::with_exposed_provenance::>(cur_l0_phys) }; + let cur_l0 = unsafe { &*ptr::with_exposed_provenance::>(cur_l0_phys) }; // Inherit kernel L0 entries verbatim (sharing the L1/L2/L3 tables // below). Only the user-space slot needs a deep copy with COW so @@ -1247,9 +1239,8 @@ pub fn copy_current_root_page_table() -> usize { let cur_l1 = unsafe { &*ptr::with_exposed_provenance::>(cur_l0_entry.address().as_usize()) }; - let new_l1 = unsafe { - &mut *ptr::with_exposed_provenance_mut::>(new_l1_phys) - }; + let new_l1 = + unsafe { &mut *ptr::with_exposed_provenance_mut::>(new_l1_phys) }; for l1_idx in 0..512usize { let cur_l1_entry = &cur_l1.entries[l1_idx]; @@ -1303,10 +1294,11 @@ pub fn copy_current_root_page_table() -> usize { // The child holds an additional reference to every // user-space frame in this page table. for entry in new_l3.entries.iter() { - let flags = PageTableEntryFlags::from_bits_truncate( - entry.physical_address_and_flags, - ); - if flags.contains(PageTableEntryFlags::PRESENT|PageTableEntryFlags::USER_ACCESSIBLE) { + let flags = + PageTableEntryFlags::from_bits_truncate(entry.physical_address_and_flags); + if flags.contains( + PageTableEntryFlags::PRESENT | PageTableEntryFlags::USER_ACCESSIBLE, + ) { crate::mm::frame_ref_inc(entry.address()); } } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 723e3d8e29..6664d4308d 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -6,11 +6,11 @@ #[cfg(target_arch = "aarch64")] mod aarch64; #[cfg(target_arch = "aarch64")] -pub(crate) use self::aarch64::*; +pub use self::aarch64::mm::clear_user_space; #[cfg(target_arch = "aarch64")] pub use self::aarch64::mm::paging::{BasePageSize, PageSize}; #[cfg(target_arch = "aarch64")] -pub use self::aarch64::mm::clear_user_space; +pub(crate) use self::aarch64::*; #[cfg(target_arch = "riscv64")] mod riscv64; @@ -20,6 +20,6 @@ pub(crate) use self::riscv64::*; #[cfg(target_arch = "x86_64")] mod x86_64; #[cfg(target_arch = "x86_64")] -pub(crate) use self::x86_64::*; +pub use self::x86_64::mm::{BasePageSize, PageSize, clear_user_space}; #[cfg(target_arch = "x86_64")] -pub use self::x86_64::mm::{BasePageSize, PageSize, clear_user_space}; \ No newline at end of file +pub(crate) use self::x86_64::*; diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index 9e6feaf6ae..34b8047327 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -6,13 +6,15 @@ use core::slice; use core::sync::atomic::{AtomicPtr, AtomicU32, Ordering}; use hermit_entry::boot_info::RawBootInfo; +#[cfg(feature = "common-os")] +use memory_addresses::{PhysAddr, VirtAddr}; use x86_64::registers::control::{Cr0, Cr4}; pub(crate) use self::apic::{set_oneshot_timer, wakeup_core}; -use crate::arch::x86_64::kernel::core_local::*; -use crate::env; #[cfg(feature = "common-os")] pub use self::switch::prepare_fork_child_stack; +use crate::arch::x86_64::kernel::core_local::*; +use crate::env; #[cfg(feature = "acpi")] pub mod acpi; @@ -205,28 +207,27 @@ where Option<&'static mut [u8]>, ) -> Result>, ()>, { + use alloc::sync::Arc; + + use ahash::RandomState; use align_address::Align; use free_list::PageLayout; - use memory_addresses::{PhysAddr, VirtAddr}; + use hashbrown::HashMap; + use hermit_sync::RwSpinLock; use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; use crate::arch::x86_64::mm::paging::{self, PageTableEntryFlags, PageTableEntryFlagsExt}; - use crate::mm::{FrameAlloc, PageRangeAllocator}; - use crate::fd::{Fd, RawFd}; - use crate::fd::stdio; + use crate::fd::{Fd, RawFd, stdio}; #[cfg(feature = "fork")] use crate::mm::frame_ref_inc; use crate::mm::vma::*; + use crate::mm::{FrameAlloc, PageRangeAllocator}; // each process has to provide its own object_map // => create a new one - let mut object_map = HashMap::< - RawFd, - Arc>, - RandomState, - >::with_hasher( - RandomState::with_seeds(0, 0, 0, 0), - ); + let mut object_map = HashMap::>, RandomState>::with_hasher( + RandomState::with_seeds(0, 0, 0, 0), + ); stdio::setup(&mut object_map); core_scheduler().set_current_task_object_map(Arc::new(RwSpinLock::new(object_map))); @@ -252,17 +253,25 @@ where // Convert at the boundary so the BTreeMap-keyed insert type-checks. { let start: x86_64::VirtAddr = VirtAddr::from(USER_START).into(); - let end: x86_64::VirtAddr = - VirtAddr::from(USER_START + code_size).align_up(BasePageSize::SIZE).into(); - core_scheduler().get_current_task().borrow_mut().vmas.write().insert( - start, - VirtualMemoryArea::new( + let end: x86_64::VirtAddr = VirtAddr::from(USER_START + code_size) + .align_up(BasePageSize::SIZE) + .into(); + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( start, - end, - VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE | VirtualMemoryAreaProt::EXECUTE, - MemoryType::CODE, - ), - ); + VirtualMemoryArea::new( + start, + end, + VirtualMemoryAreaProt::READ + | VirtualMemoryAreaProt::WRITE + | VirtualMemoryAreaProt::EXECUTE, + MemoryType::CODE, + ), + ); } let loader_start_ptr = ptr::with_exposed_provenance_mut(USER_START.as_usize()); @@ -287,29 +296,34 @@ where let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().user().execute_disable(); - let tls_virt = VirtAddr::from(USER_START.as_usize() + code_size + BasePageSize::SIZE as usize); + let tls_virt = + VirtAddr::from(USER_START.as_usize() + code_size + BasePageSize::SIZE as usize); paging::map::( tls_virt, physaddr, tls_memsz / BasePageSize::SIZE as usize, flags, ); - + { let start: x86_64::VirtAddr = tls_virt.into(); - let end: x86_64::VirtAddr = - (tls_virt + tls_memsz).align_up(BasePageSize::SIZE).into(); - core_scheduler().get_current_task().borrow_mut().vmas.write().insert( - start, - VirtualMemoryArea::new( + let end: x86_64::VirtAddr = (tls_virt + tls_memsz).align_up(BasePageSize::SIZE).into(); + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( start, - end, - VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE, - MemoryType::TLS - ), - ); + VirtualMemoryArea::new( + start, + end, + VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE, + MemoryType::TLS, + ), + ); } - + let block = unsafe { slice::from_raw_parts_mut(tls_virt.as_mut_ptr(), tls_offset + tcb_size) }; for elem in block.iter_mut() { @@ -329,12 +343,10 @@ where let tls_init = func(code_slice, Some(block))?; if let Some(init) = tls_init { - let template = Arc::new( - crate::scheduler::task::TlsTemplate { - size: tls_offset, - init, - }, - ); + let template = Arc::new(crate::scheduler::task::TlsTemplate { + size: tls_offset, + init, + }); core_scheduler() .get_current_task() .borrow_mut() @@ -354,16 +366,17 @@ pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) use align_address::Align; use free_list::PageLayout; - use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; - use x86_64::structures::paging::PageTableFlags as PageTableEntryFlags; + use x86_64::structures::paging::{ + PageSize, PageTableFlags as PageTableEntryFlags, Size4KiB as BasePageSize, + }; - use crate::arch::x86_64::mm::paging::PageTableEntryFlagsExt; - use crate::arch::x86_64::kernel::scheduler::TaskStacks; - use crate::mm::{FrameAlloc, PageRangeAllocator}; use crate::arch::mm::paging; + use crate::arch::x86_64::kernel::scheduler::TaskStacks; + use crate::arch::x86_64::mm::paging::PageTableEntryFlagsExt; #[cfg(feature = "fork")] use crate::mm::frame_ref_inc; use crate::mm::vma::*; + use crate::mm::{FrameAlloc, PageRangeAllocator}; debug!("Create new file descriptor table"); core_scheduler().recreate_objmap().unwrap(); @@ -384,8 +397,21 @@ pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) ); { let start: x86_64::VirtAddr = USER_STACK.into(); - let end: x86_64::VirtAddr = (USER_STACK+USER_STACK_SIZE).into(); - core_scheduler().get_current_task().borrow_mut().vmas.write().insert(start, VirtualMemoryArea::new(start, end, VirtualMemoryAreaProt::READ|VirtualMemoryAreaProt::WRITE, MemoryType::STACK)); + let end: x86_64::VirtAddr = (USER_STACK + USER_STACK_SIZE).into(); + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( + start, + VirtualMemoryArea::new( + start, + end, + VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE, + MemoryType::STACK, + ), + ); } #[cfg(feature = "fork")] for i in 0..USER_STACK_SIZE / BasePageSize::SIZE as usize { diff --git a/src/arch/x86_64/mm/mod.rs b/src/arch/x86_64/mm/mod.rs index 3df7988eb8..568b1344dd 100644 --- a/src/arch/x86_64/mm/mod.rs +++ b/src/arch/x86_64/mm/mod.rs @@ -4,17 +4,17 @@ pub(crate) mod paging; use core::slice; use memory_addresses::arch::x86_64::{PhysAddr, VirtAddr}; +#[cfg(all(feature = "common-os", feature = "fork"))] +pub use paging::copy_kernel_stack_to; +/// Copy the kernel stack pages of the current task to a new base address. +#[cfg(feature = "common-os")] +pub use paging::{clear_user_space, drop_user_space}; #[cfg(feature = "common-os")] pub use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; #[cfg(feature = "common-os")] use crate::arch::mm::paging::{PageTableEntryFlags, PageTableEntryFlagsExt}; use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; -/// Copy the kernel stack pages of the current task to a new base address. -#[cfg(feature = "common-os")] -pub use paging::{drop_user_space, clear_user_space}; -#[cfg(all(feature = "common-os", feature = "fork"))] -pub use paging::copy_kernel_stack_to; #[cfg(feature = "common-os")] pub fn create_new_root_page_table() -> usize { @@ -82,10 +82,8 @@ pub fn copy_current_root_page_table() -> usize { use core::ptr; use free_list::PageLayout; - use x86_64::structures::paging::PageTable; use x86_64::registers::control::Cr3; - - use crate::mm::{FrameAlloc, PageRangeAllocator}; + use x86_64::structures::paging::PageTable; let layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); @@ -184,7 +182,10 @@ pub fn copy_current_root_page_table() -> usize { // The child now holds an additional user reference to every // frame in this page table. for entry in new_pt.iter() { - if entry.flags().contains(PageTableFlags::PRESENT|PageTableFlags::USER_ACCESSIBLE) { + if entry + .flags() + .contains(PageTableFlags::PRESENT | PageTableFlags::USER_ACCESSIBLE) + { crate::mm::frame_ref_inc(entry.addr().into()); } } @@ -224,11 +225,8 @@ pub fn prepare_mem_copy_on_write() { pub fn allocate_thread_tls(template: &crate::scheduler::task::TlsTemplate) -> u64 { use align_address::Align; use free_list::PageLayout; - use memory_addresses::{PhysAddr, VirtAddr}; - use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; + use x86_64::structures::paging::Size4KiB as BasePageSize; - use crate::arch::x86_64::mm::paging::{self, PageTableEntryFlags, PageTableEntryFlagsExt}; - use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; #[cfg(feature = "fork")] use crate::mm::frame_ref_inc; diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index 4e07292ef4..61f8677056 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -106,7 +106,7 @@ impl PageTableEntryFlagsExt for PageTableEntryFlags { self } - #[cfg(all(feature = "common-os", feature = "fork"))] + #[cfg(all(feature = "common-os", feature = "fork"))] fn copy_on_write(&mut self) -> &mut Self { self.remove(PageTableEntryFlags::WRITABLE); self.remove(PageTableEntryFlags::DIRTY); @@ -338,8 +338,11 @@ pub fn mark_user_pages_copy_on_write() { }; for pt_idx in 0..512usize { let pt_entry = &mut pt[pt_idx]; - if pt_entry.flags().contains(PageTableEntryFlags::PRESENT|PageTableEntryFlags::WRITABLE|PageTableEntryFlags::USER_ACCESSIBLE) - && !pt_entry.flags().contains(PageTableEntryFlags::BIT_9) + if pt_entry.flags().contains( + PageTableEntryFlags::PRESENT + | PageTableEntryFlags::WRITABLE + | PageTableEntryFlags::USER_ACCESSIBLE, + ) && !pt_entry.flags().contains(PageTableEntryFlags::BIT_9) { let new_flags = *pt_entry.flags().copy_on_write(); pt_entry.set_addr(pt_entry.addr(), new_flags); @@ -401,10 +404,9 @@ fn clear_pml4(pml4_phys: usize) { for pt_idx in 0..512usize { let pt_entry = &mut pt[pt_idx]; - if pt_entry - .flags() - .contains(PageTableEntryFlags::PRESENT | PageTableEntryFlags::USER_ACCESSIBLE) - { + if pt_entry.flags().contains( + PageTableEntryFlags::PRESENT | PageTableEntryFlags::USER_ACCESSIBLE, + ) { let phys_addr = PhysAddr::new(pt_entry.addr().as_u64()); #[cfg(feature = "fork")] if crate::mm::frame_ref_dec(phys_addr) { @@ -448,14 +450,14 @@ pub fn drop_user_space(pml4_phys: usize) { #[cfg(feature = "common-os")] pub fn clear_user_space() { - use crate::fd::STDERR_FILENO; use crate::core_scheduler; + use crate::fd::STDERR_FILENO; core_scheduler() - .get_current_task() - .borrow() - .vmas - .write() + .get_current_task() + .borrow() + .vmas + .write() .clear(); core_scheduler() .get_current_task_object_map() @@ -555,7 +557,7 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( true } }; - + increment_irq_counter(14); let faulting_addr = Cr2::read().unwrap(); @@ -610,16 +612,23 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( { use core::ops::Bound; - use crate::mm::FrameAlloc; + use crate::mm::vma::VirtualMemoryAreaProt; let current_task = core_scheduler().get_current_task(); - let current_task_borrowed = current_task.borrow(); - let guard = current_task_borrowed.vmas.read(); + let current_task_borrowed = current_task.borrow(); + let guard = current_task_borrowed.vmas.read(); - if let Some((_, vma)) = guard.range((Bound::Unbounded, Bound::Included(virtaddr))).next_back() { + if let Some((_, vma)) = guard + .range((Bound::Unbounded, Bound::Included(virtaddr))) + .next_back() + { if virtaddr >= vma.start && virtaddr < vma.end { - let layout = PageLayout::from_size_align(BasePageSize::SIZE as usize, BasePageSize::SIZE as usize).unwrap(); + let layout = PageLayout::from_size_align( + BasePageSize::SIZE as usize, + BasePageSize::SIZE as usize, + ) + .unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let physaddr = PhysAddr::from(frame_range.start()); let mut flags = PageTableEntryFlags::empty(); @@ -633,15 +642,15 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( // `virtaddr` is `x86_64::VirtAddr` (from `Cr2::read`); the // `map` helper signs the parameter as `memory_addresses::VirtAddr`. - map::( - virtaddr.into(), - physaddr, - 1, - flags, - ); + map::(virtaddr.into(), physaddr, 1, flags); // clear page - let slice = unsafe { core::slice::from_raw_parts_mut(virtaddr.as_mut_ptr() as *mut u8, BasePageSize::SIZE as usize) }; + let slice = unsafe { + core::slice::from_raw_parts_mut( + virtaddr.as_mut_ptr() as *mut u8, + BasePageSize::SIZE as usize, + ) + }; slice.fill(0); #[cfg(feature = "fork")] From 1a461d9d43d4f262da65f58155a89c4f662305dc Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sun, 12 Jul 2026 12:38:04 +0200 Subject: [PATCH 44/48] remove some typos --- src/arch/x86_64/mm/paging.rs | 2 +- src/scheduler/task/mod.rs | 2 +- typos.toml | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index 61f8677056..d1faab3a3b 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -674,7 +674,7 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( // A spawned user thread whose entry function returns comes back here // with RIP=0: the `thread_start` wrapper in std's hermit pal just // returns, and the kernel-crafted return slot at the top of the user - // stack was zero-initialised. Treat an instruction-fetch fault at 0 + // stack was zero-initialized. Treat an instruction-fetch fault at 0 // in ring 3 as a clean thread exit. if error_code.contains(PageFaultErrorCode::USER_MODE) && error_code.contains(PageFaultErrorCode::INSTRUCTION_FETCH) diff --git a/src/scheduler/task/mod.rs b/src/scheduler/task/mod.rs index 0ad273ff4f..860de58647 100644 --- a/src/scheduler/task/mod.rs +++ b/src/scheduler/task/mod.rs @@ -459,7 +459,7 @@ impl PriorityTaskQueue { } } -/// Per-process TLS template used to initialise newly spawned threads. +/// Per-process TLS template used to initialize newly spawned threads. /// /// `size` is the byte size of the TLS data block (the offset at which the /// TCB / thread pointer lives, also the value that the parent's main thread diff --git a/typos.toml b/typos.toml index c283906f15..b1d26e8f31 100644 --- a/typos.toml +++ b/typos.toml @@ -5,6 +5,7 @@ extend-ignore-identifiers-re = [ "ist", "parm", "sie", + "Spectre", ] [default.extend-identifiers] From bc8fb709ab809c6e2fb33a8bd1858178dd365bee Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sun, 12 Jul 2026 19:22:40 +0200 Subject: [PATCH 45/48] remove clippy and compiler warnings --- src/arch/aarch64/kernel/interrupts.rs | 62 ++++++++++----------- src/arch/aarch64/kernel/mod.rs | 14 ++--- src/arch/aarch64/kernel/processor.rs | 8 ++- src/arch/aarch64/kernel/scheduler.rs | 17 +++--- src/arch/aarch64/mm/mod.rs | 4 +- src/arch/aarch64/mm/paging.rs | 6 +-- src/arch/mod.rs | 8 ++- src/arch/x86_64/kernel/core_local.rs | 2 + src/arch/x86_64/kernel/mod.rs | 11 ++-- src/arch/x86_64/mm/paging.rs | 78 +++++++++++++-------------- src/mm/vma.rs | 5 +- src/scheduler/mod.rs | 4 +- src/scheduler/task/mod.rs | 5 +- src/syscalls/table.rs | 2 +- 14 files changed, 115 insertions(+), 111 deletions(-) diff --git a/src/arch/aarch64/kernel/interrupts.rs b/src/arch/aarch64/kernel/interrupts.rs index 4701dbb8df..bc260d9dd0 100644 --- a/src/arch/aarch64/kernel/interrupts.rs +++ b/src/arch/aarch64/kernel/interrupts.rs @@ -179,6 +179,7 @@ pub(crate) extern "C" fn do_irq(_state: &State) -> *mut usize { #[unsafe(no_mangle)] pub(crate) extern "C" fn do_sync(state: &mut State) { + #[cfg(all(feature = "common-os", feature = "fork"))] use crate::arch::mm::paging::do_cow_fault; let esr = ESR_EL1.get(); @@ -255,41 +256,40 @@ pub(crate) extern "C" fn do_sync(state: &mut State) { if let Some((_, vma)) = guard .range((Bound::Unbounded, Bound::Included(addr))) - .next_back() + .next_back() && addr >= vma.start + && addr < vma.end { - if addr >= vma.start && addr < vma.end { - let layout = PageLayout::from_size_align( - BasePageSize::SIZE as usize, - BasePageSize::SIZE as usize, - ) - .unwrap(); - let frame_range = FrameAlloc::allocate(layout).unwrap(); - let physaddr = PhysAddr::from(frame_range.start()); - let mut flags = PageTableEntryFlags::empty(); - flags.normal().user(); - if vma.prot.contains(VirtualMemoryAreaProt::WRITE) { - flags.writable(); - } - if vma.prot.contains(VirtualMemoryAreaProt::EXECUTE) { - flags.execute_disable(); - } + let layout = PageLayout::from_size_align( + BasePageSize::SIZE as usize, + BasePageSize::SIZE as usize, + ) + .unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let physaddr = PhysAddr::from(frame_range.start()); + let mut flags = PageTableEntryFlags::empty(); + flags.normal().user(); + if vma.prot.contains(VirtualMemoryAreaProt::WRITE) { + flags.writable(); + } + if vma.prot.contains(VirtualMemoryAreaProt::EXECUTE) { + flags.execute_disable(); + } - paging::map::(addr, physaddr, 1, flags); + paging::map::(addr, physaddr, 1, flags); - // clear page - let slice = unsafe { - core::slice::from_raw_parts_mut( - addr.as_mut_ptr() as *mut u8, - BasePageSize::SIZE as usize, - ) - }; - slice.fill(0); + // clear page + let slice = unsafe { + core::slice::from_raw_parts_mut( + addr.as_mut_ptr::().cast::(), + BasePageSize::SIZE as usize, + ) + }; + slice.fill(0); - #[cfg(feature = "fork")] - crate::mm::frame_ref_inc(physaddr); + #[cfg(feature = "fork")] + crate::mm::frame_ref_inc(physaddr); - return; - } + return; } } @@ -379,7 +379,7 @@ fn dispatch_svc64(state: &mut State) { if nr >= NO_SYSCALLS { error!("Invalid syscall number {nr}"); - state.x0 = (-i32::from(Errno::Nosys)) as u32 as u64; + state.x0 = u64::from((-i32::from(Errno::Nosys)) as u32); return; } diff --git a/src/arch/aarch64/kernel/mod.rs b/src/arch/aarch64/kernel/mod.rs index d4e77b9464..93945ff5b5 100644 --- a/src/arch/aarch64/kernel/mod.rs +++ b/src/arch/aarch64/kernel/mod.rs @@ -14,7 +14,8 @@ pub mod serial; mod start; pub mod systemtime; -use alloc::alloc::{Layout, alloc}; +use alloc::alloc::alloc; +use core::alloc::Layout; #[cfg(feature = "common-os")] use core::arch::asm; use core::arch::global_asm; @@ -28,7 +29,7 @@ use memory_addresses::{PhysAddr, VirtAddr}; pub(crate) use self::interrupts::wakeup_core; pub(crate) use self::processor::set_oneshot_timer; -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub use self::scheduler::prepare_fork_child_stack; use crate::arch::aarch64::kernel::core_local::*; use crate::arch::aarch64::mm::paging::{BasePageSize, PageSize}; @@ -197,6 +198,7 @@ const USER_STACK_SIZE: usize = 0x8000; /// Map the user-mode binary into the address space and run the ELF-loader /// closure against the freshly-mapped pages. Mirrors the x86_64 sibling /// (`arch::x86_64::kernel::load_application`). +#[allow(clippy::result_unit_err)] #[cfg(feature = "common-os")] pub fn load_application(code_size: u64, tls_size: u64, func: F) -> Result<(), ()> where @@ -250,10 +252,10 @@ where .vmas .write() .insert( - VirtAddr::from(USER_START), + USER_START, VirtualMemoryArea::new( - VirtAddr::from(USER_START), - VirtAddr::from(USER_START + code_size).align_up(BasePageSize::SIZE), + USER_START, + (USER_START + code_size).align_up(BasePageSize::SIZE), VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE | VirtualMemoryAreaProt::EXECUTE, @@ -284,7 +286,7 @@ where let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().user().execute_disable(); - let tls_virt = VirtAddr::from(USER_START + code_size + BasePageSize::SIZE as usize); + let tls_virt = USER_START + code_size + BasePageSize::SIZE as usize; paging::map::( tls_virt, physaddr, diff --git a/src/arch/aarch64/kernel/processor.rs b/src/arch/aarch64/kernel/processor.rs index 5eef230ab0..7676314999 100644 --- a/src/arch/aarch64/kernel/processor.rs +++ b/src/arch/aarch64/kernel/processor.rs @@ -242,9 +242,13 @@ pub fn shutdown(error_code: i32) -> ! { // PSCI did not terminate (no dispatcher, or call returned). Fall back // to semihosting if the feature is compiled in — useful under TCG // where it correctly propagates `error_code` to the host shell. - if cfg!(feature = "semihosting") { + #[cfg(feature = "semihosting")] + { semihosting::process::exit(error_code) - } else { + } + + #[cfg(not(feature = "semihosting"))] + { // Last resort: park the CPU forever. loop { aarch64_cpu::asm::wfe(); diff --git a/src/arch/aarch64/kernel/scheduler.rs b/src/arch/aarch64/kernel/scheduler.rs index 9a51d97c23..427be8aa8e 100644 --- a/src/arch/aarch64/kernel/scheduler.rs +++ b/src/arch/aarch64/kernel/scheduler.rs @@ -1,10 +1,11 @@ //! Architecture dependent interface to initialize a task use core::arch::naked_asm; +#[cfg(feature = "common-os")] +use core::mem; use core::sync::atomic::Ordering; -use core::{mem, ptr}; -use aarch64_cpu::asm::barrier::{SY, dsb, isb}; +use aarch64_cpu::asm::barrier::{SY, isb}; use aarch64_cpu::registers::*; use align_address::Align; use free_list::{PageLayout, PageRange}; @@ -206,7 +207,7 @@ impl TaskStacks { } /// Returns the start address of the stack region (virt_addr of CommonStack). - #[cfg(feature = "common-os")] + #[cfg(all(feature = "common-os", feature = "fork"))] pub fn get_stack_virt_addr(&self) -> VirtAddr { match self { TaskStacks::Boot(stacks) => stacks.stack, @@ -215,7 +216,7 @@ impl TaskStacks { } /// Returns total size of all stacks combined. - #[cfg(feature = "common-os")] + #[cfg(all(feature = "common-os", feature = "fork"))] pub fn get_total_stack_size(&self) -> usize { match self { TaskStacks::Boot(_) => KERNEL_STACK_SIZE, @@ -344,7 +345,7 @@ impl Task { // register file. stack -= size_of::(); let state = stack.as_mut_ptr::(); - ptr::write_bytes(state.cast::(), 0, size_of::()); + state.cast::().write_bytes(0, size_of::()); // Initial user stack: top of the user-stack region with the // usual debug marker. AAPCS64 doesn't require any extra slop @@ -425,8 +426,6 @@ impl TaskFrame for Task { #[unsafe(no_mangle)] pub(crate) extern "C" fn get_last_stack_pointer() -> u64 { - use aarch64_cpu::asm::barrier::{ISH, ISHST}; - // Trap next FPU instruction so we can lazily restore FPU state CPACR_EL1.modify(CPACR_EL1::FPEN::TrapEl0El1); isb(SY); @@ -450,6 +449,8 @@ pub(crate) extern "C" fn get_last_stack_pointer() -> u64 { .as_usize() as u64; let cur_pt = TTBR0_EL1.get_baddr(); if cur_pt != new_pt { + use aarch64_cpu::asm::barrier::{ISH, ISHST, dsb}; + // Memory-barrier sequence per ARM ARM D8.13.2: DSB ISHST // ensures all prior PT updates are observable; the MSR // installs the new translation base; ISB flushes the @@ -526,7 +527,7 @@ pub unsafe fn prepare_fork_child_stack( // the State (ELR_EL1 = post-SVC user PC, SPSR_EL1 = EL0t, SP_EL0 // = user stack, x1..x30 = parent's user regs) is already correct. unsafe { - let state = ptr::with_exposed_provenance_mut::(child_state_addr); + let state = core::ptr::with_exposed_provenance_mut::(child_state_addr); (*state).x0 = 0; } diff --git a/src/arch/aarch64/mm/mod.rs b/src/arch/aarch64/mm/mod.rs index 23ab7a26f7..25a3a17e61 100644 --- a/src/arch/aarch64/mm/mod.rs +++ b/src/arch/aarch64/mm/mod.rs @@ -1,9 +1,9 @@ pub(crate) mod paging; -#[cfg(feature = "common-os")] -pub use paging::{clear_user_space, create_new_root_page_table, drop_user_space}; #[cfg(all(feature = "common-os", feature = "fork"))] pub use paging::{copy_current_root_page_table, copy_kernel_stack_to, prepare_mem_copy_on_write}; +#[cfg(feature = "common-os")] +pub use paging::{create_new_root_page_table, drop_user_space}; use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; diff --git a/src/arch/aarch64/mm/paging.rs b/src/arch/aarch64/mm/paging.rs index bb11c5d620..2bbde74aca 100644 --- a/src/arch/aarch64/mm/paging.rs +++ b/src/arch/aarch64/mm/paging.rs @@ -226,7 +226,7 @@ impl PageTableEntry { } /// Return whether this entry is a 4KiB page. - #[cfg(all(feature = "common-os", feature = "fork"))] + #[cfg(feature = "common-os")] fn is_table_or_4kib_page(&self) -> bool { (self.physical_address_and_flags & PageTableEntryFlags::TABLE_OR_4KIB_PAGE.bits()) != 0 } @@ -803,7 +803,7 @@ fn flush_tlb_all() { } /// Invalidate one TLB entry by virtual address (broadcast to all cores). -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] fn flush_tlb_one(virt: VirtAddr) { dsb(ISHST); unsafe { @@ -1331,7 +1331,7 @@ pub fn prepare_mem_copy_on_write() { /// base address. Used by `fork`: the child's `TaskStacks::new` has already /// allocated and mapped fresh physical frames at `stack_address`, so we /// simply `memcpy` the parent's stack pages into the child's mapping. -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub fn copy_kernel_stack_to(stack_address: usize) { use crate::arch::kernel::core_local::core_scheduler; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 6664d4308d..5865b987a5 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -5,10 +5,8 @@ // https://github.com/rust-lang/rust/issues/158400 #[cfg(target_arch = "aarch64")] mod aarch64; -#[cfg(target_arch = "aarch64")] -pub use self::aarch64::mm::clear_user_space; -#[cfg(target_arch = "aarch64")] -pub use self::aarch64::mm::paging::{BasePageSize, PageSize}; +#[cfg(all(target_arch = "aarch64", feature = "common-os"))] +pub use self::aarch64::mm::paging::{BasePageSize, PageSize, clear_user_space}; #[cfg(target_arch = "aarch64")] pub(crate) use self::aarch64::*; @@ -19,7 +17,7 @@ pub(crate) use self::riscv64::*; #[cfg(target_arch = "x86_64")] mod x86_64; -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] pub use self::x86_64::mm::{BasePageSize, PageSize, clear_user_space}; #[cfg(target_arch = "x86_64")] pub(crate) use self::x86_64::*; diff --git a/src/arch/x86_64/kernel/core_local.rs b/src/arch/x86_64/kernel/core_local.rs index d6bc47a181..15473c8fc1 100644 --- a/src/arch/x86_64/kernel/core_local.rs +++ b/src/arch/x86_64/kernel/core_local.rs @@ -29,6 +29,7 @@ pub(crate) struct CoreLocal { /// Start address of the kernel stack pub kernel_stack: Cell<*mut u8>, /// Current address of the user stack during a system call + #[cfg(feature = "common-os")] pub user_stack: Cell<*mut u8>, /// Interface to the interrupt counters irq_statistics: &'static IrqStatistics, @@ -58,6 +59,7 @@ impl CoreLocal { scheduler: Cell::new(ptr::null_mut()), tss: Cell::new(ptr::null_mut()), kernel_stack: Cell::new(ptr::null_mut()), + #[cfg(feature = "common-os")] user_stack: Cell::new(ptr::null_mut()), irq_statistics, ex: StaticLocalExecutor::new(), diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index 34b8047327..fb2d76c25b 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -11,7 +11,7 @@ use memory_addresses::{PhysAddr, VirtAddr}; use x86_64::registers::control::{Cr0, Cr4}; pub(crate) use self::apic::{set_oneshot_timer, wakeup_core}; -#[cfg(feature = "common-os")] +#[cfg(all(feature = "common-os", feature = "fork"))] pub use self::switch::prepare_fork_child_stack; use crate::arch::x86_64::kernel::core_local::*; use crate::env; @@ -199,6 +199,7 @@ const USER_STACK: VirtAddr = VirtAddr::new(0x0180_0000_0000 - USER_STACK_SIZE as #[cfg(feature = "common-os")] const USER_STACK_SIZE: usize = 0x8000; +#[allow(clippy::result_unit_err)] #[cfg(feature = "common-os")] pub fn load_application(code_size: u64, tls_size: u64, func: F) -> Result<(), ()> where @@ -243,7 +244,7 @@ where let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().user().execute_enable(); paging::map::( - VirtAddr::from(USER_START), + USER_START, physaddr, code_size / BasePageSize::SIZE as usize, flags, @@ -252,10 +253,8 @@ where // this file's local `use` brings in `memory_addresses::VirtAddr`. // Convert at the boundary so the BTreeMap-keyed insert type-checks. { - let start: x86_64::VirtAddr = VirtAddr::from(USER_START).into(); - let end: x86_64::VirtAddr = VirtAddr::from(USER_START + code_size) - .align_up(BasePageSize::SIZE) - .into(); + let start: x86_64::VirtAddr = USER_START.into(); + let end: x86_64::VirtAddr = (USER_START + code_size).align_up(BasePageSize::SIZE).into(); core_scheduler() .get_current_task() .borrow_mut() diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index d1faab3a3b..c98c7acf03 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -622,52 +622,52 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( if let Some((_, vma)) = guard .range((Bound::Unbounded, Bound::Included(virtaddr))) .next_back() + && virtaddr >= vma.start + && virtaddr < vma.end { - if virtaddr >= vma.start && virtaddr < vma.end { - let layout = PageLayout::from_size_align( - BasePageSize::SIZE as usize, - BasePageSize::SIZE as usize, - ) - .unwrap(); - let frame_range = FrameAlloc::allocate(layout).unwrap(); - let physaddr = PhysAddr::from(frame_range.start()); - let mut flags = PageTableEntryFlags::empty(); - flags.normal().user(); - if vma.prot.contains(VirtualMemoryAreaProt::WRITE) { - flags.writable(); - } - if vma.prot.contains(VirtualMemoryAreaProt::EXECUTE) { - flags.execute_disable(); - } + let layout = PageLayout::from_size_align( + BasePageSize::SIZE as usize, + BasePageSize::SIZE as usize, + ) + .unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let physaddr = PhysAddr::from(frame_range.start()); + let mut flags = PageTableEntryFlags::empty(); + flags.normal().user(); + if vma.prot.contains(VirtualMemoryAreaProt::WRITE) { + flags.writable(); + } + if vma.prot.contains(VirtualMemoryAreaProt::EXECUTE) { + flags.execute_disable(); + } - // `virtaddr` is `x86_64::VirtAddr` (from `Cr2::read`); the - // `map` helper signs the parameter as `memory_addresses::VirtAddr`. - map::(virtaddr.into(), physaddr, 1, flags); + // `virtaddr` is `x86_64::VirtAddr` (from `Cr2::read`); the + // `map` helper signs the parameter as `memory_addresses::VirtAddr`. + map::(virtaddr.into(), physaddr, 1, flags); - // clear page - let slice = unsafe { - core::slice::from_raw_parts_mut( - virtaddr.as_mut_ptr() as *mut u8, - BasePageSize::SIZE as usize, - ) - }; - slice.fill(0); + // clear page + let slice = unsafe { + core::slice::from_raw_parts_mut( + virtaddr.as_mut_ptr::().cast::(), + BasePageSize::SIZE as usize, + ) + }; + slice.fill(0); - #[cfg(feature = "fork")] - crate::mm::frame_ref_inc(physaddr); + #[cfg(feature = "fork")] + crate::mm::frame_ref_inc(physaddr); - // Restore user GS before returning to ring 3; the COW path - // above does the same. Without this, iret leaves the kernel - // GS-base installed, the user's next FS/GS access reads - // kernel state and the program either misbehaves or wedges. - if swapped_gs { - unsafe { - core::arch::asm!("swapgs", options(nostack)); - } + // Restore user GS before returning to ring 3; the COW path + // above does the same. Without this, iret leaves the kernel + // GS-base installed, the user's next FS/GS access reads + // kernel state and the program either misbehaves or wedges. + if swapped_gs { + unsafe { + core::arch::asm!("swapgs", options(nostack)); } - - return; } + + return; } } diff --git a/src/mm/vma.rs b/src/mm/vma.rs index 97e988c486..ff72cccd57 100644 --- a/src/mm/vma.rs +++ b/src/mm/vma.rs @@ -39,6 +39,7 @@ impl VirtualMemoryArea { } } +#[allow(clippy::upper_case_acronyms)] #[derive(Debug, Copy, Clone)] pub enum MemoryType { HEAP, @@ -86,7 +87,7 @@ pub extern "C" fn sys_mmap( prot_flags: VirtualMemoryAreaProt, ret: &mut *mut u8, ) -> i32 { - if *ret == core::ptr::null_mut() { + if (*ret).is_null() { let current_task = core_scheduler().get_current_task(); let current_task_borrowed = current_task.borrow(); let mut guard = current_task_borrowed.vmas.write(); @@ -154,5 +155,5 @@ pub extern "C" fn sys_mmap( return 0; } - return -i32::from(Errno::Inval); + -i32::from(Errno::Inval) } diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index 19630d26b6..1033ec644c 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -14,7 +14,7 @@ use ahash::RandomState; use crossbeam_utils::Backoff; use hashbrown::{HashMap, hash_map}; use hermit_sync::*; -#[cfg(all(feature = "common-os", not(target_arch = "x86_64")))] +#[cfg(target_arch = "aarch64")] use memory_addresses::VirtAddr; #[cfg(target_arch = "riscv64")] use riscv::register::sstatus; @@ -862,7 +862,7 @@ impl PerCoreScheduler { crate::mm::FrameAlloc::free_space() >> 10 ); #[cfg(not(all(target_arch = "x86_64", feature = "common-os")))] - debug!("Cleaned up task {}", id); + debug!("Cleaned up task {id}"); } } diff --git a/src/scheduler/task/mod.rs b/src/scheduler/task/mod.rs index 860de58647..06f3a43737 100644 --- a/src/scheduler/task/mod.rs +++ b/src/scheduler/task/mod.rs @@ -28,17 +28,14 @@ use x86_64::VirtAddr; #[cfg(not(feature = "common-os"))] use self::tls::Tls; use super::timer_interrupts::{Source, create_timer_abs}; +#[cfg(feature = "common-os")] use crate::arch; use crate::arch::kernel::core_local::*; use crate::arch::kernel::processor::{self, FPUState}; use crate::arch::kernel::scheduler::TaskStacks; #[cfg(not(feature = "common-os"))] -use crate::env; -#[cfg(not(feature = "common-os"))] use crate::fd::stdio; use crate::fd::{Fd, RawFd}; -#[cfg(not(feature = "common-os"))] -use crate::fd::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; #[cfg(feature = "common-os")] use crate::mm::vma::VirtualMemoryArea; use crate::scheduler::CoreId; diff --git a/src/syscalls/table.rs b/src/syscalls/table.rs index efaecf2938..2a8476c2f5 100644 --- a/src/syscalls/table.rs +++ b/src/syscalls/table.rs @@ -134,7 +134,7 @@ const SYSNO_GETADDRINFO: usize = 52; /// number of the system call `freeaddrinfo` #[cfg(any(feature = "net", feature = "virtio-vsock"))] const SYSNO_FREEADDRINFO: usize = 53; -/// number of the system call `availale_parallelism` +/// number of the system call `available_parallelism` const SYSNO_AVAILABLE_PARALLELISM: usize = 54; /// number of the system call `getdents64` const SYSNO_GET_DENTS64: usize = 55; From 511c6324c96895f44325f441bc2ef63b55162ae3 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sun, 12 Jul 2026 22:24:28 +0200 Subject: [PATCH 46/48] pass argument vector and environment to user processes jump_to_user_land takes the argument vector and the environment as owned CStrings and places both as NULL-terminated pointer arrays on the user stack; envp is handed over in rdx (x86_64) / x2 (aarch64). argc, argv, and envp are bound as fixed register operands in the inline assembly. With plain in(reg) operands, the register allocator is free to hand rdi/rsi/rdx (x0-x2) to other operands, which the mov sequence then clobbered before eret/iretq. The weak symbols sys_spawn_process and sys_exec take optional NULL-terminated argv/envp arrays; the loader provides the actual implementation. --- src/arch/aarch64/kernel/mod.rs | 51 ++++++++++++++++++++++----------- src/arch/x86_64/kernel/mod.rs | 52 +++++++++++++++++++++++----------- src/syscalls/table.rs | 12 ++++++-- 3 files changed, 79 insertions(+), 36 deletions(-) diff --git a/src/arch/aarch64/kernel/mod.rs b/src/arch/aarch64/kernel/mod.rs index 93945ff5b5..c3d161ef71 100644 --- a/src/arch/aarch64/kernel/mod.rs +++ b/src/arch/aarch64/kernel/mod.rs @@ -392,9 +392,11 @@ fn set_user_tpidr_el0(value: u64) { /// new PC, SPSR_EL1 the new PSTATE (mode bits select EL0t), and SP_EL0 the /// user stack. Per AAPCS64, `argc` lives in `x0` and `argv` in `x1`. #[cfg(feature = "common-os")] -pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) -> ! { - use alloc::ffi::CString; - +pub unsafe fn jump_to_user_land( + entry_point: usize, + args: alloc::vec::Vec, + envs: alloc::vec::Vec, +) -> ! { use align_address::Align; use free_list::PageLayout; @@ -441,27 +443,40 @@ pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); } - // Place the argv pointer array on the user stack. - let stack_pointer = stack_top - arg.len() * size_of::<*mut u8>(); + // Place the argv and envp pointer arrays on the user stack. Both + // arrays follow the C convention and are terminated by a null pointer. + let ptr_count = args.len() + 1 + envs.len() + 1; + let stack_pointer = stack_top - ptr_count * size_of::<*mut u8>(); let stack_ptr = ptr::with_exposed_provenance_mut::<*mut u8>(stack_pointer); - let argv = unsafe { slice::from_raw_parts_mut(stack_ptr, arg.len()) }; - let len = arg.iter().fold(0, |acc, x| acc + x.len() + 1); + let arrays = unsafe { slice::from_raw_parts_mut(stack_ptr, ptr_count) }; + let (argv, envp) = arrays.split_at_mut(args.len() + 1); + let len = args + .iter() + .chain(envs.iter()) + .fold(0, |acc, x| acc + x.as_bytes_with_nul().len()); // AAPCS64 requires SP to be 16-byte aligned at function entry. let stack_pointer = (stack_pointer - len).align_down(16); let mut pos: usize = 0; - for (i, s) in arg.iter().enumerate() { - let s = CString::new(*s).unwrap(); + for (dst, s) in argv + .iter_mut() + .zip(args.iter()) + .chain(envp.iter_mut().zip(envs.iter())) + { let bytes = s.as_bytes_with_nul(); - argv[i] = ptr::with_exposed_provenance_mut::(stack_pointer + pos); + *dst = ptr::with_exposed_provenance_mut::(stack_pointer + pos); pos += bytes.len(); unsafe { - argv[i].copy_from_nonoverlapping(bytes.as_ptr(), bytes.len()); + dst.copy_from_nonoverlapping(bytes.as_ptr(), bytes.len()); } } + argv[args.len()] = ptr::null_mut(); + envp[envs.len()] = ptr::null_mut(); - drop(arg); + let argc = args.len(); + drop(args); + drop(envs); debug!("Jump to user space at 0x{entry_point:x}, stack pointer 0x{stack_pointer:x}"); @@ -480,10 +495,11 @@ pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) "msr sp_el0, {sp}", "msr elr_el1, {pc}", "msr spsr_el1, {spsr}", - "mov x0, {argc}", - "mov x1, {argv}", // Clear scratch registers so EL1 state cannot leak into EL0. - "mov x2, xzr", "mov x3, xzr", "mov x4, xzr", "mov x5, xzr", + // `x0`, `x1`, and `x2` carry argc, argv, and envp and are + // bound as fixed register operands below, so the register + // allocator cannot hand them out for the other operands. + "mov x3, xzr", "mov x4, xzr", "mov x5, xzr", "mov x6, xzr", "mov x7, xzr", "mov x8, xzr", "mov x9, xzr", "mov x10, xzr", "mov x11, xzr", "mov x12, xzr", "mov x13, xzr", "mov x14, xzr", "mov x15, xzr", "mov x16, xzr", "mov x17, xzr", @@ -495,8 +511,9 @@ pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) sp = in(reg) stack_pointer, pc = in(reg) entry_point, spsr = in(reg) USER_SPSR, - argc = in(reg) argv.len(), - argv = in(reg) argv.as_ptr(), + in("x0") argc, + in("x1") argv.as_ptr(), + in("x2") envp.as_ptr(), options(nostack, noreturn) ); } diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index fb2d76c25b..a762c58085 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -360,9 +360,11 @@ where } #[cfg(feature = "common-os")] -pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) -> ! { - use alloc::ffi::CString; - +pub unsafe fn jump_to_user_land( + entry_point: usize, + args: alloc::vec::Vec, + envs: alloc::vec::Vec, +) -> ! { use align_address::Align; use free_list::PageLayout; use x86_64::structures::paging::{ @@ -417,26 +419,40 @@ pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); } - let stack_pointer = stack_pointer - 128 /* red zone */ - arg.len() * size_of::<*mut u8>(); + // Place the argv and envp pointer arrays on the user stack. Both + // arrays follow the C convention and are terminated by a null pointer. + let ptr_count = args.len() + 1 + envs.len() + 1; + let stack_pointer = stack_pointer - 128 /* red zone */ - ptr_count * size_of::<*mut u8>(); let stack_ptr = ptr::with_exposed_provenance_mut::<*mut u8>(stack_pointer); - let argv = unsafe { slice::from_raw_parts_mut(stack_ptr, arg.len()) }; - let len = arg.iter().fold(0, |acc, x| acc + x.len() + 1); + let arrays = unsafe { slice::from_raw_parts_mut(stack_ptr, ptr_count) }; + let (argv, envp) = arrays.split_at_mut(args.len() + 1); + let len = args + .iter() + .chain(envs.iter()) + .fold(0, |acc, x| acc + x.as_bytes_with_nul().len()); // align stack pointer to fulfill the requirements of the x86_64 ABI let stack_pointer = (stack_pointer - len).align_down(16) - size_of::(); let mut pos: usize = 0; - for (i, s) in arg.iter().enumerate() { - let s = CString::new(*s).unwrap(); + for (dst, s) in argv + .iter_mut() + .zip(args.iter()) + .chain(envp.iter_mut().zip(envs.iter())) + { let bytes = s.as_bytes_with_nul(); - argv[i] = ptr::with_exposed_provenance_mut::(stack_pointer + pos); + *dst = ptr::with_exposed_provenance_mut::(stack_pointer + pos); pos += bytes.len(); unsafe { - argv[i].copy_from_nonoverlapping(bytes.as_ptr(), bytes.len()); + dst.copy_from_nonoverlapping(bytes.as_ptr(), bytes.len()); } } + argv[args.len()] = ptr::null_mut(); + envp[envs.len()] = ptr::null_mut(); - drop(arg); + let argc = args.len(); + drop(args); + drop(envs); debug!("Jump to user space at 0x{entry_point:x}, stack pointer 0x{stack_pointer:x}"); @@ -449,10 +465,11 @@ pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) "push {3}", "push {4}", "push {5}", - "mov rdi, {6}", - "mov rsi, {7}", - // Clear registers so that state cannot leak. - "xor rax, rax", "xor rbx, rbx", "xor rcx, rcx", "xor rdx, rdx", + // Clear registers so that state cannot leak. `rdi`, `rsi`, + // and `rdx` carry argc, argv, and envp and are bound as + // fixed register operands below, so the register allocator + // cannot hand them out for the other `in(reg)` operands. + "xor rax, rax", "xor rbx, rbx", "xor rcx, rcx", "xor r8, r8", "xor r9, r9", "xor r10, r10", "xor r11, r11", "xor r12, r12", "xor r13, r13", "xor r14, r14", "xor r15, r15", @@ -463,8 +480,9 @@ pub unsafe fn jump_to_user_land(entry_point: usize, arg: alloc::vec::Vec<&str>) const 0x1202u64, const 0x2busize, in(reg) entry_point, - in(reg) argv.len(), - in(reg) argv.as_ptr(), + in("rdi") argc, + in("rsi") argv.as_ptr(), + in("rdx") envp.as_ptr(), options(nostack, noreturn) ); } diff --git a/src/syscalls/table.rs b/src/syscalls/table.rs index 2a8476c2f5..d0c27d332e 100644 --- a/src/syscalls/table.rs +++ b/src/syscalls/table.rs @@ -154,14 +154,22 @@ pub(crate) extern "C" fn invalid_syscall(sys_no: u64) -> ! { /// loader will replace this function #[linkage = "weak"] #[unsafe(no_mangle)] -pub extern "C" fn sys_spawn_process(_path: *const c_char) -> i32 { +pub extern "C" fn sys_spawn_process( + _path: *const c_char, + _argv: *const *const c_char, + _envp: *const *const c_char, +) -> i32 { -i32::from(Errno::Nosys) } /// loader will replace this function #[linkage = "weak"] #[unsafe(no_mangle)] -pub extern "C" fn sys_exec(_path: *const c_char) -> i32 { +pub extern "C" fn sys_exec( + _path: *const c_char, + _argv: *const *const c_char, + _envp: *const *const c_char, +) -> i32 { -i32::from(Errno::Nosys) } From 98f8d5e2df77c20c9eba39645925d9027a78938e Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Mon, 13 Jul 2026 22:21:38 +0200 Subject: [PATCH 47/48] add riscv64 support for spawn_process and exec (common-os) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port the common-os process layer to riscv64. User-mode execution is built on the trapframe crate's UserContext::run() loop instead of a hand-written trap path: jump_to_user_land seeds a UserContext (SPP=U, SPIE, SUM, FS=Initial) and services all user traps in user_loop — ecall dispatch through SYSHANDLER_TABLE (number in a7, arguments in a0..a5, return value in a0), VMA-backed lazy page faults, and timer/ external/software interrupts, which keeps user code preemptible. The trap return path of the trapframe crate does not restore gp, which this kernel uses as the CoreLocal pointer, so run() is wrapped in a save/restore of gp. Sv39 only covers 256 GiB, so the user layout is denser than on the other targets: the kernel owns the root slots below 64 GiB, user code starts at 64 GiB, the heap at 128 GiB, and the stack below 192 GiB. Process root tables share the kernel slots with the static kernel root; all 64 kernel L1 tables are pre-allocated at boot so kernel mappings created later are visible in every address space. User PTEs are mapped non-global and satp is switched on task switch. RISC-V has no privileged thread-pointer register, so the initial tp value prepared by load_application is handed to jump_to_user_land through a task-id-keyed map. sstatus.SUM is enabled because the kernel writes user pages directly (program loading, argv/envp, syscall buffers). fork remains ENOSYS on riscv64 for now; waitpid, spawn, and spawn2 (user threads, untested) are enabled. --- src/arch/mod.rs | 2 + src/arch/riscv64/kernel/interrupts.rs | 2 +- src/arch/riscv64/kernel/mod.rs | 475 ++++++++++++++++++++++++++ src/arch/riscv64/kernel/scheduler.rs | 70 +++- src/arch/riscv64/mm/mod.rs | 59 ++++ src/arch/riscv64/mm/paging.rs | 227 +++++++++++- src/lib.rs | 13 +- src/mm/virtualmem.rs | 11 +- src/mm/vma.rs | 7 + src/scheduler/mod.rs | 24 +- src/syscalls/table.rs | 26 +- src/syscalls/tasks.rs | 35 +- 12 files changed, 897 insertions(+), 54 deletions(-) diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 5865b987a5..e58fd5462d 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -12,6 +12,8 @@ pub(crate) use self::aarch64::*; #[cfg(target_arch = "riscv64")] mod riscv64; +#[cfg(all(target_arch = "riscv64", feature = "common-os"))] +pub use self::riscv64::mm::paging::{BasePageSize, PageSize, clear_user_space}; #[cfg(target_arch = "riscv64")] pub(crate) use self::riscv64::*; diff --git a/src/arch/riscv64/kernel/interrupts.rs b/src/arch/riscv64/kernel/interrupts.rs index 379a47373e..b076546844 100644 --- a/src/arch/riscv64/kernel/interrupts.rs +++ b/src/arch/riscv64/kernel/interrupts.rs @@ -261,7 +261,7 @@ pub extern "C" fn trap_handler(tf: &mut TrapFrame) { } /// Handles external interrupts -fn external_handler() { +pub(crate) fn external_handler() { use crate::arch::kernel::core_local::core_scheduler; use crate::scheduler::PerCoreSchedulerExt; diff --git a/src/arch/riscv64/kernel/mod.rs b/src/arch/riscv64/kernel/mod.rs index de549e7aeb..8e383e8581 100644 --- a/src/arch/riscv64/kernel/mod.rs +++ b/src/arch/riscv64/kernel/mod.rs @@ -11,6 +11,8 @@ pub mod serial; mod start; pub mod switch; pub mod systemtime; +#[cfg(feature = "common-os")] +use alloc::ffi::CString; use alloc::vec::Vec; use core::ptr; use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, Ordering}; @@ -106,6 +108,13 @@ fn finish_processor_init() { } trace!("SSTATUS FS: {:?}", sstatus::read().fs()); + // The kernel writes user pages directly (program loading, argv/envp + // setup, syscall buffers), which S-mode may only do with SUM set. + #[cfg(feature = "common-os")] + unsafe { + sstatus::set_sum(); + } + let current_hart_id = get_current_boot_id() as usize; // Add hart to HARTS_AVAILABLE, the hart id is stored in current_boot_id @@ -162,3 +171,469 @@ pub fn boot_next_processor() { pub fn print_statistics() { interrupts::print_statistics(); } + +/// Start of the user-mode binary (Sv39 root slot 64, see +/// `mm::paging::USER_SPACE_START`). +#[cfg(feature = "common-os")] +pub(crate) const USER_START: memory_addresses::VirtAddr = + memory_addresses::VirtAddr::new(crate::arch::riscv64::mm::paging::USER_SPACE_START as u64); +/// Top of the user stack: end of Sv39 root slot 191. The user heap +/// (`mm::vma::HEAP_START_ADDR`) starts at slot 128, so code, heap, and +/// stack live in disjoint 1 GiB root slots of the user region. +#[cfg(feature = "common-os")] +const USER_STACK: memory_addresses::VirtAddr = + memory_addresses::VirtAddr::new(0x30_0000_0000 - USER_STACK_SIZE as u64); +#[cfg(feature = "common-os")] +const USER_STACK_SIZE: usize = 0x8000; + +/// Thread-pointer values prepared by `load_application` for the first +/// user-mode entry of a task, keyed by task id. `jump_to_user_land` +/// consumes the entry when it builds the initial `UserContext`. +/// +/// RISC-V has no privileged thread-pointer register (`tp` is a plain +/// GPR), so the value cannot be stashed in a CSR between the two calls. +#[cfg(feature = "common-os")] +static USER_TP: hermit_sync::InterruptTicketMutex< + alloc::collections::BTreeMap, +> = hermit_sync::InterruptTicketMutex::new(alloc::collections::BTreeMap::new()); + +/// Map the user-mode binary into the address space and run the ELF-loader +/// closure against the freshly-mapped pages. Mirrors the x86_64/aarch64 +/// siblings. +#[allow(clippy::result_unit_err)] +#[cfg(feature = "common-os")] +pub fn load_application(code_size: u64, tls_size: u64, func: F) -> Result<(), ()> +where + F: FnOnce(&'static mut [u8], Option<&'static mut [u8]>) -> Result>, ()>, +{ + use alloc::sync::Arc; + + use ahash::RandomState; + use align_address::Align; + use hashbrown::HashMap; + use hermit_sync::RwSpinLock; + use memory_addresses::PhysAddr; + + use crate::arch::riscv64::kernel::core_local::core_scheduler; + use crate::arch::riscv64::mm::paging::{self, BasePageSize, PageSize, PageTableEntryFlags}; + use crate::fd::{Fd, RawFd}; + use crate::mm::vma::*; + + // Each process has its own object map. + let mut object_map = HashMap::>, RandomState>::with_hasher( + RandomState::with_seeds(0, 0, 0, 0), + ); + crate::fd::stdio::setup(&mut object_map); + core_scheduler().set_current_task_object_map(Arc::new(RwSpinLock::new(object_map))); + + let code_size = (code_size as usize).align_up(BasePageSize::SIZE as usize); + let layout = PageLayout::from_size_align(code_size, BasePageSize::SIZE as usize).unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let physaddr = PhysAddr::from(frame_range.start()); + + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user(); + paging::map::( + USER_START, + physaddr, + code_size / BasePageSize::SIZE as usize, + flags, + ); + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( + USER_START, + VirtualMemoryArea::new( + USER_START, + (USER_START + code_size).align_up(BasePageSize::SIZE), + VirtualMemoryAreaProt::READ + | VirtualMemoryAreaProt::WRITE + | VirtualMemoryAreaProt::EXECUTE, + MemoryType::CODE, + ), + ); + + let loader_start_ptr = ptr::with_exposed_provenance_mut(USER_START.as_usize()); + let code_slice = unsafe { core::slice::from_raw_parts_mut(loader_start_ptr, code_size) }; + + let current_task_id = core_scheduler().get_current_task().borrow().id; + + if tls_size > 0 { + // RISC-V uses TLS Variant I: `tp` points at the first byte of the + // TLS data; the two-word TCB sits directly below it. We allocate + // the TCB plus the TLS image as one contiguous block. + let tcb_size = 2 * size_of::<*mut ()>(); + let tls_offset = tcb_size; + + let tls_memsz = (tls_offset + tls_size as usize).align_up(BasePageSize::SIZE as usize); + let layout = PageLayout::from_size(tls_memsz).unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let physaddr = PhysAddr::from(frame_range.start()); + + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user().execute_disable(); + let tls_virt = USER_START + code_size as u64 + BasePageSize::SIZE; + paging::map::( + tls_virt, + physaddr, + tls_memsz / BasePageSize::SIZE as usize, + flags, + ); + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( + tls_virt, + VirtualMemoryArea::new( + tls_virt, + (tls_virt + tls_memsz as u64).align_up(BasePageSize::SIZE), + VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE, + MemoryType::TLS, + ), + ); + + let block = unsafe { + core::slice::from_raw_parts_mut( + tls_virt.as_mut_ptr::(), + tls_offset + tls_size as usize, + ) + }; + for elem in block.iter_mut() { + *elem = 0; + } + + // Variant I: `tp` points at the TLS data that follows the TCB. + USER_TP + .lock() + .insert(current_task_id, (tls_virt + tls_offset as u64).as_u64()); + + // The ELF loader copies the binary's PT_TLS initial image into the + // region that follows the TCB. + let tls_image = &mut block[tls_offset..]; + let tls_init = func(code_slice, Some(tls_image))?; + + if let Some(init) = tls_init { + let template = Arc::new(crate::scheduler::task::TlsTemplate { + size: tls_size as usize, + init, + }); + core_scheduler() + .get_current_task() + .borrow_mut() + .tls_template = Some(template); + } + + Ok(()) + } else { + // No TLS in the freshly loaded image. Drop any stale value left + // over from a previous image (`exec` re-enters this path). + USER_TP.lock().remove(¤t_task_id); + func(code_slice, None)?; + Ok(()) + } +} + +/// Run `ctx` in user mode until the next trap, preserving the kernel's +/// `gp` register. +/// +/// The trapframe crate's user-trap return path restores the callee-saved +/// registers, `ra`, and `tp` — but not `gp`, which this kernel uses as +/// the `CoreLocal` pointer. Interrupts stay disabled from trap entry +/// until after `gp` is restored, so no kernel code can observe the +/// user's `gp` value. +#[cfg(feature = "common-os")] +fn run_user(ctx: &mut trapframe::UserContext) { + let gp: usize; + unsafe { + core::arch::asm!("mv {}, gp", out(reg) gp); + } + ctx.run(); + unsafe { + core::arch::asm!("mv gp, {}", in(reg) gp); + } +} + +/// Handle a page fault raised in user mode: fault in one page of the +/// surrounding virtual memory area, if any. +/// +/// Returns `true` if the fault was resolved and the faulting instruction +/// can be retried. +#[cfg(feature = "common-os")] +fn do_user_page_fault(fault_addr: usize) -> bool { + use core::ops::Bound; + + use align_address::Align; + use memory_addresses::{PhysAddr, VirtAddr}; + + use crate::arch::riscv64::kernel::core_local::core_scheduler; + use crate::arch::riscv64::mm::paging::{self, BasePageSize, PageSize, PageTableEntryFlags}; + use crate::mm::vma::VirtualMemoryAreaProt; + + let addr = VirtAddr::new(fault_addr as u64).align_down(BasePageSize::SIZE); + let current_task = core_scheduler().get_current_task(); + let current_task_borrowed = current_task.borrow(); + let guard = current_task_borrowed.vmas.read(); + + if let Some((_, vma)) = guard + .range((Bound::Unbounded, Bound::Included(addr))) + .next_back() + && addr >= vma.start + && addr < vma.end + { + let layout = + PageLayout::from_size_align(BasePageSize::SIZE as usize, BasePageSize::SIZE as usize) + .unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let physaddr = PhysAddr::from(frame_range.start()); + + let mut flags = PageTableEntryFlags::empty(); + flags.normal().user(); + if vma.prot.contains(VirtualMemoryAreaProt::WRITE) { + flags.writable(); + } + if !vma.prot.contains(VirtualMemoryAreaProt::EXECUTE) { + flags.execute_disable(); + } + + paging::map::(addr, physaddr, 1, flags); + + // Clear the page through the identity mapping of physical memory, + // so this works independently of `sstatus.SUM`. + let slice = unsafe { + core::slice::from_raw_parts_mut( + ptr::with_exposed_provenance_mut::(physaddr.as_usize()), + BasePageSize::SIZE as usize, + ) + }; + slice.fill(0); + + return true; + } + + false +} + +/// Dispatch a system call raised by `ecall` from user mode. +/// +/// Calling convention (mirrors the x86_64/aarch64 variants of this +/// kernel): syscall number in `a7`, up to six arguments in `a0`..`a5`, +/// return value in `a0`. +#[cfg(feature = "common-os")] +fn dispatch_syscall(ctx: &mut trapframe::UserContext) { + use crate::syscalls::table::{NO_SYSCALLS, SYSHANDLER_TABLE, sys_invalid}; + + // Resume execution after the 4-byte `ecall` instruction. + ctx.sepc += 4; + + let nr = ctx.general.a7; + if nr >= NO_SYSCALLS { + crate::syscalls::table::invalid_syscall(nr as u64); + } + + let handler_ptr = SYSHANDLER_TABLE.handler(nr); + if handler_ptr == sys_invalid as *const usize { + crate::syscalls::table::invalid_syscall(nr as u64); + } + + let f: extern "C" fn(u64, u64, u64, u64, u64, u64) -> u64 = + unsafe { core::mem::transmute(handler_ptr) }; + + // The handlers may block; run them with interrupts enabled like the + // x86_64 syscall path does. + unsafe { sstatus::set_sie() }; + let result = f( + ctx.general.a0 as u64, + ctx.general.a1 as u64, + ctx.general.a2 as u64, + ctx.general.a3 as u64, + ctx.general.a4 as u64, + ctx.general.a5 as u64, + ); + unsafe { sstatus::clear_sie() }; + + ctx.general.a0 = result as usize; +} + +/// Enter user mode at `entry` with the given stack pointer, thread +/// pointer, and argument registers, and service all traps the user code +/// raises. This function never returns: the process leaves through the +/// `exit` system call (or is torn down after a fatal fault). +#[cfg(feature = "common-os")] +pub(crate) fn user_loop( + entry: usize, + stack_pointer: usize, + thread_pointer: u64, + args: [usize; 3], +) -> ! { + // sstatus value for user-mode entry, derived from the current state: + // SPP = User, SPIE = 1 (interrupts enabled after `sret`), SUM = 1 + // (syscall handlers access user buffers), FS = Initial (user code may + // use the FPU; the scheduler tracks the dirty state). + const SSTATUS_SPIE: usize = 1 << 5; + const SSTATUS_SPP: usize = 1 << 8; + const SSTATUS_FS_MASK: usize = 0b11 << 13; + const SSTATUS_FS_INITIAL: usize = 0b01 << 13; + const SSTATUS_SUM: usize = 1 << 18; + + let sstatus_raw: usize; + unsafe { + core::arch::asm!("csrr {}, sstatus", out(reg) sstatus_raw); + } + let user_sstatus = (sstatus_raw & !(SSTATUS_SPP | SSTATUS_FS_MASK)) + | SSTATUS_SPIE + | SSTATUS_FS_INITIAL + | SSTATUS_SUM; + + let mut ctx = trapframe::UserContext::default(); + ctx.sepc = entry; + ctx.sstatus = user_sstatus; + ctx.general.sp = stack_pointer; + ctx.general.tp = thread_pointer as usize; + ctx.general.a0 = args[0]; + ctx.general.a1 = args[1]; + ctx.general.a2 = args[2]; + + debug!("Jump to user space at {entry:#x}, stack pointer {stack_pointer:#x}"); + + use riscv::interrupt::{Exception, Interrupt, Trap}; + use riscv::register::{scause, stval}; + + use crate::arch::riscv64::kernel::core_local::core_scheduler; + use crate::scheduler::PerCoreSchedulerExt; + + loop { + run_user(&mut ctx); + + let scause = scause::read(); + let cause = Trap::::try_from(scause.cause()).unwrap(); + + match cause { + Trap::Exception(Exception::UserEnvCall) => dispatch_syscall(&mut ctx), + Trap::Exception( + Exception::InstructionPageFault + | Exception::LoadPageFault + | Exception::StorePageFault, + ) => { + let fault_addr = stval::read(); + if !do_user_page_fault(fault_addr) { + error!("Unhandled user page fault at {fault_addr:#x} ({cause:?})"); + error!("sepc = {:#x}", ctx.sepc); + core_scheduler().exit(1); + } + } + Trap::Interrupt(Interrupt::SupervisorTimer) => { + scheduler::timer_handler(); + } + Trap::Interrupt(Interrupt::SupervisorExternal) => { + interrupts::external_handler(); + } + #[cfg(feature = "smp")] + Trap::Interrupt(Interrupt::SupervisorSoft) => { + scheduler::wakeup_handler(); + } + cause => { + error!("Fatal user trap: {cause:?}"); + error!("sepc = {:#x}, stval = {:#x}", ctx.sepc, stval::read()); + core_scheduler().exit(1); + } + } + } +} + +/// Jump into the user-space application that `load_application` placed at +/// `USER_START`. +#[cfg(feature = "common-os")] +pub unsafe fn jump_to_user_land(entry_point: usize, args: Vec, envs: Vec) -> ! { + use align_address::Align; + use memory_addresses::PhysAddr; + + use crate::arch::riscv64::kernel::core_local::core_scheduler; + use crate::arch::riscv64::mm::paging::{self, BasePageSize, PageSize, PageTableEntryFlags}; + use crate::mm::vma::*; + + debug!("Create new file descriptor table"); + core_scheduler().recreate_objmap().unwrap(); + + let entry_point: usize = USER_START.as_usize() | entry_point; + let stack_top: usize = USER_STACK.as_usize() + USER_STACK_SIZE; + + let layout = PageLayout::from_size(USER_STACK_SIZE).unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let phys_addr = PhysAddr::from(frame_range.start()); + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user().execute_disable(); + paging::map::( + USER_STACK, + phys_addr, + USER_STACK_SIZE / BasePageSize::SIZE as usize, + flags, + ); + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( + USER_STACK, + VirtualMemoryArea::new( + USER_STACK, + USER_STACK + USER_STACK_SIZE as u64, + VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE, + MemoryType::STACK, + ), + ); + + // Place the argv and envp pointer arrays on the user stack. Both + // arrays follow the C convention and are terminated by a null pointer. + // The kernel accesses the user stack through `sstatus.SUM` (set at + // boot for common-os). + let ptr_count = args.len() + 1 + envs.len() + 1; + let stack_pointer = stack_top - ptr_count * size_of::<*mut u8>(); + let stack_ptr = ptr::with_exposed_provenance_mut::<*mut u8>(stack_pointer); + let arrays = unsafe { core::slice::from_raw_parts_mut(stack_ptr, ptr_count) }; + let (argv, envp) = arrays.split_at_mut(args.len() + 1); + let len = args + .iter() + .chain(envs.iter()) + .fold(0, |acc, x| acc + x.as_bytes_with_nul().len()); + // The RISC-V psABI requires SP to be 16-byte aligned at function entry. + let stack_pointer = (stack_pointer - len).align_down(16); + + let mut pos: usize = 0; + for (dst, s) in argv + .iter_mut() + .zip(args.iter()) + .chain(envp.iter_mut().zip(envs.iter())) + { + let bytes = s.as_bytes_with_nul(); + *dst = ptr::with_exposed_provenance_mut::(stack_pointer + pos); + pos += bytes.len(); + + unsafe { + dst.copy_from_nonoverlapping(bytes.as_ptr(), bytes.len()); + } + } + argv[args.len()] = ptr::null_mut(); + envp[envs.len()] = ptr::null_mut(); + + let argc = args.len(); + let argv_ptr = argv.as_ptr().expose_provenance(); + let envp_ptr = envp.as_ptr().expose_provenance(); + drop(args); + drop(envs); + + let current_task_id = core_scheduler().get_current_task().borrow().id; + let thread_pointer = USER_TP.lock().remove(¤t_task_id).unwrap_or_default(); + + user_loop( + entry_point, + stack_pointer, + thread_pointer, + [argc, argv_ptr, envp_ptr], + ) +} diff --git a/src/arch/riscv64/kernel/scheduler.rs b/src/arch/riscv64/kernel/scheduler.rs index 51827402db..9171407fad 100644 --- a/src/arch/riscv64/kernel/scheduler.rs +++ b/src/arch/riscv64/kernel/scheduler.rs @@ -146,13 +146,26 @@ impl TaskStacks { flags, ); + // User-stack flags differ between unikernel and common-os builds: + // in common-os the same VA is reachable from both S-mode (the + // kernel prepares the frame) and U-mode (the thread runs on it), + // so the mapping needs the U bit. + #[cfg(feature = "common-os")] + let user_flags = { + let mut f = PageTableEntryFlags::empty(); + f.normal().writable().user().execute_disable(); + f + }; + #[cfg(not(feature = "common-os"))] + let user_flags = flags; + // map user stack into the address space crate::arch::mm::paging::map::( virt_addr + KERNEL_STACK_SIZE + DEFAULT_STACK_SIZE + 3 * BasePageSize::SIZE, //virt_addr + KERNEL_STACK_SIZE + DEFAULT_STACK_SIZE, phys_addr + KERNEL_STACK_SIZE + DEFAULT_STACK_SIZE, user_stack_size / BasePageSize::SIZE as usize, - flags, + user_flags, ); // clear user stack @@ -272,6 +285,61 @@ extern "C" fn task_entry(func: extern "C" fn(usize), arg: usize) -> ! { core_scheduler().exit(0) } +/// First code of a new user-space thread (spawned via `spawn2` from a +/// user process), running in kernel mode on the thread's kernel stack. +/// Enters the user loop, which drops to U-mode at `entry(arg)`. +#[cfg(feature = "common-os")] +extern "C" fn user_task_entry(entry: usize, arg: usize, user_stack: usize, tp: usize) -> ! { + crate::arch::riscv64::kernel::user_loop(entry, user_stack, tp as u64, [arg, 0, 0]) +} + +#[cfg(feature = "common-os")] +impl Task { + /// Craft the initial kernel-stack frame for a new user-space thread. + /// When the scheduler first picks this task, `switch_to_task` pops the + /// `State` and returns into `user_task_entry`, which enters user mode + /// at `func(arg)` on the thread's user stack. + /// + /// `tls_thread_ptr` is the thread-pointer value prepared by + /// `scheduler::allocate_thread_tls`; zero leaves `tp` cleared. + pub(crate) fn create_user_stack_frame( + &mut self, + func: unsafe extern "C" fn(usize), + arg: usize, + tls_thread_ptr: u64, + ) { + unsafe { + // Set a marker for debugging at the very top. + let mut stack = + self.stacks.get_kernel_stack() + self.stacks.get_kernel_stack_size() - 0x10u64; + *stack.as_mut_ptr::() = 0xdead_beefu64; + + // Put the State structure expected by the ASM switch() function on the stack. + stack -= size_of::(); + + let state = stack.as_mut_ptr::(); + state.cast::().write_bytes(0, size_of::()); + + // The RISC-V psABI requires SP to be 16-byte aligned. + self.user_stack_pointer = + (self.stacks.get_user_stack() + self.stacks.get_user_stack_size() - 0x10u64) + .align_down(16u64); + + (*state).ra = core::mem::transmute::< + extern "C" fn(usize, usize, usize, usize) -> !, + unsafe extern "C" fn(extern "C" fn(usize), usize, u64), + >(user_task_entry); + (*state).sp = stack.as_usize(); + (*state).a0 = func as usize; + (*state).a1 = arg; + (*state).a2 = self.user_stack_pointer.as_usize(); + (*state).a3 = tls_thread_ptr as usize; + + self.last_stack_pointer = stack; + } + } +} + impl TaskFrame for Task { fn create_stack_frame(&mut self, func: unsafe extern "C" fn(usize), arg: usize) { // Check if the task (process or thread) uses Thread-Local-Storage. diff --git a/src/arch/riscv64/mm/mod.rs b/src/arch/riscv64/mm/mod.rs index d7ac10171e..0711456f8b 100644 --- a/src/arch/riscv64/mm/mod.rs +++ b/src/arch/riscv64/mm/mod.rs @@ -1,5 +1,8 @@ pub mod paging; +#[cfg(feature = "common-os")] +pub use paging::{create_new_root_page_table, drop_user_space}; + use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; pub unsafe fn init() { @@ -12,4 +15,60 @@ pub unsafe fn init() { unsafe { paging::enable_page_table(); } + + #[cfg(feature = "common-os")] + { + paging::prepopulate_kernel_root(); + crate::scheduler::BOOT_ROOT_PAGE_TABLE + .set(paging::kernel_root_page_table()) + .unwrap(); + } +} + +/// Allocate a fresh per-thread TLS block from the current process's +/// per-process TLS template and return the thread-pointer value for the +/// new thread. +/// +/// RISC-V uses TLS Variant I: `tp` points at the first byte of the TLS +/// data; the two-word TCB sits directly below it. +#[cfg(feature = "common-os")] +pub fn allocate_thread_tls(template: &crate::scheduler::task::TlsTemplate) -> u64 { + use align_address::Align; + use free_list::PageLayout; + use memory_addresses::{PhysAddr, VirtAddr}; + + use crate::arch::riscv64::mm::paging::{self, BasePageSize, PageSize, PageTableEntryFlags}; + + let tcb_size = 2 * size_of::<*mut ()>(); + let total = (tcb_size + template.size).align_up(BasePageSize::SIZE as usize); + + let virt_layout = PageLayout::from_size(total).unwrap(); + let virt_range = PageAlloc::allocate(virt_layout).unwrap(); + let virt_addr = VirtAddr::from(virt_range.start()); + + let frame_layout = PageLayout::from_size(total).unwrap(); + let frame_range = FrameAlloc::allocate(frame_layout).unwrap(); + let phys_addr = PhysAddr::from(frame_range.start()); + + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user().execute_disable(); + paging::map::( + virt_addr, + phys_addr, + total / BasePageSize::SIZE as usize, + flags, + ); + + unsafe { + // Zero the whole region first (covers both the TCB and the TLS BSS area). + virt_addr.as_mut_ptr::().write_bytes(0, total); + // Copy the pristine PT_TLS init image into the slot that follows the TCB. + virt_addr + .as_mut_ptr::() + .add(tcb_size) + .copy_from_nonoverlapping(template.init.as_ptr(), template.init.len()); + } + + // Variant I: `tp` points at the TLS data that follows the TCB. + (virt_addr + tcb_size as u64).as_u64() } diff --git a/src/arch/riscv64/mm/paging.rs b/src/arch/riscv64/mm/paging.rs index 0eb13e2886..77e2c5fb9e 100644 --- a/src/arch/riscv64/mm/paging.rs +++ b/src/arch/riscv64/mm/paging.rs @@ -85,6 +85,12 @@ impl PageTableEntryFlags { self.remove(PageTableEntryFlags::EXECUTABLE); self } + + pub fn user(&mut self) -> &mut Self { + self.insert(PageTableEntryFlags::USER_ACCESSIBLE); + self + } + } /// An entry in either table @@ -131,6 +137,7 @@ impl PageTableEntry { (self.physical_address_and_flags & PageTableEntryFlags::EXECUTABLE.bits()) != 0 } + /// Mark this as an invalid (not present) entry fn unset(&mut self) { self.physical_address_and_flags = PhysAddr::zero(); @@ -151,7 +158,11 @@ impl PageTableEntry { let mut flags_to_set = flags; flags_to_set.insert(PageTableEntryFlags::VALID); - flags_to_set.insert(PageTableEntryFlags::GLOBAL); + // User mappings are per-process and must not survive an address + // space switch; only kernel mappings are marked global. + if !flags_to_set.contains(PageTableEntryFlags::USER_ACCESSIBLE) { + flags_to_set.insert(PageTableEntryFlags::GLOBAL); + } self.physical_address_and_flags = PhysAddr::new((physical_address.as_u64() >> 2) | flags_to_set.bits()); } @@ -583,6 +594,43 @@ pub fn virtual_to_physical(virtual_address: VirtAddr) -> Option { panic!("virtual_to_physical should never reach this point"); } +/// Start of the user-space region (inclusive). Sv39 root slot 64. +/// +/// The kernel owns root slots 0..64 (0..64 GiB, see +/// `mm::virtualmem::kernel_heap_end`); everything from here up to the +/// end of the Sv39 address space belongs to user processes and is +/// managed per process in its own root page table. +#[cfg(feature = "common-os")] +pub(crate) const USER_SPACE_START: usize = 0x10_0000_0000; +/// End of the user-space region (exclusive). Sv39 ends at 256 GiB. +#[cfg(feature = "common-os")] +pub(crate) const USER_SPACE_END: usize = 0x40_0000_0000; + +/// First root-table index of the user-space region. +#[cfg(feature = "common-os")] +const USER_ROOT_INDEX_FIRST: usize = USER_SPACE_START >> (PAGE_BITS + 2 * PAGE_MAP_BITS); +/// Root-table index one past the user-space region. +#[cfg(feature = "common-os")] +const USER_ROOT_INDEX_LAST: usize = USER_SPACE_END >> (PAGE_BITS + 2 * PAGE_MAP_BITS); + +#[cfg(feature = "common-os")] +#[inline] +fn is_user_address(virtual_address: VirtAddr) -> bool { + (USER_SPACE_START..USER_SPACE_END).contains(&virtual_address.as_usize()) +} + +/// Returns the root page table that is currently installed in `satp`. +/// +/// Every task owns its root table (or the shared boot root), so mutating +/// it without a lock is sound as long as the caller only touches the +/// user-space slots of its own address space. +#[cfg(feature = "common-os")] +#[allow(clippy::mut_from_ref)] +fn active_root_table() -> &'static mut PageTable { + let root_phys = satp::read().ppn() << PAGE_BITS; + unsafe { &mut *ptr::with_exposed_provenance_mut::>(root_phys) } +} + pub fn map( virtual_address: VirtAddr, physical_address: PhysAddr, @@ -594,11 +642,21 @@ pub fn map( ); let range = get_page_range::(virtual_address, count); + + // User mappings are private to the current address space and go into + // the root table referenced by `satp`. Kernel mappings go into the + // static kernel root table; its L1 subtables are shared with every + // process root (see `create_new_root_page_table`), so they become + // visible in all address spaces. + #[cfg(feature = "common-os")] + if is_user_address(virtual_address) { + active_root_table().map_pages(range, physical_address, flags); + return; + } + ROOT_PAGETABLE .lock() .map_pages(range, physical_address, flags); - - //assert_eq!(virtual_address.as_u64(), physical_address.as_u64(), "Paging not implemented"); } pub fn map_heap(virt_addr: VirtAddr, count: usize) -> Result<(), usize> { @@ -624,9 +682,13 @@ pub fn unmap(virtual_address: VirtAddr, count: usize) { trace!("Unmapping virtual address {virtual_address:#X} ({count} pages)"); let range = get_page_range::(virtual_address, count); - /* let root_pagetable = unsafe { - &mut *mem::transmute::<*mut u64, *mut PageTable>(L2TABLE_ADDRESS.as_mut_ptr()) - }; */ + + #[cfg(feature = "common-os")] + if is_user_address(virtual_address) { + active_root_table().map_pages(range, PhysAddr::zero(), PageTableEntryFlags::empty()); + return; + } + ROOT_PAGETABLE .lock() .map_pages(range, PhysAddr::zero(), PageTableEntryFlags::empty()); @@ -660,3 +722,156 @@ pub unsafe fn enable_page_table() { asm::sfence_vma_all(); } } + +/// Physical address of the static kernel root page table. +#[cfg(feature = "common-os")] +pub(crate) fn kernel_root_page_table() -> usize { + ROOT_PAGETABLE.data_ptr().expose_provenance() +} + +/// Pre-populate every kernel slot of the static root page table with an +/// (initially empty) L1 subtable. +/// +/// `create_new_root_page_table` copies the kernel slots of the static +/// root *by value* into each new process root, so both share the same L1 +/// subtables. Kernel mappings created later (task stacks, TLS blocks, +/// heap growth) modify those shared subtables and become visible in all +/// address spaces — but only if the root slot already existed at copy +/// time. Allocating all 64 kernel L1 tables up front closes that hole. +#[cfg(feature = "common-os")] +pub fn prepopulate_kernel_root() { + let mut root = ROOT_PAGETABLE.lock(); + + for entry in root.entries[..USER_ROOT_INDEX_FIRST].iter_mut() { + if !entry.is_present() { + let frame_layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); + let frame_range = FrameAlloc::allocate(frame_layout).unwrap(); + let l1_phys = frame_range.start(); + + let l1 = + unsafe { &mut *ptr::with_exposed_provenance_mut::>(l1_phys) }; + for l1_entry in l1.entries.iter_mut() { + l1_entry.unset(); + } + + // A pointer entry: VALID without R/W/X. + entry.set(PhysAddr::new(l1_phys as u64), PageTableEntryFlags::empty()); + } + } +} + +/// Create a new root page table for a fresh address space. +/// +/// The kernel slots are copied verbatim from the static kernel root, so +/// the new address space shares all kernel L1 subtables. The user slots +/// start out empty. +#[cfg(feature = "common-os")] +pub fn create_new_root_page_table() -> usize { + let frame_layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); + let frame_range = FrameAlloc::allocate(frame_layout).unwrap(); + let new_root_phys = frame_range.start(); + + let new_root = + unsafe { &mut *ptr::with_exposed_provenance_mut::>(new_root_phys) }; + let kernel_root = ROOT_PAGETABLE.lock(); + + for (new_entry, kernel_entry) in new_root + .entries + .iter_mut() + .zip(kernel_root.entries.iter()) + .take(USER_ROOT_INDEX_FIRST) + { + *new_entry = *kernel_entry; + } + for new_entry in new_root.entries[USER_ROOT_INDEX_FIRST..].iter_mut() { + new_entry.unset(); + } + + new_root_phys +} + +/// Release all user-space mappings of the root table at `root_phys`: +/// every user-accessible frame and the L1/L0 subtables that held them. +#[cfg(feature = "common-os")] +fn clear_user_slots(root_phys: usize) { + let root = unsafe { &mut *ptr::with_exposed_provenance_mut::>(root_phys) }; + + for root_entry in root.entries[USER_ROOT_INDEX_FIRST..USER_ROOT_INDEX_LAST].iter_mut() { + if !root_entry.is_present() { + continue; + } + + let l1_phys = root_entry.address().as_usize(); + let l1 = unsafe { &mut *ptr::with_exposed_provenance_mut::>(l1_phys) }; + for l1_entry in l1.entries.iter_mut() { + if !l1_entry.is_present() { + continue; + } + + let l0_phys = l1_entry.address().as_usize(); + let l0 = + unsafe { &mut *ptr::with_exposed_provenance_mut::>(l0_phys) }; + for l0_entry in l0.entries.iter_mut() { + if l0_entry.is_present() && l0_entry.is_user() { + let frame_phys = l0_entry.address().as_usize(); + let range = free_list::PageRange::new( + frame_phys, + frame_phys + BasePageSize::SIZE as usize, + ) + .unwrap(); + unsafe { FrameAlloc::deallocate(range) }; + } + l0_entry.unset(); + } + + let range = + free_list::PageRange::new(l0_phys, l0_phys + BasePageSize::SIZE as usize).unwrap(); + unsafe { FrameAlloc::deallocate(range) }; + l1_entry.unset(); + } + + let range = + free_list::PageRange::new(l1_phys, l1_phys + BasePageSize::SIZE as usize).unwrap(); + unsafe { FrameAlloc::deallocate(range) }; + root_entry.unset(); + } +} + +/// Release the address space rooted at `root_phys`: all user mappings, +/// their page tables, and the root table itself. +/// +/// Must not be called for the currently installed root page table. +#[cfg(feature = "common-os")] +pub fn drop_user_space(root_phys: usize) { + clear_user_slots(root_phys); + + let range = + free_list::PageRange::new(root_phys, root_phys + BasePageSize::SIZE as usize).unwrap(); + unsafe { FrameAlloc::deallocate(range) }; +} + +/// Clear the user-space portion of the currently active address space. +/// +/// Used by `exec` to tear down the old process image before the loader +/// maps the new one. +#[cfg(feature = "common-os")] +pub fn clear_user_space() { + use crate::core_scheduler; + use crate::fd::STDERR_FILENO; + + core_scheduler() + .get_current_task() + .borrow() + .vmas + .write() + .clear(); + core_scheduler() + .get_current_task_object_map() + .write() + .retain(|&fd, _| fd <= STDERR_FILENO); + + let root_phys = satp::read().ppn() << PAGE_BITS; + clear_user_slots(root_phys); + + asm::sfence_vma_all(); +} diff --git a/src/lib.rs b/src/lib.rs index 9dde577d95..e8917993f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,8 +46,13 @@ #![cfg_attr(target_arch = "x86_64", feature(abi_x86_interrupt))] #![feature(allocator_api)] #![cfg_attr(docsrs, feature(doc_cfg))] +// `#[linkage = "weak"]` is used by the syscall table (common-os, all +// architectures) and by `sys_errno_location` (unikernel, except riscv64). #![cfg_attr( - all(not(feature = "nostd"), not(target_arch = "riscv64"),), + all( + not(feature = "nostd"), + any(feature = "common-os", not(target_arch = "riscv64")) + ), feature(linkage) )] #![feature(linked_list_cursors)] @@ -105,7 +110,11 @@ mod logging; pub mod arch; #[cfg(all( feature = "common-os", - any(target_arch = "x86_64", target_arch = "aarch64") + any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64" + ) ))] pub mod common_os; pub mod config; diff --git a/src/mm/virtualmem.rs b/src/mm/virtualmem.rs index ba313bb16d..7b92665b18 100644 --- a/src/mm/virtualmem.rs +++ b/src/mm/virtualmem.rs @@ -71,8 +71,15 @@ pub fn kernel_heap_end() -> VirtAddr { VirtAddr::new(0xFFFF_FFFF_FFFF) } target_arch = "riscv64" => { - // 256 GiB - VirtAddr::new(0x0040_0000_0000 - 1) + if cfg!(feature = "common-os") { + // The kernel owns the Sv39 root slots below 64 GiB; the + // region above belongs to user processes (see + // `arch::riscv64::mm::paging::USER_SPACE_START`). + VirtAddr::new(0x0010_0000_0000 - 1) + } else { + // 256 GiB + VirtAddr::new(0x0040_0000_0000 - 1) + } } target_arch = "x86_64" => { use x86_64::structures::paging::PageTableIndex; diff --git a/src/mm/vma.rs b/src/mm/vma.rs index ff72cccd57..c3ce7f2561 100644 --- a/src/mm/vma.rs +++ b/src/mm/vma.rs @@ -76,8 +76,15 @@ bitflags! { // loaded image while still falling inside L0[2] (which ends at // 0x0180_0000_0000). The address is canonical on x86_64 as well // (bit 47 = 0). +#[cfg(not(target_arch = "riscv64"))] const HEAP_START_ADDR: VirtAddr = VirtAddr::new(0x0140_0000_0000); +// Sv39 only covers 256 GiB, so the riscv64 user layout is denser: code +// at 64 GiB (`USER_START`), heap at 128 GiB, stack below 192 GiB — all +// in disjoint 1 GiB root slots of the user region (64..256 GiB). +#[cfg(target_arch = "riscv64")] +const HEAP_START_ADDR: VirtAddr = VirtAddr::new(0x20_0000_0000); + /// Creates a new virtual memory mapping of the `size` specified with /// protection bits specified in `prot_flags`. #[hermit_macro::system(errno)] diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index 1033ec644c..81908977a4 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -16,6 +16,8 @@ use hashbrown::{HashMap, hash_map}; use hermit_sync::*; #[cfg(target_arch = "aarch64")] use memory_addresses::VirtAddr; +#[cfg(all(target_arch = "riscv64", feature = "common-os"))] +use memory_addresses::VirtAddr; #[cfg(target_arch = "riscv64")] use riscv::register::sstatus; use timer_interrupts::TimerList; @@ -651,7 +653,11 @@ impl PerCoreScheduler { /// clone the standard descriptors. #[cfg(feature = "common-os")] #[cfg_attr( - not(any(target_arch = "x86_64", target_arch = "aarch64")), + not(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64" + )), expect(dead_code) )] pub fn recreate_objmap(&self) -> io::Result<()> { @@ -1018,6 +1024,22 @@ impl PerCoreScheduler { self.current_task.borrow_mut().last_fpu_state.save(); } task.borrow().last_fpu_state.restore(); + + // Install the new task's address space before switching to + // its kernel stack. + #[cfg(feature = "common-os")] + { + use riscv::register::satp; + + let new_ppn = task.borrow().root_page_table.as_usize() >> 12; + if satp::read().ppn() != new_ppn { + unsafe { + satp::set(satp::Mode::Sv39, 0, new_ppn); + riscv::asm::sfence_vma_all(); + } + } + } + self.current_task = task; unsafe { switch_to_task(last_stack_pointer, new_stack_pointer.as_usize()); diff --git a/src/syscalls/table.rs b/src/syscalls/table.rs index d0c27d332e..8fcda19516 100644 --- a/src/syscalls/table.rs +++ b/src/syscalls/table.rs @@ -37,10 +37,8 @@ const SYSNO_WRITEV: usize = 12; /// Number of the system call `readv` const SYSNO_READV: usize = 13; /// number of the system call `fork` -#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] const SYSNO_FORK: usize = 14; /// number of the system call `waitpid` -#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] const SYSNO_WAITPID: usize = 15; /// number of the system call `spawn_process` const SYSNO_SPAWN_PROCESS: usize = 16; @@ -185,14 +183,15 @@ pub(crate) unsafe extern "C" fn sys_invalid() { ); } -/// Sentinel placeholder for unregistered syscall slots on aarch64. +/// Sentinel placeholder for unregistered syscall slots on aarch64 and +/// riscv64. /// -/// The aarch64 dispatcher in `do_sync` compares the table entry against -/// this function pointer before invoking; if it matches, it bails out -/// with `invalid_syscall(nr)` — the body here is therefore never executed. -/// The empty `extern "C"` body still gives us a stable, no_mangle symbol -/// whose address we can compare against. -#[cfg(target_arch = "aarch64")] +/// The dispatchers (`do_sync` on aarch64, `user_loop` on riscv64) compare +/// the table entry against this function pointer before invoking; if it +/// matches, they bail out without calling it — the body here is therefore +/// never executed. The empty `extern "C"` body still gives us a stable, +/// no_mangle symbol whose address we can compare against. +#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] #[unsafe(no_mangle)] pub(crate) extern "C" fn sys_invalid() {} @@ -222,11 +221,8 @@ impl SyscallTable { table.handle[SYSNO_OPEN] = sys_open as *const _; table.handle[SYSNO_READV] = sys_readv as *const _; table.handle[SYSNO_WRITEV] = sys_writev as *const _; - #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] - { - table.handle[SYSNO_FORK] = sys_fork as *const _; - table.handle[SYSNO_WAITPID] = sys_waitpid as *const _; - } + table.handle[SYSNO_FORK] = sys_fork as *const _; + table.handle[SYSNO_WAITPID] = sys_waitpid as *const _; table.handle[SYSNO_SPAWN_PROCESS] = sys_spawn_process as *const _; table.handle[SYSNO_CLOCK_GETTIME] = sys_clock_gettime as *const _; table.handle[SYSNO_SPAWN] = sys_spawn as *const _; @@ -279,7 +275,7 @@ impl SyscallTable { } impl SyscallTable { - #[cfg(target_arch = "aarch64")] + #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] #[inline] pub(crate) fn handler(&self, nr: usize) -> *const usize { self.handle[nr] diff --git a/src/syscalls/tasks.rs b/src/syscalls/tasks.rs index bf221a79b9..c43de3c6e7 100644 --- a/src/syscalls/tasks.rs +++ b/src/syscalls/tasks.rs @@ -18,19 +18,17 @@ pub type Pid = i32; /// Fork the current process. /// Returns the child's PID to the parent, and 0 to the child. -#[cfg(all( - any(target_arch = "x86_64", target_arch = "aarch64"), - feature = "common-os" -))] +/// Fork is not implemented on riscv64 yet, so the call fails there. +#[cfg(feature = "common-os")] #[hermit_macro::system(errno)] #[unsafe(no_mangle)] pub extern "C" fn sys_fork() -> i32 { - #[cfg(feature = "fork")] + #[cfg(all(feature = "fork", any(target_arch = "x86_64", target_arch = "aarch64")))] unsafe { scheduler::fork().into() } - #[cfg(not(feature = "fork"))] + #[cfg(not(all(feature = "fork", any(target_arch = "x86_64", target_arch = "aarch64"))))] { -i32::from(Errno::Nosys) } @@ -47,10 +45,7 @@ pub extern "C" fn sys_fork() -> i32 { /// Waitpid block the current process until termination of process `pid`. /// Returns 0 is the process terminates -#[cfg(all( - any(target_arch = "x86_64", target_arch = "aarch64"), - feature = "common-os" -))] +#[cfg(feature = "common-os")] #[hermit_macro::system(errno)] #[unsafe(no_mangle)] pub extern "C" fn sys_waitpid(pid: Pid) -> i32 { @@ -233,19 +228,13 @@ pub unsafe extern "C" fn sys_spawn2( stack_size: usize, selector: isize, ) -> Tid { - #[cfg(all( - any(target_arch = "x86_64", target_arch = "aarch64"), - feature = "common-os" - ))] + #[cfg(feature = "common-os")] { unsafe { scheduler::spawn_thread(func, arg, Priority::from(prio), stack_size, selector).into() } } - #[cfg(not(all( - any(target_arch = "x86_64", target_arch = "aarch64"), - feature = "common-os" - )))] + #[cfg(not(feature = "common-os"))] { unsafe { scheduler::spawn(func, arg, Priority::from(prio), stack_size, selector).into() } } @@ -261,18 +250,12 @@ pub unsafe extern "C" fn sys_spawn( selector: isize, ) -> i32 { let new_id = { - #[cfg(all( - any(target_arch = "x86_64", target_arch = "aarch64"), - feature = "common-os" - ))] + #[cfg(feature = "common-os")] unsafe { scheduler::spawn_thread(func, arg, Priority::from(prio), USER_STACK_SIZE, selector) .into() } - #[cfg(not(all( - any(target_arch = "x86_64", target_arch = "aarch64"), - feature = "common-os" - )))] + #[cfg(not(feature = "common-os"))] unsafe { scheduler::spawn(func, arg, Priority::from(prio), USER_STACK_SIZE, selector).into() } From d574bcca24081f7c73498639055f3caee4842ea9 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Mon, 13 Jul 2026 22:22:10 +0200 Subject: [PATCH 48/48] add fork with copy-on-write for riscv64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unlike x86_64/aarch64, no kernel stack has to be cloned: the user- mode dispatcher owns the complete trap context, so fork is intercepted in user_loop (the syscall-table entry stays as an ENOSYS fallback), the child gets a copy of the parent's UserContext with a0 = 0, a fresh kernel stack, and re-enters user mode through its own user_loop_resume via fork_child_entry. The COW machinery mirrors the other targets: RSW bit 8 marks shared frames, mark_user_pages_copy_on_write drops the write bit before copy_current_root_page_table deep-copies the user slots (kernel slots stay shared), and do_cow_fault clones a frame on the first write — or re-marks it writable in place when the last reference faults. The frame refcounting protocol is completed on riscv64: every user mapping (code, TLS, stack, lazily faulted pages) takes a reference and clear_user_slots only releases frames whose count drops to zero, which also covers exec after fork. Kernel-mode stores through sstatus.SUM to COW pages are resolved in trap_handler. Verified in QEMU with the fork example: fork, exec with argv/envp in the child, waitpid, and 16k fork/exec/join cycles complete cleanly. --- src/arch/riscv64/kernel/interrupts.rs | 22 +++ src/arch/riscv64/kernel/mod.rs | 49 ++++- src/arch/riscv64/kernel/scheduler.rs | 44 +++++ src/arch/riscv64/mm/mod.rs | 2 + src/arch/riscv64/mm/paging.rs | 248 +++++++++++++++++++++++++- src/scheduler/mod.rs | 101 +++++++++++ src/syscalls/table.rs | 2 +- src/syscalls/tasks.rs | 5 +- 8 files changed, 462 insertions(+), 11 deletions(-) diff --git a/src/arch/riscv64/kernel/interrupts.rs b/src/arch/riscv64/kernel/interrupts.rs index b076546844..f277897e12 100644 --- a/src/arch/riscv64/kernel/interrupts.rs +++ b/src/arch/riscv64/kernel/interrupts.rs @@ -248,6 +248,28 @@ pub extern "C" fn trap_handler(tf: &mut TrapFrame) { Trap::Interrupt(Interrupt::SupervisorTimer) => { crate::arch::riscv64::kernel::scheduler::timer_handler(); } + // The kernel writes to user pages through `sstatus.SUM` (program + // loading, argv/envp setup, syscall buffers). Such a store can + // fault on a COW-marked or not-yet-faulted user page and must be + // resolved exactly like the corresponding user-mode fault. + #[cfg(feature = "common-os")] + Trap::Exception(Exception::LoadPageFault | Exception::StorePageFault) => { + let fault_addr = stval; + + #[cfg(feature = "fork")] + if matches!(cause, Trap::Exception(Exception::StorePageFault)) + && crate::arch::riscv64::mm::paging::do_cow_fault(memory_addresses::VirtAddr::new( + fault_addr as u64, + )) { + return; + } + + if !crate::arch::riscv64::kernel::do_user_page_fault(fault_addr) { + error!("Unhandled kernel page fault at {fault_addr:#x} ({cause:?})"); + error!("sepc = {sepc:x}"); + scheduler::abort(); + } + } cause => { error!("Interrupt: {cause:?}"); error!("tf = {tf:x?} "); diff --git a/src/arch/riscv64/kernel/mod.rs b/src/arch/riscv64/kernel/mod.rs index 8e383e8581..376f1b3a30 100644 --- a/src/arch/riscv64/kernel/mod.rs +++ b/src/arch/riscv64/kernel/mod.rs @@ -230,6 +230,10 @@ where let layout = PageLayout::from_size_align(code_size, BasePageSize::SIZE as usize).unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let physaddr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] + for i in 0..code_size / BasePageSize::SIZE as usize { + crate::mm::frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize); + } let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().user(); @@ -272,6 +276,10 @@ where let layout = PageLayout::from_size(tls_memsz).unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let physaddr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] + for i in 0..tls_memsz / BasePageSize::SIZE as usize { + crate::mm::frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize); + } let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().user().execute_disable(); @@ -364,7 +372,7 @@ fn run_user(ctx: &mut trapframe::UserContext) { /// Returns `true` if the fault was resolved and the faulting instruction /// can be retried. #[cfg(feature = "common-os")] -fn do_user_page_fault(fault_addr: usize) -> bool { +pub(crate) fn do_user_page_fault(fault_addr: usize) -> bool { use core::ops::Bound; use align_address::Align; @@ -401,6 +409,8 @@ fn do_user_page_fault(fault_addr: usize) -> bool { } paging::map::(addr, physaddr, 1, flags); + #[cfg(feature = "fork")] + crate::mm::frame_ref_inc(physaddr); // Clear the page through the identity mapping of physical memory, // so this works independently of `sstatus.SUM`. @@ -500,6 +510,15 @@ pub(crate) fn user_loop( debug!("Jump to user space at {entry:#x}, stack pointer {stack_pointer:#x}"); + user_loop_resume(ctx) +} + +/// Service all traps the user code raises, starting (or resuming, for a +/// fork child) user execution from `ctx`. Never returns; the process +/// leaves through the `exit` system call (or is torn down after a fatal +/// fault). +#[cfg(feature = "common-os")] +pub(crate) fn user_loop_resume(mut ctx: trapframe::UserContext) -> ! { use riscv::interrupt::{Exception, Interrupt, Trap}; use riscv::register::{scause, stval}; @@ -513,13 +532,35 @@ pub(crate) fn user_loop( let cause = Trap::::try_from(scause.cause()).unwrap(); match cause { - Trap::Exception(Exception::UserEnvCall) => dispatch_syscall(&mut ctx), + Trap::Exception(Exception::UserEnvCall) => { + // `fork` needs the full user context to seed the child's + // user loop, so it is dispatched here instead of through + // the syscall table. + #[cfg(feature = "fork")] + if ctx.general.a7 == crate::syscalls::table::SYSNO_FORK { + ctx.sepc += 4; + let pid: i32 = unsafe { crate::scheduler::fork_from_user_context(&ctx) }.into(); + ctx.general.a0 = pid as usize; + continue; + } + + dispatch_syscall(&mut ctx); + } Trap::Exception( Exception::InstructionPageFault | Exception::LoadPageFault | Exception::StorePageFault, ) => { let fault_addr = stval::read(); + + #[cfg(feature = "fork")] + if matches!(cause, Trap::Exception(Exception::StorePageFault)) + && crate::arch::riscv64::mm::paging::do_cow_fault( + memory_addresses::VirtAddr::new(fault_addr as u64), + ) { + continue; + } + if !do_user_page_fault(fault_addr) { error!("Unhandled user page fault at {fault_addr:#x} ({cause:?})"); error!("sepc = {:#x}", ctx.sepc); @@ -565,6 +606,10 @@ pub unsafe fn jump_to_user_land(entry_point: usize, args: Vec, envs: Ve let layout = PageLayout::from_size(USER_STACK_SIZE).unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let phys_addr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] + for i in 0..USER_STACK_SIZE / BasePageSize::SIZE as usize { + crate::mm::frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); + } let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().user().execute_disable(); paging::map::( diff --git a/src/arch/riscv64/kernel/scheduler.rs b/src/arch/riscv64/kernel/scheduler.rs index 9171407fad..340da4fbb8 100644 --- a/src/arch/riscv64/kernel/scheduler.rs +++ b/src/arch/riscv64/kernel/scheduler.rs @@ -293,6 +293,50 @@ extern "C" fn user_task_entry(entry: usize, arg: usize, user_stack: usize, tp: u crate::arch::riscv64::kernel::user_loop(entry, user_stack, tp as u64, [arg, 0, 0]) } +/// First code of a fork child, running in kernel mode on the child's +/// fresh kernel stack. Re-enters user mode with the parent's saved +/// register state (`a0` already patched to 0 by `fork_from_user_context`). +#[cfg(all(feature = "common-os", feature = "fork"))] +extern "C" fn fork_child_entry(ctx_box: usize) -> ! { + let ctx = unsafe { + alloc::boxed::Box::from_raw(core::ptr::with_exposed_provenance_mut::< + trapframe::UserContext, + >(ctx_box)) + }; + crate::arch::riscv64::kernel::user_loop_resume(*ctx) +} + +/// Craft the initial kernel-stack frame for a fork child. When the +/// scheduler first picks the child, `switch_to_task` pops the `State` +/// and returns into `fork_child_entry` with the boxed `UserContext` as +/// argument. Returns the child's initial `last_stack_pointer`. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub(crate) fn create_fork_stack_frame( + stacks: &TaskStacks, + ctx: alloc::boxed::Box, +) -> VirtAddr { + unsafe { + // Set a marker for debugging at the very top. + let mut stack = stacks.get_kernel_stack() + stacks.get_kernel_stack_size() - 0x10u64; + *stack.as_mut_ptr::() = 0xdead_beefu64; + + // Put the State structure expected by the ASM switch() function on the stack. + stack -= size_of::(); + + let state = stack.as_mut_ptr::(); + state.cast::().write_bytes(0, size_of::()); + + (*state).ra = core::mem::transmute::< + extern "C" fn(usize) -> !, + unsafe extern "C" fn(extern "C" fn(usize), usize, u64), + >(fork_child_entry); + (*state).sp = stack.as_usize(); + (*state).a0 = alloc::boxed::Box::into_raw(ctx).expose_provenance(); + + stack + } +} + #[cfg(feature = "common-os")] impl Task { /// Craft the initial kernel-stack frame for a new user-space thread. diff --git a/src/arch/riscv64/mm/mod.rs b/src/arch/riscv64/mm/mod.rs index 0711456f8b..4b0e015b93 100644 --- a/src/arch/riscv64/mm/mod.rs +++ b/src/arch/riscv64/mm/mod.rs @@ -1,5 +1,7 @@ pub mod paging; +#[cfg(all(feature = "common-os", feature = "fork"))] +pub use paging::{copy_current_root_page_table, prepare_mem_copy_on_write}; #[cfg(feature = "common-os")] pub use paging::{create_new_root_page_table, drop_user_space}; diff --git a/src/arch/riscv64/mm/paging.rs b/src/arch/riscv64/mm/paging.rs index 77e2c5fb9e..ff93264e7d 100644 --- a/src/arch/riscv64/mm/paging.rs +++ b/src/arch/riscv64/mm/paging.rs @@ -56,6 +56,11 @@ bitflags! { /// The RSW field is reserved for use by supervisor const RSW = (1 << 8) | (1 << 9); + + /// Software marker (RSW bit 8): the frame is shared Copy-On-Write. + /// Set together with a cleared WRITABLE bit; the store-fault + /// handler clones the frame on the first write. + const COW_MARKER = 1 << 8; } } @@ -91,6 +96,12 @@ impl PageTableEntryFlags { self } + #[cfg(all(feature = "common-os", feature = "fork"))] + pub fn copy_on_write(&mut self) -> &mut Self { + self.remove(PageTableEntryFlags::WRITABLE); + self.insert(PageTableEntryFlags::COW_MARKER); + self + } } /// An entry in either table @@ -137,6 +148,11 @@ impl PageTableEntry { (self.physical_address_and_flags & PageTableEntryFlags::EXECUTABLE.bits()) != 0 } + /// Returns the flag bits of this entry (low 10 bits of the PTE). + #[cfg(all(feature = "common-os", feature = "fork"))] + fn flags(self) -> PageTableEntryFlags { + PageTableEntryFlags::from_bits_truncate(self.physical_address_and_flags.as_u64() & 0x3ff) + } /// Mark this as an invalid (not present) entry fn unset(&mut self) { @@ -813,13 +829,23 @@ fn clear_user_slots(root_phys: usize) { unsafe { &mut *ptr::with_exposed_provenance_mut::>(l0_phys) }; for l0_entry in l0.entries.iter_mut() { if l0_entry.is_present() && l0_entry.is_user() { - let frame_phys = l0_entry.address().as_usize(); - let range = free_list::PageRange::new( - frame_phys, - frame_phys + BasePageSize::SIZE as usize, - ) - .unwrap(); - unsafe { FrameAlloc::deallocate(range) }; + // With fork support the frame may be COW-shared with + // another address space; it is only released when the + // last reference is dropped. + #[cfg(feature = "fork")] + let release = crate::mm::frame_ref_dec(l0_entry.address()); + #[cfg(not(feature = "fork"))] + let release = true; + + if release { + let frame_phys = l0_entry.address().as_usize(); + let range = free_list::PageRange::new( + frame_phys, + frame_phys + BasePageSize::SIZE as usize, + ) + .unwrap(); + unsafe { FrameAlloc::deallocate(range) }; + } } l0_entry.unset(); } @@ -850,6 +876,214 @@ pub fn drop_user_space(root_phys: usize) { unsafe { FrameAlloc::deallocate(range) }; } +/// Walk the user slots of the currently active root table and mark every +/// writable user page as Copy-On-Write. Must be called before duplicating +/// the page table for a fork. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn mark_user_pages_copy_on_write() { + let root = active_root_table(); + + for root_entry in root.entries[USER_ROOT_INDEX_FIRST..USER_ROOT_INDEX_LAST].iter_mut() { + if !root_entry.is_present() { + continue; + } + + let l1 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>( + root_entry.address().as_usize(), + ) + }; + for l1_entry in l1.entries.iter_mut() { + if !l1_entry.is_present() { + continue; + } + + let l0 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>( + l1_entry.address().as_usize(), + ) + }; + for l0_entry in l0.entries.iter_mut() { + let flags = l0_entry.flags(); + let user_writable = l0_entry.is_present() + && flags.contains(PageTableEntryFlags::USER_ACCESSIBLE) + && flags.contains(PageTableEntryFlags::WRITABLE) + && !flags.contains(PageTableEntryFlags::COW_MARKER); + if user_writable { + let addr = l0_entry.address(); + let mut new_flags = flags; + new_flags.copy_on_write(); + l0_entry.set(addr, new_flags); + } + } + } + } + + asm::sfence_vma_all(); +} + +/// Duplicate the currently active root page table for a fork. +/// +/// The kernel slots are shared with the parent (copied by value, pointing +/// at the same L1 subtables); the user slots are deep-copied so parent +/// and child own separate table hierarchies referencing the same +/// (COW-marked) frames. Every user frame gains one reference for the +/// child. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn copy_current_root_page_table() -> usize { + let cur_root = active_root_table(); + + let frame_layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); + let frame_range = FrameAlloc::allocate(frame_layout).unwrap(); + let new_root_phys = frame_range.start(); + let new_root = + unsafe { &mut *ptr::with_exposed_provenance_mut::>(new_root_phys) }; + + // Kernel slots are inherited verbatim (sharing the L1 subtables). + for (new_entry, cur_entry) in new_root + .entries + .iter_mut() + .zip(cur_root.entries.iter()) + .take(USER_ROOT_INDEX_FIRST) + { + *new_entry = *cur_entry; + } + + // User slots get a deep copy of the L1/L0 hierarchy. + for (idx, new_entry) in new_root.entries[USER_ROOT_INDEX_FIRST..] + .iter_mut() + .enumerate() + { + let cur_entry = &cur_root.entries[USER_ROOT_INDEX_FIRST + idx]; + if !cur_entry.is_present() || USER_ROOT_INDEX_FIRST + idx >= USER_ROOT_INDEX_LAST { + new_entry.unset(); + continue; + } + + let cur_l1 = unsafe { + &*ptr::with_exposed_provenance::>(cur_entry.address().as_usize()) + }; + + let frame_layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); + let frame_range = FrameAlloc::allocate(frame_layout).unwrap(); + let new_l1_phys = frame_range.start(); + let new_l1 = + unsafe { &mut *ptr::with_exposed_provenance_mut::>(new_l1_phys) }; + + for (new_l1_entry, cur_l1_entry) in new_l1.entries.iter_mut().zip(cur_l1.entries.iter()) { + if !cur_l1_entry.is_present() { + new_l1_entry.unset(); + continue; + } + + let cur_l0 = unsafe { + &*ptr::with_exposed_provenance::>( + cur_l1_entry.address().as_usize(), + ) + }; + + let frame_layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); + let frame_range = FrameAlloc::allocate(frame_layout).unwrap(); + let new_l0_phys = frame_range.start(); + let new_l0 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>(new_l0_phys) + }; + + // Leaf entries are copied verbatim (already COW-marked by + // `mark_user_pages_copy_on_write`); the child holds an + // additional reference to every user frame. + for (new_l0_entry, cur_l0_entry) in new_l0.entries.iter_mut().zip(cur_l0.entries.iter()) + { + *new_l0_entry = *cur_l0_entry; + if cur_l0_entry.is_present() && cur_l0_entry.is_user() { + crate::mm::frame_ref_inc(cur_l0_entry.address()); + } + } + + new_l1_entry.set( + PhysAddr::new(new_l0_phys as u64), + PageTableEntryFlags::empty(), + ); + } + + new_entry.set( + PhysAddr::new(new_l1_phys as u64), + PageTableEntryFlags::empty(), + ); + } + + new_root_phys +} + +/// Handle a store fault on a Copy-On-Write page of the currently active +/// address space. +/// +/// Returns `true` if the fault was resolved (the frame was cloned or +/// re-marked writable) and the faulting instruction can be retried. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn do_cow_fault(faulting_addr: VirtAddr) -> bool { + let vaddr = faulting_addr.align_down(BasePageSize::SIZE); + if !(USER_SPACE_START..USER_SPACE_END).contains(&vaddr.as_usize()) { + return false; + } + + let root = active_root_table(); + let page = Page::::including_address(vaddr); + + let root_entry = &root.entries[page.table_index::()]; + if !root_entry.is_present() { + return false; + } + let l1 = unsafe { + &*ptr::with_exposed_provenance::>(root_entry.address().as_usize()) + }; + let l1_entry = &l1.entries[page.table_index::()]; + if !l1_entry.is_present() { + return false; + } + let l0 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>(l1_entry.address().as_usize()) + }; + let l0_entry = &mut l0.entries[page.table_index::()]; + + let flags = l0_entry.flags(); + let is_cow = l0_entry.is_present() + && flags.contains(PageTableEntryFlags::USER_ACCESSIBLE) + && flags.contains(PageTableEntryFlags::COW_MARKER) + && !flags.contains(PageTableEntryFlags::WRITABLE); + if !is_cow { + return false; + } + + let src_phys = l0_entry.address(); + let mut new_flags = flags; + new_flags.remove(PageTableEntryFlags::COW_MARKER); + new_flags.insert(PageTableEntryFlags::WRITABLE); + + // Drop this address space's COW reference. + if crate::mm::frame_ref_dec(src_phys) { + // Last reference: no other address space uses the frame anymore, + // so it can simply be made writable again. + crate::mm::frame_ref_inc(src_phys); + l0_entry.set(src_phys, new_flags); + } else { + // The frame is still shared: clone it and map the copy writable. + let new_phys = crate::mm::copy_page(src_phys); + crate::mm::frame_ref_inc(new_phys); + l0_entry.set(new_phys, new_flags); + } + + asm::sfence_vma(0, vaddr.as_usize()); + true +} + +/// Mark the user pages of the current address space Copy-On-Write in +/// preparation of duplicating it for a fork. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn prepare_mem_copy_on_write() { + mark_user_pages_copy_on_write(); +} + /// Clear the user-space portion of the currently active address space. /// /// Used by `exec` to tear down the old process image before the loader diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index 81908977a4..dc6508ae09 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -1318,6 +1318,107 @@ pub unsafe fn fork() -> TaskId { tid } +/// Fork the current task from its saved user-mode context (riscv64). +/// +/// On riscv64 the user-mode dispatcher (`user_loop`) owns the complete +/// trap context, so — unlike the x86_64/aarch64 path — no kernel stack +/// has to be cloned: the child gets a fresh kernel stack and re-enters +/// user mode through its own `user_loop_resume`, seeded with a copy of +/// the parent's `UserContext` in which `a0` (the fork return value) is +/// set to 0. +/// +/// Returns the child's `TaskId`; only the parent executes this function. +#[cfg(all(target_arch = "riscv64", feature = "common-os", feature = "fork"))] +pub unsafe fn fork_from_user_context(parent_ctx: &trapframe::UserContext) -> TaskId { + use crate::arch::mm::{copy_current_root_page_table, prepare_mem_copy_on_write}; + + let core_id = SPAWN_COUNTER.fetch_add(1, Ordering::SeqCst) % get_processor_count(); + + // Mark user pages COW before copying the page table. + prepare_mem_copy_on_write(); + let child_root_page_table = copy_current_root_page_table(); + + let stack_size = core_scheduler() + .get_current_task() + .borrow() + .stacks + .get_user_stack_size(); + let stacks = TaskStacks::new(stack_size); + + // The child resumes user execution with the parent's register state, + // except that `fork` returns 0 in the child. + let mut child_ctx = Box::new(*parent_ctx); + child_ctx.general.a0 = 0; + let child_last_sp = kernel::scheduler::create_fork_stack_frame(&stacks, child_ctx); + + let tid = get_tid(); + let parent_user_sp = core_scheduler() + .get_current_task() + .borrow() + .user_stack_pointer; + let parent_prio = core_scheduler().get_current_task().borrow().prio; + let parent_object_map = Arc::new(RwSpinLock::new(HashMap::< + RawFd, + Arc>, + RandomState, + >::with_hasher(RandomState::with_seeds( + 0, 0, 0, 0, + )))); + for (key, val) in core_scheduler().get_current_task_object_map().read().iter() { + parent_object_map.write().insert(*key, val.clone()); + } + let parent_tls_template = core_scheduler() + .get_current_task() + .borrow() + .tls_template + .clone(); + let parent_vmas = Arc::new(RwSpinLock::new( + core_scheduler() + .get_current_task() + .borrow() + .vmas + .read() + .clone(), + )); + + let child_task = Task::new_fork( + tid, + core_id, + TaskStatus::Ready, + parent_prio, + stacks, + child_last_sp, + parent_user_sp, + parent_object_map, + Arc::new(RootPageTable::new(child_root_page_table)), + parent_tls_template, + parent_vmas, + ); + + { + #[cfg(feature = "smp")] + let _input_locked = get_scheduler_input(core_id).lock(); + WAITING_TASKS.lock().insert(tid, VecDeque::with_capacity(1)); + TASKS.lock().insert( + tid, + TaskHandle::new( + tid, + parent_prio, + #[cfg(feature = "smp")] + core_id, + ), + ); + NO_TASKS.fetch_add(1, Ordering::SeqCst); + + let task = Rc::new(RefCell::new(child_task)); + core_scheduler().ready_queue.push(task); + } + + debug!("Child was created and has the id {tid}"); + + tid +} + pub fn shutdown(arg: i32) -> ! { crate::syscalls::shutdown(arg) } diff --git a/src/syscalls/table.rs b/src/syscalls/table.rs index 8fcda19516..0e2332b584 100644 --- a/src/syscalls/table.rs +++ b/src/syscalls/table.rs @@ -37,7 +37,7 @@ const SYSNO_WRITEV: usize = 12; /// Number of the system call `readv` const SYSNO_READV: usize = 13; /// number of the system call `fork` -const SYSNO_FORK: usize = 14; +pub(crate) const SYSNO_FORK: usize = 14; /// number of the system call `waitpid` const SYSNO_WAITPID: usize = 15; /// number of the system call `spawn_process` diff --git a/src/syscalls/tasks.rs b/src/syscalls/tasks.rs index c43de3c6e7..098c96bba0 100644 --- a/src/syscalls/tasks.rs +++ b/src/syscalls/tasks.rs @@ -18,7 +18,10 @@ pub type Pid = i32; /// Fork the current process. /// Returns the child's PID to the parent, and 0 to the child. -/// Fork is not implemented on riscv64 yet, so the call fails there. +/// +/// On riscv64 fork is dispatched directly by the user-mode trap loop +/// (`user_loop`), which owns the saved user context; this table entry is +/// only a fallback and reports `ENOSYS`. #[cfg(feature = "common-os")] #[hermit_macro::system(errno)] #[unsafe(no_mangle)]