diff --git a/Cargo.toml b/Cargo.toml index 467138ff40..9650b72105 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,9 @@ default = [ ## Note that this feature is not complete yet. common-os = [] +## Enable the support of classical fork +fork = ["common-os"] + ## Enables support for memory management system calls. ## ## This feature enables functions similar to [sys/mman.h]. diff --git a/src/arch/aarch64/kernel/interrupts.rs b/src/arch/aarch64/kernel/interrupts.rs index 0717b2222f..bc260d9dd0 100644 --- a/src/arch/aarch64/kernel/interrupts.rs +++ b/src/arch/aarch64/kernel/interrupts.rs @@ -30,6 +30,14 @@ const PPI_START: u8 = 16; const SPI_START: u8 = 32; /// Software-generated interrupt for rescheduling pub(crate) const SGI_RESCHED: u8 = 1; +/// Synthetic IRQ slot used for page-fault accounting. The number does not +/// correspond to any GIC interrupt — it is purely a bookkeeping ID for +/// `IrqStatistics` so the page-fault count shows up in `print_statistics` +/// alongside the real interrupts. Picked at 14 to mirror the x86_64 +/// CPU exception vector for #PF, which keeps the two architectures +/// consistent in the diagnostic output. +#[cfg(feature = "common-os")] +pub(crate) const PAGE_FAULT_IRQ: u8 = 14; /// Number of the timer interrupt static mut TIMER_INTERRUPT: u32 = 0; @@ -170,43 +178,138 @@ pub(crate) extern "C" fn do_irq(_state: &State) -> *mut usize { } #[unsafe(no_mangle)] -pub(crate) extern "C" fn do_sync(state: &State) { +pub(crate) extern "C" fn do_sync(state: &mut State) { + #[cfg(all(feature = "common-os", feature = "fork"))] + use crate::arch::mm::paging::do_cow_fault; + let esr = ESR_EL1.get(); let ec_raw = ESR_EL1.read(ESR_EL1::EC); let ec: ESR_EL1::EC::Value = ESR_EL1.read_as_enum(ESR_EL1::EC).unwrap(); let iss = ESR_EL1.read(ESR_EL1::ISS); let pc = ELR_EL1.get(); - /* data abort from lower or current level */ - if (ec == ESR_EL1::EC::Value::DataAbortCurrentEL) - || (ec == ESR_EL1::EC::Value::DataAbortLowerEL) - { - /* check if value in far_el1 is valid */ - if (iss & (1 << 10)) == 0 { - /* read far_el1 register, which holds the faulting virtual address */ - let far = FAR_EL1.get(); - - // add page fault handler - - error!("Current stack pointer {state:p}"); - error!("Unable to handle page fault at {far:#x}"); - error!("Exception return address {:#x}", ELR_EL1.get()); - error!("Thread ID register {:#x}", TPIDR_EL0.get()); - error!("Table Base Register {:#x}", TTBR0_EL1.get()); - error!("Exception Syndrome Register {esr:#x}"); - - if let Some(irqid) = - GicCpuInterface::get_and_acknowledge_interrupt(InterruptGroup::Group1) + // SVC64 — user-space syscall (ESR_EL1.EC = 0x15). Dispatched here + // before any of the other branches so we can return early without + // touching debug/breakpoint/FPU paths. + #[cfg(feature = "common-os")] + if ec_raw == 0x15 { + dispatch_svc64(state); + return; + } + + // User-mode instruction-fetch fault at PC=0: the entry wrapper of a + // freshly spawned user thread (`std::sys::thread::hermit::Thread:: + // new_with_coreid::thread_start`) returns with `ret`, popping LR + // from the user stack. The trap frame we crafted in + // `Task::create_user_stack_frame` zeroed every register, so LR=0 + // and the implicit branch lands at PC 0 — there is no code there. + // Mirror the x86_64 page-fault handler and treat this as a clean + // thread exit instead of crashing the whole process. + #[cfg(feature = "common-os")] + if ec == ESR_EL1::EC::Value::InstrAbortLowerEL && ELR_EL1.get() == 0 { + use crate::scheduler::PerCoreSchedulerExt; + debug!( + "User thread {} returned from entry; exiting cleanly.", + core_scheduler().get_current_task_id() + ); + core_scheduler().exit(0); + } + + /* Data Abort from current or lower EL — the primary path is a COW + * write fault from EL0 (EC=0x25). EC=0x24 covers a kernel write to a + * COW-marked page, which can happen e.g. when the kernel writes + * argv/envp into the freshly-mapped user page during the loader path. + */ + if ec == ESR_EL1::EC::Value::DataAbortCurrentEL || ec == ESR_EL1::EC::Value::DataAbortLowerEL { + #[cfg(feature = "common-os")] + increment_irq_counter(PAGE_FAULT_IRQ); + + let far = FAR_EL1.get(); + // ESR_EL1.ISS layout for Data Abort (ARM ARM D24.2.45): + // bits 5:0 = DFSC (Data Fault Status Code) + // bit 6 = WnR (1 = write, 0 = read) + // Permission fault DFSC values are 0b001100..0b001111 (level 0..3). + let dfsc = iss & 0b11_1111; + let is_write = (iss & (1 << 6)) != 0; + #[cfg(all(feature = "common-os", feature = "fork"))] + let is_permission_fault = (0b00_1100..=0b00_1111).contains(&dfsc); + + #[cfg(all(feature = "common-os", feature = "fork"))] + if is_write && is_permission_fault && do_cow_fault(VirtAddr::new(far)) { + // Faulting instruction is retried on `eret` from the trap. + return; + } + + #[cfg(feature = "common-os")] + { + use core::ops::Bound; + + use align_address::Align; + + use crate::mm::FrameAlloc; + use crate::mm::vma::VirtualMemoryAreaProt; + + let addr = VirtAddr::new(far).align_down(BasePageSize::SIZE); + let current_task = core_scheduler().get_current_task(); + let current_task_borrowed = current_task.borrow(); + let guard = current_task_borrowed.vmas.read(); + + if let Some((_, vma)) = guard + .range((Bound::Unbounded, Bound::Included(addr))) + .next_back() && addr >= vma.start + && addr < vma.end { - GicCpuInterface::end_interrupt(irqid, InterruptGroup::Group1); - } else { - error!("Unable to acknowledge interrupt!"); + let layout = PageLayout::from_size_align( + BasePageSize::SIZE as usize, + BasePageSize::SIZE as usize, + ) + .unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let physaddr = PhysAddr::from(frame_range.start()); + let mut flags = PageTableEntryFlags::empty(); + flags.normal().user(); + if vma.prot.contains(VirtualMemoryAreaProt::WRITE) { + flags.writable(); + } + if vma.prot.contains(VirtualMemoryAreaProt::EXECUTE) { + flags.execute_disable(); + } + + paging::map::(addr, physaddr, 1, flags); + + // clear page + let slice = unsafe { + core::slice::from_raw_parts_mut( + addr.as_mut_ptr::().cast::(), + BasePageSize::SIZE as usize, + ) + }; + slice.fill(0); + + #[cfg(feature = "fork")] + crate::mm::frame_ref_inc(physaddr); + + return; } + } + + let kind = dfsc_kind(dfsc); + let access = if is_write { "write" } else { "read" }; + error!("Current stack pointer {state:p}"); + error!("Unhandled data abort: {kind} on {access} of {far:#x} (DFSC={dfsc:#x})"); + error!("Exception return address {:#x}", ELR_EL1.get()); + error!("Thread ID register {:#x}", TPIDR_EL0.get()); + error!("Table Base Register {:#x}", TTBR0_EL1.get()); + error!("Exception Syndrome Register {esr:#x}"); - scheduler::abort() + if let Some(irqid) = GicCpuInterface::get_and_acknowledge_interrupt(InterruptGroup::Group1) + { + GicCpuInterface::end_interrupt(irqid, InterruptGroup::Group1); } else { - error!("Unknown exception"); + error!("Unable to acknowledge interrupt!"); } + + scheduler::abort() } else if ec == ESR_EL1::EC::Value::Brk64 { error!("Trap to debugger, PC={pc:#x}"); loop { @@ -232,6 +335,64 @@ pub(crate) extern "C" fn do_sync(state: &State) { } } +/// Convert the 6-bit DFSC (Data Fault Status Code) of `ESR_EL1.ISS` into +/// a short human-readable label for diagnostics. Mapping per ARM ARM +/// D24.2.45 (ESR_EL1, Data Abort). +fn dfsc_kind(dfsc: u64) -> &'static str { + match dfsc { + 0b00_0000..=0b00_0011 => "address size fault", + 0b00_0100..=0b00_0111 => "translation fault", + 0b00_1000..=0b00_1011 => "access flag fault", + 0b00_1100..=0b00_1111 => "permission fault", + 0b01_0000 => "synchronous external abort", + 0b01_0001 => "synchronous tag check fail", + 0b01_0100..=0b01_0111 => "external abort on translation table walk", + 0b01_1000 => "synchronous parity/ECC error", + 0b01_1100..=0b01_1111 => "parity/ECC error on translation table walk", + 0b10_0001 => "alignment fault", + 0b11_0000 => "TLB conflict abort", + 0b11_0001 => "unsupported atomic hardware-update fault", + _ => "unknown data fault", + } +} + +/// Dispatch an EL0-issued AArch64 syscall (`svc #0`). +/// +/// Hermit's user-space follows the Linux-like AArch64 convention: +/// - syscall number in `x8` +/// - up to 6 arguments in `x0`..`x5` +/// - return value in `x0` +/// +/// The handler entries in `SYSHANDLER_TABLE` are typed for the SystemV +/// x86_64 ABI but are reachable just as well via the AAPCS64 ABI: in both +/// cases the first six 64-bit args land in the first six argument +/// registers and the return value goes into the first return register. +/// Registers in `state` were saved by `trap_entry` and are restored by +/// `trap_exit` — writing back `state.x0` is what propagates the return +/// value to user-space. +#[cfg(feature = "common-os")] +fn dispatch_svc64(state: &mut State) { + use crate::errno::Errno; + use crate::syscalls::table::{NO_SYSCALLS, SYSHANDLER_TABLE, invalid_syscall, sys_invalid}; + + let nr = state.x8 as usize; + + if nr >= NO_SYSCALLS { + error!("Invalid syscall number {nr}"); + state.x0 = u64::from((-i32::from(Errno::Nosys)) as u32); + return; + } + + let handler_ptr = SYSHANDLER_TABLE.handler(nr); + if handler_ptr == sys_invalid as *const usize { + invalid_syscall(state.x8); + } + + let f: extern "C" fn(u64, u64, u64, u64, u64, u64) -> u64 = + unsafe { core::mem::transmute(handler_ptr) }; + state.x0 = f(state.x0, state.x1, state.x2, state.x3, state.x4, state.x5); +} + #[unsafe(no_mangle)] pub(crate) extern "C" fn do_bad_mode(_state: &State, reason: u32) -> ! { error!("Receive unhandled exception: {reason}"); @@ -335,11 +496,26 @@ pub(crate) fn init() { if let Some(timer_node) = fdt.find_compatible(&["arm,armv8-timer", "arm,armv7-timer"]) { let irq_slice = timer_node.property("interrupts").unwrap().value; - /* Secure Phys IRQ */ + // The "arm,armv8-timer" interrupts property lists four (type, irq, + // flags) triplets in this exact order: + // 1. Secure Phys + // 2. Non-secure Phys + // 3. Virtual ← we want this one + // 4. Hypervisor Phys + // We program the Virtual Timer (CNTV_*) instead of the Physical + // Timer because virtualised guests (e.g. macOS HVF) hide the + // physical timer from EL1, and CNTV_* works identically on bare + // metal where CNTVOFF_EL2 defaults to 0. + + /* Secure Phys IRQ — skip */ let (_irqtype, irq_slice) = irq_slice.split_at(size_of::()); let (_irq, irq_slice) = irq_slice.split_at(size_of::()); let (_irqflags, irq_slice) = irq_slice.split_at(size_of::()); - /* Non-secure Phys IRQ */ + /* Non-secure Phys IRQ — skip */ + let (_irqtype, irq_slice) = irq_slice.split_at(size_of::()); + let (_irq, irq_slice) = irq_slice.split_at(size_of::()); + let (_irqflags, irq_slice) = irq_slice.split_at(size_of::()); + /* Virtual Timer IRQ */ let (irqtype, irq_slice) = irq_slice.split_at(size_of::()); let (irq, irq_slice) = irq_slice.split_at(size_of::()); let (irqflags, _irq_slice) = irq_slice.split_at(size_of::()); @@ -426,6 +602,8 @@ pub(crate) fn init() { .unwrap(); gic.enable_interrupt(reschedid, Some(cpu_id), true).unwrap(); IRQ_NAMES.lock().insert(SGI_RESCHED, "Reschedule"); + #[cfg(feature = "common-os")] + IRQ_NAMES.lock().insert(PAGE_FAULT_IRQ, "Page Fault"); *GIC.lock() = Some(gic); } diff --git a/src/arch/aarch64/kernel/mod.rs b/src/arch/aarch64/kernel/mod.rs index cb9587b0c8..c3d161ef71 100644 --- a/src/arch/aarch64/kernel/mod.rs +++ b/src/arch/aarch64/kernel/mod.rs @@ -16,12 +16,21 @@ pub mod systemtime; use alloc::alloc::alloc; use core::alloc::Layout; +#[cfg(feature = "common-os")] +use core::arch::asm; use core::arch::global_asm; use core::ptr; +#[cfg(feature = "common-os")] +use core::slice; use core::sync::atomic::{AtomicPtr, AtomicU32, Ordering}; +#[cfg(feature = "common-os")] +use memory_addresses::{PhysAddr, VirtAddr}; + pub(crate) use self::interrupts::wakeup_core; pub(crate) use self::processor::set_oneshot_timer; +#[cfg(all(feature = "common-os", feature = "fork"))] +pub use self::scheduler::prepare_fork_child_stack; use crate::arch::aarch64::kernel::core_local::*; use crate::arch::aarch64::mm::paging::{BasePageSize, PageSize}; use crate::config::*; @@ -171,3 +180,341 @@ pub fn boot_next_processor() { pub fn print_statistics() { interrupts::print_statistics(); } + +#[cfg(feature = "common-os")] +pub(crate) const USER_START: VirtAddr = VirtAddr::new(0x0100_0000_0000); +// Place the user stack at the top of the L0 slot that backs USER_START +// (L0[USER_L0_INDEX] = L0[2], spanning 0x0100_0000_0000..0x0180_0000_0000). +// Only this slot is deep-copied by `create_new_root_page_table`; every +// other L0 entry is inherited verbatim from the parent, so a stack +// outside L0[2] would share its L1/L2/L3 tables with the spawning +// process and the first `paging::map` call from the child would +// silently overwrite the parent's stack mapping. +#[cfg(feature = "common-os")] +const USER_STACK: VirtAddr = VirtAddr::new(0x0180_0000_0000 - USER_STACK_SIZE as u64); +#[cfg(feature = "common-os")] +const USER_STACK_SIZE: usize = 0x8000; + +/// Map the user-mode binary into the address space and run the ELF-loader +/// closure against the freshly-mapped pages. Mirrors the x86_64 sibling +/// (`arch::x86_64::kernel::load_application`). +#[allow(clippy::result_unit_err)] +#[cfg(feature = "common-os")] +pub fn load_application(code_size: u64, tls_size: u64, func: F) -> Result<(), ()> +where + F: FnOnce( + &'static mut [u8], + Option<&'static mut [u8]>, + ) -> Result>, ()>, +{ + use alloc::sync::Arc; + + use ahash::RandomState; + use align_address::Align; + use free_list::PageLayout; + use hashbrown::HashMap; + use hermit_sync::RwSpinLock; + + use crate::arch::aarch64::mm::paging::{self, PageTableEntryFlags}; + use crate::fd::{Fd, RawFd}; + #[cfg(feature = "fork")] + use crate::mm::frame_ref_inc; + use crate::mm::vma::*; + use crate::mm::{FrameAlloc, PageRangeAllocator}; + + // Each process has its own object map. + let mut object_map = HashMap::>, RandomState>::with_hasher( + RandomState::with_seeds(0, 0, 0, 0), + ); + crate::fd::stdio::setup(&mut object_map); + core_scheduler().set_current_task_object_map(Arc::new(RwSpinLock::new(object_map))); + + let code_size = (code_size as usize).align_up(BasePageSize::SIZE as usize); + let layout = PageLayout::from_size_align(code_size, BasePageSize::SIZE as usize).unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let physaddr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] + for i in 0..code_size / BasePageSize::SIZE as usize { + frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize); + } + + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user().execute_enable(); + paging::map::( + USER_START, + physaddr, + code_size / BasePageSize::SIZE as usize, + flags, + ); + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( + USER_START, + VirtualMemoryArea::new( + USER_START, + (USER_START + code_size).align_up(BasePageSize::SIZE), + VirtualMemoryAreaProt::READ + | VirtualMemoryAreaProt::WRITE + | VirtualMemoryAreaProt::EXECUTE, + MemoryType::CODE, + ), + ); + + let loader_start_ptr = ptr::with_exposed_provenance_mut(USER_START.as_usize()); + let code_slice = unsafe { slice::from_raw_parts_mut(loader_start_ptr, code_size) }; + + if tls_size > 0 { + // AArch64 uses TLS Variant I: the thread pointer (TPIDR_EL0) points to + // the TCB; the TLS image follows immediately after a two-word reserved + // area (`tcb[0] = dtv`, `tcb[1]` reserved). We allocate the TCB plus + // the TLS image as one contiguous block so a single `msr tpidr_el0` + // suffices and the layout matches what musl/glibc expect. + let tcb_size = 2 * size_of::<*mut ()>(); + let tls_offset = tcb_size; + + let tls_memsz = (tls_offset + tls_size as usize).align_up(BasePageSize::SIZE as usize); + let layout = PageLayout::from_size(tls_memsz).unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let physaddr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] + for i in 0..tls_memsz / BasePageSize::SIZE as usize { + frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize); + } + + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user().execute_disable(); + let tls_virt = USER_START + code_size + BasePageSize::SIZE as usize; + paging::map::( + tls_virt, + physaddr, + tls_memsz / BasePageSize::SIZE as usize, + flags, + ); + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( + tls_virt, + VirtualMemoryArea::new( + tls_virt, + (tls_virt + tls_memsz).align_up(BasePageSize::SIZE), + VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE, + MemoryType::TLS, + ), + ); + + let block = unsafe { + slice::from_raw_parts_mut(tls_virt.as_mut_ptr(), tls_offset + tls_size as usize) + }; + for elem in block.iter_mut() { + *elem = 0; + } + + // Variant I: TPIDR_EL0 points at the TCB; user code finds its data at + // TPIDR_EL0 + tls_offset. + let thread_ptr = block.as_mut_ptr().cast::<()>(); + set_user_tpidr_el0(thread_ptr.expose_provenance() as u64); + + // The ELF loader copies the binary's PT_TLS initial image into the + // region that follows the TCB. + let tls_image = &mut block[tls_offset..]; + let tls_init = func(code_slice, Some(tls_image))?; + + if let Some(init) = tls_init { + let template = Arc::new(crate::scheduler::task::TlsTemplate { + size: tls_size as usize, + init, + }); + core_scheduler() + .get_current_task() + .borrow_mut() + .tls_template = Some(template); + } + + Ok(()) + } else { + // No TLS in the freshly loaded image. We must still reset TPIDR_EL0 + // because an `exec()` re-enters this path: a stale value left over + // from the previous program's TLS would otherwise persist across + // the image swap and corrupt unrelated user-mode state on the + // next thread-local access. + set_user_tpidr_el0(0); + func(code_slice, None)?; + Ok(()) + } +} + +/// Set the user-space `TPIDR_EL0` value for the *current* task. +/// +/// The naive `msr tpidr_el0, xN` only changes the live register. When this +/// helper is called from inside an SVC handler — as it always is on the +/// `load_application` / `exec` path — `trap_exit` would later overwrite +/// `tpidr_el0` again from the value `trap_entry` saved in the on-stack +/// `State` struct (the trap-frame's `tpidr_el0` field). To make the new +/// thread pointer survive the trap-exit we therefore *also* update the +/// saved trap frame in place. The `State` lives at the very top of the +/// current kernel stack (`stack_top - MARKER_SIZE - sizeof(State)`). +#[cfg(feature = "common-os")] +fn set_user_tpidr_el0(value: u64) { + use crate::arch::aarch64::kernel::scheduler::{State, TaskStacks}; + + // Update the live system register so any pre-eret read in this kernel + // path sees the new value. + unsafe { + asm!( + "msr tpidr_el0, {0}", + in(reg) value, + options(nomem, nostack, preserves_flags), + ); + } + + // Patch the saved trap frame so trap_exit restores the new value too. + // `trap_entry` saved the user TPIDR_EL0 at SVC time; without this patch, + // `trap_exit` would clobber our just-installed value with the stale one. + let task = core_scheduler().get_current_task(); + let kernel_stack_top = task.borrow().stacks.get_kernel_stack().as_usize() + + task.borrow().stacks.get_kernel_stack_size(); + let state_addr = kernel_stack_top - TaskStacks::MARKER_SIZE - size_of::(); + unsafe { + let state = ptr::with_exposed_provenance_mut::(state_addr); + (*state).tpidr_el0 = value; + } +} + +/// Drop into EL0, executing the freshly-loaded user binary at `entry_point`. +/// +/// `iretq` on x86_64 corresponds to `eret` on AArch64: ELR_EL1 supplies the +/// new PC, SPSR_EL1 the new PSTATE (mode bits select EL0t), and SP_EL0 the +/// user stack. Per AAPCS64, `argc` lives in `x0` and `argv` in `x1`. +#[cfg(feature = "common-os")] +pub unsafe fn jump_to_user_land( + entry_point: usize, + args: alloc::vec::Vec, + envs: alloc::vec::Vec, +) -> ! { + use align_address::Align; + use free_list::PageLayout; + + use crate::arch::aarch64::kernel::scheduler::TaskStacks; + use crate::arch::aarch64::mm::paging::{self, PageTableEntryFlags}; + #[cfg(feature = "fork")] + use crate::mm::frame_ref_inc; + use crate::mm::vma::*; + use crate::mm::{FrameAlloc, PageRangeAllocator}; + + debug!("Create new file descriptor table"); + core_scheduler().recreate_objmap().unwrap(); + + let entry_point: usize = USER_START.as_usize() | entry_point; + let stack_top: usize = USER_STACK.as_usize() + USER_STACK_SIZE - TaskStacks::MARKER_SIZE; + + let layout = PageLayout::from_size(USER_STACK_SIZE).unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let phys_addr = PhysAddr::from(frame_range.start()); + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user(); + paging::map::( + USER_STACK, + phys_addr, + USER_STACK_SIZE / BasePageSize::SIZE as usize, + flags, + ); + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( + USER_STACK, + VirtualMemoryArea::new( + USER_STACK, + USER_STACK + USER_STACK_SIZE, + VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE, + MemoryType::STACK, + ), + ); + #[cfg(feature = "fork")] + for i in 0..USER_STACK_SIZE / BasePageSize::SIZE as usize { + frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); + } + + // Place the argv and envp pointer arrays on the user stack. Both + // arrays follow the C convention and are terminated by a null pointer. + let ptr_count = args.len() + 1 + envs.len() + 1; + let stack_pointer = stack_top - ptr_count * size_of::<*mut u8>(); + let stack_ptr = ptr::with_exposed_provenance_mut::<*mut u8>(stack_pointer); + let arrays = unsafe { slice::from_raw_parts_mut(stack_ptr, ptr_count) }; + let (argv, envp) = arrays.split_at_mut(args.len() + 1); + let len = args + .iter() + .chain(envs.iter()) + .fold(0, |acc, x| acc + x.as_bytes_with_nul().len()); + // AAPCS64 requires SP to be 16-byte aligned at function entry. + let stack_pointer = (stack_pointer - len).align_down(16); + + let mut pos: usize = 0; + for (dst, s) in argv + .iter_mut() + .zip(args.iter()) + .chain(envp.iter_mut().zip(envs.iter())) + { + let bytes = s.as_bytes_with_nul(); + *dst = ptr::with_exposed_provenance_mut::(stack_pointer + pos); + pos += bytes.len(); + + unsafe { + dst.copy_from_nonoverlapping(bytes.as_ptr(), bytes.len()); + } + } + argv[args.len()] = ptr::null_mut(); + envp[envs.len()] = ptr::null_mut(); + + let argc = args.len(); + drop(args); + drop(envs); + + debug!("Jump to user space at 0x{entry_point:x}, stack pointer 0x{stack_pointer:x}"); + + // SPSR_EL1: M[4:0]=0b00000 ⇒ EL0t / AArch64; DAIF=0 ⇒ all interrupts enabled. + const USER_SPSR: u64 = 0; + + unsafe { + asm!( + // Mask all interrupts during the EL1t→EL1h→EL0t transition. + "msr daifset, #0b1111", + // Switch to EL1h (SPSEL=1) so SP_EL0 is no longer the active SP + // before we overwrite it. The initd task is started by `task_start` + // in EL1t (SPSEL=0); writing SP_EL0 while it is the active stack + // pointer is UNDEFINED on AArch64. + "msr spsel, #1", + "msr sp_el0, {sp}", + "msr elr_el1, {pc}", + "msr spsr_el1, {spsr}", + // Clear scratch registers so EL1 state cannot leak into EL0. + // `x0`, `x1`, and `x2` carry argc, argv, and envp and are + // bound as fixed register operands below, so the register + // allocator cannot hand them out for the other operands. + "mov x3, xzr", "mov x4, xzr", "mov x5, xzr", + "mov x6, xzr", "mov x7, xzr", "mov x8, xzr", "mov x9, xzr", + "mov x10, xzr", "mov x11, xzr", "mov x12, xzr", "mov x13, xzr", + "mov x14, xzr", "mov x15, xzr", "mov x16, xzr", "mov x17, xzr", + "mov x18, xzr", "mov x29, xzr", "mov x30, xzr", + "eret", + // Speculation barrier behind ERET (Spectre-v1 mitigation). + "dsb nsh", + "isb", + sp = in(reg) stack_pointer, + pc = in(reg) entry_point, + spsr = in(reg) USER_SPSR, + in("x0") argc, + in("x1") argv.as_ptr(), + in("x2") envp.as_ptr(), + options(nostack, noreturn) + ); + } +} diff --git a/src/arch/aarch64/kernel/processor.rs b/src/arch/aarch64/kernel/processor.rs index c27034b922..7676314999 100644 --- a/src/arch/aarch64/kernel/processor.rs +++ b/src/arch/aarch64/kernel/processor.rs @@ -117,7 +117,12 @@ static CPU_FREQUENCY: Lazy = Lazy::new(|| { } cpu_frequency }); -// Value of CNTPCT_EL0 at boot time +// Value of CNTVCT_EL0 at boot time. We use the Virtual Timer (CNTV_*) +// rather than the Physical Timer (CNTP_*): the virtual timer is always +// available — including under hypervisors that hide CNTP_* from EL1 +// (e.g. macOS HVF on Apple Silicon, where MSR/MRS to `CNTP_CVAL_EL0` +// faults as UNDEF) — and behaves identically on bare metal because +// CNTVOFF_EL2 defaults to zero when no hypervisor is in play. static BOOT_COUNTER: OnceCell = OnceCell::new(); enum CpuFrequencySources { @@ -206,26 +211,47 @@ pub fn halt() { aarch64_cpu::asm::wfi(); } -/// Shutdown the system +/// Shutdown the system. +/// +/// We try PSCI first, with semihosting as a fallback. Rationale: +/// +/// * **PSCI `SYSTEM_OFF`** is implemented by QEMU's `virt` machine for +/// both `accel=tcg` and `accel=hvf`/`accel=kvm`, and by real-hardware +/// firmware. It is the most portable shutdown primitive on AArch64. +/// +/// * **AArch64 semihosting** uses `HLT #0xf000`. On `accel=tcg`, QEMU +/// intercepts that instruction and exits with the supplied code. +/// Under `accel=hvf`/`accel=kvm` the HLT goes straight to the guest +/// as an UNDEF exception (`EC=0x0`) because hardware virtualisation +/// does not trap it — so semihosting is *not* a viable shutdown +/// primitive in a virtualised guest. We therefore use it only as a +/// fallback for the `_exit(error_code)` semantic when PSCI declines +/// to terminate (e.g. on platforms without a PSCI dispatcher). #[allow(unused_variables)] pub fn shutdown(error_code: i32) -> ! { info!("Shutting down system"); - cfg_select! { - feature = "semihosting" => { - semihosting::process::exit(error_code) - } - _ => { - unsafe { - const PSCI_SYSTEM_OFF: u64 = 0x8400_0008; - // call hypervisor to shut down the system - asm!("hvc #0", in("x0") PSCI_SYSTEM_OFF, options(nomem, nostack)); - - // we should never reach this point - loop { - aarch64_cpu::asm::wfe(); - } - } + unsafe { + const PSCI_SYSTEM_OFF: u64 = 0x8400_0008; + // hvc #0 with x0 = PSCI_SYSTEM_OFF: QEMU virt's PSCI dispatcher + // terminates the VM. On real hardware this returns to firmware + // which performs the shutdown. + asm!("hvc #0", in("x0") PSCI_SYSTEM_OFF, options(nomem, nostack)); + } + + // PSCI did not terminate (no dispatcher, or call returned). Fall back + // to semihosting if the feature is compiled in — useful under TCG + // where it correctly propagates `error_code` to the host shell. + #[cfg(feature = "semihosting")] + { + semihosting::process::exit(error_code) + } + + #[cfg(not(feature = "semihosting"))] + { + // Last resort: park the CPU forever. + loop { + aarch64_cpu::asm::wfe(); } } } @@ -247,7 +273,7 @@ pub fn get_frequency() -> u16 { #[inline] pub fn get_timestamp() -> u64 { - CNTPCT_EL0.get() - BOOT_COUNTER.get().unwrap() + CNTVCT_EL0.get() - BOOT_COUNTER.get().unwrap() } #[inline] @@ -308,7 +334,7 @@ pub fn configure() { } pub fn detect_frequency() { - BOOT_COUNTER.set(CNTPCT_EL0.get()).unwrap(); + BOOT_COUNTER.set(CNTVCT_EL0.get()).unwrap(); Lazy::force(&CPU_FREQUENCY); } @@ -316,8 +342,8 @@ pub fn detect_frequency() { fn __set_oneshot_timer(wakeup_time: Option) { let Some(wt) = wakeup_time else { // disable timer - CNTP_CVAL_EL0.set(0); - CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::CLEAR); + CNTV_CVAL_EL0.set(0); + CNTV_CTL_EL0.write(CNTV_CTL_EL0::ENABLE::CLEAR); return; }; @@ -325,8 +351,8 @@ fn __set_oneshot_timer(wakeup_time: Option) { let freq: u64 = CPU_FREQUENCY.get().into(); // frequency in KHz let deadline = (wt / 1000) * freq; - CNTP_CVAL_EL0.set(deadline); - CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET); + CNTV_CVAL_EL0.set(deadline); + CNTV_CTL_EL0.write(CNTV_CTL_EL0::ENABLE::SET); } pub fn set_oneshot_timer(wakeup_time: Option) { diff --git a/src/arch/aarch64/kernel/scheduler.rs b/src/arch/aarch64/kernel/scheduler.rs index 985aea77e5..427be8aa8e 100644 --- a/src/arch/aarch64/kernel/scheduler.rs +++ b/src/arch/aarch64/kernel/scheduler.rs @@ -1,6 +1,8 @@ //! Architecture dependent interface to initialize a task use core::arch::naked_asm; +#[cfg(feature = "common-os")] +use core::mem; use core::sync::atomic::Ordering; use aarch64_cpu::asm::barrier::{SY, isb}; @@ -140,23 +142,40 @@ impl TaskStacks { total_size >> 10 ); - let mut flags = PageTableEntryFlags::empty(); - flags.normal().writable().execute_disable(); + let mut kernel_flags = PageTableEntryFlags::empty(); + kernel_flags.normal().writable().execute_disable(); - // map kernel stack into the address space + // map kernel stack into the address space (kernel-only access) crate::arch::mm::paging::map::( virt_addr + BasePageSize::SIZE, phys_addr, DEFAULT_STACK_SIZE / BasePageSize::SIZE as usize, - flags, + kernel_flags, ); + // User-stack flags differ between unikernel and common-os builds: + // in common-os the same VA is reachable from both EL1 (the kernel + // crafting argv during jump_to_user_land) and EL0 (the running + // thread), so it must carry USER_ACCESSIBLE. Without this, a + // freshly spawned user thread (`scheduler::spawn_thread`) faults + // as soon as it touches its own stack — TaskStacks::new is the + // only path that allocates a user stack outside the LOADER_START + // region, so the bug only manifests on the thread-spawn path. + #[cfg(feature = "common-os")] + let user_flags = { + let mut f = PageTableEntryFlags::empty(); + f.normal().writable().user().execute_disable(); + f + }; + #[cfg(not(feature = "common-os"))] + let user_flags = kernel_flags; + // map user stack into the address space crate::arch::mm::paging::map::( virt_addr + DEFAULT_STACK_SIZE + 2 * BasePageSize::SIZE, phys_addr + DEFAULT_STACK_SIZE, user_stack_size / BasePageSize::SIZE as usize, - flags, + user_flags, ); // clear user stack @@ -187,6 +206,24 @@ impl TaskStacks { } } + /// Returns the start address of the stack region (virt_addr of CommonStack). + #[cfg(all(feature = "common-os", feature = "fork"))] + pub fn get_stack_virt_addr(&self) -> VirtAddr { + match self { + TaskStacks::Boot(stacks) => stacks.stack, + TaskStacks::Common(stacks) => stacks.virt_addr, + } + } + + /// Returns total size of all stacks combined. + #[cfg(all(feature = "common-os", feature = "fork"))] + pub fn get_total_stack_size(&self) -> usize { + match self { + TaskStacks::Boot(_) => KERNEL_STACK_SIZE, + TaskStacks::Common(stacks) => stacks.total_size, + } + } + pub fn get_user_stack(&self) -> VirtAddr { match self { TaskStacks::Boot(_) => VirtAddr::zero(), @@ -277,6 +314,68 @@ extern "C" fn task_start(_f: extern "C" fn(usize), _arg: usize) -> ! { ) } +#[cfg(feature = "common-os")] +impl Task { + /// Build the initial trap frame for a freshly spawned user-space + /// thread. Mirrors the role of the x86_64 sibling: when the scheduler + /// first picks this task, the standard `trap_exit` machinery pops the + /// `State` we craft here and `eret`s straight into ring 3 at + /// `func(arg)` on the new user stack — so no naked-asm "task_start_user" + /// trampoline is needed on AArch64. + /// + /// `tls_thread_ptr` is the value that should be installed in + /// `TPIDR_EL0` for the new thread (the per-thread TLS thread pointer + /// allocated by `scheduler::allocate_thread_tls`); zero leaves the + /// register as installed by the loader. + pub(crate) fn create_user_stack_frame( + &mut self, + func: unsafe extern "C" fn(usize), + arg: usize, + tls_thread_ptr: u64, + ) { + unsafe { + // Debug marker at the very top of the kernel stack. + let mut stack = self.stacks.get_kernel_stack() + self.stacks.get_kernel_stack_size() + - TaskStacks::MARKER_SIZE; + *stack.as_mut_ptr::() = 0xdead_beefu64; + + // Allocate space for the trap frame and zero it. Anything we + // don't touch below stays zero on entry to user space, which + // keeps any leftover kernel state out of EL0's general-purpose + // register file. + stack -= size_of::(); + let state = stack.as_mut_ptr::(); + state.cast::().write_bytes(0, size_of::()); + + // Initial user stack: top of the user-stack region with the + // usual debug marker. AAPCS64 doesn't require any extra slop + // (no red zone, no shadow space), so the user starts at SP + // pointing at the byte immediately above the marker. + self.user_stack_pointer = self.stacks.get_user_stack() + + self.stacks.get_user_stack_size() + - TaskStacks::MARKER_SIZE; + *self.user_stack_pointer.as_mut_ptr::() = 0xdead_beefu64; + + (*state).elr_el1 = mem::transmute::< + unsafe extern "C" fn(usize), + extern "C" fn(extern "C" fn(usize), usize) -> !, + >(func); + // SPSR_EL1 = 0 ⇒ M[4:0] = 0b00000 (EL0t / AArch64), DAIF = 0 + // (interrupts unmasked once the thread is running). + (*state).spsr_el1 = 0; + (*state).sp_el0 = self.user_stack_pointer.as_u64(); + (*state).tpidr_el0 = tls_thread_ptr; + // AAPCS64 first argument register. + (*state).x0 = arg as u64; + // SPSEL is consumed by trap_exit but does not affect the + // post-eret EL0 SP selection (SPSR_EL1 alone determines it). + (*state).spsel = 1; + + self.last_stack_pointer = stack; + } + } +} + impl TaskFrame for Task { fn create_stack_frame(&mut self, func: unsafe extern "C" fn(usize), arg: usize) { // Check if TLS is allocated already and if the task uses thread-local storage. @@ -331,5 +430,110 @@ pub(crate) extern "C" fn get_last_stack_pointer() -> u64 { CPACR_EL1.modify(CPACR_EL1::FPEN::TrapEl0El1); isb(SY); - core_scheduler().get_last_stack_pointer().as_u64() + let scheduler = core_scheduler(); + + // Switch TTBR0_EL1 to the new task's root page table when the address + // space changes between two `common-os` processes. The IRQ-driven + // context switch (see `start.s`) calls this helper after the scheduler + // has already promoted `current_task` to the new task; if we leave + // TTBR0_EL1 pointing at the previous process's table, the new task + // runs in the OLD address space — which becomes catastrophic the + // moment that task issues `clear_user_space` (it clears the wrong + // mapping) or its user code touches its own TLS. + #[cfg(feature = "common-os")] + { + let new_pt = scheduler + .get_current_task() + .borrow() + .root_page_table + .as_usize() as u64; + let cur_pt = TTBR0_EL1.get_baddr(); + if cur_pt != new_pt { + use aarch64_cpu::asm::barrier::{ISH, ISHST, dsb}; + + // Memory-barrier sequence per ARM ARM D8.13.2: DSB ISHST + // ensures all prior PT updates are observable; the MSR + // installs the new translation base; ISB flushes the + // pipeline so subsequent instructions use the new table. + dsb(ISHST); + TTBR0_EL1.set_baddr(new_pt); + isb(SY); + // Invalidate TLB entries from the old translation regime. + unsafe { + core::arch::asm!("tlbi vmalle1is", options(nostack)); + } + dsb(ISH); + isb(SY); + } + } + + scheduler.get_last_stack_pointer().as_u64() +} + +/// Prepare the child's stack and root page table for a fork(), AArch64. +/// +/// Mirrors the role of the x86_64 `prepare_fork_child_stack`, but does not +/// need a naked-asm child-entry stub: when the SVC trapped into EL1, the +/// hardware-supplied `trap_entry` macro pushed a complete `State` struct +/// at the top of the parent's kernel stack. Copying the kernel stack page +/// for the child duplicates that `State`; if we then patch `x0 = 0` in +/// the child's copy, the existing trap-exit machinery will `eret` it +/// straight back to the user-space instruction after the SVC with the +/// fork-returns-zero contract satisfied. +/// +/// Operations performed (in order; ordering matters): +/// 1. Copy the parent's kernel stack pages into `new_stack_addr`. This +/// runs in the parent's still-active page table, so the new mappings +/// become visible immediately. +/// 2. Snapshot the current root page table for the child via +/// `copy_current_root_page_table` — the snapshot now includes the +/// just-copied stack mapping. +/// 3. Patch the child's saved-`x0` to 0. +/// 4. Compute the child's saved kernel-SP (the address of the child's +/// `State` copy) and store it through `stack_pointer`. +/// +/// Returns `false` (the parent path); the child path becomes reachable +/// once the scheduler context-switches to the new task and the existing +/// IRQ trap-exit pops the child's `State` and `eret`s. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub unsafe fn prepare_fork_child_stack( + stack_pointer: *mut usize, + root_page_table: *mut usize, + new_stack_addr: usize, +) -> bool { + use crate::arch::aarch64::mm::{copy_current_root_page_table, copy_kernel_stack_to}; + + // 1. Copy the kernel stack pages into the child's region. Must run + // before the page-table snapshot so the new mappings are visible. + copy_kernel_stack_to(new_stack_addr); + + // 2. Duplicate the root page table for the child and hand the new + // physical address back to the caller. + let new_pt = copy_current_root_page_table(); + unsafe { *root_page_table = new_pt }; + + // 3. Locate the parent's saved `State` (the structure `trap_entry` + // pushed at SVC time) and compute the matching address inside the + // child's freshly-copied kernel stack. + let task = core_scheduler().get_current_task(); + let parent_stack_base = task.borrow().stacks.get_stack_virt_addr().as_usize(); + let kernel_stack_top = task.borrow().stacks.get_kernel_stack().as_usize() + + task.borrow().stacks.get_kernel_stack_size(); + let parent_state_addr = kernel_stack_top - TaskStacks::MARKER_SIZE - size_of::(); + let offset = new_stack_addr.wrapping_sub(parent_stack_base); + let child_state_addr = parent_state_addr.wrapping_add(offset); + + // 4. Patch the child's `x0` so fork() returns 0 there. The rest of + // the State (ELR_EL1 = post-SVC user PC, SPSR_EL1 = EL0t, SP_EL0 + // = user stack, x1..x30 = parent's user regs) is already correct. + unsafe { + let state = core::ptr::with_exposed_provenance_mut::(child_state_addr); + (*state).x0 = 0; + } + + // 5. Hand the child's kernel SP back to the caller; the scheduler + // will plug it into the child's task struct as `last_stack_pointer`. + unsafe { *stack_pointer = child_state_addr }; + + false } diff --git a/src/arch/aarch64/kernel/start.rs b/src/arch/aarch64/kernel/start.rs index 7b6f0590f9..bb846fe856 100644 --- a/src/arch/aarch64/kernel/start.rs +++ b/src/arch/aarch64/kernel/start.rs @@ -266,6 +266,39 @@ unsafe extern "C" fn pre_init(boot_info: Option<&'static RawBootInfo>, cpu_id: u // Memory barrier dsb(SY); + // On CPUs that implement FEAT_PAN (ARMv8.1+, e.g. Apple Silicon under + // HVF), `PSTATE.PAN` may default to 1, which would make every kernel + // write to a USER_ACCESSIBLE page (e.g. clearing the user-space TLS + // region during `load_application`) trap as a permission fault. + // Hermit's common-os path needs the kernel to be able to set up + // user pages on behalf of the loader, so: + // 1. set SCTLR_EL1.SPAN=1 so `PSTATE.PAN` is *not* forced to 1 on + // exception entry (otherwise every SVC/IRQ would re-set PAN + // and our `msr pan, #0` below would only hold for one trap), and + // 2. clear `PSTATE.PAN` itself. + // On older CPUs without FEAT_PAN (e.g. Cortex-A72) the PAN field in + // ID_AA64MMFR1_EL1 reads zero, so we skip the `msr pan, #0` (which + // would otherwise UNDEF). Touching SCTLR.SPAN is safe even there: + // the bit is RES0 and the OR is a no-op. + unsafe { + asm!( + // SCTLR_EL1.SPAN <- 1: keep PSTATE.PAN unchanged on exception entry. + "mrs {tmp}, sctlr_el1", + "orr {tmp}, {tmp}, #(1 << 23)", + "msr sctlr_el1, {tmp}", + "isb", + // Clear PSTATE.PAN if the CPU implements FEAT_PAN. + "mrs {tmp}, id_aa64mmfr1_el1", + "ubfx {tmp}, {tmp}, #20, #4", + "cbz {tmp}, 9f", + ".arch_extension pan", + "msr pan, #0", + "9:", + tmp = out(reg) _, + options(nostack, preserves_flags), + ); + } + if cpu_id == 0 { env::set_boot_info(*boot_info.unwrap()); crate::boot_processor_main() diff --git a/src/arch/aarch64/kernel/start.s b/src/arch/aarch64/kernel/start.s index 64703fb503..3d4320d0ef 100644 --- a/src/arch/aarch64/kernel/start.s +++ b/src/arch/aarch64/kernel/start.s @@ -243,13 +243,52 @@ el1_sp0_error: .size el1_sp0_error, .-el1_sp0_error .type el1_sp0_error, @function -el0_sync_invalid: - invalid 0 -.type el0_sync_invalid, @function +/* + * Synchronous exception from a lower EL (EL0) using AArch64. This is the + * SVC entry point — the dispatcher in do_sync decodes ESR_EL1.EC. + * + * On exception entry from EL0, the hardware automatically switches to + * SP_EL1, so we don't need to flip SPSEL here. + */ +.align 6 +el0_sync: + trap_entry 0 + mov x0, sp + bl do_sync + trap_exit + eret + // speculation barrier after the ERET to prevent the CPU + // from speculating past the exception return. + dsb nsh + isb +.size el0_sync, .-el0_sync +.type el0_sync, @function -el0_irq_invalid: - invalid 1 -.type el0_irq_invalid, @function +/* + * IRQ from a lower EL (EL0). Mirrors el1_irq, including the optional + * task switch when do_irq returns the saved-SP slot of the next task. + */ +.align 6 +el0_irq: + trap_entry 0 + mov x0, sp + bl do_irq + cmp x0, 0 + b.eq 5f + // switch to the next task + mov x1, sp + str x1, [x0] /* store old sp */ + bl get_last_stack_pointer /* get new sp */ + mov sp, x0 +5: + trap_exit + eret + // speculation barrier after the ERET to prevent the CPU + // from speculating past the exception return. + dsb nsh + isb +.size el0_irq, .-el0_irq +.type el0_irq, @function el0_fiq_invalid: invalid 2 @@ -293,14 +332,14 @@ ventry el1_fiq // FIQ EL1h ventry el1_error // Error EL1h /* Lower EL using AArch64 */ -ventry el0_sync_invalid // Synchronous 64-bit EL0 -ventry el0_irq_invalid // IRQ 64-bit EL0 +ventry el0_sync // Synchronous 64-bit EL0 +ventry el0_irq // IRQ 64-bit EL0 ventry el0_fiq_invalid // FIQ 64-bit EL0 ventry el0_error_invalid // Error 64-bit EL0 -/* Lower EL using AArch32 */ -ventry el0_sync_invalid // Synchronous 32-bit EL0 -ventry el0_irq_invalid // IRQ 32-bit EL0 +/* Lower EL using AArch32 — not supported, route to invalid */ +ventry el0_fiq_invalid // Synchronous 32-bit EL0 +ventry el0_fiq_invalid // IRQ 32-bit EL0 ventry el0_fiq_invalid // FIQ 32-bit EL0 ventry el0_error_invalid // Error 32-bit EL0 .size vector_table, .-vector_table diff --git a/src/arch/aarch64/mm/mod.rs b/src/arch/aarch64/mm/mod.rs index e797888e27..25a3a17e61 100644 --- a/src/arch/aarch64/mm/mod.rs +++ b/src/arch/aarch64/mm/mod.rs @@ -1,7 +1,69 @@ -pub mod paging; +pub(crate) mod paging; + +#[cfg(all(feature = "common-os", feature = "fork"))] +pub use paging::{copy_current_root_page_table, copy_kernel_stack_to, prepare_mem_copy_on_write}; +#[cfg(feature = "common-os")] +pub use paging::{create_new_root_page_table, drop_user_space}; use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; +/// AArch64 sibling of the x86_64 [`allocate_thread_tls`]. +/// +/// Allocates a fresh user-accessible TLS region in the currently active +/// (shared) root page table and returns the value to install in +/// `TPIDR_EL0` for the new thread. AArch64 uses TLS Variant I: the +/// thread pointer points at the TCB which sits at the *start* of the +/// block, with the TLS image immediately following a 16-byte reserved +/// area (`tcb[0]`/`tcb[1]`). User code therefore reaches its TLS data +/// at `TPIDR_EL0 + 16 + offset`. +#[cfg(feature = "common-os")] +pub fn allocate_thread_tls(template: &crate::scheduler::task::TlsTemplate) -> u64 { + use align_address::Align; + use free_list::PageLayout; + use memory_addresses::arch::aarch64::{PhysAddr, VirtAddr}; + + use crate::arch::aarch64::mm::paging::{self, BasePageSize, PageSize, PageTableEntryFlags}; + #[cfg(feature = "fork")] + use crate::mm::frame_ref_inc; + + let tcb_size = 2 * size_of::<*mut ()>(); + let total = (tcb_size + template.size).align_up(BasePageSize::SIZE as usize); + + let virt_layout = PageLayout::from_size(total).unwrap(); + let virt_range = PageAlloc::allocate(virt_layout).unwrap(); + let virt_addr = VirtAddr::from(virt_range.start()); + + let frame_layout = PageLayout::from_size(total).unwrap(); + let frame_range = FrameAlloc::allocate(frame_layout).unwrap(); + let phys_addr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] + for i in 0..total / BasePageSize::SIZE as usize { + frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); + } + + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user().execute_disable(); + paging::map::( + virt_addr, + phys_addr, + total / BasePageSize::SIZE as usize, + flags, + ); + + unsafe { + // Zero the whole region first (covers both the TCB and the TLS BSS area). + virt_addr.as_mut_ptr::().write_bytes(0, total); + // Copy the pristine PT_TLS init image into the slot that follows the TCB. + virt_addr + .as_mut_ptr::() + .add(tcb_size) + .copy_from_nonoverlapping(template.init.as_ptr(), template.init.len()); + } + + // Variant I: TPIDR_EL0 is the start of the TCB block. + virt_addr.as_u64() +} + pub unsafe fn init() { unsafe { paging::init(); @@ -12,4 +74,12 @@ pub unsafe fn init() { unsafe { PageAlloc::init(); } + + #[cfg(feature = "common-os")] + { + use aarch64_cpu::registers::TTBR0_EL1; + crate::scheduler::BOOT_ROOT_PAGE_TABLE + .set(TTBR0_EL1.get_baddr() as usize) + .unwrap(); + } } diff --git a/src/arch/aarch64/mm/paging.rs b/src/arch/aarch64/mm/paging.rs index 9ec81fffc2..2bbde74aca 100644 --- a/src/arch/aarch64/mm/paging.rs +++ b/src/arch/aarch64/mm/paging.rs @@ -55,6 +55,9 @@ bitflags! { /// Set if this entry points to normal memory (cacheable) const NORMAL = 1 << 4; + /// Set if memory referenced by this entry shall also be accessible from EL0 (user mode). + const USER_ACCESSIBLE = 1 << 6; + /// Set if memory referenced by this entry shall be read-only. const READ_ONLY = 1 << 7; @@ -75,11 +78,15 @@ bitflags! { /// Self-reference to the Level 0 page table const SELF = 1 << 55; + + /// Software-defined marker for Copy-On-Write pages (bit 58 is reserved for software use). + #[cfg(feature = "common-os")] + const COW_MARKER = 1 << 58; } } impl PageTableEntryFlags { - #[expect(dead_code)] + #[allow(dead_code)] pub fn present(&mut self) -> &mut Self { self.insert(PageTableEntryFlags::PRESENT); self @@ -102,7 +109,7 @@ impl PageTableEntryFlags { self } - #[expect(dead_code)] + #[allow(dead_code)] pub fn read_only(&mut self) -> &mut Self { self.insert(PageTableEntryFlags::READ_ONLY); self @@ -118,6 +125,84 @@ impl PageTableEntryFlags { self.insert(PageTableEntryFlags::UNPRIVILEGED_EXECUTE_NEVER); self } + + #[cfg(feature = "common-os")] + pub fn execute_enable(&mut self) -> &mut Self { + self.remove(PageTableEntryFlags::PRIVILEGED_EXECUTE_NEVER); + self.remove(PageTableEntryFlags::UNPRIVILEGED_EXECUTE_NEVER); + self + } + + #[cfg(feature = "common-os")] + pub fn user(&mut self) -> &mut Self { + self.insert(PageTableEntryFlags::USER_ACCESSIBLE); + // An EL0-accessible page must not be executable from EL1. + self.insert(PageTableEntryFlags::PRIVILEGED_EXECUTE_NEVER); + self + } + + #[cfg(feature = "common-os")] + pub fn kernel(&mut self) -> &mut Self { + self.remove(PageTableEntryFlags::USER_ACCESSIBLE); + self + } + + /// Mark a page as Copy-On-Write: force read-only and set the COW marker. + #[cfg(feature = "common-os")] + pub fn copy_on_write(&mut self) -> &mut Self { + self.insert(PageTableEntryFlags::READ_ONLY); + self.insert(PageTableEntryFlags::COW_MARKER); + self + } +} + +/// Extension trait that mirrors the x86_64 `PageTableEntryFlagsExt` API for the +/// common-os layer. On AArch64 the `PageTableEntryFlags` type already carries +/// the same helpers as inherent methods; this trait exists only so that +/// architecture-independent callers can be written in terms of a single API. +#[cfg(feature = "common-os")] +#[allow(dead_code)] +pub trait PageTableEntryFlagsExt { + fn device(&mut self) -> &mut Self; + fn normal(&mut self) -> &mut Self; + fn read_only(&mut self) -> &mut Self; + fn writable(&mut self) -> &mut Self; + fn execute_disable(&mut self) -> &mut Self; + fn execute_enable(&mut self) -> &mut Self; + fn user(&mut self) -> &mut Self; + fn kernel(&mut self) -> &mut Self; + fn copy_on_write(&mut self) -> &mut Self; +} + +#[cfg(feature = "common-os")] +impl PageTableEntryFlagsExt for PageTableEntryFlags { + fn device(&mut self) -> &mut Self { + PageTableEntryFlags::device(self) + } + fn normal(&mut self) -> &mut Self { + PageTableEntryFlags::normal(self) + } + fn read_only(&mut self) -> &mut Self { + PageTableEntryFlags::read_only(self) + } + fn writable(&mut self) -> &mut Self { + PageTableEntryFlags::writable(self) + } + fn execute_disable(&mut self) -> &mut Self { + PageTableEntryFlags::execute_disable(self) + } + fn execute_enable(&mut self) -> &mut Self { + PageTableEntryFlags::execute_enable(self) + } + fn user(&mut self) -> &mut Self { + PageTableEntryFlags::user(self) + } + fn kernel(&mut self) -> &mut Self { + PageTableEntryFlags::kernel(self) + } + fn copy_on_write(&mut self) -> &mut Self { + PageTableEntryFlags::copy_on_write(self) + } } /// An entry in either table @@ -141,7 +226,7 @@ impl PageTableEntry { } /// Return whether this entry is a 4KiB page. - #[expect(dead_code)] + #[cfg(feature = "common-os")] fn is_table_or_4kib_page(&self) -> bool { (self.physical_address_and_flags & PageTableEntryFlags::TABLE_OR_4KIB_PAGE.bits()) != 0 } @@ -570,7 +655,7 @@ fn get_page_range(virtual_address: VirtAddr, count: usize) -> PageI Page::range(first_page, last_page) } -#[expect(dead_code)] +#[cfg(all(feature = "common-os", feature = "fork"))] pub fn get_page_table_entry(virtual_address: VirtAddr) -> Option { trace!("Looking up Page Table Entry for {virtual_address:p}"); @@ -706,6 +791,581 @@ pub fn unmap(virtual_address: VirtAddr, count: usize) { root_pagetable.map_pages(range, PhysAddr::zero(), PageTableEntryFlags::empty()); } +/// Flush the entire (non-global) TLB on this core and broadcast to others. +#[cfg(feature = "common-os")] +fn flush_tlb_all() { + dsb(ISHST); + unsafe { + asm!("tlbi vmalle1is", options(nostack)); + } + dsb(ISH); + isb(SY); +} + +/// Invalidate one TLB entry by virtual address (broadcast to all cores). +#[cfg(all(feature = "common-os", feature = "fork"))] +fn flush_tlb_one(virt: VirtAddr) { + dsb(ISHST); + unsafe { + asm!( + "tlbi vale1is, {addr}", + addr = in(reg) virt.as_u64() >> 12, + options(nostack), + ); + } + dsb(ISH); + isb(SY); +} + +/// Resolve a Copy-On-Write fault for a single user-space page. +/// +/// The page-fault handler in `interrupts::do_sync` calls this on a write +/// permission-fault (`ESR_EL1.EC = 0x24/0x25`, `ISS.WnR = 1`, +/// `ISS.DFSC = 0b001100..0b001111` — permission fault). Returns `true` if +/// the fault was a genuine COW miss and was handled (the trapped +/// instruction will be retried after `eret`); `false` if the entry is not +/// a COW page (caller falls back to the abort path). +/// +/// Behavior mirrors `arch::x86_64::mm::paging::page_fault_handler`'s COW +/// branch: drop this task's COW reference, then either flip the existing +/// frame back to writable if we were the last sharer, or allocate a new +/// frame and copy the contents. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn do_cow_fault(faulting_addr: VirtAddr) -> bool { + use aarch64_cpu::registers::TTBR0_EL1; + + let vaddr = faulting_addr.align_down(BasePageSize::SIZE); + let l0_phys = TTBR0_EL1.get_baddr() as usize; + let l0 = unsafe { &mut *ptr::with_exposed_provenance_mut::>(l0_phys) }; + + let l0_idx = ((vaddr.as_u64() >> 39) & 0x1ff) as usize; + let l1_idx = ((vaddr.as_u64() >> 30) & 0x1ff) as usize; + let l2_idx = ((vaddr.as_u64() >> 21) & 0x1ff) as usize; + let l3_idx = ((vaddr.as_u64() >> 12) & 0x1ff) as usize; + + // L0 entry 0 is the kernel low mapping, 511 is the self-reference — + // neither hosts user pages. + if l0_idx == 0 || l0_idx == 511 { + return false; + } + let l0_entry = &mut l0.entries[l0_idx]; + if !l0_entry.is_present() { + return false; + } + + let l1 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>(l0_entry.address().as_usize()) + }; + let l1_entry = &mut l1.entries[l1_idx]; + if !l1_entry.is_present() || !l1_entry.is_table_or_4kib_page() { + return false; + } + + let l2 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>(l1_entry.address().as_usize()) + }; + let l2_entry = &mut l2.entries[l2_idx]; + if !l2_entry.is_present() || !l2_entry.is_table_or_4kib_page() { + return false; + } + + let l3 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>(l2_entry.address().as_usize()) + }; + let l3_entry = &mut l3.entries[l3_idx]; + + let flags = PageTableEntryFlags::from_bits_truncate(l3_entry.physical_address_and_flags); + let is_cow_page = flags.contains(PageTableEntryFlags::PRESENT) + && flags.contains(PageTableEntryFlags::USER_ACCESSIBLE) + && flags.contains(PageTableEntryFlags::READ_ONLY) + && flags.contains(PageTableEntryFlags::COW_MARKER); + if !is_cow_page { + return false; + } + + let src_phys = l3_entry.address(); + let mut new_flags = flags; + new_flags.writable(); + new_flags.remove(PageTableEntryFlags::COW_MARKER); + + let last_ref = crate::mm::frame_ref_dec(src_phys); + if last_ref { + // We were the only remaining sharer — keep the frame, flip flags. + // frame_ref_dec removed the entry; reinstate it so future forks + // of this task continue to refcount correctly. + crate::mm::frame_ref_inc(src_phys); + l3_entry.set(src_phys, new_flags); + } else { + // Other tasks still share the source frame — give this task its + // own copy so it can write without disturbing the others. + let new_phys = crate::mm::copy_page(src_phys); + crate::mm::frame_ref_inc(new_phys); + l3_entry.set(new_phys, new_flags); + } + + flush_tlb_one(vaddr); + true +} + +/// Walk user pages in the currently active L0 table and mark all writable user pages as +/// Copy-On-Write. This must be called before duplicating the page table for a fork. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn mark_user_pages_copy_on_write() { + use aarch64_cpu::registers::TTBR0_EL1; + + // Physical memory is identity-mapped in the kernel's address space, so a + // physical frame address can be dereferenced directly as a page-table + // pointer. + let l0_phys = TTBR0_EL1.get_baddr() as usize; + let l0 = unsafe { &mut *ptr::with_exposed_provenance_mut::>(l0_phys) }; + + // User pages live exclusively in the L0 slot covering USER_START. + // All other L0 entries (kernel image, heap, per-task stacks…) are + // shared kernel mappings and must not be touched here. + for l0_idx in [USER_L0_INDEX].iter().copied() { + let l0_entry = &mut l0.entries[l0_idx]; + if !l0_entry.is_present() { + continue; + } + let l1 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>( + l0_entry.address().as_usize(), + ) + }; + for l1_idx in 0..512usize { + let l1_entry = &mut l1.entries[l1_idx]; + if !l1_entry.is_present() { + continue; + } + if !l1_entry.is_table_or_4kib_page() { + warn!("User space isn't able to use 1 GiB pages"); + continue; + } + let l2 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>( + l1_entry.address().as_usize(), + ) + }; + for l2_idx in 0..512usize { + let l2_entry = &mut l2.entries[l2_idx]; + if !l2_entry.is_present() { + continue; + } + if !l2_entry.is_table_or_4kib_page() { + warn!("User space isn't able to use 2 MiB pages"); + continue; + } + let l3 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>( + l2_entry.address().as_usize(), + ) + }; + for l3_idx in 0..512usize { + let l3_entry = &mut l3.entries[l3_idx]; + let flags = PageTableEntryFlags::from_bits_truncate( + l3_entry.physical_address_and_flags, + ); + let user_writable = flags.contains(PageTableEntryFlags::PRESENT) + && flags.contains(PageTableEntryFlags::USER_ACCESSIBLE) + && !flags.contains(PageTableEntryFlags::READ_ONLY) + && !flags.contains(PageTableEntryFlags::COW_MARKER); + if user_writable { + let addr = l3_entry.address(); + let mut new_flags = flags; + new_flags.copy_on_write(); + l3_entry.set(addr, new_flags); + } + } + } + } + } + + flush_tlb_all(); +} + +/// Recursively free the user-space portion of a given L0 page table. +/// +/// Entry 0 (kernel low mapping) and entry 511 (self-reference) are preserved. +/// +/// **Important**: Hermit shares TTBR0_EL1 between user space and the kernel's +/// own heap and task stacks (PageAlloc hands out addresses in the high half of +/// the low 48-bit range, e.g. starting at `0x800000000000`). The same L0 +/// entries (1..510) can therefore back **mixed** kernel + user mappings — and +/// even L1/L2/L3 tables further down might contain a mix. +/// +/// We must only free a sub-table once it is **completely empty** after the +/// user-page sweep; otherwise we would unmap the kernel's own heap or the +/// kernel stack the current task is running on, which makes the very next +/// stack access fault and the CPU spins in recursive sync exceptions. +#[cfg(feature = "common-os")] +fn clear_l0(l0_phys: usize) { + fn free_frame(phys: usize) { + let range = free_list::PageRange::new(phys, phys + BasePageSize::SIZE as usize).unwrap(); + unsafe { FrameAlloc::deallocate(range) }; + } + + fn pt_is_empty(pt: &PageTable) -> bool + where + L: PageTableLevel, + { + pt.entries.iter().all(|e| !e.is_present()) + } + + let l0 = unsafe { &mut *ptr::with_exposed_provenance_mut::>(l0_phys) }; + + // Only walk the user-space L0 slot. All other entries point at + // kernel L1 tables that are *shared* across every task's PT — clearing + // them here would unmap the kernel's own heap and stack. + for l0_idx in [USER_L0_INDEX].iter().copied() { + let l0_entry = &mut l0.entries[l0_idx]; + if !l0_entry.is_present() { + continue; + } + let l1_phys = l0_entry.address().as_usize(); + let l1 = unsafe { &mut *ptr::with_exposed_provenance_mut::>(l1_phys) }; + + for l1_idx in 0..512usize { + let l1_entry = &mut l1.entries[l1_idx]; + if !l1_entry.is_present() { + continue; + } + if !l1_entry.is_table_or_4kib_page() { + warn!("User space isn't able to use 1 GiB pages"); + continue; + } + let l2_phys = l1_entry.address().as_usize(); + let l2 = + unsafe { &mut *ptr::with_exposed_provenance_mut::>(l2_phys) }; + + for l2_idx in 0..512usize { + let l2_entry = &mut l2.entries[l2_idx]; + if !l2_entry.is_present() { + continue; + } + if !l2_entry.is_table_or_4kib_page() { + warn!("User space isn't able to use 2 MiB pages"); + continue; + } + let l3_phys = l2_entry.address().as_usize(); + let l3 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>(l3_phys) + }; + + for l3_idx in 0..512usize { + let l3_entry = &mut l3.entries[l3_idx]; + let flags = PageTableEntryFlags::from_bits_truncate( + l3_entry.physical_address_and_flags, + ); + if flags.contains(PageTableEntryFlags::PRESENT) + && flags.contains(PageTableEntryFlags::USER_ACCESSIBLE) + { + let phys_addr = l3_entry.address(); + + #[cfg(feature = "fork")] + if crate::mm::frame_ref_dec(phys_addr) { + free_frame(phys_addr.as_usize()); + } + #[cfg(not(feature = "fork"))] + { + free_frame(phys_addr.as_usize()); + } + *l3_entry = PageTableEntry::default(); + } + } + + if pt_is_empty(l3) { + free_frame(l3_phys); + *l2_entry = PageTableEntry::default(); + } + } + + if pt_is_empty(l2) { + free_frame(l2_phys); + *l1_entry = PageTableEntry::default(); + } + } + + if pt_is_empty(l1) { + free_frame(l1_phys); + *l0_entry = PageTableEntry::default(); + } + } +} + +/// Drop an inactive user address space: free all user pages and all user-space +/// page-table pages, then free the L0 table itself. +#[cfg(feature = "common-os")] +pub fn drop_user_space(l0_phys: usize) { + debug!("Drop the user space at L0 {l0_phys:#x}"); + + clear_l0(l0_phys); + + // The L0 table is not loaded on any core, so no TLB flush is necessary. + let range = free_list::PageRange::new(l0_phys, l0_phys + BasePageSize::SIZE as usize).unwrap(); + unsafe { FrameAlloc::deallocate(range) }; +} + +/// Clear the user-space portion of the currently active address space. +#[cfg(feature = "common-os")] +pub fn clear_user_space() { + use aarch64_cpu::registers::TTBR0_EL1; + + use crate::core_scheduler; + use crate::fd::STDERR_FILENO; + + core_scheduler() + .get_current_task() + .borrow() + .vmas + .write() + .clear(); + core_scheduler() + .get_current_task_object_map() + .write() + .retain(|&k, _| k <= STDERR_FILENO); + + let l0_phys = TTBR0_EL1.get_baddr() as usize; + debug!("Clear the user space at L0 {l0_phys:#x}"); + + clear_l0(l0_phys); + + flush_tlb_all(); +} + +/// Allocate a fresh L0 (root) page table for a new address space. +/// +/// The new L0 inherits the kernel low mapping from the currently active L0 +/// (entry 0) and installs a self-reference at entry 511. User-space entries +/// (1..511) are left empty. +/// Index of the L0 entry that backs the user-space load area +/// (`USER_START = 0x0100_0000_0000`, bits 47..39 = 2). +#[cfg(feature = "common-os")] +const USER_L0_INDEX: usize = (crate::arch::aarch64::kernel::USER_START.as_usize() >> 39) & 0x1ff; + +#[cfg(feature = "common-os")] +pub fn create_new_root_page_table() -> usize { + use aarch64_cpu::registers::TTBR0_EL1; + + let layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); + let frame_range = FrameAlloc::allocate(layout).expect("Failed to allocate L0 table"); + let new_l0_phys = frame_range.start(); + + let cur_l0_phys = TTBR0_EL1.get_baddr() as usize; + let cur_l0 = unsafe { &*ptr::with_exposed_provenance::>(cur_l0_phys) }; + + let new_l0 = + unsafe { &mut *ptr::with_exposed_provenance_mut::>(new_l0_phys) }; + + // Inherit every L0 entry from the current (kernel) page table EXCEPT + // the user-space slot — that one is left empty so the new task starts + // with a clean user address space. + // + // Sharing the kernel L0 entries means we share the L1/L2/L3 tables + // underneath. That is intentional: kernel mappings (kernel image, + // heap, per-CPU stacks, …) are global and must + // stay in sync across every common-os task. Without sharing, the + // kernel would lose access to its own heap as soon as the scheduler + // switches TTBR0_EL1 to this task's PT. + for (i, entry) in new_l0.entries.iter_mut().enumerate() { + *entry = if i == USER_L0_INDEX || i == 511 { + PageTableEntry::default() + } else { + cur_l0.entries[i] + }; + } + + let self_flags = PageTableEntryFlags::PRESENT + | PageTableEntryFlags::TABLE_OR_4KIB_PAGE + | PageTableEntryFlags::NORMAL + | PageTableEntryFlags::INNER_SHAREABLE + | PageTableEntryFlags::ACCESSED + | PageTableEntryFlags::SELF; + new_l0.entries[511].set(PhysAddr::new(new_l0_phys as u64), self_flags); + + new_l0_phys +} + +/// Returns the physical address of the current task's root page table. +#[allow(dead_code)] +#[cfg(feature = "common-os")] +pub fn get_current_root_page_table() -> usize { + use crate::arch::kernel::core_local::core_scheduler; + core_scheduler() + .get_current_task() + .borrow() + .root_page_table + .as_usize() +} + +/// Deep-copy the current task's L0 into a new page table, sharing data pages (COW). +/// Returns the physical address of the new L0. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn copy_current_root_page_table() -> usize { + use aarch64_cpu::registers::TTBR0_EL1; + + let layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); + + let new_l0_frame = FrameAlloc::allocate(layout).unwrap(); + let new_l0_phys = new_l0_frame.start(); + let new_l0 = + unsafe { &mut *ptr::with_exposed_provenance_mut::>(new_l0_phys) }; + + let cur_l0_phys = TTBR0_EL1.get_baddr() as usize; + let cur_l0 = unsafe { &*ptr::with_exposed_provenance::>(cur_l0_phys) }; + + // Inherit kernel L0 entries verbatim (sharing the L1/L2/L3 tables + // below). Only the user-space slot needs a deep copy with COW so + // parent and child can diverge after fork(). + for (i, entry) in new_l0.entries.iter_mut().enumerate() { + *entry = if i == USER_L0_INDEX || i == 511 { + PageTableEntry::default() + } else { + cur_l0.entries[i] + }; + } + + for l0_idx in [USER_L0_INDEX].iter().copied() { + let cur_l0_entry = &cur_l0.entries[l0_idx]; + if !cur_l0_entry.is_present() { + continue; + } + + let new_l1_frame = FrameAlloc::allocate(layout).unwrap(); + let new_l1_phys = new_l1_frame.start(); + let l0_flags = + PageTableEntryFlags::from_bits_truncate(cur_l0_entry.physical_address_and_flags); + new_l0.entries[l0_idx].set(PhysAddr::new(new_l1_phys as u64), l0_flags); + + let cur_l1 = unsafe { + &*ptr::with_exposed_provenance::>(cur_l0_entry.address().as_usize()) + }; + let new_l1 = + unsafe { &mut *ptr::with_exposed_provenance_mut::>(new_l1_phys) }; + + for l1_idx in 0..512usize { + let cur_l1_entry = &cur_l1.entries[l1_idx]; + if !cur_l1_entry.is_present() || !cur_l1_entry.is_table_or_4kib_page() { + new_l1.entries[l1_idx] = PageTableEntry::default(); + continue; + } + + let new_l2_frame = FrameAlloc::allocate(layout).unwrap(); + let new_l2_phys = new_l2_frame.start(); + let l1_flags = + PageTableEntryFlags::from_bits_truncate(cur_l1_entry.physical_address_and_flags); + new_l1.entries[l1_idx].set(PhysAddr::new(new_l2_phys as u64), l1_flags); + + let cur_l2 = unsafe { + &*ptr::with_exposed_provenance::>( + cur_l1_entry.address().as_usize(), + ) + }; + let new_l2 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>(new_l2_phys) + }; + + for l2_idx in 0..512usize { + let cur_l2_entry = &cur_l2.entries[l2_idx]; + if !cur_l2_entry.is_present() || !cur_l2_entry.is_table_or_4kib_page() { + new_l2.entries[l2_idx] = PageTableEntry::default(); + continue; + } + + let new_l3_frame = FrameAlloc::allocate(layout).unwrap(); + let new_l3_phys = new_l3_frame.start(); + let l2_flags = PageTableEntryFlags::from_bits_truncate( + cur_l2_entry.physical_address_and_flags, + ); + new_l2.entries[l2_idx].set(PhysAddr::new(new_l3_phys as u64), l2_flags); + + let cur_l3 = unsafe { + &*ptr::with_exposed_provenance::>( + cur_l2_entry.address().as_usize(), + ) + }; + let new_l3 = unsafe { + &mut *ptr::with_exposed_provenance_mut::>(new_l3_phys) + }; + + // Copy L3 entries verbatim — data pages are shared + // (already COW-marked by mark_user_pages_copy_on_write). + new_l3.entries = cur_l3.entries; + + // The child holds an additional reference to every + // user-space frame in this page table. + for entry in new_l3.entries.iter() { + let flags = + PageTableEntryFlags::from_bits_truncate(entry.physical_address_and_flags); + if flags.contains( + PageTableEntryFlags::PRESENT | PageTableEntryFlags::USER_ACCESSIBLE, + ) { + crate::mm::frame_ref_inc(entry.address()); + } + } + } + } + } + + // Entry 0 (kernel low mapping) was inherited above as part of the + // shared kernel-entry copy loop, so no extra copy is needed here. + + // Entry 511: self-reference for the new L0 + let self_flags = PageTableEntryFlags::PRESENT + | PageTableEntryFlags::TABLE_OR_4KIB_PAGE + | PageTableEntryFlags::NORMAL + | PageTableEntryFlags::INNER_SHAREABLE + | PageTableEntryFlags::ACCESSED + | PageTableEntryFlags::SELF; + new_l0.entries[511].set(PhysAddr::new(new_l0_phys as u64), self_flags); + + new_l0_phys +} + +/// Mark all writable user pages in the current page table as Copy-On-Write. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn prepare_mem_copy_on_write() { + mark_user_pages_copy_on_write(); +} + +/// Copy the contents of the current task's kernel stack to a new virtual +/// base address. Used by `fork`: the child's `TaskStacks::new` has already +/// allocated and mapped fresh physical frames at `stack_address`, so we +/// simply `memcpy` the parent's stack pages into the child's mapping. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn copy_kernel_stack_to(stack_address: usize) { + use crate::arch::kernel::core_local::core_scheduler; + + let virt_addr = core_scheduler() + .get_current_task() + .borrow() + .stacks + .get_stack_virt_addr(); + let total_size = core_scheduler() + .get_current_task() + .borrow() + .stacks + .get_total_stack_size(); + + let addr_diff = stack_address as u64 - virt_addr.as_u64(); + + // The virtual layout has 3 guard pages (unmapped) interspersed; iterate + // the full range including those so no mapped pages are missed. + let full_virt_size = total_size as u64 + 3 * BasePageSize::SIZE; + for i in (virt_addr.as_u64()..(virt_addr.as_u64() + full_virt_size)) + .step_by(BasePageSize::SIZE as usize) + { + if get_page_table_entry::(VirtAddr::new(i)).is_some() { + let src = ptr::with_exposed_provenance::(i as usize); + let dst = ptr::with_exposed_provenance_mut::((i + addr_diff) as usize); + unsafe { + dst.copy_from_nonoverlapping(src, BasePageSize::SIZE as usize); + } + } + } + + flush_tlb_all(); +} + pub unsafe fn init() { // determine physical address size let id_aa64mmfr0_el1 = ID_AA64MMFR0_EL1.extract(); diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 6596a53ac4..5865b987a5 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -5,6 +5,8 @@ // https://github.com/rust-lang/rust/issues/158400 #[cfg(target_arch = "aarch64")] mod aarch64; +#[cfg(all(target_arch = "aarch64", feature = "common-os"))] +pub use self::aarch64::mm::paging::{BasePageSize, PageSize, clear_user_space}; #[cfg(target_arch = "aarch64")] pub(crate) use self::aarch64::*; @@ -15,5 +17,7 @@ pub(crate) use self::riscv64::*; #[cfg(target_arch = "x86_64")] mod x86_64; +#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +pub use self::x86_64::mm::{BasePageSize, PageSize, clear_user_space}; #[cfg(target_arch = "x86_64")] pub(crate) use self::x86_64::*; diff --git a/src/arch/x86_64/kernel/core_local.rs b/src/arch/x86_64/kernel/core_local.rs index 000b07ca26..15473c8fc1 100644 --- a/src/arch/x86_64/kernel/core_local.rs +++ b/src/arch/x86_64/kernel/core_local.rs @@ -28,6 +28,9 @@ pub(crate) struct CoreLocal { pub tss: Cell<*mut TaskStateSegment>, /// Start address of the kernel stack pub kernel_stack: Cell<*mut u8>, + /// Current address of the user stack during a system call + #[cfg(feature = "common-os")] + pub user_stack: Cell<*mut u8>, /// Interface to the interrupt counters irq_statistics: &'static IrqStatistics, /// The core-local async executor. @@ -56,6 +59,8 @@ impl CoreLocal { scheduler: Cell::new(ptr::null_mut()), tss: Cell::new(ptr::null_mut()), kernel_stack: Cell::new(ptr::null_mut()), + #[cfg(feature = "common-os")] + user_stack: Cell::new(ptr::null_mut()), irq_statistics, ex: StaticLocalExecutor::new(), #[cfg(feature = "smp")] diff --git a/src/arch/x86_64/kernel/gdt.rs b/src/arch/x86_64/kernel/gdt.rs index c924a72aa0..603251dd98 100644 --- a/src/arch/x86_64/kernel/gdt.rs +++ b/src/arch/x86_64/kernel/gdt.rs @@ -86,6 +86,7 @@ pub extern "C" fn set_current_kernel_stack() { let (current_frame, val) = Cr3::read_raw(); if current_frame != new_frame { + debug!("Change Cr3 from {current_frame:?} to {new_frame:?}"); unsafe { Cr3::write_raw(new_frame, val); } diff --git a/src/arch/x86_64/kernel/interrupts.rs b/src/arch/x86_64/kernel/interrupts.rs index 3ed4572acb..c967707f11 100644 --- a/src/arch/x86_64/kernel/interrupts.rs +++ b/src/arch/x86_64/kernel/interrupts.rs @@ -155,6 +155,8 @@ pub(crate) fn install() { } IRQ_NAMES.lock().insert(7, "FPU"); + #[cfg(feature = "common-os")] + IRQ_NAMES.lock().insert(14, "Page Fault"); } pub(crate) fn install_handlers(handlers: InterruptHandlerMap) { diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index e0696d93a6..a762c58085 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -6,9 +6,13 @@ use core::slice; use core::sync::atomic::{AtomicPtr, AtomicU32, Ordering}; use hermit_entry::boot_info::RawBootInfo; +#[cfg(feature = "common-os")] +use memory_addresses::{PhysAddr, VirtAddr}; use x86_64::registers::control::{Cr0, Cr4}; pub(crate) use self::apic::{set_oneshot_timer, wakeup_core}; +#[cfg(all(feature = "common-os", feature = "fork"))] +pub use self::switch::prepare_fork_child_stack; use crate::arch::x86_64::kernel::core_local::*; use crate::env; @@ -189,38 +193,87 @@ unsafe extern "C" fn pre_init(boot_info: Option<&'static RawBootInfo>, cpu_id: u } #[cfg(feature = "common-os")] -const LOADER_START: usize = 0x0100_0000_0000; +pub(crate) const USER_START: VirtAddr = VirtAddr::new(0x0100_0000_0000); +#[cfg(feature = "common-os")] +const USER_STACK: VirtAddr = VirtAddr::new(0x0180_0000_0000 - USER_STACK_SIZE as u64); #[cfg(feature = "common-os")] -const LOADER_STACK_SIZE: usize = 0x8000; +const USER_STACK_SIZE: usize = 0x8000; +#[allow(clippy::result_unit_err)] #[cfg(feature = "common-os")] -pub fn load_application(code_size: u64, tls_size: u64, func: F) -> T +pub fn load_application(code_size: u64, tls_size: u64, func: F) -> Result<(), ()> where - F: FnOnce(&'static mut [u8], Option<&'static mut [u8]>) -> T, + F: FnOnce( + &'static mut [u8], + Option<&'static mut [u8]>, + ) -> Result>, ()>, { + use alloc::sync::Arc; + + use ahash::RandomState; use align_address::Align; use free_list::PageLayout; - use memory_addresses::{PhysAddr, VirtAddr}; + use hashbrown::HashMap; + use hermit_sync::RwSpinLock; use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; use crate::arch::x86_64::mm::paging::{self, PageTableEntryFlags, PageTableEntryFlagsExt}; + use crate::fd::{Fd, RawFd, stdio}; + #[cfg(feature = "fork")] + use crate::mm::frame_ref_inc; + use crate::mm::vma::*; use crate::mm::{FrameAlloc, PageRangeAllocator}; - let code_size = (code_size as usize + LOADER_STACK_SIZE).align_up(BasePageSize::SIZE as usize); + // each process has to provide its own object_map + // => create a new one + let mut object_map = HashMap::>, RandomState>::with_hasher( + RandomState::with_seeds(0, 0, 0, 0), + ); + stdio::setup(&mut object_map); + core_scheduler().set_current_task_object_map(Arc::new(RwSpinLock::new(object_map))); + + let code_size = (code_size as usize).align_up(BasePageSize::SIZE as usize); let layout = PageLayout::from_size_align(code_size, BasePageSize::SIZE as usize).unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let physaddr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] + for i in 0..code_size / BasePageSize::SIZE as usize { + frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize); + } let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().user().execute_enable(); paging::map::( - VirtAddr::from(LOADER_START), + USER_START, physaddr, code_size / BasePageSize::SIZE as usize, flags, ); + // VirtAddr's defined in vma.rs is `x86_64::VirtAddr` on x86_64 but + // this file's local `use` brings in `memory_addresses::VirtAddr`. + // Convert at the boundary so the BTreeMap-keyed insert type-checks. + { + let start: x86_64::VirtAddr = USER_START.into(); + let end: x86_64::VirtAddr = (USER_START + code_size).align_up(BasePageSize::SIZE).into(); + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( + start, + VirtualMemoryArea::new( + start, + end, + VirtualMemoryAreaProt::READ + | VirtualMemoryAreaProt::WRITE + | VirtualMemoryAreaProt::EXECUTE, + MemoryType::CODE, + ), + ); + } - let loader_start_ptr = ptr::with_exposed_provenance_mut(LOADER_START); + let loader_start_ptr = ptr::with_exposed_provenance_mut(USER_START.as_usize()); let code_slice = unsafe { slice::from_raw_parts_mut(loader_start_ptr, code_size) }; if tls_size > 0 { @@ -235,16 +288,41 @@ where let layout = PageLayout::from_size(tls_memsz).unwrap(); let frame_range = FrameAlloc::allocate(layout).unwrap(); let physaddr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] + for i in 0..tls_memsz / BasePageSize::SIZE as usize { + frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize); + } let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().user().execute_disable(); - let tls_virt = VirtAddr::from(LOADER_START + code_size + BasePageSize::SIZE as usize); + let tls_virt = + VirtAddr::from(USER_START.as_usize() + code_size + BasePageSize::SIZE as usize); paging::map::( tls_virt, physaddr, tls_memsz / BasePageSize::SIZE as usize, flags, ); + + { + let start: x86_64::VirtAddr = tls_virt.into(); + let end: x86_64::VirtAddr = (tls_virt + tls_memsz).align_up(BasePageSize::SIZE).into(); + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( + start, + VirtualMemoryArea::new( + start, + end, + VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE, + MemoryType::TLS, + ), + ); + } + let block = unsafe { slice::from_raw_parts_mut(tls_virt.as_mut_ptr(), tls_offset + tcb_size) }; for elem in block.iter_mut() { @@ -258,47 +336,123 @@ where } processor::writefs(thread_ptr.expose_provenance()); - func(code_slice, Some(block)) + // Run the ELF loader, which copies the binary's `PT_TLS` initial + // image into `block` and returns the pristine PT_TLS image + // directly from the ELF buffer. + let tls_init = func(code_slice, Some(block))?; + + if let Some(init) = tls_init { + let template = Arc::new(crate::scheduler::task::TlsTemplate { + size: tls_offset, + init, + }); + core_scheduler() + .get_current_task() + .borrow_mut() + .tls_template = Some(template); + } + + Ok(()) } else { - func(code_slice, None) + func(code_slice, None)?; + Ok(()) } } #[cfg(feature = "common-os")] -pub unsafe fn jump_to_user_land(entry_point: usize, code_size: usize, arg: &[&str]) -> ! { - use alloc::ffi::CString; - +pub unsafe fn jump_to_user_land( + entry_point: usize, + args: alloc::vec::Vec, + envs: alloc::vec::Vec, +) -> ! { use align_address::Align; - use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; + use free_list::PageLayout; + use x86_64::structures::paging::{ + PageSize, PageTableFlags as PageTableEntryFlags, Size4KiB as BasePageSize, + }; + use crate::arch::mm::paging; use crate::arch::x86_64::kernel::scheduler::TaskStacks; + use crate::arch::x86_64::mm::paging::PageTableEntryFlagsExt; + #[cfg(feature = "fork")] + use crate::mm::frame_ref_inc; + use crate::mm::vma::*; + use crate::mm::{FrameAlloc, PageRangeAllocator}; - info!("Create new file descriptor table"); + debug!("Create new file descriptor table"); core_scheduler().recreate_objmap().unwrap(); - let entry_point: usize = LOADER_START | entry_point; - let stack_pointer: usize = LOADER_START - + (code_size + LOADER_STACK_SIZE).align_up(BasePageSize::SIZE.try_into().unwrap()) - - 8; + let entry_point: usize = USER_START.as_usize() | entry_point; + let stack_pointer: usize = USER_STACK.as_usize() + USER_STACK_SIZE - 8; + + let layout = PageLayout::from_size(USER_STACK_SIZE).unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let phys_addr = PhysAddr::from(frame_range.start()); + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user(); + paging::map::( + USER_STACK, + phys_addr, + USER_STACK_SIZE / BasePageSize::SIZE as usize, + flags, + ); + { + let start: x86_64::VirtAddr = USER_STACK.into(); + let end: x86_64::VirtAddr = (USER_STACK + USER_STACK_SIZE).into(); + core_scheduler() + .get_current_task() + .borrow_mut() + .vmas + .write() + .insert( + start, + VirtualMemoryArea::new( + start, + end, + VirtualMemoryAreaProt::READ | VirtualMemoryAreaProt::WRITE, + MemoryType::STACK, + ), + ); + } + #[cfg(feature = "fork")] + for i in 0..USER_STACK_SIZE / BasePageSize::SIZE as usize { + frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); + } - let stack_pointer = stack_pointer - 128 /* red zone */ - arg.len() * size_of::<*mut u8>(); + // Place the argv and envp pointer arrays on the user stack. Both + // arrays follow the C convention and are terminated by a null pointer. + let ptr_count = args.len() + 1 + envs.len() + 1; + let stack_pointer = stack_pointer - 128 /* red zone */ - ptr_count * size_of::<*mut u8>(); let stack_ptr = ptr::with_exposed_provenance_mut::<*mut u8>(stack_pointer); - let argv = unsafe { slice::from_raw_parts_mut(stack_ptr, arg.len()) }; - let len = arg.iter().fold(0, |acc, x| acc + x.len() + 1); + let arrays = unsafe { slice::from_raw_parts_mut(stack_ptr, ptr_count) }; + let (argv, envp) = arrays.split_at_mut(args.len() + 1); + let len = args + .iter() + .chain(envs.iter()) + .fold(0, |acc, x| acc + x.as_bytes_with_nul().len()); // align stack pointer to fulfill the requirements of the x86_64 ABI let stack_pointer = (stack_pointer - len).align_down(16) - size_of::(); let mut pos: usize = 0; - for (i, s) in arg.iter().enumerate() { - let s = CString::new(*s).unwrap(); + for (dst, s) in argv + .iter_mut() + .zip(args.iter()) + .chain(envp.iter_mut().zip(envs.iter())) + { let bytes = s.as_bytes_with_nul(); - argv[i] = ptr::with_exposed_provenance_mut::(stack_pointer + pos); + *dst = ptr::with_exposed_provenance_mut::(stack_pointer + pos); pos += bytes.len(); unsafe { - argv[i].copy_from_nonoverlapping(bytes.as_ptr(), bytes.len()); + dst.copy_from_nonoverlapping(bytes.as_ptr(), bytes.len()); } } + argv[args.len()] = ptr::null_mut(); + envp[envs.len()] = ptr::null_mut(); + + let argc = args.len(); + drop(args); + drop(envs); debug!("Jump to user space at 0x{entry_point:x}, stack pointer 0x{stack_pointer:x}"); @@ -311,8 +465,14 @@ pub unsafe fn jump_to_user_land(entry_point: usize, code_size: usize, arg: &[&st "push {3}", "push {4}", "push {5}", - "mov rdi, {6}", - "mov rsi, {7}", + // Clear registers so that state cannot leak. `rdi`, `rsi`, + // and `rdx` carry argc, argv, and envp and are bound as + // fixed register operands below, so the register allocator + // cannot hand them out for the other `in(reg)` operands. + "xor rax, rax", "xor rbx, rbx", "xor rcx, rcx", + "xor r8, r8", "xor r9, r9", "xor r10, r10", + "xor r11, r11", "xor r12, r12", "xor r13, r13", + "xor r14, r14", "xor r15, r15", "iretq", const u64::MAX - (TaskStacks::MARKER_SIZE as u64 - 1), const 0x23usize, @@ -320,8 +480,9 @@ pub unsafe fn jump_to_user_land(entry_point: usize, code_size: usize, arg: &[&st const 0x1202u64, const 0x2busize, in(reg) entry_point, - in(reg) argv.len(), - in(reg) argv.as_ptr(), + in("rdi") argc, + in("rsi") argv.as_ptr(), + in("rdx") envp.as_ptr(), options(nostack, noreturn) ); } diff --git a/src/arch/x86_64/kernel/processor.rs b/src/arch/x86_64/kernel/processor.rs index 460c6713d8..cfb8c268f9 100644 --- a/src/arch/x86_64/kernel/processor.rs +++ b/src/arch/x86_64/kernel/processor.rs @@ -909,7 +909,7 @@ pub fn configure() { let syscall_handler_addr = syscall::syscall_handler as *const (); let syscall_handler_addr = VirtAddr::from_ptr(syscall_handler_addr); LStar::write(syscall_handler_addr); - SFMask::write(RFlags::INTERRUPT_FLAG); // clear IF flag during system call + SFMask::write(RFlags::INTERRUPT_FLAG | RFlags::TRAP_FLAG); // clear IF and TF flag during system call } // Initialize the FS register, which is later used for Thread-Local Storage. diff --git a/src/arch/x86_64/kernel/scheduler.rs b/src/arch/x86_64/kernel/scheduler.rs index 1a31956dd0..7c716f5d93 100644 --- a/src/arch/x86_64/kernel/scheduler.rs +++ b/src/arch/x86_64/kernel/scheduler.rs @@ -12,12 +12,12 @@ use crate::arch::x86_64::kernel::{apic, interrupts}; use crate::arch::x86_64::mm::paging::{ BasePageSize, PageSize, PageTableEntryFlags, PageTableEntryFlagsExt, }; +use crate::arch::x86_64::swapgs; use crate::config::*; use crate::env; use crate::mm::{FrameAlloc, PageAlloc, PageRangeAllocator}; use crate::scheduler::task::{Task, TaskFrame}; use crate::scheduler::{PerCoreSchedulerExt, timer_interrupts}; - #[repr(C, packed)] struct State { #[cfg(feature = "common-os")] @@ -129,12 +129,20 @@ impl TaskStacks { flags, ); - // map user stack into the address space + // map user stack into the address space (must be user-accessible under common-os) + #[cfg(feature = "common-os")] + let user_flags = { + let mut f = PageTableEntryFlags::empty(); + f.normal().writable().user().execute_disable(); + f + }; + #[cfg(not(feature = "common-os"))] + let user_flags = flags; crate::arch::mm::paging::map::( virt_addr + IST_SIZE + DEFAULT_STACK_SIZE + 3 * BasePageSize::SIZE, phys_addr + IST_SIZE + DEFAULT_STACK_SIZE, user_stack_size / BasePageSize::SIZE as usize, - flags, + user_flags, ); // clear user stack @@ -173,6 +181,24 @@ impl TaskStacks { } } + /// Returns the start address of the stack region (virt_addr of CommonStack) + #[cfg(all(feature = "common-os", feature = "fork"))] + pub fn get_stack_virt_addr(&self) -> VirtAddr { + match self { + TaskStacks::Boot(stacks) => stacks.stack, + TaskStacks::Common(stacks) => stacks.virt_addr, + } + } + + /// Returns total size of all stacks combined + #[cfg(all(feature = "common-os", feature = "fork"))] + pub fn get_total_stack_size(&self) -> usize { + match self { + TaskStacks::Boot(_) => KERNEL_STACK_SIZE, + TaskStacks::Common(stacks) => stacks.total_size, + } + } + pub fn get_user_stack(&self) -> VirtAddr { match self { TaskStacks::Boot(_) => VirtAddr::zero(), @@ -269,6 +295,76 @@ extern "C" fn task_entry(func: extern "C" fn(usize), arg: usize) -> ! { core_scheduler().exit(0) } +/// Entry trampoline for a new user-space thread (common-os). +/// +/// Called in kernel mode the first time the task is scheduled. It crafts +/// an `iretq` frame that transfers control to `func(arg)` in ring 3 on +/// the new user stack. Matches the descriptor layout used by +/// `jump_to_user_land`: user CS=0x2b, user SS=0x23, RFLAGS=0x1202. +#[cfg(feature = "common-os")] +#[unsafe(naked)] +extern "C" fn task_start_user(_f: extern "C" fn(usize), _arg: usize, _user_stack: u64) -> ! { + // rdi = func (user entry) + // rsi = arg (forwarded as first argument to `func`) + // rdx = user stack pointer + naked_asm!( + "swapgs", + "push 0x23", // user SS + "push rdx", // user RSP + "push 0x1202", // RFLAGS (IF=1) + "push 0x2b", // user CS + "push rdi", // RIP = user func + "mov rdi, rsi", // first argument for user func + "iretq", + ) +} + +#[cfg(feature = "common-os")] +impl Task { + /// Build the initial kernel-stack context for a new user-space thread. + /// + /// When this task is first scheduled, `switch_to_task` restores the + /// saved registers and `ret`s into `task_start_user`, which then + /// `iretq`s to `func(arg)` in ring 3 on the freshly allocated user + /// stack. + pub(crate) fn create_user_stack_frame( + &mut self, + func: unsafe extern "C" fn(usize), + arg: usize, + tls_fs_base: u64, + ) { + unsafe { + // Debug marker at the very top of the kernel stack. + let mut stack = self.stacks.get_kernel_stack() + self.stacks.get_kernel_stack_size() + - TaskStacks::MARKER_SIZE; + *stack.as_mut_ptr::() = 0xdead_beefu64; + + stack -= size_of::(); + let state = stack.as_mut_ptr::(); + state.cast::().write_bytes(0, size_of::()); + + (*state).rip = task_start_user; + (*state).rdi = func as usize as u64; + (*state).rsi = arg as u64; + (*state).rflags = 0x1202u64; + // FS.Base for the new user-space thread (its private TLS area). + (*state).fs = tls_fs_base; + + // `stack` / `get_user_stack()` produce `memory_addresses::VirtAddr`, + // while `Task::{last,user}_stack_pointer` is `x86_64::VirtAddr` — + // convert at the assignment boundary. + self.last_stack_pointer = stack.into(); + self.user_stack_pointer = (self.stacks.get_user_stack() + + self.stacks.get_user_stack_size() + - TaskStacks::MARKER_SIZE) + .into(); + + // rdx is used by task_start_user as the new user-mode RSP + (*state).rdx = self.user_stack_pointer.as_u64() - size_of::() as u64; + } + } +} + impl TaskFrame for Task { fn create_stack_frame(&mut self, func: unsafe extern "C" fn(usize), arg: usize) { // Check if TLS is allocated already and if the task uses thread-local storage. @@ -301,10 +397,11 @@ impl TaskFrame for Task { (*state).rflags = 0x1202u64; // Set the task's stack pointer entry to the stack we have just crafted. - self.last_stack_pointer = stack; - self.user_stack_pointer = self.stacks.get_user_stack() + self.last_stack_pointer = stack.into(); + self.user_stack_pointer = (self.stacks.get_user_stack() + self.stacks.get_user_stack_size() - - TaskStacks::MARKER_SIZE; + - TaskStacks::MARKER_SIZE) + .into(); // rdx is required to initialize the stack (*state).rdx = self.user_stack_pointer.as_u64() - size_of::() as u64; @@ -312,7 +409,8 @@ impl TaskFrame for Task { } } -extern "x86-interrupt" fn timer_handler(_stack_frame: interrupts::ExceptionStackFrame) { +extern "x86-interrupt" fn timer_handler(stack_frame: interrupts::ExceptionStackFrame) { + swapgs(&stack_frame); increment_irq_counter(apic::TIMER_INTERRUPT_NUMBER); debug!("Handle timer interrupt"); @@ -321,6 +419,7 @@ extern "x86-interrupt" fn timer_handler(_stack_frame: interrupts::ExceptionStack core_scheduler().handle_waiting_tasks(); apic::eoi(); core_scheduler().reschedule(); + swapgs(&stack_frame); } pub fn install_timer_handler() { diff --git a/src/arch/x86_64/kernel/switch.rs b/src/arch/x86_64/kernel/switch.rs index dc1675f03c..0f57d20327 100644 --- a/src/arch/x86_64/kernel/switch.rs +++ b/src/arch/x86_64/kernel/switch.rs @@ -140,6 +140,17 @@ macro_rules! save_context { } macro_rules! restore_context { + () => { + concat!( + restore_context_without_return!(), + r#" + ret + "# + ) + }; +} + +macro_rules! restore_context_without_return { () => { concat!( pop_gs!(), @@ -161,7 +172,6 @@ macro_rules! restore_context { pop rcx pop rax popfq - ret "# ) }; @@ -210,3 +220,194 @@ pub(crate) unsafe extern "C" fn switch_to_fpu_owner(_old_stack: *mut usize, _new set_current_kernel_stack = sym set_current_kernel_stack, ); } + +// ── Fork support ───────────────────────────────────────────────────────────── + +/// Entry point for the child task after a fork. +/// +/// `switch_to_task` restores the saved context and `ret`s here. Instead +/// of unwinding the whole kernel call-chain (which is fragile), we jump +/// directly to user space, mirroring `syscall_handler`'s return path: +/// • The 14 user-side registers (`rcx`, `rdx`, `rbx`, `rbp`, `rsi`, +/// `rdi`, `r8`..`r15`) sit at the top of the child's kernel stack +/// because `copy_kernel_stack_to` copied them across when the +/// parent ran `prepare_fork_child_stack`. `rcx` holds the user +/// RIP and `r11` the user RFLAGS — the `syscall` instruction +/// stashed them there before `syscall_handler` even started. +/// • The user RSP for the child is in the unused MARKER_SIZE pad of +/// the kernel stack at `kernel_top - 8`. +/// `prepare_fork_child_stack` writes it there from the per-CPU +/// `user_stack` slot at fork time, so we get a snapshot that +/// survives any other syscall taking place between the parent's +/// fork() and the child being scheduled. +/// • `rax = 0` (fork returns 0 in the child), `swapgs` restores the +/// user GS base, `sysretq` jumps back to user mode. +#[cfg(all(feature = "common-os", feature = "fork"))] +#[unsafe(naked)] +extern "C" fn fork_child_start() { + use core::arch::naked_asm; + naked_asm!( + // Position rsp at the top of the saved-register block. + // `gs:{kernel_stack}` = kernel_top - MARKER_SIZE = kernel_top - 16. + // After 14 register pushes by syscall_handler rsp was + // gs:{kernel_stack} - 14*8. + "mov rsp, gs:{core_local_kernel_stack}", + "sub rsp, 14 * 8", + // Pop the 14 saved registers (rcx = user RIP, r11 = user RFLAGS). + "pop r15", + "pop r14", + "pop r13", + "pop r12", + "pop r11", + "pop r10", + "pop r9", + "pop r8", + "pop rdi", + "pop rsi", + "pop rbp", + "pop rbx", + "pop rdx", + "pop rcx", + // fork() returns 0 in the child. + "xor eax, eax", + // Load the per-task user_rsp from kernel_top - 8 (the unused + // MARKER_SIZE pad above the marker; `prepare_fork_child_stack` + // wrote it there). gs:{kernel_stack} holds kernel_top - 16, + // so the slot we want is at [kernel_stack + 8]. Go via rsp + // itself — we are about to overwrite it anyway and must not + // clobber rax (the fork return value). + "mov rsp, gs:{core_local_kernel_stack}", + "mov rsp, [rsp + 8]", + "swapgs", + "sysretq", + core_local_kernel_stack = const core::mem::offset_of!(super::core_local::CoreLocal, kernel_stack), + ); +} + +/// Returns the base virtual address of the current task's stack allocation. +/// Used to calculate the offset for the child's stack pointer. +#[cfg(all(feature = "common-os", feature = "fork"))] +extern "C" fn get_current_stack_addr() -> usize { + use crate::arch::x86_64::kernel::core_local::core_scheduler; + core_scheduler() + .get_current_task() + .borrow() + .stacks + .get_stack_virt_addr() + .as_usize() +} + +/// C-callable wrapper: copy the current root page table and return the new PML4 physical address. +#[cfg(all(feature = "common-os", feature = "fork"))] +extern "C" fn copy_current_root_page_table() -> usize { + crate::arch::x86_64::mm::copy_current_root_page_table() +} + +/// C-callable wrapper: copy the current kernel stack to `stack_addr`. +#[cfg(all(feature = "common-os", feature = "fork"))] +extern "C" fn copy_kernel_stack_to(stack_addr: usize) { + crate::arch::x86_64::mm::copy_kernel_stack_to(stack_addr); +} + +/// Snapshot the current user RSP into the child's kernel stack so that +/// `fork_child_start` can recover it via `sysretq` regardless of what +/// other syscalls do to the per-CPU `user_stack` slot in the meantime. +/// +/// The slot we use sits at `child_kernel_top - 8`, i.e. inside the +/// `MARKER_SIZE` pad above the `0xdeadbeef` marker — see [`fork_child_start`] +/// for the matching `mov rsp, [gs:{kernel_stack} + 8]`. +/// +/// The parent's user RSP was stashed by `syscall_handler` into +/// `gs:[user_stack]` at syscall entry; we just read it back here. +#[cfg(all(feature = "common-os", feature = "fork"))] +extern "C" fn stash_user_rsp_in_child_stack(new_stack_addr: usize) { + use crate::arch::x86_64::kernel::core_local::CoreLocal; + use crate::arch::x86_64::kernel::interrupts::IST_SIZE; + use crate::arch::x86_64::mm::paging::{BasePageSize, PageSize}; + use crate::config::DEFAULT_STACK_SIZE; + + let user_rsp: usize; + unsafe { + core::arch::asm!( + "mov {0}, gs:[{1}]", + out(reg) user_rsp, + const core::mem::offset_of!(CoreLocal, user_stack), + options(nostack, preserves_flags), + ); + } + + let child_kernel_top = + new_stack_addr + IST_SIZE + 2 * BasePageSize::SIZE as usize + DEFAULT_STACK_SIZE; + unsafe { + core::ptr::with_exposed_provenance_mut::(child_kernel_top - 8).write(user_rsp); + } +} + +/// Prepare the child's stack for a fork. +/// +/// On entry (parent): +/// rdi = `stack_pointer` (*mut usize — receives the child's rsp) +/// rsi = `root_page_table` (*mut usize — receives the child's PML4 phys addr) +/// rdx = `new_stack_addr` (base virt addr of the child's stack allocation) +/// +/// Returns `false` in the parent; the child task's saved context will `ret` to +/// `fork_child_start` which jumps directly back to user space via `sysretq`. +#[cfg(all(feature = "common-os", feature = "fork"))] +#[unsafe(naked)] +pub unsafe extern "C" fn prepare_fork_child_stack( + _stack_pointer: *mut usize, + _root_page_table: *mut usize, + _new_stack_addr: usize, +) -> bool { + naked_asm!( + // Push fork_child_start as the child's future return address. + "lea rax, [rip + {fork_child_start}]", + "push rax", + // Save all caller-saved and callee-saved registers. + save_context!(), + // Spill the three parameters onto the stack so we can call C functions. + "push rdi", // [rsp+16]: stack_pointer ptr + "push rdx", // [rsp+8]: new_stack_addr + "push rsi", // [rsp+0]: root_page_table ptr + + // 1. Copy the kernel stack pages FIRST, so the copied mappings exist in the + // current page table before we snapshot it for the child's PML4. + "mov rdi, [rsp+8]", // rdi = new_stack_addr + "call {copy_kernel_stack_to}", // copy kernel stack pages + + // 1b. Stash the user RSP into the child's kernel stack at + // child_kernel_top - 8, so that fork_child_start can + // restore it via sysretq without relying on the per-CPU + // `user_stack` slot (which other syscalls may clobber). + "mov rdi, [rsp+8]", // rdi = new_stack_addr + "call {stash_user_rsp_in_child_stack}", + + // 2. Duplicate the page table (COW) — snapshot now includes the copied stack. + "call {copy_current_root_page_table}", // rax = new PML4 phys addr + "mov rsi, [rsp]", // rsi = root_page_table ptr + "mov [rsi], rax", // *root_page_table = new PML4 + + // 3. Calculate the child's stack pointer. + // child_rsp = rsp_after_save_context + (new_stack_addr - current_stack_base) + "call {get_current_stack_addr}", // rax = current stack base addr + "pop rsi", // pop root_page_table ptr (already written) + "pop rbx", // rbx = new_stack_addr + "pop rdi", // rdi = stack_pointer ptr + "sub rbx, rax", // rbx = new_stack_addr - stack_base (offset) + "add rbx, rsp", // rbx = rsp_after_save_context + offset = child rsp + "mov [rdi], rbx", // *stack_pointer = child_rsp + + // Restore registers (parent continues normally). + restore_context_without_return!(), + // Skip the fork_child_start address we pushed at the top. + "add rsp, 8", + // Return false (0) — this is the parent. + "xor rax, rax", + "ret", + fork_child_start = sym fork_child_start, + copy_current_root_page_table = sym copy_current_root_page_table, + copy_kernel_stack_to = sym copy_kernel_stack_to, + stash_user_rsp_in_child_stack = sym stash_user_rsp_in_child_stack, + get_current_stack_addr = sym get_current_stack_addr, + ); +} diff --git a/src/arch/x86_64/kernel/syscall.rs b/src/arch/x86_64/kernel/syscall.rs index cc50cab8be..3e34c6ac97 100644 --- a/src/arch/x86_64/kernel/syscall.rs +++ b/src/arch/x86_64/kernel/syscall.rs @@ -1,49 +1,70 @@ use core::arch::naked_asm; -use core::mem; -use super::core_local::CoreLocal; use crate::syscalls::table::SYSHANDLER_TABLE; #[unsafe(no_mangle)] #[unsafe(naked)] pub(crate) unsafe extern "C" fn syscall_handler() -> ! { naked_asm!( - // save context, see x86_64 ABI + "swapgs", + // switch to kernel stack + "mov gs:{core_local_user_stack}, rsp", + "mov rsp, gs:{core_local_kernel_stack}", + // save context "push rcx", "push rdx", + "push rbx", + "push rbp", "push rsi", "push rdi", "push r8", "push r9", "push r10", "push r11", - // switch to kernel stack - "swapgs", - "mov rcx, rsp", - "mov rsp, gs:{core_local_kernel_stack}", - // save user stack pointer + "push r12", + "push r13", + "push r14", + "push r15", + // All user-visible registers are now on the kernel stack, so + // rcx is free to use as scratch. Stash the user RSP onto the + // per-task kernel stack — the per-CPU `user_stack` slot would + // otherwise be clobbered by any syscall a different task runs + // while this call is blocked (e.g. waitpid waiting for a child). + "mov rcx, gs:{core_local_user_stack}", "push rcx", + "sti", // copy 4th argument to rcx to adhere x86_64 ABI "mov rcx, r10", - "sti", - "mov r10, qword ptr [rip + {table}@GOTPCREL]", + // call system call + "mov r10, [rip + {table}@GOTPCREL]", "call [r10 + 8*rax]", "cli", - // restore user stack pointer + // Pop the stashed user RSP and route it back through the + // per-CPU slot so the final `mov rsp, gs:[user_stack]` still + // works. Interrupts are off, so the slot can't be clobbered + // between here and `sysretq`. "pop rcx", - "mov rsp, rcx", - "swapgs", - // restore context, see x86_64 ABI + "mov gs:{core_local_user_stack}, rcx", + // restore context (without rax) + "pop r15", + "pop r14", + "pop r13", + "pop r12", "pop r11", "pop r10", "pop r9", "pop r8", "pop rdi", "pop rsi", + "pop rbp", + "pop rbx", "pop rdx", "pop rcx", + "mov rsp, gs:{core_local_user_stack}", + "swapgs", "sysretq", - core_local_kernel_stack = const mem::offset_of!(CoreLocal, kernel_stack), + core_local_user_stack = const core::mem::offset_of!(super::core_local::CoreLocal, user_stack), + core_local_kernel_stack = const core::mem::offset_of!(super::core_local::CoreLocal, kernel_stack), table = sym SYSHANDLER_TABLE, ); } diff --git a/src/arch/x86_64/mm/mod.rs b/src/arch/x86_64/mm/mod.rs index 43982e292c..568b1344dd 100644 --- a/src/arch/x86_64/mm/mod.rs +++ b/src/arch/x86_64/mm/mod.rs @@ -4,8 +4,13 @@ pub(crate) mod paging; use core::slice; use memory_addresses::arch::x86_64::{PhysAddr, VirtAddr}; +#[cfg(all(feature = "common-os", feature = "fork"))] +pub use paging::copy_kernel_stack_to; +/// Copy the kernel stack pages of the current task to a new base address. #[cfg(feature = "common-os")] -use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; +pub use paging::{clear_user_space, drop_user_space}; +#[cfg(feature = "common-os")] +pub use x86_64::structures::paging::{PageSize, Size4KiB as BasePageSize}; #[cfg(feature = "common-os")] use crate::arch::mm::paging::{PageTableEntryFlags, PageTableEntryFlagsExt}; @@ -58,6 +63,216 @@ pub fn create_new_root_page_table() -> usize { physaddr.as_usize() } +/// Returns the physical address of the current task's root page table (PML4). +#[allow(dead_code)] +#[cfg(feature = "common-os")] +pub fn get_current_root_page_table() -> usize { + use crate::arch::kernel::core_local::core_scheduler; + core_scheduler() + .get_current_task() + .borrow() + .root_page_table + .as_usize() +} + +/// Copy the current task's PML4 into a new page table, sharing data pages (COW). +/// Returns the physical address of the new PML4. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn copy_current_root_page_table() -> usize { + use core::ptr; + + use free_list::PageLayout; + use x86_64::registers::control::Cr3; + use x86_64::structures::paging::PageTable; + + let layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); + + // Allocate a new PML4 frame + let new_pml4_frame = FrameAlloc::allocate(layout).unwrap(); + let new_pml4_phys = PhysAddr::new(new_pml4_frame.start().try_into().unwrap()); + let new_pml4 = unsafe { + &mut *ptr::with_exposed_provenance_mut::( + new_pml4_phys.as_u64().try_into().unwrap(), + ) + }; + + // Access the current PML4 via Cr3 (identity-mapped: phys == virt) + let (frame, _) = Cr3::read(); + let cur_pml4_phys = frame.start_address().as_u64(); + let cur_pml4 = + unsafe { &*ptr::with_exposed_provenance::(cur_pml4_phys.try_into().unwrap()) }; + + use x86_64::structures::paging::PageTableFlags; + + // Clear new PML4 + for entry in new_pml4.iter_mut() { + entry.set_unused(); + } + + // Copy entries 1..511 (user-space) by deep-copying page table hierarchy. + // Data pages themselves are shared (COW); only page-table pages are duplicated. + for pml4_idx in 1..511usize { + let cur_entry = &cur_pml4[pml4_idx]; + if !cur_entry.flags().contains(PageTableFlags::PRESENT) { + continue; + } + + let new_pdpt_frame = FrameAlloc::allocate(layout).unwrap(); + let new_pdpt_phys = PhysAddr::new(new_pdpt_frame.start().try_into().unwrap()); + new_pml4[pml4_idx].set_addr(new_pdpt_phys.into(), cur_entry.flags()); + + let cur_pdpt = unsafe { + &*ptr::with_exposed_provenance::( + cur_entry.addr().as_u64().try_into().unwrap(), + ) + }; + let new_pdpt = unsafe { + &mut *ptr::with_exposed_provenance_mut::( + new_pdpt_phys.as_u64().try_into().unwrap(), + ) + }; + + for pdpt_idx in 0..512usize { + let cur_pdpt_entry = &cur_pdpt[pdpt_idx]; + if !cur_pdpt_entry.flags().contains(PageTableFlags::PRESENT) { + new_pdpt[pdpt_idx].set_unused(); + continue; + } + + let new_pd_frame = FrameAlloc::allocate(layout).unwrap(); + let new_pd_phys = PhysAddr::new(new_pd_frame.start().try_into().unwrap()); + new_pdpt[pdpt_idx].set_addr(new_pd_phys.into(), cur_pdpt_entry.flags()); + + let cur_pd = unsafe { + &*ptr::with_exposed_provenance::( + cur_pdpt_entry.addr().as_u64().try_into().unwrap(), + ) + }; + let new_pd = unsafe { + &mut *ptr::with_exposed_provenance_mut::( + new_pd_phys.as_u64().try_into().unwrap(), + ) + }; + + for pd_idx in 0..512usize { + let cur_pd_entry = &cur_pd[pd_idx]; + if !cur_pd_entry.flags().contains(PageTableFlags::PRESENT) { + new_pd[pd_idx].set_unused(); + continue; + } + + let new_pt_frame = FrameAlloc::allocate(layout).unwrap(); + let new_pt_phys = PhysAddr::new(new_pt_frame.start().try_into().unwrap()); + new_pd[pd_idx].set_addr(new_pt_phys.into(), cur_pd_entry.flags()); + + let cur_pt = unsafe { + &*ptr::with_exposed_provenance::( + cur_pd_entry.addr().as_u64().try_into().unwrap(), + ) + }; + let new_pt = unsafe { + &mut *ptr::with_exposed_provenance_mut::( + new_pt_phys.as_u64().try_into().unwrap(), + ) + }; + + // Copy PT entries verbatim — data pages are shared (already COW-marked) + *new_pt = cur_pt.clone(); + + // The child now holds an additional user reference to every + // frame in this page table. + for entry in new_pt.iter() { + if entry + .flags() + .contains(PageTableFlags::PRESENT | PageTableFlags::USER_ACCESSIBLE) + { + crate::mm::frame_ref_inc(entry.addr().into()); + } + } + } + } + } + + // Entry 0: kernel low mapping — copy from current PML4 + unsafe { + let src = &raw const cur_pml4[0]; + let dst = &raw mut new_pml4[0]; + *dst = (*src).clone(); + } + + // Entry 511: self-reference for the new PML4 + new_pml4[511].set_addr( + new_pml4_phys.into(), + PageTableFlags::PRESENT | PageTableFlags::WRITABLE, + ); + + new_pml4_phys.as_usize() +} + +/// Mark all writable user pages in the current page table as Copy-On-Write. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn prepare_mem_copy_on_write() { + paging::mark_user_pages_copy_on_write(); +} + +// Allocate and initialize a private TLS region for a new user-space thread. +/// +/// Maps a fresh user-accessible page range through the currently active +/// (shared) root page table, copies the per-process `TlsTemplate` into it, +/// sets up the trailing 8-byte TCB self-pointer, and returns the new +/// FS.Base value (the thread pointer) for the child thread. +#[cfg(feature = "common-os")] +pub fn allocate_thread_tls(template: &crate::scheduler::task::TlsTemplate) -> u64 { + use align_address::Align; + use free_list::PageLayout; + use x86_64::structures::paging::Size4KiB as BasePageSize; + + #[cfg(feature = "fork")] + use crate::mm::frame_ref_inc; + + let tcb_size = size_of::<*mut ()>(); + let total = (template.size + tcb_size).align_up(BasePageSize::SIZE as usize); + + let virt_layout = PageLayout::from_size(total).unwrap(); + let virt_range = PageAlloc::allocate(virt_layout).unwrap(); + let virt_addr = VirtAddr::from(virt_range.start()); + + let frame_layout = PageLayout::from_size(total).unwrap(); + let frame_range = FrameAlloc::allocate(frame_layout).unwrap(); + let phys_addr = PhysAddr::from(frame_range.start()); + #[cfg(feature = "fork")] + for i in 0..total / BasePageSize::SIZE as usize { + frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize); + } + + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().user().execute_disable(); + paging::map::( + virt_addr, + phys_addr, + total / BasePageSize::SIZE as usize, + flags, + ); + + unsafe { + // Copy the pristine PT_TLS image into the new block. + virt_addr + .as_mut_ptr::() + .copy_from_nonoverlapping(template.init.as_ptr(), template.init.len()); + // Zero the rest of the TLS BSS area and the trailing TCB. + virt_addr + .as_mut_ptr::() + .add(template.init.len()) + .write_bytes(0, total - template.init.len()); + // Variant II (x86_64): the thread pointer is at the start of the TCB + // (right after the TLS data block) and stores its own address. + let thread_ptr = virt_addr.as_u64() + template.size as u64; + let tcb_ptr: *mut u64 = core::ptr::with_exposed_provenance_mut(thread_ptr as usize); + tcb_ptr.write(thread_ptr); + thread_ptr + } +} + pub unsafe fn init() { paging::init(); unsafe { diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index aa56d2f599..c98c7acf03 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -1,6 +1,8 @@ use core::{fmt, ptr}; use free_list::PageLayout; +#[cfg(feature = "common-os")] +use x86_64::instructions::tlb; use x86_64::registers::control::{Cr0, Cr0Flags, Cr2, Cr3}; #[cfg(feature = "common-os")] use x86_64::registers::segmentation::SegmentSelector; @@ -52,6 +54,10 @@ pub trait PageTableEntryFlagsExt { #[expect(dead_code)] #[cfg(feature = "common-os")] fn kernel(&mut self) -> &mut Self; + + /// Mark a page as Copy-On-Write: remove WRITABLE, remove DIRTY, set BIT_9 as COW marker. + #[cfg(all(feature = "common-os", feature = "fork"))] + fn copy_on_write(&mut self) -> &mut Self; } impl PageTableEntryFlagsExt for PageTableEntryFlags { @@ -96,6 +102,15 @@ impl PageTableEntryFlagsExt for PageTableEntryFlags { #[cfg(feature = "common-os")] fn kernel(&mut self) -> &mut Self { self.remove(PageTableEntryFlags::USER_ACCESSIBLE); + self.insert(PageTableEntryFlags::GLOBAL); + self + } + + #[cfg(all(feature = "common-os", feature = "fork"))] + fn copy_on_write(&mut self) -> &mut Self { + self.remove(PageTableEntryFlags::WRITABLE); + self.remove(PageTableEntryFlags::DIRTY); + self.insert(PageTableEntryFlags::BIT_9); // BIT_9 = COW marker self } } @@ -275,6 +290,243 @@ where } } +/// Walk user pages in the current PML4 and mark all writable user pages as Copy-On-Write. +/// This must be called before duplicating the page table for a fork. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn mark_user_pages_copy_on_write() { + // Since the kernel identity-maps all physical memory (phys addr == virt addr), + // we can dereference physical addresses directly as page table pointers. + let (frame, _) = Cr3::read(); + let pml4 = unsafe { + &mut *ptr::with_exposed_provenance_mut::(frame.start_address().as_u64() as usize) + }; + + // Walk PML4 entries 1..511 (user-space entries). + // Entry 0 is for the low kernel mapping; entry 511 is the self-reference. + for pml4_idx in 1..511usize { + let pml4_entry = &mut pml4[pml4_idx]; + if !pml4_entry.flags().contains(PageTableEntryFlags::PRESENT) { + continue; + } + let pdpt = unsafe { + &mut *ptr::with_exposed_provenance_mut::(pml4_entry.addr().as_u64() as usize) + }; + for pdpt_idx in 0..512usize { + let pdpt_entry = &mut pdpt[pdpt_idx]; + if !pdpt_entry.flags().contains(PageTableEntryFlags::PRESENT) { + continue; + } + let pd = unsafe { + &mut *ptr::with_exposed_provenance_mut::( + pdpt_entry.addr().as_u64().try_into().unwrap(), + ) + }; + for pd_idx in 0..512usize { + let pd_entry = &mut pd[pd_idx]; + if !pd_entry.flags().contains(PageTableEntryFlags::PRESENT) { + continue; + } + if pd_entry.flags().contains(PageTableEntryFlags::HUGE_PAGE) { + // Skip huge pages (2 MiB) - not handled as COW here + warn!("User space isn't able to use huge pages"); + continue; + } + let pt = unsafe { + &mut *ptr::with_exposed_provenance_mut::( + pd_entry.addr().as_u64().try_into().unwrap(), + ) + }; + for pt_idx in 0..512usize { + let pt_entry = &mut pt[pt_idx]; + if pt_entry.flags().contains( + PageTableEntryFlags::PRESENT + | PageTableEntryFlags::WRITABLE + | PageTableEntryFlags::USER_ACCESSIBLE, + ) && !pt_entry.flags().contains(PageTableEntryFlags::BIT_9) + { + let new_flags = *pt_entry.flags().copy_on_write(); + pt_entry.set_addr(pt_entry.addr(), new_flags); + } + } + } + } + } + + tlb::flush_all(); + #[cfg(feature = "smp")] + crate::arch::x86_64::kernel::apic::ipi_tlb_flush(); +} + +#[cfg(feature = "common-os")] +fn clear_pml4(pml4_phys: usize) { + // Free a 4 KiB physical frame back to the frame allocator. + fn free_frame(phys: usize) { + let range = free_list::PageRange::new(phys, phys + free_list::PAGE_SIZE).unwrap(); + unsafe { FrameAlloc::deallocate(range) }; + } + + // Task::drop is usually called from another task's context (e.g. the + // idle task running `cleanup_tasks`), so `Cr3::read()` would return the + // page table of the cleaning task — not of the one being destroyed. + // Walk the stored PML4 directly through the kernel identity map. + let pml4 = unsafe { &mut *ptr::with_exposed_provenance_mut::(pml4_phys) }; + + // Walk PML4 entries 1..511 (user-space entries). + // Entry 0 is for the low kernel mapping; entry 511 is the self-reference. + for pml4_idx in 1..511usize { + let pml4_entry = &mut pml4[pml4_idx]; + if !pml4_entry.flags().contains(PageTableEntryFlags::PRESENT) { + continue; + } + let pdpt_phys = pml4_entry.addr().as_u64() as usize; + let pdpt = unsafe { &mut *ptr::with_exposed_provenance_mut::(pdpt_phys) }; + + for pdpt_idx in 0..512usize { + let pdpt_entry = &mut pdpt[pdpt_idx]; + if !pdpt_entry.flags().contains(PageTableEntryFlags::PRESENT) { + continue; + } + let pd_phys = pdpt_entry.addr().as_u64() as usize; + let pd = unsafe { &mut *ptr::with_exposed_provenance_mut::(pd_phys) }; + + for pd_idx in 0..512usize { + let pd_entry = &mut pd[pd_idx]; + if !pd_entry.flags().contains(PageTableEntryFlags::PRESENT) { + continue; + } + if pd_entry.flags().contains(PageTableEntryFlags::HUGE_PAGE) { + // Skip huge pages (2 MiB) - not handled as COW here + warn!("User space isn't able to use huge pages"); + continue; + } + let pt_phys = pd_entry.addr().as_u64() as usize; + let pt = unsafe { &mut *ptr::with_exposed_provenance_mut::(pt_phys) }; + + for pt_idx in 0..512usize { + let pt_entry = &mut pt[pt_idx]; + if pt_entry.flags().contains( + PageTableEntryFlags::PRESENT | PageTableEntryFlags::USER_ACCESSIBLE, + ) { + let phys_addr = PhysAddr::new(pt_entry.addr().as_u64()); + #[cfg(feature = "fork")] + if crate::mm::frame_ref_dec(phys_addr) { + free_frame(phys_addr.as_u64() as usize); + } + #[cfg(not(feature = "fork"))] + { + free_frame(phys_addr.as_u64() as usize); + } + pt_entry.set_unused(); + } + } + + // PT emptied → free the PT frame and clear the PD entry. + free_frame(pt_phys); + pd_entry.set_unused(); + } + + // PD emptied → free the PD frame and clear the PDPT entry. + free_frame(pd_phys); + pdpt_entry.set_unused(); + } + + // PDPT emptied → free the PDPT frame and clear the PML4 entry. + free_frame(pdpt_phys); + pml4_entry.set_unused(); + } +} + +#[cfg(feature = "common-os")] +pub fn drop_user_space(pml4_phys: usize) { + debug!("Drop the user space at PML4 {pml4_phys:#x}"); + + clear_pml4(pml4_phys); + + // Finally free the PML4 itself. No TLB flush needed: the dropped page + // table is not loaded on any core. + let range = free_list::PageRange::new(pml4_phys, pml4_phys + free_list::PAGE_SIZE).unwrap(); + unsafe { FrameAlloc::deallocate(range) }; +} + +#[cfg(feature = "common-os")] +pub fn clear_user_space() { + use crate::core_scheduler; + use crate::fd::STDERR_FILENO; + + core_scheduler() + .get_current_task() + .borrow() + .vmas + .write() + .clear(); + core_scheduler() + .get_current_task_object_map() + .write() + .retain(|&k, _| k <= STDERR_FILENO); + + let (p4_frame, _) = Cr3::read_raw(); + let pml4_phys = p4_frame.start_address().as_u64() as usize; + + debug!("Clear the user space at PML4 {pml4_phys:#x}"); + + clear_pml4(pml4_phys); + + tlb::flush_all(); + #[cfg(feature = "smp")] + crate::arch::x86_64::kernel::apic::ipi_tlb_flush(); +} + +/// Copy the contents of the current task's kernel stack to a new virtual +/// base address. Used by fork: the child's `TaskStacks::new` has already +/// allocated and mapped fresh physical frames at `stack_address`, so we +/// simply `memcpy` the parent's stack pages into the child's mapping. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn copy_kernel_stack_to(stack_address: usize) { + use crate::arch::kernel::core_local::core_scheduler; + + let virt_addr = core_scheduler() + .get_current_task() + .borrow() + .stacks + .get_stack_virt_addr(); + let total_size = core_scheduler() + .get_current_task() + .borrow() + .stacks + .get_total_stack_size(); + + let addr_diff = stack_address as u64 - virt_addr.as_u64(); + + let page_table = unsafe { identity_mapped_page_table() }; + + // The virtual layout has 4 guard pages (unmapped) interspersed; iterate + // the full range including those so no mapped pages are missed. + let full_virt_size = total_size as u64 + 4 * BasePageSize::SIZE; + for i in (virt_addr.as_u64()..(virt_addr.as_u64() + full_virt_size)) + .step_by(BasePageSize::SIZE as usize) + { + let virt = x86_64::VirtAddr::new(i); + match page_table.translate(virt) { + TranslateResult::Mapped { .. } => { + // Both src and dst are already mapped in the current page + // table (dst via TaskStacks::new), so a plain memcpy suffices. + let src = ptr::with_exposed_provenance::(i as usize); + let dst = ptr::with_exposed_provenance_mut::((i + addr_diff) as usize); + unsafe { + dst.copy_from_nonoverlapping(src, BasePageSize::SIZE as usize); + } + } + TranslateResult::NotMapped => {} + TranslateResult::InvalidFrameAddress(_) => { + error!("Unexpected translation result for stack page at {i:#X}"); + scheduler::abort(); + } + } + } + + tlb::flush_all(); +} + #[cfg(not(feature = "common-os"))] pub(crate) extern "x86-interrupt" fn page_fault_handler( stack_frame: ExceptionStackFrame, @@ -294,13 +546,156 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( mut stack_frame: ExceptionStackFrame, error_code: PageFaultErrorCode, ) { - unsafe { - if stack_frame.as_mut().read().code_segment != SegmentSelector(0x08) { - core::arch::asm!("swapgs", options(nostack)); + use core::arch::asm; + + use crate::arch::kernel::core_local::{core_scheduler, increment_irq_counter}; + let swapped_gs = unsafe { + if stack_frame.as_mut().read().code_segment == SegmentSelector(0x08) { + false + } else { + asm!("swapgs", options(nostack)); + true } + }; + + increment_irq_counter(14); + + let faulting_addr = Cr2::read().unwrap(); + let virtaddr = faulting_addr.align_down(BasePageSize::SIZE); + + debug!( + "Task {} triggers a page fault at {:p}.", + core_scheduler().get_current_task_id(), + faulting_addr + ); + + // Handle Copy-On-Write faults: write fault on a read-only page with BIT_9 set. + #[cfg(feature = "fork")] + if error_code.contains(PageFaultErrorCode::CAUSED_BY_WRITE) + && !error_code.contains(PageFaultErrorCode::INSTRUCTION_FETCH) + { + let page_table = unsafe { identity_mapped_page_table() }; + if let TranslateResult::Mapped { flags, .. } = page_table.translate(virtaddr) + && flags.contains(PageTableEntryFlags::BIT_9) + && !flags.contains(PageTableEntryFlags::WRITABLE) + { + // COW fault: allocate a new page, copy old contents, map as writable. + if let Some(src_phys) = virtual_to_physical(virtaddr.into()) { + let mut new_flags = flags; + new_flags.insert(PageTableEntryFlags::WRITABLE); + new_flags.remove(PageTableEntryFlags::BIT_9); + // Drop this task's COW reference + let ref_is_zero = crate::mm::frame_ref_dec(src_phys); + + if ref_is_zero { + crate::mm::frame_ref_inc(src_phys); + map::(virtaddr.into(), src_phys, 1, new_flags); + } else { + let new_phys = crate::mm::copy_page(src_phys); + crate::mm::frame_ref_inc(new_phys); + map::(virtaddr.into(), new_phys, 1, new_flags); + } + + if swapped_gs { + unsafe { + core::arch::asm!("swapgs", options(nostack)); + } + } + + return; + } else { + error!("COW: failed to resolve physical address for {virtaddr:p}"); + scheduler::abort(); + } + } + } + + { + use core::ops::Bound; + + use crate::mm::vma::VirtualMemoryAreaProt; + + let current_task = core_scheduler().get_current_task(); + let current_task_borrowed = current_task.borrow(); + let guard = current_task_borrowed.vmas.read(); + + if let Some((_, vma)) = guard + .range((Bound::Unbounded, Bound::Included(virtaddr))) + .next_back() + && virtaddr >= vma.start + && virtaddr < vma.end + { + let layout = PageLayout::from_size_align( + BasePageSize::SIZE as usize, + BasePageSize::SIZE as usize, + ) + .unwrap(); + let frame_range = FrameAlloc::allocate(layout).unwrap(); + let physaddr = PhysAddr::from(frame_range.start()); + let mut flags = PageTableEntryFlags::empty(); + flags.normal().user(); + if vma.prot.contains(VirtualMemoryAreaProt::WRITE) { + flags.writable(); + } + if vma.prot.contains(VirtualMemoryAreaProt::EXECUTE) { + flags.execute_disable(); + } + + // `virtaddr` is `x86_64::VirtAddr` (from `Cr2::read`); the + // `map` helper signs the parameter as `memory_addresses::VirtAddr`. + map::(virtaddr.into(), physaddr, 1, flags); + + // clear page + let slice = unsafe { + core::slice::from_raw_parts_mut( + virtaddr.as_mut_ptr::().cast::(), + BasePageSize::SIZE as usize, + ) + }; + slice.fill(0); + + #[cfg(feature = "fork")] + crate::mm::frame_ref_inc(physaddr); + + // Restore user GS before returning to ring 3; the COW path + // above does the same. Without this, iret leaves the kernel + // GS-base installed, the user's next FS/GS access reads + // kernel state and the program either misbehaves or wedges. + if swapped_gs { + unsafe { + core::arch::asm!("swapgs", options(nostack)); + } + } + + return; + } + } + + // A spawned user thread whose entry function returns comes back here + // with RIP=0: the `thread_start` wrapper in std's hermit pal just + // returns, and the kernel-crafted return slot at the top of the user + // stack was zero-initialized. Treat an instruction-fetch fault at 0 + // in ring 3 as a clean thread exit. + if error_code.contains(PageFaultErrorCode::USER_MODE) + && error_code.contains(PageFaultErrorCode::INSTRUCTION_FETCH) + && faulting_addr.as_u64() == 0 + { + debug!( + "User thread {} returned from entry; exiting cleanly.", + core_scheduler().get_current_task_id() + ); + // Do not swap GS back here. `exit` will context-switch to another + // task, so the current kernel GS must remain in place for the + // scheduler bookkeeping. `swapped_gs` is a no-op at this point: + // the handler never returns, and the next task's GS is restored + // by the context switch. + let _ = swapped_gs; + use crate::scheduler::PerCoreSchedulerExt; + core_scheduler().exit(0); } + error!("Page fault (#PF)!"); - error!("page_fault_linear_address = {:p}", Cr2::read().unwrap()); + error!("page_fault_linear_address = {faulting_addr:p}"); error!("error_code = {error_code:?}"); error!("fs = {:#X}", processor::readfs()); error!("gs = {:#X}", processor::readgs()); diff --git a/src/lib.rs b/src/lib.rs index 25745c5744..9dde577d95 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,10 +47,7 @@ #![feature(allocator_api)] #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr( - all( - not(any(feature = "common-os", feature = "nostd")), - not(target_arch = "riscv64"), - ), + all(not(feature = "nostd"), not(target_arch = "riscv64"),), feature(linkage) )] #![feature(linked_list_cursors)] @@ -106,7 +103,10 @@ mod macros; mod logging; pub mod arch; -#[cfg(all(feature = "common-os", target_arch = "x86_64"))] +#[cfg(all( + feature = "common-os", + any(target_arch = "x86_64", target_arch = "aarch64") +))] pub mod common_os; pub mod config; pub mod console; diff --git a/src/mm/mod.rs b/src/mm/mod.rs index 9f16f4104f..4b34991d28 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -44,6 +44,8 @@ pub(crate) mod device_alloc; mod page_range_alloc; mod physicalmem; mod virtualmem; +#[cfg(feature = "common-os")] +pub(crate) mod vma; use core::alloc::Layout; use core::mem::MaybeUninit; @@ -59,7 +61,11 @@ use talc::TalcLock; use talc::source::Manual; pub use self::page_range_alloc::{PageRangeAllocator, PageRangeBox}; +#[cfg(feature = "common-os")] +pub use self::physicalmem::copy_page; pub use self::physicalmem::{FrameAlloc, FrameBox}; +#[cfg(all(feature = "common-os", feature = "fork"))] +pub use self::physicalmem::{frame_ref_dec, frame_ref_inc}; pub use self::virtualmem::{PageAlloc, PageBox}; #[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] use crate::arch::mm::paging::HugePageSize; diff --git a/src/mm/physicalmem.rs b/src/mm/physicalmem.rs index 758871230c..06129a000c 100644 --- a/src/mm/physicalmem.rs +++ b/src/mm/physicalmem.rs @@ -1,3 +1,5 @@ +#[cfg(all(feature = "common-os", feature = "fork"))] +use alloc::collections::BTreeMap; use core::alloc::AllocError; use core::fmt; use core::sync::atomic::{AtomicUsize, Ordering}; @@ -18,6 +20,44 @@ static PHYSICAL_FREE_LIST: InterruptTicketMutex> = InterruptTicketMutex::new(FreeList::new()); pub static TOTAL_MEMORY: AtomicUsize = AtomicUsize::new(0); +/// Sparse per-frame COW reference counts. +/// Only frames that are actively COW-shared have an entry; exclusively-owned +/// frames are absent (equivalent to refcount 0). Stored in a `BTreeMap` so +/// that memory use scales with the number of *shared* frames, not with total +/// physical memory. +#[cfg(all(feature = "common-os", feature = "fork"))] +static PAGE_REFCOUNTS: InterruptTicketMutex> = + InterruptTicketMutex::new(BTreeMap::new()); + +/// Increment the COW reference count for `phys_addr` (4 KiB-aligned frame). +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn frame_ref_inc(phys_addr: PhysAddr) { + let frame = (phys_addr.as_u64() as usize) >> 12; + *PAGE_REFCOUNTS.lock().entry(frame).or_insert(0) += 1; +} + +/// Decrement the COW reference count for `phys_addr`. +/// If the count reaches zero the the function returned true. +#[cfg(all(feature = "common-os", feature = "fork"))] +pub fn frame_ref_dec(phys_addr: PhysAddr) -> bool { + let frame = (phys_addr.as_u64() as usize) >> 12; + let mut map = PAGE_REFCOUNTS.lock(); + match map.get_mut(&frame) { + None => { + warn!("frame_ref_dec: no refcount entry for frame {phys_addr:p}"); + false + } + Some(count) if *count <= 1 => { + map.remove(&frame); + true + } + Some(count) => { + *count -= 1; + false + } + } +} + pub struct FrameAlloc; impl PageRangeAllocator for FrameAlloc { @@ -48,6 +88,12 @@ impl PageRangeAllocator for FrameAlloc { } } +impl FrameAlloc { + pub fn free_space() -> usize { + PHYSICAL_FREE_LIST.lock().free_space() + } +} + impl fmt::Display for FrameAlloc { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let free_list = PHYSICAL_FREE_LIST.lock(); @@ -57,6 +103,43 @@ impl fmt::Display for FrameAlloc { pub type FrameBox = PageRangeBox; +/// Copy the physical page at `src_phys` into a freshly allocated page and return its address. +#[cfg(feature = "common-os")] +pub fn copy_page(src_phys: PhysAddr) -> PhysAddr { + use crate::arch::mm::paging::BasePageSize; + use crate::mm::PageBox; + + let frame_layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap(); + let frame_range = FrameAlloc::allocate(frame_layout).expect("Failed to allocate page"); + let dst_phys = PhysAddr::new(frame_range.start().try_into().unwrap()); + + let page_layout = PageLayout::from_size(2 * BasePageSize::SIZE as usize).unwrap(); + let page_box = PageBox::new(page_layout).unwrap(); + let virt = VirtAddr::from(page_box.start()); + + let flags = { + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable(); + flags + }; + paging::map::(virt, src_phys, 1, flags); + paging::map::(virt + BasePageSize::SIZE, dst_phys, 1, flags); + + unsafe { + let src = core::slice::from_raw_parts(virt.as_ptr::(), BasePageSize::SIZE as usize); + let dst = core::slice::from_raw_parts_mut( + (virt + BasePageSize::SIZE).as_mut_ptr::(), + BasePageSize::SIZE as usize, + ); + dst.copy_from_slice(src); + } + + paging::unmap::(virt, 2); + // page_box is dropped here, freeing the virtual memory + + dst_phys +} + pub fn total_memory_size() -> usize { TOTAL_MEMORY.load(Ordering::Relaxed) } diff --git a/src/mm/vma.rs b/src/mm/vma.rs new file mode 100644 index 0000000000..ff72cccd57 --- /dev/null +++ b/src/mm/vma.rs @@ -0,0 +1,159 @@ +use core::ops::Bound; + +#[cfg(not(target_arch = "x86_64"))] +use memory_addresses::VirtAddr; +#[cfg(target_arch = "x86_64")] +use x86_64::VirtAddr; + +use crate::arch::kernel::core_local::core_scheduler; +use crate::errno::Errno; + +/// A contiguous range of virtual addresses with uniform protection +/// and backing semantics, owned by one address space. +#[derive(Debug, Copy, Clone)] +pub struct VirtualMemoryArea { + /// Inclusive start, page-aligned. + pub start: VirtAddr, + /// Exclusive end, page-aligned. + pub end: VirtAddr, + /// Protection bits the kernel must install when faulting in a page. + pub prot: VirtualMemoryAreaProt, + /// Description of the memory type + #[allow(unused)] + pub mem_type: MemoryType, +} + +impl VirtualMemoryArea { + pub fn new( + start: VirtAddr, + end: VirtAddr, + prot: VirtualMemoryAreaProt, + mem_type: MemoryType, + ) -> Self { + Self { + start, + end, + prot, + mem_type, + } + } +} + +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug, Copy, Clone)] +pub enum MemoryType { + HEAP, + STACK, + CODE, + TLS, +} + +bitflags! { + #[repr(transparent)] + #[derive(Debug, Copy, Clone, Default)] + pub struct VirtualMemoryAreaProt: u32 { + /// Memory may not be accessed. + const NONE = 0; + /// Indicates that the memory region should be readable. + const READ = 1 << 0; + /// Indicates that the memory region should be writable. + const WRITE = 1 << 1; + /// Indicates that the memory region should be executable. + const EXECUTE = 1 << 2; + } +} + +// Place the user heap inside the L0 slot that covers LOADER_START +// (`USER_L0_INDEX = LOADER_START >> 39 = 2`). On aarch64 only that +// slot is COW-marked at fork and deep-copied by +// `copy_current_root_page_table`; everything else is treated as a +// shared kernel mapping. Putting the heap outside L0[2] leaves it +// shared between parent and child forks, which manifests as the +// child overwriting the parent's heap on its first user-space write. +// +// LOADER_START is 0x0100_0000_0000; binaries currently take well +// under 256 GiB, so 0x0140_0000_0000 sits comfortably above any +// loaded image while still falling inside L0[2] (which ends at +// 0x0180_0000_0000). The address is canonical on x86_64 as well +// (bit 47 = 0). +const HEAP_START_ADDR: VirtAddr = VirtAddr::new(0x0140_0000_0000); + +/// Creates a new virtual memory mapping of the `size` specified with +/// protection bits specified in `prot_flags`. +#[hermit_macro::system(errno)] +#[unsafe(no_mangle)] +pub extern "C" fn sys_mmap( + size: usize, + prot_flags: VirtualMemoryAreaProt, + ret: &mut *mut u8, +) -> i32 { + if (*ret).is_null() { + let current_task = core_scheduler().get_current_task(); + let current_task_borrowed = current_task.borrow(); + let mut guard = current_task_borrowed.vmas.write(); + + if let Some((_, vma)) = guard + .range((Bound::Unbounded, Bound::Included(HEAP_START_ADDR))) + .next_back() + { + if vma.end < HEAP_START_ADDR { + let new_vma = VirtualMemoryArea::new( + HEAP_START_ADDR, + HEAP_START_ADDR + size as u64, + prot_flags, + MemoryType::HEAP, + ); + guard.insert(HEAP_START_ADDR, new_vma); + *ret = HEAP_START_ADDR.as_mut_ptr(); + + return 0; + } else { + error!("Unable to create heap"); + + return -i32::from(Errno::Nomem); + } + } + } else { + // Extend an existing VMA whose `end` equals the user-supplied + // address. The caller passes the current upper bound of a region + // it owns; the kernel grows that VMA by `size` bytes — provided + // the next VMA (if any) starts far enough behind it. + let addr = VirtAddr::from_ptr(*ret); + let new_end = addr + size as u64; + + let current_task = core_scheduler().get_current_task(); + let current_task_borrowed = current_task.borrow(); + let mut guard = current_task_borrowed.vmas.write(); + + // 1. Locate the predecessor: the VMA with largest start < addr. + // For an extend request its `end` must match `addr` exactly. + let key = { + let Some((key, vma)) = guard.range(..addr).next_back() else { + return -i32::from(Errno::Inval); + }; + if vma.end != addr { + return -i32::from(Errno::Inval); + } + if !vma.prot.contains(prot_flags) { + return -i32::from(Errno::Inval); + } + *key + }; + + // 2. The extension must not run into the next VMA's start. + // `VirtAddr::new(u64::MAX)` panics on x86_64 (non-canonical), + // so model "no successor" with `Option` instead of a sentinel. + if let Some((&next_start, _)) = guard.range((Bound::Excluded(key), Bound::Unbounded)).next() + && new_end > next_start + { + return -i32::from(Errno::Nomem); + } + + // 3. In-place extension. + guard.get_mut(&key).unwrap().end = new_end; + + return 0; + } + + -i32::from(Errno::Inval) +} diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index e5d8ae802a..1033ec644c 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -14,9 +14,13 @@ use ahash::RandomState; use crossbeam_utils::Backoff; use hashbrown::{HashMap, hash_map}; use hermit_sync::*; +#[cfg(target_arch = "aarch64")] +use memory_addresses::VirtAddr; #[cfg(target_arch = "riscv64")] use riscv::register::sstatus; use timer_interrupts::TimerList; +#[cfg(all(feature = "common-os", target_arch = "x86_64"))] +use x86_64::VirtAddr; use crate::arch::kernel; use crate::arch::kernel::core_local::*; @@ -29,6 +33,8 @@ use crate::arch::kernel::{get_processor_count, interrupts}; use crate::errno::Errno; use crate::fd::{Fd, RawFd}; use crate::io; +#[cfg(feature = "common-os")] +use crate::mm::vma::VirtualMemoryArea; use crate::scheduler::task::*; pub mod task; @@ -45,6 +51,8 @@ static WAITING_TASKS: InterruptTicketMutex /// Map between Task ID and TaskHandle static TASKS: InterruptTicketMutex> = InterruptTicketMutex::new(BTreeMap::new()); +/// Count the number of spawned tasks +static SPAWN_COUNTER: AtomicU32 = AtomicU32::new(1); /// Unique identifier for a core. pub type CoreId = u32; @@ -212,6 +220,24 @@ struct NewTask { core_id: CoreId, stacks: TaskStacks, object_map: Arc>, RandomState>>>, + /// When `Some`, the new task is a user-space thread that shares the + /// given root page table with its parent process and inherits the + /// parent's `pid`. When `None`, the task is a regular kernel-mode + /// task with a fresh address space; its `pid` equals its `tid`. + #[cfg(feature = "common-os")] + thread_of: Option<(Arc, ProcessId)>, + /// Per-process TLS template, cloned from the spawning thread. Used by + /// `From` to propagate the template into the new task so that + /// any threads it spawns in turn can allocate their own TLS regions. + #[cfg(feature = "common-os")] + tls_template: Option>, + /// Per-thread TLS thread-pointer (FS.Base on x86_64, TPIDR_EL0 on + /// aarch64), already prepared by `spawn_thread` from the per-process + /// `TlsTemplate`. Zero means "do not install a thread pointer". + #[cfg(feature = "common-os")] + tls_base: u64, + #[cfg(feature = "common-os")] + vmas: Arc>>, } impl From for Task { @@ -224,7 +250,34 @@ impl From for Task { core_id, stacks, object_map, + #[cfg(feature = "common-os")] + thread_of, + #[cfg(feature = "common-os")] + tls_template, + #[cfg(feature = "common-os")] + tls_base, + #[cfg(feature = "common-os")] + vmas, } = value; + + #[cfg(feature = "common-os")] + if let Some((root_page_table, parent_pid)) = thread_of { + let mut task = Self::new_thread( + tid, + parent_pid, + core_id, + TaskStatus::Ready, + prio, + stacks, + object_map, + root_page_table, + tls_template, + vmas, + ); + task.create_user_stack_frame(func, arg, tls_base); + return task; + } + let mut task = Self::new(tid, core_id, TaskStatus::Ready, prio, stacks, object_map); task.create_stack_frame(func, arg); task @@ -251,6 +304,14 @@ impl PerCoreScheduler { core_id, stacks, object_map: core_scheduler().get_current_task_object_map(), + #[cfg(feature = "common-os")] + thread_of: None, + #[cfg(feature = "common-os")] + tls_template: None, + #[cfg(feature = "common-os")] + tls_base: 0, + #[cfg(feature = "common-os")] + vmas: Arc::new(RwSpinLock::new(BTreeMap::new())), }; // Add it to the task lists. @@ -297,6 +358,107 @@ impl PerCoreScheduler { tid } + /// Spawn a new user-space thread that shares the current task's + /// address space (root page table). + /// + /// `func` must be a valid ring-3 entry point mapped in the shared + /// address space. The new thread receives its own kernel/interrupt + /// stacks and a fresh user stack, all mapped into the shared PT via + /// the regular `TaskStacks::new` path. + #[cfg(feature = "common-os")] + pub unsafe fn spawn_thread( + func: unsafe extern "C" fn(usize), + arg: usize, + prio: Priority, + core_id: CoreId, + stack_size: usize, + ) -> TaskId { + let tid = get_tid(); + // TaskStacks::new maps into the current address space — which *is* + // the shared PT of the calling thread — so the new stacks are + // immediately visible to every thread in this process. + let stacks = TaskStacks::new(stack_size); + + let (root_page_table, object_map, tls_template, vmas, parent_pid) = { + let current = core_scheduler().get_current_task(); + let borrowed = current.borrow(); + ( + borrowed.root_page_table.clone(), + borrowed.object_map.clone(), + borrowed.tls_template.clone(), + borrowed.vmas.clone(), + borrowed.pid, + ) + }; + + // Allocate a private TLS region for the new thread from the pristine + // per-process TLS template captured at `load_application` time. Must + // run in the parent's address space (still active here in the syscall + // path) so that the new user-accessible pages get mapped into the + // shared root page table. + let tls_base = if let Some(ref template) = tls_template { + crate::arch::mm::allocate_thread_tls(template) + } else { + 0 + }; + + let new_task = NewTask { + tid, + func, + arg, + prio, + core_id, + stacks, + object_map, + thread_of: Some((root_page_table, parent_pid)), + tls_template, + tls_base, + vmas, + }; + + let wakeup = { + #[cfg(feature = "smp")] + let mut input_locked = get_scheduler_input(core_id).lock(); + WAITING_TASKS.lock().insert(tid, VecDeque::with_capacity(1)); + TASKS.lock().insert( + tid, + TaskHandle::new( + tid, + prio, + #[cfg(feature = "smp")] + core_id, + ), + ); + NO_TASKS.fetch_add(1, Ordering::SeqCst); + + #[cfg(feature = "smp")] + if core_id == core_scheduler().core_id { + let task = Rc::new(RefCell::new(Task::from(new_task))); + core_scheduler().ready_queue.push(task); + false + } else { + input_locked.new_tasks.push_back(new_task); + true + } + #[cfg(not(feature = "smp"))] + if core_id == 0 { + let task = Rc::new(RefCell::new(Task::from(new_task))); + core_scheduler().ready_queue.push(task); + false + } else { + panic!("Invalid core_id {core_id}!") + } + }; + + debug!("Creating user thread {tid} with priority {prio} on core {core_id}"); + + if wakeup { + kernel::wakeup_core(core_id); + } + + tid + } + #[cfg(feature = "newlib")] fn clone_impl(&self, func: extern "C" fn(usize), arg: usize) -> TaskId { static NEXT_CORE_ID: AtomicU32 = AtomicU32::new(1); @@ -328,6 +490,12 @@ impl PerCoreScheduler { core_id, stacks: TaskStacks::new(current_task_borrowed.stacks.get_user_stack_size()), object_map: current_task_borrowed.object_map.clone(), + #[cfg(feature = "common-os")] + thread_of: None, + #[cfg(feature = "common-os")] + tls_template: None, + #[cfg(feature = "common-os")] + tls_base: 0, }; // Add it to the task lists. @@ -445,6 +613,22 @@ impl PerCoreScheduler { without_interrupts(|| self.current_task.borrow().id) } + /// Returns the Rc> for the currently running task. + #[cfg(feature = "common-os")] + #[inline] + pub fn get_current_task(&self) -> Rc> { + self.current_task.clone() + } + + #[cfg(feature = "common-os")] + #[inline] + pub fn set_current_task_object_map( + &mut self, + object_map: Arc>, RandomState>>>, + ) { + without_interrupts(|| self.current_task.borrow_mut().object_map = object_map); + } + #[inline] pub fn get_current_task_object_map( &self, @@ -466,7 +650,10 @@ impl PerCoreScheduler { /// Creates a new map between file descriptor and their IO interface and /// clone the standard descriptors. #[cfg(feature = "common-os")] - #[cfg_attr(not(target_arch = "x86_64"), expect(dead_code))] + #[cfg_attr( + not(any(target_arch = "x86_64", target_arch = "aarch64")), + expect(dead_code) + )] pub fn recreate_objmap(&self) -> io::Result<()> { let mut map = HashMap::>, RandomState>::with_hasher( RandomState::with_seeds(0, 0, 0, 0), @@ -667,7 +854,15 @@ impl PerCoreScheduler { fn cleanup_tasks(&mut self) { // Pop the first finished task and remove it from the TASKS list, which implicitly deallocates all associated memory. while let Some(finished_task) = self.finished_tasks.pop_front() { - debug!("Cleaning up task {}", finished_task.borrow().id); + let id = finished_task.borrow().id; + drop(finished_task); + #[cfg(feature = "common-os")] + trace!( + "Cleaned up task {id} — free frames: {} KiB", + crate::mm::FrameAlloc::free_space() >> 10 + ); + #[cfg(not(all(target_arch = "x86_64", feature = "common-os")))] + debug!("Cleaned up task {id}"); } } @@ -722,7 +917,7 @@ impl PerCoreScheduler { #[inline] #[cfg(target_arch = "aarch64")] - pub fn get_last_stack_pointer(&self) -> memory_addresses::VirtAddr { + pub fn get_last_stack_pointer(&self) -> VirtAddr { self.current_task.borrow().last_stack_pointer } @@ -906,11 +1101,9 @@ pub unsafe fn spawn( stack_size: usize, selector: isize, ) -> TaskId { - static CORE_COUNTER: AtomicU32 = AtomicU32::new(1); - let core_id = if selector < 0 { // use Round Robin to schedule the cores - CORE_COUNTER.fetch_add(1, Ordering::SeqCst) % get_processor_count() + SPAWN_COUNTER.fetch_add(1, Ordering::SeqCst) % get_processor_count() } else { selector as u32 }; @@ -918,6 +1111,29 @@ pub unsafe fn spawn( unsafe { PerCoreScheduler::spawn(func, arg, prio, core_id, stack_size) } } +/// Spawn a user-space thread that shares the current task's address space. +/// +/// Used by `sys_spawn`/`sys_spawn2` under the `common-os` feature to +/// implement POSIX-style threads: the entry point `func` lives in user +/// space and the new thread executes in ring 3 against the parent +/// process's root page table. +#[cfg(feature = "common-os")] +pub unsafe fn spawn_thread( + func: unsafe extern "C" fn(usize), + arg: usize, + prio: Priority, + stack_size: usize, + selector: isize, +) -> TaskId { + let core_id = if selector < 0 { + SPAWN_COUNTER.fetch_add(1, Ordering::SeqCst) % get_processor_count() + } else { + selector as u32 + }; + + unsafe { PerCoreScheduler::spawn_thread(func, arg, prio, core_id, stack_size) } +} + #[allow(clippy::result_unit_err)] pub fn join(id: TaskId) -> Result<(), ()> { let core_scheduler = core_scheduler(); @@ -944,6 +1160,142 @@ pub fn join(id: TaskId) -> Result<(), ()> { } } +/// Fork the current task. +/// +/// Marks user pages as Copy-On-Write, duplicates the page table hierarchy, +/// copies the kernel stack, and enqueues a new child task that will resume +/// execution right after the `prepare_fork_child_stack` call returning 1. +/// +/// Returns the child's `TaskId` in the parent; the child itself sees `TaskId(0)`. +#[cfg(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + all(feature = "common-os", feature = "fork") +))] +pub unsafe fn fork() -> TaskId { + use crate::arch::kernel::prepare_fork_child_stack; + use crate::arch::mm::prepare_mem_copy_on_write; + + let core_id = SPAWN_COUNTER.fetch_add(1, Ordering::SeqCst) % get_processor_count(); + + // Mark user pages COW before copying the page table. + prepare_mem_copy_on_write(); + + let stack_size = core_scheduler() + .get_current_task() + .borrow() + .stacks + .get_user_stack_size(); + let stacks = TaskStacks::new(stack_size); + + let mut child_stack_pointer: usize = 0; + let mut child_root_page_table: usize = 0; + + // Copy the kernel stack and duplicate the page table; returns false in parent. + let is_child = unsafe { + prepare_fork_child_stack( + &raw mut child_stack_pointer, + &raw mut child_root_page_table, + stacks.get_stack_virt_addr().as_usize(), + ) + }; + + if is_child { + // We are in the child context (stack is already switched). + // Prevent the newly-allocated stacks from being dropped here — + // they are owned by the child task created in the parent. + core::mem::forget(stacks); + return TaskId::from(0); + } + + // Parent path: register the child task. + let tid = get_tid(); + let child_last_sp = VirtAddr::new(child_stack_pointer.try_into().unwrap()); + let parent_user_sp = core_scheduler() + .get_current_task() + .borrow() + .user_stack_pointer; + let parent_prio = core_scheduler().get_current_task().borrow().prio; + let parent_object_map = Arc::new(RwSpinLock::new(HashMap::< + RawFd, + Arc>, + RandomState, + >::with_hasher(RandomState::with_seeds( + 0, 0, 0, 0, + )))); + for (key, val) in core_scheduler().get_current_task_object_map().read().iter() { + parent_object_map.write().insert(*key, val.clone()); + } + let parent_tls_template = core_scheduler() + .get_current_task() + .borrow() + .tls_template + .clone(); + let parent_vmas = Arc::new(RwSpinLock::new( + core_scheduler() + .get_current_task() + .borrow() + .vmas + .read() + .clone(), + )); + + let child_task = Task::new_fork( + tid, + core_id, + TaskStatus::Ready, + parent_prio, + stacks, + child_last_sp, + parent_user_sp, + parent_object_map, + Arc::new(RootPageTable::new(child_root_page_table)), + parent_tls_template, + parent_vmas, + ); + + let wakeup = { + #[cfg(feature = "smp")] + let _input_locked = get_scheduler_input(core_id).lock(); + WAITING_TASKS.lock().insert(tid, VecDeque::with_capacity(1)); + TASKS.lock().insert( + tid, + TaskHandle::new( + tid, + parent_prio, + #[cfg(feature = "smp")] + core_id, + ), + ); + NO_TASKS.fetch_add(1, Ordering::SeqCst); + + #[cfg(feature = "smp")] + if core_id == core_scheduler().core_id { + let task = Rc::new(RefCell::new(child_task)); + core_scheduler().ready_queue.push(task); + false + } else { + // For SMP we'd need to send to the target core; for now push locally. + let task = Rc::new(RefCell::new(child_task)); + core_scheduler().ready_queue.push(task); + false + } + #[cfg(not(feature = "smp"))] + { + let task = Rc::new(RefCell::new(child_task)); + core_scheduler().ready_queue.push(task); + false + } + }; + + if wakeup { + kernel::wakeup_core(core_id); + } + + debug!("Child was created and has the id {tid}"); + + tid +} + pub fn shutdown(arg: i32) -> ! { crate::syscalls::shutdown(arg) } @@ -952,11 +1304,11 @@ fn get_task_handle(id: TaskId) -> Option { TASKS.lock().get(&id).copied() } -#[cfg(all(target_arch = "x86_64", feature = "common-os"))] +#[cfg(feature = "common-os")] pub(crate) static BOOT_ROOT_PAGE_TABLE: OnceCell = OnceCell::new(); #[cfg(all(target_arch = "x86_64", feature = "common-os"))] pub(crate) fn get_root_page_table() -> usize { let current_task_borrowed = core_scheduler().current_task.borrow_mut(); - current_task_borrowed.root_page_table + current_task_borrowed.root_page_table.as_usize() } diff --git a/src/scheduler/task/mod.rs b/src/scheduler/task/mod.rs index e5e1fe171d..06f3a43737 100644 --- a/src/scheduler/task/mod.rs +++ b/src/scheduler/task/mod.rs @@ -3,9 +3,13 @@ #[cfg(not(feature = "common-os"))] pub(crate) mod tls; +#[cfg(feature = "common-os")] +use alloc::collections::BTreeMap; use alloc::collections::{LinkedList, VecDeque}; use alloc::rc::Rc; use alloc::sync::Arc; +#[cfg(feature = "common-os")] +use alloc::vec::Vec; use core::cell::RefCell; use core::num::NonZeroU64; use core::{cmp, fmt}; @@ -13,18 +17,74 @@ use core::{cmp, fmt}; use ahash::RandomState; use crossbeam_utils::CachePadded; use hashbrown::HashMap; -use hermit_sync::{OnceCell, RwSpinLock}; +#[cfg(not(feature = "common-os"))] +use hermit_sync::OnceCell; +use hermit_sync::RwSpinLock; +#[cfg(not(target_arch = "x86_64"))] use memory_addresses::VirtAddr; +#[cfg(target_arch = "x86_64")] +use x86_64::VirtAddr; #[cfg(not(feature = "common-os"))] use self::tls::Tls; use super::timer_interrupts::{Source, create_timer_abs}; +#[cfg(feature = "common-os")] +use crate::arch; use crate::arch::kernel::core_local::*; use crate::arch::kernel::processor::{self, FPUState}; use crate::arch::kernel::scheduler::TaskStacks; -use crate::fd::{Fd, RawFd, stdio}; +#[cfg(not(feature = "common-os"))] +use crate::fd::stdio; +use crate::fd::{Fd, RawFd}; +#[cfg(feature = "common-os")] +use crate::mm::vma::VirtualMemoryArea; use crate::scheduler::CoreId; +/// A reference-counted handle to a process's root page table. +/// +/// Threads of the same process share the same `Arc`. When +/// the last owning task is dropped, the physical page-table hierarchy and +/// the user-space mappings are released. +#[cfg(feature = "common-os")] +pub struct RootPageTable { + pml4_phys: usize, + /// `false` for the boot page table, which must never be released. + owned: bool, +} + +#[cfg(feature = "common-os")] +impl RootPageTable { + /// Wraps a freshly allocated PML4 that this process owns. + pub fn new(pml4_phys: usize) -> Self { + Self { + pml4_phys, + owned: true, + } + } + + /// Wraps the boot page table, shared by all idle tasks. Dropping this + /// instance is a no-op. + pub fn new_boot(pml4_phys: usize) -> Self { + Self { + pml4_phys, + owned: false, + } + } + + pub fn as_usize(&self) -> usize { + self.pml4_phys + } +} + +#[cfg(feature = "common-os")] +impl Drop for RootPageTable { + fn drop(&mut self) { + if self.owned { + arch::mm::drop_user_space(self.pml4_phys); + } + } +} + /// Returns the most significant bit. /// /// # Examples @@ -50,26 +110,67 @@ pub(crate) enum TaskStatus { Idle, } -/// Unique identifier for a task (i.e. `pid`). +/// Unique identifier for a task (i.e. `tid`). #[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)] pub struct TaskId(i32); impl TaskId { - pub const fn into(self) -> i32 { - self.0 - } - pub const fn from(x: i32) -> Self { TaskId(x) } } +impl From for i32 { + fn from(tid: TaskId) -> Self { + tid.0 + } +} + impl fmt::Display for TaskId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0) } } +/// Process ID — shared between every thread of the same process. +/// +/// For the main thread of a process it carries the same numeric value +/// as the thread's [`TaskId`]; additional threads inherit it from the +/// spawning thread. A [`TaskId`] converts into a `ProcessId` via +/// [`From`] / [`Into`] (used by `Task::new` / `Task::new_fork` to seed +/// `pid = tid` for a newly created main thread). +#[cfg(feature = "common-os")] +#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)] +pub struct ProcessId(i32); + +#[cfg(feature = "common-os")] +impl ProcessId { + pub const fn from(x: i32) -> Self { + ProcessId(x) + } +} + +#[cfg(feature = "common-os")] +impl From for i32 { + fn from(pid: ProcessId) -> Self { + pid.0 + } +} + +#[cfg(feature = "common-os")] +impl From for ProcessId { + fn from(tid: TaskId) -> Self { + ProcessId(tid.0) + } +} + +#[cfg(feature = "common-os")] +impl fmt::Display for ProcessId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + /// Priority of a task #[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)] pub struct Priority(u8); @@ -355,6 +456,21 @@ impl PriorityTaskQueue { } } +/// Per-process TLS template used to initialize newly spawned threads. +/// +/// `size` is the byte size of the TLS data block (the offset at which the +/// TCB / thread pointer lives, also the value that the parent's main thread +/// uses for its own block). `init` is a snapshot of the freshly-loaded TLS +/// image taken right after the user binary's `PT_TLS` segment was copied +/// into place by `load_application`. Each new thread starts with a verbatim +/// copy of `init`, so it sees pristine `#[thread_local]` defaults rather +/// than mutated state from another thread. +#[cfg(feature = "common-os")] +pub(crate) struct TlsTemplate { + pub size: usize, + pub init: Vec, +} + /// A task control block, which identifies either a process or a thread #[cfg_attr(any(target_arch = "x86_64", target_arch = "aarch64"), repr(align(128)))] #[cfg_attr( @@ -364,6 +480,12 @@ impl PriorityTaskQueue { pub(crate) struct Task { /// The ID of this context pub id: TaskId, + /// Process ID — equal to `id` for the main thread of a process, and + /// inherited from the spawning thread for every additional thread of + /// the same process. After `fork()` the child's `pid` equals the + /// child's `id` (it becomes its own process). + #[cfg(feature = "common-os")] + pub pid: ProcessId, /// Status of a task, e.g. if the task is ready or blocked pub status: TaskStatus, /// Task priority, @@ -383,9 +505,18 @@ pub(crate) struct Task { /// Task Thread-Local-Storage (TLS) #[cfg(not(feature = "common-os"))] pub tls: Option, - // Physical address of the 1st level page table - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] - pub root_page_table: usize, + // Physical address of the 1st level page table, shared between all + // threads of the same process via `Arc`. The address space is freed when + // the last thread referencing this `RootPageTable` is dropped. + #[cfg(feature = "common-os")] + pub root_page_table: Arc, + /// Per-process TLS template used to allocate fresh TLS regions for new + /// threads. `None` for kernel-only tasks; set by `load_application` + /// when the user binary has a `PT_TLS` segment. + #[cfg(feature = "common-os")] + pub tls_template: Option>, + #[cfg(feature = "common-os")] + pub vmas: Arc>>, } pub(crate) trait TaskFrame { @@ -406,6 +537,8 @@ impl Task { Task { id: tid, + #[cfg(feature = "common-os")] + pid: tid.into(), status: task_status, prio: task_prio, last_stack_pointer: VirtAddr::zero(), @@ -416,19 +549,28 @@ impl Task { object_map, #[cfg(not(feature = "common-os"))] tls: None, - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] - root_page_table: crate::arch::mm::create_new_root_page_table(), + #[cfg(feature = "common-os")] + root_page_table: Arc::new(RootPageTable::new(arch::mm::create_new_root_page_table())), + #[cfg(feature = "common-os")] + tls_template: None, + #[cfg(feature = "common-os")] + vmas: Arc::new(RwSpinLock::new(BTreeMap::new())), } } pub fn new_idle(tid: TaskId, core_id: CoreId) -> Task { debug!("Creating idle task {tid}"); - /// All cores use the same mapping between file descriptor and the referenced object + /// In the unikernel case all cores share the same mapping between + /// file descriptor and the referenced object. Under `common-os`, + /// each process gets its own `object_map` when the application is + /// loaded, so the idle task only needs an empty placeholder. + #[cfg(not(feature = "common-os"))] static OBJECT_MAP: OnceCell< Arc>, RandomState>>>, > = OnceCell::new(); + #[cfg(not(feature = "common-os"))] if core_id == 0 { OBJECT_MAP .set(Arc::new(RwSpinLock::new(HashMap::< @@ -454,6 +596,8 @@ impl Task { Task { id: tid, + #[cfg(feature = "common-os")] + pid: tid.into(), status: TaskStatus::Idle, prio: IDLE_PRIO, last_stack_pointer: VirtAddr::zero(), @@ -461,20 +605,111 @@ impl Task { last_fpu_state: FPUState::new(), core_id, stacks: TaskStacks::from_boot_stacks(), + #[cfg(not(feature = "common-os"))] object_map: OBJECT_MAP.get().unwrap().clone(), + #[cfg(feature = "common-os")] + object_map: Arc::new(RwSpinLock::new(HashMap::< + RawFd, + Arc>, + RandomState, + >::with_hasher(RandomState::with_seeds( + 0, 0, 0, 0, + )))), #[cfg(not(feature = "common-os"))] tls, - #[cfg(all(target_arch = "x86_64", feature = "common-os"))] - root_page_table: *crate::scheduler::BOOT_ROOT_PAGE_TABLE.get().unwrap(), + #[cfg(feature = "common-os")] + root_page_table: Arc::new(RootPageTable::new_boot( + *crate::scheduler::BOOT_ROOT_PAGE_TABLE.get().unwrap(), + )), + #[cfg(feature = "common-os")] + tls_template: None, + #[cfg(feature = "common-os")] + vmas: Arc::new(RwSpinLock::new(BTreeMap::new())), + } + } + + /// Create a new user-space thread that shares its parent's address space. + /// + /// The `root_page_table` `Arc` is cloned from the parent, so all threads + /// of the same process drop together when the last one exits. + /// `parent_pid` is the spawning thread's `pid`; the new thread joins + /// the same process and reports the same `pid` value. + #[cfg(feature = "common-os")] + #[allow(clippy::too_many_arguments)] + pub fn new_thread( + tid: TaskId, + parent_pid: ProcessId, + core_id: CoreId, + task_status: TaskStatus, + task_prio: Priority, + stacks: TaskStacks, + object_map: Arc>, RandomState>>>, + root_page_table: Arc, + tls_template: Option>, + vmas: Arc>>, + ) -> Task { + debug!("Creating user thread {tid} (pid {parent_pid}) on core {core_id}"); + Task { + id: tid, + pid: parent_pid, + status: task_status, + prio: task_prio, + last_stack_pointer: VirtAddr::zero(), + user_stack_pointer: VirtAddr::zero(), + last_fpu_state: FPUState::new(), + core_id, + stacks, + object_map, + root_page_table, + tls_template, + vmas, + } + } + + /// Create a forked task that starts at the point where the parent called fork. + /// `last_stack_pointer` is set to the child's pre-computed stack pointer. + #[cfg(all(feature = "common-os", feature = "fork"))] + #[allow(clippy::too_many_arguments)] + pub fn new_fork( + tid: TaskId, + core_id: CoreId, + task_status: TaskStatus, + task_prio: Priority, + stacks: TaskStacks, + last_stack_pointer: VirtAddr, + user_stack_pointer: VirtAddr, + object_map: Arc>, RandomState>>>, + root_page_table: Arc, + tls_template: Option>, + vmas: Arc>>, + ) -> Task { + debug!("Creating forked task {tid} on core {core_id}"); + Task { + id: tid, + // A forked child becomes the main thread of a new process. + pid: tid.into(), + status: task_status, + prio: task_prio, + last_stack_pointer, + user_stack_pointer, + last_fpu_state: FPUState::new(), + core_id, + stacks, + object_map, + #[cfg(not(feature = "common-os"))] + tls: None, + root_page_table, + tls_template, + vmas, } } } -/*impl Drop for Task { +impl Drop for Task { fn drop(&mut self) { - debug!("Drop task {}", self.id); + //debug!("Drop task {}", self.id); } -}*/ +} struct BlockedTask { task: Rc>, diff --git a/src/syscalls/entropy.rs b/src/syscalls/entropy.rs index 6292ce597e..a2cda9b649 100644 --- a/src/syscalls/entropy.rs +++ b/src/syscalls/entropy.rs @@ -33,7 +33,7 @@ unsafe fn read_entropy(buf: *mut u8, len: usize, flags: u32) -> isize { entropy::read(buf, flags) .unwrap_or_else(|_| { - warn!("Unable to read entropy! Fallback to a naive implementation!"); + trace!("Unable to read entropy! Fallback to a naive implementation!"); for byte in &mut *buf { *byte = (generate_park_miller_lehmer_random_number() & 0xff) .try_into() @@ -53,7 +53,14 @@ unsafe fn read_entropy(buf: *mut u8, len: usize, flags: u32) -> isize { #[hermit_macro::system] #[unsafe(no_mangle)] pub unsafe extern "C" fn sys_read_entropy(buf: *mut u8, len: usize, flags: u32) -> isize { - unsafe { read_entropy(buf, len, flags) } + let ret = unsafe { read_entropy(buf, len, flags) }; + if ret >= 0 { + debug!("Read {ret} bytes with cryptographically secure random data"); + } else { + error!("Unable to read cryptographically secure random data"); + } + + ret } /// Create a cryptographicly secure 32bit random number with the support of diff --git a/src/syscalls/socket/addrinfo.rs b/src/syscalls/socket/addrinfo.rs index 8403510c4a..f52f6da83a 100644 --- a/src/syscalls/socket/addrinfo.rs +++ b/src/syscalls/socket/addrinfo.rs @@ -13,7 +13,7 @@ use super::{Af, Ipproto, Sock, SockFlags, sockaddr, sockaddrBox, sockaddrRef, so #[repr(C)] #[derive(Default)] -struct addrinfo { +pub(crate) struct addrinfo { ai_flags: Ai, ai_family: i32, ai_socktype: i32, @@ -89,7 +89,7 @@ impl Drop for addrinfo { #[derive(Default)] #[repr(transparent)] -struct addrinfoList(Option>); +pub(crate) struct addrinfoList(Option>); impl addrinfoList { fn is_empty(&self) -> bool { @@ -132,7 +132,7 @@ impl FromIterator for addrinfoList { } } -struct addrinfoIter<'a>(Option<&'a addrinfo>); +pub(crate) struct addrinfoIter<'a>(Option<&'a addrinfo>); impl<'a> Iterator for addrinfoIter<'a> { type Item = &'a addrinfo; diff --git a/src/syscalls/socket/mod.rs b/src/syscalls/socket/mod.rs index 8faae6c68c..01c10456f5 100644 --- a/src/syscalls/socket/mod.rs +++ b/src/syscalls/socket/mod.rs @@ -1,7 +1,7 @@ #![allow(dead_code)] #![allow(nonstandard_style)] -mod addrinfo; +pub(crate) mod addrinfo; use alloc::boxed::Box; use alloc::sync::Arc; diff --git a/src/syscalls/table.rs b/src/syscalls/table.rs index fcbb32b489..d0c27d332e 100644 --- a/src/syscalls/table.rs +++ b/src/syscalls/table.rs @@ -1,5 +1,11 @@ +#[cfg(target_arch = "x86_64")] use core::arch::naked_asm; +use crate::mm::vma::sys_mmap; +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +use crate::syscalls::socket::addrinfo::*; +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +use crate::syscalls::socket::*; use crate::syscalls::*; /// Number of the system call `exit` @@ -30,15 +36,144 @@ const SYSNO_OPEN: usize = 11; const SYSNO_WRITEV: usize = 12; /// Number of the system call `readv` const SYSNO_READV: usize = 13; +/// number of the system call `fork` +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] +const SYSNO_FORK: usize = 14; +/// number of the system call `waitpid` +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] +const SYSNO_WAITPID: usize = 15; +/// number of the system call `spawn_process` +const SYSNO_SPAWN_PROCESS: usize = 16; +/// number of the system call `clock_gettime` +const SYSNO_CLOCK_GETTIME: usize = 17; +/// number of the system call `spawn` +const SYSNO_SPAWN: usize = 18; +/// number of the system call `spawn2` +const SYSNO_SPAWN2: usize = 19; +/// number of the system call `join` +const SYSNO_JOIN: usize = 20; +/// number of the system call `unlink` +const SYSNO_UNLINK: usize = 21; +/// number of the system call `mkdir` +const SYSNO_MKDIR: usize = 22; +/// number of the system call `rmdir` +const SYSNO_RMDIR: usize = 23; +/// number of the system call `stat` +const SYSNO_STAT: usize = 24; +/// number of the system call `lstat` +const SYSNO_LSTAT: usize = 25; +/// number of the system call `fstat` +const SYSNO_FSTAT: usize = 26; +/// number of the system call `dup` +const SYSNO_DUP: usize = 27; +/// number of the system call `ioctl` +const SYSNO_IOCTL: usize = 28; +/// number of the system call `poll` +const SYSNO_POLL: usize = 29; +/// number of the system call `notify` +const SYSNO_NOTIFY: usize = 30; +/// number of the system call `add_queue` +const SYSNO_ADD_QUEUE: usize = 31; +/// number of the system call `wait` +const SYSNO_WAIT: usize = 32; +/// number of the system call `init_queue` +const SYSNO_INIT_QUEUE: usize = 33; +/// number of the system call `destroy_queue` +const SYSNO_DESTROY_QUEUE: usize = 34; +/// number of the system call `block_current_task` +const SYSNO_BLOCK_CURRENT_TASK: usize = 35; +/// number of the system call `block_current_task_with_timeout` +const SYSNO_BLOCK_CURRENT_TASK_WITH_TIMEOUT: usize = 36; +/// number of the system call `wakeup_task` +const SYSNO_WAKEUP_TASK: usize = 37; +/// number of the system call `socket` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_SOCKET: usize = 38; +/// number of the system call `bind` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_BIND: usize = 39; +/// number of the system call `listen` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_LISTEN: usize = 40; +/// number of the system call `accept` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_ACCEPT: usize = 41; +/// number of the system call `connect` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_CONNECT: usize = 42; +/// number of the system call `recv` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_RECV: usize = 43; +/// number of the system call `recvfrom` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_RECVFROM: usize = 44; +/// number of the system call `send` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_SEND: usize = 45; +/// number of the system call `sendto` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_SENDTO: usize = 46; +/// number of the system call `shutdown` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_SHUTDOWN: usize = 47; +/// number of the system call `getpeername` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_GETPEERNAME: usize = 48; +/// number of the system call `getsockname` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_GETSOCKNAME: usize = 49; +/// number of the system call `getsockopt` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_GETSOCKOPT: usize = 50; +/// number of the system call `setsockopt` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_SETSOCKOPT: usize = 51; +/// number of the system call `getaddrinfo` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_GETADDRINFO: usize = 52; +/// number of the system call `freeaddrinfo` +#[cfg(any(feature = "net", feature = "virtio-vsock"))] +const SYSNO_FREEADDRINFO: usize = 53; +/// number of the system call `available_parallelism` +const SYSNO_AVAILABLE_PARALLELISM: usize = 54; +/// number of the system call `getdents64` +const SYSNO_GET_DENTS64: usize = 55; +/// number of the system call `exec` +const SYSNO_EXEC: usize = 56; +/// number of the system call `mmap` +const SYSNO_MMAP: usize = 57; /// Total number of system calls -const NO_SYSCALLS: usize = 32; +pub(crate) const NO_SYSCALLS: usize = 64; -extern "C" fn invalid_syscall(sys_no: u64) -> ! { +pub(crate) extern "C" fn invalid_syscall(sys_no: u64) -> ! { error!("Invalid syscall {sys_no}"); sys_exit(1); } +/// loader will replace this function +#[linkage = "weak"] +#[unsafe(no_mangle)] +pub extern "C" fn sys_spawn_process( + _path: *const c_char, + _argv: *const *const c_char, + _envp: *const *const c_char, +) -> i32 { + -i32::from(Errno::Nosys) +} + +/// loader will replace this function +#[linkage = "weak"] +#[unsafe(no_mangle)] +pub extern "C" fn sys_exec( + _path: *const c_char, + _argv: *const *const c_char, + _envp: *const *const c_char, +) -> i32 { + -i32::from(Errno::Nosys) +} + +#[cfg(target_arch = "x86_64")] #[allow(unused_assignments)] #[unsafe(no_mangle)] #[unsafe(naked)] @@ -50,6 +185,17 @@ pub(crate) unsafe extern "C" fn sys_invalid() { ); } +/// Sentinel placeholder for unregistered syscall slots on aarch64. +/// +/// The aarch64 dispatcher in `do_sync` compares the table entry against +/// this function pointer before invoking; if it matches, it bails out +/// with `invalid_syscall(nr)` — the body here is therefore never executed. +/// The empty `extern "C"` body still gives us a stable, no_mangle symbol +/// whose address we can compare against. +#[cfg(target_arch = "aarch64")] +#[unsafe(no_mangle)] +pub(crate) extern "C" fn sys_invalid() {} + #[repr(align(64))] #[repr(C)] pub(crate) struct SyscallTable { @@ -76,11 +222,70 @@ impl SyscallTable { table.handle[SYSNO_OPEN] = sys_open as *const _; table.handle[SYSNO_READV] = sys_readv as *const _; table.handle[SYSNO_WRITEV] = sys_writev as *const _; + #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] + { + table.handle[SYSNO_FORK] = sys_fork as *const _; + table.handle[SYSNO_WAITPID] = sys_waitpid as *const _; + } + table.handle[SYSNO_SPAWN_PROCESS] = sys_spawn_process as *const _; + table.handle[SYSNO_CLOCK_GETTIME] = sys_clock_gettime as *const _; + table.handle[SYSNO_SPAWN] = sys_spawn as *const _; + table.handle[SYSNO_SPAWN2] = sys_spawn2 as *const _; + table.handle[SYSNO_JOIN] = sys_join as *const _; + table.handle[SYSNO_UNLINK] = sys_unlink as *const _; + table.handle[SYSNO_MKDIR] = sys_mkdir as *const _; + table.handle[SYSNO_RMDIR] = sys_rmdir as *const _; + table.handle[SYSNO_STAT] = sys_stat as *const _; + table.handle[SYSNO_LSTAT] = sys_lstat as *const _; + table.handle[SYSNO_FSTAT] = sys_fstat as *const _; + table.handle[SYSNO_DUP] = sys_dup as *const _; + table.handle[SYSNO_IOCTL] = sys_ioctl as *const _; + table.handle[SYSNO_POLL] = sys_poll as *const _; + table.handle[SYSNO_NOTIFY] = sys_notify as *const _; + table.handle[SYSNO_ADD_QUEUE] = sys_add_queue as *const _; + table.handle[SYSNO_WAIT] = sys_wait as *const _; + table.handle[SYSNO_INIT_QUEUE] = sys_init_queue as *const _; + table.handle[SYSNO_DESTROY_QUEUE] = sys_destroy_queue as *const _; + table.handle[SYSNO_BLOCK_CURRENT_TASK] = sys_block_current_task as *const _; + table.handle[SYSNO_BLOCK_CURRENT_TASK_WITH_TIMEOUT] = + sys_block_current_task_with_timeout as *const _; + table.handle[SYSNO_WAKEUP_TASK] = sys_wakeup_task as *const _; + #[cfg(any(feature = "net", feature = "virtio-vsock"))] + { + table.handle[SYSNO_SOCKET] = sys_socket as *const _; + table.handle[SYSNO_BIND] = sys_bind as *const _; + table.handle[SYSNO_LISTEN] = sys_listen as *const _; + table.handle[SYSNO_ACCEPT] = sys_accept as *const _; + table.handle[SYSNO_CONNECT] = sys_connect as *const _; + table.handle[SYSNO_RECV] = sys_recv as *const _; + table.handle[SYSNO_RECVFROM] = sys_recvfrom as *const _; + table.handle[SYSNO_SEND] = sys_send as *const _; + table.handle[SYSNO_SENDTO] = sys_sendto as *const _; + table.handle[SYSNO_SHUTDOWN] = sys_shutdown as *const _; + table.handle[SYSNO_GETPEERNAME] = sys_getpeername as *const _; + table.handle[SYSNO_GETSOCKNAME] = sys_getsockname as *const _; + table.handle[SYSNO_GETSOCKOPT] = sys_getsockopt as *const _; + table.handle[SYSNO_SETSOCKOPT] = sys_setsockopt as *const _; + table.handle[SYSNO_GETADDRINFO] = sys_getaddrinfo as *const _; + table.handle[SYSNO_FREEADDRINFO] = sys_freeaddrinfo as *const _; + } + table.handle[SYSNO_AVAILABLE_PARALLELISM] = sys_available_parallelism as *const _; + table.handle[SYSNO_GET_DENTS64] = sys_getdents64 as *const _; + table.handle[SYSNO_EXEC] = sys_exec as *const _; + table.handle[SYSNO_MMAP] = sys_mmap as *const _; table } } +impl SyscallTable { + #[cfg(target_arch = "aarch64")] + #[inline] + pub(crate) fn handler(&self, nr: usize) -> *const usize { + self.handle[nr] + } +} + unsafe impl Send for SyscallTable {} unsafe impl Sync for SyscallTable {} diff --git a/src/syscalls/tasks.rs b/src/syscalls/tasks.rs index bd2d31e73f..bf221a79b9 100644 --- a/src/syscalls/tasks.rs +++ b/src/syscalls/tasks.rs @@ -14,11 +14,77 @@ use crate::{arch, scheduler}; #[cfg(feature = "newlib")] pub type SignalHandler = extern "C" fn(i32); pub type Tid = i32; +pub type Pid = i32; + +/// Fork the current process. +/// Returns the child's PID to the parent, and 0 to the child. +#[cfg(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + feature = "common-os" +))] +#[hermit_macro::system(errno)] +#[unsafe(no_mangle)] +pub extern "C" fn sys_fork() -> i32 { + #[cfg(feature = "fork")] + unsafe { + scheduler::fork().into() + } + + #[cfg(not(feature = "fork"))] + { + -i32::from(Errno::Nosys) + } +} + +/// Fork the current process. +/// In case of a unikernel, this system call always fail. +#[cfg(not(feature = "common-os"))] +#[hermit_macro::system(errno)] +#[unsafe(no_mangle)] +pub extern "C" fn sys_fork() -> i32 { + -i32::from(Errno::Nosys) +} + +/// Waitpid block the current process until termination of process `pid`. +/// Returns 0 is the process terminates +#[cfg(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + feature = "common-os" +))] +#[hermit_macro::system(errno)] +#[unsafe(no_mangle)] +pub extern "C" fn sys_waitpid(pid: Pid) -> i32 { + match scheduler::join(TaskId::from(pid)) { + Ok(()) => 0, + _ => -i32::from(Errno::Inval), + } +} + +/// Waitpid block the current process until termination of process `pid`. +/// In case of a unikernel, this system call always fail. +#[cfg(not(feature = "common-os"))] +#[hermit_macro::system(errno)] +#[unsafe(no_mangle)] +pub extern "C" fn sys_waitpid(_pid: Pid) -> i32 { + -i32::from(Errno::Nosys) +} #[hermit_macro::system] #[unsafe(no_mangle)] -pub extern "C" fn sys_getpid() -> Tid { - 0 +pub extern "C" fn sys_getpid() -> Pid { + #[cfg(not(feature = "common-os"))] + { + // an unikernel doesn't have a pid => return always 0 + 0 + } + + #[cfg(feature = "common-os")] + { + // Return the process ID — equal for every thread of the same + // process, set by `Task::new_thread` from the spawning thread's + // `pid` and reset to the new `tid` on fork. + core_scheduler().get_current_task().borrow().pid.into() + } } #[cfg(feature = "newlib")] @@ -43,7 +109,11 @@ pub unsafe extern "C" fn sys_setprio(_id: *const Tid, _prio: i32) -> i32 { fn exit(arg: i32) -> ! { debug!("Exit program with error code {arg}!"); - super::shutdown(arg) + if cfg!(not(feature = "common-os")) { + super::shutdown(arg) + } else { + core_scheduler().exit(arg) + } } #[hermit_macro::system] @@ -132,7 +202,7 @@ pub unsafe extern "C" fn sys_clone(id: *mut Tid, func: extern "C" fn(usize), arg 0 } -#[hermit_macro::system(errno)] +#[hermit_macro::system] #[unsafe(no_mangle)] pub extern "C" fn sys_yield() { core_scheduler().reschedule(); @@ -163,7 +233,22 @@ pub unsafe extern "C" fn sys_spawn2( stack_size: usize, selector: isize, ) -> Tid { - unsafe { scheduler::spawn(func, arg, Priority::from(prio), stack_size, selector).into() } + #[cfg(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + feature = "common-os" + ))] + { + unsafe { + scheduler::spawn_thread(func, arg, Priority::from(prio), stack_size, selector).into() + } + } + #[cfg(not(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + feature = "common-os" + )))] + { + unsafe { scheduler::spawn(func, arg, Priority::from(prio), stack_size, selector).into() } + } } #[hermit_macro::system] @@ -175,8 +260,22 @@ pub unsafe extern "C" fn sys_spawn( prio: u8, selector: isize, ) -> i32 { - let new_id = unsafe { - scheduler::spawn(func, arg, Priority::from(prio), USER_STACK_SIZE, selector).into() + let new_id = { + #[cfg(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + feature = "common-os" + ))] + unsafe { + scheduler::spawn_thread(func, arg, Priority::from(prio), USER_STACK_SIZE, selector) + .into() + } + #[cfg(not(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + feature = "common-os" + )))] + unsafe { + scheduler::spawn(func, arg, Priority::from(prio), USER_STACK_SIZE, selector).into() + } }; if !id.is_null() { diff --git a/typos.toml b/typos.toml index c283906f15..b1d26e8f31 100644 --- a/typos.toml +++ b/typos.toml @@ -5,6 +5,7 @@ extend-ignore-identifiers-re = [ "ist", "parm", "sie", + "Spectre", ] [default.extend-identifiers]