From 424b449a63a0596c547a379d34da8fd1a318f7cd Mon Sep 17 00:00:00 2001 From: Daniel-De-Dev Date: Sat, 26 Sep 2026 18:16:05 +0200 Subject: [PATCH 1/9] feat: added helpers for temporary frame modifying Since the frame allocator will return addresses for physical memory, a helper is added so the kernel can later temporary make a physical address mapped/accessible. (It's mapped to the last possible gigapage on the higher half virtual address space), then performs the operation that is provided. --- kernel/src/arch/riscv64/paging.rs | 167 ++++++++++++++++++++++++++++-- kernel/src/memory.rs | 2 +- 2 files changed, 157 insertions(+), 12 deletions(-) diff --git a/kernel/src/arch/riscv64/paging.rs b/kernel/src/arch/riscv64/paging.rs index f9e52d2..a9c3736 100644 --- a/kernel/src/arch/riscv64/paging.rs +++ b/kernel/src/arch/riscv64/paging.rs @@ -5,11 +5,14 @@ use core::{arch::asm, ptr}; -use crate::memory::{PhysAddr, VirtAddr}; +use crate::memory::{PhysAddr, PhysFrame, VirtAddr}; /// Lowest canonical virtual address in the upper half of Sv39. const SV39_HIGH_HALF_BASE: usize = 0xffff_ffc0_0000_0000; +/// First root-table index belonging to the canonical Sv39 upper half. +const SV39_HIGH_HALF_VPN2: usize = (SV39_HIGH_HALF_BASE >> GIGAPAGE_SHIFT) & VPN_MASK; + /// Number of low address bits forming a page offset. const PAGE_SHIFT: usize = 12; @@ -37,9 +40,15 @@ const PTE_VALID: usize = 1 << 0; /// PTE bit permitting reads through a leaf mapping. const PTE_READ: usize = 1 << 1; +/// PTE bit permitting writes through a leaf mapping. +const PTE_WRITE: usize = 1 << 2; + /// PTE bit recording that a leaf mapping has been accessed. const PTE_ACCESSED: usize = 1 << 6; +/// PTE bit recording that a writable leaf mapping has been modified. +const PTE_DIRTY: usize = 1 << 7; + /// Flags for a read-only boot-time FDT leaf mapping. const BOOT_FDT_PTE_FLAGS: usize = PTE_VALID | PTE_READ | PTE_ACCESSED; @@ -47,7 +56,7 @@ const BOOT_FDT_PTE_FLAGS: usize = PTE_VALID | PTE_READ | PTE_ACCESSED; const BOOT_FDT_WINDOW_BASE: usize = SV39_HIGH_HALF_BASE; /// Root-table index containing the start of the boot-time FDT window. -const BOOT_FDT_WINDOW_VPN2: usize = (BOOT_FDT_WINDOW_BASE >> GIGAPAGE_SHIFT) & VPN_MASK; +const BOOT_FDT_WINDOW_VPN2: usize = SV39_HIGH_HALF_VPN2; /// Number of consecutive level-2 leaves available to the boot-time FDT window. /// @@ -55,6 +64,20 @@ const BOOT_FDT_WINDOW_VPN2: usize = (BOOT_FDT_WINDOW_BASE >> GIGAPAGE_SHIFT) & V /// end of its first 1 GiB region. const BOOT_FDT_WINDOW_GIGAPAGES: usize = 5; +/// Root-table index reserved for temporary access to physical frames. +/// +/// VPN[2] 511 corresponds to the final 1 GiB of the canonical Sv39 +/// higher-half address space. The bootstrap kernel occupies VPN[2] 510, +/// while the temporary FDT window begins at VPN[2] 256. +const BOOT_FRAME_WINDOW_VPN2: usize = VPN_MASK; + +/// First virtual address of the temporary physical-frame access window. +const BOOT_FRAME_WINDOW_BASE: usize = + SV39_HIGH_HALF_BASE + ((BOOT_FRAME_WINDOW_VPN2 - SV39_HIGH_HALF_VPN2) * GIGAPAGE_SIZE); + +/// Flags used by the temporary physical-frame mapping. +const BOOT_FRAME_PTE_FLAGS: usize = PTE_VALID | PTE_READ | PTE_WRITE | PTE_ACCESSED | PTE_DIRTY; + /// Bit position of `satp.MODE` on RV64. const SATP_MODE_SHIFT: usize = 60; @@ -134,6 +157,135 @@ fn active_root_table() -> Result { Ok(PhysAddr::new(root_ppn << PAGE_SHIFT)) } +/// Executes `operation` while `frame` is temporarily accessible through the +/// bootstrap physical-frame window. +/// +/// The temporary mapping uses a single Sv39 level-2 leaf. The physical 1 GiB +/// region containing `frame` is mapped at [`BOOT_FRAME_WINDOW_BASE`], and the +/// returned virtual address preserves the frame's offset within that region. +/// +/// The mapping exists only while `operation` executes and is removed before +/// this function returns. +/// +/// The active root page table must itself still be accessible through the +/// bootstrap identity mapping. +/// +/// # Errors +/// +/// Returns [`PagingError::UnexpectedAddressTranslationMode`] if Sv39 is not +/// active, [`PagingError::RootEntryInUse`] if the reserved temporary root entry +/// is already occupied, or +/// [`PagingError::PhysicalAddressNotRepresentable`] if the frame cannot be +/// represented by Sv39. +#[expect( + clippy::arithmetic_side_effects, + reason = "the root index and frame offset are bounded by the Sv39 page-table and gigapage sizes" +)] +fn with_bootstrap_frame( + frame: PhysFrame, + operation: impl FnOnce(VirtAddr) -> R, +) -> Result { + let boot_root = active_root_table()?; + + let entry_offset = BOOT_FRAME_WINDOW_VPN2 * PTE_SIZE; + + let entry_address = PhysAddr::new(boot_root.as_usize() + entry_offset); + + let entry_pointer = ptr::with_exposed_provenance_mut::(entry_address.as_usize()); + + // SAFETY: + // The bootstrap root page table is contained inside the physical kernel + // gigapage, which remains identity-mapped while this helper is used. + let existing = unsafe { ptr::read_volatile(entry_pointer) }; + + if existing != 0 { + return Err(PagingError::RootEntryInUse { + index: BOOT_FRAME_WINDOW_VPN2, + entry: existing, + }); + } + + let frame_address = frame.start_address(); + + if frame_address.as_usize() > MAX_PHYSICAL_ADDRESS { + return Err(PagingError::PhysicalAddressNotRepresentable { + address: frame_address, + }); + } + + let physical_base = PhysAddr::new(frame_address.as_usize() & !GIGAPAGE_MASK); + + let entry = leaf_entry(physical_base, BOOT_FRAME_PTE_FLAGS)?; + + // SAFETY: + // The reserved scratch root entry was verified to be empty and the active + // bootstrap root remains identity-mapped. + unsafe { + ptr::write_volatile(entry_pointer, entry); + } + + let frame_offset = frame_address.as_usize() & GIGAPAGE_MASK; + + let virtual_address = VirtAddr::new(BOOT_FRAME_WINDOW_BASE + frame_offset); + + flush_address(virtual_address); + + let result = operation(virtual_address); + + // SAFETY: + // This is the same bootstrap root entry installed above, and no other code + // may use the reserved scratch entry while this function executes. + unsafe { + ptr::write_volatile(entry_pointer, 0); + } + + flush_address(virtual_address); + + Ok(result) +} + +/// Constructs an Sv39 leaf page-table entry mapping `physical_address`. +/// +/// `flags` must contain the desired Sv39 leaf permissions. +/// +/// # Errors +/// +/// Returns [`PagingError::PhysicalAddressNotRepresentable`] if +/// `physical_address` cannot be encoded in an Sv39 PTE. +const fn leaf_entry(physical_address: PhysAddr, flags: usize) -> Result { + if physical_address.as_usize() > MAX_PHYSICAL_ADDRESS { + return Err(PagingError::PhysicalAddressNotRepresentable { + address: physical_address, + }); + } + + let physical_page_number = physical_address.as_usize() >> PAGE_SHIFT; + + Ok((physical_page_number << PTE_PPN_SHIFT) | flags) +} + +/// Invalidates the current hart's cached translation for `address`. +fn flush_address(address: VirtAddr) { + // SAFETY: + // `sfence.vma` only affects address-translation state on the current hart. + unsafe { + asm!( + "sfence.vma {address}, zero", + address = in(reg) address.as_usize(), + options(nostack), + ); + } +} + +/// Invalidates all cached address translations on the current hart. +fn flush_all() { + // SAFETY: + // `sfence.vma` only affects address-translation state on the current hart. + unsafe { + asm!("sfence.vma zero, zero", options(nostack)); + } +} + /// Maps the physical region containing `dtb` into a higher-half boot window. /// /// The mapping preserves `dtb`'s offset within its first 1 GiB physical region, @@ -214,9 +366,7 @@ pub(crate) fn map_bootstrap_fdt(dtb: PhysAddr) -> Result let entry_address = PhysAddr::new(boot_root.as_usize() + entry_offset); - let physical_page_number = physical_address.as_usize() >> PAGE_SHIFT; - - let entry = (physical_page_number << PTE_PPN_SHIFT) | BOOT_FDT_PTE_FLAGS; + let entry = leaf_entry(physical_address, BOOT_FDT_PTE_FLAGS)?; let entry_pointer = ptr::with_exposed_provenance_mut::(entry_address.as_usize()); @@ -228,12 +378,7 @@ pub(crate) fn map_bootstrap_fdt(dtb: PhysAddr) -> Result } } - // SAFETY: - // `sfence.vma` makes the preceding PTE updates visible to address translation - // on the current hart. - unsafe { - asm!("sfence.vma zero, zero", options(nostack)); - } + flush_all(); let dtb_offset = dtb.as_usize() & GIGAPAGE_MASK; diff --git a/kernel/src/memory.rs b/kernel/src/memory.rs index db8caf5..f8d4609 100644 --- a/kernel/src/memory.rs +++ b/kernel/src/memory.rs @@ -12,7 +12,7 @@ mod address; mod frame; pub(crate) use address::{PhysAddr, VirtAddr}; -pub(crate) use frame::BootFrameAllocator; +pub(crate) use frame::{BootFrameAllocator, PhysFrame}; /// A non-empty half-open physical address range `[start, end)`. /// From 0094c143ebf4ee52032431ac023c866e37028de5 Mon Sep 17 00:00:00 2001 From: Daniel-De-Dev Date: Sat, 26 Sep 2026 20:12:50 +0200 Subject: [PATCH 2/9] feat: added a way to map virtual addresses to physical frames Implemented a way to have a virtual address mapped to a physical frame. The method will deal with allocating and writing the page table entries and allocate tables when necessary. When needing to write to the physical memory for the physical page tables, the with_bootstrap_frame is used to have that table mapped and actually accessible. --- kernel/src/arch/riscv64/paging.rs | 305 +++++++++++++++++++++++++++++- 1 file changed, 298 insertions(+), 7 deletions(-) diff --git a/kernel/src/arch/riscv64/paging.rs b/kernel/src/arch/riscv64/paging.rs index a9c3736..abf01e2 100644 --- a/kernel/src/arch/riscv64/paging.rs +++ b/kernel/src/arch/riscv64/paging.rs @@ -5,17 +5,35 @@ use core::{arch::asm, ptr}; -use crate::memory::{PhysAddr, PhysFrame, VirtAddr}; +use crate::memory::{BootFrameAllocator, PhysAddr, PhysFrame, VirtAddr}; + +/// Sign bit of an Sv39 virtual address. +const SV39_SIGN_BIT: usize = 38; /// Lowest canonical virtual address in the upper half of Sv39. -const SV39_HIGH_HALF_BASE: usize = 0xffff_ffc0_0000_0000; +const SV39_HIGH_HALF_BASE: usize = usize::MAX << SV39_SIGN_BIT; /// First root-table index belonging to the canonical Sv39 upper half. const SV39_HIGH_HALF_VPN2: usize = (SV39_HIGH_HALF_BASE >> GIGAPAGE_SHIFT) & VPN_MASK; +/// First virtual address outside the canonical Sv39 lower half. +const SV39_LOW_HALF_END: usize = 1usize << SV39_SIGN_BIT; + /// Number of low address bits forming a page offset. const PAGE_SHIFT: usize = 12; +/// Size in bytes of one Sv39 page. +const PAGE_SIZE: usize = 1 << PAGE_SHIFT; + +/// Mask selecting an offset within one Sv39 page. +const PAGE_MASK: usize = PAGE_SIZE - 1; + +/// Bit offset of an Sv39 level-1 virtual page number. +const MEGAPAGE_SHIFT: usize = 21; + +/// Number of entries in one Sv39 page table. +const PAGE_TABLE_ENTRIES: usize = 512; + /// Bit offset of an Sv39 level-2 virtual page number. const GIGAPAGE_SHIFT: usize = 30; @@ -43,12 +61,18 @@ const PTE_READ: usize = 1 << 1; /// PTE bit permitting writes through a leaf mapping. const PTE_WRITE: usize = 1 << 2; +/// PTE bit permitting instruction fetches through a leaf mapping. +const PTE_EXECUTE: usize = 1 << 3; + /// PTE bit recording that a leaf mapping has been accessed. const PTE_ACCESSED: usize = 1 << 6; /// PTE bit recording that a writable leaf mapping has been modified. const PTE_DIRTY: usize = 1 << 7; +/// Bits distinguishing an Sv39 leaf from a non-leaf entry. +const PTE_LEAF_MASK: usize = PTE_READ | PTE_WRITE | PTE_EXECUTE; + /// Flags for a read-only boot-time FDT leaf mapping. const BOOT_FDT_PTE_FLAGS: usize = PTE_VALID | PTE_READ | PTE_ACCESSED; @@ -99,6 +123,94 @@ const PHYSICAL_ADDRESS_BITS: usize = PHYSICAL_PAGE_NUMBER_BITS + PAGE_SHIFT; /// Largest physical byte address representable by Sv39. const MAX_PHYSICAL_ADDRESS: usize = (1usize << PHYSICAL_ADDRESS_BITS) - 1; +/// One 4 KiB Sv39 page table containing 512 64-bit entries. +#[repr(C, align(4096))] +struct PageTable { + /// Raw Sv39 page-table entries. + entries: [usize; PAGE_TABLE_ENTRIES], +} + +/// An Sv39 address space being constructed by the kernel. +struct Sv39PageTable { + /// Physical frame containing the level-2 root table. + root: PhysFrame, +} + +impl Sv39PageTable { + /// Allocates a new empty Sv39 address space. + /// + /// # Errors + /// + /// Returns an error if no root frame can be allocated or initialized. + fn new(frames: &mut BootFrameAllocator<'_, '_>) -> Result { + let root = allocate_page_table(frames)?; + + Ok(Self { root }) + } + + /// Maps one 4 KiB virtual page to `physical_frame`. + /// + /// `permissions` contains the Sv39 `R`, `W`, and `X` bits for the leaf. + /// Valid/accessed state is added automatically, as is the dirty bit for + /// writable mappings. + /// + /// # Errors + /// + /// Returns an error if the virtual address is invalid or unaligned, physical + /// frames required for intermediate tables cannot be allocated, an existing + /// intermediate entry is malformed, or the virtual page is already mapped. + fn map_page( + &self, + virtual_address: VirtAddr, + physical_frame: PhysFrame, + permissions: usize, + frames: &mut BootFrameAllocator<'_, '_>, + ) -> Result<(), PagingError> { + let address = virtual_address.as_usize(); + + if address & PAGE_MASK != 0 { + return Err(PagingError::VirtualAddressNotPageAligned { address }); + } + + if address >= SV39_LOW_HALF_END && address < SV39_HIGH_HALF_BASE { + return Err(PagingError::VirtualAddressNotCanonical { address }); + } + + if permissions & !PTE_LEAF_MASK != 0 + || permissions & PTE_LEAF_MASK == 0 + || permissions & PTE_WRITE != 0 && permissions & PTE_READ == 0 + { + return Err(PagingError::InvalidLeafPermissions { permissions }); + } + + let vpn2 = (address >> GIGAPAGE_SHIFT) & VPN_MASK; + let vpn1 = (address >> MEGAPAGE_SHIFT) & VPN_MASK; + let vpn0 = (address >> PAGE_SHIFT) & VPN_MASK; + + let level_1 = next_table(self.root, vpn2, frames)?; + let level_0 = next_table(level_1, vpn1, frames)?; + + let existing = read_page_table_entry(level_0, vpn0)?; + + if existing != 0 { + return Err(PagingError::PageAlreadyMapped { + address, + entry: existing, + }); + } + + let mut flags = PTE_VALID | PTE_ACCESSED | permissions; + + if permissions & PTE_WRITE != 0 { + flags |= PTE_DIRTY; + } + + let entry = page_table_entry(physical_frame.start_address(), flags)?; + + write_page_table_entry(level_0, vpn0, entry) + } +} + /// Errors produced while inspecting or modifying Sv39 paging state. #[derive(Debug)] pub(crate) enum PagingError { @@ -122,6 +234,52 @@ pub(crate) enum PagingError { /// Physical address outside the Sv39 representable range. address: PhysAddr, }, + + /// No physical frame remained for a required page table. + OutOfPhysicalFrames, + + /// A virtual address supplied for a 4 KiB mapping was not page-aligned. + VirtualAddressNotPageAligned { + /// Unaligned virtual address. + address: usize, + }, + + /// A virtual address is not canonical under Sv39. + VirtualAddressNotCanonical { + /// Non-canonical virtual address. + address: usize, + }, + + /// A page-table entry expected to point to the next level was malformed or + /// was already a leaf mapping. + InvalidIntermediateEntry { + /// Index containing the unexpected entry. + index: usize, + + /// Raw unexpected entry. + entry: usize, + }, + + /// A requested leaf virtual page already had a mapping. + PageAlreadyMapped { + /// Virtual page that was already mapped. + address: usize, + + /// Existing raw leaf entry. + entry: usize, + }, + + /// A page-table index was outside the 512-entry table. + PageTableIndexOutOfRange { + /// Invalid entry index. + index: usize, + }, + + /// The requested Sv39 leaf permissions were invalid. + InvalidLeafPermissions { + /// Requested raw R/W/X permission bits. + permissions: usize, + }, } /// Returns the physical address of the active Sv39 root page table. @@ -215,7 +373,7 @@ fn with_bootstrap_frame( let physical_base = PhysAddr::new(frame_address.as_usize() & !GIGAPAGE_MASK); - let entry = leaf_entry(physical_base, BOOT_FRAME_PTE_FLAGS)?; + let entry = page_table_entry(physical_base, BOOT_FRAME_PTE_FLAGS)?; // SAFETY: // The reserved scratch root entry was verified to be empty and the active @@ -244,15 +402,15 @@ fn with_bootstrap_frame( Ok(result) } -/// Constructs an Sv39 leaf page-table entry mapping `physical_address`. +/// Constructs an Sv39 page-table entry pointing at `physical_address`. /// -/// `flags` must contain the desired Sv39 leaf permissions. +/// `flags` contains the desired Sv39 PTE flag bits. /// /// # Errors /// /// Returns [`PagingError::PhysicalAddressNotRepresentable`] if /// `physical_address` cannot be encoded in an Sv39 PTE. -const fn leaf_entry(physical_address: PhysAddr, flags: usize) -> Result { +const fn page_table_entry(physical_address: PhysAddr, flags: usize) -> Result { if physical_address.as_usize() > MAX_PHYSICAL_ADDRESS { return Err(PagingError::PhysicalAddressNotRepresentable { address: physical_address, @@ -264,6 +422,139 @@ const fn leaf_entry(physical_address: PhysAddr, flags: usize) -> Result Result<(), PagingError> { + with_bootstrap_frame(frame, |address| { + let table_pointer = ptr::with_exposed_provenance_mut::(address.as_usize()); + + // SAFETY: + // `address` is a temporary mapping of the complete page-aligned physical + // frame, and an all-zero Sv39 page table contains only invalid entries. + unsafe { + ptr::write_bytes(table_pointer, 0, 1); + } + }) +} + +/// Reads one entry from a physical page-table frame. +/// +/// # Errors +/// +/// Returns an error if the frame cannot be temporarily mapped or `index` is +/// outside the page table. +fn read_page_table_entry(frame: PhysFrame, index: usize) -> Result { + with_bootstrap_frame(frame, |address| { + let table_pointer = ptr::with_exposed_provenance::(address.as_usize()); + + // SAFETY: + // The temporary mapping covers the complete physical page-table frame and + // remains valid for the duration of this closure. + let table = unsafe { &*table_pointer }; + + let Some(entry) = table.entries.get(index) else { + return Err(PagingError::PageTableIndexOutOfRange { index }); + }; + + Ok(*entry) + })? +} + +/// Writes one entry in a physical page-table frame. +/// +/// # Errors +/// +/// Returns an error if the frame cannot be temporarily mapped or `index` is +/// outside the page table. +fn write_page_table_entry(frame: PhysFrame, index: usize, entry: usize) -> Result<(), PagingError> { + with_bootstrap_frame(frame, |address| { + let table_pointer = ptr::with_exposed_provenance_mut::(address.as_usize()); + + // SAFETY: + // The temporary mapping covers the complete physical page-table frame and + // remains valid exclusively for the duration of this closure. + let table = unsafe { &mut *table_pointer }; + + let Some(slot) = table.entries.get_mut(index) else { + return Err(PagingError::PageTableIndexOutOfRange { index }); + }; + + *slot = entry; + + Ok(()) + })? +} + +/// Allocates and initializes one empty physical page-table frame. +/// +/// # Errors +/// +/// Returns [`PagingError::OutOfPhysicalFrames`] if no frame remains, or an +/// error if the allocated frame cannot be initialized through the temporary +/// mapping window. +fn allocate_page_table(frames: &mut BootFrameAllocator<'_, '_>) -> Result { + let Some(frame) = frames.allocate() else { + return Err(PagingError::OutOfPhysicalFrames); + }; + + zero_page_table(frame)?; + + Ok(frame) +} + +/// Returns the page-table frame referenced by a valid non-leaf entry. +/// +/// # Errors +/// +/// Returns [`PagingError::InvalidIntermediateEntry`] if `entry` is not a valid +/// Sv39 non-leaf entry. +const fn intermediate_frame(index: usize, entry: usize) -> Result { + if entry & PTE_VALID == 0 || entry & PTE_LEAF_MASK != 0 { + return Err(PagingError::InvalidIntermediateEntry { index, entry }); + } + + let physical_page_number = (entry >> PTE_PPN_SHIFT) & SATP_PPN_MASK; + + let physical_address = PhysAddr::new(physical_page_number << PAGE_SHIFT); + + let Some(frame) = PhysFrame::from_start(physical_address) else { + return Err(PagingError::InvalidIntermediateEntry { index, entry }); + }; + + Ok(frame) +} + +/// Returns the child table referenced by `parent[index]`, allocating it when +/// the entry is currently empty. +/// +/// # Errors +/// +/// Returns an error if the parent cannot be accessed, no physical frame remains, +/// or an existing entry is not a valid non-leaf page-table pointer. +fn next_table( + parent: PhysFrame, + index: usize, + frames: &mut BootFrameAllocator<'_, '_>, +) -> Result { + let entry = read_page_table_entry(parent, index)?; + + if entry != 0 { + return intermediate_frame(index, entry); + } + + let child = allocate_page_table(frames)?; + + let entry = page_table_entry(child.start_address(), PTE_VALID)?; + + write_page_table_entry(parent, index, entry)?; + + Ok(child) +} + /// Invalidates the current hart's cached translation for `address`. fn flush_address(address: VirtAddr) { // SAFETY: @@ -366,7 +657,7 @@ pub(crate) fn map_bootstrap_fdt(dtb: PhysAddr) -> Result let entry_address = PhysAddr::new(boot_root.as_usize() + entry_offset); - let entry = leaf_entry(physical_address, BOOT_FDT_PTE_FLAGS)?; + let entry = page_table_entry(physical_address, BOOT_FDT_PTE_FLAGS)?; let entry_pointer = ptr::with_exposed_provenance_mut::(entry_address.as_usize()); From 1344d2de14465f5527efbea7e165f399621c9234 Mon Sep 17 00:00:00 2001 From: Daniel-De-Dev Date: Sun, 27 Sep 2026 00:50:50 +0200 Subject: [PATCH 3/9] refactor(pages): update paging to be more explicit and add helpers Added helpers that allow for easily read and write to *mapped* frames, but they are unsafe due to invariants which cannot be proved. Will work on restructuring the setup so invariant can better be encoded via types. A clear distinction needs to be put in place between when page operations are occurring at the start or when later normal kernel operations. Otherwise, future implementations will needs constantly use unsafe guards. --- .harper-dictionary.txt | 1 + kernel/src/arch/riscv64/paging.rs | 209 ++++++++++++++++++++---------- kernel/src/main.rs | 6 +- 3 files changed, 146 insertions(+), 70 deletions(-) diff --git a/.harper-dictionary.txt b/.harper-dictionary.txt index 0fc403e..a7eaba4 100644 --- a/.harper-dictionary.txt +++ b/.harper-dictionary.txt @@ -28,3 +28,4 @@ stackless u32 usize v0 +writably diff --git a/kernel/src/arch/riscv64/paging.rs b/kernel/src/arch/riscv64/paging.rs index abf01e2..d9c52f4 100644 --- a/kernel/src/arch/riscv64/paging.rs +++ b/kernel/src/arch/riscv64/paging.rs @@ -49,9 +49,6 @@ const VPN_MASK: usize = 0x1ff; /// Bit offset of the physical page number within an Sv39 PTE. const PTE_PPN_SHIFT: usize = 10; -/// Size in bytes of one Sv39 page-table entry. -const PTE_SIZE: usize = size_of::(); - /// PTE bit marking an entry as valid. const PTE_VALID: usize = 1 << 0; @@ -163,7 +160,7 @@ impl Sv39PageTable { &self, virtual_address: VirtAddr, physical_frame: PhysFrame, - permissions: usize, + permissions: usize, // TODO: Make permissions explicitly a type frames: &mut BootFrameAllocator<'_, '_>, ) -> Result<(), PagingError> { let address = virtual_address.as_usize(); @@ -328,6 +325,12 @@ fn active_root_table() -> Result { /// The active root page table must itself still be accessible through the /// bootstrap identity mapping. /// +/// # Safety +/// +/// The active Sv39 root page table must be completely and writably +/// identity-mapped. The reserved bootstrap frame-window root entry must not be +/// modified concurrently while this function executes. +/// /// # Errors /// /// Returns [`PagingError::UnexpectedAddressTranslationMode`] if Sv39 is not @@ -339,22 +342,19 @@ fn active_root_table() -> Result { clippy::arithmetic_side_effects, reason = "the root index and frame offset are bounded by the Sv39 page-table and gigapage sizes" )] -fn with_bootstrap_frame( +unsafe fn with_bootstrap_frame( frame: PhysFrame, operation: impl FnOnce(VirtAddr) -> R, ) -> Result { let boot_root = active_root_table()?; - let entry_offset = BOOT_FRAME_WINDOW_VPN2 * PTE_SIZE; - - let entry_address = PhysAddr::new(boot_root.as_usize() + entry_offset); - - let entry_pointer = ptr::with_exposed_provenance_mut::(entry_address.as_usize()); + let boot_root_virtual = VirtAddr::new(boot_root.as_usize()); // SAFETY: - // The bootstrap root page table is contained inside the physical kernel - // gigapage, which remains identity-mapped while this helper is used. - let existing = unsafe { ptr::read_volatile(entry_pointer) }; + // By this function's safety contract, `boot_root_virtual` maps the complete + // readable active Sv39 root page table for the duration of this operation. + let existing = + unsafe { read_mapped_page_table_entry(boot_root_virtual, BOOT_FRAME_WINDOW_VPN2)? }; if existing != 0 { return Err(PagingError::RootEntryInUse { @@ -376,10 +376,10 @@ fn with_bootstrap_frame( let entry = page_table_entry(physical_base, BOOT_FRAME_PTE_FLAGS)?; // SAFETY: - // The reserved scratch root entry was verified to be empty and the active - // bootstrap root remains identity-mapped. + // By this function's safety contract, `boot_root_virtual` maps the complete + // writable active Sv39 root page table for the duration of this operation. unsafe { - ptr::write_volatile(entry_pointer, entry); + write_mapped_page_table_entry(boot_root_virtual, BOOT_FRAME_WINDOW_VPN2, entry)?; } let frame_offset = frame_address.as_usize() & GIGAPAGE_MASK; @@ -391,10 +391,11 @@ fn with_bootstrap_frame( let result = operation(virtual_address); // SAFETY: - // This is the same bootstrap root entry installed above, and no other code - // may use the reserved scratch entry while this function executes. + // By this function's safety contract, `boot_root_virtual` maps the complete + // writable active Sv39 root page table. This function exclusively owns the + // bootstrap frame-window entry until it is cleared here. unsafe { - ptr::write_volatile(entry_pointer, 0); + write_mapped_page_table_entry(boot_root_virtual, BOOT_FRAME_WINDOW_VPN2, 0)?; } flush_address(virtual_address); @@ -429,16 +430,23 @@ const fn page_table_entry(physical_address: PhysAddr, flags: usize) -> Result Result<(), PagingError> { - with_bootstrap_frame(frame, |address| { + let operation = |address: VirtAddr| { let table_pointer = ptr::with_exposed_provenance_mut::(address.as_usize()); // SAFETY: - // `address` is a temporary mapping of the complete page-aligned physical - // frame, and an all-zero Sv39 page table contains only invalid entries. + // `with_bootstrap_frame` maps the complete physical frame at `address` + // for the duration of this operation. `frame` is a newly allocated + // page-table frame and therefore may be initialized exclusively here. unsafe { ptr::write_bytes(table_pointer, 0, 1); } - }) + }; + + // SAFETY: + // Page-table construction currently runs while the bootstrap Sv39 root is + // active and writably identity-mapped. The paging module exclusively owns + // the reserved bootstrap frame-window entry while this operation executes. + unsafe { with_bootstrap_frame(frame, operation) } } /// Reads one entry from a physical page-table frame. @@ -448,20 +456,18 @@ fn zero_page_table(frame: PhysFrame) -> Result<(), PagingError> { /// Returns an error if the frame cannot be temporarily mapped or `index` is /// outside the page table. fn read_page_table_entry(frame: PhysFrame, index: usize) -> Result { - with_bootstrap_frame(frame, |address| { - let table_pointer = ptr::with_exposed_provenance::(address.as_usize()); - + let operation = |address: VirtAddr| { // SAFETY: - // The temporary mapping covers the complete physical page-table frame and - // remains valid for the duration of this closure. - let table = unsafe { &*table_pointer }; - - let Some(entry) = table.entries.get(index) else { - return Err(PagingError::PageTableIndexOutOfRange { index }); - }; + // `with_bootstrap_frame` maps the complete physical page-table frame at + // `address` for the duration of this operation. + unsafe { read_mapped_page_table_entry(address, index) } + }; - Ok(*entry) - })? + // SAFETY: + // Page-table construction currently runs while the bootstrap Sv39 root is + // active and writably identity-mapped. The paging module exclusively owns + // the reserved bootstrap frame-window entry while this operation executes. + unsafe { with_bootstrap_frame(frame, operation) }? } /// Writes one entry in a physical page-table frame. @@ -471,22 +477,18 @@ fn read_page_table_entry(frame: PhysFrame, index: usize) -> Result Result<(), PagingError> { - with_bootstrap_frame(frame, |address| { - let table_pointer = ptr::with_exposed_provenance_mut::(address.as_usize()); - + let operation = |address: VirtAddr| { // SAFETY: - // The temporary mapping covers the complete physical page-table frame and - // remains valid exclusively for the duration of this closure. - let table = unsafe { &mut *table_pointer }; - - let Some(slot) = table.entries.get_mut(index) else { - return Err(PagingError::PageTableIndexOutOfRange { index }); - }; - - *slot = entry; + // `with_bootstrap_frame` maps the complete writable physical page-table + // frame at `address` for the duration of this operation. + unsafe { write_mapped_page_table_entry(address, index, entry) } + }; - Ok(()) - })? + // SAFETY: + // Page-table construction currently runs while the bootstrap Sv39 root is + // active and writably identity-mapped. The paging module exclusively owns + // the reserved bootstrap frame-window entry while this operation executes. + unsafe { with_bootstrap_frame(frame, operation) }? } /// Allocates and initializes one empty physical page-table frame. @@ -577,14 +579,87 @@ fn flush_all() { } } +/// Reads one entry from a page table that is currently mapped at `address`. +/// +/// # Safety +/// +/// `address` must be page-aligned and must map a complete readable +/// [`PageTable`] for the duration of this operation. +/// +/// # Errors +/// +/// Returns [`PagingError::PageTableIndexOutOfRange`] if `index` does not +/// identify an entry in the page table. +unsafe fn read_mapped_page_table_entry( + address: VirtAddr, + index: usize, +) -> Result { + if index >= PAGE_TABLE_ENTRIES { + return Err(PagingError::PageTableIndexOutOfRange { index }); + } + + let table_pointer = ptr::with_exposed_provenance::(address.as_usize()); + + // SAFETY: + // The caller guarantees that `address` maps a complete `PageTable`, and the + // bounds check above guarantees that `index` identifies one of its entries. + let entry_pointer = unsafe { table_pointer.cast::().add(index) }; + + // SAFETY: + // `entry_pointer` refers to a valid entry within the mapped page table. + Ok(unsafe { ptr::read_volatile(entry_pointer) }) +} + +/// Writes one entry in a page table that is currently mapped at `address`. +/// +/// # Safety +/// +/// `address` must be page-aligned and must map a complete writable +/// [`PageTable`] for the duration of this operation. The table must not be +/// concurrently accessed in a way that conflicts with this write. +/// +/// # Errors +/// +/// Returns [`PagingError::PageTableIndexOutOfRange`] if `index` does not +/// identify an entry in the page table. +unsafe fn write_mapped_page_table_entry( + address: VirtAddr, + index: usize, + value: usize, +) -> Result<(), PagingError> { + if index >= PAGE_TABLE_ENTRIES { + return Err(PagingError::PageTableIndexOutOfRange { index }); + } + + let table_pointer = ptr::with_exposed_provenance_mut::(address.as_usize()); + + // SAFETY: + // The caller guarantees that `address` maps a complete writable `PageTable`, + // and the bounds check above guarantees that `index` identifies one of its + // entries. + let entry_pointer = unsafe { table_pointer.cast::().add(index) }; + + // SAFETY: + // `entry_pointer` points to a writable PTE within the mapped page table. + unsafe { + ptr::write_volatile(entry_pointer, value); + } + + Ok(()) +} + /// Maps the physical region containing `dtb` into a higher-half boot window. /// /// The mapping preserves `dtb`'s offset within its first 1 GiB physical region, /// so the returned virtual address refers to the same byte as `dtb`. The window /// consists of consecutive read-only, non-executable Sv39 level-2 leaves. /// -/// The active root table must be accessible through an identity mapping while -/// this function executes. +/// # Safety +/// +/// The active Sv39 root page table must be completely and writably +/// identity-mapped for the duration of this function. No other hart or code +/// may concurrently modify the root-table entries used by the bootstrap FDT +/// window. /// /// # Errors /// @@ -598,9 +673,11 @@ fn flush_all() { the satp PPN, and the DTB address is validated before \ physical-address arithmetic" )] -pub(crate) fn map_bootstrap_fdt(dtb: PhysAddr) -> Result { +pub(crate) unsafe fn map_bootstrap_fdt(dtb: PhysAddr) -> Result { let boot_root = active_root_table()?; + let boot_root_virtual = VirtAddr::new(boot_root.as_usize()); + if dtb.as_usize() > MAX_PHYSICAL_ADDRESS { return Err(PagingError::PhysicalAddressNotRepresentable { address: dtb }); } @@ -613,16 +690,13 @@ pub(crate) fn map_bootstrap_fdt(dtb: PhysAddr) -> Result */ for window_index in 0..BOOT_FDT_WINDOW_GIGAPAGES { let root_index = BOOT_FDT_WINDOW_VPN2 + window_index; - let entry_offset = root_index * PTE_SIZE; - - let entry_address = PhysAddr::new(boot_root.as_usize() + entry_offset); - - let entry_pointer = ptr::with_exposed_provenance::(entry_address.as_usize()); // SAFETY: - // The active bootstrap root is located inside the physical kernel - // gigapage, which remains identity-mapped while this function executes. - let existing = unsafe { ptr::read_volatile(entry_pointer) }; + // `boot_root_virtual` is the identity-mapped virtual address of the active + // bootstrap Sv39 root table, which remains mapped and readable while this + // function executes. `root_index` refers to an entry within that complete + // page table. + let existing = unsafe { read_mapped_page_table_entry(boot_root_virtual, root_index)? }; if existing != 0 { return Err(PagingError::RootEntryInUse { @@ -653,19 +727,16 @@ pub(crate) fn map_bootstrap_fdt(dtb: PhysAddr) -> Result let physical_address = PhysAddr::new(physical_base.as_usize() + physical_offset); - let entry_offset = root_index * PTE_SIZE; - - let entry_address = PhysAddr::new(boot_root.as_usize() + entry_offset); - let entry = page_table_entry(physical_address, BOOT_FDT_PTE_FLAGS)?; - let entry_pointer = ptr::with_exposed_provenance_mut::(entry_address.as_usize()); - // SAFETY: - // The active bootstrap root remains identity-mapped, and the selected - // entry was verified to be unused before any mappings were installed. + // `boot_root_virtual` is the identity-mapped virtual address of the active + // bootstrap Sv39 root table, which remains mapped and writable while this + // function executes. `root_index` refers to an entry within that complete + // page table, and this entry was verified to be unused before any bootstrap + // FDT mappings were installed. unsafe { - ptr::write_volatile(entry_pointer, entry); + write_mapped_page_table_entry(boot_root_virtual, root_index, entry)?; } } diff --git a/kernel/src/main.rs b/kernel/src/main.rs index f39a65e..3d33dcf 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -41,7 +41,11 @@ extern "C" fn main(hart_id: usize, dtb: usize, kernel_phys_start: usize) -> ! { logging::info!("initializing trap handling"); arch::init_trap(); - let dtb_virtual = match arch::map_bootstrap_fdt(dtb_phys) { + // SAFETY: + // The architecture bootstrap enters `main` with the complete active Sv39 root + // page table writably identity-mapped. No other hart or code has modified the + // reserved bootstrap FDT window before this call. + let dtb_virtual = match unsafe { arch::map_bootstrap_fdt(dtb_phys) } { Ok(address) => address, Err(error) => { logging::error!( From 8467884c52f521815546d539244ff8dea878d6f3 Mon Sep 17 00:00:00 2001 From: Daniel-De-Dev Date: Sun, 27 Sep 2026 02:38:25 +0200 Subject: [PATCH 4/9] refactor: encode frame ownership and bootstrap paging state explicitly in types This is done to avoid relying on implicit global assumptions. Add `AllocatedFrame` to better encode exclusive ownership of a physical frame and that the owner is free to use the underlying memory. The frame allocator now returns this type. Define `BootstrapPaging` to represent the initial paging stage, where the kernel relies on assumptions about the existing page table and its accessibility. --- kernel/src/arch.rs | 2 +- kernel/src/arch/riscv64.rs | 2 +- kernel/src/arch/riscv64/paging.rs | 648 +++++++++++----------- kernel/src/main.rs | 29 +- kernel/src/memory.rs | 2 +- kernel/src/memory/frame.rs | 43 ++ kernel/src/memory/frame/boot_allocator.rs | 6 +- 7 files changed, 404 insertions(+), 328 deletions(-) diff --git a/kernel/src/arch.rs b/kernel/src/arch.rs index 9a28af4..a144bdc 100644 --- a/kernel/src/arch.rs +++ b/kernel/src/arch.rs @@ -7,4 +7,4 @@ mod riscv64; #[cfg(target_arch = "riscv64")] -pub(super) use riscv64::{halt, init_trap, map_bootstrap_fdt}; +pub(super) use riscv64::{BootstrapPaging, halt, init_trap}; diff --git a/kernel/src/arch/riscv64.rs b/kernel/src/arch/riscv64.rs index 9273999..4183c89 100644 --- a/kernel/src/arch/riscv64.rs +++ b/kernel/src/arch/riscv64.rs @@ -7,7 +7,7 @@ mod boot; mod paging; mod trap; -pub(crate) use paging::map_bootstrap_fdt; +pub(crate) use paging::BootstrapPaging; pub(crate) use trap::init as init_trap; use core::arch::asm; diff --git a/kernel/src/arch/riscv64/paging.rs b/kernel/src/arch/riscv64/paging.rs index d9c52f4..231be3f 100644 --- a/kernel/src/arch/riscv64/paging.rs +++ b/kernel/src/arch/riscv64/paging.rs @@ -5,7 +5,7 @@ use core::{arch::asm, ptr}; -use crate::memory::{BootFrameAllocator, PhysAddr, PhysFrame, VirtAddr}; +use crate::memory::{AllocatedFrame, BootFrameAllocator, PhysAddr, PhysFrame, VirtAddr}; /// Sign bit of an Sv39 virtual address. const SV39_SIGN_BIT: usize = 38; @@ -127,10 +127,325 @@ struct PageTable { entries: [usize; PAGE_TABLE_ENTRIES], } +/// Exclusive access to the inherited Sv39 bootstrap address space. +/// +/// Construction establishes that `root_virtual` maps the complete active Sv39 +/// root page table read/write and that this value exclusively controls the +/// bootstrap paging entries reserved by this module. +/// +/// The capability is valid only while that bootstrap address space remains +/// active. +pub(crate) struct BootstrapPaging { + /// Identity-mapped virtual address of the active bootstrap root table. + root_virtual: VirtAddr, +} + +impl BootstrapPaging { + /// Claims exclusive control of the active bootstrap Sv39 address space. + /// + /// # Safety + /// + /// The active Sv39 root page table must be completely and writably + /// identity-mapped. No other [`BootstrapPaging`] capability may exist for the + /// same active address space. The paging entries reserved by this module must + /// not be modified by any other code or hart while this capability exists, + /// and the active root must not be replaced except by consuming this + /// capability. + /// + /// # Errors + /// + /// Returns [`PagingError::UnexpectedAddressTranslationMode`] if the active + /// address-translation mode is not Sv39. + pub(crate) unsafe fn claim() -> Result { + let root = active_root_table()?; + + Ok(Self { + root_virtual: VirtAddr::new(root.as_usize()), + }) + } + + /// Maps the physical region containing `dtb` into a higher-half boot window. + /// + /// The mapping preserves `dtb`'s offset within its first 1 GiB physical region, + /// so the returned virtual address refers to the same byte as `dtb`. The window + /// consists of consecutive read-only, non-executable Sv39 level-2 leaves. + /// + /// # Errors + /// + /// Returns [`PagingError::RootEntryInUse`] if the required virtual window is + /// occupied, or [`PagingError::PhysicalAddressNotRepresentable`] if the DTB + /// window requires a physical address that cannot be represented by Sv39. + #[expect( + clippy::arithmetic_side_effects, + reason = "window indices are statically bounded and the DTB address is \ + validated before physical-address arithmetic" + )] + pub(crate) fn map_fdt(&mut self, dtb: PhysAddr) -> Result { + if dtb.as_usize() > MAX_PHYSICAL_ADDRESS { + return Err(PagingError::PhysicalAddressNotRepresentable { address: dtb }); + } + + let physical_base = PhysAddr::new(dtb.as_usize() & !GIGAPAGE_MASK); + + /* + * Validate the complete window before modifying the active page table so an + * error cannot leave a partially installed mapping. + */ + for window_index in 0..BOOT_FDT_WINDOW_GIGAPAGES { + let root_index = BOOT_FDT_WINDOW_VPN2 + window_index; + + // SAFETY: + // `BootstrapPaging` guarantees that `root_virtual` maps the complete readable + // active bootstrap root table while this capability exists. + let existing = unsafe { read_mapped_page_table_entry(self.root_virtual, root_index)? }; + + if existing != 0 { + return Err(PagingError::RootEntryInUse { + index: root_index, + entry: existing, + }); + } + + let physical_offset = window_index * GIGAPAGE_SIZE; + + let physical_address = PhysAddr::new(physical_base.as_usize() + physical_offset); + + if physical_address.as_usize() > MAX_PHYSICAL_ADDRESS { + return Err(PagingError::PhysicalAddressNotRepresentable { + address: physical_address, + }); + } + } + + /* + * Every required entry and physical address has been validated. Install the + * read-only level-2 leaves. + */ + for window_index in 0..BOOT_FDT_WINDOW_GIGAPAGES { + let root_index = BOOT_FDT_WINDOW_VPN2 + window_index; + + let physical_offset = window_index * GIGAPAGE_SIZE; + + let physical_address = PhysAddr::new(physical_base.as_usize() + physical_offset); + + let entry = page_table_entry(physical_address, BOOT_FDT_PTE_FLAGS)?; + + // SAFETY: + // `BootstrapPaging` guarantees that `root_virtual` maps the complete writable + // active bootstrap root table. This entry was verified to be unused before any + // FDT mappings were installed. + unsafe { + write_mapped_page_table_entry(self.root_virtual, root_index, entry)?; + } + } + + flush_all(); + + let dtb_offset = dtb.as_usize() & GIGAPAGE_MASK; + + let virtual_address = BOOT_FDT_WINDOW_BASE + dtb_offset; + + Ok(VirtAddr::new(virtual_address)) + } + + /// Executes `operation` while `frame` is temporarily accessible through the + /// bootstrap physical-frame window. + /// + /// The temporary mapping uses a single Sv39 level-2 leaf. The physical 1 GiB + /// region containing `frame` is mapped at [`BOOT_FRAME_WINDOW_BASE`], and the + /// returned virtual address preserves the frame's offset within that region. + /// + /// The mapping exists only while `operation` executes and is removed before + /// this function returns. + /// + /// # Errors + /// + /// Returns [`PagingError::RootEntryInUse`] if the reserved frame-window entry + /// is already occupied, or + /// [`PagingError::PhysicalAddressNotRepresentable`] if `frame` cannot be + /// represented by Sv39. + #[expect( + clippy::arithmetic_side_effects, + reason = "the root index and frame offset are bounded by the Sv39 page-table and gigapage sizes" + )] + fn with_frame( + &mut self, + frame: PhysFrame, + operation: impl FnOnce(VirtAddr) -> R, + ) -> Result { + // SAFETY: + // `BootstrapPaging` guarantees that `root_virtual` maps the complete + // readable active bootstrap root table. + let existing = + unsafe { read_mapped_page_table_entry(self.root_virtual, BOOT_FRAME_WINDOW_VPN2)? }; + + if existing != 0 { + return Err(PagingError::RootEntryInUse { + index: BOOT_FRAME_WINDOW_VPN2, + entry: existing, + }); + } + + let frame_address = frame.start_address(); + + if frame_address.as_usize() > MAX_PHYSICAL_ADDRESS { + return Err(PagingError::PhysicalAddressNotRepresentable { + address: frame_address, + }); + } + + let physical_base = PhysAddr::new(frame_address.as_usize() & !GIGAPAGE_MASK); + + let entry = page_table_entry(physical_base, BOOT_FRAME_PTE_FLAGS)?; + + // SAFETY: + // `BootstrapPaging` guarantees that `root_virtual` maps the complete + // writable active bootstrap root table. + unsafe { + write_mapped_page_table_entry(self.root_virtual, BOOT_FRAME_WINDOW_VPN2, entry)?; + } + + let frame_offset = frame_address.as_usize() & GIGAPAGE_MASK; + + let virtual_address = VirtAddr::new(BOOT_FRAME_WINDOW_BASE + frame_offset); + + flush_address(virtual_address); + + let result = operation(virtual_address); + + // SAFETY: + // `BootstrapPaging` guarantees that `root_virtual` maps the complete + // writable active bootstrap root table. + unsafe { + write_mapped_page_table_entry(self.root_virtual, BOOT_FRAME_WINDOW_VPN2, 0)?; + } + + flush_address(virtual_address); + + Ok(result) + } + + /// Clears every entry in a physical page-table frame. + /// + /// # Errors + /// + /// Returns an error if `frame` cannot be accessed through the bootstrap + /// physical-frame window. + #[expect( + clippy::needless_pass_by_ref_mut, + reason = "the mutable borrow intentionally provides exclusive access to \ + the allocated frame while its memory is initialized" + )] + fn zero_page_table(&mut self, frame: &mut AllocatedFrame) -> Result<(), PagingError> { + let physical_frame = frame.frame(); + + self.with_frame(physical_frame, |address| { + let table_pointer = ptr::with_exposed_provenance_mut::(address.as_usize()); + + // SAFETY: + // `with_frame` maps the complete allocated frame at `address` for this + // operation, and `AllocatedFrame` provides exclusive ownership of the + // physical frame being initialized. + unsafe { + ptr::write_bytes(table_pointer, 0, 1); + } + }) + } + + /// Reads one entry from a physical page-table frame. + /// + /// # Errors + /// + /// Returns an error if the frame cannot be temporarily mapped or `index` is + /// outside the page table. + fn read_page_table_entry( + &mut self, + frame: PhysFrame, + index: usize, + ) -> Result { + self.with_frame(frame, |address| { + // SAFETY: + // `with_frame` maps the complete physical frame at `address` for the + // duration of this operation. + unsafe { read_mapped_page_table_entry(address, index) } + })? + } + + /// Writes one entry in a physical page-table frame. + /// + /// # Errors + /// + /// Returns an error if the frame cannot be temporarily mapped or `index` is + /// outside the page table. + fn write_page_table_entry( + &mut self, + frame: PhysFrame, + index: usize, + entry: usize, + ) -> Result<(), PagingError> { + self.with_frame(frame, |address| { + // SAFETY: + // `with_frame` maps the complete writable physical frame at `address` for + // the duration of this operation. + unsafe { write_mapped_page_table_entry(address, index, entry) } + })? + } + + /// Allocates and initializes one empty physical page-table frame. + /// + /// # Errors + /// + /// Returns [`PagingError::OutOfPhysicalFrames`] if no frame remains, or an + /// error if the allocated frame cannot be initialized through the temporary + /// mapping window. + // TODO: Deal with ownership one day in the future for who is responsible for + // freeing. Works now since allocator never frees + fn allocate_page_table( + &mut self, + frames: &mut BootFrameAllocator<'_, '_>, + ) -> Result { + let Some(mut frame) = frames.allocate() else { + return Err(PagingError::OutOfPhysicalFrames); + }; + + self.zero_page_table(&mut frame)?; + + Ok(frame) + } + + /// Returns the child table referenced by `parent[index]`, allocating it when + /// the entry is currently empty. + /// + /// # Errors + /// + /// Returns an error if the parent cannot be accessed, no physical frame remains, + /// or an existing entry is not a valid non-leaf page-table pointer. + fn next_table( + &mut self, + parent: PhysFrame, + index: usize, + frames: &mut BootFrameAllocator<'_, '_>, + ) -> Result { + let entry = self.read_page_table_entry(parent, index)?; + + if entry != 0 { + return intermediate_frame(index, entry); + } + + let child = self.allocate_page_table(frames)?; + + let entry = page_table_entry(child.start_address(), PTE_VALID)?; + + self.write_page_table_entry(parent, index, entry)?; + + Ok(child.into_frame()) + } +} + /// An Sv39 address space being constructed by the kernel. struct Sv39PageTable { - /// Physical frame containing the level-2 root table. - root: PhysFrame, + /// Allocation containing the level-2 root table. + root: AllocatedFrame, } impl Sv39PageTable { @@ -139,8 +454,11 @@ impl Sv39PageTable { /// # Errors /// /// Returns an error if no root frame can be allocated or initialized. - fn new(frames: &mut BootFrameAllocator<'_, '_>) -> Result { - let root = allocate_page_table(frames)?; + fn new( + bootstrap: &mut BootstrapPaging, + frames: &mut BootFrameAllocator<'_, '_>, + ) -> Result { + let root = bootstrap.allocate_page_table(frames)?; Ok(Self { root }) } @@ -156,8 +474,14 @@ impl Sv39PageTable { /// Returns an error if the virtual address is invalid or unaligned, physical /// frames required for intermediate tables cannot be allocated, an existing /// intermediate entry is malformed, or the virtual page is already mapped. + #[expect( + clippy::needless_pass_by_ref_mut, + reason = "mapping a page mutates the address space represented by this \ + value, even though its page-table memory is modified indirectly" + )] fn map_page( - &self, + &mut self, + bootstrap: &mut BootstrapPaging, virtual_address: VirtAddr, physical_frame: PhysFrame, permissions: usize, // TODO: Make permissions explicitly a type @@ -184,10 +508,10 @@ impl Sv39PageTable { let vpn1 = (address >> MEGAPAGE_SHIFT) & VPN_MASK; let vpn0 = (address >> PAGE_SHIFT) & VPN_MASK; - let level_1 = next_table(self.root, vpn2, frames)?; - let level_0 = next_table(level_1, vpn1, frames)?; + let level_1 = bootstrap.next_table(self.root.frame(), vpn2, frames)?; + let level_0 = bootstrap.next_table(level_1, vpn1, frames)?; - let existing = read_page_table_entry(level_0, vpn0)?; + let existing = bootstrap.read_page_table_entry(level_0, vpn0)?; if existing != 0 { return Err(PagingError::PageAlreadyMapped { @@ -204,7 +528,7 @@ impl Sv39PageTable { let entry = page_table_entry(physical_frame.start_address(), flags)?; - write_page_table_entry(level_0, vpn0, entry) + bootstrap.write_page_table_entry(level_0, vpn0, entry) } } @@ -312,97 +636,6 @@ fn active_root_table() -> Result { Ok(PhysAddr::new(root_ppn << PAGE_SHIFT)) } -/// Executes `operation` while `frame` is temporarily accessible through the -/// bootstrap physical-frame window. -/// -/// The temporary mapping uses a single Sv39 level-2 leaf. The physical 1 GiB -/// region containing `frame` is mapped at [`BOOT_FRAME_WINDOW_BASE`], and the -/// returned virtual address preserves the frame's offset within that region. -/// -/// The mapping exists only while `operation` executes and is removed before -/// this function returns. -/// -/// The active root page table must itself still be accessible through the -/// bootstrap identity mapping. -/// -/// # Safety -/// -/// The active Sv39 root page table must be completely and writably -/// identity-mapped. The reserved bootstrap frame-window root entry must not be -/// modified concurrently while this function executes. -/// -/// # Errors -/// -/// Returns [`PagingError::UnexpectedAddressTranslationMode`] if Sv39 is not -/// active, [`PagingError::RootEntryInUse`] if the reserved temporary root entry -/// is already occupied, or -/// [`PagingError::PhysicalAddressNotRepresentable`] if the frame cannot be -/// represented by Sv39. -#[expect( - clippy::arithmetic_side_effects, - reason = "the root index and frame offset are bounded by the Sv39 page-table and gigapage sizes" -)] -unsafe fn with_bootstrap_frame( - frame: PhysFrame, - operation: impl FnOnce(VirtAddr) -> R, -) -> Result { - let boot_root = active_root_table()?; - - let boot_root_virtual = VirtAddr::new(boot_root.as_usize()); - - // SAFETY: - // By this function's safety contract, `boot_root_virtual` maps the complete - // readable active Sv39 root page table for the duration of this operation. - let existing = - unsafe { read_mapped_page_table_entry(boot_root_virtual, BOOT_FRAME_WINDOW_VPN2)? }; - - if existing != 0 { - return Err(PagingError::RootEntryInUse { - index: BOOT_FRAME_WINDOW_VPN2, - entry: existing, - }); - } - - let frame_address = frame.start_address(); - - if frame_address.as_usize() > MAX_PHYSICAL_ADDRESS { - return Err(PagingError::PhysicalAddressNotRepresentable { - address: frame_address, - }); - } - - let physical_base = PhysAddr::new(frame_address.as_usize() & !GIGAPAGE_MASK); - - let entry = page_table_entry(physical_base, BOOT_FRAME_PTE_FLAGS)?; - - // SAFETY: - // By this function's safety contract, `boot_root_virtual` maps the complete - // writable active Sv39 root page table for the duration of this operation. - unsafe { - write_mapped_page_table_entry(boot_root_virtual, BOOT_FRAME_WINDOW_VPN2, entry)?; - } - - let frame_offset = frame_address.as_usize() & GIGAPAGE_MASK; - - let virtual_address = VirtAddr::new(BOOT_FRAME_WINDOW_BASE + frame_offset); - - flush_address(virtual_address); - - let result = operation(virtual_address); - - // SAFETY: - // By this function's safety contract, `boot_root_virtual` maps the complete - // writable active Sv39 root page table. This function exclusively owns the - // bootstrap frame-window entry until it is cleared here. - unsafe { - write_mapped_page_table_entry(boot_root_virtual, BOOT_FRAME_WINDOW_VPN2, 0)?; - } - - flush_address(virtual_address); - - Ok(result) -} - /// Constructs an Sv39 page-table entry pointing at `physical_address`. /// /// `flags` contains the desired Sv39 PTE flag bits. @@ -423,91 +656,6 @@ const fn page_table_entry(physical_address: PhysAddr, flags: usize) -> Result Result<(), PagingError> { - let operation = |address: VirtAddr| { - let table_pointer = ptr::with_exposed_provenance_mut::(address.as_usize()); - - // SAFETY: - // `with_bootstrap_frame` maps the complete physical frame at `address` - // for the duration of this operation. `frame` is a newly allocated - // page-table frame and therefore may be initialized exclusively here. - unsafe { - ptr::write_bytes(table_pointer, 0, 1); - } - }; - - // SAFETY: - // Page-table construction currently runs while the bootstrap Sv39 root is - // active and writably identity-mapped. The paging module exclusively owns - // the reserved bootstrap frame-window entry while this operation executes. - unsafe { with_bootstrap_frame(frame, operation) } -} - -/// Reads one entry from a physical page-table frame. -/// -/// # Errors -/// -/// Returns an error if the frame cannot be temporarily mapped or `index` is -/// outside the page table. -fn read_page_table_entry(frame: PhysFrame, index: usize) -> Result { - let operation = |address: VirtAddr| { - // SAFETY: - // `with_bootstrap_frame` maps the complete physical page-table frame at - // `address` for the duration of this operation. - unsafe { read_mapped_page_table_entry(address, index) } - }; - - // SAFETY: - // Page-table construction currently runs while the bootstrap Sv39 root is - // active and writably identity-mapped. The paging module exclusively owns - // the reserved bootstrap frame-window entry while this operation executes. - unsafe { with_bootstrap_frame(frame, operation) }? -} - -/// Writes one entry in a physical page-table frame. -/// -/// # Errors -/// -/// Returns an error if the frame cannot be temporarily mapped or `index` is -/// outside the page table. -fn write_page_table_entry(frame: PhysFrame, index: usize, entry: usize) -> Result<(), PagingError> { - let operation = |address: VirtAddr| { - // SAFETY: - // `with_bootstrap_frame` maps the complete writable physical page-table - // frame at `address` for the duration of this operation. - unsafe { write_mapped_page_table_entry(address, index, entry) } - }; - - // SAFETY: - // Page-table construction currently runs while the bootstrap Sv39 root is - // active and writably identity-mapped. The paging module exclusively owns - // the reserved bootstrap frame-window entry while this operation executes. - unsafe { with_bootstrap_frame(frame, operation) }? -} - -/// Allocates and initializes one empty physical page-table frame. -/// -/// # Errors -/// -/// Returns [`PagingError::OutOfPhysicalFrames`] if no frame remains, or an -/// error if the allocated frame cannot be initialized through the temporary -/// mapping window. -fn allocate_page_table(frames: &mut BootFrameAllocator<'_, '_>) -> Result { - let Some(frame) = frames.allocate() else { - return Err(PagingError::OutOfPhysicalFrames); - }; - - zero_page_table(frame)?; - - Ok(frame) -} - /// Returns the page-table frame referenced by a valid non-leaf entry. /// /// # Errors @@ -530,33 +678,6 @@ const fn intermediate_frame(index: usize, entry: usize) -> Result, -) -> Result { - let entry = read_page_table_entry(parent, index)?; - - if entry != 0 { - return intermediate_frame(index, entry); - } - - let child = allocate_page_table(frames)?; - - let entry = page_table_entry(child.start_address(), PTE_VALID)?; - - write_page_table_entry(parent, index, entry)?; - - Ok(child) -} - /// Invalidates the current hart's cached translation for `address`. fn flush_address(address: VirtAddr) { // SAFETY: @@ -647,104 +768,3 @@ unsafe fn write_mapped_page_table_entry( Ok(()) } - -/// Maps the physical region containing `dtb` into a higher-half boot window. -/// -/// The mapping preserves `dtb`'s offset within its first 1 GiB physical region, -/// so the returned virtual address refers to the same byte as `dtb`. The window -/// consists of consecutive read-only, non-executable Sv39 level-2 leaves. -/// -/// # Safety -/// -/// The active Sv39 root page table must be completely and writably -/// identity-mapped for the duration of this function. No other hart or code -/// may concurrently modify the root-table entries used by the bootstrap FDT -/// window. -/// -/// # Errors -/// -/// Returns [`PagingError::UnexpectedAddressTranslationMode`] if Sv39 is not -/// active, [`PagingError::RootEntryInUse`] if the required virtual window is -/// occupied, or [`PagingError::PhysicalAddressNotRepresentable`] if the DTB -/// window requires a physical address that cannot be represented by Sv39. -#[expect( - clippy::arithmetic_side_effects, - reason = "window indices are statically bounded, the active root comes from \ - the satp PPN, and the DTB address is validated before \ - physical-address arithmetic" -)] -pub(crate) unsafe fn map_bootstrap_fdt(dtb: PhysAddr) -> Result { - let boot_root = active_root_table()?; - - let boot_root_virtual = VirtAddr::new(boot_root.as_usize()); - - if dtb.as_usize() > MAX_PHYSICAL_ADDRESS { - return Err(PagingError::PhysicalAddressNotRepresentable { address: dtb }); - } - - let physical_base = PhysAddr::new(dtb.as_usize() & !GIGAPAGE_MASK); - - /* - * Validate the complete window before modifying the active page table so an - * error cannot leave a partially installed mapping. - */ - for window_index in 0..BOOT_FDT_WINDOW_GIGAPAGES { - let root_index = BOOT_FDT_WINDOW_VPN2 + window_index; - - // SAFETY: - // `boot_root_virtual` is the identity-mapped virtual address of the active - // bootstrap Sv39 root table, which remains mapped and readable while this - // function executes. `root_index` refers to an entry within that complete - // page table. - let existing = unsafe { read_mapped_page_table_entry(boot_root_virtual, root_index)? }; - - if existing != 0 { - return Err(PagingError::RootEntryInUse { - index: root_index, - entry: existing, - }); - } - - let physical_offset = window_index * GIGAPAGE_SIZE; - - let physical_address = PhysAddr::new(physical_base.as_usize() + physical_offset); - - if physical_address.as_usize() > MAX_PHYSICAL_ADDRESS { - return Err(PagingError::PhysicalAddressNotRepresentable { - address: physical_address, - }); - } - } - - /* - * Every required entry and physical address has been validated. Install the - * read-only level-2 leaves. - */ - for window_index in 0..BOOT_FDT_WINDOW_GIGAPAGES { - let root_index = BOOT_FDT_WINDOW_VPN2 + window_index; - - let physical_offset = window_index * GIGAPAGE_SIZE; - - let physical_address = PhysAddr::new(physical_base.as_usize() + physical_offset); - - let entry = page_table_entry(physical_address, BOOT_FDT_PTE_FLAGS)?; - - // SAFETY: - // `boot_root_virtual` is the identity-mapped virtual address of the active - // bootstrap Sv39 root table, which remains mapped and writable while this - // function executes. `root_index` refers to an entry within that complete - // page table, and this entry was verified to be unused before any bootstrap - // FDT mappings were installed. - unsafe { - write_mapped_page_table_entry(boot_root_virtual, root_index, entry)?; - } - } - - flush_all(); - - let dtb_offset = dtb.as_usize() & GIGAPAGE_MASK; - - let virtual_address = BOOT_FDT_WINDOW_BASE + dtb_offset; - - Ok(VirtAddr::new(virtual_address)) -} diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 3d33dcf..533e0f1 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -42,15 +42,27 @@ extern "C" fn main(hart_id: usize, dtb: usize, kernel_phys_start: usize) -> ! { arch::init_trap(); // SAFETY: - // The architecture bootstrap enters `main` with the complete active Sv39 root - // page table writably identity-mapped. No other hart or code has modified the - // reserved bootstrap FDT window before this call. - let dtb_virtual = match unsafe { arch::map_bootstrap_fdt(dtb_phys) } { + // Architecture bootstrap enters `main` with the complete active Sv39 root + // page table writably identity-mapped. No other hart or code modifies the + // paging entries reserved for bootstrap use while this capability exists. + let mut bootstrap_paging = match unsafe { arch::BootstrapPaging::claim() } { + Ok(paging) => paging, + Err(error) => { + logging::error!( + "Failed to claim bootstrap paging state ({error:?}); \ + kernel startup is unrecoverable, halting" + ); + + arch::halt(); + } + }; + + let dtb_virtual = match bootstrap_paging.map_fdt(dtb_phys) { Ok(address) => address, Err(error) => { logging::error!( "Failed to establish higher-half DTB mapping ({error:?}); \ - kernel startup is unrecoverable, halting" + kernel startup is unrecoverable, halting" ); arch::halt(); @@ -62,9 +74,10 @@ extern "C" fn main(hart_id: usize, dtb: usize, kernel_phys_start: usize) -> ! { let dtb_ptr = core::ptr::with_exposed_provenance::(dtb_virtual.as_usize()); // SAFETY: - // `map_bootstrap_fdt` establishes a readable higher-half mapping containing - // the DTB. This mapping must remain valid and unmodified for as long as `fdt` - // and values borrowing from it remain alive. + // `BootstrapPaging::map_fdt` established a readable mapping containing the + // complete DTB. The bootstrap paging capability remains alive, so that + // mapping remains valid and unmodified while `fdt` and values borrowing from + // it are used. let fdt = match unsafe { Fdt::from_ptr(dtb_ptr) } { Ok(fdt) => fdt, Err(error) => { diff --git a/kernel/src/memory.rs b/kernel/src/memory.rs index f8d4609..c63e90f 100644 --- a/kernel/src/memory.rs +++ b/kernel/src/memory.rs @@ -12,7 +12,7 @@ mod address; mod frame; pub(crate) use address::{PhysAddr, VirtAddr}; -pub(crate) use frame::{BootFrameAllocator, PhysFrame}; +pub(crate) use frame::{AllocatedFrame, BootFrameAllocator, PhysFrame}; /// A non-empty half-open physical address range `[start, end)`. /// diff --git a/kernel/src/memory/frame.rs b/kernel/src/memory/frame.rs index 44d6eb3..ce17b43 100644 --- a/kernel/src/memory/frame.rs +++ b/kernel/src/memory/frame.rs @@ -73,6 +73,49 @@ impl PhysFrame { } } +/// A physical frame allocated exclusively for kernel use. +/// +/// Unlike [`PhysFrame`], this type represents ownership rather than just +/// describing a physical address range. Instances are created only by a frame +/// allocator after the frame has been excluded from future allocation. +#[derive(Debug)] +#[must_use = "dropping an allocated frame permanently loses its ownership token"] +// NOTE: The current allocator is monotonic, so dropping an `AllocatedFrame` does +// not return the frame to the allocator. +pub(crate) struct AllocatedFrame { + /// Physical frame owned by this allocation. + frame: PhysFrame, +} + +impl AllocatedFrame { + /// Constructs ownership of `frame`. + /// + /// This is restricted to the memory-management implementation because + /// constructing an `AllocatedFrame` asserts that the frame has been + /// exclusively allocated and will not be returned again. + const fn new(frame: PhysFrame) -> Self { + Self { frame } + } + + /// Returns the physical frame described by this allocation. + #[must_use] + pub(crate) const fn frame(&self) -> PhysFrame { + self.frame + } + + /// Returns the physical start address of this allocation. + #[must_use] + pub(crate) const fn start_address(&self) -> PhysAddr { + self.frame.start_address() + } + + /// Consumes this ownership token and returns its physical-frame descriptor. + #[must_use] + pub(crate) const fn into_frame(self) -> PhysFrame { + self.frame + } +} + /// Rounds a physical address up to the nearest frame boundary. /// /// If `address` is already frame-aligned, it is returned unchanged. diff --git a/kernel/src/memory/frame/boot_allocator.rs b/kernel/src/memory/frame/boot_allocator.rs index 137c6e9..2f17f1a 100644 --- a/kernel/src/memory/frame/boot_allocator.rs +++ b/kernel/src/memory/frame/boot_allocator.rs @@ -22,7 +22,7 @@ compile_error!("the boot frame allocator currently requires a 64-bit target"); use fdt::{Fdt, MemoryRanges}; -use super::{PhysFrame, align_down, align_up}; +use super::{AllocatedFrame, PhysFrame, align_down, align_up}; use crate::memory::{PhysAddr, PhysRange}; /// Errors that can prevent construction of a boot frame allocator. @@ -176,7 +176,7 @@ impl<'fdt, 'dtb> BootFrameAllocator<'fdt, 'dtb> { clippy::expect_used, reason = "CurrentMemoryRange guarantees that candidate cursors are frame-aligned and can contain a complete frame" )] - pub(crate) fn allocate(&mut self) -> Option { + pub(crate) fn allocate(&mut self) -> Option { loop { if self.current.is_none() && !self.advance_memory_range() { return None; @@ -218,7 +218,7 @@ impl<'fdt, 'dtb> BootFrameAllocator<'fdt, 'dtb> { }); } - return Some(frame); + return Some(AllocatedFrame::new(frame)); } } From aa8720d005e4495d26b413875b4ca9265f35c429 Mon Sep 17 00:00:00 2001 From: Daniel-De-Dev Date: Sun, 27 Sep 2026 15:58:31 +0200 Subject: [PATCH 5/9] refactor: enforce single calls of functions Some functions don't make sense to allow to have called multiple times. Like the bootstrap paging and allocator initialization. Hence, added global bools that only allow them to succeed once and then fail otherwise. This will prevent from initializing 2 instances when only 1 is supposed to be allowed to exist. --- kernel/src/arch/riscv64/paging.rs | 39 +++++++++++++++----- kernel/src/main.rs | 10 ++++-- kernel/src/memory/frame/boot_allocator.rs | 44 ++++++++++++++++++++--- 3 files changed, 78 insertions(+), 15 deletions(-) diff --git a/kernel/src/arch/riscv64/paging.rs b/kernel/src/arch/riscv64/paging.rs index 231be3f..1c6bc17 100644 --- a/kernel/src/arch/riscv64/paging.rs +++ b/kernel/src/arch/riscv64/paging.rs @@ -3,7 +3,11 @@ //! This module provides address-translation helpers and temporary mappings used //! while the kernel is running under its bootstrap Sv39 address space. -use core::{arch::asm, ptr}; +use core::{ + arch::asm, + ptr, + sync::atomic::{AtomicBool, Ordering}, +}; use crate::memory::{AllocatedFrame, BootFrameAllocator, PhysAddr, PhysFrame, VirtAddr}; @@ -140,25 +144,41 @@ pub(crate) struct BootstrapPaging { root_virtual: VirtAddr, } +/// Tracks whether the bootstrap paging capability has already been claimed. +/// +/// Once set, this flag is never cleared, preventing construction of more than +/// one [`BootstrapPaging`] capability during the lifetime of the kernel. +static BOOTSTRAP_PAGING_CLAIMED: AtomicBool = AtomicBool::new(false); + impl BootstrapPaging { - /// Claims exclusive control of the active bootstrap Sv39 address space. + /// Claims the bootstrap paging capability for the active Sv39 address space. + /// + /// This capability can be claimed only once. A successful claim remains + /// permanent even if the returned [`BootstrapPaging`] value is later dropped. /// /// # Safety /// /// The active Sv39 root page table must be completely and writably - /// identity-mapped. No other [`BootstrapPaging`] capability may exist for the - /// same active address space. The paging entries reserved by this module must - /// not be modified by any other code or hart while this capability exists, - /// and the active root must not be replaced except by consuming this - /// capability. + /// identity-mapped. The paging entries reserved by this module must not be + /// modified by any other code or hart while this capability exists, and the + /// active root must not be replaced except by consuming this capability. /// /// # Errors /// /// Returns [`PagingError::UnexpectedAddressTranslationMode`] if the active - /// address-translation mode is not Sv39. + /// address-translation mode is not Sv39, or + /// [`PagingError::BootstrapPagingAlreadyClaimed`] if the bootstrap paging + /// capability has already been claimed. pub(crate) unsafe fn claim() -> Result { let root = active_root_table()?; + if BOOTSTRAP_PAGING_CLAIMED + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_err() + { + return Err(PagingError::BootstrapPagingAlreadyClaimed); + } + Ok(Self { root_virtual: VirtAddr::new(root.as_usize()), }) @@ -601,6 +621,9 @@ pub(crate) enum PagingError { /// Requested raw R/W/X permission bits. permissions: usize, }, + + /// The bootstrap paging capability has already been claimed. + BootstrapPagingAlreadyClaimed, } /// Returns the physical address of the active Sv39 root page table. diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 533e0f1..674e3af 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -119,11 +119,17 @@ extern "C" fn main(hart_id: usize, dtb: usize, kernel_phys_start: usize) -> ! { logging::debug!("DTB Range: {:?}", dtb_range); - let frames = match BootFrameAllocator::new(&fdt, kernel_range, dtb_range) { + // SAFETY: + // The architecture bootstrap preserves the actual physical start of the live + // kernel image, so `kernel_range` covers its complete linker-defined boot + // footprint. `dtb_range` covers the same firmware-provided DTB backing `fdt`. + // The FDT is trusted to describe all other physical memory that is unavailable + // for allocation through its reservation information. + let frames = match unsafe { BootFrameAllocator::claim(&fdt, kernel_range, dtb_range) } { Ok(frames) => frames, Err(error) => { logging::error!( - "Failed to initialize physical frame allocator ({error:?}); \ + "Failed to claim physical frame allocator ({error:?}); \ kernel startup is unrecoverable, halting" ); diff --git a/kernel/src/memory/frame/boot_allocator.rs b/kernel/src/memory/frame/boot_allocator.rs index 2f17f1a..ad12eef 100644 --- a/kernel/src/memory/frame/boot_allocator.rs +++ b/kernel/src/memory/frame/boot_allocator.rs @@ -20,6 +20,8 @@ #[cfg(not(target_pointer_width = "64"))] compile_error!("the boot frame allocator currently requires a 64-bit target"); +use core::sync::atomic::{AtomicBool, Ordering}; + use fdt::{Fdt, MemoryRanges}; use super::{AllocatedFrame, PhysFrame, align_down, align_up}; @@ -36,6 +38,9 @@ pub(crate) enum BootFrameAllocatorError { /// Second overlapping physical memory range. second: PhysRange, }, + + /// The boot physical-frame allocator has already been claimed. + AlreadyClaimed, } /// State for the physical memory range currently being scanned. @@ -131,23 +136,52 @@ fn validate_memory_ranges(fdt: &Fdt<'_>) -> Result<(), BootFrameAllocatorError> Ok(()) } +/// Tracks whether the boot physical-frame allocator has already been claimed. +/// +/// Once set, this flag is never cleared, preventing multiple allocators from +/// independently issuing ownership of the same physical frames. +// TODO: If multiple physical-frame allocator implementations are introduced, +// move this one-shot ownership guard above the individual implementations. +static BOOT_FRAME_ALLOCATOR_CLAIMED: AtomicBool = AtomicBool::new(false); + impl<'fdt, 'dtb> BootFrameAllocator<'fdt, 'dtb> { - /// Constructs a boot frame allocator. + /// Claims the boot-time physical frame allocator. + /// + /// This allocator can be claimed only once. A successful claim remains + /// permanent even if the returned [`BootFrameAllocator`] value is later + /// dropped. /// - /// `kernel` and `dtb` describe physical ranges that must be excluded from - /// allocation. + /// # Safety + /// + /// Every FDT-described memory range not excluded by `kernel`, `dtb`, or the + /// FDT's reservation information must refer to physical RAM that the kernel + /// may exclusively allocate. + /// + /// `kernel` and `dtb` must completely cover the physical memory occupied by + /// the live kernel image and device-tree blob. All other physical memory that + /// must not be allocated must be described by the supplied FDT's reservation + /// information. /// /// # Errors /// /// Returns [`BootFrameAllocatorError::OverlappingMemoryRanges`] if two - /// FDT-described physical memory ranges overlap. - pub(crate) fn new( + /// FDT-described physical memory ranges overlap, or + /// [`BootFrameAllocatorError::AlreadyClaimed`] if the boot frame allocator has + /// already been claimed. + pub(crate) unsafe fn claim( fdt: &'fdt Fdt<'dtb>, kernel: PhysRange, dtb: PhysRange, ) -> Result { validate_memory_ranges(fdt)?; + if BOOT_FRAME_ALLOCATOR_CLAIMED + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_err() + { + return Err(BootFrameAllocatorError::AlreadyClaimed); + } + Ok(Self { fdt, memory_ranges: fdt.memory_ranges(), From b86d59b9887db2ed0955b55649525f36f74d000b Mon Sep 17 00:00:00 2001 From: Daniel-De-Dev Date: Mon, 5 Oct 2026 20:11:34 +0200 Subject: [PATCH 6/9] feat: expose linker-defined kernel section ranges Added also VirtRange (counterpart of PhysRange, essentially copy paste). --- kernel/linker.ld.in | 4 + kernel/src/arch/riscv64/boot.rs | 3 + kernel/src/main.rs | 6 +- kernel/src/memory.rs | 163 +++++++++++++++++++++++++++++++- kernel/src/memory/address.rs | 17 ++++ 5 files changed, 190 insertions(+), 3 deletions(-) diff --git a/kernel/linker.ld.in b/kernel/linker.ld.in index c09a15b..89de375 100644 --- a/kernel/linker.ld.in +++ b/kernel/linker.ld.in @@ -119,6 +119,10 @@ SECTIONS *(.srodata .srodata.*) *(.rodata .rodata.*) + . = ALIGN(8); + *(.eh_frame .eh_frame.*) + *(.eh_frame_hdr) + _rodata_end = .; } diff --git a/kernel/src/arch/riscv64/boot.rs b/kernel/src/arch/riscv64/boot.rs index cb8cb82..a6c9a67 100644 --- a/kernel/src/arch/riscv64/boot.rs +++ b/kernel/src/arch/riscv64/boot.rs @@ -17,6 +17,9 @@ //! `OpenSBI` supplies the hart ID in `a0` and the physical device-tree address //! in `a1`. The physical bootstrap places the kernel's physical start address //! in `a2`. All three values are preserved until `main` is entered. +// TODO: Look into if kernel physical boot address needs to be passed in the +// first place as an argument and inaccessible otherwise, else fix via linker +// script. use core::arch::global_asm; diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 674e3af..b257e3e 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -16,7 +16,7 @@ mod memory; use core::panic::PanicInfo; use fdt::Fdt; -use memory::{BootFrameAllocator, PhysAddr, PhysRange}; +use memory::{BootFrameAllocator, PhysAddr, PhysRange, kernel_sections}; /// Runs the kernel after architecture-specific initialization. /// @@ -107,6 +107,10 @@ extern "C" fn main(hart_id: usize, dtb: usize, kernel_phys_start: usize) -> ! { arch::halt(); }; + let kernel_sections = kernel_sections(); + + logging::debug!("{:#?}", kernel_sections); + logging::debug!("Kernel Range: {:?}", kernel_range); let Some(dtb_range) = PhysRange::from_start_size(dtb_phys, fdt.total_size()) else { diff --git a/kernel/src/memory.rs b/kernel/src/memory.rs index c63e90f..a2f03bb 100644 --- a/kernel/src/memory.rs +++ b/kernel/src/memory.rs @@ -4,8 +4,7 @@ //! kernel's memory-management code. //! //! Physical and virtual addresses are represented by [`PhysAddr`] and -//! [`VirtAddr`], while [`PhysRange`] represents non-empty half-open regions of -//! physical address space. Physical frame representation and boot-time frame +//! [`VirtAddr`]. Physical frame representation and boot-time frame //! allocation are implemented by the [`frame`] submodule. mod address; @@ -27,6 +26,47 @@ pub(crate) struct PhysRange { end: PhysAddr, } +/// A non-empty half-open virtual address range `[start, end)`. +/// +/// A `VirtRange` guarantees that `start` is strictly less than `end`. The +/// start address is included in the range, while the end address is excluded. +#[derive(Clone, Copy, Debug)] +pub(crate) struct VirtRange { + /// Inclusive start address. + start: VirtAddr, + + /// Exclusive end address. + end: VirtAddr, +} + +/// Linker-defined virtual ranges of the higher-half kernel sections. +/// +/// Each range includes its start and excludes its end. Section starts are +/// page-aligned, but ends describe the actual section contents and may require +/// rounding up when mapping complete pages. Empty sections have equal bounds. +/// +/// None represents an empty section. +#[derive(Debug)] +pub(crate) struct KernelSections { + /// Virtual origin corresponding to the physical kernel image start. + pub(crate) virtual_start: VirtAddr, + + /// Executable kernel code. + pub(crate) text: Option, + + /// Read-only kernel constants. + pub(crate) rodata: Option, + + /// Initialized writable kernel data. + pub(crate) data: Option, + + /// Zero-initialized writable kernel data. + pub(crate) bss: Option, + + /// Reserved boot stack. + pub(crate) stack: Option, +} + unsafe extern "C" { /// Linker-defined symbol marking the start of the kernel's complete virtual /// boot-time footprint. @@ -35,6 +75,71 @@ unsafe extern "C" { /// Linker-defined symbol marking the end of the kernel's complete virtual /// boot-time footprint. static _kernel_virtual_end: u8; + + /// Inclusive virtual start of executable kernel code. + static _text_start: u8; + + /// Exclusive virtual end of executable kernel code. + static _text_end: u8; + + /// Inclusive virtual start of read-only kernel constants. + static _rodata_start: u8; + + /// Exclusive virtual end of read-only kernel constants. + static _rodata_end: u8; + + /// Inclusive virtual start of initialized writable kernel data. + static _data_start: u8; + + /// Exclusive virtual end of initialized writable kernel data. + static _data_end: u8; + + /// Inclusive virtual start of zero-initialized kernel data. + static _bss_start: u8; + + /// Exclusive virtual end of zero-initialized kernel data. + static _bss_end: u8; + + /// Inclusive virtual start of the reserved boot stack. + static _stack_start: u8; + + /// Exclusive virtual end of the reserved boot stack. + static _stack_end: u8; +} + +/// Returns the linker-defined virtual boundaries of the kernel sections. +/// +/// Only symbol addresses are taken. The linker script guarantees ordered +/// ranges with page-aligned starts. Empty sections have equal bounds. +#[must_use] +pub(crate) fn kernel_sections() -> KernelSections { + let range = |start: *const u8, end: *const u8| { + VirtRange::new(VirtAddr::new(start.addr()), VirtAddr::new(end.addr())) + }; + + KernelSections { + virtual_start: VirtAddr::new(core::ptr::addr_of!(_kernel_virtual_start).addr()), + text: range( + core::ptr::addr_of!(_text_start), + core::ptr::addr_of!(_text_end), + ), + rodata: range( + core::ptr::addr_of!(_rodata_start), + core::ptr::addr_of!(_rodata_end), + ), + data: range( + core::ptr::addr_of!(_data_start), + core::ptr::addr_of!(_data_end), + ), + bss: range( + core::ptr::addr_of!(_bss_start), + core::ptr::addr_of!(_bss_end), + ), + stack: range( + core::ptr::addr_of!(_stack_start), + core::ptr::addr_of!(_stack_end), + ), + } } /// Returns the kernel's complete boot-time memory footprint in bytes. @@ -120,3 +225,57 @@ impl PhysRange { self.start.as_usize() < other.end.as_usize() && other.start.as_usize() < self.end.as_usize() } } + +// TODO: Merge with PhysRange as the only difference so far was the type used. +// Maybe define a trait which can be derived for both? +impl VirtRange { + /// Constructs a virtual range from its inclusive start and exclusive end. + /// + /// Returns `None` if `start` is greater than or equal to `end`. + #[must_use] + pub(crate) const fn new(start: VirtAddr, end: VirtAddr) -> Option { + if start.as_usize() >= end.as_usize() { + return None; + } + + Some(Self { start, end }) + } + + /// Constructs a virtual range beginning at `start` and spanning `size` + /// bytes. + /// + /// Returns `None` if `size` is zero or if computing the end address would + /// overflow the virtual address representation. + #[must_use] + pub(crate) const fn from_start_size(start: VirtAddr, size: usize) -> Option { + if size == 0 { + return None; + } + + let Some(end) = start.checked_add(size) else { + return None; + }; + + Self::new(start, end) + } + + /// Returns the inclusive start address. + #[must_use] + pub(crate) const fn start(self) -> VirtAddr { + self.start + } + + /// Returns the exclusive end address. + #[must_use] + pub(crate) const fn end(self) -> VirtAddr { + self.end + } + + /// Returns whether this range overlaps `other`. + /// + /// Ranges that only meet at an endpoint do not overlap. + #[must_use] + pub(crate) const fn overlaps(self, other: Self) -> bool { + self.start.as_usize() < other.end.as_usize() && other.start.as_usize() < self.end.as_usize() + } +} diff --git a/kernel/src/memory/address.rs b/kernel/src/memory/address.rs index a865483..5b308af 100644 --- a/kernel/src/memory/address.rs +++ b/kernel/src/memory/address.rs @@ -70,6 +70,12 @@ impl fmt::LowerHex for PhysAddr { #[repr(transparent)] pub(crate) struct VirtAddr(usize); +impl fmt::Debug for VirtAddr { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{:#x}", self.0) + } +} + impl VirtAddr { /// Constructs a virtual address from its machine-sized representation. #[must_use] @@ -82,4 +88,15 @@ impl VirtAddr { pub(crate) const fn as_usize(self) -> usize { self.0 } + + /// Returns the virtual address obtained by adding `bytes`. + /// + /// Returns `None` if the addition would overflow. + #[must_use] + pub(crate) const fn checked_add(self, bytes: usize) -> Option { + match self.0.checked_add(bytes) { + Some(address) => Some(Self(address)), + None => None, + } + } } From 1914b12b2445494715cc47d9f2f471e1ab8ff5de Mon Sep 17 00:00:00 2001 From: Daniel-De-Dev Date: Mon, 5 Oct 2026 20:55:25 +0200 Subject: [PATCH 7/9] feat(paging): added map_range helper Added a wrapper around map_page that now will allow to map a range of memory instead. --- kernel/src/arch/riscv64/paging.rs | 88 ++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/kernel/src/arch/riscv64/paging.rs b/kernel/src/arch/riscv64/paging.rs index 1c6bc17..2e86ff8 100644 --- a/kernel/src/arch/riscv64/paging.rs +++ b/kernel/src/arch/riscv64/paging.rs @@ -9,7 +9,7 @@ use core::{ sync::atomic::{AtomicBool, Ordering}, }; -use crate::memory::{AllocatedFrame, BootFrameAllocator, PhysAddr, PhysFrame, VirtAddr}; +use crate::memory::{AllocatedFrame, BootFrameAllocator, PhysAddr, PhysFrame, VirtAddr, VirtRange}; /// Sign bit of an Sv39 virtual address. const SV39_SIGN_BIT: usize = 38; @@ -483,6 +483,82 @@ impl Sv39PageTable { Ok(Self { root }) } + /// Maps every 4 KiB page covering `virtual_range` to consecutive physical + /// frames beginning at `physical_start`. + /// + /// The virtual start must be page-aligned. The exclusive end may be + /// unaligned, its final partial page is mapped in full. + /// + /// # Errors + /// + /// Returns an error if the virtual start is unaligned, the range includes + /// non-canonical Sv39 addresses, the physical span is not representable, or + /// an individual page cannot be mapped. + /// + /// Address spans are validated before any entries are installed. Allocation + /// or mapping conflicts can still fail partway through; mappings already + /// installed by this call remain in the address space. + #[expect( + clippy::arithmetic_side_effects, + reason = "VirtRange bounds are ordered, and page offsets fit within the \ + validated virtual and physical spans" + )] + fn map_range( + &mut self, + bootstrap: &mut BootstrapPaging, + virtual_range: VirtRange, + physical_start: PhysFrame, + permissions: usize, + frames: &mut BootFrameAllocator<'_, '_>, + ) -> Result<(), PagingError> { + let start = virtual_range.start().as_usize(); + let end = virtual_range.end().as_usize(); + + if start & PAGE_MASK != 0 { + return Err(PagingError::VirtualAddressNotPageAligned { address: start }); + } + + if (SV39_LOW_HALF_END..SV39_HIGH_HALF_BASE).contains(&start) { + return Err(PagingError::VirtualAddressNotCanonical { address: start }); + } + + if start < SV39_LOW_HALF_END && end > SV39_LOW_HALF_END { + return Err(PagingError::VirtualAddressNotCanonical { + address: SV39_LOW_HALF_END, + }); + } + + let size = end - start; + let page_count = size.div_ceil(PAGE_SIZE); + let last_offset = (page_count - 1) * PAGE_SIZE; + let physical_base = physical_start.start_address(); + + physical_base + .checked_add(last_offset) + .filter(|address| address.as_usize() <= MAX_PHYSICAL_ADDRESS) + .ok_or(PagingError::PhysicalRangeNotRepresentable { + start: physical_base, + size, + })?; + + for page_index in 0..page_count { + let offset = page_index * PAGE_SIZE; + let virtual_address = VirtAddr::new(start + offset); + let physical_address = PhysAddr::new(physical_base.as_usize() + offset); + + let Some(frame) = PhysFrame::from_start(physical_address) else { + return Err(PagingError::PhysicalRangeNotRepresentable { + start: physical_base, + size, + }); + }; + + self.map_page(bootstrap, virtual_address, frame, permissions, frames)?; + } + + Ok(()) + } + /// Maps one 4 KiB virtual page to `physical_frame`. /// /// `permissions` contains the Sv39 `R`, `W`, and `X` bits for the leaf. @@ -576,6 +652,16 @@ pub(crate) enum PagingError { address: PhysAddr, }, + /// The physical frames covering a requested byte range cannot all be + /// represented by Sv39, or computing the final frame address would overflow. + PhysicalRangeNotRepresentable { + /// Physical start address corresponding to the virtual range start. + start: PhysAddr, + + /// Requested range size in bytes before rounding up to complete pages. + size: usize, + }, + /// No physical frame remained for a required page table. OutOfPhysicalFrames, From 94a25d77dea35c35bfe5230b3bcac70c2251fdf7 Mon Sep 17 00:00:00 2001 From: Daniel-De-Dev Date: Tue, 6 Oct 2026 15:53:34 +0200 Subject: [PATCH 8/9] feature(KernelPaging): kernel switching from bootstrap paging Explicitly defined a transition from the bootstrap paging that worked using mostly using gigapages, to now removing identity mappings and using granular 4KiB pages with appropriate permissions. --- kernel/src/arch.rs | 2 +- kernel/src/arch/riscv64.rs | 2 +- kernel/src/arch/riscv64/paging.rs | 65 ++++- kernel/src/arch/riscv64/paging/kernel.rs | 332 +++++++++++++++++++++++ kernel/src/main.rs | 42 ++- nix/qemu.nix | 14 +- 6 files changed, 445 insertions(+), 12 deletions(-) create mode 100644 kernel/src/arch/riscv64/paging/kernel.rs diff --git a/kernel/src/arch.rs b/kernel/src/arch.rs index a144bdc..d0066a1 100644 --- a/kernel/src/arch.rs +++ b/kernel/src/arch.rs @@ -7,4 +7,4 @@ mod riscv64; #[cfg(target_arch = "riscv64")] -pub(super) use riscv64::{BootstrapPaging, halt, init_trap}; +pub(super) use riscv64::{BootstrapPaging, KernelPaging, halt, init_trap}; diff --git a/kernel/src/arch/riscv64.rs b/kernel/src/arch/riscv64.rs index 4183c89..458fbd6 100644 --- a/kernel/src/arch/riscv64.rs +++ b/kernel/src/arch/riscv64.rs @@ -7,7 +7,7 @@ mod boot; mod paging; mod trap; -pub(crate) use paging::BootstrapPaging; +pub(crate) use paging::{BootstrapPaging, KernelPaging}; pub(crate) use trap::init as init_trap; use core::arch::asm; diff --git a/kernel/src/arch/riscv64/paging.rs b/kernel/src/arch/riscv64/paging.rs index 2e86ff8..c535a80 100644 --- a/kernel/src/arch/riscv64/paging.rs +++ b/kernel/src/arch/riscv64/paging.rs @@ -1,7 +1,19 @@ //! RISC-V Sv39 paging support. //! -//! This module provides address-translation helpers and temporary mappings used -//! while the kernel is running under its bootstrap Sv39 address space. +//! This module provides bootstrap paging, construction of the kernel address +//! space, and temporary access to physical frames after the root transition. +// TODO: Support larger leaves for aligned, physically contiguous ranges +// with uniform permissions. +// TODO: Protect the frame window against interrupt re-entry before handlers +// begin using paging. +// TODO: Add synchronization and remote translation invalidation before +// multiple kernel harts share mutable mappings. +// TODO: Add automatic temporary-mapping cleanup if recoverable unwinding +// or early returns are introduced after installing a mapping. + +mod kernel; + +pub(crate) use kernel::KernelPaging; use core::{ arch::asm, @@ -9,7 +21,9 @@ use core::{ sync::atomic::{AtomicBool, Ordering}, }; -use crate::memory::{AllocatedFrame, BootFrameAllocator, PhysAddr, PhysFrame, VirtAddr, VirtRange}; +use crate::memory::{ + AllocatedFrame, BootFrameAllocator, PhysAddr, PhysFrame, PhysRange, VirtAddr, VirtRange, +}; /// Sign bit of an Sv39 virtual address. const SV39_SIGN_BIT: usize = 38; @@ -662,6 +676,21 @@ pub(crate) enum PagingError { size: usize, }, + /// A kernel section could not be translated into an aligned physical frame. + InvalidKernelSection { + /// Linker-defined virtual section range. + range: VirtRange, + }, + + /// The DTB alias had a different page offset or its page coverage overflowed. + InvalidDtbMapping { + /// Virtual address of the first byte of the DTB. + virtual_address: VirtAddr, + + /// Physical range covering the DTB bytes. + physical_range: PhysRange, + }, + /// No physical frame remained for a required page table. OutOfPhysicalFrames, @@ -809,6 +838,36 @@ fn flush_all() { } } +/// Installs a complete replacement root on the current hart. +/// +/// # Safety +/// +/// `root` must be representable in `satp` and identify a fully initialized Sv39 +/// hierarchy that preserves all live code, stack, trap, and borrowed-data +/// mappings. Its table frames must remain allocated while the address space +/// is active. The caller must consume the capability controlling the previous +/// root and expose only the replacement capability after this operation +/// completes. +unsafe fn activate_root(root: PhysFrame) { + let satp = (SATP_MODE_SV39 << SATP_MODE_SHIFT) | (root.start_address().as_usize() >> PAGE_SHIFT); + + // SAFETY: + // The caller establishes the replacement mapping and allocation invariants. + // The first fence orders completed table writes before the switch. The + // second invalidates old translations, including those using ASID zero. + // The assembly has a memory clobber. No fallible operation follows the satp + // write. + unsafe { + asm!( + "sfence.vma zero, zero", + "csrw satp, {satp}", + "sfence.vma zero, zero", + satp = in(reg) satp, + options(nostack), + ); + } +} + /// Reads one entry from a page table that is currently mapped at `address`. /// /// # Safety diff --git a/kernel/src/arch/riscv64/paging/kernel.rs b/kernel/src/arch/riscv64/paging/kernel.rs new file mode 100644 index 0000000..f2d46ba --- /dev/null +++ b/kernel/src/arch/riscv64/paging/kernel.rs @@ -0,0 +1,332 @@ +//! The active kernel Sv39 address space and temporary physical-frame access. + +use core::{fmt, ptr}; + +use super::{ + BOOT_FRAME_WINDOW_BASE, BootstrapPaging, GIGAPAGE_SHIFT, GIGAPAGE_SIZE, MEGAPAGE_SHIFT, + PAGE_MASK, PAGE_SHIFT, PAGE_SIZE, PTE_ACCESSED, PTE_DIRTY, PTE_EXECUTE, PTE_READ, PTE_VALID, + PTE_WRITE, PagingError, Sv39PageTable, VPN_MASK, activate_root, flush_address, page_table_entry, +}; +use crate::memory::{ + AllocatedFrame, BootFrameAllocator, KernelSections, PhysAddr, PhysFrame, PhysRange, VirtAddr, + VirtRange, +}; + +/// Virtual page mapping the L0 table that controls the kernel frame window. +/// +/// This lies in the final 2 MiB below the kernel's 1 GiB virtual region. +const FIXMAP_TABLE_BASE: usize = + BOOT_FRAME_WINDOW_BASE - GIGAPAGE_SIZE - (1usize << MEGAPAGE_SHIFT); + +/// Virtual page reserved for temporary access to one physical frame. +/// +/// This page begins unmapped. The adjacent table alias will allow its leaf +/// entry to be changed after the replacement root is activated. +const FIXMAP_FRAME_BASE: usize = FIXMAP_TABLE_BASE + PAGE_SIZE; + +/// Index of the temporary frame slot within its controlling L0 table. +const FIXMAP_FRAME_INDEX: usize = (FIXMAP_FRAME_BASE >> PAGE_SHIFT) & VPN_MASK; + +/// Exclusive control of the active kernel address space. +/// +/// This capability is created only by consuming [`BootstrapPaging`] and +/// activating the replacement root. It cannot be cloned or copied and must +/// be used on the hart where that root was activated. The root need not have a +/// permanent virtual mapping: physical frames are accessed through the one-page +/// window controlled by `frame_slot`. +/// +/// All page-table frames remain permanently reserved by the monotonic boot +/// allocator. No other code may change the frame-window entry while this +/// capability exists. +#[must_use = "retain the capability controlling the active kernel address space"] +pub(crate) struct KernelPaging { + /// Allocation containing the active Sv39 root. + root: AllocatedFrame, + + /// L0 table mapped at [`FIXMAP_TABLE_BASE`]. + fixmap_table: PhysFrame, + + /// Complete virtual pages covering the DTB at its existing alias. + dtb_mapping: VirtRange, + + /// Writable PTE for the temporary frame slot, through the fixed table alias. + frame_slot: *mut usize, +} + +impl fmt::Debug for KernelPaging { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("KernelPaging") + .field("root", &self.root.start_address()) + .field("dtb_mapping", &self.dtb_mapping) + .field("fixmap_table", &self.fixmap_table.start_address()) + .field("fixmap_table_alias", &VirtAddr::new(FIXMAP_TABLE_BASE)) + .field("frame_slot_pointer", &self.frame_slot) + .field("frame_slot_alias", &VirtAddr::new(FIXMAP_FRAME_BASE)) + .finish() + } +} + +impl BootstrapPaging { + /// Consumes bootstrap paging and enters the normal kernel address space. + /// + /// Kernel section addresses are translated relative to + /// `sections.virtual_start`. Every mapping uses a 4 KiB leaf. + /// + /// Only pages covering `dtb_physical` are retained at the existing + /// `dtb_virtual` alias. No bootstrap or identity mappings are copied. + /// All mappings are built before the root is switched, and the bootstrap + /// capability cannot be reclaimed afterward. No separate prepared capability + /// is exposed. + /// + /// # Safety + /// + /// `sections` and `kernel_physical_start` must describe the live kernel's + /// actual virtual sections and physical load address. They must cover all + /// live kernel code, constants, data, stacks, and trap state. `dtb_physical` + /// must cover the complete DTB backing any retained references, and + /// `dtb_virtual` must be its existing bootstrap alias. No other borrowed + /// memory may depend on a mapping omitted by the new root. + /// + /// The transition must run on the hart whose bootstrap root this capability + /// controls. The returned capability must be used only while the replacement + /// root remains active on that hart. + /// + /// Supervisor interrupts must remain disabled during the transition and + /// temporary frame access. No other hart or trap handler may modify these + /// paging structures or use the temporary frame window concurrently. + /// + /// # Errors + /// + /// Returns an error if a section or DTB range cannot be translated, mappings + /// conflict, or a required page-table frame cannot be allocated or accessed. + /// Errors occur before activation, leaving the bootstrap root active. + /// The consumed capability is not returned, so startup must treat failure as + /// unrecoverable. Allocated frames remain reserved because the boot allocator + /// does not reclaim frames. + pub(crate) unsafe fn into_kernel( + mut self, + kernel_physical_start: PhysAddr, + sections: &KernelSections, + dtb_virtual: VirtAddr, + dtb_physical: PhysRange, + frames: &mut BootFrameAllocator<'_, '_>, + ) -> Result { + let (dtb_mapping, dtb_frame) = dtb_page_mapping(dtb_virtual, dtb_physical)?; + let mut page_table = Sv39PageTable::new(&mut self, frames)?; + + for (section, permissions) in [ + (sections.text, PTE_READ | PTE_EXECUTE), + (sections.rodata, PTE_READ), + (sections.data, PTE_READ | PTE_WRITE), + (sections.bss, PTE_READ | PTE_WRITE), + (sections.stack, PTE_READ | PTE_WRITE), + ] { + let Some(range) = section else { + continue; + }; + + let physical_start = range + .start() + .as_usize() + .checked_sub(sections.virtual_start.as_usize()) + .and_then(|offset| kernel_physical_start.checked_add(offset)) + .and_then(PhysFrame::from_start) + .ok_or(PagingError::InvalidKernelSection { range })?; + + page_table.map_range(&mut self, range, physical_start, permissions, frames)?; + } + + page_table.map_range(&mut self, dtb_mapping, dtb_frame, PTE_READ, frames)?; + + let fixmap_table = page_table.prepare_fixmap(&mut self, frames)?; + + let kernel = KernelPaging { + root: page_table.root, + fixmap_table, + dtb_mapping, + frame_slot: ptr::with_exposed_provenance_mut::(FIXMAP_TABLE_BASE) + .wrapping_add(FIXMAP_FRAME_INDEX), + }; + + // SAFETY: + // The caller supplies the complete live kernel layout and retained DTB + // range. Construction above installs those mappings and the permanent + // control-table alias. All table frames remain reserved, this consumes the + // bootstrap capability, and only `kernel` is returned after the switch. + unsafe { + activate_root(kernel.root.frame()); + } + + Ok(kernel) + } +} + +impl KernelPaging { + /// Executes `operation` with `frame` temporarily mapped read/write. + /// + /// The alias maps exactly one 4 KiB frame, is non-executable and + /// supervisor-only, and is removed before returning. The closure must not + /// enable interrupts. Dereferencing the supplied address still requires the + /// caller to establish valid memory access and ownership for `frame`. + /// + /// # Errors + /// + /// Returns an error if the frame cannot be represented by Sv39 or the + /// temporary slot is already occupied. + pub(crate) fn with_frame( + &mut self, + frame: PhysFrame, + operation: impl FnOnce(VirtAddr) -> R, + ) -> Result { + let flags = PTE_VALID | PTE_READ | PTE_WRITE | PTE_ACCESSED | PTE_DIRTY; + let entry = page_table_entry(frame.start_address(), flags)?; + + // SAFETY: + // `KernelPaging` guarantees that the controlling L0 table is permanently + // mapped read/write and that `frame_slot` points to one of its entries. + let existing = unsafe { ptr::read_volatile(self.frame_slot) }; + + if existing != 0 { + return Err(PagingError::PageAlreadyMapped { + address: FIXMAP_FRAME_BASE, + entry: existing, + }); + } + + // SAFETY: + // The entry pointer is valid and exclusively controlled by this capability, + // and the new leaf maps one representable, aligned physical frame. + unsafe { + ptr::write_volatile(self.frame_slot, entry); + } + + let address = VirtAddr::new(FIXMAP_FRAME_BASE); + + // Make the newly installed mapping available. + flush_address(address); + + let result = operation(address); + + // SAFETY: + // `frame_slot` remains a valid, exclusively controlled pointer into the + // permanently mapped L0 table. Clear its temporary frame mapping. + unsafe { + ptr::write_volatile(self.frame_slot, 0); + } + + flush_address(address); + + Ok(result) + } + + /// Zeroes every byte in an allocated physical frame through the kernel's + /// temporary frame window. + /// + /// # Errors + /// + /// Returns an error if the physical address cannot be represented by Sv39 + /// or the temporary frame slot is already occupied. + #[expect( + clippy::needless_pass_by_ref_mut, + reason = "the mutable borrow intentionally provides exclusive access to \ + the allocated frame while its memory is initialized" + )] + pub(crate) fn zero_frame(&mut self, frame: &mut AllocatedFrame) -> Result<(), PagingError> { + self.with_frame(frame.frame(), |address| { + let pointer = ptr::with_exposed_provenance_mut::(address.as_usize()); + + // SAFETY: + // The window maps the complete 4 KiB frame for this closure. The mutable + // `AllocatedFrame` borrow provides exclusive ownership of the RAM being + // initialized, and this operation creates no persistent references. + unsafe { + ptr::write_bytes(pointer, 0, PAGE_SIZE); + } + }) + } +} + +impl Sv39PageTable { + /// Maps the frame-window L0 table into its own control page, leaving the + /// adjacent temporary frame slot empty. + /// + /// # Errors + /// + /// Returns an error if the tables cannot be allocated or accessed, or either + /// fixmap page is already occupied. + fn prepare_fixmap( + &mut self, + bootstrap: &mut BootstrapPaging, + frames: &mut BootFrameAllocator<'_, '_>, + ) -> Result { + let vpn2 = (FIXMAP_TABLE_BASE >> GIGAPAGE_SHIFT) & VPN_MASK; + let vpn1 = (FIXMAP_TABLE_BASE >> MEGAPAGE_SHIFT) & VPN_MASK; + + let level_1 = bootstrap.next_table(self.root.frame(), vpn2, frames)?; + let level_0 = bootstrap.next_table(level_1, vpn1, frames)?; + let existing = bootstrap.read_page_table_entry(level_0, FIXMAP_FRAME_INDEX)?; + + if existing != 0 { + return Err(PagingError::PageAlreadyMapped { + address: FIXMAP_FRAME_BASE, + entry: existing, + }); + } + + self.map_page( + bootstrap, + VirtAddr::new(FIXMAP_TABLE_BASE), + level_0, + PTE_READ | PTE_WRITE, + frames, + )?; + + Ok(level_0) + } +} + +/// Computes the aligned virtual range and first physical frame covering the +/// DTB. +/// +/// # Errors +/// +/// Returns an error if the physical and virtual page offsets differ or the +/// complete page coverage cannot be represented. +#[expect( + clippy::arithmetic_side_effects, + reason = "PhysRange guarantees strictly ordered physical bounds" +)] +fn dtb_page_mapping( + virtual_address: VirtAddr, + physical_range: PhysRange, +) -> Result<(VirtRange, PhysFrame), PagingError> { + let invalid_mapping = || PagingError::InvalidDtbMapping { + virtual_address, + physical_range, + }; + + let physical_start = physical_range.start().as_usize(); + let virtual_start = virtual_address.as_usize(); + + if virtual_start & PAGE_MASK != physical_start & PAGE_MASK { + return Err(invalid_mapping()); + } + + let size = physical_range.end().as_usize() - physical_start; + let virtual_end = virtual_address + .checked_add(size) + .and_then(|end| end.checked_add(PAGE_MASK)) + .ok_or_else(invalid_mapping)?; + + let virtual_range = VirtRange::new( + VirtAddr::new(virtual_start & !PAGE_MASK), + VirtAddr::new(virtual_end.as_usize() & !PAGE_MASK), + ) + .ok_or_else(invalid_mapping)?; + + let physical_frame = PhysFrame::from_start(PhysAddr::new(physical_start & !PAGE_MASK)) + .ok_or_else(invalid_mapping)?; + + Ok((virtual_range, physical_frame)) +} diff --git a/kernel/src/main.rs b/kernel/src/main.rs index b257e3e..53daea3 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -18,6 +18,7 @@ use core::panic::PanicInfo; use fdt::Fdt; use memory::{BootFrameAllocator, PhysAddr, PhysRange, kernel_sections}; +// TODO: add debug/trace info for allocations /// Runs the kernel after architecture-specific initialization. /// /// `hart_id` identifies the RISC-V hart on which the kernel was entered. @@ -75,9 +76,8 @@ extern "C" fn main(hart_id: usize, dtb: usize, kernel_phys_start: usize) -> ! { // SAFETY: // `BootstrapPaging::map_fdt` established a readable mapping containing the - // complete DTB. The bootstrap paging capability remains alive, so that - // mapping remains valid and unmodified while `fdt` and values borrowing from - // it are used. + // complete DTB. `into_kernel` transition preserves all pages backing this + // blob at the same virtual alias. let fdt = match unsafe { Fdt::from_ptr(dtb_ptr) } { Ok(fdt) => fdt, Err(error) => { @@ -129,7 +129,7 @@ extern "C" fn main(hart_id: usize, dtb: usize, kernel_phys_start: usize) -> ! { // footprint. `dtb_range` covers the same firmware-provided DTB backing `fdt`. // The FDT is trusted to describe all other physical memory that is unavailable // for allocation through its reservation information. - let frames = match unsafe { BootFrameAllocator::claim(&fdt, kernel_range, dtb_range) } { + let mut frames = match unsafe { BootFrameAllocator::claim(&fdt, kernel_range, dtb_range) } { Ok(frames) => frames, Err(error) => { logging::error!( @@ -141,7 +141,39 @@ extern "C" fn main(hart_id: usize, dtb: usize, kernel_phys_start: usize) -> ! { } }; - logging::debug!("kernel halting"); + // SAFETY: + // These sections come directly from the live kernel's linker symbols, and + // bootstrap supplied the actual physical load address. They cover the live + // code, trap vector, constants, data, and stack. `dtb_range` and `dtb_virtual` + // describe the complete blob backing `fdt` and the frame allocator. No other + // retained reference depends on a bootstrap-only mapping. Startup relies on + // the boot environment entering this hart with supervisor interrupts disabled + // and does not enable them before this transition. No other hart accesses + // these paging structures. + let kernel_paging = match unsafe { + bootstrap_paging.into_kernel( + kernel_phys_start, + &kernel_sections, + dtb_virtual, + dtb_range, + &mut frames, + ) + } { + Ok(paging) => paging, + Err(error) => { + logging::error!( + "Failed to enter kernel paging ({error:?}); \ + kernel startup is unrecoverable, halting" + ); + + arch::halt(); + } + }; + + logging::info!("Kernel paging active"); + logging::debug!("{:#?}", kernel_paging); + + logging::info!("kernel halting"); arch::halt() } diff --git a/nix/qemu.nix b/nix/qemu.nix index e9fc51b..50d550e 100644 --- a/nix/qemu.nix +++ b/nix/qemu.nix @@ -106,8 +106,18 @@ exit 1 fi - if ! grep -Fq "kernel entered" "$log"; then - echo "kernel did not reach main" >&2 + for expected in \ + "kernel entered" \ + "Kernel paging active"; do + if ! grep -Fq "$expected" "$log"; then + echo "kernel did not reach: $expected" >&2 + cat "$log" >&2 + exit 1 + fi + done + + if grep -Fq "Unhandled supervisor trap" "$log"; then + echo "kernel trapped during startup" >&2 cat "$log" >&2 exit 1 fi From 34bb8e5b1462e050f054af3c7e3c3e9dbf8d7bc5 Mon Sep 17 00:00:00 2001 From: Daniel-De-Dev Date: Tue, 6 Oct 2026 16:56:11 +0200 Subject: [PATCH 9/9] chore: review & fix up documentation --- kernel/src/arch/riscv64/paging.rs | 6 ++--- kernel/src/arch/riscv64/paging/kernel.rs | 27 ++++++++++++++--------- kernel/src/main.rs | 4 ++-- kernel/src/memory.rs | 6 ++--- kernel/src/memory/frame/boot_allocator.rs | 4 ++++ 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/kernel/src/arch/riscv64/paging.rs b/kernel/src/arch/riscv64/paging.rs index c535a80..55ba4f3 100644 --- a/kernel/src/arch/riscv64/paging.rs +++ b/kernel/src/arch/riscv64/paging.rs @@ -114,8 +114,8 @@ const BOOT_FRAME_WINDOW_VPN2: usize = VPN_MASK; const BOOT_FRAME_WINDOW_BASE: usize = SV39_HIGH_HALF_BASE + ((BOOT_FRAME_WINDOW_VPN2 - SV39_HIGH_HALF_VPN2) * GIGAPAGE_SIZE); -/// Flags used by the temporary physical-frame mapping. -const BOOT_FRAME_PTE_FLAGS: usize = PTE_VALID | PTE_READ | PTE_WRITE | PTE_ACCESSED | PTE_DIRTY; +/// Flags for read/write, non-executable supervisor frame-window leaves. +const FRAME_WINDOW_PTE_FLAGS: usize = PTE_VALID | PTE_READ | PTE_WRITE | PTE_ACCESSED | PTE_DIRTY; /// Bit position of `satp.MODE` on RV64. const SATP_MODE_SHIFT: usize = 60; @@ -330,7 +330,7 @@ impl BootstrapPaging { let physical_base = PhysAddr::new(frame_address.as_usize() & !GIGAPAGE_MASK); - let entry = page_table_entry(physical_base, BOOT_FRAME_PTE_FLAGS)?; + let entry = page_table_entry(physical_base, FRAME_WINDOW_PTE_FLAGS)?; // SAFETY: // `BootstrapPaging` guarantees that `root_virtual` maps the complete diff --git a/kernel/src/arch/riscv64/paging/kernel.rs b/kernel/src/arch/riscv64/paging/kernel.rs index f2d46ba..63e2a51 100644 --- a/kernel/src/arch/riscv64/paging/kernel.rs +++ b/kernel/src/arch/riscv64/paging/kernel.rs @@ -3,9 +3,9 @@ use core::{fmt, ptr}; use super::{ - BOOT_FRAME_WINDOW_BASE, BootstrapPaging, GIGAPAGE_SHIFT, GIGAPAGE_SIZE, MEGAPAGE_SHIFT, - PAGE_MASK, PAGE_SHIFT, PAGE_SIZE, PTE_ACCESSED, PTE_DIRTY, PTE_EXECUTE, PTE_READ, PTE_VALID, - PTE_WRITE, PagingError, Sv39PageTable, VPN_MASK, activate_root, flush_address, page_table_entry, + BOOT_FRAME_WINDOW_BASE, BootstrapPaging, FRAME_WINDOW_PTE_FLAGS, GIGAPAGE_SHIFT, GIGAPAGE_SIZE, + MEGAPAGE_SHIFT, PAGE_MASK, PAGE_SHIFT, PAGE_SIZE, PTE_EXECUTE, PTE_READ, PTE_WRITE, PagingError, + Sv39PageTable, VPN_MASK, activate_root, flush_address, page_table_entry, }; use crate::memory::{ AllocatedFrame, BootFrameAllocator, KernelSections, PhysAddr, PhysFrame, PhysRange, VirtAddr, @@ -76,8 +76,7 @@ impl BootstrapPaging { /// Only pages covering `dtb_physical` are retained at the existing /// `dtb_virtual` alias. No bootstrap or identity mappings are copied. /// All mappings are built before the root is switched, and the bootstrap - /// capability cannot be reclaimed afterward. No separate prepared capability - /// is exposed. + /// capability cannot be reclaimed afterward. /// /// # Safety /// @@ -165,10 +164,17 @@ impl BootstrapPaging { impl KernelPaging { /// Executes `operation` with `frame` temporarily mapped read/write. /// - /// The alias maps exactly one 4 KiB frame, is non-executable and - /// supervisor-only, and is removed before returning. The closure must not - /// enable interrupts. Dereferencing the supplied address still requires the - /// caller to establish valid memory access and ownership for `frame`. + /// The alias maps exactly one 4 KiB frame and is non-executable and + /// supervisor-only. The supplied address is valid only during `operation`; + /// references created through it must not outlive the closure. + /// Dereferencing it requires the caller to establish valid memory access + /// and ownership for `frame`. + /// + /// Supervisor interrupts must already be disabled and remain disabled + /// throughout this call. + /// + /// The mapping is removed when the closure returns normally. Clean up does + /// not run during unwinding; current kernel panics halt without unwinding. /// /// # Errors /// @@ -179,8 +185,7 @@ impl KernelPaging { frame: PhysFrame, operation: impl FnOnce(VirtAddr) -> R, ) -> Result { - let flags = PTE_VALID | PTE_READ | PTE_WRITE | PTE_ACCESSED | PTE_DIRTY; - let entry = page_table_entry(frame.start_address(), flags)?; + let entry = page_table_entry(frame.start_address(), FRAME_WINDOW_PTE_FLAGS)?; // SAFETY: // `KernelPaging` guarantees that the controlling L0 table is permanently diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 53daea3..80cdf48 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -127,8 +127,8 @@ extern "C" fn main(hart_id: usize, dtb: usize, kernel_phys_start: usize) -> ! { // The architecture bootstrap preserves the actual physical start of the live // kernel image, so `kernel_range` covers its complete linker-defined boot // footprint. `dtb_range` covers the same firmware-provided DTB backing `fdt`. - // The FDT is trusted to describe all other physical memory that is unavailable - // for allocation through its reservation information. + // The FDT is trusted to describe all other physical memory that is + // unavailable for allocation through its reservation information. let mut frames = match unsafe { BootFrameAllocator::claim(&fdt, kernel_range, dtb_range) } { Ok(frames) => frames, Err(error) => { diff --git a/kernel/src/memory.rs b/kernel/src/memory.rs index a2f03bb..373666f 100644 --- a/kernel/src/memory.rs +++ b/kernel/src/memory.rs @@ -1,4 +1,4 @@ -//! Core physical memory abstractions. +//! Core kernel memory abstractions. //! //! This module provides the common physical-memory vocabulary used by the //! kernel's memory-management code. @@ -43,9 +43,9 @@ pub(crate) struct VirtRange { /// /// Each range includes its start and excludes its end. Section starts are /// page-aligned, but ends describe the actual section contents and may require -/// rounding up when mapping complete pages. Empty sections have equal bounds. +/// rounding up when mapping complete pages. /// -/// None represents an empty section. +/// Empty sections have equal linker bounds and are represented by `None`. #[derive(Debug)] pub(crate) struct KernelSections { /// Virtual origin corresponding to the physical kernel image start. diff --git a/kernel/src/memory/frame/boot_allocator.rs b/kernel/src/memory/frame/boot_allocator.rs index ad12eef..1639c30 100644 --- a/kernel/src/memory/frame/boot_allocator.rs +++ b/kernel/src/memory/frame/boot_allocator.rs @@ -153,6 +153,10 @@ impl<'fdt, 'dtb> BootFrameAllocator<'fdt, 'dtb> { /// /// # Safety /// + /// This capability must be used only on the claiming hart while its + /// bootstrap root remains active. Supervisor interrupts must remain + /// disabled during temporary frame access. + /// /// Every FDT-described memory range not excluded by `kernel`, `dtb`, or the /// FDT's reservation information must refer to physical RAM that the kernel /// may exclusively allocate.