diff --git a/.harper-dictionary.txt b/.harper-dictionary.txt index 0fc403e..a7eaba4 100644 --- a/.harper-dictionary.txt +++ b/.harper-dictionary.txt @@ -28,3 +28,4 @@ stackless u32 usize v0 +writably diff --git a/kernel/linker.ld.in b/kernel/linker.ld.in index c09a15b..89de375 100644 --- a/kernel/linker.ld.in +++ b/kernel/linker.ld.in @@ -119,6 +119,10 @@ SECTIONS *(.srodata .srodata.*) *(.rodata .rodata.*) + . = ALIGN(8); + *(.eh_frame .eh_frame.*) + *(.eh_frame_hdr) + _rodata_end = .; } diff --git a/kernel/src/arch.rs b/kernel/src/arch.rs index 9a28af4..d0066a1 100644 --- a/kernel/src/arch.rs +++ b/kernel/src/arch.rs @@ -7,4 +7,4 @@ mod riscv64; #[cfg(target_arch = "riscv64")] -pub(super) use riscv64::{halt, init_trap, map_bootstrap_fdt}; +pub(super) use riscv64::{BootstrapPaging, KernelPaging, halt, init_trap}; diff --git a/kernel/src/arch/riscv64.rs b/kernel/src/arch/riscv64.rs index 9273999..458fbd6 100644 --- a/kernel/src/arch/riscv64.rs +++ b/kernel/src/arch/riscv64.rs @@ -7,7 +7,7 @@ mod boot; mod paging; mod trap; -pub(crate) use paging::map_bootstrap_fdt; +pub(crate) use paging::{BootstrapPaging, KernelPaging}; pub(crate) use trap::init as init_trap; use core::arch::asm; diff --git a/kernel/src/arch/riscv64/boot.rs b/kernel/src/arch/riscv64/boot.rs index cb8cb82..a6c9a67 100644 --- a/kernel/src/arch/riscv64/boot.rs +++ b/kernel/src/arch/riscv64/boot.rs @@ -17,6 +17,9 @@ //! `OpenSBI` supplies the hart ID in `a0` and the physical device-tree address //! in `a1`. The physical bootstrap places the kernel's physical start address //! in `a2`. All three values are preserved until `main` is entered. +// TODO: Look into if kernel physical boot address needs to be passed in the +// first place as an argument and inaccessible otherwise, else fix via linker +// script. use core::arch::global_asm; diff --git a/kernel/src/arch/riscv64/paging.rs b/kernel/src/arch/riscv64/paging.rs index f9e52d2..55ba4f3 100644 --- a/kernel/src/arch/riscv64/paging.rs +++ b/kernel/src/arch/riscv64/paging.rs @@ -1,18 +1,57 @@ //! RISC-V Sv39 paging support. //! -//! This module provides address-translation helpers and temporary mappings used -//! while the kernel is running under its bootstrap Sv39 address space. +//! This module provides bootstrap paging, construction of the kernel address +//! space, and temporary access to physical frames after the root transition. +// TODO: Support larger leaves for aligned, physically contiguous ranges +// with uniform permissions. +// TODO: Protect the frame window against interrupt re-entry before handlers +// begin using paging. +// TODO: Add synchronization and remote translation invalidation before +// multiple kernel harts share mutable mappings. +// TODO: Add automatic temporary-mapping cleanup if recoverable unwinding +// or early returns are introduced after installing a mapping. + +mod kernel; + +pub(crate) use kernel::KernelPaging; + +use core::{ + arch::asm, + ptr, + sync::atomic::{AtomicBool, Ordering}, +}; + +use crate::memory::{ + AllocatedFrame, BootFrameAllocator, PhysAddr, PhysFrame, PhysRange, VirtAddr, VirtRange, +}; + +/// Sign bit of an Sv39 virtual address. +const SV39_SIGN_BIT: usize = 38; -use core::{arch::asm, ptr}; +/// Lowest canonical virtual address in the upper half of Sv39. +const SV39_HIGH_HALF_BASE: usize = usize::MAX << SV39_SIGN_BIT; -use crate::memory::{PhysAddr, VirtAddr}; +/// First root-table index belonging to the canonical Sv39 upper half. +const SV39_HIGH_HALF_VPN2: usize = (SV39_HIGH_HALF_BASE >> GIGAPAGE_SHIFT) & VPN_MASK; -/// Lowest canonical virtual address in the upper half of Sv39. -const SV39_HIGH_HALF_BASE: usize = 0xffff_ffc0_0000_0000; +/// First virtual address outside the canonical Sv39 lower half. +const SV39_LOW_HALF_END: usize = 1usize << SV39_SIGN_BIT; /// Number of low address bits forming a page offset. const PAGE_SHIFT: usize = 12; +/// Size in bytes of one Sv39 page. +const PAGE_SIZE: usize = 1 << PAGE_SHIFT; + +/// Mask selecting an offset within one Sv39 page. +const PAGE_MASK: usize = PAGE_SIZE - 1; + +/// Bit offset of an Sv39 level-1 virtual page number. +const MEGAPAGE_SHIFT: usize = 21; + +/// Number of entries in one Sv39 page table. +const PAGE_TABLE_ENTRIES: usize = 512; + /// Bit offset of an Sv39 level-2 virtual page number. const GIGAPAGE_SHIFT: usize = 30; @@ -28,18 +67,27 @@ const VPN_MASK: usize = 0x1ff; /// Bit offset of the physical page number within an Sv39 PTE. const PTE_PPN_SHIFT: usize = 10; -/// Size in bytes of one Sv39 page-table entry. -const PTE_SIZE: usize = size_of::(); - /// PTE bit marking an entry as valid. const PTE_VALID: usize = 1 << 0; /// PTE bit permitting reads through a leaf mapping. const PTE_READ: usize = 1 << 1; +/// PTE bit permitting writes through a leaf mapping. +const PTE_WRITE: usize = 1 << 2; + +/// PTE bit permitting instruction fetches through a leaf mapping. +const PTE_EXECUTE: usize = 1 << 3; + /// PTE bit recording that a leaf mapping has been accessed. const PTE_ACCESSED: usize = 1 << 6; +/// PTE bit recording that a writable leaf mapping has been modified. +const PTE_DIRTY: usize = 1 << 7; + +/// Bits distinguishing an Sv39 leaf from a non-leaf entry. +const PTE_LEAF_MASK: usize = PTE_READ | PTE_WRITE | PTE_EXECUTE; + /// Flags for a read-only boot-time FDT leaf mapping. const BOOT_FDT_PTE_FLAGS: usize = PTE_VALID | PTE_READ | PTE_ACCESSED; @@ -47,7 +95,7 @@ const BOOT_FDT_PTE_FLAGS: usize = PTE_VALID | PTE_READ | PTE_ACCESSED; const BOOT_FDT_WINDOW_BASE: usize = SV39_HIGH_HALF_BASE; /// Root-table index containing the start of the boot-time FDT window. -const BOOT_FDT_WINDOW_VPN2: usize = (BOOT_FDT_WINDOW_BASE >> GIGAPAGE_SHIFT) & VPN_MASK; +const BOOT_FDT_WINDOW_VPN2: usize = SV39_HIGH_HALF_VPN2; /// Number of consecutive level-2 leaves available to the boot-time FDT window. /// @@ -55,6 +103,20 @@ const BOOT_FDT_WINDOW_VPN2: usize = (BOOT_FDT_WINDOW_BASE >> GIGAPAGE_SHIFT) & V /// end of its first 1 GiB region. const BOOT_FDT_WINDOW_GIGAPAGES: usize = 5; +/// Root-table index reserved for temporary access to physical frames. +/// +/// VPN[2] 511 corresponds to the final 1 GiB of the canonical Sv39 +/// higher-half address space. The bootstrap kernel occupies VPN[2] 510, +/// while the temporary FDT window begins at VPN[2] 256. +const BOOT_FRAME_WINDOW_VPN2: usize = VPN_MASK; + +/// First virtual address of the temporary physical-frame access window. +const BOOT_FRAME_WINDOW_BASE: usize = + SV39_HIGH_HALF_BASE + ((BOOT_FRAME_WINDOW_VPN2 - SV39_HIGH_HALF_VPN2) * GIGAPAGE_SIZE); + +/// Flags for read/write, non-executable supervisor frame-window leaves. +const FRAME_WINDOW_PTE_FLAGS: usize = PTE_VALID | PTE_READ | PTE_WRITE | PTE_ACCESSED | PTE_DIRTY; + /// Bit position of `satp.MODE` on RV64. const SATP_MODE_SHIFT: usize = 60; @@ -76,6 +138,510 @@ const PHYSICAL_ADDRESS_BITS: usize = PHYSICAL_PAGE_NUMBER_BITS + PAGE_SHIFT; /// Largest physical byte address representable by Sv39. const MAX_PHYSICAL_ADDRESS: usize = (1usize << PHYSICAL_ADDRESS_BITS) - 1; +/// One 4 KiB Sv39 page table containing 512 64-bit entries. +#[repr(C, align(4096))] +struct PageTable { + /// Raw Sv39 page-table entries. + entries: [usize; PAGE_TABLE_ENTRIES], +} + +/// Exclusive access to the inherited Sv39 bootstrap address space. +/// +/// Construction establishes that `root_virtual` maps the complete active Sv39 +/// root page table read/write and that this value exclusively controls the +/// bootstrap paging entries reserved by this module. +/// +/// The capability is valid only while that bootstrap address space remains +/// active. +pub(crate) struct BootstrapPaging { + /// Identity-mapped virtual address of the active bootstrap root table. + root_virtual: VirtAddr, +} + +/// Tracks whether the bootstrap paging capability has already been claimed. +/// +/// Once set, this flag is never cleared, preventing construction of more than +/// one [`BootstrapPaging`] capability during the lifetime of the kernel. +static BOOTSTRAP_PAGING_CLAIMED: AtomicBool = AtomicBool::new(false); + +impl BootstrapPaging { + /// Claims the bootstrap paging capability for the active Sv39 address space. + /// + /// This capability can be claimed only once. A successful claim remains + /// permanent even if the returned [`BootstrapPaging`] value is later dropped. + /// + /// # Safety + /// + /// The active Sv39 root page table must be completely and writably + /// identity-mapped. The paging entries reserved by this module must not be + /// modified by any other code or hart while this capability exists, and the + /// active root must not be replaced except by consuming this capability. + /// + /// # Errors + /// + /// Returns [`PagingError::UnexpectedAddressTranslationMode`] if the active + /// address-translation mode is not Sv39, or + /// [`PagingError::BootstrapPagingAlreadyClaimed`] if the bootstrap paging + /// capability has already been claimed. + pub(crate) unsafe fn claim() -> Result { + let root = active_root_table()?; + + if BOOTSTRAP_PAGING_CLAIMED + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_err() + { + return Err(PagingError::BootstrapPagingAlreadyClaimed); + } + + Ok(Self { + root_virtual: VirtAddr::new(root.as_usize()), + }) + } + + /// Maps the physical region containing `dtb` into a higher-half boot window. + /// + /// The mapping preserves `dtb`'s offset within its first 1 GiB physical region, + /// so the returned virtual address refers to the same byte as `dtb`. The window + /// consists of consecutive read-only, non-executable Sv39 level-2 leaves. + /// + /// # Errors + /// + /// Returns [`PagingError::RootEntryInUse`] if the required virtual window is + /// occupied, or [`PagingError::PhysicalAddressNotRepresentable`] if the DTB + /// window requires a physical address that cannot be represented by Sv39. + #[expect( + clippy::arithmetic_side_effects, + reason = "window indices are statically bounded and the DTB address is \ + validated before physical-address arithmetic" + )] + pub(crate) fn map_fdt(&mut self, dtb: PhysAddr) -> Result { + if dtb.as_usize() > MAX_PHYSICAL_ADDRESS { + return Err(PagingError::PhysicalAddressNotRepresentable { address: dtb }); + } + + let physical_base = PhysAddr::new(dtb.as_usize() & !GIGAPAGE_MASK); + + /* + * Validate the complete window before modifying the active page table so an + * error cannot leave a partially installed mapping. + */ + for window_index in 0..BOOT_FDT_WINDOW_GIGAPAGES { + let root_index = BOOT_FDT_WINDOW_VPN2 + window_index; + + // SAFETY: + // `BootstrapPaging` guarantees that `root_virtual` maps the complete readable + // active bootstrap root table while this capability exists. + let existing = unsafe { read_mapped_page_table_entry(self.root_virtual, root_index)? }; + + if existing != 0 { + return Err(PagingError::RootEntryInUse { + index: root_index, + entry: existing, + }); + } + + let physical_offset = window_index * GIGAPAGE_SIZE; + + let physical_address = PhysAddr::new(physical_base.as_usize() + physical_offset); + + if physical_address.as_usize() > MAX_PHYSICAL_ADDRESS { + return Err(PagingError::PhysicalAddressNotRepresentable { + address: physical_address, + }); + } + } + + /* + * Every required entry and physical address has been validated. Install the + * read-only level-2 leaves. + */ + for window_index in 0..BOOT_FDT_WINDOW_GIGAPAGES { + let root_index = BOOT_FDT_WINDOW_VPN2 + window_index; + + let physical_offset = window_index * GIGAPAGE_SIZE; + + let physical_address = PhysAddr::new(physical_base.as_usize() + physical_offset); + + let entry = page_table_entry(physical_address, BOOT_FDT_PTE_FLAGS)?; + + // SAFETY: + // `BootstrapPaging` guarantees that `root_virtual` maps the complete writable + // active bootstrap root table. This entry was verified to be unused before any + // FDT mappings were installed. + unsafe { + write_mapped_page_table_entry(self.root_virtual, root_index, entry)?; + } + } + + flush_all(); + + let dtb_offset = dtb.as_usize() & GIGAPAGE_MASK; + + let virtual_address = BOOT_FDT_WINDOW_BASE + dtb_offset; + + Ok(VirtAddr::new(virtual_address)) + } + + /// Executes `operation` while `frame` is temporarily accessible through the + /// bootstrap physical-frame window. + /// + /// The temporary mapping uses a single Sv39 level-2 leaf. The physical 1 GiB + /// region containing `frame` is mapped at [`BOOT_FRAME_WINDOW_BASE`], and the + /// returned virtual address preserves the frame's offset within that region. + /// + /// The mapping exists only while `operation` executes and is removed before + /// this function returns. + /// + /// # Errors + /// + /// Returns [`PagingError::RootEntryInUse`] if the reserved frame-window entry + /// is already occupied, or + /// [`PagingError::PhysicalAddressNotRepresentable`] if `frame` cannot be + /// represented by Sv39. + #[expect( + clippy::arithmetic_side_effects, + reason = "the root index and frame offset are bounded by the Sv39 page-table and gigapage sizes" + )] + fn with_frame( + &mut self, + frame: PhysFrame, + operation: impl FnOnce(VirtAddr) -> R, + ) -> Result { + // SAFETY: + // `BootstrapPaging` guarantees that `root_virtual` maps the complete + // readable active bootstrap root table. + let existing = + unsafe { read_mapped_page_table_entry(self.root_virtual, BOOT_FRAME_WINDOW_VPN2)? }; + + if existing != 0 { + return Err(PagingError::RootEntryInUse { + index: BOOT_FRAME_WINDOW_VPN2, + entry: existing, + }); + } + + let frame_address = frame.start_address(); + + if frame_address.as_usize() > MAX_PHYSICAL_ADDRESS { + return Err(PagingError::PhysicalAddressNotRepresentable { + address: frame_address, + }); + } + + let physical_base = PhysAddr::new(frame_address.as_usize() & !GIGAPAGE_MASK); + + let entry = page_table_entry(physical_base, FRAME_WINDOW_PTE_FLAGS)?; + + // SAFETY: + // `BootstrapPaging` guarantees that `root_virtual` maps the complete + // writable active bootstrap root table. + unsafe { + write_mapped_page_table_entry(self.root_virtual, BOOT_FRAME_WINDOW_VPN2, entry)?; + } + + let frame_offset = frame_address.as_usize() & GIGAPAGE_MASK; + + let virtual_address = VirtAddr::new(BOOT_FRAME_WINDOW_BASE + frame_offset); + + flush_address(virtual_address); + + let result = operation(virtual_address); + + // SAFETY: + // `BootstrapPaging` guarantees that `root_virtual` maps the complete + // writable active bootstrap root table. + unsafe { + write_mapped_page_table_entry(self.root_virtual, BOOT_FRAME_WINDOW_VPN2, 0)?; + } + + flush_address(virtual_address); + + Ok(result) + } + + /// Clears every entry in a physical page-table frame. + /// + /// # Errors + /// + /// Returns an error if `frame` cannot be accessed through the bootstrap + /// physical-frame window. + #[expect( + clippy::needless_pass_by_ref_mut, + reason = "the mutable borrow intentionally provides exclusive access to \ + the allocated frame while its memory is initialized" + )] + fn zero_page_table(&mut self, frame: &mut AllocatedFrame) -> Result<(), PagingError> { + let physical_frame = frame.frame(); + + self.with_frame(physical_frame, |address| { + let table_pointer = ptr::with_exposed_provenance_mut::(address.as_usize()); + + // SAFETY: + // `with_frame` maps the complete allocated frame at `address` for this + // operation, and `AllocatedFrame` provides exclusive ownership of the + // physical frame being initialized. + unsafe { + ptr::write_bytes(table_pointer, 0, 1); + } + }) + } + + /// Reads one entry from a physical page-table frame. + /// + /// # Errors + /// + /// Returns an error if the frame cannot be temporarily mapped or `index` is + /// outside the page table. + fn read_page_table_entry( + &mut self, + frame: PhysFrame, + index: usize, + ) -> Result { + self.with_frame(frame, |address| { + // SAFETY: + // `with_frame` maps the complete physical frame at `address` for the + // duration of this operation. + unsafe { read_mapped_page_table_entry(address, index) } + })? + } + + /// Writes one entry in a physical page-table frame. + /// + /// # Errors + /// + /// Returns an error if the frame cannot be temporarily mapped or `index` is + /// outside the page table. + fn write_page_table_entry( + &mut self, + frame: PhysFrame, + index: usize, + entry: usize, + ) -> Result<(), PagingError> { + self.with_frame(frame, |address| { + // SAFETY: + // `with_frame` maps the complete writable physical frame at `address` for + // the duration of this operation. + unsafe { write_mapped_page_table_entry(address, index, entry) } + })? + } + + /// Allocates and initializes one empty physical page-table frame. + /// + /// # Errors + /// + /// Returns [`PagingError::OutOfPhysicalFrames`] if no frame remains, or an + /// error if the allocated frame cannot be initialized through the temporary + /// mapping window. + // TODO: Deal with ownership one day in the future for who is responsible for + // freeing. Works now since allocator never frees + fn allocate_page_table( + &mut self, + frames: &mut BootFrameAllocator<'_, '_>, + ) -> Result { + let Some(mut frame) = frames.allocate() else { + return Err(PagingError::OutOfPhysicalFrames); + }; + + self.zero_page_table(&mut frame)?; + + Ok(frame) + } + + /// Returns the child table referenced by `parent[index]`, allocating it when + /// the entry is currently empty. + /// + /// # Errors + /// + /// Returns an error if the parent cannot be accessed, no physical frame remains, + /// or an existing entry is not a valid non-leaf page-table pointer. + fn next_table( + &mut self, + parent: PhysFrame, + index: usize, + frames: &mut BootFrameAllocator<'_, '_>, + ) -> Result { + let entry = self.read_page_table_entry(parent, index)?; + + if entry != 0 { + return intermediate_frame(index, entry); + } + + let child = self.allocate_page_table(frames)?; + + let entry = page_table_entry(child.start_address(), PTE_VALID)?; + + self.write_page_table_entry(parent, index, entry)?; + + Ok(child.into_frame()) + } +} + +/// An Sv39 address space being constructed by the kernel. +struct Sv39PageTable { + /// Allocation containing the level-2 root table. + root: AllocatedFrame, +} + +impl Sv39PageTable { + /// Allocates a new empty Sv39 address space. + /// + /// # Errors + /// + /// Returns an error if no root frame can be allocated or initialized. + fn new( + bootstrap: &mut BootstrapPaging, + frames: &mut BootFrameAllocator<'_, '_>, + ) -> Result { + let root = bootstrap.allocate_page_table(frames)?; + + Ok(Self { root }) + } + + /// Maps every 4 KiB page covering `virtual_range` to consecutive physical + /// frames beginning at `physical_start`. + /// + /// The virtual start must be page-aligned. The exclusive end may be + /// unaligned, its final partial page is mapped in full. + /// + /// # Errors + /// + /// Returns an error if the virtual start is unaligned, the range includes + /// non-canonical Sv39 addresses, the physical span is not representable, or + /// an individual page cannot be mapped. + /// + /// Address spans are validated before any entries are installed. Allocation + /// or mapping conflicts can still fail partway through; mappings already + /// installed by this call remain in the address space. + #[expect( + clippy::arithmetic_side_effects, + reason = "VirtRange bounds are ordered, and page offsets fit within the \ + validated virtual and physical spans" + )] + fn map_range( + &mut self, + bootstrap: &mut BootstrapPaging, + virtual_range: VirtRange, + physical_start: PhysFrame, + permissions: usize, + frames: &mut BootFrameAllocator<'_, '_>, + ) -> Result<(), PagingError> { + let start = virtual_range.start().as_usize(); + let end = virtual_range.end().as_usize(); + + if start & PAGE_MASK != 0 { + return Err(PagingError::VirtualAddressNotPageAligned { address: start }); + } + + if (SV39_LOW_HALF_END..SV39_HIGH_HALF_BASE).contains(&start) { + return Err(PagingError::VirtualAddressNotCanonical { address: start }); + } + + if start < SV39_LOW_HALF_END && end > SV39_LOW_HALF_END { + return Err(PagingError::VirtualAddressNotCanonical { + address: SV39_LOW_HALF_END, + }); + } + + let size = end - start; + let page_count = size.div_ceil(PAGE_SIZE); + let last_offset = (page_count - 1) * PAGE_SIZE; + let physical_base = physical_start.start_address(); + + physical_base + .checked_add(last_offset) + .filter(|address| address.as_usize() <= MAX_PHYSICAL_ADDRESS) + .ok_or(PagingError::PhysicalRangeNotRepresentable { + start: physical_base, + size, + })?; + + for page_index in 0..page_count { + let offset = page_index * PAGE_SIZE; + let virtual_address = VirtAddr::new(start + offset); + let physical_address = PhysAddr::new(physical_base.as_usize() + offset); + + let Some(frame) = PhysFrame::from_start(physical_address) else { + return Err(PagingError::PhysicalRangeNotRepresentable { + start: physical_base, + size, + }); + }; + + self.map_page(bootstrap, virtual_address, frame, permissions, frames)?; + } + + Ok(()) + } + + /// Maps one 4 KiB virtual page to `physical_frame`. + /// + /// `permissions` contains the Sv39 `R`, `W`, and `X` bits for the leaf. + /// Valid/accessed state is added automatically, as is the dirty bit for + /// writable mappings. + /// + /// # Errors + /// + /// Returns an error if the virtual address is invalid or unaligned, physical + /// frames required for intermediate tables cannot be allocated, an existing + /// intermediate entry is malformed, or the virtual page is already mapped. + #[expect( + clippy::needless_pass_by_ref_mut, + reason = "mapping a page mutates the address space represented by this \ + value, even though its page-table memory is modified indirectly" + )] + fn map_page( + &mut self, + bootstrap: &mut BootstrapPaging, + virtual_address: VirtAddr, + physical_frame: PhysFrame, + permissions: usize, // TODO: Make permissions explicitly a type + frames: &mut BootFrameAllocator<'_, '_>, + ) -> Result<(), PagingError> { + let address = virtual_address.as_usize(); + + if address & PAGE_MASK != 0 { + return Err(PagingError::VirtualAddressNotPageAligned { address }); + } + + if address >= SV39_LOW_HALF_END && address < SV39_HIGH_HALF_BASE { + return Err(PagingError::VirtualAddressNotCanonical { address }); + } + + if permissions & !PTE_LEAF_MASK != 0 + || permissions & PTE_LEAF_MASK == 0 + || permissions & PTE_WRITE != 0 && permissions & PTE_READ == 0 + { + return Err(PagingError::InvalidLeafPermissions { permissions }); + } + + let vpn2 = (address >> GIGAPAGE_SHIFT) & VPN_MASK; + let vpn1 = (address >> MEGAPAGE_SHIFT) & VPN_MASK; + let vpn0 = (address >> PAGE_SHIFT) & VPN_MASK; + + let level_1 = bootstrap.next_table(self.root.frame(), vpn2, frames)?; + let level_0 = bootstrap.next_table(level_1, vpn1, frames)?; + + let existing = bootstrap.read_page_table_entry(level_0, vpn0)?; + + if existing != 0 { + return Err(PagingError::PageAlreadyMapped { + address, + entry: existing, + }); + } + + let mut flags = PTE_VALID | PTE_ACCESSED | permissions; + + if permissions & PTE_WRITE != 0 { + flags |= PTE_DIRTY; + } + + let entry = page_table_entry(physical_frame.start_address(), flags)?; + + bootstrap.write_page_table_entry(level_0, vpn0, entry) + } +} + /// Errors produced while inspecting or modifying Sv39 paging state. #[derive(Debug)] pub(crate) enum PagingError { @@ -99,6 +665,80 @@ pub(crate) enum PagingError { /// Physical address outside the Sv39 representable range. address: PhysAddr, }, + + /// The physical frames covering a requested byte range cannot all be + /// represented by Sv39, or computing the final frame address would overflow. + PhysicalRangeNotRepresentable { + /// Physical start address corresponding to the virtual range start. + start: PhysAddr, + + /// Requested range size in bytes before rounding up to complete pages. + size: usize, + }, + + /// A kernel section could not be translated into an aligned physical frame. + InvalidKernelSection { + /// Linker-defined virtual section range. + range: VirtRange, + }, + + /// The DTB alias had a different page offset or its page coverage overflowed. + InvalidDtbMapping { + /// Virtual address of the first byte of the DTB. + virtual_address: VirtAddr, + + /// Physical range covering the DTB bytes. + physical_range: PhysRange, + }, + + /// No physical frame remained for a required page table. + OutOfPhysicalFrames, + + /// A virtual address supplied for a 4 KiB mapping was not page-aligned. + VirtualAddressNotPageAligned { + /// Unaligned virtual address. + address: usize, + }, + + /// A virtual address is not canonical under Sv39. + VirtualAddressNotCanonical { + /// Non-canonical virtual address. + address: usize, + }, + + /// A page-table entry expected to point to the next level was malformed or + /// was already a leaf mapping. + InvalidIntermediateEntry { + /// Index containing the unexpected entry. + index: usize, + + /// Raw unexpected entry. + entry: usize, + }, + + /// A requested leaf virtual page already had a mapping. + PageAlreadyMapped { + /// Virtual page that was already mapped. + address: usize, + + /// Existing raw leaf entry. + entry: usize, + }, + + /// A page-table index was outside the 512-entry table. + PageTableIndexOutOfRange { + /// Invalid entry index. + index: usize, + }, + + /// The requested Sv39 leaf permissions were invalid. + InvalidLeafPermissions { + /// Requested raw R/W/X permission bits. + permissions: usize, + }, + + /// The bootstrap paging capability has already been claimed. + BootstrapPagingAlreadyClaimed, } /// Returns the physical address of the active Sv39 root page table. @@ -134,110 +774,165 @@ fn active_root_table() -> Result { Ok(PhysAddr::new(root_ppn << PAGE_SHIFT)) } -/// Maps the physical region containing `dtb` into a higher-half boot window. +/// Constructs an Sv39 page-table entry pointing at `physical_address`. /// -/// The mapping preserves `dtb`'s offset within its first 1 GiB physical region, -/// so the returned virtual address refers to the same byte as `dtb`. The window -/// consists of consecutive read-only, non-executable Sv39 level-2 leaves. -/// -/// The active root table must be accessible through an identity mapping while -/// this function executes. +/// `flags` contains the desired Sv39 PTE flag bits. /// /// # Errors /// -/// Returns [`PagingError::UnexpectedAddressTranslationMode`] if Sv39 is not -/// active, [`PagingError::RootEntryInUse`] if the required virtual window is -/// occupied, or [`PagingError::PhysicalAddressNotRepresentable`] if the DTB -/// window requires a physical address that cannot be represented by Sv39. -#[expect( - clippy::arithmetic_side_effects, - reason = "window indices are statically bounded, the active root comes from \ - the satp PPN, and the DTB address is validated before \ - physical-address arithmetic" -)] -pub(crate) fn map_bootstrap_fdt(dtb: PhysAddr) -> Result { - let boot_root = active_root_table()?; - - if dtb.as_usize() > MAX_PHYSICAL_ADDRESS { - return Err(PagingError::PhysicalAddressNotRepresentable { address: dtb }); +/// Returns [`PagingError::PhysicalAddressNotRepresentable`] if +/// `physical_address` cannot be encoded in an Sv39 PTE. +const fn page_table_entry(physical_address: PhysAddr, flags: usize) -> Result { + if physical_address.as_usize() > MAX_PHYSICAL_ADDRESS { + return Err(PagingError::PhysicalAddressNotRepresentable { + address: physical_address, + }); } - let physical_base = PhysAddr::new(dtb.as_usize() & !GIGAPAGE_MASK); + let physical_page_number = physical_address.as_usize() >> PAGE_SHIFT; - /* - * Validate the complete window before modifying the active page table so an - * error cannot leave a partially installed mapping. - */ - for window_index in 0..BOOT_FDT_WINDOW_GIGAPAGES { - let root_index = BOOT_FDT_WINDOW_VPN2 + window_index; - let entry_offset = root_index * PTE_SIZE; + Ok((physical_page_number << PTE_PPN_SHIFT) | flags) +} - let entry_address = PhysAddr::new(boot_root.as_usize() + entry_offset); +/// Returns the page-table frame referenced by a valid non-leaf entry. +/// +/// # Errors +/// +/// Returns [`PagingError::InvalidIntermediateEntry`] if `entry` is not a valid +/// Sv39 non-leaf entry. +const fn intermediate_frame(index: usize, entry: usize) -> Result { + if entry & PTE_VALID == 0 || entry & PTE_LEAF_MASK != 0 { + return Err(PagingError::InvalidIntermediateEntry { index, entry }); + } - let entry_pointer = ptr::with_exposed_provenance::(entry_address.as_usize()); + let physical_page_number = (entry >> PTE_PPN_SHIFT) & SATP_PPN_MASK; - // SAFETY: - // The active bootstrap root is located inside the physical kernel - // gigapage, which remains identity-mapped while this function executes. - let existing = unsafe { ptr::read_volatile(entry_pointer) }; + let physical_address = PhysAddr::new(physical_page_number << PAGE_SHIFT); - if existing != 0 { - return Err(PagingError::RootEntryInUse { - index: root_index, - entry: existing, - }); - } + let Some(frame) = PhysFrame::from_start(physical_address) else { + return Err(PagingError::InvalidIntermediateEntry { index, entry }); + }; - let physical_offset = window_index * GIGAPAGE_SIZE; + Ok(frame) +} - let physical_address = PhysAddr::new(physical_base.as_usize() + physical_offset); +/// Invalidates the current hart's cached translation for `address`. +fn flush_address(address: VirtAddr) { + // SAFETY: + // `sfence.vma` only affects address-translation state on the current hart. + unsafe { + asm!( + "sfence.vma {address}, zero", + address = in(reg) address.as_usize(), + options(nostack), + ); + } +} - if physical_address.as_usize() > MAX_PHYSICAL_ADDRESS { - return Err(PagingError::PhysicalAddressNotRepresentable { - address: physical_address, - }); - } +/// Invalidates all cached address translations on the current hart. +fn flush_all() { + // SAFETY: + // `sfence.vma` only affects address-translation state on the current hart. + unsafe { + asm!("sfence.vma zero, zero", options(nostack)); } +} - /* - * Every required entry and physical address has been validated. Install the - * read-only level-2 leaves. - */ - for window_index in 0..BOOT_FDT_WINDOW_GIGAPAGES { - let root_index = BOOT_FDT_WINDOW_VPN2 + window_index; +/// Installs a complete replacement root on the current hart. +/// +/// # Safety +/// +/// `root` must be representable in `satp` and identify a fully initialized Sv39 +/// hierarchy that preserves all live code, stack, trap, and borrowed-data +/// mappings. Its table frames must remain allocated while the address space +/// is active. The caller must consume the capability controlling the previous +/// root and expose only the replacement capability after this operation +/// completes. +unsafe fn activate_root(root: PhysFrame) { + let satp = (SATP_MODE_SV39 << SATP_MODE_SHIFT) | (root.start_address().as_usize() >> PAGE_SHIFT); - let physical_offset = window_index * GIGAPAGE_SIZE; + // SAFETY: + // The caller establishes the replacement mapping and allocation invariants. + // The first fence orders completed table writes before the switch. The + // second invalidates old translations, including those using ASID zero. + // The assembly has a memory clobber. No fallible operation follows the satp + // write. + unsafe { + asm!( + "sfence.vma zero, zero", + "csrw satp, {satp}", + "sfence.vma zero, zero", + satp = in(reg) satp, + options(nostack), + ); + } +} - let physical_address = PhysAddr::new(physical_base.as_usize() + physical_offset); +/// Reads one entry from a page table that is currently mapped at `address`. +/// +/// # Safety +/// +/// `address` must be page-aligned and must map a complete readable +/// [`PageTable`] for the duration of this operation. +/// +/// # Errors +/// +/// Returns [`PagingError::PageTableIndexOutOfRange`] if `index` does not +/// identify an entry in the page table. +unsafe fn read_mapped_page_table_entry( + address: VirtAddr, + index: usize, +) -> Result { + if index >= PAGE_TABLE_ENTRIES { + return Err(PagingError::PageTableIndexOutOfRange { index }); + } - let entry_offset = root_index * PTE_SIZE; + let table_pointer = ptr::with_exposed_provenance::(address.as_usize()); - let entry_address = PhysAddr::new(boot_root.as_usize() + entry_offset); + // SAFETY: + // The caller guarantees that `address` maps a complete `PageTable`, and the + // bounds check above guarantees that `index` identifies one of its entries. + let entry_pointer = unsafe { table_pointer.cast::().add(index) }; - let physical_page_number = physical_address.as_usize() >> PAGE_SHIFT; + // SAFETY: + // `entry_pointer` refers to a valid entry within the mapped page table. + Ok(unsafe { ptr::read_volatile(entry_pointer) }) +} - let entry = (physical_page_number << PTE_PPN_SHIFT) | BOOT_FDT_PTE_FLAGS; +/// Writes one entry in a page table that is currently mapped at `address`. +/// +/// # Safety +/// +/// `address` must be page-aligned and must map a complete writable +/// [`PageTable`] for the duration of this operation. The table must not be +/// concurrently accessed in a way that conflicts with this write. +/// +/// # Errors +/// +/// Returns [`PagingError::PageTableIndexOutOfRange`] if `index` does not +/// identify an entry in the page table. +unsafe fn write_mapped_page_table_entry( + address: VirtAddr, + index: usize, + value: usize, +) -> Result<(), PagingError> { + if index >= PAGE_TABLE_ENTRIES { + return Err(PagingError::PageTableIndexOutOfRange { index }); + } - let entry_pointer = ptr::with_exposed_provenance_mut::(entry_address.as_usize()); + let table_pointer = ptr::with_exposed_provenance_mut::(address.as_usize()); - // SAFETY: - // The active bootstrap root remains identity-mapped, and the selected - // entry was verified to be unused before any mappings were installed. - unsafe { - ptr::write_volatile(entry_pointer, entry); - } - } + // SAFETY: + // The caller guarantees that `address` maps a complete writable `PageTable`, + // and the bounds check above guarantees that `index` identifies one of its + // entries. + let entry_pointer = unsafe { table_pointer.cast::().add(index) }; // SAFETY: - // `sfence.vma` makes the preceding PTE updates visible to address translation - // on the current hart. + // `entry_pointer` points to a writable PTE within the mapped page table. unsafe { - asm!("sfence.vma zero, zero", options(nostack)); + ptr::write_volatile(entry_pointer, value); } - let dtb_offset = dtb.as_usize() & GIGAPAGE_MASK; - - let virtual_address = BOOT_FDT_WINDOW_BASE + dtb_offset; - - Ok(VirtAddr::new(virtual_address)) + Ok(()) } diff --git a/kernel/src/arch/riscv64/paging/kernel.rs b/kernel/src/arch/riscv64/paging/kernel.rs new file mode 100644 index 0000000..63e2a51 --- /dev/null +++ b/kernel/src/arch/riscv64/paging/kernel.rs @@ -0,0 +1,337 @@ +//! The active kernel Sv39 address space and temporary physical-frame access. + +use core::{fmt, ptr}; + +use super::{ + BOOT_FRAME_WINDOW_BASE, BootstrapPaging, FRAME_WINDOW_PTE_FLAGS, GIGAPAGE_SHIFT, GIGAPAGE_SIZE, + MEGAPAGE_SHIFT, PAGE_MASK, PAGE_SHIFT, PAGE_SIZE, PTE_EXECUTE, PTE_READ, PTE_WRITE, PagingError, + Sv39PageTable, VPN_MASK, activate_root, flush_address, page_table_entry, +}; +use crate::memory::{ + AllocatedFrame, BootFrameAllocator, KernelSections, PhysAddr, PhysFrame, PhysRange, VirtAddr, + VirtRange, +}; + +/// Virtual page mapping the L0 table that controls the kernel frame window. +/// +/// This lies in the final 2 MiB below the kernel's 1 GiB virtual region. +const FIXMAP_TABLE_BASE: usize = + BOOT_FRAME_WINDOW_BASE - GIGAPAGE_SIZE - (1usize << MEGAPAGE_SHIFT); + +/// Virtual page reserved for temporary access to one physical frame. +/// +/// This page begins unmapped. The adjacent table alias will allow its leaf +/// entry to be changed after the replacement root is activated. +const FIXMAP_FRAME_BASE: usize = FIXMAP_TABLE_BASE + PAGE_SIZE; + +/// Index of the temporary frame slot within its controlling L0 table. +const FIXMAP_FRAME_INDEX: usize = (FIXMAP_FRAME_BASE >> PAGE_SHIFT) & VPN_MASK; + +/// Exclusive control of the active kernel address space. +/// +/// This capability is created only by consuming [`BootstrapPaging`] and +/// activating the replacement root. It cannot be cloned or copied and must +/// be used on the hart where that root was activated. The root need not have a +/// permanent virtual mapping: physical frames are accessed through the one-page +/// window controlled by `frame_slot`. +/// +/// All page-table frames remain permanently reserved by the monotonic boot +/// allocator. No other code may change the frame-window entry while this +/// capability exists. +#[must_use = "retain the capability controlling the active kernel address space"] +pub(crate) struct KernelPaging { + /// Allocation containing the active Sv39 root. + root: AllocatedFrame, + + /// L0 table mapped at [`FIXMAP_TABLE_BASE`]. + fixmap_table: PhysFrame, + + /// Complete virtual pages covering the DTB at its existing alias. + dtb_mapping: VirtRange, + + /// Writable PTE for the temporary frame slot, through the fixed table alias. + frame_slot: *mut usize, +} + +impl fmt::Debug for KernelPaging { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("KernelPaging") + .field("root", &self.root.start_address()) + .field("dtb_mapping", &self.dtb_mapping) + .field("fixmap_table", &self.fixmap_table.start_address()) + .field("fixmap_table_alias", &VirtAddr::new(FIXMAP_TABLE_BASE)) + .field("frame_slot_pointer", &self.frame_slot) + .field("frame_slot_alias", &VirtAddr::new(FIXMAP_FRAME_BASE)) + .finish() + } +} + +impl BootstrapPaging { + /// Consumes bootstrap paging and enters the normal kernel address space. + /// + /// Kernel section addresses are translated relative to + /// `sections.virtual_start`. Every mapping uses a 4 KiB leaf. + /// + /// Only pages covering `dtb_physical` are retained at the existing + /// `dtb_virtual` alias. No bootstrap or identity mappings are copied. + /// All mappings are built before the root is switched, and the bootstrap + /// capability cannot be reclaimed afterward. + /// + /// # Safety + /// + /// `sections` and `kernel_physical_start` must describe the live kernel's + /// actual virtual sections and physical load address. They must cover all + /// live kernel code, constants, data, stacks, and trap state. `dtb_physical` + /// must cover the complete DTB backing any retained references, and + /// `dtb_virtual` must be its existing bootstrap alias. No other borrowed + /// memory may depend on a mapping omitted by the new root. + /// + /// The transition must run on the hart whose bootstrap root this capability + /// controls. The returned capability must be used only while the replacement + /// root remains active on that hart. + /// + /// Supervisor interrupts must remain disabled during the transition and + /// temporary frame access. No other hart or trap handler may modify these + /// paging structures or use the temporary frame window concurrently. + /// + /// # Errors + /// + /// Returns an error if a section or DTB range cannot be translated, mappings + /// conflict, or a required page-table frame cannot be allocated or accessed. + /// Errors occur before activation, leaving the bootstrap root active. + /// The consumed capability is not returned, so startup must treat failure as + /// unrecoverable. Allocated frames remain reserved because the boot allocator + /// does not reclaim frames. + pub(crate) unsafe fn into_kernel( + mut self, + kernel_physical_start: PhysAddr, + sections: &KernelSections, + dtb_virtual: VirtAddr, + dtb_physical: PhysRange, + frames: &mut BootFrameAllocator<'_, '_>, + ) -> Result { + let (dtb_mapping, dtb_frame) = dtb_page_mapping(dtb_virtual, dtb_physical)?; + let mut page_table = Sv39PageTable::new(&mut self, frames)?; + + for (section, permissions) in [ + (sections.text, PTE_READ | PTE_EXECUTE), + (sections.rodata, PTE_READ), + (sections.data, PTE_READ | PTE_WRITE), + (sections.bss, PTE_READ | PTE_WRITE), + (sections.stack, PTE_READ | PTE_WRITE), + ] { + let Some(range) = section else { + continue; + }; + + let physical_start = range + .start() + .as_usize() + .checked_sub(sections.virtual_start.as_usize()) + .and_then(|offset| kernel_physical_start.checked_add(offset)) + .and_then(PhysFrame::from_start) + .ok_or(PagingError::InvalidKernelSection { range })?; + + page_table.map_range(&mut self, range, physical_start, permissions, frames)?; + } + + page_table.map_range(&mut self, dtb_mapping, dtb_frame, PTE_READ, frames)?; + + let fixmap_table = page_table.prepare_fixmap(&mut self, frames)?; + + let kernel = KernelPaging { + root: page_table.root, + fixmap_table, + dtb_mapping, + frame_slot: ptr::with_exposed_provenance_mut::(FIXMAP_TABLE_BASE) + .wrapping_add(FIXMAP_FRAME_INDEX), + }; + + // SAFETY: + // The caller supplies the complete live kernel layout and retained DTB + // range. Construction above installs those mappings and the permanent + // control-table alias. All table frames remain reserved, this consumes the + // bootstrap capability, and only `kernel` is returned after the switch. + unsafe { + activate_root(kernel.root.frame()); + } + + Ok(kernel) + } +} + +impl KernelPaging { + /// Executes `operation` with `frame` temporarily mapped read/write. + /// + /// The alias maps exactly one 4 KiB frame and is non-executable and + /// supervisor-only. The supplied address is valid only during `operation`; + /// references created through it must not outlive the closure. + /// Dereferencing it requires the caller to establish valid memory access + /// and ownership for `frame`. + /// + /// Supervisor interrupts must already be disabled and remain disabled + /// throughout this call. + /// + /// The mapping is removed when the closure returns normally. Clean up does + /// not run during unwinding; current kernel panics halt without unwinding. + /// + /// # Errors + /// + /// Returns an error if the frame cannot be represented by Sv39 or the + /// temporary slot is already occupied. + pub(crate) fn with_frame( + &mut self, + frame: PhysFrame, + operation: impl FnOnce(VirtAddr) -> R, + ) -> Result { + let entry = page_table_entry(frame.start_address(), FRAME_WINDOW_PTE_FLAGS)?; + + // SAFETY: + // `KernelPaging` guarantees that the controlling L0 table is permanently + // mapped read/write and that `frame_slot` points to one of its entries. + let existing = unsafe { ptr::read_volatile(self.frame_slot) }; + + if existing != 0 { + return Err(PagingError::PageAlreadyMapped { + address: FIXMAP_FRAME_BASE, + entry: existing, + }); + } + + // SAFETY: + // The entry pointer is valid and exclusively controlled by this capability, + // and the new leaf maps one representable, aligned physical frame. + unsafe { + ptr::write_volatile(self.frame_slot, entry); + } + + let address = VirtAddr::new(FIXMAP_FRAME_BASE); + + // Make the newly installed mapping available. + flush_address(address); + + let result = operation(address); + + // SAFETY: + // `frame_slot` remains a valid, exclusively controlled pointer into the + // permanently mapped L0 table. Clear its temporary frame mapping. + unsafe { + ptr::write_volatile(self.frame_slot, 0); + } + + flush_address(address); + + Ok(result) + } + + /// Zeroes every byte in an allocated physical frame through the kernel's + /// temporary frame window. + /// + /// # Errors + /// + /// Returns an error if the physical address cannot be represented by Sv39 + /// or the temporary frame slot is already occupied. + #[expect( + clippy::needless_pass_by_ref_mut, + reason = "the mutable borrow intentionally provides exclusive access to \ + the allocated frame while its memory is initialized" + )] + pub(crate) fn zero_frame(&mut self, frame: &mut AllocatedFrame) -> Result<(), PagingError> { + self.with_frame(frame.frame(), |address| { + let pointer = ptr::with_exposed_provenance_mut::(address.as_usize()); + + // SAFETY: + // The window maps the complete 4 KiB frame for this closure. The mutable + // `AllocatedFrame` borrow provides exclusive ownership of the RAM being + // initialized, and this operation creates no persistent references. + unsafe { + ptr::write_bytes(pointer, 0, PAGE_SIZE); + } + }) + } +} + +impl Sv39PageTable { + /// Maps the frame-window L0 table into its own control page, leaving the + /// adjacent temporary frame slot empty. + /// + /// # Errors + /// + /// Returns an error if the tables cannot be allocated or accessed, or either + /// fixmap page is already occupied. + fn prepare_fixmap( + &mut self, + bootstrap: &mut BootstrapPaging, + frames: &mut BootFrameAllocator<'_, '_>, + ) -> Result { + let vpn2 = (FIXMAP_TABLE_BASE >> GIGAPAGE_SHIFT) & VPN_MASK; + let vpn1 = (FIXMAP_TABLE_BASE >> MEGAPAGE_SHIFT) & VPN_MASK; + + let level_1 = bootstrap.next_table(self.root.frame(), vpn2, frames)?; + let level_0 = bootstrap.next_table(level_1, vpn1, frames)?; + let existing = bootstrap.read_page_table_entry(level_0, FIXMAP_FRAME_INDEX)?; + + if existing != 0 { + return Err(PagingError::PageAlreadyMapped { + address: FIXMAP_FRAME_BASE, + entry: existing, + }); + } + + self.map_page( + bootstrap, + VirtAddr::new(FIXMAP_TABLE_BASE), + level_0, + PTE_READ | PTE_WRITE, + frames, + )?; + + Ok(level_0) + } +} + +/// Computes the aligned virtual range and first physical frame covering the +/// DTB. +/// +/// # Errors +/// +/// Returns an error if the physical and virtual page offsets differ or the +/// complete page coverage cannot be represented. +#[expect( + clippy::arithmetic_side_effects, + reason = "PhysRange guarantees strictly ordered physical bounds" +)] +fn dtb_page_mapping( + virtual_address: VirtAddr, + physical_range: PhysRange, +) -> Result<(VirtRange, PhysFrame), PagingError> { + let invalid_mapping = || PagingError::InvalidDtbMapping { + virtual_address, + physical_range, + }; + + let physical_start = physical_range.start().as_usize(); + let virtual_start = virtual_address.as_usize(); + + if virtual_start & PAGE_MASK != physical_start & PAGE_MASK { + return Err(invalid_mapping()); + } + + let size = physical_range.end().as_usize() - physical_start; + let virtual_end = virtual_address + .checked_add(size) + .and_then(|end| end.checked_add(PAGE_MASK)) + .ok_or_else(invalid_mapping)?; + + let virtual_range = VirtRange::new( + VirtAddr::new(virtual_start & !PAGE_MASK), + VirtAddr::new(virtual_end.as_usize() & !PAGE_MASK), + ) + .ok_or_else(invalid_mapping)?; + + let physical_frame = PhysFrame::from_start(PhysAddr::new(physical_start & !PAGE_MASK)) + .ok_or_else(invalid_mapping)?; + + Ok((virtual_range, physical_frame)) +} diff --git a/kernel/src/main.rs b/kernel/src/main.rs index f39a65e..80cdf48 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -16,8 +16,9 @@ mod memory; use core::panic::PanicInfo; use fdt::Fdt; -use memory::{BootFrameAllocator, PhysAddr, PhysRange}; +use memory::{BootFrameAllocator, PhysAddr, PhysRange, kernel_sections}; +// TODO: add debug/trace info for allocations /// Runs the kernel after architecture-specific initialization. /// /// `hart_id` identifies the RISC-V hart on which the kernel was entered. @@ -41,12 +42,28 @@ extern "C" fn main(hart_id: usize, dtb: usize, kernel_phys_start: usize) -> ! { logging::info!("initializing trap handling"); arch::init_trap(); - let dtb_virtual = match arch::map_bootstrap_fdt(dtb_phys) { + // SAFETY: + // Architecture bootstrap enters `main` with the complete active Sv39 root + // page table writably identity-mapped. No other hart or code modifies the + // paging entries reserved for bootstrap use while this capability exists. + let mut bootstrap_paging = match unsafe { arch::BootstrapPaging::claim() } { + Ok(paging) => paging, + Err(error) => { + logging::error!( + "Failed to claim bootstrap paging state ({error:?}); \ + kernel startup is unrecoverable, halting" + ); + + arch::halt(); + } + }; + + let dtb_virtual = match bootstrap_paging.map_fdt(dtb_phys) { Ok(address) => address, Err(error) => { logging::error!( "Failed to establish higher-half DTB mapping ({error:?}); \ - kernel startup is unrecoverable, halting" + kernel startup is unrecoverable, halting" ); arch::halt(); @@ -58,9 +75,9 @@ extern "C" fn main(hart_id: usize, dtb: usize, kernel_phys_start: usize) -> ! { let dtb_ptr = core::ptr::with_exposed_provenance::(dtb_virtual.as_usize()); // SAFETY: - // `map_bootstrap_fdt` establishes a readable higher-half mapping containing - // the DTB. This mapping must remain valid and unmodified for as long as `fdt` - // and values borrowing from it remain alive. + // `BootstrapPaging::map_fdt` established a readable mapping containing the + // complete DTB. `into_kernel` transition preserves all pages backing this + // blob at the same virtual alias. let fdt = match unsafe { Fdt::from_ptr(dtb_ptr) } { Ok(fdt) => fdt, Err(error) => { @@ -90,6 +107,10 @@ extern "C" fn main(hart_id: usize, dtb: usize, kernel_phys_start: usize) -> ! { arch::halt(); }; + let kernel_sections = kernel_sections(); + + logging::debug!("{:#?}", kernel_sections); + logging::debug!("Kernel Range: {:?}", kernel_range); let Some(dtb_range) = PhysRange::from_start_size(dtb_phys, fdt.total_size()) else { @@ -102,11 +123,46 @@ extern "C" fn main(hart_id: usize, dtb: usize, kernel_phys_start: usize) -> ! { logging::debug!("DTB Range: {:?}", dtb_range); - let frames = match BootFrameAllocator::new(&fdt, kernel_range, dtb_range) { + // SAFETY: + // The architecture bootstrap preserves the actual physical start of the live + // kernel image, so `kernel_range` covers its complete linker-defined boot + // footprint. `dtb_range` covers the same firmware-provided DTB backing `fdt`. + // The FDT is trusted to describe all other physical memory that is + // unavailable for allocation through its reservation information. + let mut frames = match unsafe { BootFrameAllocator::claim(&fdt, kernel_range, dtb_range) } { Ok(frames) => frames, Err(error) => { logging::error!( - "Failed to initialize physical frame allocator ({error:?}); \ + "Failed to claim physical frame allocator ({error:?}); \ + kernel startup is unrecoverable, halting" + ); + + arch::halt(); + } + }; + + // SAFETY: + // These sections come directly from the live kernel's linker symbols, and + // bootstrap supplied the actual physical load address. They cover the live + // code, trap vector, constants, data, and stack. `dtb_range` and `dtb_virtual` + // describe the complete blob backing `fdt` and the frame allocator. No other + // retained reference depends on a bootstrap-only mapping. Startup relies on + // the boot environment entering this hart with supervisor interrupts disabled + // and does not enable them before this transition. No other hart accesses + // these paging structures. + let kernel_paging = match unsafe { + bootstrap_paging.into_kernel( + kernel_phys_start, + &kernel_sections, + dtb_virtual, + dtb_range, + &mut frames, + ) + } { + Ok(paging) => paging, + Err(error) => { + logging::error!( + "Failed to enter kernel paging ({error:?}); \ kernel startup is unrecoverable, halting" ); @@ -114,7 +170,10 @@ extern "C" fn main(hart_id: usize, dtb: usize, kernel_phys_start: usize) -> ! { } }; - logging::debug!("kernel halting"); + logging::info!("Kernel paging active"); + logging::debug!("{:#?}", kernel_paging); + + logging::info!("kernel halting"); arch::halt() } diff --git a/kernel/src/memory.rs b/kernel/src/memory.rs index db8caf5..373666f 100644 --- a/kernel/src/memory.rs +++ b/kernel/src/memory.rs @@ -1,18 +1,17 @@ -//! Core physical memory abstractions. +//! Core kernel memory abstractions. //! //! This module provides the common physical-memory vocabulary used by the //! kernel's memory-management code. //! //! Physical and virtual addresses are represented by [`PhysAddr`] and -//! [`VirtAddr`], while [`PhysRange`] represents non-empty half-open regions of -//! physical address space. Physical frame representation and boot-time frame +//! [`VirtAddr`]. Physical frame representation and boot-time frame //! allocation are implemented by the [`frame`] submodule. mod address; mod frame; pub(crate) use address::{PhysAddr, VirtAddr}; -pub(crate) use frame::BootFrameAllocator; +pub(crate) use frame::{AllocatedFrame, BootFrameAllocator, PhysFrame}; /// A non-empty half-open physical address range `[start, end)`. /// @@ -27,6 +26,47 @@ pub(crate) struct PhysRange { end: PhysAddr, } +/// A non-empty half-open virtual address range `[start, end)`. +/// +/// A `VirtRange` guarantees that `start` is strictly less than `end`. The +/// start address is included in the range, while the end address is excluded. +#[derive(Clone, Copy, Debug)] +pub(crate) struct VirtRange { + /// Inclusive start address. + start: VirtAddr, + + /// Exclusive end address. + end: VirtAddr, +} + +/// Linker-defined virtual ranges of the higher-half kernel sections. +/// +/// Each range includes its start and excludes its end. Section starts are +/// page-aligned, but ends describe the actual section contents and may require +/// rounding up when mapping complete pages. +/// +/// Empty sections have equal linker bounds and are represented by `None`. +#[derive(Debug)] +pub(crate) struct KernelSections { + /// Virtual origin corresponding to the physical kernel image start. + pub(crate) virtual_start: VirtAddr, + + /// Executable kernel code. + pub(crate) text: Option, + + /// Read-only kernel constants. + pub(crate) rodata: Option, + + /// Initialized writable kernel data. + pub(crate) data: Option, + + /// Zero-initialized writable kernel data. + pub(crate) bss: Option, + + /// Reserved boot stack. + pub(crate) stack: Option, +} + unsafe extern "C" { /// Linker-defined symbol marking the start of the kernel's complete virtual /// boot-time footprint. @@ -35,6 +75,71 @@ unsafe extern "C" { /// Linker-defined symbol marking the end of the kernel's complete virtual /// boot-time footprint. static _kernel_virtual_end: u8; + + /// Inclusive virtual start of executable kernel code. + static _text_start: u8; + + /// Exclusive virtual end of executable kernel code. + static _text_end: u8; + + /// Inclusive virtual start of read-only kernel constants. + static _rodata_start: u8; + + /// Exclusive virtual end of read-only kernel constants. + static _rodata_end: u8; + + /// Inclusive virtual start of initialized writable kernel data. + static _data_start: u8; + + /// Exclusive virtual end of initialized writable kernel data. + static _data_end: u8; + + /// Inclusive virtual start of zero-initialized kernel data. + static _bss_start: u8; + + /// Exclusive virtual end of zero-initialized kernel data. + static _bss_end: u8; + + /// Inclusive virtual start of the reserved boot stack. + static _stack_start: u8; + + /// Exclusive virtual end of the reserved boot stack. + static _stack_end: u8; +} + +/// Returns the linker-defined virtual boundaries of the kernel sections. +/// +/// Only symbol addresses are taken. The linker script guarantees ordered +/// ranges with page-aligned starts. Empty sections have equal bounds. +#[must_use] +pub(crate) fn kernel_sections() -> KernelSections { + let range = |start: *const u8, end: *const u8| { + VirtRange::new(VirtAddr::new(start.addr()), VirtAddr::new(end.addr())) + }; + + KernelSections { + virtual_start: VirtAddr::new(core::ptr::addr_of!(_kernel_virtual_start).addr()), + text: range( + core::ptr::addr_of!(_text_start), + core::ptr::addr_of!(_text_end), + ), + rodata: range( + core::ptr::addr_of!(_rodata_start), + core::ptr::addr_of!(_rodata_end), + ), + data: range( + core::ptr::addr_of!(_data_start), + core::ptr::addr_of!(_data_end), + ), + bss: range( + core::ptr::addr_of!(_bss_start), + core::ptr::addr_of!(_bss_end), + ), + stack: range( + core::ptr::addr_of!(_stack_start), + core::ptr::addr_of!(_stack_end), + ), + } } /// Returns the kernel's complete boot-time memory footprint in bytes. @@ -120,3 +225,57 @@ impl PhysRange { self.start.as_usize() < other.end.as_usize() && other.start.as_usize() < self.end.as_usize() } } + +// TODO: Merge with PhysRange as the only difference so far was the type used. +// Maybe define a trait which can be derived for both? +impl VirtRange { + /// Constructs a virtual range from its inclusive start and exclusive end. + /// + /// Returns `None` if `start` is greater than or equal to `end`. + #[must_use] + pub(crate) const fn new(start: VirtAddr, end: VirtAddr) -> Option { + if start.as_usize() >= end.as_usize() { + return None; + } + + Some(Self { start, end }) + } + + /// Constructs a virtual range beginning at `start` and spanning `size` + /// bytes. + /// + /// Returns `None` if `size` is zero or if computing the end address would + /// overflow the virtual address representation. + #[must_use] + pub(crate) const fn from_start_size(start: VirtAddr, size: usize) -> Option { + if size == 0 { + return None; + } + + let Some(end) = start.checked_add(size) else { + return None; + }; + + Self::new(start, end) + } + + /// Returns the inclusive start address. + #[must_use] + pub(crate) const fn start(self) -> VirtAddr { + self.start + } + + /// Returns the exclusive end address. + #[must_use] + pub(crate) const fn end(self) -> VirtAddr { + self.end + } + + /// Returns whether this range overlaps `other`. + /// + /// Ranges that only meet at an endpoint do not overlap. + #[must_use] + pub(crate) const fn overlaps(self, other: Self) -> bool { + self.start.as_usize() < other.end.as_usize() && other.start.as_usize() < self.end.as_usize() + } +} diff --git a/kernel/src/memory/address.rs b/kernel/src/memory/address.rs index a865483..5b308af 100644 --- a/kernel/src/memory/address.rs +++ b/kernel/src/memory/address.rs @@ -70,6 +70,12 @@ impl fmt::LowerHex for PhysAddr { #[repr(transparent)] pub(crate) struct VirtAddr(usize); +impl fmt::Debug for VirtAddr { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{:#x}", self.0) + } +} + impl VirtAddr { /// Constructs a virtual address from its machine-sized representation. #[must_use] @@ -82,4 +88,15 @@ impl VirtAddr { pub(crate) const fn as_usize(self) -> usize { self.0 } + + /// Returns the virtual address obtained by adding `bytes`. + /// + /// Returns `None` if the addition would overflow. + #[must_use] + pub(crate) const fn checked_add(self, bytes: usize) -> Option { + match self.0.checked_add(bytes) { + Some(address) => Some(Self(address)), + None => None, + } + } } diff --git a/kernel/src/memory/frame.rs b/kernel/src/memory/frame.rs index 44d6eb3..ce17b43 100644 --- a/kernel/src/memory/frame.rs +++ b/kernel/src/memory/frame.rs @@ -73,6 +73,49 @@ impl PhysFrame { } } +/// A physical frame allocated exclusively for kernel use. +/// +/// Unlike [`PhysFrame`], this type represents ownership rather than just +/// describing a physical address range. Instances are created only by a frame +/// allocator after the frame has been excluded from future allocation. +#[derive(Debug)] +#[must_use = "dropping an allocated frame permanently loses its ownership token"] +// NOTE: The current allocator is monotonic, so dropping an `AllocatedFrame` does +// not return the frame to the allocator. +pub(crate) struct AllocatedFrame { + /// Physical frame owned by this allocation. + frame: PhysFrame, +} + +impl AllocatedFrame { + /// Constructs ownership of `frame`. + /// + /// This is restricted to the memory-management implementation because + /// constructing an `AllocatedFrame` asserts that the frame has been + /// exclusively allocated and will not be returned again. + const fn new(frame: PhysFrame) -> Self { + Self { frame } + } + + /// Returns the physical frame described by this allocation. + #[must_use] + pub(crate) const fn frame(&self) -> PhysFrame { + self.frame + } + + /// Returns the physical start address of this allocation. + #[must_use] + pub(crate) const fn start_address(&self) -> PhysAddr { + self.frame.start_address() + } + + /// Consumes this ownership token and returns its physical-frame descriptor. + #[must_use] + pub(crate) const fn into_frame(self) -> PhysFrame { + self.frame + } +} + /// Rounds a physical address up to the nearest frame boundary. /// /// If `address` is already frame-aligned, it is returned unchanged. diff --git a/kernel/src/memory/frame/boot_allocator.rs b/kernel/src/memory/frame/boot_allocator.rs index 137c6e9..1639c30 100644 --- a/kernel/src/memory/frame/boot_allocator.rs +++ b/kernel/src/memory/frame/boot_allocator.rs @@ -20,9 +20,11 @@ #[cfg(not(target_pointer_width = "64"))] compile_error!("the boot frame allocator currently requires a 64-bit target"); +use core::sync::atomic::{AtomicBool, Ordering}; + use fdt::{Fdt, MemoryRanges}; -use super::{PhysFrame, align_down, align_up}; +use super::{AllocatedFrame, PhysFrame, align_down, align_up}; use crate::memory::{PhysAddr, PhysRange}; /// Errors that can prevent construction of a boot frame allocator. @@ -36,6 +38,9 @@ pub(crate) enum BootFrameAllocatorError { /// Second overlapping physical memory range. second: PhysRange, }, + + /// The boot physical-frame allocator has already been claimed. + AlreadyClaimed, } /// State for the physical memory range currently being scanned. @@ -131,23 +136,56 @@ fn validate_memory_ranges(fdt: &Fdt<'_>) -> Result<(), BootFrameAllocatorError> Ok(()) } +/// Tracks whether the boot physical-frame allocator has already been claimed. +/// +/// Once set, this flag is never cleared, preventing multiple allocators from +/// independently issuing ownership of the same physical frames. +// TODO: If multiple physical-frame allocator implementations are introduced, +// move this one-shot ownership guard above the individual implementations. +static BOOT_FRAME_ALLOCATOR_CLAIMED: AtomicBool = AtomicBool::new(false); + impl<'fdt, 'dtb> BootFrameAllocator<'fdt, 'dtb> { - /// Constructs a boot frame allocator. + /// Claims the boot-time physical frame allocator. + /// + /// This allocator can be claimed only once. A successful claim remains + /// permanent even if the returned [`BootFrameAllocator`] value is later + /// dropped. /// - /// `kernel` and `dtb` describe physical ranges that must be excluded from - /// allocation. + /// # Safety + /// + /// This capability must be used only on the claiming hart while its + /// bootstrap root remains active. Supervisor interrupts must remain + /// disabled during temporary frame access. + /// + /// Every FDT-described memory range not excluded by `kernel`, `dtb`, or the + /// FDT's reservation information must refer to physical RAM that the kernel + /// may exclusively allocate. + /// + /// `kernel` and `dtb` must completely cover the physical memory occupied by + /// the live kernel image and device-tree blob. All other physical memory that + /// must not be allocated must be described by the supplied FDT's reservation + /// information. /// /// # Errors /// /// Returns [`BootFrameAllocatorError::OverlappingMemoryRanges`] if two - /// FDT-described physical memory ranges overlap. - pub(crate) fn new( + /// FDT-described physical memory ranges overlap, or + /// [`BootFrameAllocatorError::AlreadyClaimed`] if the boot frame allocator has + /// already been claimed. + pub(crate) unsafe fn claim( fdt: &'fdt Fdt<'dtb>, kernel: PhysRange, dtb: PhysRange, ) -> Result { validate_memory_ranges(fdt)?; + if BOOT_FRAME_ALLOCATOR_CLAIMED + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_err() + { + return Err(BootFrameAllocatorError::AlreadyClaimed); + } + Ok(Self { fdt, memory_ranges: fdt.memory_ranges(), @@ -176,7 +214,7 @@ impl<'fdt, 'dtb> BootFrameAllocator<'fdt, 'dtb> { clippy::expect_used, reason = "CurrentMemoryRange guarantees that candidate cursors are frame-aligned and can contain a complete frame" )] - pub(crate) fn allocate(&mut self) -> Option { + pub(crate) fn allocate(&mut self) -> Option { loop { if self.current.is_none() && !self.advance_memory_range() { return None; @@ -218,7 +256,7 @@ impl<'fdt, 'dtb> BootFrameAllocator<'fdt, 'dtb> { }); } - return Some(frame); + return Some(AllocatedFrame::new(frame)); } } diff --git a/nix/qemu.nix b/nix/qemu.nix index e9fc51b..50d550e 100644 --- a/nix/qemu.nix +++ b/nix/qemu.nix @@ -106,8 +106,18 @@ exit 1 fi - if ! grep -Fq "kernel entered" "$log"; then - echo "kernel did not reach main" >&2 + for expected in \ + "kernel entered" \ + "Kernel paging active"; do + if ! grep -Fq "$expected" "$log"; then + echo "kernel did not reach: $expected" >&2 + cat "$log" >&2 + exit 1 + fi + done + + if grep -Fq "Unhandled supervisor trap" "$log"; then + echo "kernel trapped during startup" >&2 cat "$log" >&2 exit 1 fi