diff --git a/.cargo/config.toml b/.cargo/config.toml index 6728891..db240e9 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -12,8 +12,20 @@ lto = "thin" [profile.bootloader_debug] inherits = "bootloader" +strip = "none" debug-assertions = true debug = "full" +lto = "off" [target.x86_64-unknown-none] -rustflags = ["-C", "code-model=kernel", "-C", "relocation-model=static"] +rustflags = [ + "-C", + "code-model=kernel", + "-C", + "relocation-model=pie", + "-C", + "linker-flavor=ld.lld", +] + +[target.x86_64-unknown-elpytios] +rustflags = ["-Z", "codegen-backend=llvm", "-C", "link-arg=-Map=linker.map"] diff --git a/.gitignore b/.gitignore index 4d430b3..cb48098 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ /*.exe /*.pdb +/*.log /target /esp *.qcow2 +/Cargo.lock /rustc-ice-* \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index 47d3596..0000000 --- a/Cargo.lock +++ /dev/null @@ -1,640 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "addr2line" -version = "0.25.1" -dependencies = [ - "alloc", - "core", - "gimli", - "object", - "rustc-demangle", -] - -[[package]] -name = "adler2" -version = "2.0.1" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "alloc" -version = "0.0.0" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "bit_field" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" - -[[package]] -name = "bitflags" -version = "2.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" - -[[package]] -name = "bytemuck" -version = "1.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" -dependencies = [ - "bytemuck_derive", -] - -[[package]] -name = "bytemuck_derive" -version = "1.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "cc" -version = "1.2.65" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" -dependencies = [ - "find-msvc-tools", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "compiler_builtins" -version = "0.1.160" -dependencies = [ - "cc", - "core", -] - -[[package]] -name = "const_panic" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652" -dependencies = [ - "const_panic_proc_macros", - "typewit", -] - -[[package]] -name = "const_panic_proc_macros" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb0240417fe20ccf13397fa25e6f0a987dbbfaf27d9e13532419df7f593e65e8" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", -] - -[[package]] -name = "convert_case" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" -dependencies = [ - "unicode-segmentation", -] - -[[package]] -name = "core" -version = "0.0.0" - -[[package]] -name = "derive_more" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" -dependencies = [ - "derive_more-impl", -] - -[[package]] -name = "derive_more-impl" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" -dependencies = [ - "convert_case", - "proc-macro2", - "quote", - "rustc_version", - "syn", - "unicode-xid", -] - -[[package]] -name = "dlmalloc" -version = "0.2.13" -dependencies = [ - "cfg-if 1.0.4", - "core", - "libc 0.2.185", - "windows-sys", -] - -[[package]] -name = "elpytios-abi" -version = "0.1.0" -dependencies = [ - "compiler_builtins", - "core", - "elpytios-abi-macros", -] - -[[package]] -name = "elpytios-abi-macros" -version = "0.1.0" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "elpytios-bootinfo" -version = "0.1.0" -dependencies = [ - "bitflags", - "bytemuck", - "derive_more", -] - -[[package]] -name = "elpytios-bootloader" -version = "0.1.0" -dependencies = [ - "bytemuck", - "const_panic", - "elpytios-bootinfo", - "elpytios-elf", - "log", - "uefi", -] - -[[package]] -name = "elpytios-elf" -version = "0.1.0" -dependencies = [ - "bitflags", - "bytemuck", - "const_panic", -] - -[[package]] -name = "elpytios-kernel" -version = "0.1.0" -dependencies = [ - "elpytios-abi", - "elpytios-bootinfo", -] - -[[package]] -name = "find-msvc-tools" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" - -[[package]] -name = "foldhash" -version = "0.2.0" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "fortanix-sgx-abi" -version = "0.6.1" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "gimli" -version = "0.32.3" -dependencies = [ - "alloc", - "core", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -dependencies = [ - "alloc", - "core", - "foldhash", - "libc 0.2.163", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hermit-abi" -version = "0.5.2" -dependencies = [ - "alloc", - "compiler_builtins", - "core", -] - -[[package]] -name = "libc" -version = "0.2.163" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fdaeca4cf44ed4ac623e86ef41f056e848dbeab7ec043ecb7326ba300b36fd0" - -[[package]] -name = "libc" -version = "0.2.185" -dependencies = [ - "rustc-std-workspace-core", -] - -[[package]] -name = "log" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" - -[[package]] -name = "memchr" -version = "2.7.6" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "miniz_oxide" -version = "0.8.9" -dependencies = [ - "adler2", - "alloc", - "core", -] - -[[package]] -name = "moto-rt" -version = "0.16.0" -dependencies = [ - "alloc", - "core", -] - -[[package]] -name = "object" -version = "0.37.3" -dependencies = [ - "alloc", - "compiler_builtins", - "core", - "hashbrown", - "memchr", -] - -[[package]] -name = "panic_abort" -version = "0.0.0" -dependencies = [ - "alloc", - "compiler_builtins", - "core", - "libc 0.2.185", -] - -[[package]] -name = "panic_unwind" -version = "0.0.0" -dependencies = [ - "alloc", - "core", - "libc 0.2.185", - "unwind", -] - -[[package]] -name = "proc-macro2" -version = "1.0.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "ptr_meta" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" -dependencies = [ - "ptr_meta_derive", -] - -[[package]] -name = "ptr_meta_derive" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "quote" -version = "1.0.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "r-efi-alloc" -version = "2.1.0" -dependencies = [ - "core", - "r-efi", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.27" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "rustc-std-workspace-core" -version = "1.99.0" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - -[[package]] -name = "semver" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" - -[[package]] -name = "shlex" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" - -[[package]] -name = "std" -version = "0.0.0" -dependencies = [ - "addr2line", - "alloc", - "cfg-if 1.0.4", - "core", - "dlmalloc", - "elpytios-abi", - "fortanix-sgx-abi", - "hashbrown", - "hermit-abi", - "libc 0.2.185", - "miniz_oxide", - "moto-rt", - "object", - "panic_abort", - "panic_unwind", - "r-efi", - "r-efi-alloc", - "rustc-demangle", - "std_detect", - "unwind", - "vex-sdk", - "wasip1", - "wasip2", - "wasip3", - "windows-link", -] - -[[package]] -name = "std_detect" -version = "0.1.5" -dependencies = [ - "alloc", - "core", - "libc 0.2.185", -] - -[[package]] -name = "syn" -version = "2.0.67" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8655ed1d86f3af4ee3fd3263786bc14245ad17c4c7e85ba7187fb3ae028c90" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "typewit" -version = "1.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6" - -[[package]] -name = "ucs2" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79298e11f316400c57ec268f3c2c29ac3c4d4777687955cd3d4f3a35ce7eba" -dependencies = [ - "bit_field", -] - -[[package]] -name = "uefi" -version = "0.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66ab9569afdd1e33a31d8002343aa1df594f055347b1a66136bf9dd6cbc3ec37" -dependencies = [ - "bitflags", - "cfg-if 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "log", - "ptr_meta", - "ucs2", - "uefi-macros", - "uefi-raw", - "uguid", -] - -[[package]] -name = "uefi-macros" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4687412b5ac74d245d5bfb1733ede50c31be19bf8a4b6a967a29b451bab49e67" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "uefi-raw" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3775e5934877acaef4b00f254f252df1e2266903c31e51455c117f4f2824eda" -dependencies = [ - "bitflags", - "uguid", -] - -[[package]] -name = "uguid" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8352f8c05e47892e7eaf13b34abd76a7f4aeaf817b716e88789381927f199c" - -[[package]] -name = "unicode-ident" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" - -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - -[[package]] -name = "unwind" -version = "0.0.0" -dependencies = [ - "core", - "libc 0.2.185", - "unwinding", -] - -[[package]] -name = "unwinding" -version = "0.2.8" -dependencies = [ - "core", - "gimli", - "libc 0.2.185", -] - -[[package]] -name = "vex-sdk" -version = "0.27.1" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "wasip1" -version = "1.0.0" -dependencies = [ - "alloc", - "compiler_builtins", - "core", -] - -[[package]] -name = "wasip2" -version = "1.0.3+wasi-0.2.9" -dependencies = [ - "alloc", - "core", - "wit-bindgen", -] - -[[package]] -name = "wasip3" -version = "0.6.0+wasi-0.3.0-rc-2026-03-15" -dependencies = [ - "alloc", - "core", - "wit-bindgen", -] - -[[package]] -name = "windows-link" -version = "0.0.0" - -[[package]] -name = "windows-sys" -version = "0.61.100" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -dependencies = [ - "alloc", - "core", -] diff --git a/Cargo.toml b/Cargo.toml index 9bb137d..ae62aba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,12 @@ [workspace] resolver = "3" -members = ["abi", "abi/macros", "elf", "std", "kernel", "bootloader", "bootinfo"] +members = [ + "abi", + "abi/macros", + "elf", + "std", + "kernel", + "bootloader", + "bootinfo", + "user/shell", +] diff --git a/abi/src/userspace/x86_64.rs b/abi/src/userspace/x86_64.rs index 7f62c44..410c0b4 100644 --- a/abi/src/userspace/x86_64.rs +++ b/abi/src/userspace/x86_64.rs @@ -8,7 +8,7 @@ pub unsafe fn syscall0(sys: usize) -> usize { "syscall", inlateout("rax") sys => ret, - options(nostack, preserves_flags), + options(nomem, nostack, preserves_flags), ); } ret @@ -24,7 +24,7 @@ pub unsafe fn syscall1(sys: usize, a0: usize) -> usize { in("rdi") a0, - options(nostack, preserves_flags), + options(nomem, nostack, preserves_flags), ); } ret @@ -41,7 +41,7 @@ pub unsafe fn syscall2(sys: usize, a0: usize, a1: usize) -> usize { in("rdi") a0, in("rsi") a1, - options(nostack, preserves_flags), + options(nomem, nostack, preserves_flags), ); } ret @@ -59,7 +59,7 @@ pub unsafe fn syscall3(sys: usize, a0: usize, a1: usize, a2: usize) -> usize { in("rsi") a1, in("rdx") a2, - options(nostack, preserves_flags), + options(nomem, nostack, preserves_flags), ); } ret @@ -78,7 +78,7 @@ pub unsafe fn syscall4(sys: usize, a0: usize, a1: usize, a2: usize, a3: usize) - in("rdx") a2, in("r10") a3, - options(nostack, preserves_flags), + options(nomem, nostack, preserves_flags), ); } ret @@ -98,7 +98,7 @@ pub unsafe fn syscall5(sys: usize, a0: usize, a1: usize, a2: usize, a3: usize, a in("r10") a3, in("r8") a4, - options(nostack, preserves_flags), + options(nomem, nostack, preserves_flags), ); } ret @@ -119,7 +119,7 @@ pub unsafe fn syscall6(sys: usize, a0: usize, a1: usize, a2: usize, a3: usize, a in("r8") a4, in("r9") a5, - options(nostack, preserves_flags), + options(nomem, nostack, preserves_flags), ); } ret diff --git a/bootinfo/Cargo.toml b/bootinfo/Cargo.toml index 4e2d839..53a9a02 100644 --- a/bootinfo/Cargo.toml +++ b/bootinfo/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" rust-version = "1.98" [dependencies] +arrayvec = { version = "0.7", default-features = false } bitflags = "2" bytemuck = { version = "1", features = ["derive", "latest_stable_rust"] } derive_more = { version = "2", default-features = false, features = [ diff --git a/bootinfo/src/lib.rs b/bootinfo/src/lib.rs index 343fcd8..132f120 100644 --- a/bootinfo/src/lib.rs +++ b/bootinfo/src/lib.rs @@ -1,63 +1,22 @@ #![feature(custom_inner_attributes)] #![rustfmt::skip] - #![no_std] pub mod paddr; -pub mod vaddr { - use super::*; - - #[derive(Debug, Clone, Copy)] - #[repr(transparent)] - pub struct VFlags(usize); - bitflags! { - impl VFlags: usize { - const WRITABLE = 1 << 0; - const USER_MODE = 1 << 1; - const WRITE_THROUGH = 1 << 2; - const CACHE_DISABLED = 1 << 3; - const ACCESSED = 1 << 4; - - /// Don't flush translation lookaside buffers when switching virtual map tables - const GLOBAL = 1 << 5; - } - } - - #[derive(Debug, Display, Clone, Copy)] - #[repr(C)] - pub enum VirtualMapError { - #[display("Couldn't allocate a page table")] - PageTable, - #[display("Couldn't map {v_addr} to {p_addr}: the virtual address is reserved")] - Reserved { p_addr: PAddr, v_addr: VAddr }, - #[display("Couldn't map {v_addr} to {p_addr}: the virtual address is already mapped to {p_addr_existing}")] - AlreadyMapped { p_addr: PAddr, v_addr: VAddr, p_addr_existing: PAddr } - } - - cfg_select! { - target_arch = "x86_64" => { - mod x86_64; - pub use x86_64::*; - } - _ => { - compile_error!("Unsupported architecture"); - } - } -} +use arrayvec::ArrayVec; use bitflags::bitflags; -use derive_more::Display; - -use core::{mem::MaybeUninit, slice}; - use paddr::PAddr; -use vaddr::{VAddr, VirtualMap}; pub const PAGE_SIZE: usize = 4096; + pub const MAX_MEMORY_REGIONS: usize = 128; +pub const MAX_IDENTITY_MAPS: usize = 32; +pub const MAX_SCRATCH: usize = 4; +pub const MAX_RELOCATIONS: usize = 8; #[allow(non_camel_case_types)] -#[derive(Clone, Copy)] +#[derive(Debug, Clone, Copy)] #[repr(usize)] pub enum PixelFormat { RGB_8_BIT, @@ -66,44 +25,84 @@ pub enum PixelFormat { BLT_ONLY } +#[derive(Debug, Clone, Copy)] #[repr(C)] -#[derive(Clone, Copy)] pub struct GraphicsInfo { pub w: usize, pub h: usize, pub stride: usize, pub pixel_format: PixelFormat, - pub frame_buffer: *mut u8, + pub frame_buffer: PAddr, pub frame_buffer_size: usize } -#[derive(Clone, Copy)] +#[derive(Debug, Clone, Copy)] +#[repr(C)] pub struct MemoryRegion { pub base: PAddr, - pub pages: usize + pub pages: usize, } -#[repr(C, align(4096))] -pub struct BootInfo { - pub graphics_info: GraphicsInfo, - pub virtual_map: VirtualMap, - /// Leftover identity-mapping from the bootloader, to be unmapped by the kernel - pub switcher_map: VAddr, - - pub memory_regions_base: [MaybeUninit; MAX_MEMORY_REGIONS], - pub memory_regions_size: usize, - pub v_addr_start: VAddr, - pub v_addr_end: VAddr, +impl MemoryRegion { + #[inline] + pub const fn at(base: PAddr, pages: usize) -> Self { + Self { base, pages } + } } -impl BootInfo { - #[inline] - pub fn memory_regions(&self) -> &[MemoryRegion] { - unsafe { slice::from_raw_parts(&raw const self.memory_regions_base as _, self.memory_regions_size) } +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct IdentityMap { + pub region: MemoryRegion, + pub flags: IdentityMapFlags, +} + +bitflags! { + #[derive(Debug, Clone, Copy)] + #[repr(transparent)] + pub struct IdentityMapFlags: u8 { + const EXECUTABLE = 1 << 0; + const WRITABLE = 1 << 1; + const READABLE = 1 << 2; } +} +impl IdentityMap { #[inline] - pub fn v_addr_range(&self) -> [VAddr; 2] { - [self.v_addr_start, self.v_addr_end] + pub const fn new(base: PAddr, pages: usize, flags: IdentityMapFlags) -> Self { + Self { + region: MemoryRegion { base, pages }, + flags, + } } +} + +#[derive(Debug, Clone, Copy)] +pub struct Reloc { + pub offset: usize, + pub size: usize, + pub stride: usize, +} + +#[derive(Debug, Clone, Copy)] +pub enum DeviceTree { + Acpi(PAddr), + Acpi2(PAddr), +} + +// Note: Must uphold `BootInfo: Sync` +#[derive(Debug)] +#[repr(C, align(4096))] +pub struct BootInfo { + pub graphics_info: GraphicsInfo, + pub device_tree: DeviceTree, + + /// Used to calculate slide for virtual mapping + pub kernel_elf_base: PAddr, + pub kernel_virt_base: usize, + + pub memory_regions: ArrayVec, + pub identity_maps: ArrayVec, + pub scratch_pages: ArrayVec, + pub relocations: ArrayVec, } \ No newline at end of file diff --git a/bootinfo/src/paddr.rs b/bootinfo/src/paddr.rs index 47ac40c..2653d72 100644 --- a/bootinfo/src/paddr.rs +++ b/bootinfo/src/paddr.rs @@ -24,12 +24,12 @@ impl PAddr { impl fmt::Debug for PAddr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{self}") + fmt::Pointer::fmt(self, f) } } -impl fmt::Display for PAddr { +impl fmt::Pointer for PAddr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:#018p}", self.0 as *const ()) + write!(f, "{:#018x}", self.0) } } diff --git a/bootinfo/src/vaddr/x86_64/virt.rs b/bootinfo/src/vaddr/x86_64/virt.rs deleted file mode 100644 index 360a296..0000000 --- a/bootinfo/src/vaddr/x86_64/virt.rs +++ /dev/null @@ -1,383 +0,0 @@ -use core::{cell::RefCell, fmt, hint::unreachable_unchecked, ops::DerefMut}; - -use bytemuck::Zeroable; - -use crate::{ - paddr::PAddr, - vaddr::{Entry, NodeEntry, PdEntry, PdTable, PdptEntry, PdptTable, Pml4Table, PtEntry, PtTable, UnionEntry, VFlags, VirtualMapError}, -}; - -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Zeroable)] -#[repr(transparent)] -pub struct VAddr(usize); -impl VAddr { - #[inline] - pub const fn new(virtual_address: usize) -> Self { - Self(virtual_address) - } - - #[inline] - pub const fn addr(self) -> usize { - self.0 - } - - #[inline] - pub const fn ptr(self) -> *const T { - self.0 as *const T - } - - #[inline] - pub const fn ptr_mut(self) -> *mut T { - self.0 as *mut T - } - - #[inline] - pub(crate) const fn info(self) -> VAddrInfo { - VAddrInfo { - page_offset: self.0 & 0xfff, - pt_index: (self.0 >> 12) & 0x1ff, - pd_index: (self.0 >> 21) & 0x1ff, - pdpt_index: (self.0 >> 30) & 0x1ff, - pml4_index: (self.0 >> 39) & 0x1ff, - } - } - - #[inline] - pub(crate) const fn from_info( - VAddrInfo { - page_offset, - pt_index, - pd_index, - pdpt_index, - pml4_index, - }: VAddrInfo, - ) -> Self { - let addr = - page_offset & 0xfff | (pt_index & 0x1ff) << 12 | (pd_index & 0x1ff) << 21 | (pdpt_index & 0x1ff) << 30 | (pml4_index & 0x1ff) << 39; - Self((addr.cast_signed() << 16 >> 16).cast_unsigned()) - } -} - -#[derive(Debug, Clone, Copy, Zeroable)] -#[repr(C)] -pub(crate) struct VAddrInfo { - pub page_offset: usize, - pub pt_index: usize, - pub pd_index: usize, - pub pdpt_index: usize, - pub pml4_index: usize, -} - -impl fmt::Debug for VAddr { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self.info()) - } -} - -impl fmt::Display for VAddr { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:#018p}", self.0 as *const ()) - } -} - -#[derive(Debug)] -#[repr(C)] -pub struct VirtualMapBuilder { - map: VirtualMap, - new_page_table: fn() -> Option, -} - -impl VirtualMapBuilder { - /// # Safety - /// - `new_page_table` must return a [`PAGE_SIZE`](crate::PAGE_SIZE)-aligned physical address - /// that is: - /// - Completely zeroed out. - /// - Completely free to be written to (nothing else "owns" it). - /// - `page_table_ptr` must convert physical addresses returned by `new_page_table` into a - /// pointer that points to a page table. - #[inline] - pub const unsafe fn new(recursion_index: usize, new_page_table: fn() -> Option, page_table_ptr: unsafe fn(PAddr) -> *mut ()) -> Self { - Self { - map: VirtualMap { - mapper: LocalMapper { - table: RefCell::new(bytemuck::zeroed()), - page_table_ptr, - recursion_index, - }, - }, - new_page_table, - } - } - - #[inline] - pub fn map(&mut self, p_addr: PAddr, v_addr: VAddr, flags: VFlags) -> Result<(), VirtualMapError> { - unsafe { self.map.map(p_addr, v_addr, flags, self.new_page_table) } - } - - pub fn finish(self) -> Result<(PAddr, VirtualMap), VirtualMapError> { - let Self { - map: VirtualMap { mut mapper }, - new_page_table, - } = self; - - let pml4_phys = (new_page_table)().ok_or(VirtualMapError::PageTable)?; - match mapper.table.get_mut().pdpt_entries[mapper.recursion_index] { - e if e.is_present() => unreachable!("`recursion_index` is checked in earlier methods"), - ref mut e => *e = unsafe { NodeEntry::new(Entry::WRITABLE, pml4_phys) }, - } - - unsafe { - (mapper.page_table_ptr)(pml4_phys).cast::().write(mapper.table.into_inner()); - } - - Ok((pml4_phys, VirtualMap { - mapper: unsafe { sealed::RecursiveMapper::new(mapper.recursion_index) }, - })) - } -} - -#[derive(Debug)] -#[repr(C)] -pub struct LocalMapper { - table: RefCell, - page_table_ptr: unsafe fn(PAddr) -> *mut (), - recursion_index: usize, -} - -unsafe impl sealed::VirtualMapper for LocalMapper { - #[inline] - fn reserved(&self, v_addr: VAddr) -> bool { - let VAddrInfo { pml4_index, .. } = v_addr.info(); - pml4_index == self.recursion_index - } - - #[inline] - fn pml4(&self) -> impl DerefMut { - self.table.borrow_mut() - } - - #[inline] - unsafe fn pdpt(&self, pml4_index: usize) -> &mut PdptTable { - unsafe { - (self.page_table_ptr)(self.table.borrow_mut().pdpt_entries.get(pml4_index).unwrap_unchecked().child_addr()) - .cast::() - .as_mut_unchecked() - } - } - - #[inline] - unsafe fn pd(&self, pml4_index: usize, pdpt_index: usize) -> &mut PdTable { - unsafe { - (self.page_table_ptr)(match self.pdpt(pml4_index).pd_entries.get(pdpt_index).unwrap_unchecked().kind() { - UnionEntry::Node(e) => e.child_addr(), - UnionEntry::Leaf(..) => unreachable_unchecked(), - }) - .cast::() - .as_mut_unchecked() - } - } - - #[inline] - unsafe fn pt(&self, pml4_index: usize, pdpt_index: usize, pd_index: usize) -> &mut PtTable { - unsafe { - (self.page_table_ptr)(match self.pd(pml4_index, pdpt_index).pt_entries.get(pd_index).unwrap_unchecked().kind() { - UnionEntry::Node(e) => e.child_addr(), - UnionEntry::Leaf(..) => unreachable_unchecked(), - }) - .cast::() - .as_mut_unchecked() - } - } -} - -#[derive(Debug)] -#[repr(transparent)] -// Note: From user-facing API perspective, `VirtualMap` must have no trait bounds. -pub struct VirtualMap { - mapper: T, -} - -impl VirtualMap { - /// # Safety - /// - `new_page_table` must return a [`PAGE_SIZE`](crate::PAGE_SIZE)-aligned physical address - /// that is: - /// - Completely zeroed out. - /// - Completely free to be written to (nothing else "owns" it). - /// - There must never be concurrent (multithreaded) calls to this method that have the same - /// virtual page occupied by `v_addr`. - pub unsafe fn map( - &self, - p_addr: PAddr, - v_addr: VAddr, - flags: VFlags, - mut new_page_table: impl FnMut() -> Option, - ) -> Result<(), VirtualMapError> { - if self.mapper.reserved(v_addr) { - return Err(VirtualMapError::Reserved { p_addr, v_addr }) - } - - let VAddrInfo { - pt_index, - pd_index, - pdpt_index, - pml4_index, - .. - } = v_addr.info(); - unsafe { - match &mut self.mapper.pml4().pdpt_entries[pml4_index] { - e if !e.is_present() => *e = NodeEntry::new(Entry::WRITABLE, new_page_table().ok_or(VirtualMapError::PageTable)?), - _ => {} - } - - match &mut self.mapper.pdpt(pml4_index).pd_entries[pdpt_index] { - e if !e.is_present() => *e = PdptEntry::node(NodeEntry::new(Entry::WRITABLE, new_page_table().ok_or(VirtualMapError::PageTable)?)), - e if let UnionEntry::Leaf(e) = e.kind() => { - return Err(VirtualMapError::AlreadyMapped { - p_addr, - v_addr, - p_addr_existing: e.addr(), - }) - } - _ => {} - } - - match &mut self.mapper.pd(pml4_index, pdpt_index).pt_entries[pd_index] { - e if !e.is_present() => *e = PdEntry::node(NodeEntry::new(Entry::WRITABLE, new_page_table().ok_or(VirtualMapError::PageTable)?)), - e if let UnionEntry::Leaf(e) = e.kind() => { - return Err(VirtualMapError::AlreadyMapped { - p_addr, - v_addr, - p_addr_existing: e.addr(), - }) - } - _ => {} - } - - match &mut self.mapper.pt(pml4_index, pdpt_index, pd_index).phys_pages[pt_index] { - e if e.is_present() => { - return Err(VirtualMapError::AlreadyMapped { - p_addr, - v_addr, - p_addr_existing: e.addr(), - }) - } - e => *e = PtEntry::new(flags.into(), p_addr) | if flags.contains(VFlags::GLOBAL) { PtEntry::GLOBAL } else { PtEntry::empty() }, - } - } - - Ok(()) - } -} - -mod sealed { - use super::*; - - #[allow(unused_variables, reason = "Available for implementors, not defaults")] - pub unsafe trait VirtualMapper { - #[inline] - fn reserved(&self, v_addr: VAddr) -> bool { - false - } - - fn pml4(&self) -> impl DerefMut; - - /// # Safety - /// - [`pml4_index`] must be within `0..512` (exclusive). - unsafe fn pdpt(&self, pml4_index: usize) -> &mut PdptTable; - - /// # Safety: - /// - [`Self::pdpt()`] to the given indices must return a node entry, not leaf. - /// - [`pml4_index`] and [`pdpt_index`] must be within `0..512` (exclusive). - unsafe fn pd(&self, pml4_index: usize, pdpt_index: usize) -> &mut PdTable; - - /// # Safety: - /// - [`Self::pd()`] to the given indices must return a node entry, not leaf. - /// - [`pml4_index`], [`pdpt_index`], and [`pd_index`] must be within `0..512` (exclusive). - unsafe fn pt(&self, pml4_index: usize, pdpt_index: usize, pd_index: usize) -> &mut PtTable; - } - - #[derive(Debug)] - #[repr(transparent)] - pub struct RecursiveMapper { - recursion_index: usize, - } - - impl RecursiveMapper { - /// # Safety: - /// `recursion_index` must be N where - /// [`pdpt_entries[N]`](crate::vaddr::Pml4Table::pdpt_entries) points to the - /// physical address of the PML4 table itself (i.e. recursive slot). - #[inline] - pub const unsafe fn new(recursion_index: usize) -> Self { - Self { recursion_index } - } - } - - unsafe impl VirtualMapper for RecursiveMapper { - #[inline] - fn reserved(&self, v_addr: VAddr) -> bool { - let VAddrInfo { pml4_index, .. } = v_addr.info(); - pml4_index == self.recursion_index - } - - #[inline] - fn pml4(&self) -> impl DerefMut { - unsafe { - VAddr::from_info(VAddrInfo { - page_offset: 0, // Keep recursing so it ends up with the PML4 table itself - pt_index: self.recursion_index, - pd_index: self.recursion_index, - pdpt_index: self.recursion_index, - pml4_index: self.recursion_index, - }) - .ptr_mut::() - .as_mut_unchecked() - } - } - - #[inline] - unsafe fn pdpt(&self, pml4_index: usize) -> &mut PdptTable { - unsafe { - VAddr::from_info(VAddrInfo { - page_offset: 0, // Stop recursing at PT index so it ends up with the PDPT entry - pt_index: pml4_index, - pd_index: self.recursion_index, - pdpt_index: self.recursion_index, - pml4_index: self.recursion_index, - }) - .ptr_mut::() - .as_mut_unchecked() - } - } - - #[inline] - unsafe fn pd(&self, pml4_index: usize, pdpt_index: usize) -> &mut PdTable { - unsafe { - VAddr::from_info(VAddrInfo { - page_offset: 0, // Stop recursing at PD index so it ends up with the PD entry - pt_index: pdpt_index, - pd_index: pml4_index, - pdpt_index: self.recursion_index, - pml4_index: self.recursion_index, - }) - .ptr_mut::() - .as_mut_unchecked() - } - } - - #[inline] - unsafe fn pt(&self, pml4_index: usize, pdpt_index: usize, pd_index: usize) -> &mut PtTable { - unsafe { - VAddr::from_info(VAddrInfo { - page_offset: 0, // Stop recursing at PDPT index so it ends up with the PT entry - pt_index: pd_index, - pd_index: pdpt_index, - pdpt_index: pml4_index, - pml4_index: self.recursion_index, - }) - .ptr_mut::() - .as_mut_unchecked() - } - } - } -} diff --git a/bootloader/Cargo.toml b/bootloader/Cargo.toml index 2e655d7..06465e8 100644 --- a/bootloader/Cargo.toml +++ b/bootloader/Cargo.toml @@ -12,7 +12,6 @@ forced-target = "x86_64-unknown-uefi" elpytios-elf = { path = "../elf" } elpytios-bootinfo = { path = "../bootinfo" } -bytemuck = { version = "1", features = ["derive", "latest_stable_rust"] } +arrayvec = { version = "0.7", default-features = false } const_panic = { version = "0.2", features = ["derive", "rust_latest_stable"] } -log = "0.4" -uefi = { version = "0.37", features = ["logger", "panic_handler"] } +uefi = { version = "0.38", features = ["logger", "panic_handler"] } diff --git a/bootloader/src/lib.rs b/bootloader/src/lib.rs deleted file mode 100644 index 6f2b4ee..0000000 --- a/bootloader/src/lib.rs +++ /dev/null @@ -1,3 +0,0 @@ -#![no_std] - -pub mod region_grouper; diff --git a/bootloader/src/main.rs b/bootloader/src/main.rs index bc77cb2..9bb713e 100644 --- a/bootloader/src/main.rs +++ b/bootloader/src/main.rs @@ -1,21 +1,27 @@ -#![feature(const_cmp, const_convert, const_iter, const_trait_impl, custom_inner_attributes, fn_align)] +#![feature(const_cmp, const_convert, const_iter, const_trait_impl, custom_inner_attributes)] #![rustfmt::skip] #![no_std] #![no_main] -use core::{arch::naked_asm, mem::{self, MaybeUninit}, slice}; +use core::{arch::asm, mem::{self, MaybeUninit}}; +use arrayvec::ArrayVec; use const_panic::concat_panic; -use elpytios_elf::{Elf, Elf64, ElfSegment64, ElfSegmentType, sys::ElfProgramFlags}; -use elpytios_bootinfo::{BootInfo, GraphicsInfo, MAX_MEMORY_REGIONS, MemoryRegion, paddr::PAddr, vaddr::{VAddr, VFlags, VirtualMapBuilder}}; -use uefi::{Status, boot::{self, AllocateType, MemoryType}, entry, helpers, mem::memory_map::{MemoryMap, MemoryMapOwned}, proto::console::gop::*}; +use elpytios_elf::{Elf, Elf64, ElfSegment64, ElfSegmentType, sys::{ElfProgramFlags, ElfRela64, ElfRela64Type}}; +use elpytios_bootinfo::{BootInfo, DeviceTree, GraphicsInfo, IdentityMap, IdentityMapFlags, MAX_SCRATCH, MemoryRegion, PAGE_SIZE, Reloc, paddr::PAddr}; +use uefi::{Status, boot::{self, AllocateType, MemoryType}, entry, helpers, mem::memory_map::MemoryMap, proto::console::gop::*, table::cfg::ConfigTableEntry}; -const PAGE_SIZE: usize = 4096; +const _: () = assert!(PAGE_SIZE == boot::PAGE_SIZE); -const ELPYTI_KERNEL_CODE: MemoryType = MemoryType::custom(0x8000_0000); -const ELPYTI_KERNEL_STACK: MemoryType = MemoryType::custom(0x8000_0001); -const ELPYTI_PAGE_TABLE: MemoryType = MemoryType::custom(0x8000_0002); +const MEM_KERNEL_CODE: MemoryType = MemoryType::custom(0x8000_0000); +const MEM_STACK: MemoryType = MemoryType::custom(0x8000_0001); +const MEM_BOOT_INFO: MemoryType = MemoryType::custom(0x8000_0002); +const MEM_SCRATCH: MemoryType = MemoryType::custom(0x8000_0003); + +const MEM_STACK_LEN: usize = 16; // Note: `opt-level = 0` makes the code consume way too much stack space +const MEM_BOOT_INFO_LEN: usize = size_of::().div_ceil(PAGE_SIZE); +const MEM_SCRATCH_LEN: usize = MAX_SCRATCH; const KERNEL_BINARY: Elf64 = match Elf::from_bytes(include_bytes!(concat!("../../target/x86_64-unknown-none/", cfg_select! { debug_assertions => "bootloader_debug", @@ -36,7 +42,7 @@ const KERNEL_SEGMENTS: [ElfSegment64; KERNEL_BINARY.program_header_count()] = { Err(e) => concat_panic!(e), }; - if segment.alignment != PAGE_SIZE as u64 { + if let ElfSegmentType::Load(..) = segment.segment_type && segment.alignment != PAGE_SIZE as u64 { concat_panic!("Kernel segments must be aligned to ", PAGE_SIZE, "! Found: ", segment.alignment); } @@ -49,53 +55,31 @@ const KERNEL_SEGMENTS: [ElfSegment64; KERNEL_BINARY.program_header_count()] = { unsafe { out.assume_init() } }; -const KERNEL_BOOTINFO_ADDRESS: usize = { - let mut ret = 0; - for section in KERNEL_BINARY.sections() { - let section = match section { - Ok(section) => section, - Err(e) => concat_panic!(e), - }; - - if section.name(&KERNEL_BINARY) == b".bootinfo" { - ret = match usize::try_from(section.virtual_address) { - Ok(max) => max.next_multiple_of(PAGE_SIZE), - _ => concat_panic!("Integer doesn't fit: ", section.virtual_address), - }; - break - } - } - - if ret == 0 { - panic!("`.bootinfo` section not found") - } else if ret % PAGE_SIZE != 0 { - concat_panic!("`.bootinfo` section not aligned: ", ret) - } else { - ret - } -}; - /// Index 0: Lowest virtual address of the kernel. -/// Index 1: New virtual addresses can take this spot. +/// Index 1: Highest virtual address of the kernel, page-aligned. const KERNEL_VIRTUAL_ADDRESSES: [usize; 2] = { let mut min = u64::MAX; let mut max = u64::MIN; let mut i = 0; loop { - let segment = KERNEL_SEGMENTS[i]; - if segment.segment_type != ElfSegmentType::Load { continue } + if i == KERNEL_SEGMENTS.len() { break } + + let segment = &KERNEL_SEGMENTS[i]; + if !matches!(segment.segment_type, ElfSegmentType::Load(..)) { + i += 1; + continue + } min = min.min(segment.virtual_address); max = max.max(segment.virtual_address + segment.memory_size); i += 1; - if i == KERNEL_SEGMENTS.len() { break } } match (usize::try_from(min), usize::try_from(max)) { (Ok(min), Ok(max)) => [ - if min % PAGE_SIZE == 0 { + if min.is_multiple_of(PAGE_SIZE) { min } else { concat_panic!("Virtual address base (", min, ") isn't aligned to ", PAGE_SIZE) @@ -106,40 +90,24 @@ const KERNEL_VIRTUAL_ADDRESSES: [usize; 2] = { } }; -const KERNEL_STACK_PAGES: usize = 8; - struct UefiInfo { - pub memory_map: MemoryMapOwned, - pub pml4_phys: PAddr, - - pub kernel_stack_base: VAddr, - pub kernel_entry: VAddr, + pub kernel_entry: *mut u8, + pub kernel_stack_base: *mut u8, + pub boot_info: *mut BootInfo, } fn setup_uefi_and_exit() -> UefiInfo { - let memory_map: MemoryMapOwned; - let pml4_phys: PAddr; - - let boot_info_ptr: *mut BootInfo; - let kernel_stack_base: VAddr; - let kernel_entry: VAddr; + let kernel_entry: *mut u8; + let kernel_stack_base: *mut u8; + let boot_info: *mut BootInfo; helpers::init().unwrap(); { // Graphics Info Fetching - let graphics_output_protocol_handle = boot::get_handle_for_protocol::().expect("No Graphics Output Protocol"); - let mut graphics_output_protocol; - unsafe { - graphics_output_protocol = boot::open_protocol::( - boot::OpenProtocolParams { - handle: graphics_output_protocol_handle, - agent: boot::image_handle(), - controller: None - }, - boot::OpenProtocolAttributes::GetProtocol - ).expect("Error opening Graphics Output Protocol"); - } + let mut graphics_output_protocol = boot::open_protocol_exclusive::( + boot::get_handle_for_protocol::().expect("No Graphics Output Protocol") + ).expect("Error opening Graphics Output Protocol"); let mut max_area: usize = 0; let mut max_mode: Option = None; @@ -169,121 +137,133 @@ fn setup_uefi_and_exit() -> UefiInfo { let pixel_format = mode_info.pixel_format(); let frame_buffer_ptr = frame_buffer.as_mut_ptr(); - let mut virtual_map = unsafe { - VirtualMapBuilder::new( - // 511 used for higher-half addressing - 510, - // Allocate 1 page via UEFI's allocator - || boot::allocate_pages(AllocateType::AnyPages, ELPYTI_PAGE_TABLE, 1).ok().map(|ptr| { - let ptr = ptr.as_ptr(); - ptr.write_bytes(0, PAGE_SIZE); - PAddr::new(ptr.addr()) - }), - // Identity mapping is still enabled at this point - |ptr| ptr.addr() as *mut (), - ) + let graphics_info = GraphicsInfo { + w, + h, + stride, + pixel_format: match pixel_format { + PixelFormat::Rgb => elpytios_bootinfo::PixelFormat::RGB_8_BIT, + PixelFormat::Bgr => elpytios_bootinfo::PixelFormat::BGR_8_BIT, + PixelFormat::Bitmask => elpytios_bootinfo::PixelFormat::BIT_MASK, + PixelFormat::BltOnly => elpytios_bootinfo::PixelFormat::BLT_ONLY + }, + frame_buffer: PAddr::new(frame_buffer_ptr.addr()), + frame_buffer_size: frame_buffer_size, }; - let [virtual_base, mut next_v_addr] = KERNEL_VIRTUAL_ADDRESSES; - let kernel_ptr = boot::allocate_pages(AllocateType::AnyPages, ELPYTI_KERNEL_CODE, (next_v_addr - virtual_base) / PAGE_SIZE).unwrap().as_ptr(); + let [virtual_base, virtual_max] = KERNEL_VIRTUAL_ADDRESSES; + let mut identity_maps = ArrayVec::new(); + + let kernel_base_pages = (virtual_max - virtual_base) / PAGE_SIZE; + let kernel_ptr = boot::allocate_pages(AllocateType::AnyPages, MEM_KERNEL_CODE, kernel_base_pages).unwrap().as_ptr(); + for segment in KERNEL_SEGMENTS { - if segment.segment_type != ElfSegmentType::Load { continue } + let ElfSegmentType::Load(data) = segment.segment_type else { continue }; + identity_maps.push(IdentityMap::new( + PAddr::new(kernel_ptr.addr() + segment.virtual_address as usize - virtual_base), + (segment.memory_size as usize).div_ceil(PAGE_SIZE), + { + let mut flags = IdentityMapFlags::empty(); + if segment.flags.contains(ElfProgramFlags::EXECUTABLE) { flags |= IdentityMapFlags::EXECUTABLE } + if segment.flags.contains(ElfProgramFlags::READABLE) { flags |= IdentityMapFlags::READABLE } + if segment.flags.contains(ElfProgramFlags::WRITABLE) { flags |= IdentityMapFlags::WRITABLE } + + flags + }, + )); + unsafe { kernel_ptr .add(segment.virtual_address as usize - virtual_base) - .copy_from_nonoverlapping(segment.data.as_ptr(), segment.data.len()); + .copy_from_nonoverlapping(data.as_ptr(), data.len()); kernel_ptr - .add(segment.virtual_address as usize - virtual_base + segment.data.len()) - .add(segment.data.len()).write_bytes(0, segment.memory_size as usize - segment.data.len()); + .add(segment.virtual_address as usize - virtual_base + data.len()) + .write_bytes(0, segment.memory_size as usize - data.len()); } + } - for i in (0..segment.memory_size as usize).step_by(PAGE_SIZE) { - virtual_map.map( - PAddr::new(unsafe { kernel_ptr.add(segment.virtual_address as usize - virtual_base).addr() } + i), - VAddr::new(segment.virtual_address as usize + i), - VFlags::GLOBAL | match segment.flags.contains(ElfProgramFlags::WRITABLE) { - false => VFlags::empty(), - true => VFlags::WRITABLE, - }, - ).unwrap_or_else(|e| panic!("{e}")); + let mut relocations = ArrayVec::new(); + for segment in KERNEL_SEGMENTS { + let ElfSegmentType::Dynamic { offset, size, stride } = segment.segment_type else { continue }; + relocations.push(Reloc { offset, size, stride }); + for i in 0..size / stride { + unsafe { + let rela = kernel_ptr.cast::().byte_add(offset - virtual_base).add(i).read_unaligned(); + match rela.info.kind { + ElfRela64Type::X86_64_RELATIVE => { + let slide = kernel_ptr.addr() as i64 - virtual_base as i64; + let patch_addr = kernel_ptr.add(rela.offset as usize - virtual_base); + let value = slide + rela.addend; + patch_addr.cast::().write(value); + } + kind => panic!("Unsupported Elf64_Rela kind: {}", kind.0), + } + } } } - kernel_entry = VAddr::new(KERNEL_BINARY.program_entry() as usize); - - // Identity-map the kernel switcher - let switcher_addr = (switch_to_kernel as *const ()).addr(); - assert_eq!(switcher_addr % PAGE_SIZE, 0, "`switch_to_kernel` must be page-aligned"); - virtual_map.map(PAddr::new(switcher_addr), VAddr::new(switcher_addr), VFlags::empty()).unwrap_or_else(|e| panic!("{e}")); - - let mut next_free_page = |p_addr: PAddr, page_count: usize, flags: VFlags| { - let v_addr = next_v_addr; - next_v_addr += page_count * PAGE_SIZE; - - for i in 0..page_count { - let offset = i * PAGE_SIZE; - virtual_map.map( - PAddr::new(p_addr.addr() + offset), - VAddr::new(v_addr + offset), - VFlags::GLOBAL | flags, - ).unwrap_or_else(|e| panic!("{e}")); - } - VAddr::new(v_addr) - }; + let stack_ptr = boot::allocate_pages(AllocateType::AnyPages, MEM_STACK, MEM_STACK_LEN).unwrap().as_ptr(); + identity_maps.push(IdentityMap::new(PAddr::new(stack_ptr.addr()), MEM_STACK_LEN, IdentityMapFlags::READABLE | IdentityMapFlags::WRITABLE)); - // Map the stack pointer - let stack_ptr = boot::allocate_pages(AllocateType::AnyPages, ELPYTI_KERNEL_STACK, KERNEL_STACK_PAGES).unwrap().as_ptr(); - let stack = next_free_page(PAddr::new(stack_ptr.addr()), KERNEL_STACK_PAGES, VFlags::WRITABLE); - kernel_stack_base = VAddr::new(stack.addr() + KERNEL_STACK_PAGES * PAGE_SIZE); + kernel_entry = unsafe { kernel_ptr.add(KERNEL_BINARY.program_entry() as usize - virtual_base) }; + kernel_stack_base = unsafe { stack_ptr.add(MEM_STACK_LEN * PAGE_SIZE) }; - // Map the framebuffer - let fb_phys = frame_buffer_ptr.addr(); - let fb_size = frame_buffer_size; + boot_info = boot::allocate_pages(AllocateType::AnyPages, MEM_BOOT_INFO, MEM_BOOT_INFO_LEN).unwrap().as_ptr().cast(); + identity_maps.push(IdentityMap::new(PAddr::new(boot_info.addr()), MEM_BOOT_INFO_LEN, IdentityMapFlags::READABLE)); - let fb_phys_base = fb_phys & !(PAGE_SIZE - 1); - let fb_phys_end = (fb_phys + fb_size).next_multiple_of(PAGE_SIZE); - let fb_page_count = (fb_phys_end - fb_phys_base) / PAGE_SIZE; + // + unsafe { + let scratch_ptr = boot::allocate_pages(AllocateType::MaxAddress(1 << 16), MEM_SCRATCH, MEM_SCRATCH_LEN).unwrap().as_ptr(); + scratch_ptr.write_bytes(0, MEM_SCRATCH_LEN * PAGE_SIZE); + let mut scratch_pages = ArrayVec::new(); + for i in 0..MEM_SCRATCH_LEN { + scratch_pages.push(PAddr::new(scratch_ptr.addr() + i * PAGE_SIZE)); + } - let graphics_info = GraphicsInfo { - w, - h, - stride, - pixel_format: match pixel_format { - PixelFormat::Rgb => elpytios_bootinfo::PixelFormat::RGB_8_BIT, - PixelFormat::Bgr => elpytios_bootinfo::PixelFormat::BGR_8_BIT, - PixelFormat::Bitmask => elpytios_bootinfo::PixelFormat::BIT_MASK, - PixelFormat::BltOnly => elpytios_bootinfo::PixelFormat::BLT_ONLY - }, - frame_buffer: next_free_page(PAddr::new(fb_phys), fb_page_count, VFlags::WRITABLE | VFlags::WRITE_THROUGH | VFlags::CACHE_DISABLED).ptr_mut(), - frame_buffer_size: frame_buffer_size, - }; + let device_tree = uefi::system::with_config_table(|slice| { + let mut out = None; + for i in slice { + match i.guid { + ConfigTableEntry::ACPI_GUID if out.is_none() => out = Some(DeviceTree::Acpi(PAddr::new(i.address.addr()))), + ConfigTableEntry::ACPI2_GUID => { + out = Some(DeviceTree::Acpi2(PAddr::new(i.address.addr()))); + break + }, + _ => {} + } + } - let (pml4_phys_ret, virtual_map) = virtual_map.finish().unwrap(); - pml4_phys = pml4_phys_ret; - unsafe { - boot_info_ptr = kernel_ptr.add(KERNEL_BOOTINFO_ADDRESS - virtual_base).cast(); - boot_info_ptr.write(BootInfo { + out.expect("No ACPI or ACPI2 table found") + }); + + boot_info.write(BootInfo { graphics_info, - virtual_map, - switcher_map: VAddr::new(switcher_addr), + device_tree, - // Initialized after exiting UEFI boot services - memory_regions_base: [MaybeUninit::uninit(); _], - memory_regions_size: 0, + kernel_elf_base: PAddr::new(kernel_ptr.addr()), + kernel_virt_base: virtual_base, - v_addr_start: VAddr::new(next_v_addr), - v_addr_end: VAddr::new(usize::MAX), + memory_regions: ArrayVec::new(), + identity_maps, + scratch_pages, + relocations, }); } } unsafe { - memory_map = boot::exit_boot_services(Some(MemoryType::LOADER_DATA)); + let memory_map = boot::exit_boot_services(Some(MemoryType::LOADER_DATA)); + for entry in memory_map.entries() { + if matches!(entry.ty, MemoryType::ACPI_RECLAIM) { + (*boot_info).identity_maps.push(IdentityMap { + region: MemoryRegion::at(PAddr::new(entry.phys_start as usize), entry.page_count as usize), + flags: IdentityMapFlags::READABLE, + }); + } + } - let mut len = 0; let mut region = None; - for entry in memory_map.entries() { if !matches!(entry.ty, MemoryType::LOADER_CODE | MemoryType::LOADER_DATA | @@ -292,75 +272,51 @@ fn setup_uefi_and_exit() -> UefiInfo { ) { continue } - + let start = entry.phys_start as usize; let count = entry.page_count as usize; match region.as_mut() { - None => region = Some(MemoryRegion { - base: PAddr::new(start), - pages: count, - }), + None => region = Some(MemoryRegion::at(PAddr::new(start), count)), Some(reg) => { if reg.base.addr() + reg.pages * PAGE_SIZE == start { reg.pages += count; - } else if reg.pages >= 8 { - (&raw mut (*boot_info_ptr).memory_regions_base[len]) - .cast::() - .write(mem::replace(reg, MemoryRegion { - base: PAddr::new(start), - pages: count, - })); - - len += 1; - if len == MAX_MEMORY_REGIONS { + } else { + if (*boot_info).memory_regions.try_push(mem::replace(reg, MemoryRegion::at(PAddr::new(start), count))).is_err() { break } - } else { - region = None; } } } } - if len < MAX_MEMORY_REGIONS && let Some(region) = region { - (&raw mut (*boot_info_ptr).memory_regions_base[len]).cast::().write(region); - len += 1; + if let Some(region) = region { + _ = (*boot_info).memory_regions.try_push(region); } - slice::from_raw_parts_mut(&raw mut (*boot_info_ptr).memory_regions_base as *mut MemoryRegion, len) - .sort_unstable_by(|a, b| b.pages.cmp(&a.pages)); - - (&raw mut (*boot_info_ptr).memory_regions_size).write(len); + (*boot_info).memory_regions.sort_unstable_by(|a, b| b.pages.cmp(&a.pages)); } UefiInfo { - memory_map, - pml4_phys, - kernel_stack_base, kernel_entry, + boot_info, } } -#[rustc_align(4096)] -#[unsafe(naked)] -unsafe extern "sysv64" fn switch_to_kernel( - pml4_phys: usize, - stack_base: usize, - kernel_entry: usize, -) -> ! { - naked_asm!( - "mov cr3, rdi", - "lea rsp, [rsi - 8]", - "jmp rdx", - ) -} - #[entry] fn entry() -> Status { - let UefiInfo { memory_map, pml4_phys, kernel_stack_base, kernel_entry } = setup_uefi_and_exit(); + let UefiInfo { kernel_entry, kernel_stack_base, boot_info } = setup_uefi_and_exit(); unsafe { - switch_to_kernel(pml4_phys.addr(), kernel_stack_base.addr(), kernel_entry.addr()) + asm!( + "mov rsp, {kernel_stack_base}", + "jmp {kernel_entry}", + + in("rdi") boot_info, + kernel_stack_base = in(reg) kernel_stack_base, + kernel_entry = in(reg) kernel_entry, + + options(noreturn) + ) } } diff --git a/bootloader/src/region_grouper.rs b/bootloader/src/region_grouper.rs deleted file mode 100644 index 26d62e5..0000000 --- a/bootloader/src/region_grouper.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub struct RegionGrouper { - -} - diff --git a/elf/src/lib.rs b/elf/src/lib.rs index 4c364d4..bdfdfc4 100644 --- a/elf/src/lib.rs +++ b/elf/src/lib.rs @@ -2,7 +2,7 @@ //! //! ``` //! pub fn load_kernel_code() -> Result<(), ElfError> { -//! let kernel_code = include_bytes!("../Cargo.toml"); +//! let kernel_code = include_bytes!("path/to/binary.elf"); //! //! match Elf::from_bytes(kernel_code)? { //! Elf::N32(..) => unreachable!("kernel ELF is 64-bits, silly"), @@ -11,7 +11,6 @@ //! let segment = segment?; //! //! segment.segment_type; // `ElfSegmentType`: null, load, dynamic, interp, and note -//! segment.data; // `&[u8]`, program segment data //! segment.flags; // `ElfProgramFlags`: 1 = executable, 2 = writable, 4 = readable //! segment.virtual_address; // `usize`, virtual address that `segment_data` should be copied into //! segment.physical_address; // `usize`, physical address that `segment_data` could be copied into, usually ignored @@ -28,7 +27,6 @@ #![no_std] #![feature( const_clone, - const_cmp, const_convert, const_destruct, const_index, @@ -49,7 +47,7 @@ use bytemuck::AnyBitPattern; use const_panic::PanicFmt; use sys::{ElfHeader64, ElfHeaderPrologue, ElfProgramFlags, ElfProgramHeader64}; -use crate::sys::ElfSectionHeader64; +use crate::sys::{ElfDt64, ElfDyn64, ElfSectionHeader64}; #[derive(Debug, Clone, Copy, PanicFmt)] pub enum ElfError { @@ -59,6 +57,7 @@ pub enum ElfError { InvalidSegmentType(u32), IntDoesntFit, MissingStringTable, + MalformedDynHeader(&'static str), Eof, } @@ -207,18 +206,33 @@ const impl<'a> Iterator for Elf64Programs<'a> { self.program_table_reader.take(self.stride as usize); Some(try { - let data = self - .file_reader - .fork(int_fit(program_header.segment_offset)?) - .ok_or(ElfError::Eof)? - .take(int_fit(program_header.segment_file_size)?) - .ok_or(ElfError::Eof)?; - + let mut data = self.file_reader.fork(int_fit(program_header.segment_offset)?).ok_or(ElfError::Eof)?; ElfSegment64 { segment_type: match program_header.segment_type { 0 => ElfSegmentType::Null, - 1 => ElfSegmentType::Load, - 2 => ElfSegmentType::Dynamic, + 1 => ElfSegmentType::Load(data.take(int_fit(program_header.segment_file_size)?).ok_or(ElfError::Eof)?), + 2 => { + let mut rela_offset = None; + let mut rela_size = None; + let mut rela_stride = None; + loop { + let dyn_entry = data.read::().ok_or(ElfError::Eof)?; + match dyn_entry.tag { + ElfDt64::NULL => break, + ElfDt64::RELA => rela_offset = Some(int_fit(dyn_entry.val)?), + ElfDt64::RELASZ => rela_size = Some(int_fit(dyn_entry.val)?), + ElfDt64::RELAENT => rela_stride = Some(int_fit(dyn_entry.val)?), + _ => {} + } + } + + // TODO `DT_RELA` isn't the only way to relocate things + let offset = rela_offset.ok_or(ElfError::MalformedDynHeader("`DT_RELA` not found"))?; + let size = rela_size.ok_or(ElfError::MalformedDynHeader("`DT_RELASZ` not found"))?; + let stride = rela_stride.ok_or(ElfError::MalformedDynHeader("`DT_RELAENT` not found"))?; + + ElfSegmentType::Dynamic { offset, size, stride } + } 3 => ElfSegmentType::Interp, 4 => ElfSegmentType::Note, 5 => ElfSegmentType::Shlib, @@ -226,7 +240,6 @@ const impl<'a> Iterator for Elf64Programs<'a> { 7 => ElfSegmentType::Tls, n => ElfSegmentType::Unknown(n), }, - data, flags: program_header.flags, virtual_address: program_header.segment_virtual_address, physical_address: program_header.segment_physical_address, @@ -252,10 +265,9 @@ impl ExactSizeIterator for Elf64Programs<'_> { impl FusedIterator for Elf64Programs<'_> {} -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] pub struct ElfSegment64<'a> { - pub segment_type: ElfSegmentType, - pub data: &'a [u8], + pub segment_type: ElfSegmentType<'a>, pub flags: ElfProgramFlags, /// [`Self::data`] should be copied to this v-address pub virtual_address: u64, @@ -267,15 +279,15 @@ pub struct ElfSegment64<'a> { pub alignment: u64, } -#[derive(Debug, Clone, Copy, Hash)] +#[derive(Debug, Clone)] #[repr(u32)] -pub enum ElfSegmentType { +pub enum ElfSegmentType<'a> { /// Ignore the entry Null = 0, /// Clear p_memsz bytes at p_vaddr to 0, then copy p_filesz bytes from p_offset to p_vaddr - Load = 1, + Load(&'a [u8]) = 1, /// Requires dynamic linking - Dynamic = 2, + Dynamic { offset: usize, size: usize, stride: usize } = 2, /// Contains a file path to an executable to use as an interpreter for the segment Interp = 3, /// Note section. There are more values, but mostly contain architecture/environment specific @@ -291,25 +303,6 @@ pub enum ElfSegmentType { Unknown(u32) = u32::MAX, } -const impl Eq for ElfSegmentType {} -const impl PartialEq for ElfSegmentType { - #[inline] - fn eq(&self, other: &Self) -> bool { - match (*self, *other) { - (Self::Null, Self::Null) - | (Self::Load, Self::Load) - | (Self::Dynamic, Self::Dynamic) - | (Self::Interp, Self::Interp) - | (Self::Note, Self::Note) - | (Self::Shlib, Self::Shlib) - | (Self::Header, Self::Header) - | (Self::Tls, Self::Tls) => true, - (Self::Unknown(l), Self::Unknown(r)) if l == r => true, - _ => false, - } - } -} - #[derive(Debug)] pub struct Elf64Sections<'a> { len: u16, diff --git a/elf/src/sys/elf_header.rs b/elf/src/sys/elf_header.rs index 83ba97a..0a26f06 100644 --- a/elf/src/sys/elf_header.rs +++ b/elf/src/sys/elf_header.rs @@ -1,7 +1,7 @@ -use bytemuck::{Pod, Zeroable}; +use bytemuck::AnyBitPattern; /// The ELF header is always found at the start of the file. -#[derive(Debug, Clone, Copy, Pod, Zeroable)] +#[derive(Debug, Clone, Copy, AnyBitPattern)] #[repr(C)] pub struct ElfHeaderPrologue { /// Magic number - 0x7F, then 'ELF' in ASCII 0-3 @@ -25,7 +25,7 @@ pub struct ElfHeaderPrologue { } /// Continuation of [`ElfHeaderPrologue`] in 64-bit format (arch == 2). -#[derive(Debug, Clone, Copy, Pod, Zeroable)] +#[derive(Debug, Clone, Copy, AnyBitPattern)] #[repr(C)] pub struct ElfHeader64 { /// Program entry offset 24-31 diff --git a/elf/src/sys/mod.rs b/elf/src/sys/mod.rs index ddef6bd..19c17c7 100644 --- a/elf/src/sys/mod.rs +++ b/elf/src/sys/mod.rs @@ -1,6 +1,8 @@ mod elf_header; mod program_header; +mod relocations; mod section_header; pub use elf_header::*; pub use program_header::*; +pub use relocations::*; pub use section_header::*; diff --git a/elf/src/sys/program_header.rs b/elf/src/sys/program_header.rs index 8e8d5c9..8ba4a81 100644 --- a/elf/src/sys/program_header.rs +++ b/elf/src/sys/program_header.rs @@ -1,7 +1,7 @@ use bitflags::bitflags; -use bytemuck::{Pod, Zeroable}; +use bytemuck::AnyBitPattern; -#[derive(Debug, Clone, Copy, Pod, Zeroable)] +#[derive(Debug, Clone, Copy, AnyBitPattern)] #[repr(C)] pub struct ElfProgramHeader64 { /// Type of segment (see below) 0-3 @@ -22,7 +22,7 @@ pub struct ElfProgramHeader64 { pub section_alignment: u64, } -#[derive(Debug, Clone, Copy, Pod, Zeroable)] +#[derive(Debug, Clone, Copy, AnyBitPattern)] #[repr(transparent)] pub struct ElfProgramFlags(u32); bitflags! { diff --git a/elf/src/sys/relocations.rs b/elf/src/sys/relocations.rs new file mode 100644 index 0000000..ebc69a3 --- /dev/null +++ b/elf/src/sys/relocations.rs @@ -0,0 +1,43 @@ +use bytemuck::AnyBitPattern; + +#[derive(Debug, Clone, Copy, AnyBitPattern)] +#[repr(C)] +pub struct ElfDyn64 { + pub tag: ElfDt64, + pub val: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, AnyBitPattern)] +#[repr(transparent)] +pub struct ElfDt64(pub i64); +impl ElfDt64 { + pub const NULL: Self = Self(0); + pub const RELA: Self = Self(7); + pub const RELASZ: Self = Self(8); + pub const RELAENT: Self = Self(9); +} + +#[derive(Debug, Clone, Copy, AnyBitPattern)] +#[repr(C)] +pub struct ElfRela64 { + pub offset: u64, + pub info: ElfRela64Info, + pub addend: i64, +} + +#[derive(Debug, Clone, Copy, AnyBitPattern)] +#[repr(C, align(8))] +pub struct ElfRela64Info { + #[cfg(target_endian = "little")] + pub kind: ElfRela64Type, + pub index: u32, + #[cfg(target_endian = "big")] + pub kind: ElfRela64Type, +} + +#[derive(Debug, Clone, Copy, AnyBitPattern, PartialEq, Eq)] +#[repr(transparent)] +pub struct ElfRela64Type(pub u32); +impl ElfRela64Type { + pub const X86_64_RELATIVE: Self = Self(8); +} diff --git a/elf/src/sys/section_header.rs b/elf/src/sys/section_header.rs index 6a2e0fa..c896ad0 100644 --- a/elf/src/sys/section_header.rs +++ b/elf/src/sys/section_header.rs @@ -1,7 +1,7 @@ use bitflags::bitflags; -use bytemuck::{Pod, Zeroable}; +use bytemuck::AnyBitPattern; -#[derive(Debug, Clone, Copy, Pod, Zeroable)] +#[derive(Debug, Clone, Copy, AnyBitPattern)] #[repr(C)] pub struct ElfSectionHeader64 { /// Offset into the section-name string table 0-3 @@ -26,7 +26,7 @@ pub struct ElfSectionHeader64 { pub entry_size: u64, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Pod, Zeroable)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, AnyBitPattern)] #[repr(transparent)] pub struct ElfSectionType(pub u32); impl ElfSectionType { @@ -44,7 +44,7 @@ impl ElfSectionType { pub const DYNSYM: Self = Self(11); } -#[derive(Debug, Clone, Copy, Pod, Zeroable)] +#[derive(Debug, Clone, Copy, AnyBitPattern)] #[repr(transparent)] pub struct ElfSectionFlags(u64); bitflags! { diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index abd61e9..3c7300a 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -11,3 +11,10 @@ forced-target = "x86_64-unknown-none" [dependencies] elpytios-abi = { path = "../abi" } elpytios-bootinfo = { path = "../bootinfo" } +elpytios-elf = { path = "../elf" } + +arrayvec = { version = "0.7", default-features = false } +bitflags = "2" +bytemuck = { version = "1", features = ["derive", "latest_stable_rust"] } +log = "*" +nonmax = { version = "0.5", default-features = false } diff --git a/kernel/src/allocator/heap.rs b/kernel/src/allocator/heap.rs new file mode 100644 index 0000000..17fde85 --- /dev/null +++ b/kernel/src/allocator/heap.rs @@ -0,0 +1,283 @@ +use core::{ + self, + alloc::{GlobalAlloc, Layout}, + hint::{cold_path, spin_loop, unreachable_unchecked}, + mem::Alignment, + ptr, + sync::atomic::{ + AtomicPtr, AtomicU16, + Ordering::{Acquire, Relaxed, Release}, + }, +}; + +use elpytios_bootinfo::PAGE_SIZE; + +use crate::{ + allocator::PHYS_ALLOC_ALIGNMENT, + statics::{get_phys_alloc, phys_to_virt}, +}; + +const MICRO_SIZES: [usize; 8] = [8, 16, 24, 32, 48, 64, 96, 128]; +const SMALL_SIZES: [usize; 8] = [192, 256, 384, 512, 768, 1024, 1536, 2048]; +const MEDIUM_SIZES: [usize; 4] = [3072, 4096, 6144, 8192]; +const LARGE_SIZES: [usize; 4] = [12288, 16384, 24576, 32768]; + +pub struct HeapAllocator { + micro_bins: [AtomicPtr; MICRO_SIZES.len()], + small_bins: [AtomicPtr; SMALL_SIZES.len()], + medium_bins: [AtomicPtr; MEDIUM_SIZES.len()], + large_bins: [AtomicPtr; LARGE_SIZES.len()], +} + +#[repr(C, align(4096))] +struct Segment { + data: [u8; N], + meta: SegmentMeta, +} + +impl Segment { + const LOCK: u16 = 1 << (u16::BITS - 1); + const MASK: u16 = !Self::LOCK; + + #[cold] + fn new(size_class: usize) -> *mut Self { + // Ensure that segment allocations are always aligned + _ = const { + assert!(size_of::().is_power_of_two()); + assert!(PHYS_ALLOC_ALIGNMENT.as_usize().is_multiple_of(size_of::())); + }; + + let addr = get_phys_alloc() + .lock() + .alloc(size_of::().ilog2()) + .expect("Couldn't allocate pages for heap allocator"); + let this = phys_to_virt(addr).ptr_mut::(); + debug_assert!( + this.is_aligned_to(size_of::()), + "Physical page allocator didn't allocate an aligned heap" + ); + + unsafe { + let offset = (this as usize % size_class) as u16; + let available = ((N - offset as usize) / size_class) as u16; + for i in 0..available { + (&raw mut (*this).data) + .cast::() + .byte_add(offset as usize + i as usize * size_class) + .cast::() + .write((i + 1) % available); + } + + (&raw mut (*this).meta).write(SegmentMeta { + next: ptr::null_mut(), + head_and_lock: AtomicU16::new(0), + available, + offset, + size_class: size_class as u16, + }); + } + + this + } + + unsafe fn alloc(self: *mut Self, new_head: impl FnOnce(*mut Self)) -> *mut u8 { + #[cold] + fn new_head_never(_: *mut T) { + unreachable!("Newly allocated page was immediately full, somehow") + } + + unsafe { + let mut curr_head = (*self).meta.head_and_lock.load(Relaxed) & Self::MASK; + loop { + // Short circuit without obtaining a lock if the segment is already full + // Pointer is only ever non-null if this segment is full + if !(*self).meta.next.is_null() { + return (*self).meta.next.cast::().alloc(new_head) + } + + match (*self) + .meta + .head_and_lock + .compare_exchange_weak(curr_head, curr_head | Self::LOCK, Acquire, Relaxed) + { + Ok(..) => { + if let Some(new_available) = (*self).meta.available.checked_sub(1) { + let data = (&raw mut (*self).data) + .cast::() + .byte_add((*self).meta.offset as usize + curr_head as usize * (*self).meta.size_class as usize); + + let next_head = data.cast::().read(); + (&raw mut (*self).meta.available).write(new_available); + (*self).meta.head_and_lock.store(next_head, Release); + + return data + } else { + cold_path(); + if (*self).meta.next.is_null() { + let new_segment = Self::new((*self).meta.size_class as usize); + let result = new_segment.alloc(new_head_never::); + new_head(new_segment); + + (&raw mut (*self).meta.next).write(new_segment.cast()); + (*self).meta.head_and_lock.store(curr_head, Release); + return result + } else { + (*self).meta.head_and_lock.store(curr_head, Release); + return (*self).meta.next.cast::().alloc(new_head) + } + } + } + Err(updated_head) => { + curr_head = updated_head & Self::MASK; + spin_loop(); + } + } + } + } + } +} + +#[repr(C)] +struct SegmentMeta { + next: *mut (), + head_and_lock: AtomicU16, + available: u16, + offset: u16, + size_class: u16, +} + +type MicroSegment = Segment<{ PAGE_SIZE - size_of::() }>; +type SmallSegment = Segment<{ (4 * PAGE_SIZE) - size_of::() }>; +type MediumSegment = Segment<{ (16 * PAGE_SIZE) - size_of::() }>; +type LargeSegment = Segment<{ (32 * PAGE_SIZE) - size_of::() }>; + +enum SizeClassify<'a> { + Micro(&'a AtomicPtr, usize), + Small(&'a AtomicPtr, usize), + Medium(&'a AtomicPtr, usize), + Large(&'a AtomicPtr, usize), + Huge, +} + +impl<'a> SizeClassify<'a> { + #[inline] + unsafe fn new(alloc: &'a HeapAllocator, size: usize) -> Self { + #[inline] + unsafe fn get<'a, const N: usize, T, R>( + ptrs: &'a [AtomicPtr], + size: usize, + segment_sizes: [usize; N], + f: impl FnOnce(&'a AtomicPtr, usize) -> R, + ) -> R { + unsafe { + let i = segment_sizes + .into_iter() + .enumerate() + .filter_map(|(i, segment_size)| Some((i, segment_size.checked_sub(size)?))) + .min_by_key(|&(.., segment_size)| segment_size) + .unwrap_unchecked() + .0; + + f(ptrs.get_unchecked(i), *segment_sizes.get_unchecked(i)) + } + } + + const MICRO_LARGEST: usize = MICRO_SIZES[MICRO_SIZES.len() - 1]; + const SMALL_LARGEST: usize = SMALL_SIZES[SMALL_SIZES.len() - 1]; + const MEDIUM_LARGEST: usize = MEDIUM_SIZES[MEDIUM_SIZES.len() - 1]; + const LARGE_LARGEST: usize = LARGE_SIZES[LARGE_SIZES.len() - 1]; + + unsafe { + match size { + 0 => unreachable_unchecked(), + ..=MICRO_LARGEST => get(&alloc.micro_bins, size, MICRO_SIZES, Self::Micro), + ..=SMALL_LARGEST => get(&alloc.small_bins, size, SMALL_SIZES, Self::Small), + ..=MEDIUM_LARGEST => get(&alloc.medium_bins, size, MEDIUM_SIZES, Self::Medium), + ..=LARGE_LARGEST => get(&alloc.large_bins, size, LARGE_SIZES, Self::Large), + _ => Self::Huge, + } + } + } +} + +impl HeapAllocator { + pub const fn new() -> Self { + Self { + micro_bins: [const { AtomicPtr::null() }; _], + small_bins: [const { AtomicPtr::null() }; _], + medium_bins: [const { AtomicPtr::null() }; _], + large_bins: [const { AtomicPtr::null() }; _], + } + } + + fn alloc(ptr: &AtomicPtr>, size_class: usize) -> *mut u8 { + let mut segment_ptr = ptr.load(Acquire); + if segment_ptr.is_null() { + cold_path(); + loop { + // 0th bit is used as allocation lock + match ptr.compare_exchange_weak(segment_ptr, segment_ptr.wrapping_byte_add(1), Acquire, Relaxed) { + Ok(..) => { + let new = Segment::new(size_class); + ptr.store(new, Release); + + segment_ptr = new; + break + } + Err(curr_ptr) => { + segment_ptr = curr_ptr; + break + } + } + } + } + + // If 0th bit is set, means the segment is still being allocated + if !segment_ptr.is_aligned() { + cold_path(); + loop { + spin_loop(); + segment_ptr = ptr.load(Acquire); + if segment_ptr.is_aligned() { + break + } + } + } + + unsafe { segment_ptr.alloc(|new_head| ptr.store(new_head, Release)) } + } + + #[inline] + const fn unionize(layout: Layout) -> Layout { + let new_size = layout.size().next_multiple_of(size_of::()); + let new_align = layout.alignment().max(Alignment::of::()); + unsafe { Layout::from_size_alignment_unchecked(new_size, new_align).pad_to_align() } + } +} + +unsafe impl Sync for HeapAllocator {} +unsafe impl GlobalAlloc for HeapAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let layout = Self::unionize(layout); + match unsafe { SizeClassify::new(self, layout.size()) } { + SizeClassify::Micro(ptr, size_class) => Self::alloc(ptr, size_class), + SizeClassify::Small(ptr, size_class) => Self::alloc(ptr, size_class), + SizeClassify::Medium(ptr, size_class) => { + cold_path(); + Self::alloc(ptr, size_class) + } + SizeClassify::Large(ptr, size_class) => { + cold_path(); + Self::alloc(ptr, size_class) + } + SizeClassify::Huge => { + cold_path(); + todo!() + } + } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + todo!("deallocation not yet implemented") + } +} diff --git a/kernel/src/allocator/mod.rs b/kernel/src/allocator/mod.rs new file mode 100644 index 0000000..703147e --- /dev/null +++ b/kernel/src/allocator/mod.rs @@ -0,0 +1,16 @@ +use core::mem::Alignment; + +use elpytios_bootinfo::PAGE_SIZE; + +mod heap; +mod phys; +mod slab; +mod tree; +pub use heap::*; +pub use phys::*; +pub use slab::*; +pub use tree::*; + +/// Physical allocators are aligned to 32 pages (128 KiB). +/// This means pointers of allocations up to 32 pages are guaranteed to be aligned. +pub const PHYS_ALLOC_ALIGNMENT: Alignment = unsafe { Alignment::new_unchecked(1 << (32 * PAGE_SIZE).ilog2()) }; diff --git a/kernel/src/allocator/phys.rs b/kernel/src/allocator/phys.rs new file mode 100644 index 0000000..190332a --- /dev/null +++ b/kernel/src/allocator/phys.rs @@ -0,0 +1,98 @@ +use core::{ + hint::cold_path, + sync::atomic::{AtomicBool, Ordering::Relaxed}, +}; + +use arrayvec::ArrayVec; +use elpytios_bootinfo::{PAGE_SIZE, paddr::PAddr}; + +use crate::allocator::{AllocTree, TreeAllocError}; + +#[derive(Debug)] +pub struct PhysicalPageAllocator { + trees: ArrayVec() }>, +} + +impl PhysicalPageAllocator { + /// # Safety + /// [`Self::sort_tree`] must be called before allocating. + #[inline] + pub unsafe fn new() -> PhysicalPageAllocator { + static CREATED: AtomicBool = AtomicBool::new(false); + + if CREATED.swap(true, Relaxed) { + panic!("Only one `PhysicalPageAllocator` instance may be created") + } + + Self { + trees: ArrayVec::new_const(), + } + } + + #[inline] + pub const fn tree_count(&self) -> usize { + self.trees.len() + } + + /// # Safety + /// The resulting tree's allocations must be aligned to + /// [`PHYS_ALLOC_ALIGNMENT`](super::PHYS_ALLOC_ALIGNMENT). + #[inline] + pub unsafe fn push_tree(&mut self, base: PAddr, tree: *mut AllocTree) { + self.trees.push(Entry { base, tree }); + } + + #[inline] + pub fn sort_tree(&mut self) { + self.trees.sort_unstable_by_key(|e| e.base); + } + + pub fn alloc(&mut self, order: u32) -> Result { + let mut last_error = TreeAllocError::InsufficientSpace { requested_order: order }; + for &Entry { base, tree } in &self.trees { + let tree = unsafe { tree.as_mut_unchecked() }; + match tree.alloc(order) { + Ok(index) => return Ok(base.byte_add(index as usize * PAGE_SIZE)), + Err(e @ TreeAllocError::InsufficientSpace { .. }) => last_error = e, + } + } + + Err(last_error) + } + + /// # Safety + /// - `addr` must have been obtained through [`Self::alloc`]. + /// - `order` must be the same value passed through the same [`Self::alloc`] invocation. + pub unsafe fn dealloc(&mut self, addr: PAddr, order: u32) { + let tree_index = match self.trees.binary_search_by_key(&addr, |e| e.base) { + Ok(i) => i, + Err(i) => match i.checked_sub(1) { + Some(i) => i, + None => { + cold_path(); + panic!("`PhysicalPageAllocator` has absolutely no trees"); + } + }, + }; + + unsafe { + let &Entry { base, tree } = self.trees.get_unchecked(tree_index); + let tree = tree.as_mut_unchecked(); + let index = u32::try_from((addr.addr() - base.addr()) / PAGE_SIZE).unwrap_unchecked(); + + // `dealloc` *may* be called for pages that didn't originally come with this allocator + // But in the case that they do, callers must ensure the safety invariants + if index < tree.node_count() { + tree.dealloc(index, order); + } + } + } +} + +unsafe impl Sync for PhysicalPageAllocator {} + +#[derive(Debug, Clone, Copy)] +struct Entry { + base: PAddr, + tree: *mut AllocTree, +} diff --git a/kernel/src/allocator/slab.rs b/kernel/src/allocator/slab.rs new file mode 100644 index 0000000..071c20b --- /dev/null +++ b/kernel/src/allocator/slab.rs @@ -0,0 +1,286 @@ +use core::{ + marker::PhantomData, + mem::{self, ManuallyDrop, MaybeUninit}, + num::NonZeroU16, + ops::{Deref, DerefMut}, + ptr::{self, NonNull}, +}; + +use elpytios_bootinfo::PAGE_SIZE; + +use crate::{ + spin_sync::SpinMutex, + statics::{get_phys_alloc, phys_to_virt, virt_to_phys}, + vaddr::VAddr, +}; + +#[repr(C)] +pub struct SlabAllocator { + slots: SpinMutex>; 2]>>, +} + +unsafe impl Sync for SlabAllocator {} +impl SlabAllocator { + #[inline] + pub const fn new() -> Self { + Self { slots: SpinMutex::new(None) } + } + + #[inline] + fn as_uninit(&self) -> &SlabAllocator> { + unsafe { mem::transmute(self) } + } + + #[inline] + pub fn alloc(&self, item: T) -> SlabId<'_, T> { + let mut slab = self.alloc_uninit(); + slab.write(item); + unsafe { SlabId::assume_init(slab) } + } + + pub fn alloc_uninit(&self) -> SlabId<'_, MaybeUninit> { + let mut slots = self.slots.lock(); + let [free, ..] = slots.get_or_insert_with(|| { + let slot = Slab::new(); + [slot, slot] + }); + + let ptr = loop { + match unsafe { free.as_mut() }.alloc() { + Ok(ptr) => break ptr, + Err(next_free) => *free = next_free, + } + }; + + SlabId { + alloc: self.as_uninit(), + slot: free.cast(), + ptr: ptr.cast(), + } + } + + unsafe fn dealloc(&self, mut slot: NonNull>, ptr: NonNull) { + let mut slots = self.slots.lock(); + unsafe { + slot.as_mut().dealloc(ptr); + slots.as_mut().unwrap_unchecked()[0] = slot; + } + } +} + +impl SlabAllocator> { + #[inline] + fn assume_init(&self) -> &SlabAllocator { + unsafe { mem::transmute(self) } + } +} + +impl Drop for SlabAllocator { + fn drop(&mut self) { + let Some([.., mut slot]) = self.slots.get_mut().take() else { return }; + loop { + unsafe { + let (entries, meta) = slot.as_ptr().fields(); + match *meta { + SlabMeta::Full { next_slot } => { + for i in 0..Slab::::LEN { + ManuallyDrop::drop(&mut (*entries.add(i)).taken) + } + + get_phys_alloc().lock().dealloc(virt_to_phys(VAddr::from(slot.as_ptr())), 0); + slot = next_slot; + } + SlabMeta::Available { mut free } => { + // Since `generic_const_exprs` is a million years away, pessimistically assume `Slot` is at least + // 2 bytes (which is the size of the union's `free` field) + let mut mask = [1u64; (PAGE_SIZE / size_of::()).div_ceil(u64::BITS as usize)]; + loop { + let block_index = free as usize / u64::BITS as usize; + let block_bit = 1 << (free as usize & (u64::BITS - 1) as usize); + mask[block_index] &= !block_bit; + + match (*entries.add(free as usize)).free { + Some(next_free) => free = next_free.get(), + None => break, + } + } + + for i in 0..Slab::::LEN { + let block_index = i / u64::BITS as usize; + let block_bit = 1 << (i & (u64::BITS - 1) as usize); + if mask[block_index] & block_bit != 0 { + ManuallyDrop::drop(&mut (*entries.add(i)).taken); + } + } + + get_phys_alloc().lock().dealloc(virt_to_phys(VAddr::from(slot.as_ptr())), 0); + break + } + } + } + } + } +} + +#[repr(C)] +pub struct SlabId<'a, T: 'a> { + alloc: &'a SlabAllocator, + slot: NonNull>, + ptr: NonNull, +} + +impl<'a, T> SlabId<'a, T> { + #[inline] + pub fn into_inner(this: Self) -> T { + let this = ManuallyDrop::new(this); + unsafe { + let out = this.ptr.read(); + this.alloc.dealloc(this.slot, this.ptr); + out + } + } + + #[inline] + pub fn leak(this: Self) -> &'a mut T { + let mut this = ManuallyDrop::new(this); + unsafe { this.ptr.as_mut() } + } +} + +impl<'a, T> SlabId<'a, MaybeUninit> { + #[inline] + pub unsafe fn assume_init(this: Self) -> SlabId<'a, T> { + let this = ManuallyDrop::new(this); + SlabId { + alloc: this.alloc.assume_init(), + slot: this.slot.cast(), + ptr: this.ptr.cast(), + } + } +} + +impl Drop for SlabId<'_, T> { + #[inline] + fn drop(&mut self) { + unsafe { + self.ptr.drop_in_place(); + self.alloc.dealloc(self.slot, self.ptr); + } + } +} + +impl Deref for SlabId<'_, T> { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + unsafe { self.ptr.as_ref() } + } +} + +impl DerefMut for SlabId<'_, T> { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { self.ptr.as_mut() } + } +} + +#[repr(C, align(4096))] +struct Slab { + data: [MaybeUninit; PAGE_SIZE], + _marker: PhantomData<[Slot]>, +} + +impl Slab { + const LEN: usize = { + let len = (PAGE_SIZE - size_of::>()) / size_of::>(); + assert!(len != 0, "`T` is too large!"); + len + }; + + fn new() -> NonNull { + let addr = get_phys_alloc().lock().alloc(0).expect("Couldn't allocate a page for slot allocator"); + unsafe { + let ptr = phys_to_virt(addr).ptr_mut::(); + let (entries, meta) = ptr.fields(); + meta.write(SlabMeta::Available { free: 0 }); + + for i in 0..Self::LEN { + entries.add(i).write(Slot { + free: NonZeroU16::new(((i + 1) % Self::LEN) as u16), + }); + } + + NonNull::new_unchecked(ptr) + } + } + + #[inline] + unsafe fn fields(self: *mut Self) -> (*mut Slot, *mut SlabMeta) { + unsafe { (self.cast(), self.byte_add(Self::LEN * size_of::>()).cast()) } + } + + fn alloc(&mut self) -> Result, NonNull> { + unsafe { + let (entries, meta) = (&raw mut *self).fields(); + let meta = meta.as_mut_unchecked(); + match meta { + SlabMeta::Full { next_slot } => Err(*next_slot), + SlabMeta::Available { free } => Ok({ + let out = entries.add(*free as usize); + match (*out).free { + Some(next_free) => *free = next_free.get(), + None => *meta = SlabMeta::Full { next_slot: Self::new() }, + } + + NonNull::new_unchecked(&raw mut (*out).taken as *mut T) + }), + } + } + } + + unsafe fn dealloc(&mut self, ptr: NonNull) { + unsafe { + let (entries, meta) = (&raw mut *self).fields(); + let meta = meta.as_mut_unchecked(); + let ptr = ptr.as_ptr().cast::>(); + + let index = ptr.offset_from_unsigned(entries) as u16; + match meta { + SlabMeta::Full { .. } => { + ptr.write(Slot { free: None }); + *meta = SlabMeta::Available { free: index }; + } + SlabMeta::Available { free } => { + // `free` must be the lowest index + let free_ptr = entries.add(*free as usize); + if *free < index { + // From: head:free --> [next_free] + // To : head:free --> index --> [next_free] + ptr.write(ptr::replace(free_ptr, Slot { + free: Some(NonZeroU16::new_unchecked(index)), + })); + } else { + // From: head:free --> [next_free] + // To : head:index --> free --> [next_free] + ptr.write(Slot { + free: Some(NonZeroU16::new_unchecked(*free)), + }); + *free = index; + } + } + } + } + } +} + +#[repr(C)] +enum SlabMeta { + Full { next_slot: NonNull> }, + Available { free: u16 }, +} + +union Slot { + taken: ManuallyDrop, + free: Option, +} diff --git a/kernel/src/allocator/tree.rs b/kernel/src/allocator/tree.rs new file mode 100644 index 0000000..8842b5b --- /dev/null +++ b/kernel/src/allocator/tree.rs @@ -0,0 +1,322 @@ +use core::{ + alloc::{Layout, LayoutError}, + fmt, + hint::assert_unchecked, + mem::MaybeUninit, + ptr, slice, +}; + +use nonmax::NonMaxU32; + +#[derive(Clone, Copy)] +pub enum TreeAllocError { + InsufficientSpace { requested_order: u32 }, +} + +impl fmt::Debug for TreeAllocError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl fmt::Display for TreeAllocError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InsufficientSpace { requested_order } => write!(f, "Tree can no longer contain allocation of size 2^{requested_order}"), + } + } +} + +/// A binary buddy tree, implemented with a split bitset and free lists. The tree operates on number +/// of "order," not leaf counts; i.e., the leaf count must be `2 ^ order`. +#[repr(C)] +pub struct AllocTree { + max_order: u32, + // The offsets here are relative to the offset of `data`... + nodes_offset: usize, + split_bitset_offset: usize, + data: AllocTreeData, +} + +struct AllocTreeFields<'a> { + max_order: u32, + free_lists: FreeLists<'a>, + list_nodes: &'a mut [ListNode], + split_bitset: &'a mut AllocBitset, +} + +impl AllocTree { + pub unsafe fn new(at: *mut (), layout: AllocTreeLayout) -> *mut Self { + let this = ptr::from_raw_parts_mut::(at, layout.size() - layout.free_nodes_offset_abs); + unsafe { + (&raw mut (*this).max_order).write(layout.max_order); + (&raw mut (*this).nodes_offset).write(layout.nodes_offset_abs - layout.free_nodes_offset_abs); + (&raw mut (*this).split_bitset_offset).write(layout.split_bitset_offset_abs - layout.free_nodes_offset_abs); + + let free_lists = (&raw mut (*this).data.0).as_mut_ptr().cast::(); + for i in 0..=layout.max_order as usize { + free_lists.add(i).write(ListHead { + head: if i == layout.max_order as usize { Some(NonMaxU32::ZERO) } else { None }, + }); + } + + let list_nodes = (&raw mut (*this).data.0) + .as_mut_ptr() + .add(layout.nodes_offset_abs - layout.free_nodes_offset_abs) + .cast::(); + for i in 0..1 << layout.max_order { + list_nodes.add(i).write(ListNode { prev: None, next: None }); + } + + (&raw mut (*this).data.0) + .as_mut_ptr() + .add(layout.split_bitset_offset_abs - layout.free_nodes_offset_abs) + .cast::() + .write_bytes(0, AllocBitset::size_for((1 << layout.max_order) - 1)); + } + + this + } + + #[inline] + fn fields(&mut self) -> AllocTreeFields<'_> { + let max_order = self.max_order; + let data = self.data.0.as_mut_ptr(); + unsafe { + AllocTreeFields { + max_order, + free_lists: FreeLists { + heads: slice::from_raw_parts_mut(data.cast(), max_order as usize + 1), + }, + list_nodes: slice::from_raw_parts_mut(data.add(self.nodes_offset).cast(), 1 << max_order), + split_bitset: ptr::from_raw_parts_mut::(data.add(self.split_bitset_offset), AllocBitset::size_for((1 << max_order) - 1)) + .as_mut_unchecked(), + } + } + } + + #[inline] + pub fn node_count(&self) -> u32 { + 1 << self.max_order + } + + pub fn alloc(&mut self, order: u32) -> Result { + let AllocTreeFields { + max_order, + free_lists, + list_nodes, + split_bitset, + } = self.fields(); + + if order > max_order { + return Err(TreeAllocError::InsufficientSpace { requested_order: order }) + } + + let mut current = None; + for i in order..=max_order { + if let Some(head) = free_lists.heads[i as usize].head.take() { + if let Some(new_head) = list_nodes[head.get() as usize].next.take() { + list_nodes[new_head.get() as usize].prev = None; + free_lists.heads[i as usize].head = Some(new_head); + } + + current = Some((head.get(), i)); + break + } + } + + let Some((index, mut current_order)) = current else { + return Err(TreeAllocError::InsufficientSpace { requested_order: order }) + }; + + if current_order < max_order { + unsafe { + split_bitset.get_and_toggle(Self::bit_index(index, current_order, max_order)); + } + } + + while current_order > order { + let next_order = current_order - 1; + + let free = NonMaxU32::new(index + (1 << next_order)).expect("Allocation index >= u32::MAX"); + list_nodes[free.get() as usize].prev = None; + + if let Some(prev_head) = free_lists.heads[next_order as usize].head.replace(free) { + list_nodes[prev_head.get() as usize].prev = Some(free); + list_nodes[free.get() as usize].next = Some(prev_head); + } else { + list_nodes[free.get() as usize].next = None; + } + + current_order = next_order; + unsafe { + // False: Either both buddies are occupied or both are allocated + // True: Exactly one buddy is occupied + split_bitset.get_and_toggle(Self::bit_index(index, current_order, max_order)); + } + } + + Ok(index) + } + + pub unsafe fn dealloc(&mut self, mut index: u32, mut order: u32) { + let AllocTreeFields { + max_order, + free_lists, + list_nodes, + split_bitset, + } = self.fields(); + + unsafe { + assert_unchecked(index.is_multiple_of(1 << order)); + assert_unchecked(index < 1 << max_order); + assert_unchecked(order <= max_order); + } + + while order < max_order { + // Was false, now true: Can't merge, exactly one buddy is still occupied + // Was true, now false: Can merge, no buddies are occupied + let can_merge = unsafe { split_bitset.get_and_toggle(Self::bit_index(index, order, max_order)) }; + if can_merge { + let buddy_index = index ^ (1 << order); + + // Remove the buddy from the free list + let buddy_node = &mut list_nodes[buddy_index as usize]; + match [buddy_node.prev.take(), buddy_node.next.take()] { + // `prev.is_none()` means this is the head in the free list` + [None, new_head] => { + free_lists.heads[order as usize].head = new_head; + if let Some(new_head) = new_head { + list_nodes[new_head.get() as usize].prev = None; + } + } + [Some(prev), next] => { + list_nodes[prev.get() as usize].next = next; + if let Some(next) = next { + list_nodes[next.get() as usize].prev = Some(prev); + } + } + } + + index &= !(1 << order); + order += 1; + } else { + break + } + } + + list_nodes[index as usize].prev = None; + if let Some(prev_head) = free_lists.heads[order as usize].head.replace(unsafe { NonMaxU32::new_unchecked(index) }) { + list_nodes[prev_head.get() as usize].prev = NonMaxU32::new(index); + list_nodes[index as usize].next = Some(prev_head); + } + } + + #[inline] + fn bit_index(index: u32, order: u32, max_order: u32) -> u32 { + debug_assert!(order < max_order, "`order` ({order}) must be less than `max_order` ({max_order})"); + + let layer_base = (1 << (max_order - order - 1)) - 1; + let pair_index = index >> (order + 1); + + layer_base + pair_index + } + + pub const fn layout(count: usize) -> Result { + let max_order = count.ilog2(); + let taken = 1 << max_order; + + // `order` + let layout = Layout::new::(); + // `nodes_offset` + let (layout, ..) = layout.extend(Layout::new::())?; + // `split_bitset_offset` + let (layout, ..) = layout.extend(Layout::new::())?; + + // `free_nodes` + let (layout, free_nodes_offset_abs) = layout.extend(Layout::array::((max_order + 1) as usize)?)?; + // `nodes`split_bitset_offset + let (layout, nodes_offset_abs) = layout.extend(Layout::array::(taken)?)?; + // `split_bitset` + let (layout, split_bitset_offset_abs) = layout.extend(Layout::array::(AllocBitset::size_for(taken - 1))?)?; + let layout = layout.pad_to_align(); + + Ok(AllocTreeLayout { + layout, + free_nodes_offset_abs, + nodes_offset_abs, + split_bitset_offset_abs, + max_order, + }) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct AllocTreeLayout { + layout: Layout, + // ..while the offsets here are absolute. + free_nodes_offset_abs: usize, + nodes_offset_abs: usize, + split_bitset_offset_abs: usize, + max_order: u32, +} + +impl AllocTreeLayout { + #[inline] + pub const fn node_count(&self) -> usize { + 1 << self.max_order + } + + #[inline] + pub const fn size(&self) -> usize { + self.layout.size() + } + + #[inline] + pub const fn align(&self) -> usize { + self.layout.align() + } +} + +// Safety notes: The repr and align must be the same as `ListHead`. +#[repr(C, align(4))] +struct AllocTreeData([MaybeUninit]); + +#[repr(transparent)] +struct FreeLists<'a> { + heads: &'a mut [ListHead], +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, align(4))] +struct ListHead { + head: Option, +} + +#[derive(Clone, Copy)] +#[repr(C)] +struct ListNode { + prev: Option, + next: Option, +} + +#[repr(transparent)] +struct AllocBitset([u64]); +impl AllocBitset { + #[inline] + const fn size_for(bits: usize) -> usize { + bits.div_ceil(u64::BITS as usize) + } + + #[inline] + unsafe fn get_and_toggle(&mut self, bit: u32) -> bool { + let block_index = bit / u64::BITS; + let block_bit = 1 << (bit & (u64::BITS - 1)); + + let block = unsafe { self.0.get_unchecked_mut(block_index as usize) }; + let old_block = *block; + + *block = old_block ^ block_bit; + old_block & block_bit != 0 + } +} diff --git a/kernel/src/arch/mod.rs b/kernel/src/arch/mod.rs new file mode 100644 index 0000000..3e4818f --- /dev/null +++ b/kernel/src/arch/mod.rs @@ -0,0 +1,8 @@ +cfg_select! { + target_arch = "x86_64" => { + pub mod x86_64; + } + _ => { + compile_error!("Unsupported architecture"); + } +} diff --git a/kernel/src/arch/x86_64.rs b/kernel/src/arch/x86_64.rs new file mode 100644 index 0000000..3507b8e --- /dev/null +++ b/kernel/src/arch/x86_64.rs @@ -0,0 +1,160 @@ +use core::{arch::asm, hint::spin_loop, time::Duration}; + +#[inline(always)] +pub unsafe fn outb(port: u16, value: u8) { + unsafe { + asm!( + "out dx, al", + in("dx") port, + in("al") value, + + options(nomem, nostack, preserves_flags) + ) + } +} + +#[inline(always)] +pub unsafe fn inb(port: u16) -> u8 { + unsafe { + let value: u8; + asm!( + "in al, dx", + + out("al") value, + in("dx") port, + + options(nomem, nostack, preserves_flags), + ); + value + } +} + +pub fn pit_delay(mut duration: Duration) { + const TICK_PER_SECOND: u64 = 1_193_182; + const TICK_MAX: u64 = 1 << u16::BITS; + const NANO_PER_SECOND: u64 = 1_000_000_000; + const WAIT_MAX: Duration = Duration::from_nanos(TICK_MAX * NANO_PER_SECOND / TICK_PER_SECOND); + + while duration > Duration::ZERO { + let wait = duration.min(WAIT_MAX); + duration -= wait; + + let ticks = (wait.subsec_nanos() as u64 * TICK_PER_SECOND + (NANO_PER_SECOND - 1)) / NANO_PER_SECOND; + let ticks = match ticks { + 0 => continue, + t @ 1..TICK_MAX => t as u16, + TICK_MAX => 0, + _ => unreachable!("Tick arithmetics should ensure max wait doesn't exceed 65536"), + }; + + unsafe { + let mut port_b = inb(0x61); + port_b &= 0xFC; + outb(0x61, port_b); + + outb(0x43, 0b10110000); + + outb(0x42, (ticks & 0xff) as u8); + outb(0x42, ((ticks >> 8) & 0xff) as u8); + + outb(0x61, port_b | 0x01); + loop { + if (inb(0x61) & 0x20) != 0 { + break + } + spin_loop(); + } + + outb(0x61, port_b); + } + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(u32)] +pub enum Msr { + /// - Read-write register. + /// - Bit 10: x2APIC enable (turns off MMIO, maps registers to MSR). + /// - Bit 11: APIC global enable (must be 1 for APIC functionality). + /// - Bit 12-51: [`PAddr::addr()`](elpytios_bootinfo::PAddr::addr) without bits 0..=11 (i.e. + /// must be page-aligned). + Ia32ApicBase = 0x001b, + /// - Read-write register. + /// - Bit 0-63: Pointer to CPU-local data. + Ia32FsBase = 0xc0000100, + /// - Read-write register. + /// - Bit 0-63: Pointer to CPU-local data. + Ia32GsBase = 0xc0000101, + /// - Read-write register. + /// - Bit 0: `syscall` and `sysret` enable. + /// - Bit 8: Long-mode enable. + Ia32Efer = 0xc000_0080, + /// - Read-write register. + /// - Bit 0-31: + /// - Bit 32-63: + Ia32Star = 0xc000_0081, + /// - Read-write register. + /// - Bit 0-63: `syscall` entry stub naked function pointer. + Ia32Lstar = 0xc000_0082, + /// - Read-write register. + /// - + Ia32Fmask = 0xc000_0084, + /// - Read-only register. + /// - Bit 0-31: Unique 32-bit physical hardware ID. + Ia32X2ApicId = 0x802, + /// - Read-only register. + /// - Bit 0-7: Version number. + /// - Bit 16-23: Max LVT entries. + Ia32X2ApicVersion = 0x803, + /// - Write-only register. + /// - Write a dummy 0 to clear in-service flag. + Ia32X2ApicEoi = 0x80b, + /// - Read-write register. + /// - Bit 0-7: Set fallback handler vector. + /// - Bit 8: APIC enable in software. + Ia32X2ApicSivr = 0x80f, + /// - Read-write register. + /// - Bit 0-7: Vector. + /// - Bit 8-10: Delivery mode (100=NMI, 101=Init, 110=Startup). + /// - Bit 14: Assert flag. + /// - Bit 32-63: Target core destination APIC ID (as specified in [`Self::X2ApicId`]). + Ia32X2ApicIcr = 0x830, + /// - Read-write register. + /// - Bit 0-7: Vector. + /// - Bit 17-18: Mode (00=One-shot, 01=Periodic). + Ia32X2ApicLvtTimer = 0x832, +} + +#[inline(always)] +pub unsafe fn rdmsr(address: Msr) -> u64 { + let low: u32; + let high: u32; + unsafe { + asm!( + "rdmsr", + + in("ecx") address as u32, + out("eax") low, + out("edx") high, + + options(nomem, nostack, preserves_flags) + ); + } + + ((high as u64) << 32) | (low as u64) +} + +#[inline(always)] +pub unsafe fn wrmsr(address: Msr, value: u64) { + unsafe { + asm!( + "wrmsr", + + in("ecx") address as u32, + in("eax") value as u32, + in("edx") (value >> 32) as u32, + + options(nomem, nostack, preserves_flags) + ); + } +} diff --git a/kernel/src/device/acpi/madt.rs b/kernel/src/device/acpi/madt.rs new file mode 100644 index 0000000..9438030 --- /dev/null +++ b/kernel/src/device/acpi/madt.rs @@ -0,0 +1,344 @@ +#![allow(unused, reason = "Not all ACPI features are used, but they are needed for parsing")] + +use core::{iter, ops::RangeInclusive}; + +use bitflags::bitflags; + +use crate::device::{ + SystemTable, + acpi::{PackedPtr, TypedSystemTable}, +}; + +#[derive(Clone, Copy)] +pub struct Madt<'root> { + pub local_interrupt_control_addr: u32, + pub flags: ApicFlags, + payload: PackedPtr<'root>, +} + +impl<'root> IntoIterator for Madt<'root> { + type IntoIter = impl Iterator + 'root; + type Item = ::Item; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + let mut payload = self.payload; + iter::from_fn(move || { + while payload.len() != 0 { + const RESERVED_FOR_OSPM_START: u8 = *RESERVED_FOR_OSPM.start(); + const RESERVED_FOR_OSPM_END: u8 = *RESERVED_FOR_OSPM.end(); + const RESERVED_FOR_OEM_START: u8 = *RESERVED_FOR_OEM.start(); + const RESERVED_FOR_OEM_END: u8 = *RESERVED_FOR_OEM.end(); + + unsafe { + let kind: u8 = payload.read(); + let len: u8 = payload.read(); + + return Some(match kind { + PROCESSOR_LOCAL => Pic::ProcessorLocal(payload.read()), + IO => Pic::Io(payload.read()), + INTERRUPT_SOURCE_OVERRIDE => Pic::InterruptSourceOverride(payload.read()), + NMI_SOURCE => Pic::NmiSource(payload.read()), + LOCAL_NMI => Pic::LocalNmi(payload.read()), + LOCAL_ADDR_OVERRIDE => Pic::LocalAddrOverride(payload.read()), + IO_STREAMLINED => Pic::IoStreamlined(payload.read()), + PROCESSOR_LOCAL_STREAMLINED => { + let data = payload.read(); + let _acpi_processor_uid_string = payload.slice(len as usize - 2 - size_of::()); + Pic::ProcessorLocalStreamlined(data) + } + PLATFORM_INTERRUPT_SOURCES => Pic::PlatformInterruptSources(payload.read()), + PROCESSOR_LOCAL_X2 => Pic::ProcessLocalX2(payload.read()), + LOCAL_NMI_X2 => Pic::LocalNmiX2(payload.read()), + GIC_CPU_INTERFACE => Pic::GicCpuInterface(payload.read()), + GIC_DISTRIBUTOR => Pic::GicDistributor(payload.read()), + GIC_MSI_FRAME => Pic::GicMsiFrame(payload.read()), + GIC_REDISTRIBUTOR => Pic::GicRedistributor(payload.read()), + GIC_INTERRUPT_TRANSLATION_SERVICE => Pic::GicInterruptTranslationService(payload.read()), + RESERVED_FOR_OSPM_START..=RESERVED_FOR_OSPM_END | RESERVED_FOR_OEM_START..=RESERVED_FOR_OEM_END => continue, + }) + } + } + None + }) + } +} + +unsafe impl TypedSystemTable for Madt<'_> { + const SIGNATURE: [u8; 4] = *b"APIC"; + type Out<'root> = Madt<'root>; + + #[inline] + unsafe fn from_table<'root>(table: &SystemTable<'root>) -> Madt<'root> { + let mut payload = PackedPtr::new(table.entries); + unsafe { + Madt { + local_interrupt_control_addr: payload.read(), + flags: payload.read(), + payload, + } + } + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct ApicFlags(u32); +bitflags! { + impl ApicFlags: u32 { + const PCAT_COMPAT = 1 << 0; + } +} + +pub const PROCESSOR_LOCAL: u8 = 0x00; +pub const IO: u8 = 0x01; +pub const INTERRUPT_SOURCE_OVERRIDE: u8 = 0x02; +pub const NMI_SOURCE: u8 = 0x03; +pub const LOCAL_NMI: u8 = 0x04; +pub const LOCAL_ADDR_OVERRIDE: u8 = 0x05; +pub const IO_STREAMLINED: u8 = 0x06; +pub const PROCESSOR_LOCAL_STREAMLINED: u8 = 0x07; +pub const PLATFORM_INTERRUPT_SOURCES: u8 = 0x08; +pub const PROCESSOR_LOCAL_X2: u8 = 0x09; +pub const LOCAL_NMI_X2: u8 = 0x0a; +pub const GIC_CPU_INTERFACE: u8 = 0x0b; +pub const GIC_DISTRIBUTOR: u8 = 0x0c; +pub const GIC_MSI_FRAME: u8 = 0x0d; +pub const GIC_REDISTRIBUTOR: u8 = 0x0e; +pub const GIC_INTERRUPT_TRANSLATION_SERVICE: u8 = 0x0f; +pub const RESERVED_FOR_OSPM: RangeInclusive = 0x10..=0x7f; +pub const RESERVED_FOR_OEM: RangeInclusive = 0x80..=0xff; + +#[derive(Debug, Clone, Copy)] +#[repr(u8)] +pub enum Pic { + ProcessorLocal(ProcessorLocal) = PROCESSOR_LOCAL, + Io(Io) = IO, + InterruptSourceOverride(InterruptSourceOverride) = INTERRUPT_SOURCE_OVERRIDE, + NmiSource(NmiSource) = NMI_SOURCE, + LocalNmi(LocalNmi) = LOCAL_NMI, + LocalAddrOverride(LocalAddrOverride) = LOCAL_ADDR_OVERRIDE, + IoStreamlined(IoStreamlined) = IO_STREAMLINED, + ProcessorLocalStreamlined(ProcessorLocalStreamlined) = PROCESSOR_LOCAL_STREAMLINED, + PlatformInterruptSources(PlatformInterruptSources) = PLATFORM_INTERRUPT_SOURCES, + ProcessLocalX2(ProcessLocalX2) = PROCESSOR_LOCAL_X2, + LocalNmiX2(LocalNmiX2) = LOCAL_NMI_X2, + GicCpuInterface(GicCpuInterface) = GIC_CPU_INTERFACE, + GicDistributor(GicDistributor) = GIC_DISTRIBUTOR, + GicMsiFrame(GicMsiFrame) = GIC_MSI_FRAME, + GicRedistributor(GicRedistributor) = GIC_REDISTRIBUTOR, + GicInterruptTranslationService(GicInterruptTranslationService) = GIC_INTERRUPT_TRANSLATION_SERVICE, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct ProcessorLocal { + pub acpi_processor_uid: u8, + pub apic_id: u8, + pub flags: LocalApicFlags, +} + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct LocalApicFlags(u32); +bitflags! { + impl LocalApicFlags: u32 { + const ENABLED = 1 << 0; + const ONLINE_CAPABLE = 1 << 1; + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct Io { + pub io_apic_id: u8, + pub reserved: [u8; 1], + pub io_apic_addr: u32, + pub global_system_interrupt_base: u32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct InterruptSourceOverride { + pub bus: u8, + pub source: u8, + pub global_system_interrupt: u32, + pub flags: MpsIntiFlags, +} + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct MpsIntiFlags(u16); +bitflags! { + impl MpsIntiFlags: u16 { + const ACTIVE_HIGH = 0b01; + const ACTIVE_LOW = 0b11; + const POLARITY = 0b11; + + const EDGE_TRIGGERED = 0b01 << 2; + const LEVEL_TRIGGERED = 0b11 << 2; + const TRIGGER_MODE = 0b11 << 2; + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct NmiSource { + pub flags: MpsIntiFlags, + pub global_system_interrupt: u32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct LocalNmi { + pub acpi_processor_uid: u8, + pub flags: MpsIntiFlags, + pub local_apic_lint_num: u8, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct LocalAddrOverride { + pub reserved: [u8; 2], + pub local_apic_addr: u64, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct IoStreamlined { + pub io_apic_id: u8, + pub reserved: [u8; 1], + pub global_system_interrupt_base: u32, + pub io_sapic_addr: u64, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct ProcessorLocalStreamlined { + pub acpi_processor_id: u8, + pub local_sapic_id: u8, + pub local_sapic_eid: u8, + pub reserved: [u8; 3], + pub flags: LocalApicFlags, + pub acpi_processor_uid_value: u32, + // Note: `acpi_processor_uid_string: [u8; leftover]` +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct PlatformInterruptSources { + pub flags: MpsIntiFlags, + pub interrupt_type: u8, + pub processor_id: u8, + pub processor_eid: u8, + pub io_sapic_vector: u8, + pub global_system_interrupt: u32, + pub platform_interrupt_source_flags: PlatformInterruptSourceFlags, +} + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct PlatformInterruptSourceFlags(u32); +bitflags! { + impl PlatformInterruptSourceFlags: u32 { + const CPEI_PROCESSOR_OVERRIDE = 1 << 0; + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct ProcessLocalX2 { + pub reserved: [u8; 2], + pub x2apic_id: u32, + pub flags: LocalApicFlags, + pub acpi_processor_uid: u32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct LocalNmiX2 { + pub flags: MpsIntiFlags, + pub acpi_processor_uid: u32, + pub local_x2apic_lint_num: u8, + pub reserved: [u8; 3], +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct GicCpuInterface { + pub reserved: [u8; 2], + pub cpu_interface_num: u32, + pub acpi_processor_uid: u32, + pub flags: GicCpuInterfaceFlags, + pub parking_protocol_version: u32, + pub performance_interrupt_gsiv: u32, + pub parked_addr: u64, + pub physical_base_addr: u64, + pub gicv: u64, + pub gich: u64, + pub vgic_maintenance_interrupt: u32, + pub gicr_base_addr: u64, + pub mpidr: u64, + pub processor_power_efficiency_class: u8, + pub reserved2: [u8; 1], + pub spe_overflow_interrupt: u16, +} + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct GicCpuInterfaceFlags(u32); +bitflags! { + impl GicCpuInterfaceFlags: u32 { + const ENABLED = 1 << 0; + const PERFORMANCE_INTERRUPT_MODE = 1 << 1; + const VGIC_MAINTENANCE_INTERRUPT_MODE = 1 << 2; + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct GicDistributor { + pub reserved: [u8; 2], + pub gic_id: u32, + pub physical_base_addr: u64, + pub system_vector_base: u32, + pub gic_version: u8, + pub reserved2: [u8; 3], +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct GicMsiFrame { + pub reserved: [u8; 2], + pub gic_msi_frame_id: u32, + pub physical_base_addr: u64, + pub flags: GicMsiFrameFlags, + pub spi_count: u16, + pub spi_base: u16, +} + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct GicMsiFrameFlags(u32); +bitflags! { + impl GicMsiFrameFlags: u32 { + const SPI_COUNT_BASE_SELECT = 1 << 0; + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct GicRedistributor { + pub reserved: [u8; 2], + pub discovery_range_base_addr: u64, + pub discovery_range_len: u32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct GicInterruptTranslationService { + pub reserved: [u8; 2], + pub gic_its_id: u32, + pub physical_base_addr: u64, + pub reserved2: [u8; 4], +} diff --git a/kernel/src/device/acpi/mod.rs b/kernel/src/device/acpi/mod.rs new file mode 100644 index 0000000..2eca441 --- /dev/null +++ b/kernel/src/device/acpi/mod.rs @@ -0,0 +1,308 @@ +use core::{any::type_name, fmt, iter::FusedIterator, marker::PhantomData, mem::offset_of, num::NonZeroU8, ptr, slice}; + +mod madt; +mod root; +pub use madt::*; +pub use root::*; + +pub const ROOT_SIGNATURE: [u8; 8] = *b"RSD PTR "; + +pub type AcpiResult = Result; + +#[derive(Clone, Copy)] +pub struct SignatureFmt<'a>(pub &'a [u8]); +impl fmt::Debug for SignatureFmt<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl fmt::Display for SignatureFmt<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for chunk in self.0.utf8_chunks() { + for ch in chunk.valid().chars() { + write!(f, "{}", ch.escape_debug())?; + } + for byte in chunk.invalid() { + write!(f, "\\x{:02x}", byte)?; + } + } + Ok(()) + } +} + +#[derive(Clone, Copy)] +pub enum AcpiError { + InvalidRootSignature { found: [u8; 8] }, + InvalidTableSignature { expected: [u8; 4], found: [u8; 4] }, + InvalidChecksum(NonZeroU8), +} + +impl fmt::Debug for AcpiError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl fmt::Display for AcpiError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidRootSignature { found } => write!(f, "Expected 'RSD PTR ', found '{}'", SignatureFmt(found)), + Self::InvalidTableSignature { expected, found } => write!(f, "Expected '{}', found '{}'", SignatureFmt(expected), SignatureFmt(found)), + Self::InvalidChecksum(checksum) => write!(f, "Invalid checksum, found {checksum}"), + } + } +} + +trait AcpiParse: Sized { + type Input; + + unsafe fn parse(input: *const Self::Input) -> AcpiResult; +} + +#[repr(C, packed)] +pub struct Rsdp { + signature: [u8; 8], + checksum: u8, + oem_id: [u8; 6], + revision: u8, + rsdt_addr: u32, // Note: Explicitly use `u32` because it's a 32-bit address regardless of architecture +} + +impl Rsdp { + /// # Safety + /// - The pointer must be valid for unaligned reads of `Rsdp`. + /// - The pointer must point to a firmware-provided data to ensure validations. + /// - The resulting output must be dropped before identity-mapping is disabled. + #[inline] + pub unsafe fn new(this: *const Self) -> AcpiResult { + unsafe { Self::parse(this) } + } + + #[inline] + pub fn rsdt(&self) -> AcpiResult> { + unsafe { SystemTable::parse(self.rsdt_addr as usize as *const SystemTableHeader).and_then(|table| table.typed()) } + } +} + +impl AcpiParse for Rsdp { + type Input = Self; + + unsafe fn parse(input: *const Self::Input) -> AcpiResult { + unsafe { + ((*input).signature == ROOT_SIGNATURE).ok_or(AcpiError::InvalidRootSignature { found: (*input).signature })?; + let mut checksum = 0u8; + for &byte in slice::from_raw_parts(&raw const *input as *const u8, size_of::()) { + checksum = checksum.wrapping_add(byte); + } + + match NonZeroU8::new(checksum) { + None => Ok(input.read_unaligned()), + Some(invalid) => Err(AcpiError::InvalidChecksum(invalid)), + } + } + } +} + +#[repr(C, packed)] +pub struct Xsdp { + rsdp: Rsdp, + len: u32, + xsdt_addr: u64, // Note: Explicitly use `u64` because it's a 64-bit address regardless of architecture + ext_checksum: u8, + _reserved: [u8; 3], +} + +impl Xsdp { + /// # Safety + /// - The pointer must be valid for unaligned reads of `Xsdp`. + /// - The pointer must point to a firmware-provided data to ensure validations. + /// - The resulting output must be dropped before identity-mapping is disabled. + #[inline] + pub unsafe fn new(this: *const Self) -> AcpiResult { + unsafe { Self::parse(this) } + } + + #[inline] + pub fn xsdt(&self) -> AcpiResult> { + unsafe { SystemTable::parse(self.xsdt_addr as usize as *const SystemTableHeader).and_then(|table| table.typed()) } + } +} + +impl AcpiParse for Xsdp { + type Input = Self; + + unsafe fn parse(input: *const Self::Input) -> AcpiResult { + unsafe { + Rsdp::parse(&raw const (*input).rsdp)?; + + let mut checksum = 0u8; + for &byte in slice::from_raw_parts(&raw const (*input).len as *const u8, size_of::() - offset_of!(Self, len)) { + checksum = checksum.wrapping_add(byte); + } + + match NonZeroU8::new(checksum) { + None => Ok(input.read_unaligned()), + Some(invalid) => Err(AcpiError::InvalidChecksum(invalid)), + } + } + } +} + +#[repr(C, packed)] +pub struct SystemTableHeader { + signature: [u8; 4], + length: u32, + revision: u8, + checksum: u8, + oem_id: [u8; 6], + oem_table_id: u64, + oem_revision: u32, + creator_id: u32, + creator_revision: u32, + system_table: (), +} + +#[derive(Clone, Copy)] +pub struct SystemTable<'root> { + signature: [u8; 4], + entries: *const [u8], + _marker: PhantomData<&'root ()>, +} + +impl<'root> SystemTable<'root> { + #[inline] + pub fn typed = T>>(&self) -> AcpiResult { + (self.signature == T::SIGNATURE) + .then(|| unsafe { T::from_table(self) }) + .ok_or(AcpiError::InvalidTableSignature { + expected: T::SIGNATURE, + found: self.signature, + }) + } +} + +impl fmt::Debug for SystemTable<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct(type_name::()) + .field("signature", &SignatureFmt(&self.signature)) + .field("byte_len", &self.entries.len()) + .finish() + } +} + +impl AcpiParse for SystemTable<'_> { + type Input = SystemTableHeader; + + unsafe fn parse(input: *const Self::Input) -> AcpiResult { + unsafe { + let entries = slice::from_raw_parts( + &raw const (*input).system_table as *const u8, + (*input).length as usize - size_of::(), + ); + + let mut checksum = 0u8; + for &byte in slice::from_raw_parts(&raw const *input as *const u8, size_of::()) { + checksum = checksum.wrapping_add(byte); + } + + for &byte in entries { + checksum = checksum.wrapping_add(byte); + } + + match NonZeroU8::new(checksum) { + None => Ok(Self { + signature: (*input).signature, + entries, + _marker: PhantomData, + }), + Some(invalid) => Err(AcpiError::InvalidChecksum(invalid)), + } + } + } +} + +pub unsafe trait TypedSystemTable: Sized { + const SIGNATURE: [u8; 4]; + type Out<'root>: TypedSystemTable + 'root; + + unsafe fn from_table<'root>(table: &SystemTable<'root>) -> Self::Out<'root>; +} + +#[derive(Clone, Copy)] +struct PackedPtr<'root> { + ptr: *const [u8], + _marker: PhantomData<&'root ()>, +} + +impl<'root> PackedPtr<'root> { + #[inline] + fn new(ptr: *const [u8]) -> Self { + Self { ptr, _marker: PhantomData } + } + + #[inline] + fn len(self) -> usize { + self.ptr.len() + } + + #[inline] + unsafe fn slice(&mut self, len: usize) -> &'root [u8] { + unsafe { + let new_len = self.ptr.len().checked_sub(len).expect("Not enough bytes"); + let first = self.ptr.as_ptr(); + let result = slice::from_raw_parts(first, len); + + self.ptr = ptr::slice_from_raw_parts(first.add(len), new_len); + result + } + } + + #[inline] + unsafe fn read(&mut self) -> T { + unsafe { + let new_len = self.ptr.len().checked_sub(size_of::()).expect("Not enough bytes"); + let first = self.ptr.cast::(); + let result = first.read_unaligned(); + + self.ptr = ptr::slice_from_raw_parts(first.add(1).cast(), new_len); + result + } + } +} + +struct UnalignedPtrIter<'root, T: 'root> { + entries: PackedPtr<'root>, + _marker: PhantomData<&'root [T]>, +} + +impl<'root, T: 'root> UnalignedPtrIter<'root, T> { + fn new(entries: PackedPtr<'root>) -> Self { + Self { + entries, + _marker: PhantomData, + } + } +} + +impl<'root, T: 'root> Iterator for UnalignedPtrIter<'root, T> { + type Item = T; + + fn next(&mut self) -> Option { + (self.entries.len() != 0).then(|| unsafe { self.entries.read() }) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + (self.entries.len() / size_of::(), Some(self.entries.len() / size_of::())) + } +} + +impl<'root, T: 'root> ExactSizeIterator for UnalignedPtrIter<'root, T> { + #[inline] + fn len(&self) -> usize { + self.entries.len() / size_of::() + } +} + +impl<'root, T: 'root> FusedIterator for UnalignedPtrIter<'root, T> {} diff --git a/kernel/src/device/acpi/root.rs b/kernel/src/device/acpi/root.rs new file mode 100644 index 0000000..bfde65b --- /dev/null +++ b/kernel/src/device/acpi/root.rs @@ -0,0 +1,76 @@ +use core::{any::type_name, fmt, iter::FusedIterator}; + +use crate::device::{ + AcpiResult, SystemTable, SystemTableHeader, + acpi::{AcpiParse, PackedPtr, TypedSystemTable, UnalignedPtrIter}, +}; + +#[derive(Clone, Copy)] +pub struct Rsdt<'root> { + entries: PackedPtr<'root>, +} + +impl fmt::Debug for Rsdt<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct(type_name::()) + .field_with("entries", |f| f.debug_list().entries(self.into_iter()).finish()) + .finish() + } +} + +impl<'root> IntoIterator for Rsdt<'root> { + type IntoIter = impl Iterator>> + ExactSizeIterator + FusedIterator + 'root; + type Item = ::Item; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + UnalignedPtrIter::::new(self.entries).map(|addr| unsafe { SystemTable::parse(addr as usize as *const SystemTableHeader) }) + } +} + +unsafe impl TypedSystemTable for Rsdt<'_> { + const SIGNATURE: [u8; 4] = *b"RSDT"; + type Out<'root> = Rsdt<'root>; + + #[inline] + unsafe fn from_table<'root>(table: &SystemTable<'root>) -> Rsdt<'root> { + Rsdt { + entries: PackedPtr::new(table.entries), + } + } +} + +#[derive(Clone, Copy)] +pub struct Xsdt<'root> { + entries: PackedPtr<'root>, +} + +impl fmt::Debug for Xsdt<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct(type_name::()) + .field_with("entries", |f| f.debug_list().entries(self.into_iter()).finish()) + .finish() + } +} + +impl<'root> IntoIterator for Xsdt<'root> { + type IntoIter = impl Iterator>> + ExactSizeIterator + FusedIterator + 'root; + type Item = ::Item; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + UnalignedPtrIter::::new(self.entries).map(|addr| unsafe { SystemTable::parse(addr as usize as *const SystemTableHeader) }) + } +} + +unsafe impl TypedSystemTable for Xsdt<'_> { + const SIGNATURE: [u8; 4] = *b"XSDT"; + type Out<'root> = Xsdt<'root>; + + #[inline] + unsafe fn from_table<'root>(table: &SystemTable<'root>) -> Xsdt<'root> { + Xsdt { + entries: PackedPtr::new(table.entries), + } + } +} diff --git a/kernel/src/device/imp/trampolines/x86_64.s b/kernel/src/device/imp/trampolines/x86_64.s new file mode 100644 index 0000000..c41dd1a --- /dev/null +++ b/kernel/src/device/imp/trampolines/x86_64.s @@ -0,0 +1,144 @@ +.section .ap_trampoline, "a" + +.global __ap_trampoline_start +.global __ap_cr3 +.global __ap_cr4 +.global __ap_stack +.global __ap_kernel_entry +.global __ap_kernel_arg0 +.global __ap_kernel_arg1 +.global __ap_kernel_arg2 +.global __ap_trampoline_end + +__ap_trampoline_start: +.code16 +ap_entry_16: + # Clear interrupts and zero out segments + cli + xor ax, ax + mov ds, ax + mov es, ax + mov ss, ax + + # Get the allocated address of `__ap_trampoline_start` from `cs` + xor bx, bx + mov bx, cs + shl bx, 4 + + # Set `.gdt_desc + 2` to `.gdt_start` relative to `bx` + mov si, bx + add si, [bx + GDT_START_LOOKUP] + mov di, [bx + GDT_DESC_LOOKUP] + mov [bx + di + 2], si + lgdt [bx + di] + + # Set `.jmp_32 + 2` to `ap_entry_32` relative to `bx` + mov si, bx + add si, [bx + AP_ENTRY_32_LOOKUP] + mov di, [bx + JMP_32_LOOKUP] + mov [bx + di + 2], si + + # Enable protected mode + mov ecx, cr0 + or ecx, 0x00000001 + mov cr0, ecx + + # Raw bytes for a far jump; necessary since the address is dynamically written +.jmp_32: + .byte 0x66, 0xea + .long 0x00000000 + .word 0x0008 +.align 4 +.gdt_start: + .quad 0x0000000000000000 # Null + .quad 0x00cf9a000000ffff # Kernel code 32-bit + .quad 0x00cf92000000ffff # Kernel data + .quad 0x00af9a000000ffff # Kernel code 64-bit +.gdt_end: +.gdt_desc: + .word .gdt_end - .gdt_start - 1 + .long 0x00000000 + + .gdt_start_lookup: .word .gdt_start - __ap_trampoline_start + .gdt_desc_lookup: .word .gdt_desc - __ap_trampoline_start + .jmp_32_lookup: .word .jmp_32 - __ap_trampoline_start + .ap_entry_32_lookup: .word ap_entry_32 - __ap_trampoline_start + .equ GDT_START_LOOKUP, .gdt_start_lookup - __ap_trampoline_start + .equ GDT_DESC_LOOKUP, .gdt_desc_lookup - __ap_trampoline_start + .equ JMP_32_LOOKUP, .jmp_32_lookup - __ap_trampoline_start + .equ AP_ENTRY_32_LOOKUP, .ap_entry_32_lookup - __ap_trampoline_start + +.code32 +ap_entry_32: + # Reinitialize segments and zero-extend base address + mov ax, 0x10 + mov ds, ax + mov es, ax + mov ss, ax + movzx ebx, bx + + # Zero out unused segments + xor ax, ax + mov fs, ax + mov gs, ax + + # Set physical page table + mov eax, [ebx + AP_CR3] + mov cr3, eax + + # Copy CR4 (which importantly includes Page Address Extension) + mov eax, [ebx + AP_CR4] + mov cr4, eax + + # Enable 64-bit long mode + mov ecx, 0xc0000080 # `IA32_EFER` + rdmsr + or eax, 1 << 8 + or eax, 1 << 11 + wrmsr + + # Set `.jmp_64 + 2` to `ap_entry_64` relative to `ebx` + mov esi, ebx + add esi, [ebx + AP_ENTRY_64_LOOKUP] + mov edi, [ebx + JMP_64_LOOKUP] + mov [ebx + edi + 1], esi + + # Enable virtual paging + mov eax, cr0 + or eax, 1 << 31 + mov cr0, eax + + # Raw bytes for a far jump; necessary since the address is dynamically written +.jmp_64: + .byte 0xea + .long 0x00000000 + .word 0x0018 + + __ap_cr3: .long 0x00000000 + __ap_cr4: .long 0x00000000 + .jmp_64_lookup: .long .jmp_64 - __ap_trampoline_start + .ap_entry_64_lookup: .long ap_entry_64 - __ap_trampoline_start + .equ AP_CR3, __ap_cr3 - __ap_trampoline_start + .equ AP_CR4, __ap_cr4 - __ap_trampoline_start + .equ JMP_64_LOOKUP, .jmp_64_lookup - __ap_trampoline_start + .equ AP_ENTRY_64_LOOKUP, .ap_entry_64_lookup - __ap_trampoline_start + +.code64 +ap_entry_64: + # Setup stack pointer to the stack given by BSP + mov rsp, [rip + __ap_stack] + and rsp, -16 + + # Call an `extern "sysv64"` function given by the kernel + mov rdi, [rip + __ap_kernel_arg0] + mov rsi, [rip + __ap_kernel_arg1] + mov rdx, [rip + __ap_kernel_arg2] + jmp [rip + __ap_kernel_entry] + + __ap_stack: .quad 0x0000000000000000 + __ap_kernel_entry: .quad 0x0000000000000000 + __ap_kernel_arg0: .quad 0x0000000000000000 + __ap_kernel_arg1: .quad 0x0000000000000000 + __ap_kernel_arg2: .quad 0x0000000000000000 + +__ap_trampoline_end: \ No newline at end of file diff --git a/kernel/src/device/imp/x86_64.rs b/kernel/src/device/imp/x86_64.rs new file mode 100644 index 0000000..7b3a88b --- /dev/null +++ b/kernel/src/device/imp/x86_64.rs @@ -0,0 +1,329 @@ +use core::{ + arch::{asm, global_asm, x86_64::__cpuid_count}, + hint::spin_loop, + mem::ManuallyDrop, + ptr::{self, NonNull}, + sync::atomic::{ + AtomicBool, AtomicU32, + Ordering::{AcqRel, Acquire, Relaxed, Release}, + }, + time::Duration, +}; + +use elpytios_bootinfo::{PAGE_SIZE, paddr::PAddr}; +use log::{debug, error, info}; + +use crate::{ + ScratchPages, + allocator::{SlabAllocator, SlabId}, + arch::x86_64::{Msr, pit_delay, rdmsr, wrmsr}, + device::acpi::{LocalApicFlags, Madt, Pic}, + interrupt::init_interrupts, + statics::{get_phys_alloc, get_virtual_map, phys_to_virt}, + vaddr::{VAddr, VFlags}, +}; + +global_asm!(include_str!("trampolines/x86_64.s")); +unsafe extern "sysv64" { + static __ap_trampoline_start: u8; + static __ap_cr3: u8; + static __ap_cr4: u8; + static __ap_stack: u8; + static __ap_kernel_entry: u8; + static __ap_kernel_arg0: u8; + static __ap_kernel_arg1: u8; + static __ap_kernel_arg2: u8; + static __ap_trampoline_end: u8; +} + +#[derive(Clone, Copy)] +enum ApicDriver { + XApic { mmr: NonNull }, + X2Apic, +} + +impl ApicDriver { + #[inline] + fn apic_id(self) -> u32 { + match self { + Self::XApic { mmr } => unsafe { (mmr.byte_add(0x20).read_volatile() >> 24) & 0xff }, + Self::X2Apic => unsafe { rdmsr(Msr::Ia32X2ApicId) as u32 }, + } + } + + #[inline] + unsafe fn init(self, apic_id: u32) { + match self { + Self::XApic { .. } => unimplemented!("Waking up cores via legacy xAPIC isn't implemented yet"), + Self::X2Apic => unsafe { + let id = (apic_id as u64) << 32; + let assert = 1 << 14; + + wrmsr(Msr::Ia32X2ApicIcr, (5 << 8) | assert | id); + }, + } + } + + #[inline] + unsafe fn startup(self, send_init: bool, trampoline_phys: PAddr, apic_id: u32) { + match self { + Self::XApic { .. } => unimplemented!("Waking up cores via legacy xAPIC isn't implemented yet"), + Self::X2Apic => unsafe { + // IA32_X2APIC_ICR: + // - Bit 0-7: Vector + // - Bit 8-10: Delivery mode (4=NMI, 5=Init, 6=Startup) + // - Bit 14: Assert flag + // - Bit 32-63: Target core destination APIC ID + let id = (apic_id as u64) << 32; + let assert = 1 << 14; + + if send_init { + self.init(apic_id); + pit_delay(Duration::from_millis(10)); + } + + wrmsr( + Msr::Ia32X2ApicIcr, + ((trampoline_phys.addr() / PAGE_SIZE) & 0xff) as u64 | (6 << 8) | assert | id, + ); + }, + } + } +} + +#[repr(C)] +pub struct CpuContext { + this: *const Self, + apic: ApicDriver, + is_bootstrap: bool, + apic_id: u32, + cpu_id: u32, +} + +impl CpuContext { + unsafe fn new(apic: ApicDriver, is_bootstrap: bool, apic_id: u32, cpu_id: u32) { + static CTX_ALLOC: SlabAllocator = SlabAllocator::new(); + + if let ApicDriver::X2Apic = apic { + unsafe { wrmsr(Msr::Ia32ApicBase, rdmsr(Msr::Ia32ApicBase) | (1 << 10) | (1 << 11)) } + } + + let this = &raw mut *SlabId::leak(CTX_ALLOC.alloc(Self { + this: ptr::null(), + apic, + is_bootstrap, + apic_id, + cpu_id, + })); + + unsafe { + (*this).this = this; + wrmsr(Msr::Ia32GsBase, this as u64); + } + } + + #[inline(always)] + pub fn get() -> &'static Self { + let ptr: *const Self; + unsafe { + asm!( + "mov {}, gs:[0]", + + out(reg) ptr, + options(pure, nomem, nostack, preserves_flags), + ); + ptr.as_ref_unchecked() + } + } + + #[inline] + pub fn is_bootstrap(&self) -> bool { + self.is_bootstrap + } + + #[inline] + pub fn apic_id(&self) -> u32 { + self.apic_id + } + + #[inline] + pub fn cpu_id(&self) -> u32 { + self.cpu_id + } +} + +pub unsafe fn init_device_tree ! + Clone + Send>(scratch_pages: &mut ScratchPages, processor_entry: F, madt: Madt) -> ! { + unsafe { + init_interrupts(); + + let v_map = get_virtual_map(); + let driver = if __cpuid_count(0x01, 0x00).ecx & (1 << 21) != 0 { + info!("x2APIC is supported on this hardware; using Model-Specific Registers for APIC"); + + wrmsr(Msr::Ia32ApicBase, rdmsr(Msr::Ia32ApicBase) | (1 << 10) | (1 << 11)); + ApicDriver::X2Apic + } else { + info!("x2APIC is unsupported on this hardware; falling back to legacy memory-mapped xAPIC"); + let mmr_phys = PAddr::new(madt.local_interrupt_control_addr as usize); + let mmr = phys_to_virt(mmr_phys); + + v_map + .map( + mmr_phys, + mmr, + 1, + VFlags::GLOBAL | VFlags::WRITABLE | VFlags::CACHE_DISABLED | VFlags::EXECUTE_DISABLE, + ) + .expect("Couldn't virtual-map xAPIC MMR"); + + ApicDriver::XApic { + mmr: NonNull::new_unchecked(mmr.ptr_mut::()), + } + }; + + let trampoline_phys = scratch_pages.take().expect("Not enough scratch pages for AP trampoline entry"); + let trampoline = phys_to_virt(trampoline_phys); + v_map + .map(trampoline_phys, VAddr::new(trampoline_phys.addr()), 1, VFlags::empty()) + .expect("Couldn't identity-map trampoline code"); + v_map + .map(trampoline_phys, trampoline, 1, VFlags::WRITABLE | VFlags::EXECUTE_DISABLE) + .expect("Couldn't virtual-map trampoline code"); + let trampoline = trampoline.ptr_mut::(); + let trampoline_len = (&raw const __ap_trampoline_end).offset_from_unsigned(&raw const __ap_trampoline_start); + assert!( + trampoline_len <= PAGE_SIZE, + "AP entry trampoline size {trampoline_len} is bigger than {PAGE_SIZE}" + ); + + trampoline.copy_from_nonoverlapping(&raw const __ap_trampoline_start, trampoline_len); + trampoline + .add((&raw const __ap_cr3).offset_from_unsigned(&raw const __ap_trampoline_start)) + .cast::() + .write_unaligned({ + let cr3: usize; + asm!("mov {}, cr3", out(reg) cr3, options(nomem, nostack, preserves_flags)); + u32::try_from(cr3).expect("Page table physical address must be within 32-bit address") + }); + trampoline + .add((&raw const __ap_cr4).offset_from_unsigned(&raw const __ap_trampoline_start)) + .cast::() + .write_unaligned({ + let cr4: usize; + asm!("mov {}, cr4", out(reg) cr4, options(nomem, nostack, preserves_flags)); + u32::try_from(cr4).expect("Page table physical address must be within 32-bit address") + }); + trampoline + .add((&raw const __ap_kernel_entry).offset_from_unsigned(&raw const __ap_trampoline_start)) + .cast::() + .write_unaligned(ap_kernel_entry:: as *const () as u64); + + debug!("\tCopied {trampoline_len} bytes into {trampoline:p} (physical address at {trampoline_phys:p}) for AP cores entry"); + + static AP_INIT: AtomicBool = AtomicBool::new(false); + static AP_PROCEED: AtomicU32 = AtomicU32::new(0); + + unsafe extern "sysv64" fn ap_kernel_entry ! + Clone + Send>( + processor_entry: *const (), + apic: ApicDriver, + ids: *const [u32; 2], + ) -> ! { + let processor_entry = unsafe { + let [apic_id, cpu_id] = ids.read_unaligned(); + init_interrupts(); + CpuContext::new(apic, false, apic_id, cpu_id); + + (processor_entry as *const F).read_unaligned() + }; + + AP_INIT.store(true, Release); + let cpu_count = loop { + match AP_PROCEED.load(Acquire) { + 0 => { + spin_loop(); + continue + } + n => break n, + } + }; + + processor_entry(cpu_count) + } + + let bsp_id = driver.apic_id(); + let mut cpu_id = 0; + + let mut init_cpu = |apic_id: u32| { + if bsp_id == apic_id { + CpuContext::new(driver, true, apic_id, cpu_id); + cpu_id += 1; + } else { + const STACK_PAGES: usize = 16; + + let stack = get_phys_alloc() + .lock() + .alloc(STACK_PAGES.ilog2()) + .expect("Couldn't allocate stack for AP core"); + let stack_top = phys_to_virt(stack).byte_add(STACK_PAGES * PAGE_SIZE).addr() as u64; + let processor_entry = ManuallyDrop::new(processor_entry.clone()); + let ids = [apic_id, cpu_id]; + + trampoline + .add((&raw const __ap_stack).offset_from_unsigned(&raw const __ap_trampoline_start)) + .cast::() + .write_unaligned(stack_top); + trampoline + .add((&raw const __ap_kernel_arg0).offset_from_unsigned(&raw const __ap_trampoline_start)) + .cast::() + .write_unaligned(&raw const processor_entry as u64); + trampoline + .add((&raw const __ap_kernel_arg1).offset_from_unsigned(&raw const __ap_trampoline_start)) + .cast::() + .write_unaligned(driver); + trampoline + .add((&raw const __ap_kernel_arg2).offset_from_unsigned(&raw const __ap_trampoline_start)) + .cast::<*const [u32; 2]>() + .write_unaligned(&raw const ids); + + AP_INIT.store(false, Release); + driver.startup(true, trampoline_phys, apic_id); + + for i in 0..2 { + pit_delay(Duration::from_micros(500)); + match AP_INIT.compare_exchange(true, false, AcqRel, Relaxed) { + Ok(..) => { + cpu_id += 1; + debug!("\tAP core {apic_id} is up and running"); + break + } + Err(..) => { + if i == 0 { + driver.startup(false, trampoline_phys, apic_id); + error!("\tCouldn't start up AP core {apic_id}, retrying one more time"); + } else { + // Send one last INIT IPI to ensure the AP core isn't doing anything + driver.init(apic_id); + error!("\tCouldn't start up AP core {apic_id} even after retrying, giving up"); + } + } + } + } + } + }; + + for pic in madt { + match pic { + Pic::ProcessorLocal(proc) if (*&raw const proc.flags).contains(LocalApicFlags::ENABLED) => { + init_cpu(proc.apic_id as u32); + } + Pic::ProcessLocalX2(proc) if (*&raw const proc.flags).contains(LocalApicFlags::ENABLED) => { + init_cpu(proc.x2apic_id as u32); + } + _ => {} + } + } + + AP_PROCEED.store(cpu_id, Release); + processor_entry(cpu_id) + } +} diff --git a/kernel/src/device/mod.rs b/kernel/src/device/mod.rs new file mode 100644 index 0000000..f79c497 --- /dev/null +++ b/kernel/src/device/mod.rs @@ -0,0 +1,75 @@ +mod imp { + cfg_select! { + target_arch = "x86_64" => { + mod x86_64; + pub use x86_64::*; + } + _ => { + compile_error!("Unsupported architecture"); + } + } +} +use elpytios_bootinfo::DeviceTree; +pub use imp::CpuContext; +use log::info; + +mod acpi; +use acpi::*; + +use crate::ScratchPages; + +unsafe fn init_device_tree_impl( + scratch_pages: &mut ScratchPages, + processor_entry: impl FnOnce(u32) -> ! + Clone + Send, + system_tables: impl IntoIterator>, +) -> ! { + let system_tables = system_tables.into_iter(); + info!("Initializing device tree: found {} system tables", system_tables.len()); + + macro_rules! tables { + ($($output:ident: $type:ty;)*) => { + $(let mut $output = None::<$type>;)* + for system_table in system_tables { + let system_table = system_table.expect("Couldn't parse system table"); + $(match system_table.typed::<$type>() { + Ok(_table) => { + if $output.replace(_table).is_some() { + panic!("Duplicate '{}' entries", SignatureFmt(&<$type as TypedSystemTable>::SIGNATURE)); + } + } + Err(AcpiError::InvalidTableSignature { .. }) => {} + Err(e) => panic!("Couldn't parse typed system table: {e}"), + })* + } + $(let $output = $output.unwrap_or_else(|| panic!("Missing '{}' entries", SignatureFmt(&<$type as TypedSystemTable>::SIGNATURE)));)* + }; + } + + tables! { + madt: Madt; + } + + unsafe { imp::init_device_tree(scratch_pages, processor_entry, madt) } +} + +/// # Safety +/// - Only call this once in setup phase after higher-half addressing is finished. +/// - Identity-mapping must still be available. +pub unsafe fn init_device_tree( + device_tree: DeviceTree, + scratch_pages: &mut ScratchPages, + processor_entry: impl FnOnce(u32) -> ! + Clone + Send, +) -> ! { + match device_tree { + DeviceTree::Acpi(addr) => { + let rsdp = unsafe { Rsdp::new(addr.addr() as *const Rsdp) }.expect("Couldn't parse RSDP"); + let rsdt = rsdp.rsdt().expect("Couldn't parse RSDT"); + unsafe { init_device_tree_impl(scratch_pages, processor_entry, rsdt) } + } + DeviceTree::Acpi2(addr) => { + let xsdp = unsafe { Xsdp::new(addr.addr() as *const Xsdp) }.expect("Couldn't parse XSDP"); + let xsdt = xsdp.xsdt().expect("Couldn't parse XSDT"); + unsafe { init_device_tree_impl(scratch_pages, processor_entry, xsdt) } + } + }; +} diff --git a/kernel/src/framebuffer.rs b/kernel/src/framebuffer.rs new file mode 100644 index 0000000..d4e5367 --- /dev/null +++ b/kernel/src/framebuffer.rs @@ -0,0 +1,10 @@ +use elpytios_bootinfo::PixelFormat; + +#[derive(Debug)] +pub struct FrameBuffer { + pub width: usize, + pub height: usize, + pub stride: usize, + pub format: PixelFormat, + pub pointer: *mut u8, +} diff --git a/kernel/src/interrupt/mod.rs b/kernel/src/interrupt/mod.rs new file mode 100644 index 0000000..90d27ff --- /dev/null +++ b/kernel/src/interrupt/mod.rs @@ -0,0 +1,30 @@ +pub use imp::init_interrupts; + +cfg_select! { + target_arch = "x86_64" => { + mod x86_64; + use x86_64 as imp; + } + _ => { + compile_error!("Unsupported architecture"); + } +} + +#[inline] +unsafe fn page_fault( + ptr: *mut (), + missing_or_protected: bool, + caused_by_write: bool, + triggered_by_user: bool, + overwritten_reserved_bits: bool, + instruction_fetch_violation: bool, +) { + panic!( + "Page fault at address {ptr:p}\n\ + Missing/access\t: {missing_or_protected}\n\ + Caused by write\t: {caused_by_write}\n\ + From userland\t: {triggered_by_user}\n\ + Reserved bits\t: {overwritten_reserved_bits}\n\ + Inst fetch\t: {instruction_fetch_violation}" + ) +} diff --git a/kernel/src/interrupt/x86_64.rs b/kernel/src/interrupt/x86_64.rs new file mode 100644 index 0000000..88f62a7 --- /dev/null +++ b/kernel/src/interrupt/x86_64.rs @@ -0,0 +1,270 @@ +use core::{ + arch::{asm, naked_asm}, + hint::{cold_path, spin_loop}, + mem::size_of_val_raw, + sync::atomic::{ + AtomicU8, + Ordering::{Acquire, Relaxed, Release}, + }, +}; + +use bitflags::bitflags; +use bytemuck::Zeroable; + +use crate::allocator::{SlabAllocator, SlabId}; + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct GdtEntry(u64); +bitflags! { + impl GdtEntry: u64 { + const ACCESSED = 1 << 40; + const WRITABLE = 1 << 41; + const EXECUTABLE = 1 << 43; + const DESCRIPTOR_TYPE = 1 << 44; + const PRESENT = 1 << 47; + const LONG_MODE = 1 << 53; + } +} + +impl GdtEntry { + pub const NULL: Self = Self(0); + pub const KERNEL_CODE: Self = Self(Self::PRESENT.0 | Self::DESCRIPTOR_TYPE.0 | Self::EXECUTABLE.0 | Self::LONG_MODE.0); + pub const KERNEL_DATA: Self = Self(Self::PRESENT.0 | Self::DESCRIPTOR_TYPE.0 | Self::WRITABLE.0); +} + +pub unsafe fn init_gdt() { + static GDT_ALLOC: SlabAllocator<[GdtEntry; 3]> = SlabAllocator::new(); + + #[repr(C, packed)] + struct GdtPointer { + limit: u16, + base: *mut GdtEntry, + } + + unsafe { + let entries = SlabId::leak(GDT_ALLOC.alloc([GdtEntry::NULL, GdtEntry::KERNEL_CODE, GdtEntry::KERNEL_DATA])); + let ptr = GdtPointer { + limit: u16::try_from(size_of_val(entries) - 1).unwrap(), + base: (&raw mut *entries).cast(), + }; + + asm!( + "lgdt [{ptr}]", + // `KERNEL_DATA` selector is 0x10 + "mov ax, 0x10", + "mov ds, ax", + "mov es, ax", + "mov ss, ax", + // `KERNEL_CODE` selector is 0x08 + "push 0x08", + // Perform a long jump, loading the GDT entries + "lea rax, [rip + 2f]", + "push rax", + "retfq", + "2:", + + ptr = in(reg) &ptr, + out("rax") _, + ); + } +} + +#[derive(Copy, Clone, Zeroable)] +#[repr(C, packed)] +pub struct IdtEntry { + pointer_low: u16, + gdt_selector: u16, + options: IdtOptions, + pointer_middle: u16, + pointer_high: u32, + reserved: u32, +} + +impl IdtEntry { + #[inline] + pub unsafe fn new(handler: unsafe extern "sysv64" fn() -> !) -> Self { + let addr = handler as usize; + Self { + pointer_low: addr as u16, + pointer_middle: (addr >> 16) as u16, + pointer_high: (addr >> 32) as u32, + gdt_selector: 0x08, // `KERNEL_CODE` selector + options: IdtOptions(IdtOptions::PRESENT.0 | IdtOptions::DPL_RING_0.0 | IdtOptions::TYPE_INTERRUPT.0), + reserved: 0, + } + } +} + +#[derive(Debug, Clone, Copy, Zeroable)] +#[repr(transparent)] +pub struct IdtOptions(u16); +bitflags! { + impl IdtOptions: u16 { + const PRESENT = 1 << 15; + + const DPL_RING_0 = 0 << 12; + const DPL_RING_3 = 3 << 12; + + const TYPE_INTERRUPT = 0xe << 8; + const TYPE_TRAP = 0xf << 8; + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(usize)] +pub enum IdtIndex { + // Hard-coded by CPU + DoubleFault = 8, + PageFault = 14, +} + +const CLOBBERED: usize = 9 * size_of::(); +macro_rules! clobbered { + (push) => { + r#" + push rax + push rcx + push rdx + push rsi + push rdi + push r8 + push r9 + push r10 + push r11 + "# + }; + (pop) => { + r#" + pop r11 + pop r10 + pop r9 + pop r8 + pop rdi + pop rsi + pop rdx + pop rcx + pop rax + "# + }; +} + +#[unsafe(naked)] +pub unsafe extern "sysv64" fn double_fault() -> ! { + unsafe extern "sysv64" fn handle(code: usize) -> ! { + panic!("Double-fault caught (Hardware error code: {code})") + } + + naked_asm!( + clobbered!(push), + + "mov rdi, [rsp + {clobbered}]", + "call {handle}", + + clobbered!(pop), + "add rsp, 8", + "iretq", + + clobbered = const CLOBBERED, + handle = sym handle, + ) +} + +#[unsafe(naked)] +pub unsafe extern "sysv64" fn page_fault() -> ! { + #[repr(transparent)] + struct ErrorCode(usize); + bitflags! { + impl ErrorCode: usize { + // 0=protection violation, 1=not present + const NOT_PRESENT = 1 << 0; + // 0=caused by read, 1=caused by read + const IS_WRITE = 1 << 1; + // 0=triggered in ring 0, 1=triggered in ring 3 + const IS_USER = 1 << 2; + // overwrote reserved bits in page table + const RESERVED = 1 << 3; + // instruction fetch violation + const EXECUTE = 1 << 4; + } + } + + unsafe extern "sysv64" fn handle(code: ErrorCode) { + unsafe { + let ptr: *mut (); + asm!("mov {ptr}, cr2", ptr = out(reg) ptr); + + super::page_fault( + ptr, + code.contains(ErrorCode::NOT_PRESENT), + code.contains(ErrorCode::IS_WRITE), + code.contains(ErrorCode::IS_USER), + code.contains(ErrorCode::RESERVED), + code.contains(ErrorCode::EXECUTE), + ); + } + } + + naked_asm!( + clobbered!(push), + + "mov rdi, [rsp + {clobbered}]", + "call {handle}", + + clobbered!(pop), + "add rsp, 8", + "iretq", + + clobbered = const CLOBBERED, + handle = sym handle, + ) +} + +/// # Safety +/// Only call this once per CPU core in setup phase after higher-half addressing is finished. +pub unsafe fn init_interrupts() { + static mut IDT_ENTRIES: [IdtEntry; 256] = [bytemuck::zeroed(); 256]; + static IDT_STATE: AtomicU8 = AtomicU8::new(UNINIT); + + const UNINIT: u8 = 0; + const LOCKED: u8 = 1; + const INIT: u8 = 2; + + loop { + match IDT_STATE.compare_exchange_weak(UNINIT, LOCKED, Acquire, Relaxed) { + Ok(..) => unsafe { + cold_path(); + IDT_ENTRIES[IdtIndex::DoubleFault as usize] = IdtEntry::new(double_fault); + IDT_ENTRIES[IdtIndex::PageFault as usize] = IdtEntry::new(page_fault); + + IDT_STATE.store(INIT, Release); + }, + Err(INIT) => break, + Err(..) => { + cold_path(); + spin_loop(); + } + } + } + + #[repr(C, packed)] + struct IdtPointer { + limit: u16, + base: *mut IdtEntry, + } + + unsafe { + // Global descriptor table is x86-specific + init_gdt(); + + let ptr = IdtPointer { + limit: u16::try_from(size_of_val_raw(&raw const IDT_ENTRIES) - 1).unwrap(), + base: (&raw mut IDT_ENTRIES).cast(), + }; + + asm!( + "lidt [{ptr}]", + ptr = in(reg) &ptr, + ); + } +} diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index 316c44b..51475c1 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -1,20 +1,137 @@ -#![feature(custom_inner_attributes)] -#![rustfmt::skip] - +#![forbid(unfulfilled_lint_expectations)] +#![allow(internal_features, reason = "`pattern_type!()` is just too useful")] +#![feature( + anonymous_lifetime_in_impl_trait, + arbitrary_self_types_pointers, + atomic_ptr_null, + const_cmp, + const_trait_impl, + const_try, + debug_closure_helpers, + impl_trait_in_assoc_type, + layout_for_ptr, + never_type, + pointer_is_aligned_to, + ptr_alignment_type, + ptr_metadata, + slice_ptr_get, + sync_unsafe_cell +)] #![no_std] -pub mod page_alloc; -pub mod page_alloc_tree; +extern crate alloc; + +pub mod allocator; +pub mod arch; +pub mod device; +pub mod framebuffer; +pub mod interrupt; pub mod rendering; +pub mod serial; +pub mod spin_sync; +pub mod vaddr; use core::mem::MaybeUninit; -use elpytios_bootinfo::BootInfo; +use allocator::{HeapAllocator, PhysicalPageAllocator}; +use elpytios_bootinfo::paddr::PAddr; +use framebuffer::FrameBuffer; +use spin_sync::SpinMutex; +use vaddr::{VAddr, VirtualMap}; + +#[repr(transparent)] +pub struct ScratchPages<'a> { + pub pages: &'a [PAddr], +} + +impl ScratchPages<'_> { + pub fn take(&mut self) -> Option { + loop { + match self.pages.split_at_checked(1) { + Some((&[next], pages)) => { + self.pages = pages; + if next.addr() == 0 { continue } else { break Some(next) } + } + _ => break None, + } + } + } +} + +/// # Safety +/// Every single one of these statics must be set by their corresponding `set_*` functions below in +/// the setup-phase of the kernel. +/// +/// See `main.rs`. +pub mod statics { + use super::*; + + static mut DIRECT_MAP_OFFSET: MaybeUninit = MaybeUninit::uninit(); + static mut VIRTUAL_MAP: MaybeUninit = MaybeUninit::uninit(); + static mut PHYS_ALLOC: MaybeUninit> = MaybeUninit::uninit(); + static mut FRAME_BUFFER: MaybeUninit = MaybeUninit::uninit(); + + #[global_allocator] + static ALLOC: HeapAllocator = HeapAllocator::new(); + + #[inline] + pub unsafe fn set_direct_map_offset(offset: usize) { + unsafe { + DIRECT_MAP_OFFSET = MaybeUninit::new(offset); + } + } + + #[inline] + pub unsafe fn set_virtual_map(virtual_map: VirtualMap) { + unsafe { + VIRTUAL_MAP = MaybeUninit::new(virtual_map); + } + } + + #[inline] + pub unsafe fn set_phys_alloc(phys_alloc: PhysicalPageAllocator) { + unsafe { + PHYS_ALLOC = MaybeUninit::new(SpinMutex::new(phys_alloc)); + } + } + + #[inline] + pub unsafe fn set_frame_buffer(frame_buffer: FrameBuffer) { + unsafe { + FRAME_BUFFER = MaybeUninit::new(frame_buffer); + } + } + + #[inline] + pub fn phys_to_virt(p_addr: PAddr) -> VAddr { + VAddr::new( + p_addr + .addr() + .wrapping_add(unsafe { (&raw const DIRECT_MAP_OFFSET as *const usize).read() }), + ) + } + + #[inline] + pub fn virt_to_phys(v_addr: VAddr) -> PAddr { + PAddr::new( + v_addr + .addr() + .wrapping_sub(unsafe { (&raw const DIRECT_MAP_OFFSET as *const usize).read() }), + ) + } + + #[inline] + pub fn get_virtual_map() -> &'static VirtualMap { + unsafe { (&raw const VIRTUAL_MAP as *const VirtualMap).as_ref_unchecked() } + } -#[unsafe(link_section = ".bootinfo")] -#[used] -static mut BOOT_INFO: MaybeUninit = MaybeUninit::uninit(); + #[inline] + pub fn get_phys_alloc() -> &'static SpinMutex { + unsafe { (&raw const PHYS_ALLOC as *const SpinMutex).as_ref_unchecked() } + } -pub fn boot_info() -> &'static BootInfo { - unsafe { (&raw const BOOT_INFO as *const BootInfo).as_ref_unchecked() } -} \ No newline at end of file + #[inline] + pub fn get_frame_buffer() -> &'static FrameBuffer { + unsafe { (&raw const FRAME_BUFFER as *const FrameBuffer).as_ref_unchecked() } + } +} diff --git a/kernel/src/linker.ld b/kernel/src/linker.ld index 18ac51c..8a12af0 100644 --- a/kernel/src/linker.ld +++ b/kernel/src/linker.ld @@ -2,15 +2,16 @@ ENTRY(_start) PHDRS { - text PT_LOAD FLAGS(5); - rodata PT_LOAD FLAGS(4); - data PT_LOAD FLAGS(6); + text PT_LOAD FLAGS(5); + rodata PT_LOAD FLAGS(4); + data PT_LOAD FLAGS(6); + dynamic PT_DYNAMIC FLAGS(4); } SECTIONS { __kernel_start = .; - . = 0xffffffff80000000; + . = 0; .text : ALIGN(4K) { @@ -27,13 +28,6 @@ SECTIONS __rodata_end = .; } :rodata - .bootinfo (NOLOAD) : ALIGN(4K) - { - __bootinfo_start = .; - KEEP(*(.bootinfo)) - __bootinfo_end = .; - } :rodata - .data : ALIGN(4K) { __data_start = .; @@ -49,5 +43,10 @@ SECTIONS __bss_end = .; } :data + .dynamic : ALIGN(8) + { + *(.dynamic) + } :data :dynamic + __kernel_end = .; } \ No newline at end of file diff --git a/kernel/src/main.rs b/kernel/src/main.rs index ffac2e5..2504f08 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -1,71 +1,449 @@ -#![feature(custom_inner_attributes)] -#![rustfmt::skip] - +#![forbid(unfulfilled_lint_expectations)] +#![feature(core_float_math, ptr_alignment_type)] #![no_std] #![no_main] -use core::{arch::naked_asm, fmt::Write, panic::PanicInfo}; +extern crate alloc; +use alloc::string::String; +use core::{ + arch::{asm, naked_asm}, + cell::RefCell, + fmt::Write, + iter::once, + panic::PanicInfo, +}; -use elpytios_bootinfo::PAGE_SIZE; -use elpytios_kernel::{boot_info, rendering::DisplayWriter}; +use elpytios_bootinfo::{BootInfo, IdentityMapFlags, MemoryRegion, PAGE_SIZE, Reloc, paddr::PAddr}; +use elpytios_elf::sys::{ElfRela64, ElfRela64Type}; +use elpytios_kernel::{ + ScratchPages, + allocator::{AllocTree, PHYS_ALLOC_ALIGNMENT, PhysicalPageAllocator}, + device::{CpuContext, init_device_tree}, + framebuffer::FrameBuffer, + serial::{Com, Serial, serial_init}, + statics::{get_virtual_map, phys_to_virt, set_direct_map_offset, set_frame_buffer, set_phys_alloc, set_virtual_map}, + vaddr::{VAddr, VFlags, VirtualMapBuilder}, +}; +use log::{debug, error, info}; #[panic_handler] -fn hanic_pandler(_info: &PanicInfo) -> ! { +fn panic_handler(info: &PanicInfo) -> ! { + error!("{info}"); loop {} } -#[cfg(debug_assertions)] -#[unsafe(no_mangle)] -#[used] -static mut DEBUG_HALT: u8 = 1; - -#[cfg(debug_assertions)] -#[inline(never)] -fn pause() { - loop { - if unsafe { DEBUG_HALT } == 0 { - break +#[unsafe(naked)] +#[unsafe(export_name = "_start")] // Tell the linker that this is our entry point +unsafe extern "sysv64" fn jump_from_bootloader(info: &'static BootInfo) -> ! { + naked_asm!( + // Clear interrupt handlers, will be reinitialized by `setup_virtual_mapped()` + "cli", + "jmp {setup_identity_mapped}", + + setup_identity_mapped = sym setup_identity_mapped, + ) +} + +const HIGHER_HALF_ADDRESS_BASE: VAddr = VAddr::new(0xffff_8000_0000_0000); + +struct SerialLogger(Com); +impl log::Log for SerialLogger { + fn enabled(&self, _metadata: &log::Metadata) -> bool { + true + } + + fn log(&self, record: &log::Record) { + if self.enabled(record.metadata()) { + _ = match (record.file(), record.line()) { + (Some(file), Some(line)) => writeln!(Serial(self.0), "[{}]\t{}:{}\t> {}", record.level(), file, line, record.args()), + _ => writeln!(Serial(self.0), "[{}]\t{}\t> {}", record.level(), record.target(), record.args()), + } } + } - core::hint::spin_loop(); + fn flush(&self) {} +} + +static LOGGER: SerialLogger = SerialLogger(Com::Com3); + +struct MemoryRegions<'a> { + available: &'a [MemoryRegion], + head: MemoryRegion, +} + +impl<'a> MemoryRegions<'a> { + fn new(source: &'a [MemoryRegion]) -> Self { + let &[ref available @ .., head] = source else { panic!("Not enough memory to start the kernel") }; + Self { available, head } + } + + fn take_head(&mut self) -> Option { + let head = loop { + match self.head.pages { + 0 => { + let &[ref available @ .., head] = self.available else { return None }; + self.available = available; + self.head = head; + } + n => { + self.head.pages = n - 1; + break self.head.base.byte_add((n - 1) * PAGE_SIZE) + } + } + }; + Some(head) } } -#[unsafe(naked)] -#[unsafe(export_name = "_start")] -unsafe extern "sysv64" fn jump_from_bootloader() -> ! { - naked_asm!( - "jmp {main}", - main = sym main - ) +/// # Safety +/// - Available memory regions must *not* include the kernel code, stack, and boot info itself; +/// i.e., they must be usable immediately. +/// - Any references must point to the defined custom `MEM_*` memory types in the bootloader. +/// - See safety notes of [`setup_virtual_mapped`]. +unsafe extern "sysv64" fn setup_identity_mapped(info: &'static BootInfo) -> ! { + // Notes: + // - `log` mustn't be setup here; wait until symbols are relocated + + let regions = RefCell::new(MemoryRegions::new(&info.memory_regions)); + let mut scratch_pages = ScratchPages { pages: &info.scratch_pages }; + + let kernel_base = info + .identity_maps + .iter() + .min_by_key(|map| map.region.base) + .expect("Didn't find any identity maps") + .region + .base; + let v_slide = HIGHER_HALF_ADDRESS_BASE + .addr() + .checked_sub(kernel_base.addr()) + .expect("Kernel physical address somehow higher than higher-half addressing base"); + + let page_table_phys = scratch_pages.take().expect("Not enough scratch pages for page table"); + let setup_virtual_mapped = { + let mut v_map = unsafe { VirtualMapBuilder::new(page_table_phys, || regions.borrow_mut().take_head(), |p_addr| p_addr.addr() as *mut ()) }; + + let mut direct_map_offset = usize::MIN; + for map in &info.identity_maps { + let mut flags = VFlags::GLOBAL; + if map.flags.contains(IdentityMapFlags::WRITABLE) { + flags |= VFlags::WRITABLE; + } + if !map.flags.contains(IdentityMapFlags::EXECUTABLE) { + flags |= VFlags::EXECUTE_DISABLE; + } + + v_map + .map(map.region.base, VAddr::new(map.region.base.addr()), map.region.pages, flags) + .expect("Couldn't identity map kernel segment"); + + v_map + .map(map.region.base, VAddr::new(map.region.base.addr() + v_slide), map.region.pages, flags) + .expect("Couldn't virtual map kernel segment"); + + direct_map_offset = direct_map_offset.max(map.region.base.addr() + v_slide + map.region.pages * PAGE_SIZE); + } + + // Direct map *all* of RAM to the specified direct-map offset + let direct_map_offset = direct_map_offset.next_multiple_of(2 << 30); // Align to a gigabyte + unsafe { set_direct_map_offset(direct_map_offset) } + + v_map + .map( + page_table_phys, + VAddr::new(page_table_phys.addr() + direct_map_offset), + 1, + VFlags::GLOBAL | VFlags::WRITABLE | VFlags::EXECUTE_DISABLE, + ) + .unwrap(); + + for region in &info.memory_regions { + v_map + .map( + region.base, + VAddr::new(region.base.addr() + direct_map_offset), + region.pages, + VFlags::GLOBAL | VFlags::WRITABLE | VFlags::EXECUTE_DISABLE, + ) + .unwrap(); + } + + let virtual_map = unsafe { v_map.finish() }; + let setup_virtual_mapped = (setup_virtual_mapped as *const ()) + .addr() + .checked_add(v_slide) + .expect("`setup_virtual_mapped()` virtual address overflowed"); + + unsafe { set_virtual_map(virtual_map) } + setup_virtual_mapped + }; + + unsafe { + let tmp = 0usize; + asm!( + // Enable `GLOBAL` mapping, i.e. pages in TLB that don't get flushed + "mov {tmp}, cr4", + "or {tmp}, 1 << 7", + "mov cr4, {tmp}", + + "mov cr3, {page_table_phys}", + "add rsp, {v_slide}", + "and rsp, -16", + "jmp {setup_virtual_mapped}", + + tmp = in(reg) tmp, + page_table_phys = in(reg) page_table_phys.addr(), + v_slide = in(reg) v_slide, + setup_virtual_mapped = in(reg) setup_virtual_mapped, + in("rdi") (info as *const BootInfo).byte_add(v_slide).as_ref_unchecked(), + in("rsi") ®ions.into_inner(), + in("rdx") &mut scratch_pages, + in("rcx") kernel_base.addr(), + + options(noreturn), + ) + } } -unsafe extern "sysv64" fn main() -> ! { +unsafe extern "sysv64" fn setup_virtual_mapped( + info: &'static BootInfo, + regions: &MemoryRegions, + scratch_pages: &mut ScratchPages, + kernel_base: PAddr, +) -> ! { + // Relocate all symbols to higher-half addressing + // Identity-mapping is still present at this point, so it is okay to cast `PAddr` into pointers + let kernel_ptr = info.kernel_elf_base; + let v_slide = HIGHER_HALF_ADDRESS_BASE + .addr() + .checked_sub(kernel_base.addr()) + .expect("Kernel physical address somehow higher than higher-half addressing base") + .cast_signed() as i64; + + for &Reloc { offset, size, stride } in &info.relocations { + for i in 0..size / stride { + unsafe { + let rela = (kernel_ptr.addr() as *mut u8) + .cast::() + .byte_add(offset - info.kernel_virt_base) + .add(i) + .read_unaligned(); + + match rela.info.kind { + ElfRela64Type::X86_64_RELATIVE => { + let slide = v_slide + kernel_ptr.addr() as i64 - info.kernel_virt_base as i64; + let patch_addr = (kernel_ptr.addr() as *mut u8).add(rela.offset as usize - info.kernel_virt_base); + let value = slide + rela.addend; + patch_addr.cast::().write(value); + } + kind => panic!("Unsupported Elf64_Rela kind: {}", kind.0), + } + } + } + } + + // Setup `log` here, symbols have been relocated + unsafe { + serial_init(Com::Com3); + log::set_logger_racy(&LOGGER).expect("Log already setup before symbol relocations"); + log::set_max_level_racy(cfg_select! { + debug_assertions => log::LevelFilter::Trace, + not(debug_assertions) => log::LevelFilter::Info, + }); + + debug!( + "Setting up kernel at {kernel_ptr:p} -> {:p}", + VAddr::new(kernel_ptr.addr().wrapping_add_signed(v_slide as isize)) + ); + } + + // When running through `x qemu run --debug`, wait until a corresponding GDB client executes this: + // + // target remote [host, usuallty `localhost`]:[port, usually `1234`] + // add-symbol-file [path/to]/elpytios-kernel -o [offset; see "Setting up kernel at ..." log] + // + // set language c + // set *(unsigned char*)&__DEBUG_HALT = 0 + // set language rust + // continue + // + // This is to ensure the kernel has been loaded to memory at higher-half address before inserting + // software breakpoints and looking up symbols at the same offset #[cfg(debug_assertions)] - pause(); + { + #[unsafe(no_mangle)] + #[used] + static mut __DEBUG_HALT: u8 = 1; - let mut display_writer = DisplayWriter { - graphics_info: &boot_info().graphics_info, - line: 0, - col: 0, - }; + debug!("Waiting for debugger..."); + while unsafe { (&raw const __DEBUG_HALT).read_volatile() } != 0 { + core::hint::spin_loop(); + } + + debug!("Continuing!"); + } + + // Setup global physical page allocator + { + info!( + "Initializing physical page allocator: found {} usable memory regions", + info.memory_regions.len() + ); + + let mut phys_alloc = unsafe { PhysicalPageAllocator::new() }; + for MemoryRegion { base, pages } in regions.available.iter().copied().chain(once(regions.head)) { + debug!( + "\tAvailable memory region found at [{base:p}..{:p}], {pages} pages", + base.byte_add(pages * PAGE_SIZE) + ); + + let usable_start = base.addr(); + let mut usable_end = usable_start + pages * PAGE_SIZE; + + // Trees need to be aligned to `PHYS_ALLOC_ALIGNMENT`, so round down and manually fill "ghost" pages + let mut tree_start = usable_start & !(PHYS_ALLOC_ALIGNMENT.as_usize() - 1); + let mut tree_end = usable_end & !(PHYS_ALLOC_ALIGNMENT.as_usize() - 1); + + while tree_start < tree_end { + let layout = AllocTree::layout((tree_end - tree_start) / PAGE_SIZE).expect("`AllocTree` layout error"); + let meta_pages = layout.size().div_ceil(PAGE_SIZE); + + if (usable_end - tree_end) / PAGE_SIZE >= meta_pages { + debug!( + "\t\tBuilding tree at [{tree_start:#018x}..{:#018x}], {} pages", + tree_start + layout.node_count() * PAGE_SIZE, + layout.node_count(), + ); + + usable_end -= meta_pages * PAGE_SIZE; + unsafe { + let tree = AllocTree::new(phys_to_virt(PAddr::new(usable_end)).ptr_mut(), layout); + if let Some(mut ghost_size) = usable_start.checked_sub(tree_start) + && ghost_size != 0 + { + ghost_size /= PAGE_SIZE; + debug!("\t\t\tReserving {ghost_size} ghost pages"); + + #[cfg(debug_assertions)] + let [mut prev, mut prev_len] = [0, 0]; + + while ghost_size != 0 { + let order = ghost_size.ilog2(); + ghost_size -= 1 << order; + + #[cfg_attr(not(debug_assertions), expect(unused))] + let index = (*tree).alloc(order).expect("Couldn't reserve ghost pages"); + + #[cfg(debug_assertions)] + if index == prev + prev_len { + prev = index; + prev_len = 1 << order; + } else { + panic!("Ghost page reservation wasn't contiguous"); + } + } + } - writeln!(&mut display_writer, "Hello World from the Kernel, calling at address {:p}!!!!", main as *const ()).unwrap(); - - let regions = boot_info().memory_regions(); - writeln!(&mut display_writer, "Found {} usable physical memory regions!", regions.len()).unwrap(); - for region in regions { - writeln!( - &mut display_writer, - "Usable physical memory in {}..{}, {} pages!", - region.base, - region.base.byte_add(region.pages * PAGE_SIZE), - region.pages, - ).unwrap(); + phys_alloc.push_tree(PAddr::new(tree_start), tree); + } + tree_start += layout.node_count() * PAGE_SIZE; + tree_end = usable_end & !(PHYS_ALLOC_ALIGNMENT.as_usize() - 1); + } else { + tree_end -= (tree_end - tree_start) / 2; + } + } + } + + phys_alloc.sort_tree(); + match phys_alloc.tree_count() { + 0 => panic!("Couldn't initialize physical page allocator, found no usable memory regions"), + n => { + info!("Initialized physical page allocator with {n} trees"); + unsafe { set_phys_alloc(phys_alloc) } + } + } } - let [virt_start, virt_end] = boot_info().v_addr_range(); - writeln!(&mut display_writer, "Higher-half virtual addressing available in range {virt_start}..{virt_end}").unwrap(); + // Virtual-map the framebuffer + { + let v_map = get_virtual_map(); + let fb_phys = info.graphics_info.frame_buffer.addr(); + let fb_size = info.graphics_info.frame_buffer_size; + + let fb_phys_base = fb_phys & !(PAGE_SIZE - 1); + let fb_phys_end = (fb_phys + fb_size).next_multiple_of(PAGE_SIZE); + let fb_page_count = (fb_phys_end - fb_phys_base) / PAGE_SIZE; + + unsafe { + let p_addr = PAddr::new(fb_phys_base); + v_map + .map( + p_addr, + phys_to_virt(p_addr), + fb_page_count, + VFlags::GLOBAL | VFlags::WRITABLE | VFlags::WRITE_THROUGH | VFlags::EXECUTE_DISABLE, + ) + .unwrap(); + } + + unsafe { + set_frame_buffer(FrameBuffer { + width: info.graphics_info.w, + height: info.graphics_info.h, + stride: info.graphics_info.stride, + format: info.graphics_info.pixel_format, + pointer: phys_to_virt(PAddr::new(fb_phys)).ptr_mut(), + }) + } + } + + // Setup device tree, which includes waking up all AP and setting up interrupt handlers + // This calls the closure once for every CPU cores locally + unsafe { init_device_tree(info.device_tree, scratch_pages, main) } +} + +/// # Safety +/// - All [`statics`](elpytios_kernel::statics) must have been initialized prior to calling this +/// function. +/// - This function must be able to be run in parallel with itself on other threads +fn main(core_count: u32) -> ! { + if CpuContext::get().is_bootstrap() { + info!("Hello, world! Kernel is now up and running on {core_count} logical processors!"); + } + + { + use elpytios_bootinfo::PixelFormat; + use elpytios_kernel::statics::get_frame_buffer; + + let fbo = get_frame_buffer(); + let fbo_div = fbo.height.div_ceil(core_count as usize); + let cpu_id = CpuContext::get().cpu_id() as usize; + + match fbo.format { + fmt @ (PixelFormat::RGB_8_BIT | PixelFormat::BGR_8_BIT) => { + let invert_br = matches!(fmt, PixelFormat::BGR_8_BIT); + for y in (cpu_id * fbo_div)..((cpu_id + 1) * fbo_div).min(fbo.height) { + for x in 0..fbo.width { + let fx = x as f32 / (fbo.width - 1) as f32; + let fy = y as f32 / (fbo.height - 1) as f32; + + let r = (fx * 255.) as u8; + let g = (fy * 255.) as u8; + let b = (core::f32::math::sqrt((fx * 2. - 1.).abs() * (fy * 2. - 1.).abs()) * 255.) as u8; + let a = 255; + + unsafe { + fbo.pointer.cast::<[u8; 4]>().add(y * fbo.stride + x).write_volatile(match invert_br { + false => [r, g, b, a], + true => [b, g, r, a], + }); + } + } + } + } + _ => {} + } + } loop {} -} \ No newline at end of file +} diff --git a/kernel/src/page_alloc.rs b/kernel/src/page_alloc.rs deleted file mode 100644 index 0554168..0000000 --- a/kernel/src/page_alloc.rs +++ /dev/null @@ -1,171 +0,0 @@ -/*use core::{fmt::{Display, Formatter}, mem::offset_of, ptr::null_mut}; - -use uefi::{boot::MemoryType, mem::memory_map::{MemoryMap, MemoryMapOwned}}; - -use crate::page_alloc_tree::{BinaryBuddyTree, PAGE_SIZE}; - -const MINIMUM_PAGES_TO_MANAGE: usize = 4; -const PAGES_RESERVED_FOR_TREE_POINTERS: usize = 2; // TODO Fix, This is STUPID - -#[repr(C)] -pub struct PhysicalPageAllocator { - size: usize, - trees: [*mut BinaryBuddyTree; 0] -} - -impl PhysicalPageAllocator { - unsafe fn push_tree(&mut self, tree: *mut BinaryBuddyTree) { - unsafe { - (self as *mut Self) - .byte_add(offset_of!(Self, trees)) - .cast::<*mut BinaryBuddyTree>() - .add(self.size) - .write_volatile(tree); - } - - self.size += 1; - } - - unsafe fn get_tree(&self, n: usize) -> *mut BinaryBuddyTree { - unsafe { - (self as *const Self) - .byte_add(offset_of!(Self, trees)) - .cast::<*mut BinaryBuddyTree>() - .add(n) - .cast_mut() - .read_volatile() - } - } - - unsafe fn create_trees_(&mut self, start: usize, pages: usize, loader_memory: bool) -> Result<(), ()> { - if pages < MINIMUM_PAGES_TO_MANAGE { - return Err(()); - } - - let mut region_size = 1 << (pages.ilog2() as usize); - let free_pages = pages - region_size; - - let mut required_pages_for_tree = BinaryBuddyTree::required_pages_for_tree(region_size); - - while required_pages_for_tree > free_pages { - region_size >>= 1; - - if region_size < MINIMUM_PAGES_TO_MANAGE { - return Err(()); - } - - required_pages_for_tree = BinaryBuddyTree::required_pages_for_tree(region_size); - } - - let memory_region_start = start + required_pages_for_tree * PAGE_SIZE; - - unsafe { - if let Some(tree) = BinaryBuddyTree::new( - start as *mut BinaryBuddyTree, - memory_region_start as *mut u8, - region_size, - loader_memory - ) { - self.push_tree(tree); - - if let Ok(_) = self.create_trees_( - memory_region_start + region_size * PAGE_SIZE, - pages - region_size - required_pages_for_tree, - loader_memory - ) { - return Ok(()); - } else { - return Err(()); - } - } else { - return Err(()); - } - } - } - - pub fn new(memory_map: &MemoryMapOwned) -> Result<&mut Self, ()> { - let mut allocator: *mut Self = null_mut(); - - for i in memory_map.entries() { - let valid; - let loader_memory; - match i.ty { - MemoryType::CONVENTIONAL | - MemoryType::PERSISTENT_MEMORY => { - valid = true; - loader_memory = false; - }, - MemoryType::BOOT_SERVICES_CODE | - MemoryType::BOOT_SERVICES_DATA | - MemoryType::LOADER_CODE | - MemoryType::LOADER_DATA => { - valid = false; - loader_memory = true; - } - _ => { - valid = false; - loader_memory = false; - } - } - - if valid { - let mut start = i.phys_start as usize; - let mut size = i.page_count as usize; - - if allocator.is_null() && start.is_multiple_of(align_of::()) && - size == PAGES_RESERVED_FOR_TREE_POINTERS { - allocator = start as *mut Self; - - unsafe { - allocator.write_volatile(PhysicalPageAllocator { size: 0, trees: [] }); - } - - start = start + PAGE_SIZE * PAGES_RESERVED_FOR_TREE_POINTERS; - size -= PAGES_RESERVED_FOR_TREE_POINTERS; - } - - if !allocator.is_null() && size >= MINIMUM_PAGES_TO_MANAGE { - unsafe { - let _ = (&mut*allocator).create_trees_(start, size, loader_memory); - } - } - } - } - - unsafe { - if allocator.is_null() { - Err(()) - } else { - Ok(&mut*allocator) - } - } - } - - pub unsafe fn alloc(&mut self, pages: usize) -> Option<*mut u8> { - for i in 0..self.size { - unsafe { - if let Some(address) = (*self.get_tree(i)).alloc(pages) { - return Some(address); - } - } - } - - return None; - } - - pub unsafe fn free(&mut self, _region: *mut u8) -> Result<(), ()> { - Ok(()) - } -} - -impl Display for PhysicalPageAllocator { - fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { - for i in 0..self.size { - unsafe { - writeln!(f, "{:?}", self.get_tree(i))?; - } - } - - Ok(()) - } -}*/ diff --git a/kernel/src/page_alloc_tree.rs b/kernel/src/page_alloc_tree.rs deleted file mode 100644 index 8e70741..0000000 --- a/kernel/src/page_alloc_tree.rs +++ /dev/null @@ -1,280 +0,0 @@ -/*use core::mem::offset_of; - -#[repr(C)] -#[derive(Debug)] -pub struct BinaryBuddyTree { - base: *mut u8, - pages: usize, - pub loader_memory: bool, - children: [BinaryBuddyTreeNode; 0] -} - -const HAS_CHILDREN_FLAG: u8 = 0b0000_0001; -const IS_OCCUPIED_FLAG: u8 = 0b0000_0010; -pub const PAGE_SIZE: usize = 4096; - -#[derive(Debug, Clone, Copy)] -struct BinaryBuddyTreeNode { - flags: u8 -} - -const _: () = assert!( - offset_of!(BinaryBuddyTree, children).is_multiple_of(align_of::()) -); - -impl BinaryBuddyTree { - fn get_child_start_and_pages(&self, node: usize) -> (usize, usize) { - let level = (node + 1).ilog2() as usize; - - let first_in_level = (1 << level) - 1; - let position_in_level = node - first_in_level; - - let pages = self.pages >> level; - let start = position_in_level * pages; - - debug_assert!(pages.is_power_of_two()); - - (start, pages) - } - - unsafe fn write_child(&mut self, offset: usize, node: &BinaryBuddyTreeNode) { - unsafe { - (self as *mut BinaryBuddyTree) - .byte_add(offset_of!(Self, children)) - .cast::() - .add(offset) - .write_volatile(*node); - } - } - - unsafe fn get_child(&self, offset: usize) -> *mut BinaryBuddyTreeNode { - unsafe { - (self as *const BinaryBuddyTree) - .byte_add(offset_of!(Self, children)) - .cast::() - .cast_mut() - .add(offset) - } - } - - /// Get the amount of necessary pages to store a tree that keeps track of `pages` pages. - /// - /// * `pages`: The amount of pages the tree will keep track of. - pub fn required_pages_for_tree(pages: usize) -> usize { - let required_size_bytes = size_of::() + (pages * 2 - 1) * size_of::(); - - return (required_size_bytes + (PAGE_SIZE - 1)) / PAGE_SIZE; - } - - /// Create a new tree. - /// - /// * `tree_base`: Where to store the tree. - /// * `memory_base`: Where the memory the tree keeps track of starts at. - /// * `pages`: The amount of pages to keep track of. - /// * `loader_memory`: Whether or not this tree is keeping memory in the UEFI LoaderCode or - /// LoaderData regions. - pub unsafe fn new(tree_base: *mut BinaryBuddyTree, memory_base: *mut u8, pages: usize, loader_memory: bool) -> Option<&'static mut Self> { - unsafe { - if pages == 0 || !pages.is_power_of_two() { - return None; - } - - if !(tree_base as usize).is_multiple_of(align_of::()) { - return None; - } - - tree_base.write_volatile(BinaryBuddyTree { - base: memory_base, - pages: pages, - loader_memory: loader_memory, - children: [] - }); - - (&mut *tree_base).write_child(0, &BinaryBuddyTreeNode { flags: 0 }); - - Some(&mut *tree_base) - } - } - - unsafe fn find_fitting_leaf_(&mut self, pages: usize, node: usize, min_pages: usize) -> Option<(usize, usize)> { - unsafe { - let child = self.get_child(node); - - if (*child).flags & HAS_CHILDREN_FLAG == 0 { - let (_, child_pages) = self.get_child_start_and_pages(node); - - if (*child).flags & IS_OCCUPIED_FLAG == 0 && - child_pages >= pages && child_pages < min_pages { - return Some((node, child_pages)); - } else { - return None; - } - } else { - let result_left = self.find_fitting_leaf_(pages, node * 2 + 1, min_pages); - - let mut found_node = None; - let mut size_to_beat = min_pages; - if let Some((node_left, min_pages_left)) = result_left { - size_to_beat = min_pages_left; - found_node = Some(node_left); - } - - let result_right = self.find_fitting_leaf_(pages, node * 2 + 2, size_to_beat); - - if let Some((node_right, min_pages_right)) = result_right { - size_to_beat = min_pages_right; - found_node = Some(node_right); - } - - if let Some(found_node) = found_node { - return Some((found_node, size_to_beat)); - } else { - return None; - } - } - } - } - - unsafe fn find_fitting_leaf(&mut self, pages: usize) -> Option { - unsafe { - if let Some((node, _)) = self.find_fitting_leaf_(pages, 0, usize::MAX) { - return Some(node); - } - } - - return None; - } - - unsafe fn split_node(&mut self, node: usize) -> (usize, usize) { - unsafe { - let base_child = self.get_child(node); - (*base_child).flags |= HAS_CHILDREN_FLAG; - - self.write_child(node * 2 + 1, &BinaryBuddyTreeNode { flags: 0 }); - self.write_child(node * 2 + 2, &BinaryBuddyTreeNode { flags: 0 }); - - (node * 2 + 1, node * 2 + 2) - } - } - - unsafe fn find_node_(&mut self, start: usize, node: usize) -> Option { - unsafe { - let child = self.get_child(node); - - let (child_start, child_pages) = self.get_child_start_and_pages(node); - - if start >= child_start && start < child_start + child_pages { - if (*child).flags & HAS_CHILDREN_FLAG != 0 { - if start < child_start + child_pages / 2 { - return self.find_node_(start, node * 2 + 1); - } else { - return self.find_node_(start, node * 2 + 2); - } - } else { - if child_start == start { - return Some(node); - } else { - return None; - } - } - } - - return None; - } - } - - unsafe fn find_node(&mut self, start: usize) -> Option { - unsafe { - return self.find_node_(start, 0); - } - } - - unsafe fn try_merge(&mut self, node: usize) { - if node == 0 { - return; - } - - unsafe { - let child = self.get_child(node); - if (*child).flags & (HAS_CHILDREN_FLAG | IS_OCCUPIED_FLAG) != 0 { - return; - } - - let parent_node = (node - 1) / 2; - - let sibling; - if node % 2 == 1 { // Left child - sibling = self.get_child(node + 1); - } else { - sibling = self.get_child(node - 1); - } - - if (*sibling).flags & (HAS_CHILDREN_FLAG | IS_OCCUPIED_FLAG) == 0 { - (*self.get_child(parent_node)).flags &= !HAS_CHILDREN_FLAG; - - self.try_merge(parent_node); - } - } - } - - /// Try to allocate `pages` consecutive pages. - /// - /// * `pages`: The amount of consecutive pages to allocate. - pub unsafe fn alloc(&mut self, pages: usize) -> Option<*mut u8> { - if pages == 0 { - return None; - } - - unsafe { - if let Some(node) = self.find_fitting_leaf(pages) { - let mut child_node = node; - let mut child = self.get_child(node); - - let (mut child_start, mut child_pages) = self.get_child_start_and_pages(node); - - while child_pages / 2 >= pages { - let (next_child, _) = self.split_node(child_node); - - child_node = next_child; - child = self.get_child(child_node); - - (child_start, child_pages) = self.get_child_start_and_pages(child_node); - } - - (*child).flags |= IS_OCCUPIED_FLAG; - - return Some((child_start * PAGE_SIZE + self.base as usize) as *mut u8); - } - } - - return None; - } - - /// Try to free a region beginning at `page`. - /// - /// * `page`: The position of the region to free. - pub unsafe fn free(&mut self, page: *mut u8) -> Result<(), ()> { - debug_assert_eq!(page as usize % PAGE_SIZE, 0); - - let start = (page as usize - self.base as usize) / PAGE_SIZE; - unsafe { - if (page as usize) < (self.base as usize) || (page as usize) >= (self.base as usize) + self.pages * PAGE_SIZE { - return Err(()); - } - - if let Some(node) = self.find_node(start) { - let child = self.get_child(node); - - if (*child).flags & IS_OCCUPIED_FLAG != 0 { - (*child).flags &= !IS_OCCUPIED_FLAG; - - self.try_merge(node); - - return Ok(()); - } - } - } - - Err(()) - } -}*/ diff --git a/kernel/src/rendering.rs b/kernel/src/rendering.rs index fa031c9..ca6e63b 100644 --- a/kernel/src/rendering.rs +++ b/kernel/src/rendering.rs @@ -1,4 +1,4 @@ -#![rustfmt::skip] +/*#![rustfmt::skip] use core::fmt::Write; @@ -145,7 +145,7 @@ impl<'a> DisplayWriter<'a> { if self.col >= (self.graphics_info.w / (FONT_WIDTH + HORIZONTAL_SPACING)) { self.col = 0; - + self.advance_line(); } } @@ -195,3 +195,4 @@ impl<'a> Write for DisplayWriter<'a> { Ok(()) } } +*/ diff --git a/kernel/src/serial/mod.rs b/kernel/src/serial/mod.rs new file mode 100644 index 0000000..2fdb15d --- /dev/null +++ b/kernel/src/serial/mod.rs @@ -0,0 +1,9 @@ +cfg_select! { + target_arch = "x86_64" => { + mod uart; + pub use uart::*; + } + _ => { + compile_error!("Unsupported architecture"); + } +} diff --git a/kernel/src/serial/uart.rs b/kernel/src/serial/uart.rs new file mode 100644 index 0000000..db3c60d --- /dev/null +++ b/kernel/src/serial/uart.rs @@ -0,0 +1,69 @@ +use core::{fmt, hint::spin_loop}; + +use crate::arch::x86_64::{inb, outb}; + +#[derive(Debug, Clone, Copy)] +#[repr(u16)] +pub enum Com { + Com1 = 0x3f8, + Com2 = 0x2f8, + Com3 = 0x3e8, + Com4 = 0x2e8, +} + +pub unsafe fn serial_init(port: Com) { + let port = port as u16; + unsafe { + // Disable interrupts + outb(port + 1, 0x00); + + // Enable DLAB + outb(port + 3, 0x80); + + // Baud divisor = 3 (38400 baud assuming 115200 clock) + outb(port + 0, 0x03); + outb(port + 1, 0x00); + + // 8 bits, no parity, one stop bit + outb(port + 3, 0x03); + + // Enable FIFO, clear them, 14-byte threshold + outb(port + 2, 0xC7); + + // IRQs disabled, RTS/DSR set + outb(port + 4, 0x03); + } +} + +#[inline] +fn tx_ready(port: Com) -> bool { + unsafe { inb(port as u16 + 5) & 0x20 != 0 } +} + +pub fn serial_write_byte(port: Com, byte: u8) { + while !tx_ready(port) { + spin_loop(); + } + + unsafe { + outb(port as u16, byte); + } +} + +pub fn serial_write(port: Com, s: &str) { + for b in s.bytes() { + if b == b'\n' { + serial_write_byte(port, b'\r'); + } + + serial_write_byte(port, b); + } +} + +pub struct Serial(pub Com); +impl fmt::Write for Serial { + fn write_str(&mut self, s: &str) -> fmt::Result { + serial_write(self.0, s); + Ok(()) + } +} diff --git a/kernel/src/spin_sync/mod.rs b/kernel/src/spin_sync/mod.rs new file mode 100644 index 0000000..94de464 --- /dev/null +++ b/kernel/src/spin_sync/mod.rs @@ -0,0 +1,2 @@ +mod mutex; +pub use mutex::*; diff --git a/kernel/src/spin_sync/mutex.rs b/kernel/src/spin_sync/mutex.rs new file mode 100644 index 0000000..76b6a02 --- /dev/null +++ b/kernel/src/spin_sync/mutex.rs @@ -0,0 +1,81 @@ +use core::{ + cell::SyncUnsafeCell, + hint::spin_loop, + ops::{Deref, DerefMut}, + sync::atomic::{AtomicBool, Ordering::*}, +}; + +#[derive(Debug)] +#[repr(C)] +pub struct SpinMutex { + locked: AtomicBool, + value: SyncUnsafeCell, +} + +impl SpinMutex { + #[inline] + pub const fn new(value: T) -> Self + where T: Sized { + Self { + locked: AtomicBool::new(false), + value: SyncUnsafeCell::new(value), + } + } + + #[inline] + pub fn lock(&self) -> SpinMutexGuard<'_, T> { + loop { + match self.locked.compare_exchange_weak(false, true, Acquire, Relaxed) { + Ok(..) => { + break SpinMutexGuard { + locked: &self.locked, + value: unsafe { self.value.get().as_mut_unchecked() }, + } + } + Err(..) => spin_loop(), + } + } + } + + #[inline] + pub fn get_mut(&mut self) -> &mut T { + self.value.get_mut() + } +} + +#[derive(Debug)] +pub struct SpinMutexGuard<'a, T: ?Sized> { + locked: &'a AtomicBool, + value: &'a mut T, +} + +impl Deref for SpinMutexGuard<'_, T> { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + self.value + } +} + +impl DerefMut for SpinMutexGuard<'_, T> { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + self.value + } +} + +impl Drop for SpinMutexGuard<'_, T> { + fn drop(&mut self) { + cfg_select! { + debug_assertions => { + if !self.locked.swap(false, Release) { + unreachable!("Spin-mutex incorrectly unlocked!") + } + } + not(debug_assertions) => { + self.locked.store(false, Release); + } + } + } +} diff --git a/kernel/src/vaddr/mod.rs b/kernel/src/vaddr/mod.rs new file mode 100644 index 0000000..120a7c2 --- /dev/null +++ b/kernel/src/vaddr/mod.rs @@ -0,0 +1,63 @@ +pub(crate) use imp::*; +pub use imp::{VAddr, VirtualMap, VirtualMapBuilder}; + +cfg_select! { + target_arch = "x86_64" => { + mod x86_64; + use x86_64 as imp; + } + _ => { + compile_error!("Unsupported architecture"); + } +} + +use core::fmt; + +use bitflags::bitflags; +use elpytios_bootinfo::paddr::PAddr; + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct VFlags(usize); +bitflags! { + impl VFlags: usize { + const WRITABLE = 1 << 0; + const USER_MODE = 1 << 1; + const WRITE_THROUGH = 1 << 2; + const CACHE_DISABLED = 1 << 3; + const ACCESSED = 1 << 4; + + /// Don't flush translation lookaside buffers when switching virtual map tables + const GLOBAL = 1 << 5; + const EXECUTE_DISABLE = 1 << 6; + } +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub enum VirtualMapError { + PageTable, + AlreadyMapped { p_addr: PAddr, v_addr: VAddr, p_addr_existing: PAddr }, +} + +impl fmt::Debug for VirtualMapError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl fmt::Display for VirtualMapError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::PageTable => writeln!(f, "Couldn't allocate a page table"), + Self::AlreadyMapped { + p_addr, + v_addr, + p_addr_existing, + } => write!( + f, + "Couldn't map {v_addr:p} to {p_addr:p}: the virtual address is already mapped to {p_addr_existing:p}" + ), + } + } +} diff --git a/bootinfo/src/vaddr/x86_64/mod.rs b/kernel/src/vaddr/x86_64/mod.rs similarity index 83% rename from bootinfo/src/vaddr/x86_64/mod.rs rename to kernel/src/vaddr/x86_64/mod.rs index 4b20f7b..ad7476a 100644 --- a/bootinfo/src/vaddr/x86_64/mod.rs +++ b/kernel/src/vaddr/x86_64/mod.rs @@ -1,10 +1,9 @@ mod phys; mod virt; -pub(crate) use phys::*; +use elpytios_bootinfo::PAGE_SIZE; +pub use phys::*; pub use virt::*; -use crate::PAGE_SIZE; - const _: () = assert!(size_of::() == size_of::()); const fn assert_size_align() { diff --git a/bootinfo/src/vaddr/x86_64/phys.rs b/kernel/src/vaddr/x86_64/phys.rs similarity index 82% rename from bootinfo/src/vaddr/x86_64/phys.rs rename to kernel/src/vaddr/x86_64/phys.rs index c73a09a..2a76df4 100644 --- a/bootinfo/src/vaddr/x86_64/phys.rs +++ b/kernel/src/vaddr/x86_64/phys.rs @@ -2,9 +2,10 @@ use core::{fmt, mem}; use bitflags::bitflags; use bytemuck::Zeroable; +use elpytios_bootinfo::{PAGE_SIZE, paddr::PAddr}; use super::assert_size_align; -use crate::{PAGE_SIZE, paddr::PAddr, vaddr::VFlags}; +use crate::vaddr::VFlags; const _: () = assert_size_align::(); const _: () = assert_size_align::(); @@ -48,7 +49,9 @@ impl From for Entry { if value.contains(VFlags::ACCESSED) { out |= Self::ACCESSED } - + if value.contains(VFlags::EXECUTE_DISABLE) { + out |= Self::EXECUTE_DISABLE; + } out } } @@ -84,13 +87,13 @@ bitflags! { #[derive(Debug, Zeroable)] #[repr(C, align(4096))] pub struct Pml4Table { - pub pdpt_entries: [NodeEntry; PAGE_SIZE / size_of::()], + pub pml4_to_pdpt: [NodeEntry; PAGE_SIZE / size_of::()], } #[derive(Debug, Zeroable)] #[repr(C, align(4096))] pub struct PdptTable { - pub pd_entries: [PdptEntry; PAGE_SIZE / size_of::()], + pub pdpt_to_pd: [PdptEntry; PAGE_SIZE / size_of::()], } #[derive(Debug, Copy, Clone, Zeroable)] @@ -102,15 +105,10 @@ impl PdptLeafEntry { PAddr::new(self.0 & Self::ADDRESS_MASK.0) } - /*#[inline] + #[inline] pub const fn new(entry: Entry, addr: PAddr) -> Self { Self((entry.0 | Entry::PRESENT.0) & !Self::ADDRESS_MASK.0 | addr.addr() & Self::ADDRESS_MASK.0) } - - #[inline] - pub const fn is_present(self) -> bool { - self.0 & Entry::PRESENT.0 != 0 - }*/ } bitflags! { impl PdptLeafEntry: usize { @@ -122,6 +120,17 @@ bitflags! { } } +impl From for PdptLeafEntry { + #[inline] + fn from(value: VFlags) -> Self { + let mut out = Self::empty(); + if value.contains(VFlags::GLOBAL) { + out |= Self::GLOBAL; + } + out + } +} + #[derive(Copy, Clone)] pub enum UnionEntry { Node(Node), @@ -134,12 +143,6 @@ impl UnionEntry { let Self::Node(node) = self else { panic!("Not a node!") }; node } - - /*#[inline] - pub fn force_leaf(self) -> Leaf { - let Self::Leaf(leaf) = self else { panic!("Not a node!") }; - leaf - }*/ } #[derive(Copy, Clone, Zeroable)] @@ -162,10 +165,10 @@ impl PdptEntry { Self { node } } - /*#[inline] + #[inline] pub const fn leaf(leaf: PdptLeafEntry) -> Self { - unsafe { mem::transmute::(mem::transmute::(Self { leaf }) | NODE_IS_LEAF) } - }*/ + unsafe { mem::transmute::(mem::transmute::(Self { leaf }) | UNION_IS_LEAF) } + } #[inline] pub const fn kind(self) -> UnionEntry { @@ -178,17 +181,6 @@ impl PdptEntry { } } - /*#[inline] - pub const fn kind_mut(&mut self) -> UnionEntry<&mut NodeEntry, &mut PdptLeafEntry> { - unsafe { - if mem::transmute::(*self) & UNION_IS_LEAF != 0 { - UnionEntry::Leaf(&mut self.leaf) - } else { - UnionEntry::Node(&mut self.node) - } - } - }*/ - #[inline] pub const fn is_present(self) -> bool { unsafe { mem::transmute::(self) & Entry::PRESENT.0 != 0 } @@ -198,7 +190,7 @@ impl PdptEntry { #[derive(Zeroable)] #[repr(C, align(4096))] pub struct PdTable { - pub pt_entries: [PdEntry; PAGE_SIZE / size_of::()], + pub pd_to_pt: [PdEntry; PAGE_SIZE / size_of::()], } #[derive(Debug, Copy, Clone, Zeroable)] @@ -210,15 +202,10 @@ impl PdLeafEntry { PAddr::new(self.0 & Self::ADDRESS_MASK.0) } - /*#[inline] + #[inline] pub const fn new(entry: Entry, addr: PAddr) -> Self { Self((entry.0 | Entry::PRESENT.0) & !Self::ADDRESS_MASK.0 | addr.addr() & Self::ADDRESS_MASK.0) } - - #[inline] - pub const fn is_present(self) -> bool { - unsafe { mem::transmute::(self) & Entry::PRESENT.0 != 0 } - }*/ } bitflags! { impl PdLeafEntry: usize { @@ -230,6 +217,17 @@ bitflags! { } } +impl From for PdLeafEntry { + #[inline] + fn from(value: VFlags) -> Self { + let mut out = Self::empty(); + if value.contains(VFlags::GLOBAL) { + out |= Self::GLOBAL; + } + out + } +} + #[derive(Copy, Clone, Zeroable)] #[repr(C)] pub union PdEntry { @@ -250,10 +248,10 @@ impl PdEntry { Self { node } } - /*#[inline] + #[inline] pub const fn leaf(leaf: PdLeafEntry) -> Self { - unsafe { mem::transmute::(mem::transmute::(Self { leaf }) | NODE_IS_LEAF) } - }*/ + unsafe { mem::transmute::(mem::transmute::(Self { leaf }) | UNION_IS_LEAF) } + } #[inline] pub const fn kind(self) -> UnionEntry { @@ -266,17 +264,6 @@ impl PdEntry { } } - /*#[inline] - pub const fn kind_mut(&mut self) -> UnionEntry<&mut NodeEntry, &mut PdLeafEntry> { - unsafe { - if mem::transmute::(*self) & NODE_IS_LEAF != 0 { - UnionEntry::Leaf(&mut self.leaf) - } else { - UnionEntry::Node(&mut self.node) - } - } - }*/ - #[inline] pub const fn is_present(self) -> bool { unsafe { mem::transmute::(self) & Entry::PRESENT.0 != 0 } @@ -305,7 +292,7 @@ impl PtEntry { #[inline] pub const fn is_present(self) -> bool { - unsafe { mem::transmute::(self) & Entry::PRESENT.0 != 0 } + self.0 & Entry::PRESENT.0 != 0 } } bitflags! { @@ -317,3 +304,14 @@ bitflags! { const PROTECTION_KEY = ((1 << 4) - 1) << 59; } } + +impl From for PtEntry { + #[inline] + fn from(value: VFlags) -> Self { + let mut out = Self::empty(); + if value.contains(VFlags::GLOBAL) { + out |= Self::GLOBAL; + } + out + } +} diff --git a/kernel/src/vaddr/x86_64/virt.rs b/kernel/src/vaddr/x86_64/virt.rs new file mode 100644 index 0000000..76093c8 --- /dev/null +++ b/kernel/src/vaddr/x86_64/virt.rs @@ -0,0 +1,374 @@ +use core::fmt; + +use bytemuck::Zeroable; +use elpytios_bootinfo::{PAGE_SIZE, paddr::PAddr}; + +use crate::{ + statics::phys_to_virt, + vaddr::{ + Entry, NodeEntry, PdEntry, PdLeafEntry, PdTable, PdptEntry, PdptLeafEntry, PdptTable, Pml4Table, PtEntry, PtTable, UnionEntry, VFlags, + VirtualMapError, + }, +}; + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Zeroable)] +#[repr(transparent)] +pub struct VAddr(usize); +impl VAddr { + #[inline] + pub const fn new(virtual_address: usize) -> Self { + Self(virtual_address) + } + + #[inline] + pub const fn addr(self) -> usize { + self.0 + } + + #[inline] + pub const fn ptr(self) -> *const T { + self.0 as *const T + } + + #[inline] + pub const fn ptr_mut(self) -> *mut T { + self.0 as *mut T + } + + #[inline] + pub const fn byte_add(self, offset: usize) -> Self { + Self(self.0 + offset) + } + + #[inline] + pub(crate) const fn info(self) -> VAddrInfo { + VAddrInfo { + page_offset: self.0 & 0xfff, + pt_index: (self.0 >> 12) & 0x1ff, + pd_index: (self.0 >> 21) & 0x1ff, + pdpt_index: (self.0 >> 30) & 0x1ff, + pml4_index: (self.0 >> 39) & 0x1ff, + } + } +} + +impl From<*const T> for VAddr { + #[inline] + fn from(value: *const T) -> Self { + Self(value as usize) + } +} + +impl From<*mut T> for VAddr { + #[inline] + fn from(value: *mut T) -> Self { + Self(value as usize) + } +} + +#[derive(Debug, Clone, Copy, Zeroable)] +#[repr(C)] +pub(crate) struct VAddrInfo { + pub page_offset: usize, + pub pt_index: usize, + pub pd_index: usize, + pub pdpt_index: usize, + pub pml4_index: usize, +} + +impl fmt::Debug for VAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.info()) + } +} + +impl fmt::Pointer for VAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:#018p}", self.0 as *const ()) + } +} + +#[derive(Debug)] +#[repr(transparent)] +pub struct VirtualMapBuilder Option> { + map: VirtualMap>, +} + +impl Option> VirtualMapBuilder { + /// # Safety + /// - `page_table_new` must return a [`PAGE_SIZE`]-aligned physical address that is completely + /// free to be written to (nothing else "owns" it). + /// - `page_table_phys` must be one such pointer that satisfies to be a return value of + /// `page_table_new` + /// - `page_table_ptr` must convert physical addresses returned by `new_page_table` into a + /// pointer that points to a page table. + #[inline] + pub const unsafe fn new(page_table_phys: PAddr, page_table_new: T, page_table_ptr: unsafe fn(PAddr) -> *mut ()) -> Self { + Self { + map: VirtualMap { + mapper: LocalMapper { + page_table_phys, + page_table_new, + page_table_ptr, + }, + }, + } + } + + #[inline] + pub fn map(&mut self, p_addr: PAddr, v_addr: VAddr, page_count: usize, flags: VFlags) -> Result<(), VirtualMapError> { + unsafe { self.map.map(p_addr, v_addr, page_count, flags) } + } + + /// # Safety + /// Direct-map offset must have been set and usable by the time *any* methods in the returned + /// [`VirtualMap`] is called. + pub unsafe fn finish(self) -> VirtualMap { + VirtualMap { + mapper: unsafe { sealed::OffsetMapper::new(phys_to_virt(self.map.mapper.page_table_phys).ptr_mut()) }, + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct LocalMapper Option> { + page_table_phys: PAddr, + page_table_new: T, + page_table_ptr: unsafe fn(PAddr) -> *mut (), +} + +unsafe impl Option> sealed::VirtualMapper for LocalMapper { + #[inline] + fn new_page_table(&self) -> Option { + (self.page_table_new)().inspect(|&addr| unsafe { (self.page_table_ptr)(addr).cast::().write_bytes(0, PAGE_SIZE) }) + } + + #[inline] + unsafe fn pml4(&self) -> &mut Pml4Table { + unsafe { (self.page_table_ptr)(self.page_table_phys).cast::().as_mut_unchecked() } + } + + #[inline] + unsafe fn pdpt(&self, pml4_index: usize) -> &mut PdptTable { + unsafe { + (self.page_table_ptr)(self.pml4().pml4_to_pdpt[pml4_index].child_addr()) + .cast::() + .as_mut_unchecked() + } + } + + #[inline] + unsafe fn pd(&self, pml4_index: usize, pdpt_index: usize) -> &mut PdTable { + unsafe { + (self.page_table_ptr)(match self.pdpt(pml4_index).pdpt_to_pd[pdpt_index].kind() { + UnionEntry::Node(e) => e.child_addr(), + UnionEntry::Leaf(..) => unreachable!("PDPT entry is a huge page entry"), + }) + .cast::() + .as_mut_unchecked() + } + } + + #[inline] + unsafe fn pt(&self, pml4_index: usize, pdpt_index: usize, pd_index: usize) -> &mut PtTable { + unsafe { + (self.page_table_ptr)(match self.pd(pml4_index, pdpt_index).pd_to_pt[pd_index].kind() { + UnionEntry::Node(e) => e.child_addr(), + UnionEntry::Leaf(..) => unreachable!("PD entry is a huge page entry"), + }) + .cast::() + .as_mut_unchecked() + } + } +} + +#[derive(Debug)] +#[repr(transparent)] +// Note: From user-facing API perspective, `VirtualMap` must have no trait bounds. +pub struct VirtualMap { + mapper: T, +} + +impl VirtualMap { + /// # Safety + /// There must never be concurrent (multithreaded) calls to this method that have the same + /// virtual page occupied by `v_addr`. + pub unsafe fn map(&self, mut p_addr: PAddr, mut v_addr: VAddr, mut page_count: usize, flags: VFlags) -> Result<(), VirtualMapError> { + while page_count > 0 { + let VAddrInfo { + pt_index, + pd_index, + pdpt_index, + pml4_index, + .. + } = v_addr.info(); + + unsafe { + match &mut self.mapper.pml4().pml4_to_pdpt[pml4_index] { + e if !e.is_present() => *e = NodeEntry::new(Entry::WRITABLE, self.mapper.new_page_table().ok_or(VirtualMapError::PageTable)?), + _ => {} + } + + match &mut self.mapper.pdpt(pml4_index).pdpt_to_pd[pdpt_index] { + e if !e.is_present() => { + if pd_index == 0 && page_count >= 512 * 512 { + *e = PdptEntry::leaf(PdptLeafEntry::new(flags.into(), p_addr) | flags.into()); + p_addr = p_addr.byte_add(512 * 512 * PAGE_SIZE); + v_addr = v_addr.byte_add(512 * 512 * PAGE_SIZE); + page_count -= 512 * 512; + continue + } else { + *e = PdptEntry::node(NodeEntry::new( + Entry::WRITABLE, + self.mapper.new_page_table().ok_or(VirtualMapError::PageTable)?, + )) + } + } + e if let UnionEntry::Leaf(e) = e.kind() => { + return Err(VirtualMapError::AlreadyMapped { + p_addr, + v_addr, + p_addr_existing: e.addr(), + }) + } + _ => {} + } + + match &mut self.mapper.pd(pml4_index, pdpt_index).pd_to_pt[pd_index] { + e if !e.is_present() => { + if pt_index == 0 && page_count >= 512 { + *e = PdEntry::leaf(PdLeafEntry::new(flags.into(), p_addr) | flags.into()); + p_addr = p_addr.byte_add(512 * PAGE_SIZE); + v_addr = v_addr.byte_add(512 * PAGE_SIZE); + page_count -= 512; + continue + } else { + *e = PdEntry::node(NodeEntry::new( + Entry::WRITABLE, + self.mapper.new_page_table().ok_or(VirtualMapError::PageTable)?, + )) + } + } + e if let UnionEntry::Leaf(e) = e.kind() => { + return Err(VirtualMapError::AlreadyMapped { + p_addr, + v_addr, + p_addr_existing: e.addr(), + }) + } + _ => {} + } + + match &mut self.mapper.pt(pml4_index, pdpt_index, pd_index).phys_pages[pt_index] { + e if e.is_present() => { + return Err(VirtualMapError::AlreadyMapped { + p_addr, + v_addr, + p_addr_existing: e.addr(), + }) + } + e => { + *e = PtEntry::new(flags.into(), p_addr) | flags.into(); + p_addr = p_addr.byte_add(PAGE_SIZE); + v_addr = v_addr.byte_add(PAGE_SIZE); + page_count -= 1; + continue + } + } + } + } + + Ok(()) + } +} + +mod sealed { + use core::hint::unreachable_unchecked; + + use super::*; + use crate::statics::get_phys_alloc; + + #[allow(unused_variables, reason = "Available for implementors, not defaults")] + pub unsafe trait VirtualMapper { + fn new_page_table(&self) -> Option; + + unsafe fn pml4(&self) -> &mut Pml4Table; + + /// # Safety + /// - [`pml4_index`] must be within `0..512` (exclusive). + unsafe fn pdpt(&self, pml4_index: usize) -> &mut PdptTable; + + /// # Safety: + /// - [`Self::pdpt()`] to the given indices must return a node entry, not leaf. + /// - [`pml4_index`] and [`pdpt_index`] must be within `0..512` (exclusive). + unsafe fn pd(&self, pml4_index: usize, pdpt_index: usize) -> &mut PdTable; + + /// # Safety: + /// - [`Self::pd()`] to the given indices must return a node entry, not leaf. + /// - [`pml4_index`], [`pdpt_index`], and [`pd_index`] must be within `0..512` (exclusive). + unsafe fn pt(&self, pml4_index: usize, pdpt_index: usize, pd_index: usize) -> &mut PtTable; + } + + #[derive(Debug)] + #[repr(transparent)] + pub struct OffsetMapper { + pml4: *mut Pml4Table, + } + + impl OffsetMapper { + #[inline] + pub const unsafe fn new(pml4: *mut Pml4Table) -> Self { + Self { pml4 } + } + } + + unsafe impl VirtualMapper for OffsetMapper { + #[inline] + fn new_page_table(&self) -> Option { + get_phys_alloc() + .lock() + .alloc(0) + .ok() + .inspect(|&addr| unsafe { phys_to_virt(addr).ptr_mut::().write_bytes(0, PAGE_SIZE) }) + } + + #[inline] + unsafe fn pml4(&self) -> &mut Pml4Table { + unsafe { self.pml4.as_mut_unchecked() } + } + + #[inline] + unsafe fn pdpt(&self, pml4_index: usize) -> &mut PdptTable { + unsafe { + phys_to_virt(self.pml4().pml4_to_pdpt.get_unchecked_mut(pml4_index).child_addr()) + .ptr_mut::() + .as_mut_unchecked() + } + } + + #[inline] + unsafe fn pd(&self, pml4_index: usize, pdpt_index: usize) -> &mut PdTable { + unsafe { + phys_to_virt(match self.pdpt(pml4_index).pdpt_to_pd.get_unchecked_mut(pdpt_index).kind() { + UnionEntry::Leaf(..) => unreachable_unchecked(), + UnionEntry::Node(e) => e.child_addr(), + }) + .ptr_mut::() + .as_mut_unchecked() + } + } + + #[inline] + unsafe fn pt(&self, pml4_index: usize, pdpt_index: usize, pd_index: usize) -> &mut PtTable { + unsafe { + phys_to_virt(match self.pd(pml4_index, pdpt_index).pd_to_pt.get_unchecked_mut(pd_index).kind() { + UnionEntry::Leaf(..) => unreachable_unchecked(), + UnionEntry::Node(e) => e.child_addr(), + }) + .ptr_mut::() + .as_mut_unchecked() + } + } + } +} diff --git a/std/sys-src/mod.rs b/std/sys-src/mod.rs index c369d01..87b0c29 100644 --- a/std/sys-src/mod.rs +++ b/std/sys-src/mod.rs @@ -1,8 +1,8 @@ mod alloc; mod configure_builtins; mod helpers; -mod pal; mod personality; +mod startup; pub mod args; pub mod backtrace; @@ -26,7 +26,7 @@ pub mod thread; pub mod thread_local; pub mod time; -pub use pal::*; +pub use startup::*; /// A trait for viewing representations from std types. #[cfg_attr(not(target_os = "linux"), allow(unused))] diff --git a/std/sys-src/pal.rs b/std/sys-src/pal.rs deleted file mode 100644 index ece6ebc..0000000 --- a/std/sys-src/pal.rs +++ /dev/null @@ -1,11 +0,0 @@ -// SAFETY: must be called only once during runtime initialization. -// NOTE: this is not guaranteed to run, for example when Rust code is called externally. -pub unsafe fn init(_argc: isize, _argv: *const *const u8, _sigpipe: u8) {} - -// SAFETY: must be called only once during runtime cleanup. -// NOTE: this is not guaranteed to run, for example when the program aborts. -pub unsafe fn cleanup() {} - -pub fn abort_internal() -> ! { - core::intrinsics::abort(); -} diff --git a/std/sys-src/startup.rs b/std/sys-src/startup.rs new file mode 100644 index 0000000..b7f3f6b --- /dev/null +++ b/std/sys-src/startup.rs @@ -0,0 +1,34 @@ +use crate::arch::naked_asm; + +/// SAFETY: must be called only once during runtime initialization. +pub unsafe fn init(_argc: isize, _argv: *const *const u8, _sigpipe: u8) {} + +/// SAFETY: must be called only once during runtime cleanup. +pub unsafe fn cleanup() {} + +pub fn abort_internal() -> ! { + crate::intrinsics::abort() +} + +unsafe extern "sysv64" { + fn main(argc: i32, argv: *const *const u8) -> i32; +} + +cfg_select! { + target_arch = "x86_64" => { + #[unsafe(naked)] + #[unsafe(no_mangle)] + pub unsafe extern "sysv64" fn _start() -> ! { + naked_asm!( + "mov rsi, 0", + "mov rdi, 0", + "jmp {main}", + + main = sym main, + ) + } + } + _ => { + compile_error!("Unsupported architecture"); + } +} diff --git a/target-specs/x86_64-unknown-elpytios.json b/target-specs/x86_64-unknown-elpytios.json index 4f209ed..c18906a 100644 --- a/target-specs/x86_64-unknown-elpytios.json +++ b/target-specs/x86_64-unknown-elpytios.json @@ -3,7 +3,7 @@ "cpu": "x86-64", "crt-static-respected": true, "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", - "llvm-target": "x86_64-unknown-none", + "llvm-target": "x86_64-unknown-elpytios", "max-atomic-width": 64, "has-thread-local": false, "metadata": { @@ -13,16 +13,11 @@ "tier": 3 }, "os": "elpytios", + "executables": true, + "emit-debug-gdb-scripts": false, + "entry-abi": "sysv64", "target-pointer-width": 64, "linker-flavor": "ld.lld", "linker": "rust-lld", - "panic-strategy": "abort", - "pre-link-args": { - "gnu-cc": [ - "-m64" - ], - "gnu-lld-cc": [ - "-m64" - ] - } + "panic-strategy": "abort" } \ No newline at end of file diff --git a/user/shell/Cargo.toml b/user/shell/Cargo.toml new file mode 100644 index 0000000..acfdd7f --- /dev/null +++ b/user/shell/Cargo.toml @@ -0,0 +1,9 @@ +cargo-features = ['per-package-target'] + +[package] +name = "elpytios-shell" +version = "0.1.0" +edition = "2024" +rust-version = "1.98" + +forced-target = "target-specs/x86_64-unknown-elpytios.json" diff --git a/user/shell/src/main.rs b/user/shell/src/main.rs new file mode 100644 index 0000000..b181381 --- /dev/null +++ b/user/shell/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + loop {} +} diff --git a/x b/x index 0371129..8fb2ef1 100755 --- a/x +++ b/x @@ -283,15 +283,7 @@ def create_file_qemu(_args): runner_esp.mkdir(parents=True, exist_ok=True) -def run_qemu(args): - if not runner_fs.exists(): - create_file_qemu(args) - - accel = "tcg" - match sys.platform: - case "win32": accel = "whpx" - case "linux": accel = "kvm" - +def build_boot(args): profile = "bootloader_debug" if args.debug else "bootloader" subprocess.run( [ @@ -321,6 +313,18 @@ def run_qemu(args): check=True ) +def run_qemu(args): + profile = "bootloader_debug" if args.debug else "bootloader" + if not runner_fs.exists(): + create_file_qemu(args) + + build_boot(args) + + accel = "tcg" + match sys.platform: + case "win32": accel = "whpx" + case "linux": accel = "kvm" + runner_boot_dir.mkdir(parents=True, exist_ok=True) shutil.copy(root / "target" / "x86_64-unknown-uefi" / profile / "elpytios-bootloader.efi", runner_boot_file) @@ -328,6 +332,8 @@ def run_qemu(args): [ f"qemu-system-{platform.machine().replace("AMD64", "x86_64")}", "-accel", accel, + "-cpu", "host", + "-smp", "cores=4,threads=2,sockets=1", "-drive", f"if=pflash,format=raw,readonly=on,file={runner_ovmf / "OVMF_CODE.4m.fd"}", "-drive", f"if=pflash,format=raw,readonly=on,file={runner_ovmf / "OVMF_VARS.4m.fd"}", "-drive", f"format=raw,file=fat:rw:{runner_esp}", @@ -336,11 +342,15 @@ def run_qemu(args): "-m", "4G", "-device", "virtio-vga", "-vga", "virtio", - "-monitor", "stdio", + # Using `COM3` is the cleanest for the kernel, for some reason. + "-serial", "file:platform.log", + "-serial", "null", + "-serial", "stdio", + "-d", "int,cpu,guest_errors", "-no-reboot", "-no-shutdown", "-D", "qemu_except.log", *(["-s"] if args.debug else []), ], stdout=None, - stderr=None, + stderr=open("qemu_stderr.log", "w"), check=True ) @@ -370,6 +380,14 @@ def main(): std_sub.add_parser("build").set_defaults(func=build_std) std_sub.add_parser("clean").set_defaults(func=clean_std) + # `x boot` + boot = sub.add_parser("boot") + boot_sub = boot.add_subparsers(dest="boot_cmd") + + boot_build = boot_sub.add_parser("build") + boot_build.set_defaults(func=build_boot) + boot_build.add_argument("--debug", action="store_true") + # `x qemu` qemu = sub.add_parser("qemu") qemu.set_defaults(func=run_qemu) # Default to `x qemu run`