ARM64 port, stages 0 to 3: shared groundwork, aarch64-unknown-toyos with std and the userland, and the loader and kernel reach the PL011 on virt

Cargo.toml

@@ -37,6 +37,7 @@ members = [
     "toyos-inspect",
     "toyos-keymap",
     "toyos-ld",
+    "toyos-libc-copies",
     "toyos-logstream",
     "toyos-manifest",
     "toyos-mdns",

NOTICE

@@ -267,6 +267,32 @@ instead — which would put it inside the "comes with QEMU" allowance and delete
 6,291,456 bytes from the repository.
 
 
+
+aavmf/*.fd — EDK II firmware for QEMU `virt`, BSD-2-Clause-Patent AND Apache-2.0
+------------------------------------------------------------
+
+    AAVMF_CODE.fd  67,108,864 bytes
+      sha256 47765fe344818cbc464b1c14ae658fb4b854f5c2ceffa982411731eb4865594d
+    AAVMF_VARS.fd  67,108,864 bytes
+      sha256 b3b855c5a80310168051164986855692d1bdb06e67619856177965cd87c6774f
+
+    Copyright (c) 2019, TianoCore and contributors
+    Licence text: licenses/BSD-2-Clause-Patent-EDK2.txt
+    Copyright 1995-2023 The OpenSSL Project Authors
+    Licence text: licenses/Apache-2.0-OpenSSL.txt
+    SPDX-License-Identifier: BSD-2-Clause-Patent AND Apache-2.0
+
+QEMU's own prebuilt ArmVirtQemu firmware, unmodified: `edk2-aarch64-code.fd`
+and `edk2-arm-vars.fd` as QEMU 11.1.0 installs them under `share/qemu/`,
+whose version string reads `edk2-stable202408-prebuilt.qemu.org` (a
+`DEBUG_GCC5` build of 2024-09-12). The variable store is the template: every
+boot gives it a writable snapshot QEMU discards, because this `DEBUG` build
+asserts on a read-only store. QEMU builds it with `NETWORK_TLS_ENABLE`
+(`roms/edk2-build.config`, `[opts.common]`), and edk2-stable202408's
+ArmVirtQemu links edk2's bundled OpenSSL either way (`OpensslLib` with TLS,
+`OpensslLibCrypto` without): OpenSSL 3.0.9, whose `LICENSE.txt` is the
+Apache-2.0 text above and which carries no `NOTICE`.
+
 tests/fixtures/gbae-v0.2.0-* — gbae, MIT
 -----------------------------------------
 

bootloader/.cargo/config.toml

@@ -10,4 +10,4 @@ rustflags = [
 ]
 
 [target.x86_64-unknown-uefi]
-linker = "toyos-ld"
+linker = "toyos-ld"

bootloader/rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-targets = ["x86_64-unknown-uefi"]
+targets = ["x86_64-unknown-uefi", "aarch64-unknown-uefi"]

bootloader/src/arch/aarch64.rs

@@ -0,0 +1,138 @@
+//! AArch64: the loader's every instruction Rust has no portable spelling for.
+
+use toyos_abi::boot::KernelArgs;
+use toyos_bootmap::Typing;
+
+/// The machine the kernel image must be built for: the loader's own.
+pub const ELF_MACHINE: toyos_elf::Machine = toyos_elf::Machine::Aarch64;
+
+/// How the boot map's descriptors are encoded.
+pub use toyos_bootmap::aarch64 as encoding;
+
+/// How the boot map types memory: by firmware's map, since AArch64 has no
+/// range registers to do it and every descriptor names its own type.
+pub fn typing(write_back: &[(u64, u64)]) -> Typing<'_> {
+    Typing::ByMap(write_back)
+}
+
+/// The generic timer's virtual count, `CNTVCT_EL0`.
+pub fn counter() -> u64 {
+    let count: u64;
+    // SAFETY: reads a counter EL1 and EL2 may always read; the `ISB` keeps the
+    // read in program order.
+    unsafe { core::arch::asm!("isb", "mrs {}, cntvct_el0", out(reg) count, options(nomem, nostack, preserves_flags)) };
+    count
+}
+
+/// What the loader's report says beside the counter: its rate. Where it counts
+/// from is firmware's to say and no register here does.
+pub fn counter_origin() -> alloc::string::String {
+    let hz: u64;
+    // SAFETY: reads a register EL1 and EL2 may always read.
+    unsafe { core::arch::asm!("mrs {}, cntfrq_el0", out(reg) hz, options(nomem, nostack, preserves_flags)) };
+    alloc::format!("CNTFRQ_EL0 {hz} Hz; the counter's origin is firmware's")
+}
+
+/// What the loader says about the CPU as firmware handed it over, or why the
+/// kernel cannot run on it: entered at EL2 on a CPU without FEAT_E2H0,
+/// `HCR_EL2.E2H` is RES1, so the kernel's entry cannot clear it and every
+/// `_el1` register its drop programs would be EL2's own. Refused here, where
+/// the console still prints; the entry's read-back of `HCR_EL2` stays as the
+/// last line of defence.
+pub fn cpu_as_entered() -> Result<Option<alloc::string::String>, alloc::string::String> {
+    let current: u64;
+    // SAFETY: reads `CurrentEL`, which EL1 and above may read.
+    unsafe { core::arch::asm!("mrs {}, currentel", out(reg) current, options(nomem, nostack, preserves_flags)) };
+    let el = (current >> 2) & 0b11;
+    if el != 2 {
+        return Ok(Some(alloc::format!("CPU: entered at EL{el}")));
+    }
+    let (hcr, mmfr4): (u64, u64);
+    // SAFETY: at EL2 both are readable. `ID_AA64MMFR4_EL1` by its encoding,
+    // `S3_0_C0_C7_4`, which sits in the ID space an older CPU reads as zero.
+    unsafe {
+        core::arch::asm!("mrs {}, hcr_el2", out(reg) hcr, options(nomem, nostack, preserves_flags));
+        core::arch::asm!("mrs {}, S3_0_C0_C7_4", out(reg) mmfr4, options(nomem, nostack, preserves_flags));
+    }
+    let e2h = (hcr >> 34) & 1;
+    let e2h0 = (mmfr4 >> 24) & 0xF;
+    let state = alloc::format!("CPU: entered at EL2, HCR_EL2.E2H {e2h}, ID_AA64MMFR4_EL1.E2H0 {e2h0:#x}");
+    if e2h0 != 0 {
+        return Err(alloc::format!(
+            "{state}: REFUSED, HCR_EL2.E2H is RES1 on a CPU without FEAT_E2H0, and the kernel's \
+             drop to EL1 needs it clear"
+        ));
+    }
+    Ok(Some(alloc::format!("{state}: the kernel's entry writes E2H clear")))
+}
+
+/// The smallest data cache line on this machine, from `CTR_EL0.DminLine`.
+fn line() -> u64 {
+    let ctr: u64;
+    // SAFETY: reads `CTR_EL0`, which every level may read.
+    unsafe { core::arch::asm!("mrs {}, ctr_el0", out(reg) ctr, options(nomem, nostack, preserves_flags)) };
+    4 << ((ctr >> 16) & 0xF)
+}
+
+/// Every line of `[at, at + len)` cleaned to the point of coherency and
+/// invalidated, then `DSB SY` for their completion.
+pub fn write_back(at: u64, len: usize) {
+    let step = line();
+    let mut addr = at & !(step - 1);
+    while addr < at + len as u64 {
+        // SAFETY: `DC CIVAC` cleans and invalidates the line holding an
+        // address the caller allocated; it changes no memory's contents.
+        unsafe { core::arch::asm!("dc civac, {}", in(reg) addr, options(nostack, preserves_flags)) };
+        addr += step;
+    }
+    // SAFETY: a barrier; waits for the maintenance above to complete.
+    unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) };
+}
+
+/// AArch64 has no I/O port space: every caller checks [`pio::EXISTS`] first.
+pub mod pio {
+    pub const EXISTS: bool = false;
+
+    /// # Safety
+    /// Never called: [`EXISTS`] is false.
+    pub unsafe fn outw(_port: u16, _value: u16) {
+        unreachable!("AArch64 has no I/O port space")
+    }
+
+    pub fn inw(_port: u16) -> u16 {
+        unreachable!("AArch64 has no I/O port space")
+    }
+}
+
+/// Hand the CPU to the kernel as firmware left it — its exception level, its
+/// identity tables — at the image's physical entry, with `x0 = args`. The
+/// kernel's entry switches to the boot map (`args.boot_pml4_addr`) itself (`kernel/src/arch/aarch64/boot.rs`),
+/// because at EL2 only the kernel's own drop to EL1 can install it.
+///
+/// The image is cleaned to the point of coherency first: that entry fetches
+/// instructions with the MMU off for a few of them, straight from memory.
+///
+/// # Safety
+/// `args.boot_pml4_addr` is the boot map, `image` is the relocated kernel image, `entry_offset`
+/// is its entry point's offset in it, and `args` stays where it is until the
+/// kernel copies it.
+pub unsafe fn enter_kernel(image: (u64, u64), entry_offset: u64, args: &KernelArgs) -> ! {
+    write_back(image.0, image.1 as usize);
+    let entry = image.0 + entry_offset;
+    // SAFETY: interrupts masked for good — the kernel's vectors are not
+    // installed yet — then every instruction cache line invalidated against
+    // the image just cleaned, and a branch to its entry with `x0 = args`, the
+    // boot protocol `toyos-abi::boot` and the kernel's `_start` define.
+    unsafe {
+        core::arch::asm!(
+            "msr daifset, #0xf",
+            "ic iallu",
+            "dsb ish",
+            "isb",
+            "br {entry}",
+            entry = in(reg) entry,
+            in("x0") args as *const KernelArgs,
+            options(noreturn),
+        );
+    }
+}

bootloader/src/arch/mod.rs

@@ -0,0 +1,7 @@
+//! The machine the loader runs on, and the only part of it that knows which one.
+
+#[cfg_attr(target_arch = "x86_64", path = "x86_64.rs")]
+#[cfg_attr(target_arch = "aarch64", path = "aarch64.rs")]
+mod imp;
+
+pub use imp::*;

bootloader/src/arch/x86_64.rs

@@ -0,0 +1,119 @@
+//! x86-64: the loader's every instruction Rust has no portable spelling for.
+
+use toyos_abi::boot::KernelArgs;
+
+/// The machine the kernel image must be built for: the loader's own.
+pub const ELF_MACHINE: toyos_elf::Machine = toyos_elf::Machine::X86_64;
+
+/// How the boot map's entries are encoded.
+pub use toyos_bootmap::x86_64 as encoding;
+
+/// How the boot map types memory: by the MTRRs firmware programmed, beneath
+/// entries that select plain memory.
+pub fn typing(_write_back: &[(u64, u64)]) -> toyos_bootmap::Typing<'_> {
+    toyos_bootmap::Typing::Firmware
+}
+
+/// The time-stamp counter, which counts from reset.
+pub fn counter() -> u64 {
+    // SAFETY: RDTSC reads a counter and nothing else; every x86-64 has it.
+    unsafe { core::arch::x86_64::_rdtsc() }
+}
+
+/// What the loader's report says beside the counter: `IA32_TSC_ADJUST`,
+/// where CPUID says the CPU has it — every write to the TSC since reset is
+/// added to it (Intel SDM Vol. 3B, "Time-Stamp Counter Adjustment"), so zero
+/// is a counter firmware never wrote and the TSC is time since power-on.
+pub fn counter_origin() -> alloc::string::String {
+    let max = core::arch::x86_64::__cpuid(0).eax;
+    // Leaf 7 exists when the maximum leaf reaches it.
+    if max < 7 || core::arch::x86_64::__cpuid_count(7, 0).ebx & (1 << 1) == 0 {
+        return alloc::string::String::from("IA32_TSC_ADJUST not on this CPU");
+    }
+    let (lo, hi): (u32, u32);
+    // SAFETY: the loader runs at CPL 0, and CPUID.07H:EBX[1] says the MSR exists.
+    unsafe {
+        core::arch::asm!("rdmsr", in("ecx") 0x3bu32, out("eax") lo, out("edx") hi, options(nomem, nostack))
+    };
+    alloc::format!("IA32_TSC_ADJUST {}", ((u64::from(hi) << 32) | u64::from(lo)) as i64)
+}
+
+/// `CLFLUSH`'s line on every x86-64 part.
+const LINE: u64 = 64;
+
+/// Every line of `[at, at + len)` written back out of this CPU's caches and
+/// every other CPU's, before this returns.
+pub fn write_back(at: u64, len: usize) {
+    let mut line = at & !(LINE - 1);
+    while line < at + len as u64 {
+        // SAFETY: `CLFLUSH` writes back and invalidates the line containing the
+        // address and touches nothing else; the caller names memory it
+        // allocated, and the instruction faults on nothing a canonical address
+        // can be.
+        unsafe {
+            core::arch::asm!("clflush [{addr}]", addr = in(reg) line as *const u8, options(nostack, preserves_flags));
+        }
+        line += LINE;
+    }
+    // SAFETY: `SFENCE` orders those writebacks ahead of whatever ends this
+    // machine; it touches no memory or register.
+    unsafe { core::arch::asm!("sfence", options(nostack, preserves_flags)) };
+}
+
+/// What the loader says about the CPU as firmware handed it over, or why the
+/// kernel cannot run on it. A UEFI x86-64 loader runs in long mode at CPL 0,
+/// the state the kernel's entry takes, so there is nothing to say or refuse.
+pub fn cpu_as_entered() -> Result<Option<alloc::string::String>, alloc::string::String> {
+    Ok(None)
+}
+
+/// The I/O port space the chipset's TCO block answers in.
+pub mod pio {
+    /// Whether this architecture has an I/O port space at all.
+    pub const EXISTS: bool = true;
+
+    /// # Safety
+    /// No fault in Ring 0; the caller owns which device answers at `port` and
+    /// what the word commands it to do. `kernel/src/arch/x86_64/cpu.rs` states
+    /// the same contract for the same instruction.
+    pub unsafe fn outw(port: u16, value: u16) {
+        // SAFETY: the caller's contract.
+        unsafe {
+            core::arch::asm!("out dx, ax", in("dx") port, in("ax") value, options(nomem, nostack, preserves_flags))
+        };
+    }
+
+    /// One word from an I/O port; safe because a read has no value a caller can
+    /// get wrong, as `kernel/src/arch/x86_64/cpu.rs`'s `inw` is.
+    pub fn inw(port: u16) -> u16 {
+        let value: u16;
+        // SAFETY: one instruction into the declared output, no memory operand.
+        unsafe {
+            core::arch::asm!("in ax, dx", out("ax") value, in("dx") port, options(nomem, nostack, preserves_flags));
+        }
+        value
+    }
+}
+
+/// Switch to the boot map at `args.boot_pml4_addr` and jump to the kernel
+/// image's entry through the high half, handing it `args`.
+///
+/// # Safety
+/// The boot map identity-maps the memory this code and its stack run from and
+/// maps the kernel image at `PHYS_OFFSET`; `image` is that relocated image,
+/// `entry_offset` its entry point's offset in it, and `args` stays where it is
+/// until the kernel copies it.
+pub unsafe fn enter_kernel(image: (u64, u64), entry_offset: u64, args: &KernelArgs) -> ! {
+    let root = args.boot_pml4_addr;
+    let entry = crate::PHYS_OFFSET + image.0 + entry_offset;
+    // SAFETY: the caller's contract: the switch keeps this code and stack
+    // mapped, and the jump lands in the image it mapped.
+    unsafe { core::arch::asm!("mov cr3, {}", in(reg) root, options(nostack)) };
+    // SAFETY: `kernel.elf`'s entry point takes `&KernelArgs` in `rdi` by the boot
+    // protocol `toyos-abi::boot` and the kernel side of it define between them —
+    // `sysv64`, because this target's own `"C"` is the Microsoft convention —
+    // and this bootloader has no way to check the callee's signature, only to
+    // keep its own side of that contract.
+    let entry: extern "sysv64" fn(&KernelArgs) -> ! = unsafe { core::mem::transmute(entry) };
+    entry(args)
+}

bootloader/src/blackbox.rs

@@ -252,30 +252,11 @@ fn when(stamp: u64) -> String {
     alloc::format!("{HEAD} the record below is from the boot armed at {}", Civil::from_unix_secs(stamp).stem())
 }
 
-/// Write the page back out of this CPU's caches, and every other CPU's.
-///
-/// The kernel's `blackbox::flush` is the same loop for the same reason; the
-/// instruction is each binary's because `toyos-blackbox` forbids unsafe code,
-/// and `toyos_blackbox::CACHE_LINE` is the one decision they share.
+/// Write the page back out of every CPU's caches, as the kernel's
+/// `blackbox::flush` does and for the same reason: a reset does not write dirty
+/// lines back.
 fn flush(page: Page) {
-    let mut line = 0usize;
-    while line < BYTES {
-        // SAFETY: `CLFLUSH` writes back and invalidates the line containing the
-        // address and touches nothing else; the address is inside the page this
-        // image allocated, and the instruction faults on nothing a canonical
-        // address can be.
-        unsafe {
-            core::arch::asm!(
-                "clflush [{addr}]",
-                addr = in(reg) (page.0 + line as u64) as *const u8,
-                options(nostack, preserves_flags),
-            );
-        }
-        line += toyos_blackbox::CACHE_LINE;
-    }
-    // SAFETY: `SFENCE` orders those writebacks ahead of whatever ends this
-    // machine; it touches no memory or register.
-    unsafe { core::arch::asm!("sfence", options(nostack, preserves_flags)) };
+    crate::arch::write_back(page.0, BYTES);
 }
 
 /// A sealed [`toyos_blackbox::Fault`] as lines for the log.