✨ feat: 添加 per-CPU 预分配支持 (#62)

* ✨ feat: 添加 per-CPU 预分配支持

* ✨ feat: 优化内存布局计算，添加对对齐的调试断言

* chore(workflows): 更新 checkout action 至 v6

Author: ZR233

.github/workflows/test-real.yml

@@ -35,7 +35,7 @@ jobs:
           [ -d .git ] && git clean -ffdx || true
 
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           clean: true
 

crates/someboot/Cargo.toml

@@ -13,6 +13,7 @@ version = "0.1.10"
 efi = []
 hv = []
 mmu = []
+percpu-prealloc = []
 uspace = ["mmu"]
 
 [dependencies]

crates/someboot/src/smp/legacy.rs

@@ -0,0 +1,115 @@
+use crate::mem::{__kimage_va, __percpu, phys_to_virt, stack_size};
+
+use super::{
+    __cpu_id_list, PerCpuMeta, align_up_pow2, alloc_percpu_region, cpu_count, meta_align,
+    percpu_data_range, percpu_link_range, percpu_region_align, set_percpu_range,
+};
+
+fn meta_offset() -> usize {
+    let link_size = percpu_link_range().len();
+    let offset = align_up_pow2(link_size, meta_align());
+    debug_assert_eq!(offset % meta_align(), 0);
+    offset
+}
+
+fn stack_offset() -> usize {
+    let meta_offset = meta_offset();
+    let meta_size = core::mem::size_of::<PerCpuMeta>();
+    align_up_pow2(meta_offset + meta_size, crate::mem::page_size())
+}
+
+fn percpu_data_size() -> usize {
+    align_up_pow2(stack_offset() + stack_size(), percpu_region_align())
+}
+
+/// Per-CPU data layout:
+///
+/// | Linker percpu data | align to 0x8 | PerCpuMeta | align padding to page size | Stack |
+pub fn alloc_percpu() {
+    println!("Initializing per-CPU data");
+    let cpu_num = cpu_count();
+    let link_range = percpu_link_range();
+    let link_size = link_range.len();
+
+    let percpu_size = percpu_data_size();
+    println!("Per-CPU data one cpu size: {:#x} bytes", percpu_size);
+    let percpu_all_secondary_size = percpu_size * cpu_num;
+
+    println!(
+        "Total per-CPU data size for secondary CPUs: {:#x} bytes ({} CPUs)",
+        percpu_all_secondary_size, cpu_num
+    );
+
+    let percpu_data = alloc_percpu_region(percpu_all_secondary_size);
+
+    set_percpu_range(percpu_data, percpu_data + percpu_all_secondary_size);
+
+    unsafe {
+        core::ptr::write_bytes(phys_to_virt(percpu_data), 0, percpu_all_secondary_size);
+    }
+
+    println!(
+        "Per-CPU data allocated at {:#x} - {:#x}",
+        percpu_data_range().start,
+        percpu_data_range().end
+    );
+
+    let entry_virt = __kimage_va(super::super::entry::secondary_entry as *const () as usize);
+
+    for (idx, hard_id) in __cpu_id_list().enumerate() {
+        let cpu_percpu_start = percpu_data_range().start + idx * percpu_size;
+        println!(
+            "Initializing per-CPU RAM for CPU{idx} - hard id {hard_id:#x} @ {cpu_percpu_start:#x}"
+        );
+        unsafe {
+            core::ptr::copy_nonoverlapping(
+                phys_to_virt(link_range.start) as *const u8,
+                phys_to_virt(cpu_percpu_start),
+                link_size,
+            );
+        }
+        let meta_start = cpu_percpu_start + meta_offset();
+        let meta_va = phys_to_virt(meta_start);
+        debug_assert_eq!(meta_start % meta_align(), 0);
+        debug_assert_eq!((meta_va as usize) % meta_align(), 0);
+
+        let stack_top = cpu_percpu_start + stack_offset() + stack_size();
+        let stack_top_virt = __percpu(stack_top);
+
+        let meta = PerCpuMeta {
+            stack_top,
+            cpu_id: hard_id,
+            cpu_idx: idx,
+            stack_top_virt: stack_top_virt as _,
+            entry_virt: entry_virt as _,
+            boot_table_paddr: 0,
+            primary_table_paddr: 0,
+        };
+        unsafe {
+            *meta_va.cast::<PerCpuMeta>() = meta;
+        }
+    }
+
+    for meta in super::cpu_meta_list() {
+        println!(
+            "CPU{} - hard id {:#x}, stack top @{:#x}, stack top virt @{:#x}, entry virt @{:#x}",
+            meta.cpu_idx, meta.cpu_id, meta.stack_top, meta.stack_top_virt, meta.entry_virt
+        );
+    }
+}
+
+pub(crate) fn cpu_meta_addr(idx: usize) -> Option<usize> {
+    let base = percpu_data_range().start + idx * percpu_data_size();
+    if base >= percpu_data_range().end {
+        return None;
+    }
+    Some(base + meta_offset())
+}
+
+pub(crate) fn percpu_data_ptr(idx: usize) -> Option<*mut u8> {
+    let base = percpu_data_range().start + idx * percpu_data_size();
+    if base >= percpu_data_range().end {
+        return None;
+    }
+    Some(phys_to_virt(base))
+}

crates/someboot/src/smp/mod.rs

@@ -1,15 +1,22 @@
-use core::alloc::Layout;
-
 use kernutil::memory::MemoryType;
 
 use crate::{
     ArchTrait, DCacheOp,
     arch::Arch,
     kernel_page_table_paddr,
-    mem::{__kimage_va, __percpu, dcache_range, page_size, phys_to_virt, stack_size},
+    mem::{__percpu, dcache_range, page_size, phys_to_virt},
 };
 
 mod cpu_iter;
+#[cfg(not(feature = "percpu-prealloc"))]
+mod legacy;
+#[cfg(feature = "percpu-prealloc")]
+mod prealloc;
+
+#[cfg(not(feature = "percpu-prealloc"))]
+use legacy as layout;
+#[cfg(feature = "percpu-prealloc")]
+use prealloc as layout;
 
 static mut PERCPU_START: usize = 0;
 static mut PERCPU_END: usize = 0;
@@ -31,97 +38,8 @@ fn percpu_region_align() -> usize {
     page_size().max(meta_align())
 }
 
-fn meta_offset() -> usize {
-    let link_size = percpu_link_range().len();
-    let offset = align_up_pow2(link_size, meta_align());
-    debug_assert_eq!(offset % meta_align(), 0);
-    offset
-}
-
-/// Per-CPU data layout:
-///
-///
-/// | Linker percpu data | align to 0x8 | PerCpuMeta | align padding to page size | Stack |
 pub fn alloc_percpu() {
-    println!("Initializing per-CPU data");
-    let cpu_num = __cpu_id_list().count();
-    let link_range = percpu_link_range();
-    let link_size = link_range.len();
-
-    let percpu_size = percpu_data_size();
-    println!("Per-CPU data one cpu size: {:#x} bytes", percpu_size);
-    let percpu_all_secondary_size = percpu_size * cpu_num;
-
-    println!(
-        "Total per-CPU data size for secondary CPUs: {:#x} bytes ({} CPUs)",
-        percpu_all_secondary_size, cpu_num
-    );
-
-    unsafe { crate::mem::ram::flush_to_memory_map(MemoryType::Reserved) };
-
-    let percpu_data = unsafe {
-        crate::mem::ram::alloc_and_flush_to_memory_map(
-            Layout::from_size_align(percpu_all_secondary_size, percpu_region_align()).unwrap(),
-            MemoryType::PerCpuData,
-        )
-        .unwrap()
-    };
-
-    unsafe {
-        PERCPU_START = percpu_data;
-        PERCPU_END = PERCPU_START + percpu_all_secondary_size;
-
-        core::ptr::write_bytes(phys_to_virt(percpu_data), 0, percpu_all_secondary_size);
-    }
-
-    println!(
-        "Per-CPU data allocated at {:#x} - {:#x}",
-        unsafe { PERCPU_START },
-        unsafe { PERCPU_END }
-    );
-
-    let entry_virt = __kimage_va(super::entry::secondary_entry as *const () as usize);
-
-    for (idx, hard_id) in __cpu_id_list().enumerate() {
-        let cpu_percpu_start = percpu_data_range().start + idx * percpu_size;
-        println!(
-            "Initializing per-CPU RAM for CPU{idx} - hard id {hard_id:#x} @ {cpu_percpu_start:#x}"
-        );
-        unsafe {
-            core::ptr::copy_nonoverlapping(
-                phys_to_virt(link_range.start) as *const u8,
-                phys_to_virt(cpu_percpu_start),
-                link_size,
-            );
-        }
-        let meta_start = cpu_percpu_start + meta_offset();
-        let meta_va = phys_to_virt(meta_start);
-        debug_assert_eq!(meta_start % meta_align(), 0);
-        debug_assert_eq!((meta_va as usize) % meta_align(), 0);
-
-        let stack_top = cpu_percpu_start + stack_offset() + stack_size();
-        let stack_top_virt = __percpu(stack_top);
-
-        let meta = PerCpuMeta {
-            stack_top,
-            cpu_id: hard_id,
-            cpu_idx: idx,
-            stack_top_virt: stack_top_virt as _,
-            entry_virt: entry_virt as _,
-            boot_table_paddr: 0,
-            primary_table_paddr: 0,
-        };
-        unsafe {
-            *meta_va.cast::<PerCpuMeta>() = meta;
-        }
-    }
-
-    for meta in cpu_meta_list() {
-        println!(
-            "CPU{} - hard id {:#x}, stack top @{:#x}, stack top virt @{:#x}, entry virt @{:#x}",
-            meta.cpu_idx, meta.cpu_id, meta.stack_top, meta.stack_top_virt, meta.entry_virt
-        );
-    }
+    layout::alloc_percpu();
 }
 
 pub(crate) fn init_percpu() {
@@ -153,16 +71,6 @@ pub struct PerCpuMeta {
     pub primary_table_paddr: usize,
 }
 
-fn stack_offset() -> usize {
-    let meta_offset = meta_offset();
-    let meta_size = core::mem::size_of::<PerCpuMeta>();
-    align_up_pow2(meta_offset + meta_size, page_size())
-}
-
-fn percpu_data_size() -> usize {
-    align_up_pow2(stack_offset() + stack_size(), percpu_region_align())
-}
-
 #[allow(dead_code)]
 /// Physical RAM allocated for per-CPU data should be mapped to this virtual address range in the kernel
 pub(crate) fn percpu_range() -> core::ops::Range<usize> {
@@ -189,19 +97,11 @@ pub fn cpu_meta(idx: usize) -> Option<PerCpuMeta> {
 
 /// Physical address of cpu meta
 pub(crate) fn cpu_meta_addr(idx: usize) -> Option<usize> {
-    let base = percpu_data_range().start + idx * percpu_data_size();
-    if base >= percpu_data_range().end {
-        return None;
-    }
-    Some(base + meta_offset())
+    layout::cpu_meta_addr(idx)
 }
 
 pub fn percpu_data_ptr(idx: usize) -> Option<*mut u8> {
-    let base = percpu_data_range().start + idx * percpu_data_size();
-    if base >= percpu_data_range().end {
-        return None;
-    }
-    Some(phys_to_virt(base))
+    layout::percpu_data_ptr(idx)
 }
 
 pub fn cpu_hart_id() -> usize {
@@ -270,10 +170,6 @@ impl Iterator for CpuMetaIterMutable {
     }
 }
 
-fn percpu_data_range() -> core::ops::Range<usize> {
-    unsafe { PERCPU_START..PERCPU_END }
-}
-
 fn percpu_link_range() -> core::ops::Range<usize> {
     unsafe extern "C" {
         fn __percpu_start();
@@ -283,3 +179,26 @@ fn percpu_link_range() -> core::ops::Range<usize> {
     let end = __percpu_end as *const () as usize;
     start..end
 }
+
+fn set_percpu_range(start: usize, end: usize) {
+    unsafe {
+        PERCPU_START = start;
+        PERCPU_END = end;
+    }
+}
+
+fn percpu_data_range() -> core::ops::Range<usize> {
+    unsafe { PERCPU_START..PERCPU_END }
+}
+
+fn alloc_percpu_region(size: usize) -> usize {
+    unsafe { crate::mem::ram::flush_to_memory_map(MemoryType::Reserved) };
+
+    unsafe {
+        crate::mem::ram::alloc_and_flush_to_memory_map(
+            core::alloc::Layout::from_size_align(size, percpu_region_align()).unwrap(),
+            MemoryType::PerCpuData,
+        )
+        .unwrap()
+    }
+}