From 269b3b3039816728d24b8ab07736de68a7953758 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Wed, 12 Aug 2026 20:21:39 -0700 Subject: [PATCH 1/3] x86/multikernel: Rebuild the spawn E820 from the live grant at exec Growing an instance after its image is loaded (kerf update between load and exec) silently lost the added memory. The added regions go on the instance's region list and the manifest is rebuilt from that list on every exec, but the E820 table was baked into boot_params once, at load time. The spawn then booted with the load-time map while its manifest advertised the full grant: a 2040 MB instance came up managing 1662 MB, and the missing regions surfaced as unexplained holes in the spawn's memory map, first-fit sized and placed exactly like the update that added them. Rebuild the E820 at exec time, right after the pristine boot_params are copied into the spawn context, so the map and the manifest are derived from the same region list at the same moment. The two loaders' duplicate E820 builders collapse into the same helper, which now also refuses to emit a truncated map when the regions exceed the table's capacity instead of warning and dropping the rest. Two hardening changes in the same area: - Refuse to remove a memory region that backs a segment of the loaded image. A shrink into that range would free the pristine kernel and initrd copies out from under the next exec. - When the pool is fragmented, halve the chunk size until a piece fits instead of dropping straight to 1 MB chunks. A large grant otherwise splinters into more regions than an E820 table can ever carry. Signed-off-by: Cong Wang --- arch/x86/include/asm/multikernel.h | 5 +++ arch/x86/kernel/kexec-bzimage64.c | 60 +---------------------------- arch/x86/kernel/kexec-vmlinux.c | 48 +---------------------- arch/x86/multikernel/Makefile | 2 +- arch/x86/multikernel/e820.c | 61 ++++++++++++++++++++++++++++++ arch/x86/multikernel/spawn.c | 13 +++++++ kernel/multikernel/mem.c | 54 ++++++++++++++++++-------- 7 files changed, 121 insertions(+), 122 deletions(-) create mode 100644 arch/x86/multikernel/e820.c diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index f85b2a831aa205..de4615698967ac 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -109,6 +109,11 @@ void mk_park_cpu(void); /* True only in a spawn kernel whose boot context carries a park area */ bool mk_cpu_parkable(void); +/* Build a spawn E820 table from the instance's current memory grant */ +struct mk_instance; +struct boot_params; +int mk_e820_fill(struct mk_instance *instance, struct boot_params *params); + /* Park an offlined pool CPU (host park area, or instance context) */ void mk_pool_park_cpu(void); diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 0759f9d6b64c23..3a5b3fca1ab352 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -125,63 +125,6 @@ static int setup_e820_entries(struct boot_params *params) return 0; } -static int setup_e820_entries_multikernel(struct kimage *image, struct boot_params *params) -{ - struct mk_instance *instance = image->mk_instance; - struct mk_memory_region *region; - unsigned int nr_e820_entries = 0; - - /* - * Don't include first 1MB in e820 for multikernel spawn kernels. - * Multikernel doesn't use real-mode trampoline (init_trampoline is - * skipped), and including unmapped low memory causes sparse_init() - * to fail when trying to populate vmemmap for those sections. - * Only include the assigned memory pool regions. - * - * However, we need at least 2 e820 entries or the kernel's - * append_e820_table() will reject our map and fall back to - * creating a fake memory map that includes low memory. - * Add a small reserved region at 0x0 as a dummy entry. - */ - params->e820_table[nr_e820_entries].addr = 0; - params->e820_table[nr_e820_entries].size = 0x1000; - params->e820_table[nr_e820_entries].type = E820_TYPE_RESERVED; - nr_e820_entries++; - - if (instance && !list_empty(&instance->memory_regions)) { - list_for_each_entry(region, &instance->memory_regions, list) { - if (nr_e820_entries >= E820_MAX_ENTRIES_ZEROPAGE) { - pr_warn("E820 table full, cannot add all memory regions\n"); - break; - } - - params->e820_table[nr_e820_entries].addr = region->res.start; - params->e820_table[nr_e820_entries].size = resource_size(®ion->res); - params->e820_table[nr_e820_entries].type = E820_TYPE_RAM; - - pr_info("Added memory region to e820: 0x%llx-0x%llx (%llu MB)\n", - (unsigned long long)region->res.start, - (unsigned long long)region->res.end, - (unsigned long long)resource_size(®ion->res) >> 20); - - nr_e820_entries++; - } - } - - params->e820_entries = nr_e820_entries; - - pr_info("Final multikernel e820 map has %d total entries:\n", - nr_e820_entries); - for (int i = 0; i < nr_e820_entries; i++) { - pr_info(" e820[%d]: 0x%llx-0x%llx type=%d\n", i, - params->e820_table[i].addr, - params->e820_table[i].addr + params->e820_table[i].size, - params->e820_table[i].type); - } - - return 0; -} - enum { RNG_SEED_LENGTH = 32 }; static void @@ -804,7 +747,8 @@ static void *bzImage64_load(struct kimage *image, char *kernel, /* For multikernel, setup custom e820 map */ if (image->type == KEXEC_TYPE_MULTIKERNEL) { - ret = setup_e820_entries_multikernel(image, params); + image->arch.mk_boot_params = bootparam_load_addr; + ret = mk_e820_fill(image->mk_instance, params); if (ret) goto out_free_params; } diff --git a/arch/x86/kernel/kexec-vmlinux.c b/arch/x86/kernel/kexec-vmlinux.c index e8279270018c30..a8d9afb1d20ac6 100644 --- a/arch/x86/kernel/kexec-vmlinux.c +++ b/arch/x86/kernel/kexec-vmlinux.c @@ -306,52 +306,6 @@ static int vmlinux_probe(const char *buf, unsigned long len) return 0; } -/* - * Setup e820 memory map for multikernel spawn kernel - */ -static int setup_e820_entries_multikernel(struct kimage *image, struct boot_params *params) -{ - struct mk_instance *instance = image->mk_instance; - struct mk_memory_region *region; - unsigned int nr_e820_entries = 0; - int i; - - /* - * Only include the assigned memory pool regions for multikernel spawn. - * Don't include first 1MB - multikernel doesn't use real-mode trampoline - * and including unmapped low memory causes sparse_init() to fail. - * - * The spawn kernel uses e820__memory_setup_multikernel() which accepts - * any number of entries without fallback to legacy BIOS memory probing. - */ - if (instance && !list_empty(&instance->memory_regions)) { - list_for_each_entry(region, &instance->memory_regions, list) { - if (nr_e820_entries >= E820_MAX_ENTRIES_ZEROPAGE) { - pr_warn("Too many e820 entries, truncating\n"); - break; - } - - params->e820_table[nr_e820_entries].addr = region->res.start; - params->e820_table[nr_e820_entries].size = resource_size(®ion->res); - params->e820_table[nr_e820_entries].type = E820_TYPE_RAM; - nr_e820_entries++; - } - } - - params->e820_entries = nr_e820_entries; - - pr_info("Final multikernel e820 map has %d total entries:\n", - nr_e820_entries); - for (i = 0; i < nr_e820_entries; i++) { - pr_info(" e820[%d]: 0x%llx-0x%llx type=%d\n", i, - params->e820_table[i].addr, - params->e820_table[i].addr + params->e820_table[i].size, - params->e820_table[i].type); - } - - return 0; -} - /* * Load function - load ELF vmlinux and setup boot parameters */ @@ -606,7 +560,7 @@ static void *vmlinux_load(struct kimage *image, char *kernel, /* For multikernel, setup custom e820 map */ if (image->type == KEXEC_TYPE_MULTIKERNEL) { - ret = setup_e820_entries_multikernel(image, params); + ret = mk_e820_fill(image->mk_instance, params); if (ret) { kvfree(ldata->kernel_buf); kfree(ldata); diff --git a/arch/x86/multikernel/Makefile b/arch/x86/multikernel/Makefile index 331bd895af1f7b..5989cbbb691989 100644 --- a/arch/x86/multikernel/Makefile +++ b/arch/x86/multikernel/Makefile @@ -3,4 +3,4 @@ # Makefile for multikernel spawn support # -obj-y += spawn.o direct_boot.o head_64.o +obj-y += spawn.o direct_boot.o head_64.o e820.o diff --git a/arch/x86/multikernel/e820.c b/arch/x86/multikernel/e820.c new file mode 100644 index 00000000000000..dc78c288007dd4 --- /dev/null +++ b/arch/x86/multikernel/e820.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Spawn kernel E820 construction from an instance's memory grant. + */ +#include +#include +#include + +#include +#include +#include + +/* + * Fill @params' E820 table from @instance's current memory regions. + * + * Called by the image loaders at load time and again on every exec, + * after the pristine boot_params are copied into the spawn context. + * The grant can grow or shrink between load and exec (kerf update), + * and a map baked at load time would hand the spawn the memory it + * owned back then, silently losing the difference; the device-tree + * manifest is rebuilt at exec, so without this the two disagree. + */ +int mk_e820_fill(struct mk_instance *instance, struct boot_params *params) +{ + struct mk_memory_region *region; + unsigned int nr = 0; + u64 total = 0; + + if (!instance) + return -EINVAL; + + /* + * The first 1MB stays out: multikernel skips the real-mode + * trampoline, and unmapped low memory makes sparse_init() fail. + * A single reserved page keeps the table from being empty. + */ + params->e820_table[nr].addr = 0; + params->e820_table[nr].size = 0x1000; + params->e820_table[nr].type = E820_TYPE_RESERVED; + nr++; + + list_for_each_entry(region, &instance->memory_regions, list) { + if (nr >= E820_MAX_ENTRIES_ZEROPAGE) { + pr_err("mk_e820: instance %d has more memory regions than the E820 table can carry (%d); refusing a truncated map\n", + instance->id, E820_MAX_ENTRIES_ZEROPAGE); + return -E2BIG; + } + + params->e820_table[nr].addr = region->res.start; + params->e820_table[nr].size = resource_size(®ion->res); + params->e820_table[nr].type = E820_TYPE_RAM; + total += params->e820_table[nr].size; + nr++; + } + + params->e820_entries = nr; + + pr_info("mk_e820: instance %d: %llu MB RAM in %u regions\n", + instance->id, total >> 20, nr - 1); + return 0; +} diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index 924a3fa5137d7d..41bd71d216a586 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -364,6 +364,7 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, int cpu) { struct boot_params *src_bp; + int ret; if (!instance->ident_pgt) { struct mk_ident_pgtable *pgt; @@ -441,6 +442,18 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, sizeof(struct boot_params)); memunmap(src_bp); + /* + * The loader's E820 reflects the grant at load time; regions added + * or removed since then only exist in the live region list, which + * the manifest is also rebuilt from on every exec. Rebuild the map + * too, or the spawn boots with the memory it owned at load and the + * difference silently vanishes. + */ + ret = mk_e820_fill(instance, + mk_spawn_context_boot_params(instance->spawn_ctx)); + if (ret) + return ret; + mk_set_spawn_context(instance->spawn_ctx, mk_get_identity_cr3(instance->ident_pgt), image->arch.mk_kernel_entry, diff --git a/kernel/multikernel/mem.c b/kernel/multikernel/mem.c index 5c2da8dadb12ed..51420befc51101 100644 --- a/kernel/multikernel/mem.c +++ b/kernel/multikernel/mem.c @@ -129,22 +129,15 @@ void *multikernel_create_instance_pool(int instance_id, size_t pool_size, int mi chunk_base = multikernel_alloc(chunk_size); if (!chunk_base) { - /* If we can't get the full remaining size, try smaller chunks */ - if (chunk_size > (1024 * 1024)) { - /* Try 1MB chunks */ - chunk_size = 1024 * 1024; - chunk_base = multikernel_alloc(chunk_size); - } - - if (!chunk_base && chunk_size > (256 * 1024)) { - /* Try 256KB chunks */ - chunk_size = 256 * 1024; - chunk_base = multikernel_alloc(chunk_size); - } - - if (!chunk_base && chunk_size > (1 << min_alloc_order)) { - /* Try minimum allocation size */ - chunk_size = 1 << min_alloc_order; + /* + * Fragmented pool: halve until a piece fits. Dropping + * straight to tiny chunks would splinter the grant + * into more regions than an E820 table can carry. + */ + while (!chunk_base && + chunk_size > (1UL << min_alloc_order)) { + chunk_size = ALIGN_DOWN(chunk_size / 2, + 1UL << min_alloc_order); chunk_base = multikernel_alloc(chunk_size); } @@ -345,6 +338,27 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size) * * Returns: 0 on success, -ENOENT if region not found */ +/* Does [phys_addr, phys_addr+size) back a segment of the loaded image? */ +static bool mk_range_backs_kimage(struct mk_instance *instance, + phys_addr_t phys_addr, size_t size) +{ + struct kimage *image = instance->kimage; + unsigned long i; + + if (!image) + return false; + + for (i = 0; i < image->nr_segments; i++) { + unsigned long start = image->segment[i].mem; + unsigned long end = start + image->segment[i].memsz; + + if (phys_addr < end && start < phys_addr + size) + return true; + } + + return false; +} + int mk_instance_remove_memory_region(struct mk_instance *instance, phys_addr_t phys_addr, size_t size) { @@ -354,6 +368,14 @@ int mk_instance_remove_memory_region(struct mk_instance *instance, if (!instance) return -EINVAL; + if (mk_range_backs_kimage(instance, phys_addr, size)) { + pr_err("Refusing to remove 0x%llx-0x%llx from instance %d (%s): the loaded kernel image lives there\n", + (unsigned long long)phys_addr, + (unsigned long long)(phys_addr + size - 1), + instance->id, instance->name); + return -EBUSY; + } + list_for_each_entry_safe(region, tmp, &instance->memory_regions, list) { if (region->res.start == phys_addr && resource_size(®ion->res) == size) { From 211a2124d9b1f2c2cc35e05b7a2f121b7b2abdd3 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 23:46:02 +0300 Subject: [PATCH 2/3] x86/multikernel: inherit host timer calibration Spawn kernels cannot calibrate against host-owned PIT, PIC, or IO-APIC resources. Carry the host loops-per-jiffy, CPU and TSC frequencies, and local APIC timer calibration in the spawn boot context. Install fixed calibration callbacks before x86 timer initialization. Keep explicit command-line calibration authoritative. Signed-off-by: Nikolay Nikolaev --- arch/x86/include/asm/multikernel.h | 26 +++++++++++++++---- arch/x86/kernel/platform-quirks.c | 40 +++++++++++++++++++++++++++++- arch/x86/multikernel/spawn.c | 22 ++++++++++++++++ 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index de4615698967ac..1949b086c2d538 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -10,6 +10,8 @@ #ifndef __ASSEMBLY__ +#include +#include #include #include #include @@ -54,7 +56,7 @@ static inline int arch_cpu_from_physical_id(u64 phys_id) * from a page written by the host while running on CPUs parked by (possibly * differently built) spawn kernels. * - * The fields fall into two classes that must not be mixed up: + * The fields fall into three classes that must not be mixed up: * * - Anchor fields (self_phys, park_phys, park_cr3, ctrl_phys, * ctrl_size): the context's own identity, written once when the @@ -62,9 +64,13 @@ static inline int arch_cpu_from_physical_id(u64 phys_id) * CPU on halt or offline, so they must stay valid for the context's * whole lifetime. * - * - Dispatch fields (everything else): the wake mailbox, rewritten for - * every publication and staged into registers by the CPU that claims - * it. Reparking gets its own repark_* dispatch fields precisely so a + * - Primary boot data (bp and the calibration values appended after it): + * written before the boot CPU is released and consumed while that kernel + * initializes. Secondary and repark publications do not rewrite it. + * + * - Dispatch fields (the remaining fixed-size fields): the wake mailbox, + * rewritten for every publication and staged into registers by the CPU + * that claims it. Reparking gets its own repark_* dispatch fields so a * repark publication never overwrites the anchor: the two used to * share fields, and a repark left the anchor pointing at another * kernel's park area, which triple-faulted the next halt. @@ -91,10 +97,20 @@ struct mk_spawn_context { u32 flags; /* MK_SPAWN_F_* flags */ u32 ready; /* Signal flag */ u32 reserved; /* Padding for alignment */ - /* Variable-size struct last - size depends on kernel config */ + /* Keep all existing context offsets unchanged. */ struct boot_params bp; /* Standard x86 boot params */ + /* Optional boot data belongs after boot_params, in the zeroed tail. */ + unsigned long boot_lps; /* Host delay loops per second */ + unsigned long boot_cpu_khz; /* Host CPU frequency calibration */ + unsigned long boot_tsc_khz; /* Host TSC frequency calibration */ + unsigned long boot_apic_hz; /* Host local APIC timer frequency */ } __aligned(PAGE_SIZE); +static_assert(offsetof(struct mk_spawn_context, bp) == 144); +static_assert(offsetof(struct mk_spawn_context, boot_lps) == + 144 + sizeof(struct boot_params)); +static_assert(sizeof(struct mk_spawn_context) == 2 * PAGE_SIZE); + /* Pool park loop code, copied by the host into per-instance park pages */ extern char mk_pool_park_start[]; extern char mk_pool_park_end[]; diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index 95d2cd2ccf74f5..feea109497efde 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -14,8 +14,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -28,6 +30,35 @@ extern pmd_t *populate_extra_pmd(unsigned long vaddr); extern unsigned long orig_boot_params; #ifdef CONFIG_MULTIKERNEL +static unsigned long multikernel_cpu_khz; +static unsigned long multikernel_tsc_khz; + +static unsigned long multikernel_calibrate_cpu(void) +{ + return multikernel_cpu_khz; +} + +static unsigned long multikernel_calibrate_tsc(void) +{ + return multikernel_tsc_khz; +} + +static void __init multikernel_setup_calibration(void) +{ + phys_addr_t ctx_phys = orig_boot_params - + offsetof(struct mk_spawn_context, bp); + struct mk_spawn_context *ctx = __va(ctx_phys); + + if (ctx->self_phys != ctx_phys || !ctx->boot_tsc_khz) + return; + + multikernel_tsc_khz = ctx->boot_tsc_khz; + multikernel_cpu_khz = ctx->boot_cpu_khz ?: ctx->boot_tsc_khz; + x86_platform.calibrate_cpu = multikernel_calibrate_cpu; + x86_platform.calibrate_tsc = multikernel_calibrate_tsc; + setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ); +} + /* * Custom wakeup for multikernel spawn kernels. * Uses shared spawn table instead of realmode trampoline. @@ -105,6 +136,10 @@ static void __init multikernel_parse_smp_config(void) */ apic_update_callback(wakeup_secondary_cpu_64, multikernel_wakeup_cpu); } +#else +static inline void multikernel_setup_calibration(void) +{ +} #endif /* CONFIG_MULTIKERNEL */ void __init x86_early_init_platform_quirks(void) @@ -135,6 +170,7 @@ void __init x86_early_init_platform_quirks(void) x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; break; case X86_SUBARCH_MULTIKERNEL: + multikernel_setup_calibration(); x86_platform.legacy.devices.pnpbios = 0; x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; x86_platform.legacy.rtc = 0; @@ -175,7 +211,9 @@ void __init x86_early_init_platform_quirks(void) * the PIT - which belongs to the host - and then request * legacy IRQ0, which can never reach an instance CPU that * has neither a PIC nor an IO-APIC. Ticks come from the - * local APIC timer via setup_percpu_clockev() instead. + * local APIC timer initialized by setup_percpu_clockev(). + * Keeping global_clock_event unset bypasses LAPIC timer + * verification, whose fallback path requires legacy IRQ0. */ x86_init.timers.timer_init = x86_init_noop; x86_init.timers.wallclock_init = x86_init_noop; diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index 41bd71d216a586..8290e691257058 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include #include @@ -44,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -460,6 +463,14 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, (unsigned long)instance->trampoline_va, virt_to_phys(instance->trampoline_va), virt_to_phys(instance->park_va)); + instance->spawn_ctx->boot_lps = cpu_data(cpu).loops_per_jiffy; + if (!instance->spawn_ctx->boot_lps) + instance->spawn_ctx->boot_lps = loops_per_jiffy; + instance->spawn_ctx->boot_lps *= HZ; + instance->spawn_ctx->boot_cpu_khz = cpu_khz; + instance->spawn_ctx->boot_tsc_khz = tsc_khz; + instance->spawn_ctx->boot_apic_hz = + (unsigned long)lapic_timer_period * HZ; return mk_spawn_cpu(instance, cpu, instance->spawn_ctx); } @@ -718,6 +729,17 @@ void mk_init_boot_context(phys_addr_t ctx_phys) } mk_boot_context = ctx; + /* + * A spawn kernel cannot calibrate against legacy timers because they + * belong to the host. Reuse the selected physical CPU's delay and local + * APIC timer calibration, while keeping explicit command-line values + * authoritative. + */ + if (!preset_lpj && ctx->boot_lps) + preset_lpj = DIV_ROUND_CLOSEST_ULL(ctx->boot_lps, HZ); + if (!lapic_timer_period && ctx->boot_apic_hz) + lapic_timer_period = + DIV_ROUND_CLOSEST_ULL(ctx->boot_apic_hz, HZ); /* * The host's control area (this context, the trampoline and park From 5913818ce289e51ca315671639d1b9355b8b05f1 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Wed, 12 Aug 2026 13:51:10 +0300 Subject: [PATCH 3/3] multikernel: make IPI publication ordered and recoverable Serialize shared-ring producers with a bounded owner-aware gate. Preserve FIFO publication and recover a gate only after its producer CPU is known to be parked. The gate and slot-state protocol change the private shared transport layout. Add an exact pre-launch layout and protocol check at this boundary, then require a transport initialization acknowledgment after both rings have been validated and before marking the instance active. Fail invalid manifests and missing acknowledgments closed. A spawn started by a host without the pre-launch check validates the boot-context anchor before using any shifted field and enters a local interrupt-disabled halt loop on mismatch without trusting shared park state or touching reset and APIC hardware. Signed-off-by: Nikolay Nikolaev --- arch/x86/boot/header.S | 8 +- arch/x86/include/asm/multikernel.h | 6 +- arch/x86/include/uapi/asm/bootparam.h | 1 + arch/x86/kernel/kexec-bzimage64.c | 5 + arch/x86/kernel/kexec-vmlinux.c | 24 +- arch/x86/kernel/platform-quirks.c | 4 +- arch/x86/multikernel/head_64.S | 3 + arch/x86/multikernel/spawn.c | 132 +++++-- include/linux/multikernel.h | 84 +++- include/linux/multikernel_abi.h | 9 + kernel/kexec_core.c | 56 ++- kernel/multikernel/core.c | 133 +++++-- kernel/multikernel/instance_dt.c | 65 ++- kernel/multikernel/internal.h | 8 + kernel/multikernel/ipi.c | 547 +++++++++++++++++++++----- kernel/multikernel/manifest.c | 14 +- 16 files changed, 880 insertions(+), 219 deletions(-) create mode 100644 include/linux/multikernel_abi.h diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 9bea5a1e2c52cb..6758247c93fd7e 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -379,7 +379,13 @@ xloadflags: #define XLF56 0 #endif - .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 +#ifdef CONFIG_MULTIKERNEL +# define XLF_MK XLF_MULTIKERNEL_IPI +#else +# define XLF_MK 0 +#endif + + .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF_MK cmdline_size: .long COMMAND_LINE_SIZE-1 #length of the command line, #added with boot protocol diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index 1949b086c2d538..293f2fd60987ab 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -104,11 +104,14 @@ struct mk_spawn_context { unsigned long boot_cpu_khz; /* Host CPU frequency calibration */ unsigned long boot_tsc_khz; /* Host TSC frequency calibration */ unsigned long boot_apic_hz; /* Host local APIC timer frequency */ + u64 abi_magic; /* Validated context producer */ } __aligned(PAGE_SIZE); static_assert(offsetof(struct mk_spawn_context, bp) == 144); static_assert(offsetof(struct mk_spawn_context, boot_lps) == 144 + sizeof(struct boot_params)); +static_assert(offsetof(struct mk_spawn_context, abi_magic) == + 144 + sizeof(struct boot_params) + 4 * sizeof(unsigned long)); static_assert(sizeof(struct mk_spawn_context) == 2 * PAGE_SIZE); /* Pool park loop code, copied by the host into per-instance park pages */ @@ -163,7 +166,8 @@ void mk_set_spawn_context(struct mk_spawn_context *ctx, int mk_spawn_cpu(struct mk_instance *instance, int cpu, struct mk_spawn_context *ctx); -/* Initialize boot context tracking in spawn kernel */ +/* Validate and initialize boot context tracking in spawn kernel */ +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys); void mk_init_boot_context(phys_addr_t ctx_phys); /* Identity page table and trampoline setup */ diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h index c70be687a3ecc7..7099f7cd167dce 100644 --- a/arch/x86/include/uapi/asm/bootparam.h +++ b/arch/x86/include/uapi/asm/bootparam.h @@ -25,6 +25,7 @@ #define XLF_5LEVEL (1<<5) #define XLF_5LEVEL_ENABLED (1<<6) #define XLF_MEM_ENCRYPTION (1<<7) +#define XLF_MULTIKERNEL_IPI 0x0100 #ifndef __ASSEMBLER__ diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 3a5b3fca1ab352..5fc9c5b5904afd 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -554,6 +554,11 @@ static void *bzImage64_load(struct kimage *image, char *kernel, .buf_max = ULONG_MAX, .top_down = true }; header = (struct setup_header *)(kernel + setup_hdr_offset); + if (image->type == KEXEC_TYPE_MULTIKERNEL && + !(header->xloadflags & XLF_MULTIKERNEL_IPI)) { + pr_err("Loaded kernel lacks the required shared transport layout\n"); + return ERR_PTR(-EPROTONOSUPPORT); + } setup_sects = header->setup_sects; if (setup_sects == 0) setup_sects = 4; diff --git a/arch/x86/kernel/kexec-vmlinux.c b/arch/x86/kernel/kexec-vmlinux.c index a8d9afb1d20ac6..5a8fe67add2f2c 100644 --- a/arch/x86/kernel/kexec-vmlinux.c +++ b/arch/x86/kernel/kexec-vmlinux.c @@ -60,6 +60,12 @@ struct elf_kernel_info { unsigned long reloc_size; /* Size of relocation data */ }; +struct mk_elf_note_desc { + u64 entry; + u32 ipi_abi_version; + u32 reserved; +}; + /* * Find multikernel entry point from PT_NOTE section. * Looks for note with name "Linux" and type 0x4d4b ('MK'). @@ -93,12 +99,20 @@ static unsigned long find_multikernel_entry_note(const void *buf, size_t len, if (nhdr->n_type == 0x4d4b && nhdr->n_namesz == 6 && - nhdr->n_descsz == sizeof(u64) && + nhdr->n_descsz == sizeof(struct mk_elf_note_desc) && !memcmp(ptr + sizeof(*nhdr), "Linux", 6)) { - u64 entry = *(u64 *)(ptr + sizeof(*nhdr) + - ALIGN(nhdr->n_namesz, 4)); - pr_info("multikernel: entry=0x%llx\n", entry); - return entry; + const struct mk_elf_note_desc *desc; + + desc = ptr + sizeof(*nhdr) + + ALIGN(nhdr->n_namesz, 4); + if (desc->ipi_abi_version != MK_IPI_ABI_VERSION) { + pr_err("multikernel IPI ABI %u is not supported\n", + desc->ipi_abi_version); + return 0; + } + pr_info("multikernel: entry=0x%llx, IPI ABI=%u\n", + desc->entry, desc->ipi_abi_version); + return desc->entry; } ptr += note_size; } diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index feea109497efde..9209b77d1c54e3 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -47,9 +47,9 @@ static void __init multikernel_setup_calibration(void) { phys_addr_t ctx_phys = orig_boot_params - offsetof(struct mk_spawn_context, bp); - struct mk_spawn_context *ctx = __va(ctx_phys); + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); - if (ctx->self_phys != ctx_phys || !ctx->boot_tsc_khz) + if (!ctx || !ctx->boot_tsc_khz) return; multikernel_tsc_khz = ctx->boot_tsc_khz; diff --git a/arch/x86/multikernel/head_64.S b/arch/x86/multikernel/head_64.S index 3784147fd82f62..b726e74ff3e402 100644 --- a/arch/x86/multikernel/head_64.S +++ b/arch/x86/multikernel/head_64.S @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -273,4 +274,6 @@ SYM_CODE_END(multikernel_secondary_startup) 1: .asciz "Linux" 2: .balign 4 3: .quad multikernel_startup_64 - __START_KERNEL_map + .long MK_IPI_ABI_VERSION + .long 0 4: .balign 4 diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index 8290e691257058..adbf35ff3c2727 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -71,6 +71,7 @@ /* Set in spawn kernels: the context this kernel booted from */ static struct mk_spawn_context *mk_boot_context; +static phys_addr_t mk_boot_context_phys; /* * Spawn kernel's own trampoline for secondary CPU wakeup. @@ -84,6 +85,16 @@ static struct mk_spawn_context *mk_boot_context; */ static void *spawn_trampoline_va; static unsigned long spawn_trampoline_phys; +static bool spawn_trampoline_prepared; +static bool spawn_pool_park_prepared; +static bool spawn_park_ready; +static int spawn_park_error; + +bool mk_arch_park_ready(void) +{ + /* Pair with publication after both executable park mappings succeed. */ + return smp_load_acquire(&spawn_park_ready); +} extern char multikernel_relocate_kernel_start[]; extern char multikernel_relocate_kernel_end[]; @@ -471,6 +482,7 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, instance->spawn_ctx->boot_tsc_khz = tsc_khz; instance->spawn_ctx->boot_apic_hz = (unsigned long)lapic_timer_period * HZ; + instance->spawn_ctx->abi_magic = MK_BOOT_CONTEXT_MAGIC; return mk_spawn_cpu(instance, cpu, instance->spawn_ctx); } @@ -693,18 +705,38 @@ void __init mk_arch_register_cpu(u64 phys_id) topology_register_apic((u32)phys_id, CPU_ACPIID_INVALID, true); } -/* - * Initialize boot context tracking in spawn kernel. - * Called early during spawn kernel boot. - */ -void mk_init_boot_context(phys_addr_t ctx_phys) +static __noreturn void mk_reject_spawn_context(void) +{ + /* + * The host context layout is unknown, so neither its park state nor any + * shared context field is safe to use. Keep this CPU local and inert. An NMI + * can wake HLT, but returns to this loop with maskable interrupts still + * disabled; disable them again before every halt for defense in depth. + */ + for (;;) { + native_irq_disable(); + native_halt(); + } +} + +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys) { struct mk_spawn_context *ctx; + phys_addr_t stamped_phys; + u64 abi_magic; if (!ctx_phys) { pr_err("mk_spawn: Boot context physical address is 0!\n"); - return; + return NULL; } + if (mk_boot_context) { + if (ctx_phys != mk_boot_context_phys) + mk_reject_spawn_context(); + return mk_boot_context; + } + /* Reject an invalid derived address before mapping or dereferencing it. */ + if (!IS_ALIGNED(ctx_phys, PAGE_SIZE)) + mk_reject_spawn_context(); /* * The spawn context is in the multikernel pool which is regular RAM, @@ -721,14 +753,28 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * work and then fails much later, when this kernel shuts down and * its CPUs park on nonsense addresses. */ - if (ctx->self_phys != ctx_phys) { - pr_err("mk_spawn: Boot context at %pa is stamped %pa\n", - &ctx_phys, &ctx->self_phys); - pr_err("mk_spawn: Spawn context layout mismatch - host and spawn kernels must be built from the same source\n"); - return; - } - + stamped_phys = READ_ONCE(ctx->self_phys); + if (stamped_phys != ctx_phys) + mk_reject_spawn_context(); + abi_magic = READ_ONCE(ctx->abi_magic); + if (abi_magic != MK_BOOT_CONTEXT_MAGIC) + mk_reject_spawn_context(); + + mk_boot_context_phys = ctx_phys; mk_boot_context = ctx; + return ctx; +} + +/* + * Initialize boot context tracking in spawn kernel. + * Called early during spawn kernel boot. + */ +void mk_init_boot_context(phys_addr_t ctx_phys) +{ + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); + + if (!ctx) + return; /* * A spawn kernel cannot calibrate against legacy timers because they * belong to the host. Reuse the selected physical CPU's delay and local @@ -764,17 +810,25 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * * One physical page serves every wake path of this instance: the host * allocates it once in mk_setup_trampoline() and reuses it across - * re-spawns, and mk_prepare_trampoline() places our own trampoline copy + * re-spawns, and mk_arch_prepare_park() places our own trampoline copy * (including the secondary entry) in the same page. */ -static int __init mk_prepare_trampoline(void) +int __init mk_arch_prepare_park(void) { struct mk_spawn_context *ctx = mk_boot_context; unsigned long virt; int ret; + if (mk_arch_park_ready()) + return 0; + if (spawn_park_error) + return spawn_park_error; if (!ctx) return 0; + if (!ctx->trampoline_phys || !ctx->park_phys || !ctx->park_cr3) { + ret = -EINVAL; + goto fail; + } /* * Put our own copy of the trampoline in the page the host set @@ -782,34 +836,48 @@ static int __init mk_prepare_trampoline(void) * is entered from an offline CPU, where changing page attributes * is not allowed. */ - spawn_trampoline_phys = ctx->trampoline_phys; - spawn_trampoline_va = __va(spawn_trampoline_phys); - memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, - multikernel_relocate_kernel_end - multikernel_relocate_kernel_start); + if (!spawn_trampoline_prepared) { + spawn_trampoline_phys = ctx->trampoline_phys; + spawn_trampoline_va = __va(spawn_trampoline_phys); + memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, + multikernel_relocate_kernel_end - + multikernel_relocate_kernel_start); - /* - * Both pages are executed from the direct map, which is writable, - * so drop write before adding execute. Leaving them writable and - * executable trips the kernel's own W^X check. - */ - virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; - ret = set_memory_ro(virt, 1); - if (!ret) - ret = set_memory_x(virt, 1); - if (ret) - return ret; + /* + * Both pages are executed from the direct map, which is writable, + * so drop write before adding execute. Leaving them writable and + * executable trips the kernel's own W^X check. + */ + virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; + ret = set_memory_ro(virt, 1); + if (!ret) + ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_trampoline_prepared = true; + } /* The pool park page is entered the same way when this kernel dies */ - if (ctx->park_phys) { + if (!spawn_pool_park_prepared) { virt = (unsigned long)__va(ctx->park_phys) & PAGE_MASK; ret = set_memory_ro(virt, 1); if (!ret) ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_pool_park_prepared = true; } + /* Publish executable mappings before any reject or abort can park. */ + smp_store_release(&spawn_park_ready, true); + return 0; + +fail: + /* A partial W^X transition is not safe to retry. */ + spawn_park_error = ret; return ret; } -early_initcall(mk_prepare_trampoline); +early_initcall(mk_arch_prepare_park); /* * Add a 2MB executable mapping to a page table. diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 84a98632729401..17607e15d34347 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -14,6 +14,10 @@ #include #include #include +#include +#include + +struct pci_bus; /** * Physical CPU identifiers @@ -75,8 +79,17 @@ static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) /* IPI ring buffer size - must be power of 2 for efficient modulo */ #define MK_IPI_RING_SIZE 64 +#define MK_IPI_SLOT_EMPTY 0 +#define MK_IPI_SLOT_WRITING 1 +#define MK_IPI_SLOT_READY 2 +#define MK_IPI_SLOT_CONSUMING 3 +#define MK_IPI_SLOT_CANCELLED 4 +#define MK_IPI_ABI_MAGIC 0x4d4b495049303033ULL /* "MKIPI003" */ +#define MK_IPI_READY_TIMEOUT_MS 120000 + /* Data structure for passing parameters via IPI */ struct mk_ipi_data { + atomic_t state; u64 sender_cpu; /* Physical ID of the CPU that sent this IPI */ unsigned int type; /* User-defined type identifier */ size_t data_size; /* Size of the data */ @@ -85,9 +98,15 @@ struct mk_ipi_data { /* IPI ring buffer for queuing messages */ struct mk_ipi_ring { - atomic_t head; /* Producer index */ - atomic_t tail; /* Consumer index */ + atomic_t head; /* Producer allocation cursor */ + atomic_t tail; /* Consumer scan cursor */ struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */ + /* Appended shared ABI: do not move fields above this line. */ + atomic64_t producer_gate; /* Owner CPU and claimed slot */ + atomic_t producer_contention; /* Sends that observed a busy gate */ + atomic_t full_failures; /* Sends rejected by a full ring */ + atomic_t invalid_state; /* Invalid slot state observations */ + atomic_t cancelled_writes; /* Halted producer writes recovered */ }; /* Shared memory structures - per-instance design */ @@ -101,8 +120,47 @@ struct mk_shared_data { * CPUs the first one missed. */ u32 force_halt; + /* Appended ABI handshake; existing shared offsets stay unchanged. */ + u64 abi_magic; + u32 abi_version; + u32 abi_size; + s32 ready_instance_id; + atomic_t ready; }; +static inline void mk_ipi_ring_reset_contents(struct mk_ipi_ring *ring) +{ + unsigned int i; + + atomic_set(&ring->head, 0); + atomic_set(&ring->tail, 0); + for (i = 0; i < MK_IPI_RING_SIZE; i++) { + WRITE_ONCE(ring->entries[i].data_size, 0); + atomic_set(&ring->entries[i].state, MK_IPI_SLOT_EMPTY); + } + atomic_set(&ring->producer_contention, 0); + atomic_set(&ring->full_failures, 0); + atomic_set(&ring->invalid_state, 0); + atomic_set(&ring->cancelled_writes, 0); +} + +static inline void mk_ipi_ring_reset(struct mk_ipi_ring *ring) +{ + mk_ipi_ring_reset_contents(ring); + atomic64_set(&ring->producer_gate, 0); +} + +static inline void mk_shared_data_reset(struct mk_shared_data *shared) +{ + mk_ipi_ring_reset(&shared->ring); + WRITE_ONCE(shared->force_halt, 0); + WRITE_ONCE(shared->abi_magic, MK_IPI_ABI_MAGIC); + WRITE_ONCE(shared->abi_version, MK_IPI_ABI_VERSION); + WRITE_ONCE(shared->abi_size, sizeof(*shared)); + WRITE_ONCE(shared->ready_instance_id, -1); + atomic_set(&shared->ready, 0); +} + /* Function pointer type for IPI callbacks */ typedef void (*mk_ipi_callback_t)(struct mk_ipi_data *data, void *ctx); @@ -145,8 +203,14 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns void generic_multikernel_interrupt(void); -/* Discard everything queued in this kernel's ring (instance re-spawn) */ -void mk_ipi_ring_drop_pending(void); +int mk_ipi_shared_validate(const struct mk_shared_data *shared); +int mk_ipi_shared_mark_ready(struct mk_shared_data *shared, int instance_id); +int mk_ipi_shared_wait_ready(struct mk_shared_data *shared, int instance_id, + unsigned int timeout_ms); +int mk_ipi_shared_reset_downlink(struct mk_shared_data *shared); + +/* Recover a producer only after every CPU in @halted_cpus is parked. */ +int mk_ipi_ring_recover_halted(const struct mk_cpu_set *halted_cpus); /* * Multikernel Messaging System @@ -766,6 +830,7 @@ struct mk_instance *mk_instance_find(int mk_id); void mk_instance_put(struct mk_instance *instance); void mk_instance_set_state(struct mk_instance *instance, enum mk_instance_state state); +int mk_instance_abort_spawn(struct mk_instance *instance); /* Kimage-based access to the instance memory pool */ void *mk_kimage_alloc(struct kimage *image, size_t size, size_t align); @@ -780,6 +845,7 @@ void mk_register_cpus_from_manifest(void); /* Accept the manifest handed over at boot (spawn kernels) */ void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len); +bool mk_manifest_rejected(void); /* Build the manifest for a spawn (host, kexec path) */ int mk_manifest_finalize(struct kimage *image); @@ -837,6 +903,11 @@ static inline void mk_register_cpus_from_manifest(void) static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) { } + +static inline bool mk_manifest_rejected(void) +{ + return false; +} #endif /** @@ -844,7 +915,8 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) */ #define MK_DT_CONFIG_VERSION_1 1 #define MK_DT_CONFIG_CURRENT MK_DT_CONFIG_VERSION_1 -#define MK_FDT_COMPATIBLE "multikernel-v1" +/* Bumped whenever the shared-memory layout or message semantics change. */ +#define MK_FDT_COMPATIBLE "multikernel-v3" /** * Property Names @@ -990,6 +1062,8 @@ void mk_set_pool_cpu(int cpu, bool is_pool); /* Park the calling CPU in the pool wait loop; never returns */ void __noreturn mk_enter_pool_state(void *info); +int __init mk_arch_prepare_park(void); +bool mk_arch_park_ready(void); /* * Forcible stop of another instance's CPUs (NMI on x86). Registration diff --git a/include/linux/multikernel_abi.h b/include/linux/multikernel_abi.h new file mode 100644 index 00000000000000..9f64fe3c398de1 --- /dev/null +++ b/include/linux/multikernel_abi.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MULTIKERNEL_ABI_H +#define _LINUX_MULTIKERNEL_ABI_H + +/* Private host/spawn transport compatibility constants. */ +#define MK_IPI_ABI_VERSION 3 +#define MK_BOOT_CONTEXT_MAGIC 0x4d4b435458303032ULL /* "MKCTX002" */ + +#endif /* _LINUX_MULTIKERNEL_ABI_H */ diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 01382f40aeb724..1206eec512f3e5 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1702,6 +1702,12 @@ int multikernel_kexec_by_id(int mk_id) } instance = mk_image->mk_instance; + if (instance->state != MK_STATE_LOADED) { + pr_err("Multikernel instance %d is not loadable (state=%d)\n", + mk_id, instance->state); + rc = -EINVAL; + goto unlock; + } if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); @@ -1756,10 +1762,11 @@ int multikernel_kexec_by_id(int mk_id) } rc = mk_manifest_finalize(mk_image); - if (rc) - pr_warn("Manifest finalization failed: %d\n", rc); - else - pr_info("Manifest finalized for multikernel instance\n"); + if (rc) { + pr_err("Manifest finalization failed: %d\n", rc); + goto unlock; + } + pr_info("Manifest finalized for multikernel instance\n"); /* * Point at the ring this image actually carries. Every load @@ -1775,26 +1782,33 @@ int multikernel_kexec_by_id(int mk_id) } /* - * Start the instance with an empty ring. It outlives the kernel - * that was using it, so a new instance would otherwise inherit that - * kernel's indices and any slot it left half written - which stalls - * the reader, since an unpublished slot means "the sender is still - * filling this one". Anything left in there was addressed to a - * kernel that is gone. - */ - if (instance->ipi_data) - memset(instance->ipi_data, 0, sizeof(*instance->ipi_data)); - - /* - * Same for the other direction: whatever the halted instance left - * queued for us is addressed from a kernel that no longer exists, - * and a slot it claimed but never published stalls our ring for - * good. + * Start the instance with an empty downlink after its CPUs have been + * confirmed parked. The host is the only producer for this ring, so no + * publisher can race the reset once the old receiver is quiesced. */ - mk_ipi_ring_drop_pending(); - + if (instance->ipi_data) { + rc = mk_ipi_shared_reset_downlink(instance->ipi_data); + if (rc) { + pr_err("Failed to reset instance %d IPI downlink: %d\n", + mk_id, rc); + goto unlock; + } + } rc = mk_arch_spawn_instance(mk_image, instance, cpu); if (rc == 0) { + rc = mk_ipi_shared_wait_ready(instance->ipi_data, mk_id, + MK_IPI_READY_TIMEOUT_MS); + if (rc) { + int abort_ret; + + pr_err("Instance %d did not acknowledge IPI ABI %u: %d\n", + mk_id, MK_IPI_ABI_VERSION, rc); + abort_ret = mk_instance_abort_spawn(instance); + if (abort_ret) + pr_crit("Instance %d IPI ABI timeout abort failed: %d\n", + mk_id, abort_ret); + goto unlock; + } rc = mk_instance_set_kexec_active(mk_image->mk_id); if (rc) pr_warn("Failed to set instance %d as active: %d\n", mk_image->mk_id, rc); diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 0a40038a954773..5712ae986b30e0 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -286,6 +286,7 @@ bool multikernel_allow_emergency_restart(void) */ int mk_instance_confirm_parked(struct mk_instance *instance) { + struct mk_cpu_set *snapshot; mk_phys_cpu_t phys_cpu; unsigned int i; int ret, failed = 0; @@ -293,8 +294,22 @@ int mk_instance_confirm_parked(struct mk_instance *instance) /* Never started, so nothing of it is running */ if (!instance->spawn_ctx) return 0; + if (!instance->cpus_on_slot) { + pr_err("Instance %d (%s): missing parked-CPU tracking for a started instance\n", + instance->id, instance->name); + return -EINVAL; + } - mk_cpu_set_for_each(i, phys_cpu, instance->cpus_on_slot) { + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + ret = mk_cpu_set_copy(snapshot, instance->cpus_on_slot); + if (ret) { + mk_cpu_set_free(snapshot); + return ret; + } + + mk_cpu_set_for_each(i, phys_cpu, snapshot) { ret = mk_arch_confirm_parked(instance, phys_cpu); if (ret) { pr_err("Instance %d (%s): CPU %llu is not parked: %d\n", @@ -302,6 +317,15 @@ int mk_instance_confirm_parked(struct mk_instance *instance) failed++; } } + if (!failed) { + ret = mk_ipi_ring_recover_halted(snapshot); + if (ret) { + pr_err("Instance %d (%s): failed to recover halted IPI producer: %d\n", + instance->id, instance->name, ret); + failed++; + } + } + mk_cpu_set_free(snapshot); return failed ? -EBUSY : 0; } @@ -1386,38 +1410,16 @@ int multikernel_halt_by_id(int mk_id) return ret; } -/** - * multikernel_force_halt_by_id - Forcible shutdown of a multikernel instance via NMI - * @mk_id: Instance ID to halt - * - * Forces a spawn kernel's CPUs to stop by arming the force-halt marker - * in the instance's shared IPI area and sending NMIs directly to each - * CPU. The NMI handler tests the marker and parks the CPU in the pool. - * - * No message is queued and no doorbell is rung: a ring message is - * consumed by the instance's ordinary interrupt path, which on a - * responsive kernel races the NMIs for it and can leave them with - * nothing to act on. The marker is host-owned and survives until the - * instance is re-executed, so the NMIs act on it regardless of timing. - * - * Use when: The spawn kernel is stuck/crashed and not responding to graceful - * shutdown, or when graceful shutdown has failed. May be repeated: an - * already-halted instance absorbs the NMIs in the park loop, so a rerun - * only rescues CPUs an earlier halt missed. - * - * Returns: 0 on success, negative error code on failure - */ -int multikernel_force_halt_by_id(int mk_id) +static int __mk_instance_force_halt(struct mk_instance *instance, + bool allow_loaded) { - struct mk_instance *instance; mk_phys_cpu_t phys_cpu; unsigned int i; int cpu_count = 0; int ret; - instance = mk_instance_find(mk_id); if (!instance) - return -ENOENT; + return -EINVAL; /* * LOADED is allowed for the retry case: a previous halt already @@ -1426,20 +1428,19 @@ int multikernel_force_halt_by_id(int mk_id) * a rerun the instance is stuck for good. */ if (instance->state != MK_STATE_ACTIVE && - instance->state != MK_STATE_LOADED) { + (!allow_loaded || instance->state != MK_STATE_LOADED)) { pr_err("Instance %d not running (state=%d), nothing to force halt\n", - mk_id, instance->state); - mk_instance_put(instance); + instance->id, instance->state); return -EINVAL; } if (mk_cpu_set_empty(instance->cpus)) { - pr_err("Instance %d has no CPUs assigned\n", mk_id); - mk_instance_put(instance); + pr_err("Instance %d has no CPUs assigned\n", instance->id); return -EINVAL; } - pr_info("Force halting multikernel instance %d via NMI\n", mk_id); + pr_info("Force halting multikernel instance %d via NMI\n", + instance->id); ret = mk_arm_force_halt(instance); if (ret) @@ -1451,18 +1452,61 @@ int multikernel_force_halt_by_id(int mk_id) cpu_count++; } - pr_info("Sent NMI to %d CPUs in instance %d\n", cpu_count, mk_id); + pr_info("Sent NMI to %d CPUs in instance %d\n", + cpu_count, instance->id); + + ret = mk_instance_confirm_parked(instance); + if (ret) { + pr_err("Instance %d CPUs did not park after force halt: %d\n", + instance->id, ret); + return ret; + } - /* - * The NMI handler parks each CPU on the instance's context. Wait - * for them to arrive before reporting the instance re-spawnable, - * exactly as the graceful path does after its shutdown ACK. - */ mk_instance_settle_halted(instance); - mk_instance_put(instance); return 0; } +int mk_instance_abort_spawn(struct mk_instance *instance) +{ + int ret; + + ret = __mk_instance_force_halt(instance, true); + if (ret && instance) + mk_instance_set_state(instance, MK_STATE_FAILED); + return ret; +} + +/** + * mk_instance_force_halt - Forcibly stop an instance via NMI + * @instance: Instance to stop + * + * Forces a spawn kernel's CPUs to stop by arming the persistent force-halt + * marker and sending NMIs directly to each CPU. The NMI handler checks the + * marker and parks the CPU if it is set. + * + * Use when: The spawn kernel is stuck/crashed and not responding to graceful + * shutdown, or when graceful shutdown has failed. + * + * Returns: 0 on success, negative error code on failure + */ +int mk_instance_force_halt(struct mk_instance *instance) +{ + return __mk_instance_force_halt(instance, false); +} + +int multikernel_force_halt_by_id(int mk_id) +{ + struct mk_instance *instance; + int ret; + + instance = mk_instance_find(mk_id); + if (!instance) + return -ENOENT; + ret = mk_instance_force_halt(instance); + mk_instance_put(instance); + return ret; +} + static int __init multikernel_init(void) { int ret; @@ -1504,6 +1548,17 @@ static int __init multikernel_init(void) return ret; } + ret = mk_ipi_shared_mark_ready(root_instance->ipi_data, + root_instance->id); + if (ret < 0) { + pr_err("Failed to publish multikernel IPI readiness: %d\n", ret); + mk_kernfs_cleanup(); + mk_hotplug_cleanup(); + mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); + mk_messaging_cleanup(); + return ret; + } + pr_info("Multikernel support initialized\n"); return 0; } diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index f21cc947149f10..c3351017a86338 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "internal.h" #define PROP_SUB_FDT "fdt" @@ -34,6 +35,23 @@ struct mk_instance *root_instance = NULL; EXPORT_SYMBOL_GPL(root_instance); +static void __init __noreturn mk_manifest_reject_and_park(int error) +{ + int ret; + + ret = mk_arch_prepare_park(); + if (ret || !mk_arch_park_ready()) + panic("multikernel: rejected manifest before park path became ready"); + ret = mk_register_stop_nmi_handler(); + if (ret) + pr_emerg("multikernel: stop-NMI registration failed while rejecting manifest: %d\n", + ret); + pr_emerg("multikernel: parking CPUs after rejecting supplied manifest: %d\n", + error); + smp_call_function(mk_enter_pool_state, NULL, 0); + mk_enter_pool_state(NULL); +} + /* * Collect every CPU the instance might receive through hotplug later: * the unassigned pool plus every other kernel's CPUs (the host's and @@ -386,6 +404,7 @@ static struct mk_instance * __init alloc_mk_instance(int instance_id, const char pr_err("Failed to allocate IPI buffer for instance %d\n", instance_id); goto err_free_name; } + mk_shared_data_reset(instance->ipi_data); instance->ipi_phys = virt_to_phys(instance->ipi_data); instance->ipi_pages = (sizeof(struct mk_shared_data) + PAGE_SIZE - 1) / PAGE_SIZE; @@ -543,6 +562,12 @@ static int __init mk_restore_instance_ipi(const void *manifest, struct mk_instan (unsigned long long)ipi_phys, ipi_pages); return 0; } + if (ipi_size < sizeof(struct mk_shared_data)) { + pr_err("IPI buffer is too small for ABI %u: %zu < %zu\n", + MK_IPI_ABI_VERSION, ipi_size, + sizeof(struct mk_shared_data)); + return -EPROTO; + } instance->ipi_data = memremap(ipi_phys, ipi_size, MEMREMAP_WB); if (!instance->ipi_data) { @@ -591,6 +616,12 @@ static struct mk_instance * __init mk_restore_host_instance(const void *manifest (unsigned long long)host_ipi_phys, host_ipi_pages); return NULL; } + if (host_ipi_size < sizeof(struct mk_shared_data)) { + pr_err("Host IPI buffer is too small for ABI %u: %zu < %zu\n", + MK_IPI_ABI_VERSION, host_ipi_size, + sizeof(struct mk_shared_data)); + return NULL; + } host_instance = alloc_mk_instance(0, "", false); if (!host_instance) @@ -645,6 +676,9 @@ int __init mk_instance_restore_from_manifest(void) const void *manifest = NULL; phys_addr_t fdt_phys; + if (mk_manifest_rejected()) + mk_manifest_reject_and_park(-EPROTO); + fdt_phys = mk_manifest_phys(); if (!fdt_phys) { pr_info("No manifest available for multikernel DTB restoration\n"); @@ -682,15 +716,15 @@ int __init mk_instance_restore_from_manifest(void) int mk_node = fdt_subnode_offset(manifest, 0, "multikernel"); if (mk_node < 0) { - pr_info("No multikernel node found in manifest\n"); - ret = 0; + pr_err("No multikernel node found in supplied manifest\n"); + ret = -EINVAL; goto cleanup_fdt; } const void *dtb_data = fdt_getprop(manifest, mk_node, "dtb-data", &dtb_len); if (!dtb_data || dtb_len <= 0) { - pr_info("No dtb-data property found in multikernel node\n"); - ret = 0; + pr_err("No dtb-data property found in multikernel node\n"); + ret = -EINVAL; goto cleanup_fdt; } @@ -787,8 +821,25 @@ int __init mk_instance_restore_from_manifest(void) host_instance = mk_restore_host_instance(manifest); if (!host_instance) - pr_warn("Failed to restore host instance (spawn→host communication unavailable)\n"); - + mk_manifest_reject_and_park(-ENODEV); + + ret = mk_ipi_shared_validate(instance->ipi_data); + if (ret) + mk_manifest_reject_and_park(ret); + ret = mk_ipi_shared_validate(host_instance->ipi_data); + if (ret) + mk_manifest_reject_and_park(ret); + if (!atomic_read_acquire(&host_instance->ipi_data->ready) || + READ_ONCE(host_instance->ipi_data->ready_instance_id) != 0) + mk_manifest_reject_and_park(-EHOSTDOWN); + ret = mk_arch_prepare_park(); + if (ret) + mk_manifest_reject_and_park(ret); + if (!mk_arch_park_ready()) + mk_manifest_reject_and_park(-EIO); + ret = mk_register_stop_nmi_handler(); + if (ret) + mk_manifest_reject_and_park(ret); pr_info("Successfully restored multikernel root instance %d ('%s') from manifest (%d bytes)\n", instance_id, instance_name, dtb_len); mk_dt_config_free(&config); @@ -822,6 +873,8 @@ int __init mk_instance_restore_from_manifest(void) kfree(dtb_virt); cleanup_fdt: early_memunmap((void *)manifest, PAGE_SIZE); + if (ret) + mk_manifest_reject_and_park(ret); return ret; } diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 30a19ae3f23645..0d11f466c8141a 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -5,6 +5,14 @@ extern struct idr mk_instance_idr; extern struct list_head mk_instance_list; extern struct mk_instance *root_instance; +/* core.c */ +int mk_instance_force_halt(struct mk_instance *instance); + +/* ipi.c */ +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type); +void mk_poll_ipi_messages(void); + /* kernfs.c */ extern struct kernfs_node *mk_root_kn; extern struct kernfs_node *mk_instances_kn; diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 749500ff1f6a2f..69231f1cf65b8c 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "internal.h" /* Callback management */ @@ -20,41 +21,367 @@ static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lo static void mk_ipi_drain_ring(void); +#define MK_IPI_PRODUCER_RETRIES 10000 +#define MK_IPI_GATE_INDEX_BITS 6 +#define MK_IPI_GATE_INDEX_MASK (MK_IPI_RING_SIZE - 1) + /* - * Ring indices live in memory another kernel instance can write, so every - * read is masked before it indexes the entry array. An instance that dies - * mid-update must not be able to walk this kernel off the end of its ring. + * A nonzero gate records both the physical producer CPU and the slot at head. + * This makes the serialization recoverable after that exact CPU is confirmed + * parked. A boolean shared lock would be unsafe because the force-stop NMI may + * prevent its owner from ever returning to release it. */ -static inline unsigned int mk_ring_idx(unsigned int i) +static u64 mk_ipi_gate_token(mk_phys_cpu_t owner, unsigned int idx) +{ + BUILD_BUG_ON(BIT(MK_IPI_GATE_INDEX_BITS) != MK_IPI_RING_SIZE); + if (owner >= (U64_MAX >> MK_IPI_GATE_INDEX_BITS)) + return 0; + + return ((owner + 1) << MK_IPI_GATE_INDEX_BITS) | idx; +} + +static mk_phys_cpu_t mk_ipi_gate_owner(u64 token) +{ + return (token >> MK_IPI_GATE_INDEX_BITS) - 1; +} + +static unsigned int mk_ipi_gate_index(u64 token) +{ + return token & MK_IPI_GATE_INDEX_MASK; +} + +/* + * Serialize producers with preemption disabled so the physical owner encoded + * in the gate remains stable. Keep local IRQs enabled while waiting for a + * producer in another kernel, then disable them only for the short publish + * critical section. No NMI path sends general messages; force halt uses a + * persistent host-owned marker. + * Advancing head before READY lets recovery distinguish both interruption + * windows without allowing another producer to pass the gate. + */ +static int mk_ipi_ring_publish(struct mk_shared_data *shared, int instance_id, + const void *data, size_t data_size, + unsigned long type) +{ + struct mk_ipi_ring *ring = &shared->ring; + struct mk_ipi_data *slot; + mk_phys_cpu_t owner; + unsigned long flags; + bool contended = false; + unsigned int retry; + unsigned int idx; + u64 token, old; + int state; + int head; + int ret; + + preempt_disable(); + owner = arch_cpu_physical_id(smp_processor_id()); + + for (retry = 0; retry < MK_IPI_PRODUCER_RETRIES; retry++) { + head = atomic_read(&ring->head); + idx = head & MK_IPI_GATE_INDEX_MASK; + token = mk_ipi_gate_token(owner, idx); + if (!token) { + ret = -EOVERFLOW; + goto out_enable; + } + + old = atomic64_cmpxchg_acquire(&ring->producer_gate, 0, token); + if (!old) { + if ((atomic_read(&ring->head) & + MK_IPI_GATE_INDEX_MASK) != idx) { + atomic64_set_release(&ring->producer_gate, 0); + contended = true; + cpu_relax(); + continue; + } + break; + } + contended = true; + if (mk_ipi_gate_owner(old) == owner) { + ret = -EDEADLK; + goto out_count_contention; + } + cpu_relax(); + } + + if (retry == MK_IPI_PRODUCER_RETRIES) { + ret = -EAGAIN; + goto out_count_contention; + } + + local_irq_save(flags); + if (contended) + atomic_inc(&ring->producer_contention); + if (!atomic_read_acquire(&shared->ready) || + READ_ONCE(shared->ready_instance_id) != instance_id) { + ret = -ESHUTDOWN; + goto out_release_gate; + } + + slot = &ring->entries[idx]; + state = atomic_cmpxchg(&slot->state, MK_IPI_SLOT_EMPTY, + MK_IPI_SLOT_WRITING); + if (state != MK_IPI_SLOT_EMPTY) { + if (state == MK_IPI_SLOT_READY || + state == MK_IPI_SLOT_CONSUMING || + state == MK_IPI_SLOT_CANCELLED) { + atomic_inc(&ring->full_failures); + ret = -ENOSPC; + } else { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + goto out_release_gate; + } + + WRITE_ONCE(slot->data_size, 0); + WRITE_ONCE(slot->sender_cpu, owner); + WRITE_ONCE(slot->type, type); + if (data_size) + memcpy(slot->buffer, data, data_size); + WRITE_ONCE(slot->data_size, data_size); + atomic_set(&ring->head, (idx + 1) & MK_IPI_GATE_INDEX_MASK); + atomic_set_release(&slot->state, MK_IPI_SLOT_READY); + ret = 0; + +out_release_gate: + atomic64_set_release(&ring->producer_gate, 0); + local_irq_restore(flags); +out_enable: + preempt_enable(); + return ret; + +out_count_contention: + atomic_inc(&ring->producer_contention); + goto out_enable; +} + +static bool mk_ipi_slot_is_pending(int state) { - return i & (MK_IPI_RING_SIZE - 1); + return state == MK_IPI_SLOT_READY || + state == MK_IPI_SLOT_CANCELLED; +} + +static void mk_ipi_slot_release(struct mk_ipi_data *slot) +{ + WRITE_ONCE(slot->data_size, 0); + atomic_set_release(&slot->state, MK_IPI_SLOT_EMPTY); +} + +int mk_ipi_shared_validate(const struct mk_shared_data *shared) +{ + if (!shared) + return -ENODEV; + if (READ_ONCE(shared->abi_magic) != MK_IPI_ABI_MAGIC || + READ_ONCE(shared->abi_version) != MK_IPI_ABI_VERSION || + READ_ONCE(shared->abi_size) != sizeof(*shared)) + return -EPROTO; + + return 0; +} + +int mk_ipi_shared_mark_ready(struct mk_shared_data *shared, int instance_id) +{ + int ret; + + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + + WRITE_ONCE(shared->ready_instance_id, instance_id); + atomic_set_release(&shared->ready, 1); + return 0; +} + +int mk_ipi_shared_wait_ready(struct mk_shared_data *shared, int instance_id, + unsigned int timeout_ms) +{ + unsigned long deadline; + int ret; + + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + + deadline = jiffies + msecs_to_jiffies(timeout_ms); + do { + if (atomic_read_acquire(&shared->ready)) + return READ_ONCE(shared->ready_instance_id) == instance_id ? + 0 : -EPROTO; + msleep(20); + } while (time_before(jiffies, deadline)); + + return -ETIMEDOUT; +} + +int mk_ipi_shared_reset_downlink(struct mk_shared_data *shared) +{ + struct mk_ipi_ring *ring; + mk_phys_cpu_t owner; + unsigned long flags; + unsigned int retry; + unsigned int idx; + u64 token, old; + int head; + int ret = 0; + + if (!shared) + return -EINVAL; + + /* Exclude new publishers before waiting for an in-flight one. */ + atomic_set_release(&shared->ready, 0); + /* Pair exclusion with the readiness recheck after gate acquisition. */ + smp_mb(); + ring = &shared->ring; + preempt_disable(); + owner = arch_cpu_physical_id(smp_processor_id()); + for (retry = 0; retry < MK_IPI_PRODUCER_RETRIES; retry++) { + head = atomic_read(&ring->head); + idx = head & MK_IPI_GATE_INDEX_MASK; + token = mk_ipi_gate_token(owner, idx); + if (!token) { + ret = -EOVERFLOW; + goto out_enable; + } + old = atomic64_cmpxchg_acquire(&ring->producer_gate, 0, token); + if (!old) + break; + if (mk_ipi_gate_owner(old) == owner) { + ret = -EDEADLK; + goto out_enable; + } + cpu_relax(); + } + if (retry == MK_IPI_PRODUCER_RETRIES) { + ret = -EAGAIN; + goto out_enable; + } + + local_irq_save(flags); + /* The old receiver is parked and every pre-existing publisher drained. */ + mk_ipi_ring_reset_contents(ring); + WRITE_ONCE(shared->force_halt, 0); + WRITE_ONCE(shared->abi_magic, MK_IPI_ABI_MAGIC); + WRITE_ONCE(shared->abi_version, MK_IPI_ABI_VERSION); + WRITE_ONCE(shared->abi_size, sizeof(*shared)); + WRITE_ONCE(shared->ready_instance_id, -1); + atomic_set(&shared->ready, 0); + atomic64_set_release(&ring->producer_gate, 0); + + local_irq_restore(flags); +out_enable: + preempt_enable(); + return ret; } /** - * mk_ipi_ring_drop_pending - Discard everything queued in this kernel's ring + * mk_ipi_ring_recover_halted - Recover a ring producer after it is parked + * @halted_cpus: Exact set of CPUs confirmed parked by the caller * - * Called when an instance is re-spawned. A halting instance parks its CPUs - * wherever they were, including between claiming a ring slot and publishing - * it, and the drain stops at such a slot forever. Anything still queued was - * sent by a kernel that is gone, so drop it all rather than let one - * abandoned slot wedge the ring. + * A force-stop NMI can park a producer while it owns the shared gate. Only its + * receiver may recover the write, and only after the owner CPU is proven not + * to be executing it. Other instances may still publish into the host ring, + * so this function never resets the ring or touches another owner's gate. */ -void mk_ipi_ring_drop_pending(void) +int mk_ipi_ring_recover_halted(const struct mk_cpu_set *halted_cpus) { + struct mk_ipi_data *slot; struct mk_ipi_ring *ring; - unsigned int head, tail; - - if (!root_instance || !root_instance->ipi_data) - return; + mk_phys_cpu_t owner; + mk_phys_cpu_t target; + unsigned int idx; + unsigned int next; + u64 token; + int state; + int head; + int ret = 0; + + if (!halted_cpus || !root_instance || !root_instance->ipi_data) + return -EINVAL; ring = &root_instance->ipi_data->ring; - head = mk_ring_idx(atomic_read(&ring->head)); + token = atomic64_read_acquire(&ring->producer_gate); + if (!token) + goto kick; + if (!(token >> MK_IPI_GATE_INDEX_BITS)) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + goto kick; + } - for (tail = mk_ring_idx(atomic_read(&ring->tail)); tail != head; - tail = mk_ring_idx(tail + 1)) - ring->entries[tail].data_size = 0; + owner = mk_ipi_gate_owner(token); + if (!mk_cpu_set_contains(halted_cpus, owner)) + goto kick; + + idx = mk_ipi_gate_index(token); + next = (idx + 1) & MK_IPI_GATE_INDEX_MASK; + slot = &ring->entries[idx]; + state = atomic_read_acquire(&slot->state); + head = atomic_read(&ring->head) & MK_IPI_GATE_INDEX_MASK; + switch (state) { + case MK_IPI_SLOT_EMPTY: + if (head != idx && head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + case MK_IPI_SLOT_WRITING: + if (head == idx) { + atomic_set(&ring->head, next); + } else if (head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + break; + } + atomic_set_release(&slot->state, MK_IPI_SLOT_CANCELLED); + atomic_inc(&ring->cancelled_writes); + break; + case MK_IPI_SLOT_READY: + case MK_IPI_SLOT_CANCELLED: + if (head == idx) { + /* Repair an interrupted publication before releasing its gate. */ + atomic_set(&ring->head, next); + atomic_inc(&ring->invalid_state); + } else if (head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + case MK_IPI_SLOT_CONSUMING: + /* + * The consumer may claim a full-ring tail while this producer + * waits to test it, leaving head at idx with nothing published. + * Head at next means the previous publication was claimed before + * its now-stale gate could be released. Both cursors are valid. + */ + if (head != idx && head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + default: + atomic_inc(&ring->invalid_state); + ret = -EIO; + break; + } + + /* Keep the gate closed when cursor repair cannot make the FIFO safe. */ + if (ret) + goto kick; + if (atomic64_cmpxchg_release(&ring->producer_gate, token, 0) != token) { + atomic_inc(&ring->invalid_state); + ret = -EAGAIN; + } - atomic_set(&ring->tail, head); +kick: + /* The producer may have published and parked before ringing the bell. */ + target = mk_cpu_set_first(root_instance->cpus); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + + return ret; } /** @@ -166,7 +493,7 @@ int mk_arm_force_halt(struct mk_instance *instance) /** * multikernel_send_ipi_data - Send data to another CPU via IPI - * @instance_id: Target multikernel instance ID + * @instance: Target multikernel instance * @data: Pointer to data to send * @data_size: Size of data * @type: User-defined type identifier @@ -176,121 +503,119 @@ int mk_arm_force_halt(struct mk_instance *instance) * * Returns 0 on success, negative error code on failure */ -int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) { - struct mk_ipi_data *slot; - struct mk_instance *instance = mk_instance_find(instance_id); - unsigned int head, next_head, tail; mk_phys_cpu_t target; + int instance_id; + int ret; if (!instance) return -EINVAL; - if (data_size > MK_MAX_DATA_SIZE) { - mk_instance_put(instance); + instance_id = instance->id; + if (data_size > MK_MAX_DATA_SIZE || (data_size && !data)) return -EINVAL; - } target = mk_cpu_set_first(instance->cpus); if (target == MK_PHYS_CPU_INVALID) { pr_err("Instance %d has no CPUs to receive the IPI\n", instance_id); - mk_instance_put(instance); return -ENODEV; } if (!mk_instance_ipi_area(instance)) { pr_err("Multikernel IPI buffer not available for instance %d\n", instance_id); - mk_instance_put(instance); return -ENODEV; } + ret = mk_ipi_shared_validate(instance->ipi_data); + if (ret) + return ret; + if (!atomic_read_acquire(&instance->ipi_data->ready)) + return -EAGAIN; + if (READ_ONCE(instance->ipi_data->ready_instance_id) != instance_id) + return -EPROTO; + + ret = mk_ipi_ring_publish(instance->ipi_data, instance_id, data, + data_size, type); + if (ret) { + /* + * A doorbell can be coalesced while the target is draining this + * ring. Kick it again before reporting backpressure so READY + * entries cannot remain stranded without another notification. + */ + mk_arch_send_ipi(target); + if (ret == -ENOSPC) + pr_warn_ratelimited("multikernel: IPI ring full for instance %d\n", + instance_id); + else if (ret == -EAGAIN) + pr_warn_ratelimited("multikernel: IPI producer busy for instance %d\n", + instance_id); + else if (ret != -EDEADLK) + pr_err_ratelimited("multikernel: IPI publish failed for instance %d: %d\n", + instance_id, ret); + return ret; + } + mk_arch_send_ipi(target); - /* Try to enqueue the message in the ring buffer */ - do { - head = mk_ring_idx(atomic_read(&instance->ipi_data->ring.head)); - next_head = mk_ring_idx(head + 1); - tail = mk_ring_idx(atomic_read(&instance->ipi_data->ring.tail)); - - /* Check if ring buffer is full */ - if (next_head == tail) { - /* - * Console output reaches this path, so a plain printk - * here re-enters the console write that called us and - * deadlocks on its lock with interrupts already off. - */ - printk_deferred(KERN_WARNING - "multikernel: IPI ring full for instance %d (head=%u, tail=%u)\n", - instance_id, head, tail); - mk_instance_put(instance); - return -ENOSPC; - } - - /* Try to claim this slot atomically */ - } while (atomic_cmpxchg(&instance->ipi_data->ring.head, head, next_head) != head); - - /* We've claimed slot 'head', now fill it */ - slot = &instance->ipi_data->ring.entries[head]; - - slot->sender_cpu = arch_cpu_physical_id(smp_processor_id()); - slot->type = type; - - if (data && data_size > 0) - memcpy(slot->buffer, data, data_size); - - /* - * data_size publishes the slot: the reader treats a zero as "the - * producer has claimed this slot but has not filled it yet" and - * waits. Claiming the slot advanced head, so a reader can already - * be looking at it; everything above must be visible first. - */ - smp_store_release(&slot->data_size, data_size); + return 0; +} - mk_arch_send_ipi(target); +int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, + unsigned long type) +{ + struct mk_instance *instance; + int ret; + instance = mk_instance_find(instance_id); + if (!instance) + return -EINVAL; + ret = mk_send_ipi_data(instance, data, data_size, type); mk_instance_put(instance); - return 0; + return ret; } static void mk_ipi_drain_ring(void) { struct mk_ipi_data *slot; struct mk_ipi_handler *handler; - unsigned int head, tail, next_tail; + struct mk_ipi_ring *ring; + unsigned int tail, idx; size_t data_size; + int state; int messages_processed = 0; if (!root_instance || !root_instance->ipi_data) return; - while (1) { - tail = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.head)); - - if (tail == head) + ring = &root_instance->ipi_data->ring; + while (messages_processed < MK_IPI_RING_SIZE) { + tail = atomic_read(&ring->tail); + idx = tail & (MK_IPI_RING_SIZE - 1); + slot = &ring->entries[idx]; + + state = atomic_read_acquire(&slot->state); + if (!mk_ipi_slot_is_pending(state)) { + if (state != MK_IPI_SLOT_EMPTY && + state != MK_IPI_SLOT_WRITING && + state != MK_IPI_SLOT_CONSUMING) { + atomic_inc(&ring->invalid_state); + pr_warn_once("Multikernel IPI slot %u has bad state %d\n", + idx, state); + } break; + } - slot = &root_instance->ipi_data->ring.entries[tail]; - - /* - * Pairs with the store_release in multikernel_send_ipi_data(). - * Zero means the sender claimed this slot but has not - * finished writing it. Leave it alone: skipping it would - * drop the message it is about to publish. Its own IPI, or - * the next one, brings us back here. - * - * A sender stopped before publishing leaves its slot zero - * forever; mk_ipi_ring_drop_pending() clears those out when - * the instance is re-spawned. - */ - data_size = smp_load_acquire(&slot->data_size); - if (data_size == 0) + if (atomic_cmpxchg_acquire(&slot->state, state, + MK_IPI_SLOT_CONSUMING) != state) break; + if (state == MK_IPI_SLOT_CANCELLED) + goto advance_tail; + + data_size = READ_ONCE(slot->data_size); if (data_size > MK_MAX_DATA_SIZE) { pr_warn_once("Multikernel IPI slot %u has bad size %zu\n", - tail, data_size); - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&root_instance->ipi_data->ring.tail, next_tail); - continue; + idx, data_size); + goto advance_tail; } /* Dispatch to registered handler */ @@ -308,17 +633,29 @@ static void mk_ipi_drain_ring(void) raw_spin_unlock(&mk_handlers_lock); advance_tail: - /* Mark consumed so the slot reads as unpublished again */ - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&root_instance->ipi_data->ring.tail, next_tail); + mk_ipi_slot_release(slot); + atomic_set(&ring->tail, (idx + 1) & (MK_IPI_RING_SIZE - 1)); messages_processed++; - - if (messages_processed >= MK_IPI_RING_SIZE) - break; } } +void mk_poll_ipi_messages(void) +{ + unsigned long flags; + mk_phys_cpu_t target; + + if (!root_instance) + return; + target = mk_cpu_set_first(root_instance->cpus); + if (target == MK_PHYS_CPU_INVALID || + target != arch_cpu_physical_id(smp_processor_id())) + return; + + local_irq_save(flags); + mk_ipi_drain_ring(); + local_irq_restore(flags); +} + /** * multikernel_interrupt_handler - Handle the multikernel IPI * diff --git a/kernel/multikernel/manifest.c b/kernel/multikernel/manifest.c index 89a8ee0d2c59b3..19cdc625e9e04d 100644 --- a/kernel/multikernel/manifest.c +++ b/kernel/multikernel/manifest.c @@ -23,12 +23,18 @@ /* Physical address of the manifest this kernel booted with, 0 if none */ static phys_addr_t mk_manifest_fdt_phys; +static bool mk_manifest_fdt_rejected; phys_addr_t mk_manifest_phys(void) { return mk_manifest_fdt_phys; } +bool mk_manifest_rejected(void) +{ + return READ_ONCE(mk_manifest_fdt_rejected); +} + /** * mk_manifest_populate() - Accept the manifest handed over at boot * @fdt_phys: Physical address of the manifest FDT @@ -50,6 +56,7 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) if (!fdt) { pr_warn("multikernel: failed to memremap manifest (0x%llx)\n", fdt_phys); + err = -ENOMEM; goto out; } @@ -68,14 +75,17 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) } mk_manifest_fdt_phys = fdt_phys; + mk_manifest_fdt_rejected = false; pr_info("multikernel: manifest accepted\n"); out: if (fdt) early_memunmap(fdt, fdt_len); - if (err) - pr_warn("multikernel: ignoring invalid manifest\n"); + if (err) { + mk_manifest_fdt_rejected = true; + pr_warn("multikernel: supplied manifest rejected: %d\n", err); + } } /**