diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 9bea5a1e2c52cb..6758247c93fd7e 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -379,7 +379,13 @@ xloadflags: #define XLF56 0 #endif - .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 +#ifdef CONFIG_MULTIKERNEL +# define XLF_MK XLF_MULTIKERNEL_IPI +#else +# define XLF_MK 0 +#endif + + .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF_MK cmdline_size: .long COMMAND_LINE_SIZE-1 #length of the command line, #added with boot protocol diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index f85b2a831aa205..293f2fd60987ab 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -10,6 +10,8 @@ #ifndef __ASSEMBLY__ +#include +#include #include #include #include @@ -54,7 +56,7 @@ static inline int arch_cpu_from_physical_id(u64 phys_id) * from a page written by the host while running on CPUs parked by (possibly * differently built) spawn kernels. * - * The fields fall into two classes that must not be mixed up: + * The fields fall into three classes that must not be mixed up: * * - Anchor fields (self_phys, park_phys, park_cr3, ctrl_phys, * ctrl_size): the context's own identity, written once when the @@ -62,9 +64,13 @@ static inline int arch_cpu_from_physical_id(u64 phys_id) * CPU on halt or offline, so they must stay valid for the context's * whole lifetime. * - * - Dispatch fields (everything else): the wake mailbox, rewritten for - * every publication and staged into registers by the CPU that claims - * it. Reparking gets its own repark_* dispatch fields precisely so a + * - Primary boot data (bp and the calibration values appended after it): + * written before the boot CPU is released and consumed while that kernel + * initializes. Secondary and repark publications do not rewrite it. + * + * - Dispatch fields (the remaining fixed-size fields): the wake mailbox, + * rewritten for every publication and staged into registers by the CPU + * that claims it. Reparking gets its own repark_* dispatch fields so a * repark publication never overwrites the anchor: the two used to * share fields, and a repark left the anchor pointing at another * kernel's park area, which triple-faulted the next halt. @@ -91,10 +97,23 @@ struct mk_spawn_context { u32 flags; /* MK_SPAWN_F_* flags */ u32 ready; /* Signal flag */ u32 reserved; /* Padding for alignment */ - /* Variable-size struct last - size depends on kernel config */ + /* Keep all existing context offsets unchanged. */ struct boot_params bp; /* Standard x86 boot params */ + /* Optional boot data belongs after boot_params, in the zeroed tail. */ + unsigned long boot_lps; /* Host delay loops per second */ + unsigned long boot_cpu_khz; /* Host CPU frequency calibration */ + unsigned long boot_tsc_khz; /* Host TSC frequency calibration */ + unsigned long boot_apic_hz; /* Host local APIC timer frequency */ + u64 abi_magic; /* Validated context producer */ } __aligned(PAGE_SIZE); +static_assert(offsetof(struct mk_spawn_context, bp) == 144); +static_assert(offsetof(struct mk_spawn_context, boot_lps) == + 144 + sizeof(struct boot_params)); +static_assert(offsetof(struct mk_spawn_context, abi_magic) == + 144 + sizeof(struct boot_params) + 4 * sizeof(unsigned long)); +static_assert(sizeof(struct mk_spawn_context) == 2 * PAGE_SIZE); + /* Pool park loop code, copied by the host into per-instance park pages */ extern char mk_pool_park_start[]; extern char mk_pool_park_end[]; @@ -109,6 +128,11 @@ void mk_park_cpu(void); /* True only in a spawn kernel whose boot context carries a park area */ bool mk_cpu_parkable(void); +/* Build a spawn E820 table from the instance's current memory grant */ +struct mk_instance; +struct boot_params; +int mk_e820_fill(struct mk_instance *instance, struct boot_params *params); + /* Park an offlined pool CPU (host park area, or instance context) */ void mk_pool_park_cpu(void); @@ -142,7 +166,8 @@ void mk_set_spawn_context(struct mk_spawn_context *ctx, int mk_spawn_cpu(struct mk_instance *instance, int cpu, struct mk_spawn_context *ctx); -/* Initialize boot context tracking in spawn kernel */ +/* Validate and initialize boot context tracking in spawn kernel */ +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys); void mk_init_boot_context(phys_addr_t ctx_phys); /* Identity page table and trampoline setup */ diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h index c70be687a3ecc7..7099f7cd167dce 100644 --- a/arch/x86/include/uapi/asm/bootparam.h +++ b/arch/x86/include/uapi/asm/bootparam.h @@ -25,6 +25,7 @@ #define XLF_5LEVEL (1<<5) #define XLF_5LEVEL_ENABLED (1<<6) #define XLF_MEM_ENCRYPTION (1<<7) +#define XLF_MULTIKERNEL_IPI 0x0100 #ifndef __ASSEMBLER__ diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 0759f9d6b64c23..5fc9c5b5904afd 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -125,63 +125,6 @@ static int setup_e820_entries(struct boot_params *params) return 0; } -static int setup_e820_entries_multikernel(struct kimage *image, struct boot_params *params) -{ - struct mk_instance *instance = image->mk_instance; - struct mk_memory_region *region; - unsigned int nr_e820_entries = 0; - - /* - * Don't include first 1MB in e820 for multikernel spawn kernels. - * Multikernel doesn't use real-mode trampoline (init_trampoline is - * skipped), and including unmapped low memory causes sparse_init() - * to fail when trying to populate vmemmap for those sections. - * Only include the assigned memory pool regions. - * - * However, we need at least 2 e820 entries or the kernel's - * append_e820_table() will reject our map and fall back to - * creating a fake memory map that includes low memory. - * Add a small reserved region at 0x0 as a dummy entry. - */ - params->e820_table[nr_e820_entries].addr = 0; - params->e820_table[nr_e820_entries].size = 0x1000; - params->e820_table[nr_e820_entries].type = E820_TYPE_RESERVED; - nr_e820_entries++; - - if (instance && !list_empty(&instance->memory_regions)) { - list_for_each_entry(region, &instance->memory_regions, list) { - if (nr_e820_entries >= E820_MAX_ENTRIES_ZEROPAGE) { - pr_warn("E820 table full, cannot add all memory regions\n"); - break; - } - - params->e820_table[nr_e820_entries].addr = region->res.start; - params->e820_table[nr_e820_entries].size = resource_size(®ion->res); - params->e820_table[nr_e820_entries].type = E820_TYPE_RAM; - - pr_info("Added memory region to e820: 0x%llx-0x%llx (%llu MB)\n", - (unsigned long long)region->res.start, - (unsigned long long)region->res.end, - (unsigned long long)resource_size(®ion->res) >> 20); - - nr_e820_entries++; - } - } - - params->e820_entries = nr_e820_entries; - - pr_info("Final multikernel e820 map has %d total entries:\n", - nr_e820_entries); - for (int i = 0; i < nr_e820_entries; i++) { - pr_info(" e820[%d]: 0x%llx-0x%llx type=%d\n", i, - params->e820_table[i].addr, - params->e820_table[i].addr + params->e820_table[i].size, - params->e820_table[i].type); - } - - return 0; -} - enum { RNG_SEED_LENGTH = 32 }; static void @@ -611,6 +554,11 @@ static void *bzImage64_load(struct kimage *image, char *kernel, .buf_max = ULONG_MAX, .top_down = true }; header = (struct setup_header *)(kernel + setup_hdr_offset); + if (image->type == KEXEC_TYPE_MULTIKERNEL && + !(header->xloadflags & XLF_MULTIKERNEL_IPI)) { + pr_err("Loaded kernel lacks the required shared transport layout\n"); + return ERR_PTR(-EPROTONOSUPPORT); + } setup_sects = header->setup_sects; if (setup_sects == 0) setup_sects = 4; @@ -804,7 +752,8 @@ static void *bzImage64_load(struct kimage *image, char *kernel, /* For multikernel, setup custom e820 map */ if (image->type == KEXEC_TYPE_MULTIKERNEL) { - ret = setup_e820_entries_multikernel(image, params); + image->arch.mk_boot_params = bootparam_load_addr; + ret = mk_e820_fill(image->mk_instance, params); if (ret) goto out_free_params; } diff --git a/arch/x86/kernel/kexec-vmlinux.c b/arch/x86/kernel/kexec-vmlinux.c index e8279270018c30..5a8fe67add2f2c 100644 --- a/arch/x86/kernel/kexec-vmlinux.c +++ b/arch/x86/kernel/kexec-vmlinux.c @@ -60,6 +60,12 @@ struct elf_kernel_info { unsigned long reloc_size; /* Size of relocation data */ }; +struct mk_elf_note_desc { + u64 entry; + u32 ipi_abi_version; + u32 reserved; +}; + /* * Find multikernel entry point from PT_NOTE section. * Looks for note with name "Linux" and type 0x4d4b ('MK'). @@ -93,12 +99,20 @@ static unsigned long find_multikernel_entry_note(const void *buf, size_t len, if (nhdr->n_type == 0x4d4b && nhdr->n_namesz == 6 && - nhdr->n_descsz == sizeof(u64) && + nhdr->n_descsz == sizeof(struct mk_elf_note_desc) && !memcmp(ptr + sizeof(*nhdr), "Linux", 6)) { - u64 entry = *(u64 *)(ptr + sizeof(*nhdr) + - ALIGN(nhdr->n_namesz, 4)); - pr_info("multikernel: entry=0x%llx\n", entry); - return entry; + const struct mk_elf_note_desc *desc; + + desc = ptr + sizeof(*nhdr) + + ALIGN(nhdr->n_namesz, 4); + if (desc->ipi_abi_version != MK_IPI_ABI_VERSION) { + pr_err("multikernel IPI ABI %u is not supported\n", + desc->ipi_abi_version); + return 0; + } + pr_info("multikernel: entry=0x%llx, IPI ABI=%u\n", + desc->entry, desc->ipi_abi_version); + return desc->entry; } ptr += note_size; } @@ -306,52 +320,6 @@ static int vmlinux_probe(const char *buf, unsigned long len) return 0; } -/* - * Setup e820 memory map for multikernel spawn kernel - */ -static int setup_e820_entries_multikernel(struct kimage *image, struct boot_params *params) -{ - struct mk_instance *instance = image->mk_instance; - struct mk_memory_region *region; - unsigned int nr_e820_entries = 0; - int i; - - /* - * Only include the assigned memory pool regions for multikernel spawn. - * Don't include first 1MB - multikernel doesn't use real-mode trampoline - * and including unmapped low memory causes sparse_init() to fail. - * - * The spawn kernel uses e820__memory_setup_multikernel() which accepts - * any number of entries without fallback to legacy BIOS memory probing. - */ - if (instance && !list_empty(&instance->memory_regions)) { - list_for_each_entry(region, &instance->memory_regions, list) { - if (nr_e820_entries >= E820_MAX_ENTRIES_ZEROPAGE) { - pr_warn("Too many e820 entries, truncating\n"); - break; - } - - params->e820_table[nr_e820_entries].addr = region->res.start; - params->e820_table[nr_e820_entries].size = resource_size(®ion->res); - params->e820_table[nr_e820_entries].type = E820_TYPE_RAM; - nr_e820_entries++; - } - } - - params->e820_entries = nr_e820_entries; - - pr_info("Final multikernel e820 map has %d total entries:\n", - nr_e820_entries); - for (i = 0; i < nr_e820_entries; i++) { - pr_info(" e820[%d]: 0x%llx-0x%llx type=%d\n", i, - params->e820_table[i].addr, - params->e820_table[i].addr + params->e820_table[i].size, - params->e820_table[i].type); - } - - return 0; -} - /* * Load function - load ELF vmlinux and setup boot parameters */ @@ -606,7 +574,7 @@ static void *vmlinux_load(struct kimage *image, char *kernel, /* For multikernel, setup custom e820 map */ if (image->type == KEXEC_TYPE_MULTIKERNEL) { - ret = setup_e820_entries_multikernel(image, params); + ret = mk_e820_fill(image->mk_instance, params); if (ret) { kvfree(ldata->kernel_buf); kfree(ldata); diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index 95d2cd2ccf74f5..9209b77d1c54e3 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -14,8 +14,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -28,6 +30,35 @@ extern pmd_t *populate_extra_pmd(unsigned long vaddr); extern unsigned long orig_boot_params; #ifdef CONFIG_MULTIKERNEL +static unsigned long multikernel_cpu_khz; +static unsigned long multikernel_tsc_khz; + +static unsigned long multikernel_calibrate_cpu(void) +{ + return multikernel_cpu_khz; +} + +static unsigned long multikernel_calibrate_tsc(void) +{ + return multikernel_tsc_khz; +} + +static void __init multikernel_setup_calibration(void) +{ + phys_addr_t ctx_phys = orig_boot_params - + offsetof(struct mk_spawn_context, bp); + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); + + if (!ctx || !ctx->boot_tsc_khz) + return; + + multikernel_tsc_khz = ctx->boot_tsc_khz; + multikernel_cpu_khz = ctx->boot_cpu_khz ?: ctx->boot_tsc_khz; + x86_platform.calibrate_cpu = multikernel_calibrate_cpu; + x86_platform.calibrate_tsc = multikernel_calibrate_tsc; + setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ); +} + /* * Custom wakeup for multikernel spawn kernels. * Uses shared spawn table instead of realmode trampoline. @@ -105,6 +136,10 @@ static void __init multikernel_parse_smp_config(void) */ apic_update_callback(wakeup_secondary_cpu_64, multikernel_wakeup_cpu); } +#else +static inline void multikernel_setup_calibration(void) +{ +} #endif /* CONFIG_MULTIKERNEL */ void __init x86_early_init_platform_quirks(void) @@ -135,6 +170,7 @@ void __init x86_early_init_platform_quirks(void) x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; break; case X86_SUBARCH_MULTIKERNEL: + multikernel_setup_calibration(); x86_platform.legacy.devices.pnpbios = 0; x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; x86_platform.legacy.rtc = 0; @@ -175,7 +211,9 @@ void __init x86_early_init_platform_quirks(void) * the PIT - which belongs to the host - and then request * legacy IRQ0, which can never reach an instance CPU that * has neither a PIC nor an IO-APIC. Ticks come from the - * local APIC timer via setup_percpu_clockev() instead. + * local APIC timer initialized by setup_percpu_clockev(). + * Keeping global_clock_event unset bypasses LAPIC timer + * verification, whose fallback path requires legacy IRQ0. */ x86_init.timers.timer_init = x86_init_noop; x86_init.timers.wallclock_init = x86_init_noop; diff --git a/arch/x86/multikernel/Makefile b/arch/x86/multikernel/Makefile index 331bd895af1f7b..5989cbbb691989 100644 --- a/arch/x86/multikernel/Makefile +++ b/arch/x86/multikernel/Makefile @@ -3,4 +3,4 @@ # Makefile for multikernel spawn support # -obj-y += spawn.o direct_boot.o head_64.o +obj-y += spawn.o direct_boot.o head_64.o e820.o diff --git a/arch/x86/multikernel/e820.c b/arch/x86/multikernel/e820.c new file mode 100644 index 00000000000000..dc78c288007dd4 --- /dev/null +++ b/arch/x86/multikernel/e820.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Spawn kernel E820 construction from an instance's memory grant. + */ +#include +#include +#include + +#include +#include +#include + +/* + * Fill @params' E820 table from @instance's current memory regions. + * + * Called by the image loaders at load time and again on every exec, + * after the pristine boot_params are copied into the spawn context. + * The grant can grow or shrink between load and exec (kerf update), + * and a map baked at load time would hand the spawn the memory it + * owned back then, silently losing the difference; the device-tree + * manifest is rebuilt at exec, so without this the two disagree. + */ +int mk_e820_fill(struct mk_instance *instance, struct boot_params *params) +{ + struct mk_memory_region *region; + unsigned int nr = 0; + u64 total = 0; + + if (!instance) + return -EINVAL; + + /* + * The first 1MB stays out: multikernel skips the real-mode + * trampoline, and unmapped low memory makes sparse_init() fail. + * A single reserved page keeps the table from being empty. + */ + params->e820_table[nr].addr = 0; + params->e820_table[nr].size = 0x1000; + params->e820_table[nr].type = E820_TYPE_RESERVED; + nr++; + + list_for_each_entry(region, &instance->memory_regions, list) { + if (nr >= E820_MAX_ENTRIES_ZEROPAGE) { + pr_err("mk_e820: instance %d has more memory regions than the E820 table can carry (%d); refusing a truncated map\n", + instance->id, E820_MAX_ENTRIES_ZEROPAGE); + return -E2BIG; + } + + params->e820_table[nr].addr = region->res.start; + params->e820_table[nr].size = resource_size(®ion->res); + params->e820_table[nr].type = E820_TYPE_RAM; + total += params->e820_table[nr].size; + nr++; + } + + params->e820_entries = nr; + + pr_info("mk_e820: instance %d: %llu MB RAM in %u regions\n", + instance->id, total >> 20, nr - 1); + return 0; +} diff --git a/arch/x86/multikernel/head_64.S b/arch/x86/multikernel/head_64.S index 3784147fd82f62..b726e74ff3e402 100644 --- a/arch/x86/multikernel/head_64.S +++ b/arch/x86/multikernel/head_64.S @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -273,4 +274,6 @@ SYM_CODE_END(multikernel_secondary_startup) 1: .asciz "Linux" 2: .balign 4 3: .quad multikernel_startup_64 - __START_KERNEL_map + .long MK_IPI_ABI_VERSION + .long 0 4: .balign 4 diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index 924a3fa5137d7d..adbf35ff3c2727 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include #include @@ -44,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -68,6 +71,7 @@ /* Set in spawn kernels: the context this kernel booted from */ static struct mk_spawn_context *mk_boot_context; +static phys_addr_t mk_boot_context_phys; /* * Spawn kernel's own trampoline for secondary CPU wakeup. @@ -81,6 +85,16 @@ static struct mk_spawn_context *mk_boot_context; */ static void *spawn_trampoline_va; static unsigned long spawn_trampoline_phys; +static bool spawn_trampoline_prepared; +static bool spawn_pool_park_prepared; +static bool spawn_park_ready; +static int spawn_park_error; + +bool mk_arch_park_ready(void) +{ + /* Pair with publication after both executable park mappings succeed. */ + return smp_load_acquire(&spawn_park_ready); +} extern char multikernel_relocate_kernel_start[]; extern char multikernel_relocate_kernel_end[]; @@ -364,6 +378,7 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, int cpu) { struct boot_params *src_bp; + int ret; if (!instance->ident_pgt) { struct mk_ident_pgtable *pgt; @@ -441,12 +456,33 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, sizeof(struct boot_params)); memunmap(src_bp); + /* + * The loader's E820 reflects the grant at load time; regions added + * or removed since then only exist in the live region list, which + * the manifest is also rebuilt from on every exec. Rebuild the map + * too, or the spawn boots with the memory it owned at load and the + * difference silently vanishes. + */ + ret = mk_e820_fill(instance, + mk_spawn_context_boot_params(instance->spawn_ctx)); + if (ret) + return ret; + mk_set_spawn_context(instance->spawn_ctx, mk_get_identity_cr3(instance->ident_pgt), image->arch.mk_kernel_entry, (unsigned long)instance->trampoline_va, virt_to_phys(instance->trampoline_va), virt_to_phys(instance->park_va)); + instance->spawn_ctx->boot_lps = cpu_data(cpu).loops_per_jiffy; + if (!instance->spawn_ctx->boot_lps) + instance->spawn_ctx->boot_lps = loops_per_jiffy; + instance->spawn_ctx->boot_lps *= HZ; + instance->spawn_ctx->boot_cpu_khz = cpu_khz; + instance->spawn_ctx->boot_tsc_khz = tsc_khz; + instance->spawn_ctx->boot_apic_hz = + (unsigned long)lapic_timer_period * HZ; + instance->spawn_ctx->abi_magic = MK_BOOT_CONTEXT_MAGIC; return mk_spawn_cpu(instance, cpu, instance->spawn_ctx); } @@ -669,18 +705,38 @@ void __init mk_arch_register_cpu(u64 phys_id) topology_register_apic((u32)phys_id, CPU_ACPIID_INVALID, true); } -/* - * Initialize boot context tracking in spawn kernel. - * Called early during spawn kernel boot. - */ -void mk_init_boot_context(phys_addr_t ctx_phys) +static __noreturn void mk_reject_spawn_context(void) +{ + /* + * The host context layout is unknown, so neither its park state nor any + * shared context field is safe to use. Keep this CPU local and inert. An NMI + * can wake HLT, but returns to this loop with maskable interrupts still + * disabled; disable them again before every halt for defense in depth. + */ + for (;;) { + native_irq_disable(); + native_halt(); + } +} + +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys) { struct mk_spawn_context *ctx; + phys_addr_t stamped_phys; + u64 abi_magic; if (!ctx_phys) { pr_err("mk_spawn: Boot context physical address is 0!\n"); - return; + return NULL; } + if (mk_boot_context) { + if (ctx_phys != mk_boot_context_phys) + mk_reject_spawn_context(); + return mk_boot_context; + } + /* Reject an invalid derived address before mapping or dereferencing it. */ + if (!IS_ALIGNED(ctx_phys, PAGE_SIZE)) + mk_reject_spawn_context(); /* * The spawn context is in the multikernel pool which is regular RAM, @@ -697,14 +753,39 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * work and then fails much later, when this kernel shuts down and * its CPUs park on nonsense addresses. */ - if (ctx->self_phys != ctx_phys) { - pr_err("mk_spawn: Boot context at %pa is stamped %pa\n", - &ctx_phys, &ctx->self_phys); - pr_err("mk_spawn: Spawn context layout mismatch - host and spawn kernels must be built from the same source\n"); - return; - } - + stamped_phys = READ_ONCE(ctx->self_phys); + if (stamped_phys != ctx_phys) + mk_reject_spawn_context(); + abi_magic = READ_ONCE(ctx->abi_magic); + if (abi_magic != MK_BOOT_CONTEXT_MAGIC) + mk_reject_spawn_context(); + + mk_boot_context_phys = ctx_phys; mk_boot_context = ctx; + return ctx; +} + +/* + * Initialize boot context tracking in spawn kernel. + * Called early during spawn kernel boot. + */ +void mk_init_boot_context(phys_addr_t ctx_phys) +{ + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); + + if (!ctx) + return; + /* + * A spawn kernel cannot calibrate against legacy timers because they + * belong to the host. Reuse the selected physical CPU's delay and local + * APIC timer calibration, while keeping explicit command-line values + * authoritative. + */ + if (!preset_lpj && ctx->boot_lps) + preset_lpj = DIV_ROUND_CLOSEST_ULL(ctx->boot_lps, HZ); + if (!lapic_timer_period && ctx->boot_apic_hz) + lapic_timer_period = + DIV_ROUND_CLOSEST_ULL(ctx->boot_apic_hz, HZ); /* * The host's control area (this context, the trampoline and park @@ -729,17 +810,25 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * * One physical page serves every wake path of this instance: the host * allocates it once in mk_setup_trampoline() and reuses it across - * re-spawns, and mk_prepare_trampoline() places our own trampoline copy + * re-spawns, and mk_arch_prepare_park() places our own trampoline copy * (including the secondary entry) in the same page. */ -static int __init mk_prepare_trampoline(void) +int __init mk_arch_prepare_park(void) { struct mk_spawn_context *ctx = mk_boot_context; unsigned long virt; int ret; + if (mk_arch_park_ready()) + return 0; + if (spawn_park_error) + return spawn_park_error; if (!ctx) return 0; + if (!ctx->trampoline_phys || !ctx->park_phys || !ctx->park_cr3) { + ret = -EINVAL; + goto fail; + } /* * Put our own copy of the trampoline in the page the host set @@ -747,34 +836,48 @@ static int __init mk_prepare_trampoline(void) * is entered from an offline CPU, where changing page attributes * is not allowed. */ - spawn_trampoline_phys = ctx->trampoline_phys; - spawn_trampoline_va = __va(spawn_trampoline_phys); - memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, - multikernel_relocate_kernel_end - multikernel_relocate_kernel_start); + if (!spawn_trampoline_prepared) { + spawn_trampoline_phys = ctx->trampoline_phys; + spawn_trampoline_va = __va(spawn_trampoline_phys); + memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, + multikernel_relocate_kernel_end - + multikernel_relocate_kernel_start); - /* - * Both pages are executed from the direct map, which is writable, - * so drop write before adding execute. Leaving them writable and - * executable trips the kernel's own W^X check. - */ - virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; - ret = set_memory_ro(virt, 1); - if (!ret) - ret = set_memory_x(virt, 1); - if (ret) - return ret; + /* + * Both pages are executed from the direct map, which is writable, + * so drop write before adding execute. Leaving them writable and + * executable trips the kernel's own W^X check. + */ + virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; + ret = set_memory_ro(virt, 1); + if (!ret) + ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_trampoline_prepared = true; + } /* The pool park page is entered the same way when this kernel dies */ - if (ctx->park_phys) { + if (!spawn_pool_park_prepared) { virt = (unsigned long)__va(ctx->park_phys) & PAGE_MASK; ret = set_memory_ro(virt, 1); if (!ret) ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_pool_park_prepared = true; } + /* Publish executable mappings before any reject or abort can park. */ + smp_store_release(&spawn_park_ready, true); + return 0; + +fail: + /* A partial W^X transition is not safe to retry. */ + spawn_park_error = ret; return ret; } -early_initcall(mk_prepare_trampoline); +early_initcall(mk_arch_prepare_park); /* * Add a 2MB executable mapping to a page table. diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 84a98632729401..17607e15d34347 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -14,6 +14,10 @@ #include #include #include +#include +#include + +struct pci_bus; /** * Physical CPU identifiers @@ -75,8 +79,17 @@ static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) /* IPI ring buffer size - must be power of 2 for efficient modulo */ #define MK_IPI_RING_SIZE 64 +#define MK_IPI_SLOT_EMPTY 0 +#define MK_IPI_SLOT_WRITING 1 +#define MK_IPI_SLOT_READY 2 +#define MK_IPI_SLOT_CONSUMING 3 +#define MK_IPI_SLOT_CANCELLED 4 +#define MK_IPI_ABI_MAGIC 0x4d4b495049303033ULL /* "MKIPI003" */ +#define MK_IPI_READY_TIMEOUT_MS 120000 + /* Data structure for passing parameters via IPI */ struct mk_ipi_data { + atomic_t state; u64 sender_cpu; /* Physical ID of the CPU that sent this IPI */ unsigned int type; /* User-defined type identifier */ size_t data_size; /* Size of the data */ @@ -85,9 +98,15 @@ struct mk_ipi_data { /* IPI ring buffer for queuing messages */ struct mk_ipi_ring { - atomic_t head; /* Producer index */ - atomic_t tail; /* Consumer index */ + atomic_t head; /* Producer allocation cursor */ + atomic_t tail; /* Consumer scan cursor */ struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */ + /* Appended shared ABI: do not move fields above this line. */ + atomic64_t producer_gate; /* Owner CPU and claimed slot */ + atomic_t producer_contention; /* Sends that observed a busy gate */ + atomic_t full_failures; /* Sends rejected by a full ring */ + atomic_t invalid_state; /* Invalid slot state observations */ + atomic_t cancelled_writes; /* Halted producer writes recovered */ }; /* Shared memory structures - per-instance design */ @@ -101,8 +120,47 @@ struct mk_shared_data { * CPUs the first one missed. */ u32 force_halt; + /* Appended ABI handshake; existing shared offsets stay unchanged. */ + u64 abi_magic; + u32 abi_version; + u32 abi_size; + s32 ready_instance_id; + atomic_t ready; }; +static inline void mk_ipi_ring_reset_contents(struct mk_ipi_ring *ring) +{ + unsigned int i; + + atomic_set(&ring->head, 0); + atomic_set(&ring->tail, 0); + for (i = 0; i < MK_IPI_RING_SIZE; i++) { + WRITE_ONCE(ring->entries[i].data_size, 0); + atomic_set(&ring->entries[i].state, MK_IPI_SLOT_EMPTY); + } + atomic_set(&ring->producer_contention, 0); + atomic_set(&ring->full_failures, 0); + atomic_set(&ring->invalid_state, 0); + atomic_set(&ring->cancelled_writes, 0); +} + +static inline void mk_ipi_ring_reset(struct mk_ipi_ring *ring) +{ + mk_ipi_ring_reset_contents(ring); + atomic64_set(&ring->producer_gate, 0); +} + +static inline void mk_shared_data_reset(struct mk_shared_data *shared) +{ + mk_ipi_ring_reset(&shared->ring); + WRITE_ONCE(shared->force_halt, 0); + WRITE_ONCE(shared->abi_magic, MK_IPI_ABI_MAGIC); + WRITE_ONCE(shared->abi_version, MK_IPI_ABI_VERSION); + WRITE_ONCE(shared->abi_size, sizeof(*shared)); + WRITE_ONCE(shared->ready_instance_id, -1); + atomic_set(&shared->ready, 0); +} + /* Function pointer type for IPI callbacks */ typedef void (*mk_ipi_callback_t)(struct mk_ipi_data *data, void *ctx); @@ -145,8 +203,14 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns void generic_multikernel_interrupt(void); -/* Discard everything queued in this kernel's ring (instance re-spawn) */ -void mk_ipi_ring_drop_pending(void); +int mk_ipi_shared_validate(const struct mk_shared_data *shared); +int mk_ipi_shared_mark_ready(struct mk_shared_data *shared, int instance_id); +int mk_ipi_shared_wait_ready(struct mk_shared_data *shared, int instance_id, + unsigned int timeout_ms); +int mk_ipi_shared_reset_downlink(struct mk_shared_data *shared); + +/* Recover a producer only after every CPU in @halted_cpus is parked. */ +int mk_ipi_ring_recover_halted(const struct mk_cpu_set *halted_cpus); /* * Multikernel Messaging System @@ -766,6 +830,7 @@ struct mk_instance *mk_instance_find(int mk_id); void mk_instance_put(struct mk_instance *instance); void mk_instance_set_state(struct mk_instance *instance, enum mk_instance_state state); +int mk_instance_abort_spawn(struct mk_instance *instance); /* Kimage-based access to the instance memory pool */ void *mk_kimage_alloc(struct kimage *image, size_t size, size_t align); @@ -780,6 +845,7 @@ void mk_register_cpus_from_manifest(void); /* Accept the manifest handed over at boot (spawn kernels) */ void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len); +bool mk_manifest_rejected(void); /* Build the manifest for a spawn (host, kexec path) */ int mk_manifest_finalize(struct kimage *image); @@ -837,6 +903,11 @@ static inline void mk_register_cpus_from_manifest(void) static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) { } + +static inline bool mk_manifest_rejected(void) +{ + return false; +} #endif /** @@ -844,7 +915,8 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) */ #define MK_DT_CONFIG_VERSION_1 1 #define MK_DT_CONFIG_CURRENT MK_DT_CONFIG_VERSION_1 -#define MK_FDT_COMPATIBLE "multikernel-v1" +/* Bumped whenever the shared-memory layout or message semantics change. */ +#define MK_FDT_COMPATIBLE "multikernel-v3" /** * Property Names @@ -990,6 +1062,8 @@ void mk_set_pool_cpu(int cpu, bool is_pool); /* Park the calling CPU in the pool wait loop; never returns */ void __noreturn mk_enter_pool_state(void *info); +int __init mk_arch_prepare_park(void); +bool mk_arch_park_ready(void); /* * Forcible stop of another instance's CPUs (NMI on x86). Registration diff --git a/include/linux/multikernel_abi.h b/include/linux/multikernel_abi.h new file mode 100644 index 00000000000000..9f64fe3c398de1 --- /dev/null +++ b/include/linux/multikernel_abi.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MULTIKERNEL_ABI_H +#define _LINUX_MULTIKERNEL_ABI_H + +/* Private host/spawn transport compatibility constants. */ +#define MK_IPI_ABI_VERSION 3 +#define MK_BOOT_CONTEXT_MAGIC 0x4d4b435458303032ULL /* "MKCTX002" */ + +#endif /* _LINUX_MULTIKERNEL_ABI_H */ diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 01382f40aeb724..1206eec512f3e5 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1702,6 +1702,12 @@ int multikernel_kexec_by_id(int mk_id) } instance = mk_image->mk_instance; + if (instance->state != MK_STATE_LOADED) { + pr_err("Multikernel instance %d is not loadable (state=%d)\n", + mk_id, instance->state); + rc = -EINVAL; + goto unlock; + } if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); @@ -1756,10 +1762,11 @@ int multikernel_kexec_by_id(int mk_id) } rc = mk_manifest_finalize(mk_image); - if (rc) - pr_warn("Manifest finalization failed: %d\n", rc); - else - pr_info("Manifest finalized for multikernel instance\n"); + if (rc) { + pr_err("Manifest finalization failed: %d\n", rc); + goto unlock; + } + pr_info("Manifest finalized for multikernel instance\n"); /* * Point at the ring this image actually carries. Every load @@ -1775,26 +1782,33 @@ int multikernel_kexec_by_id(int mk_id) } /* - * Start the instance with an empty ring. It outlives the kernel - * that was using it, so a new instance would otherwise inherit that - * kernel's indices and any slot it left half written - which stalls - * the reader, since an unpublished slot means "the sender is still - * filling this one". Anything left in there was addressed to a - * kernel that is gone. - */ - if (instance->ipi_data) - memset(instance->ipi_data, 0, sizeof(*instance->ipi_data)); - - /* - * Same for the other direction: whatever the halted instance left - * queued for us is addressed from a kernel that no longer exists, - * and a slot it claimed but never published stalls our ring for - * good. + * Start the instance with an empty downlink after its CPUs have been + * confirmed parked. The host is the only producer for this ring, so no + * publisher can race the reset once the old receiver is quiesced. */ - mk_ipi_ring_drop_pending(); - + if (instance->ipi_data) { + rc = mk_ipi_shared_reset_downlink(instance->ipi_data); + if (rc) { + pr_err("Failed to reset instance %d IPI downlink: %d\n", + mk_id, rc); + goto unlock; + } + } rc = mk_arch_spawn_instance(mk_image, instance, cpu); if (rc == 0) { + rc = mk_ipi_shared_wait_ready(instance->ipi_data, mk_id, + MK_IPI_READY_TIMEOUT_MS); + if (rc) { + int abort_ret; + + pr_err("Instance %d did not acknowledge IPI ABI %u: %d\n", + mk_id, MK_IPI_ABI_VERSION, rc); + abort_ret = mk_instance_abort_spawn(instance); + if (abort_ret) + pr_crit("Instance %d IPI ABI timeout abort failed: %d\n", + mk_id, abort_ret); + goto unlock; + } rc = mk_instance_set_kexec_active(mk_image->mk_id); if (rc) pr_warn("Failed to set instance %d as active: %d\n", mk_image->mk_id, rc); diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 0a40038a954773..5712ae986b30e0 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -286,6 +286,7 @@ bool multikernel_allow_emergency_restart(void) */ int mk_instance_confirm_parked(struct mk_instance *instance) { + struct mk_cpu_set *snapshot; mk_phys_cpu_t phys_cpu; unsigned int i; int ret, failed = 0; @@ -293,8 +294,22 @@ int mk_instance_confirm_parked(struct mk_instance *instance) /* Never started, so nothing of it is running */ if (!instance->spawn_ctx) return 0; + if (!instance->cpus_on_slot) { + pr_err("Instance %d (%s): missing parked-CPU tracking for a started instance\n", + instance->id, instance->name); + return -EINVAL; + } - mk_cpu_set_for_each(i, phys_cpu, instance->cpus_on_slot) { + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + ret = mk_cpu_set_copy(snapshot, instance->cpus_on_slot); + if (ret) { + mk_cpu_set_free(snapshot); + return ret; + } + + mk_cpu_set_for_each(i, phys_cpu, snapshot) { ret = mk_arch_confirm_parked(instance, phys_cpu); if (ret) { pr_err("Instance %d (%s): CPU %llu is not parked: %d\n", @@ -302,6 +317,15 @@ int mk_instance_confirm_parked(struct mk_instance *instance) failed++; } } + if (!failed) { + ret = mk_ipi_ring_recover_halted(snapshot); + if (ret) { + pr_err("Instance %d (%s): failed to recover halted IPI producer: %d\n", + instance->id, instance->name, ret); + failed++; + } + } + mk_cpu_set_free(snapshot); return failed ? -EBUSY : 0; } @@ -1386,38 +1410,16 @@ int multikernel_halt_by_id(int mk_id) return ret; } -/** - * multikernel_force_halt_by_id - Forcible shutdown of a multikernel instance via NMI - * @mk_id: Instance ID to halt - * - * Forces a spawn kernel's CPUs to stop by arming the force-halt marker - * in the instance's shared IPI area and sending NMIs directly to each - * CPU. The NMI handler tests the marker and parks the CPU in the pool. - * - * No message is queued and no doorbell is rung: a ring message is - * consumed by the instance's ordinary interrupt path, which on a - * responsive kernel races the NMIs for it and can leave them with - * nothing to act on. The marker is host-owned and survives until the - * instance is re-executed, so the NMIs act on it regardless of timing. - * - * Use when: The spawn kernel is stuck/crashed and not responding to graceful - * shutdown, or when graceful shutdown has failed. May be repeated: an - * already-halted instance absorbs the NMIs in the park loop, so a rerun - * only rescues CPUs an earlier halt missed. - * - * Returns: 0 on success, negative error code on failure - */ -int multikernel_force_halt_by_id(int mk_id) +static int __mk_instance_force_halt(struct mk_instance *instance, + bool allow_loaded) { - struct mk_instance *instance; mk_phys_cpu_t phys_cpu; unsigned int i; int cpu_count = 0; int ret; - instance = mk_instance_find(mk_id); if (!instance) - return -ENOENT; + return -EINVAL; /* * LOADED is allowed for the retry case: a previous halt already @@ -1426,20 +1428,19 @@ int multikernel_force_halt_by_id(int mk_id) * a rerun the instance is stuck for good. */ if (instance->state != MK_STATE_ACTIVE && - instance->state != MK_STATE_LOADED) { + (!allow_loaded || instance->state != MK_STATE_LOADED)) { pr_err("Instance %d not running (state=%d), nothing to force halt\n", - mk_id, instance->state); - mk_instance_put(instance); + instance->id, instance->state); return -EINVAL; } if (mk_cpu_set_empty(instance->cpus)) { - pr_err("Instance %d has no CPUs assigned\n", mk_id); - mk_instance_put(instance); + pr_err("Instance %d has no CPUs assigned\n", instance->id); return -EINVAL; } - pr_info("Force halting multikernel instance %d via NMI\n", mk_id); + pr_info("Force halting multikernel instance %d via NMI\n", + instance->id); ret = mk_arm_force_halt(instance); if (ret) @@ -1451,18 +1452,61 @@ int multikernel_force_halt_by_id(int mk_id) cpu_count++; } - pr_info("Sent NMI to %d CPUs in instance %d\n", cpu_count, mk_id); + pr_info("Sent NMI to %d CPUs in instance %d\n", + cpu_count, instance->id); + + ret = mk_instance_confirm_parked(instance); + if (ret) { + pr_err("Instance %d CPUs did not park after force halt: %d\n", + instance->id, ret); + return ret; + } - /* - * The NMI handler parks each CPU on the instance's context. Wait - * for them to arrive before reporting the instance re-spawnable, - * exactly as the graceful path does after its shutdown ACK. - */ mk_instance_settle_halted(instance); - mk_instance_put(instance); return 0; } +int mk_instance_abort_spawn(struct mk_instance *instance) +{ + int ret; + + ret = __mk_instance_force_halt(instance, true); + if (ret && instance) + mk_instance_set_state(instance, MK_STATE_FAILED); + return ret; +} + +/** + * mk_instance_force_halt - Forcibly stop an instance via NMI + * @instance: Instance to stop + * + * Forces a spawn kernel's CPUs to stop by arming the persistent force-halt + * marker and sending NMIs directly to each CPU. The NMI handler checks the + * marker and parks the CPU if it is set. + * + * Use when: The spawn kernel is stuck/crashed and not responding to graceful + * shutdown, or when graceful shutdown has failed. + * + * Returns: 0 on success, negative error code on failure + */ +int mk_instance_force_halt(struct mk_instance *instance) +{ + return __mk_instance_force_halt(instance, false); +} + +int multikernel_force_halt_by_id(int mk_id) +{ + struct mk_instance *instance; + int ret; + + instance = mk_instance_find(mk_id); + if (!instance) + return -ENOENT; + ret = mk_instance_force_halt(instance); + mk_instance_put(instance); + return ret; +} + static int __init multikernel_init(void) { int ret; @@ -1504,6 +1548,17 @@ static int __init multikernel_init(void) return ret; } + ret = mk_ipi_shared_mark_ready(root_instance->ipi_data, + root_instance->id); + if (ret < 0) { + pr_err("Failed to publish multikernel IPI readiness: %d\n", ret); + mk_kernfs_cleanup(); + mk_hotplug_cleanup(); + mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); + mk_messaging_cleanup(); + return ret; + } + pr_info("Multikernel support initialized\n"); return 0; } diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index f21cc947149f10..c3351017a86338 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "internal.h" #define PROP_SUB_FDT "fdt" @@ -34,6 +35,23 @@ struct mk_instance *root_instance = NULL; EXPORT_SYMBOL_GPL(root_instance); +static void __init __noreturn mk_manifest_reject_and_park(int error) +{ + int ret; + + ret = mk_arch_prepare_park(); + if (ret || !mk_arch_park_ready()) + panic("multikernel: rejected manifest before park path became ready"); + ret = mk_register_stop_nmi_handler(); + if (ret) + pr_emerg("multikernel: stop-NMI registration failed while rejecting manifest: %d\n", + ret); + pr_emerg("multikernel: parking CPUs after rejecting supplied manifest: %d\n", + error); + smp_call_function(mk_enter_pool_state, NULL, 0); + mk_enter_pool_state(NULL); +} + /* * Collect every CPU the instance might receive through hotplug later: * the unassigned pool plus every other kernel's CPUs (the host's and @@ -386,6 +404,7 @@ static struct mk_instance * __init alloc_mk_instance(int instance_id, const char pr_err("Failed to allocate IPI buffer for instance %d\n", instance_id); goto err_free_name; } + mk_shared_data_reset(instance->ipi_data); instance->ipi_phys = virt_to_phys(instance->ipi_data); instance->ipi_pages = (sizeof(struct mk_shared_data) + PAGE_SIZE - 1) / PAGE_SIZE; @@ -543,6 +562,12 @@ static int __init mk_restore_instance_ipi(const void *manifest, struct mk_instan (unsigned long long)ipi_phys, ipi_pages); return 0; } + if (ipi_size < sizeof(struct mk_shared_data)) { + pr_err("IPI buffer is too small for ABI %u: %zu < %zu\n", + MK_IPI_ABI_VERSION, ipi_size, + sizeof(struct mk_shared_data)); + return -EPROTO; + } instance->ipi_data = memremap(ipi_phys, ipi_size, MEMREMAP_WB); if (!instance->ipi_data) { @@ -591,6 +616,12 @@ static struct mk_instance * __init mk_restore_host_instance(const void *manifest (unsigned long long)host_ipi_phys, host_ipi_pages); return NULL; } + if (host_ipi_size < sizeof(struct mk_shared_data)) { + pr_err("Host IPI buffer is too small for ABI %u: %zu < %zu\n", + MK_IPI_ABI_VERSION, host_ipi_size, + sizeof(struct mk_shared_data)); + return NULL; + } host_instance = alloc_mk_instance(0, "", false); if (!host_instance) @@ -645,6 +676,9 @@ int __init mk_instance_restore_from_manifest(void) const void *manifest = NULL; phys_addr_t fdt_phys; + if (mk_manifest_rejected()) + mk_manifest_reject_and_park(-EPROTO); + fdt_phys = mk_manifest_phys(); if (!fdt_phys) { pr_info("No manifest available for multikernel DTB restoration\n"); @@ -682,15 +716,15 @@ int __init mk_instance_restore_from_manifest(void) int mk_node = fdt_subnode_offset(manifest, 0, "multikernel"); if (mk_node < 0) { - pr_info("No multikernel node found in manifest\n"); - ret = 0; + pr_err("No multikernel node found in supplied manifest\n"); + ret = -EINVAL; goto cleanup_fdt; } const void *dtb_data = fdt_getprop(manifest, mk_node, "dtb-data", &dtb_len); if (!dtb_data || dtb_len <= 0) { - pr_info("No dtb-data property found in multikernel node\n"); - ret = 0; + pr_err("No dtb-data property found in multikernel node\n"); + ret = -EINVAL; goto cleanup_fdt; } @@ -787,8 +821,25 @@ int __init mk_instance_restore_from_manifest(void) host_instance = mk_restore_host_instance(manifest); if (!host_instance) - pr_warn("Failed to restore host instance (spawn→host communication unavailable)\n"); - + mk_manifest_reject_and_park(-ENODEV); + + ret = mk_ipi_shared_validate(instance->ipi_data); + if (ret) + mk_manifest_reject_and_park(ret); + ret = mk_ipi_shared_validate(host_instance->ipi_data); + if (ret) + mk_manifest_reject_and_park(ret); + if (!atomic_read_acquire(&host_instance->ipi_data->ready) || + READ_ONCE(host_instance->ipi_data->ready_instance_id) != 0) + mk_manifest_reject_and_park(-EHOSTDOWN); + ret = mk_arch_prepare_park(); + if (ret) + mk_manifest_reject_and_park(ret); + if (!mk_arch_park_ready()) + mk_manifest_reject_and_park(-EIO); + ret = mk_register_stop_nmi_handler(); + if (ret) + mk_manifest_reject_and_park(ret); pr_info("Successfully restored multikernel root instance %d ('%s') from manifest (%d bytes)\n", instance_id, instance_name, dtb_len); mk_dt_config_free(&config); @@ -822,6 +873,8 @@ int __init mk_instance_restore_from_manifest(void) kfree(dtb_virt); cleanup_fdt: early_memunmap((void *)manifest, PAGE_SIZE); + if (ret) + mk_manifest_reject_and_park(ret); return ret; } diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 30a19ae3f23645..0d11f466c8141a 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -5,6 +5,14 @@ extern struct idr mk_instance_idr; extern struct list_head mk_instance_list; extern struct mk_instance *root_instance; +/* core.c */ +int mk_instance_force_halt(struct mk_instance *instance); + +/* ipi.c */ +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type); +void mk_poll_ipi_messages(void); + /* kernfs.c */ extern struct kernfs_node *mk_root_kn; extern struct kernfs_node *mk_instances_kn; diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 749500ff1f6a2f..69231f1cf65b8c 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "internal.h" /* Callback management */ @@ -20,41 +21,367 @@ static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lo static void mk_ipi_drain_ring(void); +#define MK_IPI_PRODUCER_RETRIES 10000 +#define MK_IPI_GATE_INDEX_BITS 6 +#define MK_IPI_GATE_INDEX_MASK (MK_IPI_RING_SIZE - 1) + /* - * Ring indices live in memory another kernel instance can write, so every - * read is masked before it indexes the entry array. An instance that dies - * mid-update must not be able to walk this kernel off the end of its ring. + * A nonzero gate records both the physical producer CPU and the slot at head. + * This makes the serialization recoverable after that exact CPU is confirmed + * parked. A boolean shared lock would be unsafe because the force-stop NMI may + * prevent its owner from ever returning to release it. */ -static inline unsigned int mk_ring_idx(unsigned int i) +static u64 mk_ipi_gate_token(mk_phys_cpu_t owner, unsigned int idx) +{ + BUILD_BUG_ON(BIT(MK_IPI_GATE_INDEX_BITS) != MK_IPI_RING_SIZE); + if (owner >= (U64_MAX >> MK_IPI_GATE_INDEX_BITS)) + return 0; + + return ((owner + 1) << MK_IPI_GATE_INDEX_BITS) | idx; +} + +static mk_phys_cpu_t mk_ipi_gate_owner(u64 token) +{ + return (token >> MK_IPI_GATE_INDEX_BITS) - 1; +} + +static unsigned int mk_ipi_gate_index(u64 token) +{ + return token & MK_IPI_GATE_INDEX_MASK; +} + +/* + * Serialize producers with preemption disabled so the physical owner encoded + * in the gate remains stable. Keep local IRQs enabled while waiting for a + * producer in another kernel, then disable them only for the short publish + * critical section. No NMI path sends general messages; force halt uses a + * persistent host-owned marker. + * Advancing head before READY lets recovery distinguish both interruption + * windows without allowing another producer to pass the gate. + */ +static int mk_ipi_ring_publish(struct mk_shared_data *shared, int instance_id, + const void *data, size_t data_size, + unsigned long type) +{ + struct mk_ipi_ring *ring = &shared->ring; + struct mk_ipi_data *slot; + mk_phys_cpu_t owner; + unsigned long flags; + bool contended = false; + unsigned int retry; + unsigned int idx; + u64 token, old; + int state; + int head; + int ret; + + preempt_disable(); + owner = arch_cpu_physical_id(smp_processor_id()); + + for (retry = 0; retry < MK_IPI_PRODUCER_RETRIES; retry++) { + head = atomic_read(&ring->head); + idx = head & MK_IPI_GATE_INDEX_MASK; + token = mk_ipi_gate_token(owner, idx); + if (!token) { + ret = -EOVERFLOW; + goto out_enable; + } + + old = atomic64_cmpxchg_acquire(&ring->producer_gate, 0, token); + if (!old) { + if ((atomic_read(&ring->head) & + MK_IPI_GATE_INDEX_MASK) != idx) { + atomic64_set_release(&ring->producer_gate, 0); + contended = true; + cpu_relax(); + continue; + } + break; + } + contended = true; + if (mk_ipi_gate_owner(old) == owner) { + ret = -EDEADLK; + goto out_count_contention; + } + cpu_relax(); + } + + if (retry == MK_IPI_PRODUCER_RETRIES) { + ret = -EAGAIN; + goto out_count_contention; + } + + local_irq_save(flags); + if (contended) + atomic_inc(&ring->producer_contention); + if (!atomic_read_acquire(&shared->ready) || + READ_ONCE(shared->ready_instance_id) != instance_id) { + ret = -ESHUTDOWN; + goto out_release_gate; + } + + slot = &ring->entries[idx]; + state = atomic_cmpxchg(&slot->state, MK_IPI_SLOT_EMPTY, + MK_IPI_SLOT_WRITING); + if (state != MK_IPI_SLOT_EMPTY) { + if (state == MK_IPI_SLOT_READY || + state == MK_IPI_SLOT_CONSUMING || + state == MK_IPI_SLOT_CANCELLED) { + atomic_inc(&ring->full_failures); + ret = -ENOSPC; + } else { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + goto out_release_gate; + } + + WRITE_ONCE(slot->data_size, 0); + WRITE_ONCE(slot->sender_cpu, owner); + WRITE_ONCE(slot->type, type); + if (data_size) + memcpy(slot->buffer, data, data_size); + WRITE_ONCE(slot->data_size, data_size); + atomic_set(&ring->head, (idx + 1) & MK_IPI_GATE_INDEX_MASK); + atomic_set_release(&slot->state, MK_IPI_SLOT_READY); + ret = 0; + +out_release_gate: + atomic64_set_release(&ring->producer_gate, 0); + local_irq_restore(flags); +out_enable: + preempt_enable(); + return ret; + +out_count_contention: + atomic_inc(&ring->producer_contention); + goto out_enable; +} + +static bool mk_ipi_slot_is_pending(int state) { - return i & (MK_IPI_RING_SIZE - 1); + return state == MK_IPI_SLOT_READY || + state == MK_IPI_SLOT_CANCELLED; +} + +static void mk_ipi_slot_release(struct mk_ipi_data *slot) +{ + WRITE_ONCE(slot->data_size, 0); + atomic_set_release(&slot->state, MK_IPI_SLOT_EMPTY); +} + +int mk_ipi_shared_validate(const struct mk_shared_data *shared) +{ + if (!shared) + return -ENODEV; + if (READ_ONCE(shared->abi_magic) != MK_IPI_ABI_MAGIC || + READ_ONCE(shared->abi_version) != MK_IPI_ABI_VERSION || + READ_ONCE(shared->abi_size) != sizeof(*shared)) + return -EPROTO; + + return 0; +} + +int mk_ipi_shared_mark_ready(struct mk_shared_data *shared, int instance_id) +{ + int ret; + + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + + WRITE_ONCE(shared->ready_instance_id, instance_id); + atomic_set_release(&shared->ready, 1); + return 0; +} + +int mk_ipi_shared_wait_ready(struct mk_shared_data *shared, int instance_id, + unsigned int timeout_ms) +{ + unsigned long deadline; + int ret; + + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + + deadline = jiffies + msecs_to_jiffies(timeout_ms); + do { + if (atomic_read_acquire(&shared->ready)) + return READ_ONCE(shared->ready_instance_id) == instance_id ? + 0 : -EPROTO; + msleep(20); + } while (time_before(jiffies, deadline)); + + return -ETIMEDOUT; +} + +int mk_ipi_shared_reset_downlink(struct mk_shared_data *shared) +{ + struct mk_ipi_ring *ring; + mk_phys_cpu_t owner; + unsigned long flags; + unsigned int retry; + unsigned int idx; + u64 token, old; + int head; + int ret = 0; + + if (!shared) + return -EINVAL; + + /* Exclude new publishers before waiting for an in-flight one. */ + atomic_set_release(&shared->ready, 0); + /* Pair exclusion with the readiness recheck after gate acquisition. */ + smp_mb(); + ring = &shared->ring; + preempt_disable(); + owner = arch_cpu_physical_id(smp_processor_id()); + for (retry = 0; retry < MK_IPI_PRODUCER_RETRIES; retry++) { + head = atomic_read(&ring->head); + idx = head & MK_IPI_GATE_INDEX_MASK; + token = mk_ipi_gate_token(owner, idx); + if (!token) { + ret = -EOVERFLOW; + goto out_enable; + } + old = atomic64_cmpxchg_acquire(&ring->producer_gate, 0, token); + if (!old) + break; + if (mk_ipi_gate_owner(old) == owner) { + ret = -EDEADLK; + goto out_enable; + } + cpu_relax(); + } + if (retry == MK_IPI_PRODUCER_RETRIES) { + ret = -EAGAIN; + goto out_enable; + } + + local_irq_save(flags); + /* The old receiver is parked and every pre-existing publisher drained. */ + mk_ipi_ring_reset_contents(ring); + WRITE_ONCE(shared->force_halt, 0); + WRITE_ONCE(shared->abi_magic, MK_IPI_ABI_MAGIC); + WRITE_ONCE(shared->abi_version, MK_IPI_ABI_VERSION); + WRITE_ONCE(shared->abi_size, sizeof(*shared)); + WRITE_ONCE(shared->ready_instance_id, -1); + atomic_set(&shared->ready, 0); + atomic64_set_release(&ring->producer_gate, 0); + + local_irq_restore(flags); +out_enable: + preempt_enable(); + return ret; } /** - * mk_ipi_ring_drop_pending - Discard everything queued in this kernel's ring + * mk_ipi_ring_recover_halted - Recover a ring producer after it is parked + * @halted_cpus: Exact set of CPUs confirmed parked by the caller * - * Called when an instance is re-spawned. A halting instance parks its CPUs - * wherever they were, including between claiming a ring slot and publishing - * it, and the drain stops at such a slot forever. Anything still queued was - * sent by a kernel that is gone, so drop it all rather than let one - * abandoned slot wedge the ring. + * A force-stop NMI can park a producer while it owns the shared gate. Only its + * receiver may recover the write, and only after the owner CPU is proven not + * to be executing it. Other instances may still publish into the host ring, + * so this function never resets the ring or touches another owner's gate. */ -void mk_ipi_ring_drop_pending(void) +int mk_ipi_ring_recover_halted(const struct mk_cpu_set *halted_cpus) { + struct mk_ipi_data *slot; struct mk_ipi_ring *ring; - unsigned int head, tail; - - if (!root_instance || !root_instance->ipi_data) - return; + mk_phys_cpu_t owner; + mk_phys_cpu_t target; + unsigned int idx; + unsigned int next; + u64 token; + int state; + int head; + int ret = 0; + + if (!halted_cpus || !root_instance || !root_instance->ipi_data) + return -EINVAL; ring = &root_instance->ipi_data->ring; - head = mk_ring_idx(atomic_read(&ring->head)); + token = atomic64_read_acquire(&ring->producer_gate); + if (!token) + goto kick; + if (!(token >> MK_IPI_GATE_INDEX_BITS)) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + goto kick; + } - for (tail = mk_ring_idx(atomic_read(&ring->tail)); tail != head; - tail = mk_ring_idx(tail + 1)) - ring->entries[tail].data_size = 0; + owner = mk_ipi_gate_owner(token); + if (!mk_cpu_set_contains(halted_cpus, owner)) + goto kick; + + idx = mk_ipi_gate_index(token); + next = (idx + 1) & MK_IPI_GATE_INDEX_MASK; + slot = &ring->entries[idx]; + state = atomic_read_acquire(&slot->state); + head = atomic_read(&ring->head) & MK_IPI_GATE_INDEX_MASK; + switch (state) { + case MK_IPI_SLOT_EMPTY: + if (head != idx && head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + case MK_IPI_SLOT_WRITING: + if (head == idx) { + atomic_set(&ring->head, next); + } else if (head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + break; + } + atomic_set_release(&slot->state, MK_IPI_SLOT_CANCELLED); + atomic_inc(&ring->cancelled_writes); + break; + case MK_IPI_SLOT_READY: + case MK_IPI_SLOT_CANCELLED: + if (head == idx) { + /* Repair an interrupted publication before releasing its gate. */ + atomic_set(&ring->head, next); + atomic_inc(&ring->invalid_state); + } else if (head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + case MK_IPI_SLOT_CONSUMING: + /* + * The consumer may claim a full-ring tail while this producer + * waits to test it, leaving head at idx with nothing published. + * Head at next means the previous publication was claimed before + * its now-stale gate could be released. Both cursors are valid. + */ + if (head != idx && head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + default: + atomic_inc(&ring->invalid_state); + ret = -EIO; + break; + } + + /* Keep the gate closed when cursor repair cannot make the FIFO safe. */ + if (ret) + goto kick; + if (atomic64_cmpxchg_release(&ring->producer_gate, token, 0) != token) { + atomic_inc(&ring->invalid_state); + ret = -EAGAIN; + } - atomic_set(&ring->tail, head); +kick: + /* The producer may have published and parked before ringing the bell. */ + target = mk_cpu_set_first(root_instance->cpus); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + + return ret; } /** @@ -166,7 +493,7 @@ int mk_arm_force_halt(struct mk_instance *instance) /** * multikernel_send_ipi_data - Send data to another CPU via IPI - * @instance_id: Target multikernel instance ID + * @instance: Target multikernel instance * @data: Pointer to data to send * @data_size: Size of data * @type: User-defined type identifier @@ -176,121 +503,119 @@ int mk_arm_force_halt(struct mk_instance *instance) * * Returns 0 on success, negative error code on failure */ -int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) { - struct mk_ipi_data *slot; - struct mk_instance *instance = mk_instance_find(instance_id); - unsigned int head, next_head, tail; mk_phys_cpu_t target; + int instance_id; + int ret; if (!instance) return -EINVAL; - if (data_size > MK_MAX_DATA_SIZE) { - mk_instance_put(instance); + instance_id = instance->id; + if (data_size > MK_MAX_DATA_SIZE || (data_size && !data)) return -EINVAL; - } target = mk_cpu_set_first(instance->cpus); if (target == MK_PHYS_CPU_INVALID) { pr_err("Instance %d has no CPUs to receive the IPI\n", instance_id); - mk_instance_put(instance); return -ENODEV; } if (!mk_instance_ipi_area(instance)) { pr_err("Multikernel IPI buffer not available for instance %d\n", instance_id); - mk_instance_put(instance); return -ENODEV; } + ret = mk_ipi_shared_validate(instance->ipi_data); + if (ret) + return ret; + if (!atomic_read_acquire(&instance->ipi_data->ready)) + return -EAGAIN; + if (READ_ONCE(instance->ipi_data->ready_instance_id) != instance_id) + return -EPROTO; + + ret = mk_ipi_ring_publish(instance->ipi_data, instance_id, data, + data_size, type); + if (ret) { + /* + * A doorbell can be coalesced while the target is draining this + * ring. Kick it again before reporting backpressure so READY + * entries cannot remain stranded without another notification. + */ + mk_arch_send_ipi(target); + if (ret == -ENOSPC) + pr_warn_ratelimited("multikernel: IPI ring full for instance %d\n", + instance_id); + else if (ret == -EAGAIN) + pr_warn_ratelimited("multikernel: IPI producer busy for instance %d\n", + instance_id); + else if (ret != -EDEADLK) + pr_err_ratelimited("multikernel: IPI publish failed for instance %d: %d\n", + instance_id, ret); + return ret; + } + mk_arch_send_ipi(target); - /* Try to enqueue the message in the ring buffer */ - do { - head = mk_ring_idx(atomic_read(&instance->ipi_data->ring.head)); - next_head = mk_ring_idx(head + 1); - tail = mk_ring_idx(atomic_read(&instance->ipi_data->ring.tail)); - - /* Check if ring buffer is full */ - if (next_head == tail) { - /* - * Console output reaches this path, so a plain printk - * here re-enters the console write that called us and - * deadlocks on its lock with interrupts already off. - */ - printk_deferred(KERN_WARNING - "multikernel: IPI ring full for instance %d (head=%u, tail=%u)\n", - instance_id, head, tail); - mk_instance_put(instance); - return -ENOSPC; - } - - /* Try to claim this slot atomically */ - } while (atomic_cmpxchg(&instance->ipi_data->ring.head, head, next_head) != head); - - /* We've claimed slot 'head', now fill it */ - slot = &instance->ipi_data->ring.entries[head]; - - slot->sender_cpu = arch_cpu_physical_id(smp_processor_id()); - slot->type = type; - - if (data && data_size > 0) - memcpy(slot->buffer, data, data_size); - - /* - * data_size publishes the slot: the reader treats a zero as "the - * producer has claimed this slot but has not filled it yet" and - * waits. Claiming the slot advanced head, so a reader can already - * be looking at it; everything above must be visible first. - */ - smp_store_release(&slot->data_size, data_size); + return 0; +} - mk_arch_send_ipi(target); +int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, + unsigned long type) +{ + struct mk_instance *instance; + int ret; + instance = mk_instance_find(instance_id); + if (!instance) + return -EINVAL; + ret = mk_send_ipi_data(instance, data, data_size, type); mk_instance_put(instance); - return 0; + return ret; } static void mk_ipi_drain_ring(void) { struct mk_ipi_data *slot; struct mk_ipi_handler *handler; - unsigned int head, tail, next_tail; + struct mk_ipi_ring *ring; + unsigned int tail, idx; size_t data_size; + int state; int messages_processed = 0; if (!root_instance || !root_instance->ipi_data) return; - while (1) { - tail = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.head)); - - if (tail == head) + ring = &root_instance->ipi_data->ring; + while (messages_processed < MK_IPI_RING_SIZE) { + tail = atomic_read(&ring->tail); + idx = tail & (MK_IPI_RING_SIZE - 1); + slot = &ring->entries[idx]; + + state = atomic_read_acquire(&slot->state); + if (!mk_ipi_slot_is_pending(state)) { + if (state != MK_IPI_SLOT_EMPTY && + state != MK_IPI_SLOT_WRITING && + state != MK_IPI_SLOT_CONSUMING) { + atomic_inc(&ring->invalid_state); + pr_warn_once("Multikernel IPI slot %u has bad state %d\n", + idx, state); + } break; + } - slot = &root_instance->ipi_data->ring.entries[tail]; - - /* - * Pairs with the store_release in multikernel_send_ipi_data(). - * Zero means the sender claimed this slot but has not - * finished writing it. Leave it alone: skipping it would - * drop the message it is about to publish. Its own IPI, or - * the next one, brings us back here. - * - * A sender stopped before publishing leaves its slot zero - * forever; mk_ipi_ring_drop_pending() clears those out when - * the instance is re-spawned. - */ - data_size = smp_load_acquire(&slot->data_size); - if (data_size == 0) + if (atomic_cmpxchg_acquire(&slot->state, state, + MK_IPI_SLOT_CONSUMING) != state) break; + if (state == MK_IPI_SLOT_CANCELLED) + goto advance_tail; + + data_size = READ_ONCE(slot->data_size); if (data_size > MK_MAX_DATA_SIZE) { pr_warn_once("Multikernel IPI slot %u has bad size %zu\n", - tail, data_size); - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&root_instance->ipi_data->ring.tail, next_tail); - continue; + idx, data_size); + goto advance_tail; } /* Dispatch to registered handler */ @@ -308,17 +633,29 @@ static void mk_ipi_drain_ring(void) raw_spin_unlock(&mk_handlers_lock); advance_tail: - /* Mark consumed so the slot reads as unpublished again */ - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&root_instance->ipi_data->ring.tail, next_tail); + mk_ipi_slot_release(slot); + atomic_set(&ring->tail, (idx + 1) & (MK_IPI_RING_SIZE - 1)); messages_processed++; - - if (messages_processed >= MK_IPI_RING_SIZE) - break; } } +void mk_poll_ipi_messages(void) +{ + unsigned long flags; + mk_phys_cpu_t target; + + if (!root_instance) + return; + target = mk_cpu_set_first(root_instance->cpus); + if (target == MK_PHYS_CPU_INVALID || + target != arch_cpu_physical_id(smp_processor_id())) + return; + + local_irq_save(flags); + mk_ipi_drain_ring(); + local_irq_restore(flags); +} + /** * multikernel_interrupt_handler - Handle the multikernel IPI * diff --git a/kernel/multikernel/manifest.c b/kernel/multikernel/manifest.c index 89a8ee0d2c59b3..19cdc625e9e04d 100644 --- a/kernel/multikernel/manifest.c +++ b/kernel/multikernel/manifest.c @@ -23,12 +23,18 @@ /* Physical address of the manifest this kernel booted with, 0 if none */ static phys_addr_t mk_manifest_fdt_phys; +static bool mk_manifest_fdt_rejected; phys_addr_t mk_manifest_phys(void) { return mk_manifest_fdt_phys; } +bool mk_manifest_rejected(void) +{ + return READ_ONCE(mk_manifest_fdt_rejected); +} + /** * mk_manifest_populate() - Accept the manifest handed over at boot * @fdt_phys: Physical address of the manifest FDT @@ -50,6 +56,7 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) if (!fdt) { pr_warn("multikernel: failed to memremap manifest (0x%llx)\n", fdt_phys); + err = -ENOMEM; goto out; } @@ -68,14 +75,17 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) } mk_manifest_fdt_phys = fdt_phys; + mk_manifest_fdt_rejected = false; pr_info("multikernel: manifest accepted\n"); out: if (fdt) early_memunmap(fdt, fdt_len); - if (err) - pr_warn("multikernel: ignoring invalid manifest\n"); + if (err) { + mk_manifest_fdt_rejected = true; + pr_warn("multikernel: supplied manifest rejected: %d\n", err); + } } /** diff --git a/kernel/multikernel/mem.c b/kernel/multikernel/mem.c index 5c2da8dadb12ed..51420befc51101 100644 --- a/kernel/multikernel/mem.c +++ b/kernel/multikernel/mem.c @@ -129,22 +129,15 @@ void *multikernel_create_instance_pool(int instance_id, size_t pool_size, int mi chunk_base = multikernel_alloc(chunk_size); if (!chunk_base) { - /* If we can't get the full remaining size, try smaller chunks */ - if (chunk_size > (1024 * 1024)) { - /* Try 1MB chunks */ - chunk_size = 1024 * 1024; - chunk_base = multikernel_alloc(chunk_size); - } - - if (!chunk_base && chunk_size > (256 * 1024)) { - /* Try 256KB chunks */ - chunk_size = 256 * 1024; - chunk_base = multikernel_alloc(chunk_size); - } - - if (!chunk_base && chunk_size > (1 << min_alloc_order)) { - /* Try minimum allocation size */ - chunk_size = 1 << min_alloc_order; + /* + * Fragmented pool: halve until a piece fits. Dropping + * straight to tiny chunks would splinter the grant + * into more regions than an E820 table can carry. + */ + while (!chunk_base && + chunk_size > (1UL << min_alloc_order)) { + chunk_size = ALIGN_DOWN(chunk_size / 2, + 1UL << min_alloc_order); chunk_base = multikernel_alloc(chunk_size); } @@ -345,6 +338,27 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size) * * Returns: 0 on success, -ENOENT if region not found */ +/* Does [phys_addr, phys_addr+size) back a segment of the loaded image? */ +static bool mk_range_backs_kimage(struct mk_instance *instance, + phys_addr_t phys_addr, size_t size) +{ + struct kimage *image = instance->kimage; + unsigned long i; + + if (!image) + return false; + + for (i = 0; i < image->nr_segments; i++) { + unsigned long start = image->segment[i].mem; + unsigned long end = start + image->segment[i].memsz; + + if (phys_addr < end && start < phys_addr + size) + return true; + } + + return false; +} + int mk_instance_remove_memory_region(struct mk_instance *instance, phys_addr_t phys_addr, size_t size) { @@ -354,6 +368,14 @@ int mk_instance_remove_memory_region(struct mk_instance *instance, if (!instance) return -EINVAL; + if (mk_range_backs_kimage(instance, phys_addr, size)) { + pr_err("Refusing to remove 0x%llx-0x%llx from instance %d (%s): the loaded kernel image lives there\n", + (unsigned long long)phys_addr, + (unsigned long long)(phys_addr + size - 1), + instance->id, instance->name); + return -EBUSY; + } + list_for_each_entry_safe(region, tmp, &instance->memory_regions, list) { if (region->res.start == phys_addr && resource_size(®ion->res) == size) {