multikernel: make IPI publication ordered and recoverable

arch/x86/boot/header.S

@@ -379,7 +379,13 @@ xloadflags:
 #define XLF56 0
 #endif
 
-			.word XLF0 | XLF1 | XLF23 | XLF4 | XLF56
+#ifdef CONFIG_MULTIKERNEL
+# define XLF_MK XLF_MULTIKERNEL_IPI
+#else
+# define XLF_MK 0
+#endif
+
+			.word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF_MK
 
 cmdline_size:   .long   COMMAND_LINE_SIZE-1     #length of the command line,
                                                 #added with boot protocol

arch/x86/include/asm/multikernel.h

@@ -10,6 +10,8 @@
 
 #ifndef __ASSEMBLY__
 
+#include <linux/build_bug.h>
+#include <linux/stddef.h>
 #include <linux/types.h>
 #include <linux/cpumask.h>
 #include <linux/sizes.h>
@@ -54,17 +56,21 @@ static inline int arch_cpu_from_physical_id(u64 phys_id)
  * from a page written by the host while running on CPUs parked by (possibly
  * differently built) spawn kernels.
  *
- * The fields fall into two classes that must not be mixed up:
+ * The fields fall into three classes that must not be mixed up:
  *
  *  - Anchor fields (self_phys, park_phys, park_cr3, ctrl_phys,
  *    ctrl_size): the context's own identity, written once when the
  *    context is set up. The spawn kernel reads them whenever it parks a
  *    CPU on halt or offline, so they must stay valid for the context's
  *    whole lifetime.
  *
- *  - Dispatch fields (everything else): the wake mailbox, rewritten for
- *    every publication and staged into registers by the CPU that claims
- *    it. Reparking gets its own repark_* dispatch fields precisely so a
+ *  - Primary boot data (bp and the calibration values appended after it):
+ *    written before the boot CPU is released and consumed while that kernel
+ *    initializes. Secondary and repark publications do not rewrite it.
+ *
+ *  - Dispatch fields (the remaining fixed-size fields): the wake mailbox,
+ *    rewritten for every publication and staged into registers by the CPU
+ *    that claims it. Reparking gets its own repark_* dispatch fields so a
  *    repark publication never overwrites the anchor: the two used to
  *    share fields, and a repark left the anchor pointing at another
  *    kernel's park area, which triple-faulted the next halt.
@@ -91,10 +97,23 @@ struct mk_spawn_context {
 	u32 flags;			/* MK_SPAWN_F_* flags */
 	u32 ready;			/* Signal flag */
 	u32 reserved;			/* Padding for alignment */
-	/* Variable-size struct last - size depends on kernel config */
+	/* Keep all existing context offsets unchanged. */
 	struct boot_params bp;		/* Standard x86 boot params */
+	/* Optional boot data belongs after boot_params, in the zeroed tail. */
+	unsigned long boot_lps;		/* Host delay loops per second */
+	unsigned long boot_cpu_khz;	/* Host CPU frequency calibration */
+	unsigned long boot_tsc_khz;	/* Host TSC frequency calibration */
+	unsigned long boot_apic_hz;	/* Host local APIC timer frequency */
+	u64 abi_magic;			/* Validated context producer */
 } __aligned(PAGE_SIZE);
 
+static_assert(offsetof(struct mk_spawn_context, bp) == 144);
+static_assert(offsetof(struct mk_spawn_context, boot_lps) ==
+	      144 + sizeof(struct boot_params));
+static_assert(offsetof(struct mk_spawn_context, abi_magic) ==
+	      144 + sizeof(struct boot_params) + 4 * sizeof(unsigned long));
+static_assert(sizeof(struct mk_spawn_context) == 2 * PAGE_SIZE);
+
 /* Pool park loop code, copied by the host into per-instance park pages */
 extern char mk_pool_park_start[];
 extern char mk_pool_park_end[];
@@ -109,6 +128,11 @@ void mk_park_cpu(void);
 /* True only in a spawn kernel whose boot context carries a park area */
 bool mk_cpu_parkable(void);
 
+/* Build a spawn E820 table from the instance's current memory grant */
+struct mk_instance;
+struct boot_params;
+int mk_e820_fill(struct mk_instance *instance, struct boot_params *params);
+
 /* Park an offlined pool CPU (host park area, or instance context) */
 void mk_pool_park_cpu(void);
 
@@ -142,7 +166,8 @@ void mk_set_spawn_context(struct mk_spawn_context *ctx,
 int mk_spawn_cpu(struct mk_instance *instance, int cpu,
 		 struct mk_spawn_context *ctx);
 
-/* Initialize boot context tracking in spawn kernel */
+/* Validate and initialize boot context tracking in spawn kernel */
+struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys);
 void mk_init_boot_context(phys_addr_t ctx_phys);
 
 /* Identity page table and trampoline setup */

arch/x86/include/uapi/asm/bootparam.h

@@ -25,6 +25,7 @@
 #define XLF_5LEVEL			(1<<5)
 #define XLF_5LEVEL_ENABLED		(1<<6)
 #define XLF_MEM_ENCRYPTION		(1<<7)
+#define XLF_MULTIKERNEL_IPI		0x0100
 
 #ifndef __ASSEMBLER__
 

arch/x86/kernel/kexec-bzimage64.c

@@ -125,63 +125,6 @@ static int setup_e820_entries(struct boot_params *params)
 	return 0;
 }
 
-static int setup_e820_entries_multikernel(struct kimage *image, struct boot_params *params)
-{
-	struct mk_instance *instance = image->mk_instance;
-	struct mk_memory_region *region;
-	unsigned int nr_e820_entries = 0;
-
-	/*
-	 * Don't include first 1MB in e820 for multikernel spawn kernels.
-	 * Multikernel doesn't use real-mode trampoline (init_trampoline is
-	 * skipped), and including unmapped low memory causes sparse_init()
-	 * to fail when trying to populate vmemmap for those sections.
-	 * Only include the assigned memory pool regions.
-	 *
-	 * However, we need at least 2 e820 entries or the kernel's
-	 * append_e820_table() will reject our map and fall back to
-	 * creating a fake memory map that includes low memory.
-	 * Add a small reserved region at 0x0 as a dummy entry.
-	 */
-	params->e820_table[nr_e820_entries].addr = 0;
-	params->e820_table[nr_e820_entries].size = 0x1000;
-	params->e820_table[nr_e820_entries].type = E820_TYPE_RESERVED;
-	nr_e820_entries++;
-
-	if (instance && !list_empty(&instance->memory_regions)) {
-		list_for_each_entry(region, &instance->memory_regions, list) {
-			if (nr_e820_entries >= E820_MAX_ENTRIES_ZEROPAGE) {
-				pr_warn("E820 table full, cannot add all memory regions\n");
-				break;
-			}
-
-			params->e820_table[nr_e820_entries].addr = region->res.start;
-			params->e820_table[nr_e820_entries].size = resource_size(&region->res);
-			params->e820_table[nr_e820_entries].type = E820_TYPE_RAM;
-
-			pr_info("Added memory region to e820: 0x%llx-0x%llx (%llu MB)\n",
-				(unsigned long long)region->res.start,
-				(unsigned long long)region->res.end,
-				(unsigned long long)resource_size(&region->res) >> 20);
-
-			nr_e820_entries++;
-		}
-	}
-
-	params->e820_entries = nr_e820_entries;
-
-	pr_info("Final multikernel e820 map has %d total entries:\n",
-		nr_e820_entries);
-	for (int i = 0; i < nr_e820_entries; i++) {
-		pr_info("  e820[%d]: 0x%llx-0x%llx type=%d\n", i,
-			params->e820_table[i].addr,
-			params->e820_table[i].addr + params->e820_table[i].size,
-			params->e820_table[i].type);
-	}
-
-	return 0;
-}
-
 enum { RNG_SEED_LENGTH = 32 };
 
 static void
@@ -611,6 +554,11 @@ static void *bzImage64_load(struct kimage *image, char *kernel,
 				  .buf_max = ULONG_MAX, .top_down = true };
 
 	header = (struct setup_header *)(kernel + setup_hdr_offset);
+	if (image->type == KEXEC_TYPE_MULTIKERNEL &&
+	    !(header->xloadflags & XLF_MULTIKERNEL_IPI)) {
+		pr_err("Loaded kernel lacks the required shared transport layout\n");
+		return ERR_PTR(-EPROTONOSUPPORT);
+	}
 	setup_sects = header->setup_sects;
 	if (setup_sects == 0)
 		setup_sects = 4;
@@ -804,7 +752,8 @@ static void *bzImage64_load(struct kimage *image, char *kernel,
 
 	/* For multikernel, setup custom e820 map */
 	if (image->type == KEXEC_TYPE_MULTIKERNEL) {
-		ret = setup_e820_entries_multikernel(image, params);
+		image->arch.mk_boot_params = bootparam_load_addr;
+		ret = mk_e820_fill(image->mk_instance, params);
 		if (ret)
 			goto out_free_params;
 	}

arch/x86/kernel/kexec-vmlinux.c

@@ -60,6 +60,12 @@ struct elf_kernel_info {
 	unsigned long reloc_size;   /* Size of relocation data */
 };
 
+struct mk_elf_note_desc {
+	u64 entry;
+	u32 ipi_abi_version;
+	u32 reserved;
+};
+
 /*
  * Find multikernel entry point from PT_NOTE section.
  * Looks for note with name "Linux" and type 0x4d4b ('MK').
@@ -93,12 +99,20 @@ static unsigned long find_multikernel_entry_note(const void *buf, size_t len,
 
 			if (nhdr->n_type == 0x4d4b &&
 			    nhdr->n_namesz == 6 &&
-			    nhdr->n_descsz == sizeof(u64) &&
+			    nhdr->n_descsz == sizeof(struct mk_elf_note_desc) &&
 			    !memcmp(ptr + sizeof(*nhdr), "Linux", 6)) {
-				u64 entry = *(u64 *)(ptr + sizeof(*nhdr) +
-						     ALIGN(nhdr->n_namesz, 4));
-				pr_info("multikernel: entry=0x%llx\n", entry);
-				return entry;
+				const struct mk_elf_note_desc *desc;
+
+				desc = ptr + sizeof(*nhdr) +
+				       ALIGN(nhdr->n_namesz, 4);
+				if (desc->ipi_abi_version != MK_IPI_ABI_VERSION) {
+					pr_err("multikernel IPI ABI %u is not supported\n",
+					       desc->ipi_abi_version);
+					return 0;
+				}
+				pr_info("multikernel: entry=0x%llx, IPI ABI=%u\n",
+					desc->entry, desc->ipi_abi_version);
+				return desc->entry;
 			}
 			ptr += note_size;
 		}
@@ -306,52 +320,6 @@ static int vmlinux_probe(const char *buf, unsigned long len)
 	return 0;
 }
 
-/*
- * Setup e820 memory map for multikernel spawn kernel
- */
-static int setup_e820_entries_multikernel(struct kimage *image, struct boot_params *params)
-{
-	struct mk_instance *instance = image->mk_instance;
-	struct mk_memory_region *region;
-	unsigned int nr_e820_entries = 0;
-	int i;
-
-	/*
-	 * Only include the assigned memory pool regions for multikernel spawn.
-	 * Don't include first 1MB - multikernel doesn't use real-mode trampoline
-	 * and including unmapped low memory causes sparse_init() to fail.
-	 *
-	 * The spawn kernel uses e820__memory_setup_multikernel() which accepts
-	 * any number of entries without fallback to legacy BIOS memory probing.
-	 */
-	if (instance && !list_empty(&instance->memory_regions)) {
-		list_for_each_entry(region, &instance->memory_regions, list) {
-			if (nr_e820_entries >= E820_MAX_ENTRIES_ZEROPAGE) {
-				pr_warn("Too many e820 entries, truncating\n");
-				break;
-			}
-
-			params->e820_table[nr_e820_entries].addr = region->res.start;
-			params->e820_table[nr_e820_entries].size = resource_size(&region->res);
-			params->e820_table[nr_e820_entries].type = E820_TYPE_RAM;
-			nr_e820_entries++;
-		}
-	}
-
-	params->e820_entries = nr_e820_entries;
-
-	pr_info("Final multikernel e820 map has %d total entries:\n",
-		nr_e820_entries);
-	for (i = 0; i < nr_e820_entries; i++) {
-		pr_info("  e820[%d]: 0x%llx-0x%llx type=%d\n", i,
-			params->e820_table[i].addr,
-			params->e820_table[i].addr + params->e820_table[i].size,
-			params->e820_table[i].type);
-	}
-
-	return 0;
-}
-
 /*
  * Load function - load ELF vmlinux and setup boot parameters
  */
@@ -606,7 +574,7 @@ static void *vmlinux_load(struct kimage *image, char *kernel,
 
 	/* For multikernel, setup custom e820 map */
 	if (image->type == KEXEC_TYPE_MULTIKERNEL) {
-		ret = setup_e820_entries_multikernel(image, params);
+		ret = mk_e820_fill(image->mk_instance, params);
 		if (ret) {
 			kvfree(ldata->kernel_buf);
 			kfree(ldata);

arch/x86/kernel/platform-quirks.c

@@ -14,8 +14,10 @@
 #include <asm/e820/api.h>
 #include <asm/apic.h>
 #include <asm/apicdef.h>
+#include <linux/cpufeature.h>
 #include <asm/mpspec.h>
 #include <asm/numa.h>
+#include <asm/page.h>
 #include <linux/pgtable.h>
 #include <asm/pgtable.h>
 #include <asm/pgtable_64_types.h>
@@ -28,6 +30,35 @@ extern pmd_t *populate_extra_pmd(unsigned long vaddr);
 extern unsigned long orig_boot_params;
 
 #ifdef CONFIG_MULTIKERNEL
+static unsigned long multikernel_cpu_khz;
+static unsigned long multikernel_tsc_khz;
+
+static unsigned long multikernel_calibrate_cpu(void)
+{
+	return multikernel_cpu_khz;
+}
+
+static unsigned long multikernel_calibrate_tsc(void)
+{
+	return multikernel_tsc_khz;
+}
+
+static void __init multikernel_setup_calibration(void)
+{
+	phys_addr_t ctx_phys = orig_boot_params -
+		offsetof(struct mk_spawn_context, bp);
+	struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys);
+
+	if (!ctx || !ctx->boot_tsc_khz)
+		return;
+
+	multikernel_tsc_khz = ctx->boot_tsc_khz;
+	multikernel_cpu_khz = ctx->boot_cpu_khz ?: ctx->boot_tsc_khz;
+	x86_platform.calibrate_cpu = multikernel_calibrate_cpu;
+	x86_platform.calibrate_tsc = multikernel_calibrate_tsc;
+	setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ);
+}
+
 /*
  * Custom wakeup for multikernel spawn kernels.
  * Uses shared spawn table instead of realmode trampoline.
@@ -105,6 +136,10 @@ static void __init multikernel_parse_smp_config(void)
 	 */
 	apic_update_callback(wakeup_secondary_cpu_64, multikernel_wakeup_cpu);
 }
+#else
+static inline void multikernel_setup_calibration(void)
+{
+}
 #endif /* CONFIG_MULTIKERNEL */
 
 void __init x86_early_init_platform_quirks(void)
@@ -135,6 +170,7 @@ void __init x86_early_init_platform_quirks(void)
 		x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT;
 		break;
 	case X86_SUBARCH_MULTIKERNEL:
+		multikernel_setup_calibration();
 		x86_platform.legacy.devices.pnpbios = 0;
 		x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT;
 		x86_platform.legacy.rtc = 0;
@@ -175,7 +211,9 @@ void __init x86_early_init_platform_quirks(void)
 		 * the PIT - which belongs to the host - and then request
 		 * legacy IRQ0, which can never reach an instance CPU that
 		 * has neither a PIC nor an IO-APIC. Ticks come from the
-		 * local APIC timer via setup_percpu_clockev() instead.
+		 * local APIC timer initialized by setup_percpu_clockev().
+		 * Keeping global_clock_event unset bypasses LAPIC timer
+		 * verification, whose fallback path requires legacy IRQ0.
 		 */
 		x86_init.timers.timer_init = x86_init_noop;
 		x86_init.timers.wallclock_init = x86_init_noop;

arch/x86/multikernel/Makefile

@@ -3,4 +3,4 @@
 # Makefile for multikernel spawn support
 #
 
-obj-y += spawn.o direct_boot.o head_64.o
+obj-y += spawn.o direct_boot.o head_64.o e820.o