build_mp: a GIGA build picks its M4 boot padding from 4 KB, not four sizes

build_mp.py

@@ -1168,25 +1168,34 @@ def m4_reaches_memory(sp, pc, nmi, hf):
     return any(lo <= w < hi for w in ((sp - 4) & 0xFFFFFFFF, pc & ~1, hf & ~1) for lo, hi in M4_BUS_REGIONS)
 
 
-#: Padding tried, in bytes, when an image's M4 vector table reaches memory.
-M4_PAD_SIZES = (16, 32, 48, 64)
+#: Paddings tried step by this many bytes: a multiple of every alignment in
+#: .text (8 at most on the GIGA), so a pad that grows by it moves all the code
+#: after it on by just as much, and the bytes it brings to the M4's address can
+#: be read off the image already linked.
+M4_PAD_STEP = 16
+#: The most padding tried. The windows it can bring to the M4's address lie 16
+#: bytes apart over 4 KB, so no literal pool or table of addresses (a few dozen
+#: bytes) can refuse them all, however the image grows around them.
+M4_PAD_MAX = 4096
+#: Relinks tried before giving up, when the bytes read off the image weren't
+#: what the relink put there.
+M4_PAD_TRIES = 4
 M4_PAD_SYMBOL = "pydevices_m4_pad"
 
 
 def stm32_m4_pad_module(build, pad):
-    """A generated C module whose only code is *pad* bytes of nops, kept by
-    the linker and placed with the other modules, before the M4's address:
-    it moves the code there. With pad 0 it compiles nothing. Returns its
-    directory, for USER_C_MODULES; files are rewritten only when they change."""
+    """A generated C module whose only code is a function with *pad* bytes of
+    nops, kept by the linker and placed with the other modules, before the M4's
+    address: growing it moves the code there. Returns its directory, for
+    USER_C_MODULES; files are rewritten only when they change."""
     d = build / "m4pad-src" / "m4pad"
     d.mkdir(parents=True, exist_ok=True)
-    mk = "# Generated by build_mp.py: padding that moves the code at the M4's boot address.\n"
-    c = "// Generated by build_mp.py: padding that moves the code at the M4's boot address.\n"
-    if pad:
-        mk += f"SRC_USERMOD_C += $(USERMOD_DIR)/m4pad.c\nLDFLAGS_USERMOD += -Wl,-u,{M4_PAD_SYMBOL}\n"
-        c += (f"void {M4_PAD_SYMBOL}(void);\n"
-              f"void {M4_PAD_SYMBOL}(void) {{\n"
-              f"    __asm volatile (\".rept {pad // 2}\\n nop\\n .endr\");\n}}\n")
+    mk = ("# Generated by build_mp.py: padding that moves the code at the M4's boot address.\n"
+          f"SRC_USERMOD_C += $(USERMOD_DIR)/m4pad.c\nLDFLAGS_USERMOD += -Wl,-u,{M4_PAD_SYMBOL}\n")
+    c = ("// Generated by build_mp.py: padding that moves the code at the M4's boot address.\n"
+         f"void {M4_PAD_SYMBOL}(void);\n"
+         f"void {M4_PAD_SYMBOL}(void) {{\n"
+         f"    __asm volatile (\".rept {pad // 2}\\n nop\\n .endr\");\n}}\n")
     for name, text in (("micropython.mk", mk), ("m4pad.c", c)):
         f = d / name
         if not f.exists() or f.read_text() != text:
@@ -1201,22 +1210,70 @@ def stm32_m4_pad_now(build):
     return int(m.group(1)) * 2 if m else 0
 
 
-def stm32_m4_settle(image, text, pad, relink):
+def elf_symbol(elf, name):
+    """The value of symbol *name* in a 32-bit little-endian ELF file, or None."""
+    data = Path(elf).read_bytes() if Path(elf).exists() else b""
+    if data[:6] != b"\x7fELF\x01\x01":
+        return None
+    shoff, = struct.unpack_from("<I", data, 0x20)
+    shentsize, shnum = struct.unpack_from("<HH", data, 0x2E)
+    sections = [struct.unpack_from("<IIIIIIIIII", data, shoff + i * shentsize) for i in range(shnum)]
+    want = name.encode() + b"\0"
+    for _name, kind, _flags, _addr, off, size, link, _info, _align, entsize in sections:
+        if kind != 2:  # SHT_SYMTAB
+            continue
+        strtab = sections[link][4]
+        for at in range(off, off + size, entsize or 16):
+            st_name, value = struct.unpack_from("<II", data, at)
+            if data.startswith(want, strtab + st_name):
+                return value
+    return None
+
+
+def stm32_m4_pad_candidates(image, text, pad, pad_at):
+    """The paddings, smallest first, that would leave a vector table that
+    doesn't reach memory at the M4's boot address, read off *image* (linked
+    with *pad* bytes of padding, the padding function at address *pad_at*):
+    with p bytes, the code that was p - pad bytes before that address is there.
+    None when the padding lies after the address and can't move it."""
+    boot = STM32_M4_BOOT_DEFAULT
+    if pad_at is None or not text <= pad_at & ~1 < boot:
+        return None
+    pad_at &= ~1  # a Thumb function's address has its low bit set
+    data = Path(image).read_bytes()
+    out = []
+    for p in range(0, M4_PAD_MAX + 1, M4_PAD_STEP):
+        src = boot - text - (p - pad)
+        # The window must come from code after the padding, wholly in the image.
+        if src < pad_at - text + pad + 16 or src + 16 > len(data):
+            continue
+        if not m4_reaches_memory(*struct.unpack_from("<IIII", data, src)):
+            out.append(p)
+    return out
+
+
+def stm32_m4_settle(image, text, pad, pad_at, relink):
     """Make sure *image* doesn't hand the M4 a vector table that reaches
-    memory. Tries the current *pad* first, then each of 0 and M4_PAD_SIZES,
-    calling relink(pad) -> make's exit code for each. Returns (pad, words,
-    refused words or None), or (None, words, None) when relink failed."""
-    first = None
-    for p in [pad] + [q for q in (0,) + M4_PAD_SIZES if q != pad]:
-        if p != pad:
-            rc = relink(p)
-            if rc != 0:
-                return None, None, first
-        words = stm32_m4_boot_words(image, text)
+    memory. Keeps the current *pad* when its table passes; otherwise picks the
+    smallest padding whose table would pass, read off the image
+    (stm32_m4_pad_candidates), and calls relink(pad) -> make's exit code, up to
+    M4_PAD_TRIES times. *pad_at* () returns the padding function's address in
+    the image linked now. Returns (pad, words, refused words or None), or
+    (None, None, refused words) when relink failed."""
+    words = stm32_m4_boot_words(image, text)
+    if not (words and m4_reaches_memory(*words)):
+        return pad, words, None
+    first, tried = words, {pad}
+    for _ in range(M4_PAD_TRIES):
+        p = next((q for q in stm32_m4_pad_candidates(image, text, pad, pad_at()) or () if q not in tried), None)
+        if p is None:
+            break
+        tried.add(p)
+        if relink(p) != 0:
+            return None, None, first
+        pad, words = p, stm32_m4_boot_words(image, text)
         if not (words and m4_reaches_memory(*words)):
-            return p, words, first
-        first = first or words
-        pad = p
+            return pad, words, first
     return pad, words, first
 
 
@@ -2296,15 +2353,16 @@ def relink(p):
                     (build / "firmware.elf").unlink(missing_ok=True)
                     return shell(make, log=True)[0]
 
-                pad, words, refused = stm32_m4_settle(image, STM32_M4_SLOT[board][1], pad_before, relink)
+                pad, words, refused = stm32_m4_settle(image, STM32_M4_SLOT[board][1], pad_before,
+                                                      lambda: elf_symbol(build / "firmware.elf", M4_PAD_SYMBOL), relink)
                 if pad is None:
                     rc = 1
                 elif words and m4_reaches_memory(*words):
                     die(f"{image.name} hands the M4 core a vector table that reaches memory (stack {hex(words[0])}, "
                         f"reset {hex(words[1])}, HardFault {hex(words[3])}) at {hex(STM32_M4_BOOT_DEFAULT)}, where "
                         "the M4 of a board as shipped starts at reset on its own (BOOT_CM4_ADD0, with BCM4 set), "
-                        f"and none of {', '.join(str(p) for p in M4_PAD_SIZES)} bytes of padding moved it clear. "
-                        "Change the build a little, or turn BCM4 off on your board: the flash guide says how.")
+                        f"and no padding up to {M4_PAD_MAX:,} bytes before it moved it clear. Look at what "
+                        f"{build / 'firmware.map'} puts there, or turn BCM4 off on your board: the flash guide says how.")
                 else:
                     stm32_m4_pad = pad
                     if refused:

tests/test_build_mp.py

@@ -816,46 +816,118 @@ def test_a_stack_in_memory_is_refused_whatever_the_reset_address(self):
     def test_the_m7s_tightly_coupled_memory_is_out_of_the_m4s_reach(self):
         self.assertFalse(build_mp.m4_reaches_memory(0x20010000, 0x00000101, 0, 0x20004791))
 
+    # Where the padding function sits in the fake images below, well before the
+    # M4's address, as it does in a real GIGA image.
+    PAD_AT = 0x080405ED
+
+    def _code(self, windows):
+        """An image whose code before the M4's address is *windows*, 16-byte
+        vector tables in order, the last one at the address."""
+        off = build_mp.STM32_M4_BOOT_DEFAULT - 0x08040000
+        body = b"".join(struct.pack("<IIII", *w) for w in windows)
+        image = self.build / "firmware0.bin"
+        self.build.mkdir(parents=True, exist_ok=True)
+        image.write_bytes(bytes(off + 16 - len(body)) + body + bytes(64))
+        return image
+
     def test_a_refused_image_is_padded_until_one_passes(self):
         ulab = self.IMAGES["ulab, hung at the DAC"][0]
         moved = self.IMAGES["ulab moved 16 bytes, ran"][0]
-        image = self._image(ulab)
+        # 64 bytes of literal pool full of flash addresses, and code before it.
+        pool = (0x081F3C58, 0x081DF094, 0x081DD7B0, 0x081DCF18)
+        windows = [moved] + [pool] * 4 + [ulab]
+        image = self._code(windows)
+        links = []
+
+        def relink(pad):
+            links.append(pad)
+            # Each 16 bytes of padding brings the window before to the address.
+            self._code(windows[:len(windows) - pad // 16])
+            return 0
+
+        pad, words, refused = build_mp.stm32_m4_settle(image, 0x08040000, 0, lambda: self.PAD_AT, relink)
+        self.assertEqual((pad, words, refused), (80, moved, ulab))
+        self.assertEqual(links, [80])  # read off the image: one relink, not five
+
+    def test_a_prediction_the_relink_doesnt_bear_out_is_tried_again(self):
+        ulab = self.IMAGES["ulab, hung at the DAC"][0]
+        moved = self.IMAGES["ulab moved 16 bytes, ran"][0]
+        image = self._code([moved, moved, ulab])
         links = []
 
         def relink(pad):
             links.append(pad)
-            stm32 = moved if pad == 32 else ulab  # 16 still refused, 32 passes
-            image.write_bytes(bytes(build_mp.STM32_M4_BOOT_DEFAULT - 0x08040000) + struct.pack("<IIII", *stm32))
+            # The first relink moves the code less than the image said it would.
+            self._code([moved, ulab] if pad == 16 else [ulab, moved])
             return 0
 
-        pad, words, refused = build_mp.stm32_m4_settle(image, 0x08040000, 0, relink)
-        self.assertEqual((pad, words, refused), (32, moved, ulab))
+        pad, words, _refused = build_mp.stm32_m4_settle(image, 0x08040000, 0, lambda: self.PAD_AT, relink)
         self.assertEqual(links, [16, 32])
+        self.assertEqual((pad, words), (32, moved))
 
     def test_an_image_that_passes_is_not_relinked(self):
         image = self._image(self.IMAGES["lean, ran"][0])
-        pad, _words, refused = build_mp.stm32_m4_settle(image, 0x08040000, 0, lambda p: self.fail("relinked"))
+        pad, _words, refused = build_mp.stm32_m4_settle(image, 0x08040000, 0, lambda: self.PAD_AT,
+                                                        lambda p: self.fail("relinked"))
         self.assertEqual((pad, refused), (0, None))
 
-    def test_every_size_refused_leaves_the_image_refused(self):
+    def test_a_smaller_padding_is_taken_when_it_passes(self):
         ulab = self.IMAGES["ulab, hung at the DAC"][0]
-        image = self._image(ulab)
-        links = []
-        pad, words, _refused = build_mp.stm32_m4_settle(image, 0x08040000, 0, lambda p: links.append(p) or 0)
-        self.assertEqual(links, list(build_mp.M4_PAD_SIZES))
+        lean = self.IMAGES["lean, ran"][0]
+        # Linked with 48 bytes; with 32 the window after the address comes back.
+        off = build_mp.STM32_M4_BOOT_DEFAULT - 0x08040000
+        image = self.build / "firmware0.bin"
+        self.build.mkdir(parents=True, exist_ok=True)
+        after = [ulab, lean, ulab, ulab, ulab]
+        image.write_bytes(bytes(off) + b"".join(struct.pack("<IIII", *w) for w in after) + bytes(64))
+        self.assertEqual(build_mp.stm32_m4_pad_candidates(image, 0x08040000, 48, self.PAD_AT)[0], 32)
+
+    def test_no_padding_that_passes_leaves_the_image_refused(self):
+        ulab = self.IMAGES["ulab, hung at the DAC"][0]
+        image = self._code([ulab] * (build_mp.M4_PAD_MAX // 16 + 1))
+        pad, words, _refused = build_mp.stm32_m4_settle(image, 0x08040000, 0, lambda: self.PAD_AT,
+                                                        lambda p: self.fail("relinked"))
+        self.assertEqual(pad, 0)
         self.assertTrue(build_mp.m4_reaches_memory(*words))
 
+    def test_padding_after_the_m4s_address_cant_move_it(self):
+        image = self._code([self.IMAGES["lean, ran"][0], self.IMAGES["ulab, hung at the DAC"][0]])
+        self.assertIsNone(build_mp.stm32_m4_pad_candidates(image, 0x08040000, 0, 0x08100101))
+        self.assertIsNone(build_mp.stm32_m4_pad_candidates(image, 0x08040000, 0, None))
+
     def test_the_padding_module(self):
         d = build_mp.stm32_m4_pad_module(self.build, 0)
-        self.assertNotIn("SRC_USERMOD_C", (d / "micropython.mk").read_text())
-        self.assertEqual(build_mp.stm32_m4_pad_now(self.build), 0)
-        build_mp.stm32_m4_pad_module(self.build, 48)
         mk = (d / "micropython.mk").read_text()
+        # Compiled even when empty, so the next build can find where it is.
         self.assertIn("SRC_USERMOD_C += $(USERMOD_DIR)/m4pad.c", mk)
         self.assertIn("-Wl,-u," + build_mp.M4_PAD_SYMBOL, mk)
+        self.assertIn(".rept 0", (d / "m4pad.c").read_text())
+        self.assertEqual(build_mp.stm32_m4_pad_now(self.build), 0)
+        build_mp.stm32_m4_pad_module(self.build, 48)
         self.assertIn(".rept 24", (d / "m4pad.c").read_text())
         self.assertEqual(build_mp.stm32_m4_pad_now(self.build), 48)
 
+    def test_a_symbol_is_read_from_an_elf_file(self):
+        names = b"\0" + build_mp.M4_PAD_SYMBOL.encode() + b"\0other\0"
+        syms = struct.pack("<IIIBBH", 0, 0, 0, 0, 0, 0) + struct.pack("<IIIBBH", 1, self.PAD_AT, 8, 0x12, 0, 1)
+        shstr = b"\0"
+        head = bytearray(52)
+        head[:6] = b"\x7fELF\x01\x01"
+        strtab_off = 52
+        symtab_off = strtab_off + len(names)
+        shoff = symtab_off + len(syms)
+        struct.pack_into("<I", head, 0x20, shoff)
+        struct.pack_into("<HH", head, 0x2E, 40, 3)
+        sh = (bytes(40)
+              + struct.pack("<IIIIIIIIII", 0, 3, 0, 0, strtab_off, len(names), 0, 0, 1, 0)
+              + struct.pack("<IIIIIIIIII", 0, 2, 0, 0, symtab_off, len(syms), 1, 1, 4, 16))
+        elf = self.build / "firmware.elf"
+        self.build.mkdir(parents=True, exist_ok=True)
+        elf.write_bytes(bytes(head) + names + syms + sh + shstr)
+        self.assertEqual(build_mp.elf_symbol(elf, build_mp.M4_PAD_SYMBOL), self.PAD_AT)
+        self.assertIsNone(build_mp.elf_symbol(elf, "missing"))
+        self.assertIsNone(build_mp.elf_symbol(self.build / "nothing.elf", "x"))
+
 
 class Stm32BoardDir(unittest.TestCase):
     """A variant of ours on stm32 gets a board directory: upstream's, linked