From cb299a30c79b7213f4c752dd308122058d6adb65 Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:16:55 -0600 Subject: [PATCH 1/3] firmware flash: reach a native-USB ESP32 through its ROM port; take --before/--after A board running MicroPython on its native USB shows TinyUSB's CDC (303a:4001), which ignores DTR and RTS, so esptool's default reset kept talking to the interpreter and failed with "Invalid head of packet". firmware flash now sends such a board to its bootloader the way `mpftp bootloader` does, waits for the ROM's USB-Serial/JTAG port (303a:1001) whose serial number carries the same MAC, and flashes there. A UART bridge, a board already in its ROM, and --before no-reset are flashed where they are. The outer firmware flash parser now declares --before and --after and passes them to the engine, which already accepted them. Checked on an ESP32-S3 running MicroPython on native USB: an app-only flash from its running state went to the ROM port, wrote and verified, and the board booted back to its app with its files in place. Unit tests cover the port matching, the fallbacks and the option passthrough. Closes #208 Closes #209 --- cli/src/mpftp/cli.py | 111 ++++++++++++++++ cli/tests/test_flash_native_usb.py | 141 +++++++++++++++++++++ docs/guides/download-and-flash-firmware.md | 11 +- 3 files changed, 260 insertions(+), 3 deletions(-) create mode 100644 cli/tests/test_flash_native_usb.py diff --git a/cli/src/mpftp/cli.py b/cli/src/mpftp/cli.py index 4bcbe5a..b865535 100755 --- a/cli/src/mpftp/cli.py +++ b/cli/src/mpftp/cli.py @@ -2313,6 +2313,96 @@ def format_tree(tree: dict, port: str, board: str, variant: str, interpreter: st return None +# MicroPython's own USB CDC on a native-USB ESP32 (TinyUSB, Espressif's VID), +# and the ROM's USB-Serial/JTAG port the chip shows in download mode. The CDC +# ignores DTR and RTS, so esptool's reset can't reach the ROM through it. +_ESP_TINYUSB_CDC = (0x303A, 0x4001) +_ESP_ROM_USB_JTAG = (0x303A, 0x1001) +_ROM_PORT_WAIT_S = 20.0 + + +def _usb_id(port: dict) -> tuple[Any, Any]: + return port.get("vid"), port.get("pid") + + +def _chip_mac(serial: Any) -> str: + """The MAC in a USB serial number, as 12 hex digits ("" when there is none). + + MicroPython's CDC reports it as ``C04E3003F6300000`` and the ROM as + ``C0:4E:30:03:F6:30``; both start with the chip's MAC. + """ + digits = re.sub(r"[^0-9A-Fa-f]", "", str(serial or "")).upper() + return digits[:12] if len(digits) >= 12 else "" + + +def _wants_rom_port(ns: argparse.Namespace) -> bool: + """True when this flash should first send the board to its ROM loader.""" + if getattr(ns, "port", "") != "esp32" or getattr(ns, "uf2", False): + return False + if (getattr(ns, "before", "") or "") == "no-reset": + return False # the caller says the board is already in its ROM + device = ns.device or "" + return "://" not in device + + +def _find_rom_port(ports: list[dict], mac: str, before: set[str]) -> Optional[str]: + """The ROM download port for the chip with ``mac``, or a new one if no MAC.""" + for p in ports: + if _usb_id(p) != _ESP_ROM_USB_JTAG: + continue + if mac and _chip_mac(p.get("serial_number")) == mac: + return p["device"] + if not mac and p["device"] not in before: + return p["device"] + return None + + +def _enter_esp_rom(device: str, baud: int) -> str: + """Return the port to flash: ``device``, or the ROM port it reboots onto. + + A board on a UART bridge, or already in its ROM, flashes where it is. One + running MicroPython on its native USB is sent to its bootloader the way + ``mpftp bootloader`` does it, and then flashed on the ROM's own port, + picked by the MAC in its USB serial number. + """ + client, mode = get_client() + try: + ports = client.call("list_ports") or [] + cur = next((p for p in ports if p.get("device") == device), None) + if cur is None or _usb_id(cur) != _ESP_TINYUSB_CDC: + return device + mac = _chip_mac(cur.get("serial_number")) + before = {p.get("device") for p in ports} + print( + f"[mpftp] {device} is MicroPython's native USB, which esptool can't " + "reset: sending the board to its bootloader", + file=sys.stderr, + ) + ensure_device(client, device, baud) + res = client.call("bootloader") + if not res.get("ok"): + _die( + f"could not send {device} to its bootloader: {res.get('error')}. " + "Hold BOOT while tapping RESET, then flash the port it comes back on." + ) + deadline = time.monotonic() + _ROM_PORT_WAIT_S + while time.monotonic() < deadline: + time.sleep(0.7) + rom = _find_rom_port(client.call("list_ports") or [], mac, before) + if rom: + print(f"[mpftp] flashing on the ROM download port {rom}", file=sys.stderr) + return rom + _die( + f"{device} went to its bootloader, but no ROM download port " + f"(USB 303a:1001) appeared within {_ROM_PORT_WAIT_S:.0f} s. Hold BOOT " + "while tapping RESET, then flash the port it comes back on." + ) + finally: + if mode.startswith("sidecar"): + client.close() + return device # not reached: _die raises + + def cmd_firmware(ns: argparse.Namespace) -> None: sub = ns.fw_cmd if sub == "list": @@ -2392,6 +2482,12 @@ def cmd_firmware(ns: argparse.Namespace) -> None: extra.append("--uf2") if getattr(ns, "uf2_timeout", 0): extra += ["--uf2-timeout", str(ns.uf2_timeout)] + for name in ("before", "after"): + if getattr(ns, name, ""): + extra += [f"--{name}", getattr(ns, name)] + if ns.device and _wants_rom_port(ns): + rom = _enter_esp_rom(ns.device, ns.baud) + extra[extra.index("--device") + 1] = rom res = _engine_stream("flash", extra) out(res) if not res.get("ok"): @@ -3111,6 +3207,21 @@ def build_parser() -> argparse.ArgumentParser: help="Copy a .uf2 to a bootloader volume instead of flashing over serial") fwf.add_argument("--uf2-timeout", dest="uf2_timeout", type=float, default=0.0, help="Seconds to wait for the volume to unmount (default 30)") + fwf.add_argument( + "--before", + default="", + type=lambda s: str(s).replace("_", "-"), + choices=["", "default-reset", "no-reset", "usb-reset"], + help="esp32: esptool's reset before flashing (default default-reset; no-reset " + "for a board already in its ROM loader)", + ) + fwf.add_argument( + "--after", + default="", + type=lambda s: str(s).replace("_", "-"), + choices=["", "hard-reset", "soft-reset", "no-reset", "watchdog-reset"], + help="esp32: esptool's reset after flashing (default hard-reset)", + ) fwf.set_defaults(func=cmd_firmware) fwdt = fwsub.add_parser( diff --git a/cli/tests/test_flash_native_usb.py b/cli/tests/test_flash_native_usb.py new file mode 100644 index 0000000..f0eafee --- /dev/null +++ b/cli/tests/test_flash_native_usb.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: 2026 Brad Barnett +# +# SPDX-License-Identifier: MIT + +"""`firmware flash` reaches a native-USB ESP32 running MicroPython (mpftp#208), +and passes --before and --after through to the engine (mpftp#209). + +MicroPython's TinyUSB CDC (303A:4001) ignores DTR and RTS, so esptool's reset +keeps talking to the running interpreter. The flash has to send the board to +its bootloader and flash the ROM's port (303A:1001) it comes back on. +""" + +from __future__ import annotations + +import io +import unittest +from contextlib import redirect_stderr +from unittest import mock + +from mpftp import cli + +CDC = {"device": "COM104", "vid": 0x303A, "pid": 0x4001, "serial_number": "C04E3003F6300000"} +OTHER_CDC = {"device": "COM105", "vid": 0x303A, "pid": 0x4001, "serial_number": "70041DAD86C40000"} +ROM = {"device": "COM103", "vid": 0x303A, "pid": 0x1001, "serial_number": "C0:4E:30:03:F6:30"} +OTHER_ROM = {"device": "COM102", "vid": 0x303A, "pid": 0x1001, "serial_number": "70:04:1D:AD:86:C4"} +UART = {"device": "COM7", "vid": 0x10C4, "pid": 0xEA60, "serial_number": "0001"} + + +class FakeClient: + """Answers list_ports from a script of port lists, and records calls.""" + + def __init__(self, port_lists, bootloader_ok=True): + self.port_lists = list(port_lists) + self.calls = [] + self.bootloader_ok = bootloader_ok + + def call(self, method, params=None): + self.calls.append(method) + if method == "list_ports": + return self.port_lists.pop(0) if len(self.port_lists) > 1 else self.port_lists[0] + if method == "connect": + return {"ok": True} + if method == "bootloader": + return {"ok": self.bootloader_ok, "error": None if self.bootloader_ok else "no REPL"} + raise AssertionError(method) + + def close(self): + pass + + +def run_flash(argv, client, engine_result=None): + """Parse ``argv`` with the real parser, run it, return the engine's args.""" + ns = cli.build_parser().parse_args(argv) + seen = {} + + def engine(cmd, extra): + seen["cmd"], seen["extra"] = cmd, extra + return engine_result or {"ok": True} + + with mock.patch.object(cli, "get_client", return_value=(client, "sidecar")), \ + mock.patch.object(cli, "_engine_stream", side_effect=engine), \ + mock.patch.object(cli, "_sel_args", return_value=["--port", ns.port]), \ + mock.patch.object(cli, "boards"), \ + mock.patch.object(cli.time, "sleep"), \ + mock.patch.object(cli, "out"), \ + redirect_stderr(io.StringIO()): + cli.cmd_firmware(ns) + return seen["extra"] + + +BASE = ["firmware", "flash", "--port", "esp32", "--artifact", "fw.bin", "--offset", "0x10000"] + + +class NativeUsbFlash(unittest.TestCase): + def test_native_usb_board_is_flashed_on_its_rom_port(self): + client = FakeClient([[CDC, OTHER_CDC, OTHER_ROM], [OTHER_CDC, OTHER_ROM], [OTHER_CDC, OTHER_ROM, ROM]]) + extra = run_flash(BASE + ["-d", "COM104"], client) + self.assertIn("bootloader", client.calls) + self.assertEqual(extra[extra.index("--device") + 1], "COM103") + + def test_rom_port_is_matched_by_mac_not_by_being_new(self): + # Another board's ROM port appears first; it is not this board. + client = FakeClient([[CDC], [OTHER_ROM], [OTHER_ROM, ROM]]) + extra = run_flash(BASE + ["-d", "COM104"], client) + self.assertEqual(extra[extra.index("--device") + 1], "COM103") + + def test_uart_bridge_board_is_flashed_where_it_is(self): + client = FakeClient([[UART]]) + extra = run_flash(BASE + ["-d", "COM7"], client) + self.assertNotIn("bootloader", client.calls) + self.assertEqual(extra[extra.index("--device") + 1], "COM7") + + def test_board_already_in_rom_is_flashed_where_it_is(self): + client = FakeClient([[ROM]]) + extra = run_flash(BASE + ["-d", "COM103"], client) + self.assertNotIn("bootloader", client.calls) + self.assertEqual(extra[extra.index("--device") + 1], "COM103") + + def test_no_reset_skips_the_bootloader_step(self): + client = FakeClient([[CDC]]) + extra = run_flash(BASE + ["-d", "COM104", "--before", "no-reset"], client) + self.assertEqual(client.calls, []) + self.assertEqual(extra[extra.index("--device") + 1], "COM104") + + def test_no_rom_port_fails_without_flashing_the_dead_port(self): + client = FakeClient([[CDC], [OTHER_ROM]]) + with mock.patch.object(cli, "_ROM_PORT_WAIT_S", 0.0), \ + mock.patch.object(cli.time, "monotonic", side_effect=[0.0, 1.0, 2.0]): + with self.assertRaises(SystemExit): + run_flash(BASE + ["-d", "COM104"], client) + + def test_refused_bootloader_fails(self): + client = FakeClient([[CDC]], bootloader_ok=False) + with self.assertRaises(SystemExit): + run_flash(BASE + ["-d", "COM104"], client) + + +class ResetModes(unittest.TestCase): + def test_before_and_after_reach_the_engine(self): + client = FakeClient([[ROM]]) + extra = run_flash(BASE + ["-d", "COM103", "--before", "no_reset", "--after", "watchdog-reset"], client) + self.assertEqual(extra[extra.index("--before") + 1], "no-reset") + self.assertEqual(extra[extra.index("--after") + 1], "watchdog-reset") + + def test_engine_accepts_what_the_cli_passes(self): + from mpftp import firmware + + ns = firmware.build_parser().parse_args( + ["flash", "--port", "esp32", "--before", "no-reset", "--after", "watchdog-reset"] + ) + self.assertEqual((ns.before, ns.after), ("no-reset", "watchdog-reset")) + + +class ChipMac(unittest.TestCase): + def test_both_serial_forms_give_the_same_mac(self): + self.assertEqual(cli._chip_mac(CDC["serial_number"]), cli._chip_mac(ROM["serial_number"])) + self.assertEqual(cli._chip_mac(None), "") + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/guides/download-and-flash-firmware.md b/docs/guides/download-and-flash-firmware.md index 61244c0..f00cdd9 100644 --- a/docs/guides/download-and-flash-firmware.md +++ b/docs/guides/download-and-flash-firmware.md @@ -44,9 +44,14 @@ ESP32 and S2, 0x2000 on the P4 and C5, 0x0 on the rest. mpftp lets go of the board first, shows progress, and reconnects when the board restarts. Tick **Erase all of flash first** (`--erase`) for a clean board; it deletes the files on it. A board whose REPL is its own USB (an S2 or S3 on native USB) is -put into its bootloader first. If no download port appears, hold BOOT, tap -RESET, choose the new port and flash again; [Recover a board](recover-a-board.md) -covers the stubborn cases. +put into its bootloader first, and flashed on the ROM download port it comes +back on, found by the MAC in its USB serial number. If no download port +appears, hold BOOT, tap RESET, choose the new port and flash again; +[Recover a board](recover-a-board.md) covers the stubborn cases. + +`--before` and `--after` choose esptool's reset on either side of the write: +`--before no-reset` for a board already in its ROM loader, or `--after +watchdog-reset` for one that a plain reset leaves in download mode. Flashing without a rebuild to another board is `mpftp firmware flash -d OTHER_PORT` after a build. From 418d1cdcc3b4cfaf90a59ebaf7df38e814ccbb09 Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:17:29 -0600 Subject: [PATCH 2/3] firmware flash: take an S2's ROM port too when a native-USB board drops to its bootloader --- cli/src/mpftp/cli.py | 22 ++++++++++++++-------- cli/tests/test_flash_native_usb.py | 6 ++++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/cli/src/mpftp/cli.py b/cli/src/mpftp/cli.py index b865535..075e0ec 100755 --- a/cli/src/mpftp/cli.py +++ b/cli/src/mpftp/cli.py @@ -2314,10 +2314,11 @@ def format_tree(tree: dict, port: str, board: str, variant: str, interpreter: st # MicroPython's own USB CDC on a native-USB ESP32 (TinyUSB, Espressif's VID), -# and the ROM's USB-Serial/JTAG port the chip shows in download mode. The CDC -# ignores DTR and RTS, so esptool's reset can't reach the ROM through it. +# and the ports the ROM shows in download mode: USB-Serial/JTAG on the S3 and +# later chips, USB-OTG on the S2. The CDC ignores DTR and RTS, so esptool's +# reset can't reach the ROM through it. _ESP_TINYUSB_CDC = (0x303A, 0x4001) -_ESP_ROM_USB_JTAG = (0x303A, 0x1001) +_ESP_ROM_PORTS = {(0x303A, 0x1001), (0x303A, 0x0002)} _ROM_PORT_WAIT_S = 20.0 @@ -2346,13 +2347,18 @@ def _wants_rom_port(ns: argparse.Namespace) -> bool: def _find_rom_port(ports: list[dict], mac: str, before: set[str]) -> Optional[str]: - """The ROM download port for the chip with ``mac``, or a new one if no MAC.""" + """The ROM download port for the chip with ``mac``. + + A port whose serial number carries no MAC (the S2's ROM) can only be + told by being new. + """ for p in ports: - if _usb_id(p) != _ESP_ROM_USB_JTAG: + if _usb_id(p) not in _ESP_ROM_PORTS: continue - if mac and _chip_mac(p.get("serial_number")) == mac: + rom_mac = _chip_mac(p.get("serial_number")) + if mac and rom_mac == mac: return p["device"] - if not mac and p["device"] not in before: + if not (mac and rom_mac) and p["device"] not in before: return p["device"] return None @@ -2394,7 +2400,7 @@ def _enter_esp_rom(device: str, baud: int) -> str: return rom _die( f"{device} went to its bootloader, but no ROM download port " - f"(USB 303a:1001) appeared within {_ROM_PORT_WAIT_S:.0f} s. Hold BOOT " + f"appeared within {_ROM_PORT_WAIT_S:.0f} s. Hold BOOT " "while tapping RESET, then flash the port it comes back on." ) finally: diff --git a/cli/tests/test_flash_native_usb.py b/cli/tests/test_flash_native_usb.py index f0eafee..b0d9a31 100644 --- a/cli/tests/test_flash_native_usb.py +++ b/cli/tests/test_flash_native_usb.py @@ -23,6 +23,7 @@ OTHER_CDC = {"device": "COM105", "vid": 0x303A, "pid": 0x4001, "serial_number": "70041DAD86C40000"} ROM = {"device": "COM103", "vid": 0x303A, "pid": 0x1001, "serial_number": "C0:4E:30:03:F6:30"} OTHER_ROM = {"device": "COM102", "vid": 0x303A, "pid": 0x1001, "serial_number": "70:04:1D:AD:86:C4"} +S2_ROM = {"device": "COM29", "vid": 0x303A, "pid": 0x0002, "serial_number": "0"} UART = {"device": "COM7", "vid": 0x10C4, "pid": 0xEA60, "serial_number": "0001"} @@ -84,6 +85,11 @@ def test_rom_port_is_matched_by_mac_not_by_being_new(self): extra = run_flash(BASE + ["-d", "COM104"], client) self.assertEqual(extra[extra.index("--device") + 1], "COM103") + def test_an_s2_rom_port_with_no_mac_is_taken_when_new(self): + client = FakeClient([[CDC, OTHER_ROM], [OTHER_ROM, S2_ROM]]) + extra = run_flash(BASE + ["-d", "COM104"], client) + self.assertEqual(extra[extra.index("--device") + 1], "COM29") + def test_uart_bridge_board_is_flashed_where_it_is(self): client = FakeClient([[UART]]) extra = run_flash(BASE + ["-d", "COM7"], client) From 2d7bca36f6981aa8845cbf64a2290f7aa591ede2 Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:21:08 -0600 Subject: [PATCH 3/3] firmware flash tests: one with statement for the ROM-port timeout --- cli/tests/test_flash_native_usb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/tests/test_flash_native_usb.py b/cli/tests/test_flash_native_usb.py index b0d9a31..60853bd 100644 --- a/cli/tests/test_flash_native_usb.py +++ b/cli/tests/test_flash_native_usb.py @@ -111,9 +111,9 @@ def test_no_reset_skips_the_bootloader_step(self): def test_no_rom_port_fails_without_flashing_the_dead_port(self): client = FakeClient([[CDC], [OTHER_ROM]]) with mock.patch.object(cli, "_ROM_PORT_WAIT_S", 0.0), \ - mock.patch.object(cli.time, "monotonic", side_effect=[0.0, 1.0, 2.0]): - with self.assertRaises(SystemExit): - run_flash(BASE + ["-d", "COM104"], client) + mock.patch.object(cli.time, "monotonic", side_effect=[0.0, 1.0, 2.0]), \ + self.assertRaises(SystemExit): + run_flash(BASE + ["-d", "COM104"], client) def test_refused_bootloader_fails(self): client = FakeClient([[CDC]], bootloader_ok=False)