Fix m33mu SDK downloads and publish CI images after merge

.github/workflows/build-and-publish.yml

@@ -2,6 +2,8 @@ name: Build and Publish CI Images
 
 on:
   push:
+    branches:
+      - master
     tags:
       - 'v*'
   workflow_dispatch:
@@ -11,6 +13,10 @@ on:
         required: false
         default: 'all'
 
+concurrency:
+  group: publish-ci-images-${{ github.ref }}
+  cancel-in-progress: false
+
 env:
   REGISTRY: ghcr.io
   OWNER: wolfssl

.github/workflows/pr-build.yml

@@ -0,0 +1,64 @@
+name: Build Changed CI Images
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      images: ${{ steps.diff.outputs.images }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: List image directories touched by this PR
+        id: diff
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+          import subprocess
+
+          dockerfiles = {p.parent.name: p for p in Path('.').glob('*/Dockerfile')}
+          paths = subprocess.check_output([
+              'git', 'diff', '--name-only', '-z', os.environ['BASE_SHA'], 'HEAD'
+          ]).decode().split('\0')
+          images = {p.split('/')[0] for p in paths} & dockerfiles.keys()
+          if 'base' in images:
+              images.update(name for name, path in dockerfiles.items()
+                            if 'FROM ghcr.io/wolfssl/wolfboot-ci-base:' in path.read_text())
+          images = json.dumps(sorted(images))
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as output:
+              output.write(f'images={images}\n')
+          print(f'Images to build: {images}')
+          PY
+
+  build:
+    needs: changes
+    if: needs.changes.outputs.images != '[]'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        image: ${{ fromJSON(needs.changes.outputs.images) }}
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Build ${{ matrix.image }}
+        env:
+          IMAGE: ${{ matrix.image }}
+        run: |
+          if grep -q '^FROM ghcr.io/wolfssl/wolfboot-ci-base:' "$IMAGE/Dockerfile"; then
+            docker build --pull --no-cache -t ghcr.io/wolfssl/wolfboot-ci-base:pr base
+            docker build --no-cache --build-arg BASE_TAG=pr "$IMAGE"
+          else
+            docker build --pull --no-cache "$IMAGE"
+          fi

README.md

@@ -21,9 +21,11 @@ All images use Devuan testing as the base (except renode).
 
 ## Building locally
 
+Tag the base with its full image name so derived builds find the local image.
+
 ```bash
 # Build base first
-podman build --no-cache -t wolfboot-ci-base:testing base/
+podman build --no-cache -t ghcr.io/wolfssl/wolfboot-ci-base:testing base/
 
 # Build derived images (replace 'arm' with any image name)
 podman build --no-cache --build-arg BASE_TAG=testing -t wolfboot-ci-arm:testing arm/
@@ -42,28 +44,26 @@ directory`. The m33mu image ships both, and exports
     MCUXPRESSO_CMSIS=/opt/nxp/CMSIS_5/CMSIS
 
 wolfBoot's example configs set these with `?=`, so the environment wins and
-jobs need no extra make arguments — plain `make` works.
+jobs need no extra make arguments. Plain `make` works.
 
-Boards are provisioned into one shared west workspace. `west update_board`
-updates only its own board's projects, so boards accumulate:
+Only the west projects behind the paths wolfBoot's makefiles name are fetched
+(`core`, `CMSIS`, components, examples, the LPC, MCX, RT and Wireless device trees,
+`els_pkc` and `secure-subsystem`).
+Use `MCUX_PROJECTS` to extend this set; `MCUX_BOARDS` is no longer used.
+The NXP board manifests also pull TF-M, eIQ and other middleware whose GitLab
+submodules break the build whenever GitLab is overloaded, so the image skips
+them:
 
 ```bash
-# Add or change the board set (default: lpcxpresso55s69 frdmmcxn947 mimxrt700evk)
+# Add a device family (for example Kinetis)
 podman build --no-cache \
-  --build-arg MCUX_BOARDS="lpcxpresso55s69 frdmmcxn947 frdmmcxa153" \
+  --build-arg MCUX_PROJECTS="core CMSIS mcu-sdk-components mcu-sdk-examples mcux-devices-lpc mcux-devices-mcx mcux-devices-rt mcux-devices-wireless els_pkc secure-subsystem mcux-devices-kinetis" \
   -t wolfboot-ci-m33mu:testing m33mu/
 
-# Keep the full SDK instead of pruning to the subtrees wolfBoot references
+# Keep every example instead of only examples/_boards
 podman build --no-cache --build-arg MCUX_PRUNE=0 -t wolfboot-ci-m33mu:testing m33mu/
 ```
 
-Boards share most repositories, so extra boards are far cheaper than the
-first: `lpcxpresso55s69` alone costs ~3.0 GB, `+frdmmcxn947` ~2.5 GB more,
-and `+mimxrt700evk` only ~145 MB. The default prune (dropping `middleware`,
-`rtos`, `docs`, the non-`_boards` examples, and the `.git` directories, none
-of which any wolfBoot makefile names) takes the three-board default from
-~5.8 GB to ~2.8 GB.
-
 ## Usage in GitHub Actions
 
 ```yaml
@@ -84,9 +84,19 @@ jobs:
         run: make
 ```
 
+## Pull request builds
+
+Pull requests build each changed image without publishing it. Changes to `base/`
+also build its dependent images. Each dependent image builds against the base
+Dockerfile in the pull request, so changes to both are tested together.
+
 ## Publishing
 
-Images are automatically built and published to GHCR when a version tag is pushed:
+Pushes to `master`, including merged pull requests, build and publish all images
+to GHCR with the `master` and `latest` tags. Pull request checks build images
+without publishing them.
+
+Version tag pushes also publish images with the version tag and `latest`:
 
 ```bash
 git tag v1.0

m33mu/Dockerfile

@@ -84,20 +84,14 @@ RUN printf 'int main(void){return 0;}\n' > /tmp/arm-none-eabi-smoke.c \
 # west-provisioned MCUXpresso SDK tree plus NXP's CMSIS_5 fork. Without them a
 # job in this image dies at `fsl_common.h: No such file or directory`.
 #
-# `west update_board` runs `west update -n -o=--depth=1 <projects>`, updating
-# only that board's projects, so boards accumulate in one workspace: add a
-# board by appending it to MCUX_BOARDS.
-#
-# The se_hostlib/emwin projects are dropped from each board manifest (they are
-# not publicly fetchable), matching wolfBoot's
-# .github/workflows/test-build-mcux-sdk-manifests.yml.
-ARG MCUX_BOARDS="lpcxpresso55s69 frdmmcxn947 mimxrt700evk"
-
-# Keep only the subtrees wolfBoot's build actually references: devices,
-# drivers, components, CMSIS, arch and examples/_boards/<board>. middleware,
-# rtos, docs and the non-_boards examples are never named by any wolfBoot
-# makefile and dominate the tree, so they are pruned. Set MCUX_PRUNE=0 to keep
-# the full SDK if a future target needs more of it.
+# Only the west projects behind the paths wolfBoot's makefiles name are
+# fetched. The board manifests' repo_list also pulls TF-M, eIQ and friends,
+# whose GitLab submodules (eigen) make the build fail whenever GitLab is busy.
+# A new NXP family needs its mcux-devices-<family> project added here.
+ARG MCUX_PROJECTS="core CMSIS mcu-sdk-components mcu-sdk-examples mcux-devices-lpc mcux-devices-mcx mcux-devices-rt mcux-devices-wireless els_pkc secure-subsystem"
+
+# Drop the non-_boards examples, which no wolfBoot makefile names. Set
+# MCUX_PRUNE=0 to keep them.
 ARG MCUX_PRUNE=1
 
 ENV MCUX_ROOT=/opt/nxp
@@ -113,22 +107,16 @@ RUN set -eux; \
     /opt/venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel; \
     /opt/venv/bin/pip install --no-cache-dir west
 
-# update_board runs a bare "west update" in a subprocess
 RUN set -eux; \
-    export PATH="/opt/venv/bin:${PATH}"; \
     mkdir -p "${MCUX_ROOT}"; \
     cd "${MCUX_ROOT}"; \
     /opt/venv/bin/west init \
         -m https://github.com/nxp-mcuxpresso/mcuxsdk-manifests.git mcuxpresso-sdk; \
     cd "${MCUX_ROOT}/mcuxpresso-sdk"; \
-    for board in ${MCUX_BOARDS}; do \
-        sed -i '/se_hostlib\|emwin/d' "manifests/boards/${board}.yml"; \
-        /opt/venv/bin/west update_board --set board "${board}"; \
-    done; \
+    /opt/venv/bin/west update -n -o=--depth=1 ${MCUX_PROJECTS}; \
     git clone --depth 1 https://github.com/nxp-mcuxpresso/CMSIS_5.git \
         "${MCUX_ROOT}/CMSIS_5"; \
     if [ "${MCUX_PRUNE}" = "1" ]; then \
-        rm -rf "${MCUXPRESSO}/docs" "${MCUXPRESSO}/middleware" "${MCUXPRESSO}/rtos"; \
         find "${MCUXPRESSO}/examples" -mindepth 1 -maxdepth 1 \
             ! -name '_boards' -exec rm -rf {} +; \
     fi; \
@@ -142,9 +130,13 @@ RUN set -eux; \
 RUN set -eux; \
     test -f "${MCUXPRESSO}/drivers/common/fsl_common.h"; \
     test -f "${MCUXPRESSO}/drivers/puf/fsl_puf.h"; \
+    test -f "${MCUXPRESSO}/components/els_pkc/includes/mcuxClEls.h"; \
+    test -f "${MCUXPRESSO}/middleware/secure-subsystem/inc/fsl_sscp_mu.h"; \
     test -f "${MCUXPRESSO_CMSIS}/Core/Include/core_cm33.h"; \
     test -d "${MCUXPRESSO}/devices/LPC/LPC5500/LPC55S69"; \
     test -d "${MCUXPRESSO}/devices/MCX/MCXN/MCXN947"; \
+    test -d "${MCUXPRESSO}/devices/RT/RT700"; \
+    test -d "${MCUXPRESSO}/devices/Wireless/RW/RW612"; \
     test -d "${MCUXPRESSO}/examples/_boards/lpcxpresso55s69/project_template"; \
     test -d "${MCUXPRESSO}/examples/_boards/frdmmcxn947/project_template"; \
     test -d "${MCUXPRESSO}/examples/_boards/mimxrt700evk"