diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 6d1ebff..f64895d 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -2,6 +2,8 @@ name: Build and Publish CI Images on: push: + branches: + - master tags: - 'v*' workflow_dispatch: @@ -11,6 +13,10 @@ on: required: false default: 'all' +concurrency: + group: publish-ci-images-${{ github.ref }} + cancel-in-progress: false + env: REGISTRY: ghcr.io OWNER: wolfssl diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml new file mode 100644 index 0000000..c18ee46 --- /dev/null +++ b/.github/workflows/pr-build.yml @@ -0,0 +1,64 @@ +name: Build Changed CI Images + +on: + pull_request: + +permissions: + contents: read + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + images: ${{ steps.diff.outputs.images }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: List image directories touched by this PR + id: diff + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + python3 - <<'PY' + import json + import os + from pathlib import Path + import subprocess + + dockerfiles = {p.parent.name: p for p in Path('.').glob('*/Dockerfile')} + paths = subprocess.check_output([ + 'git', 'diff', '--name-only', '-z', os.environ['BASE_SHA'], 'HEAD' + ]).decode().split('\0') + images = {p.split('/')[0] for p in paths} & dockerfiles.keys() + if 'base' in images: + images.update(name for name, path in dockerfiles.items() + if 'FROM ghcr.io/wolfssl/wolfboot-ci-base:' in path.read_text()) + images = json.dumps(sorted(images)) + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f'images={images}\n') + print(f'Images to build: {images}') + PY + + build: + needs: changes + if: needs.changes.outputs.images != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: ${{ fromJSON(needs.changes.outputs.images) }} + steps: + - uses: actions/checkout@v7 + + - name: Build ${{ matrix.image }} + env: + IMAGE: ${{ matrix.image }} + run: | + if grep -q '^FROM ghcr.io/wolfssl/wolfboot-ci-base:' "$IMAGE/Dockerfile"; then + docker build --pull --no-cache -t ghcr.io/wolfssl/wolfboot-ci-base:pr base + docker build --no-cache --build-arg BASE_TAG=pr "$IMAGE" + else + docker build --pull --no-cache "$IMAGE" + fi diff --git a/README.md b/README.md index 8db464e..02b8703 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,11 @@ All images use Devuan testing as the base (except renode). ## Building locally +Tag the base with its full image name so derived builds find the local image. + ```bash # Build base first -podman build --no-cache -t wolfboot-ci-base:testing base/ +podman build --no-cache -t ghcr.io/wolfssl/wolfboot-ci-base:testing base/ # Build derived images (replace 'arm' with any image name) podman build --no-cache --build-arg BASE_TAG=testing -t wolfboot-ci-arm:testing arm/ @@ -42,28 +44,26 @@ directory`. The m33mu image ships both, and exports MCUXPRESSO_CMSIS=/opt/nxp/CMSIS_5/CMSIS wolfBoot's example configs set these with `?=`, so the environment wins and -jobs need no extra make arguments — plain `make` works. +jobs need no extra make arguments. Plain `make` works. -Boards are provisioned into one shared west workspace. `west update_board` -updates only its own board's projects, so boards accumulate: +Only the west projects behind the paths wolfBoot's makefiles name are fetched +(`core`, `CMSIS`, components, examples, the LPC, MCX, RT and Wireless device trees, +`els_pkc` and `secure-subsystem`). +Use `MCUX_PROJECTS` to extend this set; `MCUX_BOARDS` is no longer used. +The NXP board manifests also pull TF-M, eIQ and other middleware whose GitLab +submodules break the build whenever GitLab is overloaded, so the image skips +them: ```bash -# Add or change the board set (default: lpcxpresso55s69 frdmmcxn947 mimxrt700evk) +# Add a device family (for example Kinetis) podman build --no-cache \ - --build-arg MCUX_BOARDS="lpcxpresso55s69 frdmmcxn947 frdmmcxa153" \ + --build-arg MCUX_PROJECTS="core CMSIS mcu-sdk-components mcu-sdk-examples mcux-devices-lpc mcux-devices-mcx mcux-devices-rt mcux-devices-wireless els_pkc secure-subsystem mcux-devices-kinetis" \ -t wolfboot-ci-m33mu:testing m33mu/ -# Keep the full SDK instead of pruning to the subtrees wolfBoot references +# Keep every example instead of only examples/_boards podman build --no-cache --build-arg MCUX_PRUNE=0 -t wolfboot-ci-m33mu:testing m33mu/ ``` -Boards share most repositories, so extra boards are far cheaper than the -first: `lpcxpresso55s69` alone costs ~3.0 GB, `+frdmmcxn947` ~2.5 GB more, -and `+mimxrt700evk` only ~145 MB. The default prune (dropping `middleware`, -`rtos`, `docs`, the non-`_boards` examples, and the `.git` directories, none -of which any wolfBoot makefile names) takes the three-board default from -~5.8 GB to ~2.8 GB. - ## Usage in GitHub Actions ```yaml @@ -84,9 +84,19 @@ jobs: run: make ``` +## Pull request builds + +Pull requests build each changed image without publishing it. Changes to `base/` +also build its dependent images. Each dependent image builds against the base +Dockerfile in the pull request, so changes to both are tested together. + ## Publishing -Images are automatically built and published to GHCR when a version tag is pushed: +Pushes to `master`, including merged pull requests, build and publish all images +to GHCR with the `master` and `latest` tags. Pull request checks build images +without publishing them. + +Version tag pushes also publish images with the version tag and `latest`: ```bash git tag v1.0 diff --git a/m33mu/Dockerfile b/m33mu/Dockerfile index fade568..8c3aef4 100644 --- a/m33mu/Dockerfile +++ b/m33mu/Dockerfile @@ -84,20 +84,14 @@ RUN printf 'int main(void){return 0;}\n' > /tmp/arm-none-eabi-smoke.c \ # west-provisioned MCUXpresso SDK tree plus NXP's CMSIS_5 fork. Without them a # job in this image dies at `fsl_common.h: No such file or directory`. # -# `west update_board` runs `west update -n -o=--depth=1 `, updating -# only that board's projects, so boards accumulate in one workspace: add a -# board by appending it to MCUX_BOARDS. -# -# The se_hostlib/emwin projects are dropped from each board manifest (they are -# not publicly fetchable), matching wolfBoot's -# .github/workflows/test-build-mcux-sdk-manifests.yml. -ARG MCUX_BOARDS="lpcxpresso55s69 frdmmcxn947 mimxrt700evk" - -# Keep only the subtrees wolfBoot's build actually references: devices, -# drivers, components, CMSIS, arch and examples/_boards/. middleware, -# rtos, docs and the non-_boards examples are never named by any wolfBoot -# makefile and dominate the tree, so they are pruned. Set MCUX_PRUNE=0 to keep -# the full SDK if a future target needs more of it. +# Only the west projects behind the paths wolfBoot's makefiles name are +# fetched. The board manifests' repo_list also pulls TF-M, eIQ and friends, +# whose GitLab submodules (eigen) make the build fail whenever GitLab is busy. +# A new NXP family needs its mcux-devices- project added here. +ARG MCUX_PROJECTS="core CMSIS mcu-sdk-components mcu-sdk-examples mcux-devices-lpc mcux-devices-mcx mcux-devices-rt mcux-devices-wireless els_pkc secure-subsystem" + +# Drop the non-_boards examples, which no wolfBoot makefile names. Set +# MCUX_PRUNE=0 to keep them. ARG MCUX_PRUNE=1 ENV MCUX_ROOT=/opt/nxp @@ -113,22 +107,16 @@ RUN set -eux; \ /opt/venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel; \ /opt/venv/bin/pip install --no-cache-dir west -# update_board runs a bare "west update" in a subprocess RUN set -eux; \ - export PATH="/opt/venv/bin:${PATH}"; \ mkdir -p "${MCUX_ROOT}"; \ cd "${MCUX_ROOT}"; \ /opt/venv/bin/west init \ -m https://github.com/nxp-mcuxpresso/mcuxsdk-manifests.git mcuxpresso-sdk; \ cd "${MCUX_ROOT}/mcuxpresso-sdk"; \ - for board in ${MCUX_BOARDS}; do \ - sed -i '/se_hostlib\|emwin/d' "manifests/boards/${board}.yml"; \ - /opt/venv/bin/west update_board --set board "${board}"; \ - done; \ + /opt/venv/bin/west update -n -o=--depth=1 ${MCUX_PROJECTS}; \ git clone --depth 1 https://github.com/nxp-mcuxpresso/CMSIS_5.git \ "${MCUX_ROOT}/CMSIS_5"; \ if [ "${MCUX_PRUNE}" = "1" ]; then \ - rm -rf "${MCUXPRESSO}/docs" "${MCUXPRESSO}/middleware" "${MCUXPRESSO}/rtos"; \ find "${MCUXPRESSO}/examples" -mindepth 1 -maxdepth 1 \ ! -name '_boards' -exec rm -rf {} +; \ fi; \ @@ -142,9 +130,13 @@ RUN set -eux; \ RUN set -eux; \ test -f "${MCUXPRESSO}/drivers/common/fsl_common.h"; \ test -f "${MCUXPRESSO}/drivers/puf/fsl_puf.h"; \ + test -f "${MCUXPRESSO}/components/els_pkc/includes/mcuxClEls.h"; \ + test -f "${MCUXPRESSO}/middleware/secure-subsystem/inc/fsl_sscp_mu.h"; \ test -f "${MCUXPRESSO_CMSIS}/Core/Include/core_cm33.h"; \ test -d "${MCUXPRESSO}/devices/LPC/LPC5500/LPC55S69"; \ test -d "${MCUXPRESSO}/devices/MCX/MCXN/MCXN947"; \ + test -d "${MCUXPRESSO}/devices/RT/RT700"; \ + test -d "${MCUXPRESSO}/devices/Wireless/RW/RW612"; \ test -d "${MCUXPRESSO}/examples/_boards/lpcxpresso55s69/project_template"; \ test -d "${MCUXPRESSO}/examples/_boards/frdmmcxn947/project_template"; \ test -d "${MCUXPRESSO}/examples/_boards/mimxrt700evk"