From e6b09596cf717de4b67b47012a1ecd6d38dcd6e9 Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Mon, 9 Nov 2026 13:07:44 +0000 Subject: [PATCH 1/3] Add parallel pull for docker images for testcontainers --- .github/workflows/maven-ci.yml | 25 +++++++++++++++++++++++++ README.md | 1 + 2 files changed, 26 insertions(+) diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index 629e991..71b0581 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -38,6 +38,11 @@ on: required: false type: boolean default: false + pre_pull_docker_images: + description: "Comma-separated list of Docker images to pull in the background before the build starts (e.g. images used by Testcontainers). Default: empty, nothing is pulled" + required: false + type: string + default: "" # cancel in-progress runs of the same workflow # to avoid unnecessary runs when multiple commits pushed @@ -59,6 +64,26 @@ jobs: fetch-depth: ${{ inputs.fetch_all_commits == true && '0' || '1' }} persist-credentials: false + # Detached pulls carry on after this step ends, so they overlap with the Java/Maven setup and the build. + # Failures are harmless: Testcontainers pulls any missing image itself when it is first needed. + - name: Pre-pull Docker images in background + if: inputs.pre_pull_docker_images != '' + env: + INPUTS_PRE_PULL_DOCKER_IMAGES: ${{ inputs.pre_pull_docker_images }} + run: | + IFS=',' read -ra images <<< "${INPUTS_PRE_PULL_DOCKER_IMAGES}" + for image in "${images[@]}"; do + image="$(echo "${image}" | xargs)" + [ -n "${image}" ] || continue + if [[ ! "${image}" =~ ^[a-zA-Z0-9][a-zA-Z0-9._/:@-]*$ ]]; then + echo "::warning::Skipping invalid Docker image name '${image}'" + continue + fi + echo "Pre-pulling ${image} in background" + setsid docker pull -- "${image}" > /dev/null 2>&1 < /dev/null & + disown + done + - uses: project-ncl/shared-github-actions/.github/actions/maven-build@e7102dd3bf74f91aaccae7a3539241c03a652d1a # v0.0.27 with: java_version: ${{ inputs.java_version }} diff --git a/README.md b/README.md index 974390d..5de384e 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ the Java version etc. It is possible to use this within a matrix job. * fetch_all_commits (default: `false`) * maven_version (default: `3.9.15`) * upload_artifacts (default: `false`) + * pre_pull_docker_images (default: empty, opt-in): comma-separated list of Docker images (e.g. `quay.io/keycloak/keycloak:21.1.0,redis:7.2`) to `docker pull` in the background right after checkout, so the download overlaps with Java/Maven setup and the build instead of Testcontainers pulling them lazily and serially. No images are cached and no registry login is performed (public images only). A failed pull is harmless: the image is then pulled on demand as usual - **Permissions**: `contents: read`
From 5e422f01a75542e950b8b08b03913b7fe0f39509 Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Mon, 9 Nov 2026 13:13:52 +0000 Subject: [PATCH 2/3] Add logging to docker pull --- .github/workflows/maven-ci.yml | 44 +++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index 71b0581..f2dad71 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -66,11 +66,16 @@ jobs: # Detached pulls carry on after this step ends, so they overlap with the Java/Maven setup and the build. # Failures are harmless: Testcontainers pulls any missing image itself when it is first needed. + # A detached process cannot log into this step once it has ended, so each pull writes its output and + # result under RUNNER_TEMP, which the 'Report Docker pre-pull results' step below prints. - name: Pre-pull Docker images in background if: inputs.pre_pull_docker_images != '' env: INPUTS_PRE_PULL_DOCKER_IMAGES: ${{ inputs.pre_pull_docker_images }} run: | + dir="${RUNNER_TEMP}/pre-pull-docker-images" + mkdir -p -- "${dir}" + n=0 IFS=',' read -ra images <<< "${INPUTS_PRE_PULL_DOCKER_IMAGES}" for image in "${images[@]}"; do image="$(echo "${image}" | xargs)" @@ -79,8 +84,19 @@ jobs: echo "::warning::Skipping invalid Docker image name '${image}'" continue fi + n=$((n + 1)) + base="${dir}/${n}" + printf '%s\n' "${image}" > "${base}.image" echo "Pre-pulling ${image} in background" - setsid docker pull -- "${image}" > /dev/null 2>&1 < /dev/null & + # image and path are passed as arguments, never interpolated into the script. + # The .status file (exit code and seconds taken) appears atomically once the pull has finished. + # shellcheck disable=SC2016 + setsid bash -c ' + start=$(date +%s) + docker pull -- "$1" > "$2.log" 2>&1 + rc=$? + printf "%s %s\n" "${rc}" "$(( $(date +%s) - start ))" > "$2.status.tmp" && mv -f -- "$2.status.tmp" "$2.status" + ' _ "${image}" "${base}" > /dev/null 2>&1 < /dev/null & disown done @@ -90,6 +106,32 @@ jobs: build_command: ${{ inputs.build_command }} maven_version: ${{ inputs.maven_version }} + - name: Report Docker pre-pull results + # Also on failure/cancel, a failed pre-pull can explain a slow or failing integration test + if: always() && inputs.pre_pull_docker_images != '' + run: | + dir="${RUNNER_TEMP}/pre-pull-docker-images" + for imagefile in "${dir}"/*.image; do + [ -e "${imagefile}" ] || continue + base="${imagefile%.image}" + image="$(cat "${imagefile}")" + if [ -f "${base}.status" ]; then + read -r rc secs < "${base}.status" + if [ "${rc}" = "0" ]; then + echo "::group::Pre-pulled ${image} in ${secs}s" + else + echo "::warning::Pre-pull of ${image} failed (exit code ${rc}) after ${secs}s, Testcontainers pulls it on demand instead" + echo "::group::Pre-pull of ${image} failed" + fi + else + echo "::notice::Pre-pull of ${image} was still running when the build ended" + echo "::group::Pre-pull of ${image} still running" + fi + # Registry output is untrusted: stop it from being interpreted as workflow commands + tail -n 20 -- "${base}.log" 2>/dev/null | sed 's/::/: :/g' + echo "::endgroup::" + done + - name: Check for wrong code formatting # Straight after a maven-release the release plugin might have altered the format slightly. Don't fail # dependabot updates on formatting changes. From cef0ef0b2342b50ff98028b1777f9d30f6b3621d Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Mon, 9 Nov 2026 16:09:51 +0000 Subject: [PATCH 3/3] Simplify logging --- .github/workflows/maven-ci.yml | 54 +++++++++++++--------------------- README.md | 2 +- 2 files changed, 21 insertions(+), 35 deletions(-) diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index f2dad71..8ed8442 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -66,15 +66,14 @@ jobs: # Detached pulls carry on after this step ends, so they overlap with the Java/Maven setup and the build. # Failures are harmless: Testcontainers pulls any missing image itself when it is first needed. - # A detached process cannot log into this step once it has ended, so each pull writes its output and - # result under RUNNER_TEMP, which the 'Report Docker pre-pull results' step below prints. + # Each pull writes .log (image name on the first line, then the pull output) and, once finished, + # .rc (exit code and seconds taken). The 'Report Docker pre-pull results' step prints them. - name: Pre-pull Docker images in background if: inputs.pre_pull_docker_images != '' env: INPUTS_PRE_PULL_DOCKER_IMAGES: ${{ inputs.pre_pull_docker_images }} run: | - dir="${RUNNER_TEMP}/pre-pull-docker-images" - mkdir -p -- "${dir}" + mkdir -p -- "${RUNNER_TEMP}/pre-pull" n=0 IFS=',' read -ra images <<< "${INPUTS_PRE_PULL_DOCKER_IMAGES}" for image in "${images[@]}"; do @@ -84,19 +83,11 @@ jobs: echo "::warning::Skipping invalid Docker image name '${image}'" continue fi - n=$((n + 1)) - base="${dir}/${n}" - printf '%s\n' "${image}" > "${base}.image" echo "Pre-pulling ${image} in background" - # image and path are passed as arguments, never interpolated into the script. - # The .status file (exit code and seconds taken) appears atomically once the pull has finished. + base="${RUNNER_TEMP}/pre-pull/$((++n))" + echo "${image}" > "${base}.log" # shellcheck disable=SC2016 - setsid bash -c ' - start=$(date +%s) - docker pull -- "$1" > "$2.log" 2>&1 - rc=$? - printf "%s %s\n" "${rc}" "$(( $(date +%s) - start ))" > "$2.status.tmp" && mv -f -- "$2.status.tmp" "$2.status" - ' _ "${image}" "${base}" > /dev/null 2>&1 < /dev/null & + setsid bash -c 'docker pull -- "$1" >> "$2.log" 2>&1; echo "$? ${SECONDS}" > "$2.rc"' _ "${image}" "${base}" > /dev/null 2>&1 < /dev/null & disown done @@ -107,29 +98,24 @@ jobs: maven_version: ${{ inputs.maven_version }} - name: Report Docker pre-pull results - # Also on failure/cancel, a failed pre-pull can explain a slow or failing integration test + # Also on failure: a failed pre-pull can explain a slow or failing integration test if: always() && inputs.pre_pull_docker_images != '' run: | - dir="${RUNNER_TEMP}/pre-pull-docker-images" - for imagefile in "${dir}"/*.image; do - [ -e "${imagefile}" ] || continue - base="${imagefile%.image}" - image="$(cat "${imagefile}")" - if [ -f "${base}.status" ]; then - read -r rc secs < "${base}.status" - if [ "${rc}" = "0" ]; then - echo "::group::Pre-pulled ${image} in ${secs}s" - else - echo "::warning::Pre-pull of ${image} failed (exit code ${rc}) after ${secs}s, Testcontainers pulls it on demand instead" - echo "::group::Pre-pull of ${image} failed" - fi + for log in "${RUNNER_TEMP}"/pre-pull/*.log; do + [ -e "${log}" ] || continue + image="$(head -n 1 "${log}")" + rc="" secs="" + [ ! -f "${log%.log}.rc" ] || read -r rc secs < "${log%.log}.rc" + if [ "${rc}" = "0" ]; then + echo "✅ Pre-pulled ${image} in ${secs}s" else - echo "::notice::Pre-pull of ${image} was still running when the build ended" - echo "::group::Pre-pull of ${image} still running" + status="${rc:+exit ${rc} after ${secs}s}" + echo "::warning::❌ Pre-pull of ${image} did not succeed (${status:-still running}), Testcontainers pulls it on demand instead" + echo "::group::Pre-pull log for ${image}" + # Registry output is untrusted: stop it from being interpreted as workflow commands + sed 's/::/: :/g' "${log}" + echo "::endgroup::" fi - # Registry output is untrusted: stop it from being interpreted as workflow commands - tail -n 20 -- "${base}.log" 2>/dev/null | sed 's/::/: :/g' - echo "::endgroup::" done - name: Check for wrong code formatting diff --git a/README.md b/README.md index 5de384e..a88b097 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ the Java version etc. It is possible to use this within a matrix job. * fetch_all_commits (default: `false`) * maven_version (default: `3.9.15`) * upload_artifacts (default: `false`) - * pre_pull_docker_images (default: empty, opt-in): comma-separated list of Docker images (e.g. `quay.io/keycloak/keycloak:21.1.0,redis:7.2`) to `docker pull` in the background right after checkout, so the download overlaps with Java/Maven setup and the build instead of Testcontainers pulling them lazily and serially. No images are cached and no registry login is performed (public images only). A failed pull is harmless: the image is then pulled on demand as usual + * pre_pull_docker_images (default: empty, opt-in): comma-separated list of Docker images (e.g. `quay.io/keycloak/keycloak:21.1.0,redis:7.2`) to `docker pull` in the background right after checkout, so the download overlaps with Java/Maven setup and the build instead of Testcontainers pulling them lazily and serially. No images are cached and no registry login is performed (public images only). A failed pull is harmless: the image is then pulled on demand as usual. As the pulls are detached, a `Report Docker pre-pull results` step after the build prints one line per image (✅ with the time taken), and a warning plus the full pull log for any pull that failed or was still running - **Permissions**: `contents: read`