ci: balance acceptance slices by measured suite duration

.github/acceptance/plan-slices.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Pack ownCloud core's acceptance suites into slices that take about equally
+# long, by the minutes in suite-minutes.tsv, for the matrix of the tests job.
+# run.sh --part can't do this: it splits the suites by count, not duration.
+#
+# usage: plan-slices.sh <suites> <mode> <target> <api-tags> <cli-tags>
+#   suites    file listing one suite per line, e.g. apiMain
+#   mode      smoke or full, the column of suite-minutes.tsv to use
+#   target    minutes a slice should take, unless one suite alone takes longer
+#   api-tags  Behat tag filter for the api suites
+#   cli-tags  Behat tag filter for the cli suites
+#
+# Prints the matrix as JSON: [{type, slice, slices, suites, tags, minutes}],
+# where suites is the comma-separated list for run.sh --suites.
+set -euo pipefail
+
+suites=$1 mode=$2 target=$3 api_tags=$4 cli_tags=$5
+table="$(dirname "$0")/suite-minutes.tsv"
+[[ "${mode}" == smoke || "${mode}" == full ]] || { echo "unknown mode ${mode}" >&2 && exit 1; }
+
+# "suite<TAB>minutes" for every suite to run. The cli suites always run in full,
+# so they take the longer of both columns. A suite the table doesn't know yet
+# still runs, with a default duration that is easy to spot in the plan.
+weights=$(awk -F'\t' -v mode="${mode}" '
+  NR == FNR { if (!/^#/ && NF) { smoke[$1] = $2; full[$1] = $3 }; next }
+  NF {
+    if ($1 in full) {
+      if ($1 ~ /^cli/) minutes = (smoke[$1] > full[$1]) ? smoke[$1] : full[$1]
+      else minutes = (mode == "smoke") ? smoke[$1] : full[$1]
+    } else {
+      minutes = (mode == "smoke" && $1 !~ /^cli/) ? 3 : 20
+      printf "::warning::suite %s is not in suite-minutes.tsv, assuming %s minutes\n", $1, minutes > "/dev/stderr"
+    }
+    print $1 "\t" minutes
+  }' "${table}" "${suites}")
+[[ -n "${weights}" ]] || { echo "no suites in ${suites}" >&2 && exit 1; }
+
+# Longest suite first, each into the slice with the fewest minutes so far. No
+# more slices than suites: run.sh --suites "" would run all suites of the type.
+matrix=$(jq -Rnc --argjson target "${target}" --arg api "${api_tags}" --arg cli "${cli_tags}" '
+  [inputs | split("\t") | {suite: .[0], minutes: (.[1] | tonumber), type: (.[0] | capture("^(?<t>api|cli)").t)}]
+  | group_by(.type)
+  | map(
+      .[0].type as $type
+      | ([1, ([length, ((map(.minutes) | add) / $target | ceil)] | min)] | max) as $n
+      | reduce (sort_by(-.minutes, .suite) | .[]) as $s ([range($n) | {suites: [], minutes: 0}];
+          (to_entries | min_by([.value.minutes, (.value.suites | length)]).key) as $i
+          | .[$i].suites += [$s.suite] | .[$i].minutes += $s.minutes)
+      | to_entries
+      | map({type: $type, slice: (.key + 1), slices: $n, suites: (.value.suites | sort | join(",")),
+          tags: (if $type == "api" then $api else $cli end), minutes: (.value.minutes * 10 | round / 10)}))
+  | flatten' <<<"${weights}")
+
+# every suite in exactly one slice, and no slice empty
+jq -e 'all(.suites != "")' <<<"${matrix}" >/dev/null || { echo "a slice has no suites" >&2 && exit 1; }
+planned=$(jq -r '.[].suites | split(",")[]' <<<"${matrix}" | sort)
+[[ "${planned}" == "$(cut -f1 <<<"${weights}" | sort -u)" ]] || {
+  echo "the slices don't run every suite exactly once" >&2 && exit 1
+}
+
+jq -r '.[] | "\(.type) \(.slice)/\(.slices): \(.minutes) min, \(.suites)"' <<<"${matrix}" >&2
+echo "${matrix}"

.github/acceptance/run-part.sh

@@ -1,12 +1,12 @@
 #!/usr/bin/env bash
 # Run one part of ownCloud core's acceptance tests against a server image.
 #
-# usage: run-part.sh <image> <php> <type> <part> <parts> <tags>
-#   image  server image under test, e.g. owncloud/server:test
-#   php    PHP version of the test runner (owncloudci/php:<php>), 7.4 or 8.3
-#   type   suite type for run.sh: api or cli
-#   part   part to run, 1..parts (run.sh --part)
-#   tags   Behat tag filter
+# usage: run-part.sh <image> <php> <type> <suites> <tags>
+#   image   server image under test, e.g. owncloud/server:test
+#   php     PHP version of the test runner (owncloudci/php:<php>), 7.4 or 8.3
+#   type    suite type for run.sh: api or cli
+#   suites  comma-separated suites to run (run.sh --suites), see plan-slices.sh
+#   tags    Behat tag filter
 #
 # environment:
 #   QA_DIR                  extracted owncloud-complete-*-qa tarball with Behat installed
@@ -15,7 +15,7 @@
 #   LOG_DIR                 where to write the server and database logs
 set -uo pipefail
 
-image=$1 php=$2 type=$3 part=$4 parts=$5 tags=$6
+image=$1 php=$2 type=$3 suites=$4 tags=$5
 : "${QA_DIR:?}" "${TESTING_DIR:?}" "${LOG_DIR:?}"
 net=acceptance db=acceptance-db redis=acceptance-redis srv=acceptance-server
 mkdir -p "${LOG_DIR}"
@@ -91,4 +91,4 @@ docker run --rm --network "${net}" -v "${QA_DIR}:/src" -w /src "${expected[@]}"
   -e REGULAR_USER_PASSWORD='AaBb2Cc7Dd!' -e SUB_ADMIN_PASSWORD='AaBb2Cc8Dd!' \
   -e PLAIN_OUTPUT=true \
   "owncloudci/php:${php}" \
-  bash tests/acceptance/run.sh --remote --type "${type}" --tags "${tags}" --part "${part}" "${parts}"
+  bash tests/acceptance/run.sh --remote --type "${type}" --tags "${tags}" --suites "${suites}"

.github/acceptance/suite-minutes.tsv

@@ -0,0 +1,76 @@
+# Minutes each Behat suite takes, the longest seen in CI on 10.16.x, 11.0.1 and 11.1.0.
+# plan-slices.sh packs suites into slices by these. The cli suites always run in
+# full, so it weighs them by the longer of both columns in either mode.
+# suite	smoke	full
+apiAuth	1.3	11.4
+apiAuthOcs	0.9	2.3
+apiAuthWebDav	4.7	15
+apiCapabilities	0.8	4.4
+apiComments	1.1	3.7
+apiFavorites	1.3	8.9
+apiFederationToRoot1	0.1	0.1
+apiFederationToRoot2	0.1	0.1
+apiFederationToShares1	0.1	0.1
+apiFederationToShares2	0.1	0.1
+apiMain	1.5	20.9
+apiProvisioning-v1	3.2	20.1
+apiProvisioning-v2	3.2	19.9
+apiProvisioningGroups-v1	1.6	10.5
+apiProvisioningGroups-v2	1.5	10.4
+apiShareCreateSpecialToRoot1	0.2	22.5
+apiShareCreateSpecialToRoot2	0.1	9.1
+apiShareCreateSpecialToShares1	0.2	20.6
+apiShareCreateSpecialToShares2	0.1	19
+apiSharees	0.3	9.9
+apiShareManagementBasicToRoot	1.6	20
+apiShareManagementBasicToShares	3.2	25.4
+apiShareManagementToRoot	3.8	25.9
+apiShareManagementToShares	1.4	22
+apiShareOperationsToRoot1	3.5	13.6
+apiShareOperationsToRoot2	1	9.9
+apiShareOperationsToShares1	4.3	17.6
+apiShareOperationsToShares2	1.1	15.3
+apiSharePublicLink1	1	12.3
+apiSharePublicLink2	0.3	12.3
+apiSharePublicLink3	1	9.5
+apiShareReshareToRoot1	0.6	12.7
+apiShareReshareToRoot2	0.8	10
+apiShareReshareToRoot3	0.1	14.8
+apiShareReshareToShares1	0.8	15.6
+apiShareReshareToShares2	0.9	12.1
+apiShareReshareToShares3	0.1	17.6
+apiShareUpdateToRoot	0.4	6.5
+apiShareUpdateToShares	0.5	8.8
+apiSharingNotificationsToRoot	0.2	1.4
+apiSharingNotificationsToShares	0.2	1.4
+apiTags	1.1	8.2
+apiTranslation	0.1	1
+apiTrashbin	2.3	15.2
+apiTrashbinRestore	1.3	10.8
+apiVersions	1	26.1
+apiWebdavDelete	0.6	9.1
+apiWebdavEtagPropagation1	0.1	9
+apiWebdavEtagPropagation2	0.1	8.5
+apiWebdavLocks	6.6	14.9
+apiWebdavLocks2	11.1	20.2
+apiWebdavLocks3	0.1	12.2
+apiWebdavLocksUnlock	0.5	20.3
+apiWebdavMove1	0.1	13.6
+apiWebdavMove2	1.4	15.2
+apiWebdavOperations	1.4	23.7
+apiWebdavPreviews	0.1	6.3
+apiWebdavProperties1	1	27.1
+apiWebdavProperties2	1.7	13.8
+apiWebdavUpload1	1.1	16.3
+apiWebdavUpload2	0.7	8.1
+apiWebdavUploadTUS	0.1	0.1
+cliBackground	1	1
+cliCreateLocalStorage	8.7	8.7
+cliDbConversion	0.2	0.2
+cliEncryption	0.1	0.1
+cliExternalStorage	5.7	5.7
+cliLocalStorage	20.5	20.2
+cliMain	16.6	16.6
+cliManageApps	1	1
+cliProvisioning	14.1	14.1
+cliTrashbin	2.1	2.1

.github/workflows/acceptance.yml

@@ -43,22 +43,31 @@ jobs:
     outputs:
       suites: ${{ steps.suites.outputs.suites }}
     steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
       - id: suites
         env:
           FULL: ${{ inputs.full }}
+          TARBALL: ${{ inputs.tarball }}
+          QA_SHA256: ${{ inputs.qa-sha256 }}
+          # how long a slice of the suites should take, in minutes
+          SMOKE_SLICE_MINUTES: 25
+          FULL_SLICE_MINUTES: 30
         run: |
+          qa="${TARBALL%.tar.bz2}-qa.tar.bz2"
+          curl -sSfL --retry 3 -o "${RUNNER_TEMP}/qa.tar.bz2" "${qa}"
+          echo "${QA_SHA256}  ${RUNNER_TEMP}/qa.tar.bz2" | sha256sum -c
+          # the suites are the directories directly below tests/acceptance/features
+          tar -tjf "${RUNNER_TEMP}/qa.tar.bz2" \
+            | sed -nE 's#^[^/]+/tests/acceptance/features/((api|cli)[^/]*)/.*#\1#p' \
+            | sort -u > "${RUNNER_TEMP}/suites"
           base='~@skip&&~@skipOnDockerContainerTesting&&~@federation-app-required'
           if [ "${FULL}" = true ]; then
-            api_parts=10 api_tags="${base}"
+            mode=full minutes="${FULL_SLICE_MINUTES}" api_tags="${base}"
           else
-            api_parts=2 api_tags="@smokeTest&&${base}"
+            mode=smoke minutes="${SMOKE_SLICE_MINUTES}" api_tags="@smokeTest&&${base}"
           fi
-          cli_parts=2
-          suites=$(
-            { for p in $(seq "${api_parts}"); do jq -nc --arg t "${api_tags}" --argjson p "$p" --argjson n "${api_parts}" '{type: "api", part: $p, parts: $n, tags: $t}'; done
-              for p in $(seq "${cli_parts}"); do jq -nc --arg t "${base}" --argjson p "$p" --argjson n "${cli_parts}" '{type: "cli", part: $p, parts: $n, tags: $t}'; done
-            } | jq -sc .
-          )
+          suites=$(.github/acceptance/plan-slices.sh "${RUNNER_TEMP}/suites" "${mode}" "${minutes}" "${api_tags}" "${base}")
           echo "suites=${suites}" >> "$GITHUB_OUTPUT"
 
   image:
@@ -88,7 +97,7 @@ jobs:
           compression-level: 1
 
   tests:
-    name: ${{ matrix.suite.type }} ${{ matrix.suite.part }}/${{ matrix.suite.parts }}
+    name: ${{ matrix.suite.type }} ${{ matrix.suite.slice }}/${{ matrix.suite.slices }}
     needs: [plan, image]
     runs-on: ubuntu-latest
     strategy:
@@ -152,19 +161,18 @@ jobs:
         env:
           PHP: ${{ inputs.php }}
           TYPE: ${{ matrix.suite.type }}
-          PART: ${{ matrix.suite.part }}
-          PARTS: ${{ matrix.suite.parts }}
+          SUITES: ${{ matrix.suite.suites }}
           TAGS: ${{ matrix.suite.tags }}
           EXPECTED: .github/acceptance/expected-failures-${{ inputs.version }}-${{ case(inputs.full, 'full', 'smoke') }}.md
         run: |
           export QA_DIR="${RUNNER_TEMP}/qa" TESTING_DIR="${RUNNER_TEMP}/testing" LOG_DIR="${RUNNER_TEMP}/logs"
           export EXPECTED_FAILURES_FILE="${GITHUB_WORKSPACE}/${EXPECTED}"
-          .github/acceptance/run-part.sh "${IMAGE}" "${PHP}" "${TYPE}" "${PART}" "${PARTS}" "${TAGS}"
+          .github/acceptance/run-part.sh "${IMAGE}" "${PHP}" "${TYPE}" "${SUITES}" "${TAGS}"
 
       - name: Upload server logs
         if: failure()
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
-          name: acceptance-logs-${{ inputs.version }}-${{ matrix.suite.type }}-${{ matrix.suite.part }}
+          name: acceptance-logs-${{ inputs.version }}-${{ matrix.suite.type }}-${{ matrix.suite.slice }}
           path: ${{ runner.temp }}/logs
           retention-days: 7

.github/workflows/main.yml

@@ -9,6 +9,13 @@ on:
     - cron: 0 0 * * 0
   workflow_dispatch:
 
+# A new push to a pull request cancels its older runs, which would otherwise hold
+# the organization's runners. Every other run (master, tags, schedule, manual)
+# gets a group of its own, so none is cancelled or waits for another.
+concurrency:
+  group: ${{ github.workflow }}-${{ case(github.event_name == 'pull_request', github.ref, github.run_id) }}
+  cancel-in-progress: true
+
 jobs:
   lint:
     uses: owncloud-docker/ubuntu/.github/workflows/lint-editorconfig.yml@master