diff --git a/.github/acceptance/plan-slices.sh b/.github/acceptance/plan-slices.sh new file mode 100755 index 0000000..cfa5fab --- /dev/null +++ b/.github/acceptance/plan-slices.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Pack ownCloud core's acceptance suites into slices that take about equally +# long, by the minutes in suite-minutes.tsv, for the matrix of the tests job. +# run.sh --part can't do this: it splits the suites by count, not duration. +# +# usage: plan-slices.sh +# suites file listing one suite per line, e.g. apiMain +# mode smoke or full, the column of suite-minutes.tsv to use +# target minutes a slice should take, unless one suite alone takes longer +# api-tags Behat tag filter for the api suites +# cli-tags Behat tag filter for the cli suites +# +# Prints the matrix as JSON: [{type, slice, slices, suites, tags, minutes}], +# where suites is the comma-separated list for run.sh --suites. +set -euo pipefail + +suites=$1 mode=$2 target=$3 api_tags=$4 cli_tags=$5 +table="$(dirname "$0")/suite-minutes.tsv" +[[ "${mode}" == smoke || "${mode}" == full ]] || { echo "unknown mode ${mode}" >&2 && exit 1; } + +# "suiteminutes" for every suite to run. The cli suites always run in full, +# so they take the longer of both columns. A suite the table doesn't know yet +# still runs, with a default duration that is easy to spot in the plan. +weights=$(awk -F'\t' -v mode="${mode}" ' + NR == FNR { if (!/^#/ && NF) { smoke[$1] = $2; full[$1] = $3 }; next } + NF { + if ($1 in full) { + if ($1 ~ /^cli/) minutes = (smoke[$1] > full[$1]) ? smoke[$1] : full[$1] + else minutes = (mode == "smoke") ? smoke[$1] : full[$1] + } else { + minutes = (mode == "smoke" && $1 !~ /^cli/) ? 3 : 20 + printf "::warning::suite %s is not in suite-minutes.tsv, assuming %s minutes\n", $1, minutes > "/dev/stderr" + } + print $1 "\t" minutes + }' "${table}" "${suites}") +[[ -n "${weights}" ]] || { echo "no suites in ${suites}" >&2 && exit 1; } + +# Longest suite first, each into the slice with the fewest minutes so far. No +# more slices than suites: run.sh --suites "" would run all suites of the type. +matrix=$(jq -Rnc --argjson target "${target}" --arg api "${api_tags}" --arg cli "${cli_tags}" ' + [inputs | split("\t") | {suite: .[0], minutes: (.[1] | tonumber), type: (.[0] | capture("^(?api|cli)").t)}] + | group_by(.type) + | map( + .[0].type as $type + | ([1, ([length, ((map(.minutes) | add) / $target | ceil)] | min)] | max) as $n + | reduce (sort_by(-.minutes, .suite) | .[]) as $s ([range($n) | {suites: [], minutes: 0}]; + (to_entries | min_by([.value.minutes, (.value.suites | length)]).key) as $i + | .[$i].suites += [$s.suite] | .[$i].minutes += $s.minutes) + | to_entries + | map({type: $type, slice: (.key + 1), slices: $n, suites: (.value.suites | sort | join(",")), + tags: (if $type == "api" then $api else $cli end), minutes: (.value.minutes * 10 | round / 10)})) + | flatten' <<<"${weights}") + +# every suite in exactly one slice, and no slice empty +jq -e 'all(.suites != "")' <<<"${matrix}" >/dev/null || { echo "a slice has no suites" >&2 && exit 1; } +planned=$(jq -r '.[].suites | split(",")[]' <<<"${matrix}" | sort) +[[ "${planned}" == "$(cut -f1 <<<"${weights}" | sort -u)" ]] || { + echo "the slices don't run every suite exactly once" >&2 && exit 1 +} + +jq -r '.[] | "\(.type) \(.slice)/\(.slices): \(.minutes) min, \(.suites)"' <<<"${matrix}" >&2 +echo "${matrix}" diff --git a/.github/acceptance/run-part.sh b/.github/acceptance/run-part.sh index 7b87328..d799af2 100755 --- a/.github/acceptance/run-part.sh +++ b/.github/acceptance/run-part.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # Run one part of ownCloud core's acceptance tests against a server image. # -# usage: run-part.sh -# image server image under test, e.g. owncloud/server:test -# php PHP version of the test runner (owncloudci/php:), 7.4 or 8.3 -# type suite type for run.sh: api or cli -# part part to run, 1..parts (run.sh --part) -# tags Behat tag filter +# usage: run-part.sh +# image server image under test, e.g. owncloud/server:test +# php PHP version of the test runner (owncloudci/php:), 7.4 or 8.3 +# type suite type for run.sh: api or cli +# suites comma-separated suites to run (run.sh --suites), see plan-slices.sh +# tags Behat tag filter # # environment: # QA_DIR extracted owncloud-complete-*-qa tarball with Behat installed @@ -15,7 +15,7 @@ # LOG_DIR where to write the server and database logs set -uo pipefail -image=$1 php=$2 type=$3 part=$4 parts=$5 tags=$6 +image=$1 php=$2 type=$3 suites=$4 tags=$5 : "${QA_DIR:?}" "${TESTING_DIR:?}" "${LOG_DIR:?}" net=acceptance db=acceptance-db redis=acceptance-redis srv=acceptance-server mkdir -p "${LOG_DIR}" @@ -91,4 +91,4 @@ docker run --rm --network "${net}" -v "${QA_DIR}:/src" -w /src "${expected[@]}" -e REGULAR_USER_PASSWORD='AaBb2Cc7Dd!' -e SUB_ADMIN_PASSWORD='AaBb2Cc8Dd!' \ -e PLAIN_OUTPUT=true \ "owncloudci/php:${php}" \ - bash tests/acceptance/run.sh --remote --type "${type}" --tags "${tags}" --part "${part}" "${parts}" + bash tests/acceptance/run.sh --remote --type "${type}" --tags "${tags}" --suites "${suites}" diff --git a/.github/acceptance/suite-minutes.tsv b/.github/acceptance/suite-minutes.tsv new file mode 100644 index 0000000..7f23688 --- /dev/null +++ b/.github/acceptance/suite-minutes.tsv @@ -0,0 +1,76 @@ +# Minutes each Behat suite takes, the longest seen in CI on 10.16.x, 11.0.1 and 11.1.0. +# plan-slices.sh packs suites into slices by these. The cli suites always run in +# full, so it weighs them by the longer of both columns in either mode. +# suite smoke full +apiAuth 1.3 11.4 +apiAuthOcs 0.9 2.3 +apiAuthWebDav 4.7 15 +apiCapabilities 0.8 4.4 +apiComments 1.1 3.7 +apiFavorites 1.3 8.9 +apiFederationToRoot1 0.1 0.1 +apiFederationToRoot2 0.1 0.1 +apiFederationToShares1 0.1 0.1 +apiFederationToShares2 0.1 0.1 +apiMain 1.5 20.9 +apiProvisioning-v1 3.2 20.1 +apiProvisioning-v2 3.2 19.9 +apiProvisioningGroups-v1 1.6 10.5 +apiProvisioningGroups-v2 1.5 10.4 +apiShareCreateSpecialToRoot1 0.2 22.5 +apiShareCreateSpecialToRoot2 0.1 9.1 +apiShareCreateSpecialToShares1 0.2 20.6 +apiShareCreateSpecialToShares2 0.1 19 +apiSharees 0.3 9.9 +apiShareManagementBasicToRoot 1.6 20 +apiShareManagementBasicToShares 3.2 25.4 +apiShareManagementToRoot 3.8 25.9 +apiShareManagementToShares 1.4 22 +apiShareOperationsToRoot1 3.5 13.6 +apiShareOperationsToRoot2 1 9.9 +apiShareOperationsToShares1 4.3 17.6 +apiShareOperationsToShares2 1.1 15.3 +apiSharePublicLink1 1 12.3 +apiSharePublicLink2 0.3 12.3 +apiSharePublicLink3 1 9.5 +apiShareReshareToRoot1 0.6 12.7 +apiShareReshareToRoot2 0.8 10 +apiShareReshareToRoot3 0.1 14.8 +apiShareReshareToShares1 0.8 15.6 +apiShareReshareToShares2 0.9 12.1 +apiShareReshareToShares3 0.1 17.6 +apiShareUpdateToRoot 0.4 6.5 +apiShareUpdateToShares 0.5 8.8 +apiSharingNotificationsToRoot 0.2 1.4 +apiSharingNotificationsToShares 0.2 1.4 +apiTags 1.1 8.2 +apiTranslation 0.1 1 +apiTrashbin 2.3 15.2 +apiTrashbinRestore 1.3 10.8 +apiVersions 1 26.1 +apiWebdavDelete 0.6 9.1 +apiWebdavEtagPropagation1 0.1 9 +apiWebdavEtagPropagation2 0.1 8.5 +apiWebdavLocks 6.6 14.9 +apiWebdavLocks2 11.1 20.2 +apiWebdavLocks3 0.1 12.2 +apiWebdavLocksUnlock 0.5 20.3 +apiWebdavMove1 0.1 13.6 +apiWebdavMove2 1.4 15.2 +apiWebdavOperations 1.4 23.7 +apiWebdavPreviews 0.1 6.3 +apiWebdavProperties1 1 27.1 +apiWebdavProperties2 1.7 13.8 +apiWebdavUpload1 1.1 16.3 +apiWebdavUpload2 0.7 8.1 +apiWebdavUploadTUS 0.1 0.1 +cliBackground 1 1 +cliCreateLocalStorage 8.7 8.7 +cliDbConversion 0.2 0.2 +cliEncryption 0.1 0.1 +cliExternalStorage 5.7 5.7 +cliLocalStorage 20.5 20.2 +cliMain 16.6 16.6 +cliManageApps 1 1 +cliProvisioning 14.1 14.1 +cliTrashbin 2.1 2.1 diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 3288abb..5f0c450 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -43,22 +43,31 @@ jobs: outputs: suites: ${{ steps.suites.outputs.suites }} steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - id: suites env: FULL: ${{ inputs.full }} + TARBALL: ${{ inputs.tarball }} + QA_SHA256: ${{ inputs.qa-sha256 }} + # how long a slice of the suites should take, in minutes + SMOKE_SLICE_MINUTES: 25 + FULL_SLICE_MINUTES: 30 run: | + qa="${TARBALL%.tar.bz2}-qa.tar.bz2" + curl -sSfL --retry 3 -o "${RUNNER_TEMP}/qa.tar.bz2" "${qa}" + echo "${QA_SHA256} ${RUNNER_TEMP}/qa.tar.bz2" | sha256sum -c + # the suites are the directories directly below tests/acceptance/features + tar -tjf "${RUNNER_TEMP}/qa.tar.bz2" \ + | sed -nE 's#^[^/]+/tests/acceptance/features/((api|cli)[^/]*)/.*#\1#p' \ + | sort -u > "${RUNNER_TEMP}/suites" base='~@skip&&~@skipOnDockerContainerTesting&&~@federation-app-required' if [ "${FULL}" = true ]; then - api_parts=10 api_tags="${base}" + mode=full minutes="${FULL_SLICE_MINUTES}" api_tags="${base}" else - api_parts=2 api_tags="@smokeTest&&${base}" + mode=smoke minutes="${SMOKE_SLICE_MINUTES}" api_tags="@smokeTest&&${base}" fi - cli_parts=2 - suites=$( - { for p in $(seq "${api_parts}"); do jq -nc --arg t "${api_tags}" --argjson p "$p" --argjson n "${api_parts}" '{type: "api", part: $p, parts: $n, tags: $t}'; done - for p in $(seq "${cli_parts}"); do jq -nc --arg t "${base}" --argjson p "$p" --argjson n "${cli_parts}" '{type: "cli", part: $p, parts: $n, tags: $t}'; done - } | jq -sc . - ) + suites=$(.github/acceptance/plan-slices.sh "${RUNNER_TEMP}/suites" "${mode}" "${minutes}" "${api_tags}" "${base}") echo "suites=${suites}" >> "$GITHUB_OUTPUT" image: @@ -88,7 +97,7 @@ jobs: compression-level: 1 tests: - name: ${{ matrix.suite.type }} ${{ matrix.suite.part }}/${{ matrix.suite.parts }} + name: ${{ matrix.suite.type }} ${{ matrix.suite.slice }}/${{ matrix.suite.slices }} needs: [plan, image] runs-on: ubuntu-latest strategy: @@ -152,19 +161,18 @@ jobs: env: PHP: ${{ inputs.php }} TYPE: ${{ matrix.suite.type }} - PART: ${{ matrix.suite.part }} - PARTS: ${{ matrix.suite.parts }} + SUITES: ${{ matrix.suite.suites }} TAGS: ${{ matrix.suite.tags }} EXPECTED: .github/acceptance/expected-failures-${{ inputs.version }}-${{ case(inputs.full, 'full', 'smoke') }}.md run: | export QA_DIR="${RUNNER_TEMP}/qa" TESTING_DIR="${RUNNER_TEMP}/testing" LOG_DIR="${RUNNER_TEMP}/logs" export EXPECTED_FAILURES_FILE="${GITHUB_WORKSPACE}/${EXPECTED}" - .github/acceptance/run-part.sh "${IMAGE}" "${PHP}" "${TYPE}" "${PART}" "${PARTS}" "${TAGS}" + .github/acceptance/run-part.sh "${IMAGE}" "${PHP}" "${TYPE}" "${SUITES}" "${TAGS}" - name: Upload server logs if: failure() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: acceptance-logs-${{ inputs.version }}-${{ matrix.suite.type }}-${{ matrix.suite.part }} + name: acceptance-logs-${{ inputs.version }}-${{ matrix.suite.type }}-${{ matrix.suite.slice }} path: ${{ runner.temp }}/logs retention-days: 7 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 392712b..c75e313 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,6 +9,13 @@ on: - cron: 0 0 * * 0 workflow_dispatch: +# A new push to a pull request cancels its older runs, which would otherwise hold +# the organization's runners. Every other run (master, tags, schedule, manual) +# gets a group of its own, so none is cancelled or waits for another. +concurrency: + group: ${{ github.workflow }}-${{ case(github.event_name == 'pull_request', github.ref, github.run_id) }} + cancel-in-progress: true + jobs: lint: uses: owncloud-docker/ubuntu/.github/workflows/lint-editorconfig.yml@master