diff --git a/.github/actions/common/build-target/action.yaml b/.github/actions/common/build-target/action.yaml index 75e9637d8..c207cae0a 100644 --- a/.github/actions/common/build-target/action.yaml +++ b/.github/actions/common/build-target/action.yaml @@ -33,13 +33,8 @@ outputs: runs: using: "composite" steps: - - name: Install dependencies - shell: bash - working-directory: riscv-gnu-toolchain - run: | - sudo ./.github/setup-apt.sh - sudo apt install python3 -y - sudo apt install python-is-python3 -y + - name: Prepare dependencies + uses: ./.github/actions/common/setup-dependencies - name: Configure shell: bash diff --git a/.github/actions/common/init-and-pull-gcc/action.yaml b/.github/actions/common/init-and-pull-gcc/action.yaml index 20a059f88..c88ef4b3d 100644 --- a/.github/actions/common/init-and-pull-gcc/action.yaml +++ b/.github/actions/common/init-and-pull-gcc/action.yaml @@ -68,9 +68,11 @@ runs: id: pull-coordination run: | cd gcc - git config --add remote.origin.fetch "+refs/vendors/riscv/*:refs/remotes/upstream/vendors/riscv/*" - git fetch - git checkout upstream/vendors/riscv/heads/gcc-15-with-riscv-opts + # The GitHub mirror does not publish GCC vendor refs. Fetch this + # branch explicitly from upstream without changing the mirror remote. + git fetch --no-tags https://gcc.gnu.org/git/gcc.git \ + +refs/vendors/riscv/heads/gcc-15-with-riscv-opts:refs/remotes/upstream/vendors/riscv/heads/gcc-15-with-riscv-opts + git checkout --detach upstream/vendors/riscv/heads/gcc-15-with-riscv-opts continue-on-error: true - name: Sleep and retry @@ -81,9 +83,9 @@ runs: echo "Failed to checkout and pull gcc. Retrying in 1 min" sleep 60 cd gcc - git config --add remote.origin.fetch "+refs/vendors/riscv/*:refs/remotes/upstream/vendors/riscv/*" - git fetch - git checkout upstream/vendors/riscv/heads/gcc-15-with-riscv-opts + git fetch --no-tags https://gcc.gnu.org/git/gcc.git \ + +refs/vendors/riscv/heads/gcc-15-with-riscv-opts:refs/remotes/upstream/vendors/riscv/heads/gcc-15-with-riscv-opts + git checkout --detach upstream/vendors/riscv/heads/gcc-15-with-riscv-opts - name: Checkout release branch shell: bash diff --git a/.github/actions/common/setup-dependencies/action.yaml b/.github/actions/common/setup-dependencies/action.yaml new file mode 100644 index 000000000..f18bfed21 --- /dev/null +++ b/.github/actions/common/setup-dependencies/action.yaml @@ -0,0 +1,17 @@ +name: Prepare toolchain dependencies +description: Install dependencies on GitHub-hosted runners or verify a provisioned self-hosted runner +runs: + using: composite + steps: + - name: Install dependencies on the GitHub-hosted image + if: ${{ runner.environment == 'github-hosted' }} + shell: bash + working-directory: riscv-gnu-toolchain + run: | + sudo ./.github/setup-apt.sh + sudo apt install python3 python-is-python3 -y + + - name: Verify preinstalled self-hosted dependencies + if: ${{ runner.environment != 'github-hosted' }} + shell: bash + run: bash "$GITHUB_ACTION_PATH/check-dependencies.sh" diff --git a/.github/actions/common/setup-dependencies/check-dependencies.sh b/.github/actions/common/setup-dependencies/check-dependencies.sh new file mode 100644 index 000000000..8fd1bd615 --- /dev/null +++ b/.github/actions/common/setup-dependencies/check-dependencies.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Provision Ubuntu self-hosted machines using the maintenance account before +# starting a runner. A CI job only checks prerequisites; it does not use sudo. +set -euo pipefail + +missing=0 +for tool in gcc g++ make git gh python python3 pip3 autoconf automake bison flex \ + gawk gperf cmake ninja pkg-config expect tclsh dtc zip unzip tar \ + bc patch makeinfo curl flock dpkg-query; do + if ! command -v "$tool" >/dev/null; then + printf 'Missing runner command: %s\n' "$tool" >&2 + missing=1 + fi +done + +if command -v dpkg-query >/dev/null; then + for package in libgmp-dev libmpfr-dev libmpc-dev zlib1g-dev libexpat1-dev \ + libglib2.0-dev libpixman-1-dev tcl-dev python3-venv python3-pyelftools; do + status=$(dpkg-query -W -f='${db:Status-Status}' "$package" 2>/dev/null) || status=missing + if test "$status" != installed; then + printf 'Missing runner package: %s\n' "$package" >&2 + missing=1 + fi + done +fi + +if ! python3 -c 'import elftools, venv'; then + echo 'Missing runner Python modules: elftools or venv' >&2 + missing=1 +fi + +if test "$missing" -ne 0; then + echo 'Provision the missing dependencies with the maintenance account before starting CI.' >&2 + exit 1 +fi +echo 'Preinstalled toolchain dependencies verified.' diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 1988f453d..3d9fe8833 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -140,8 +140,8 @@ jobs: cd gcc git rev-parse HEAD - - name: Install dependencies - run: sudo ./.github/setup-apt.sh + - name: Prepare dependencies + uses: ./.github/actions/common/setup-dependencies # Only 32/64gc multilibs run on github-hosted runners - name: Configure @@ -217,11 +217,8 @@ jobs: cd gcc git rev-parse HEAD - - name: Install dependencies - run: | - sudo ./.github/setup-apt.sh - sudo apt install python3 -y - sudo apt install python-is-python3 -y + - name: Prepare dependencies + uses: ./.github/actions/common/setup-dependencies # Only 32/64gc multilibs run on github-hosted runners - name: Configure diff --git a/.github/workflows/rvv-intrinsic-test.yaml b/.github/workflows/rvv-intrinsic-test.yaml index 4dfecae56..ef8a63dd2 100644 --- a/.github/workflows/rvv-intrinsic-test.yaml +++ b/.github/workflows/rvv-intrinsic-test.yaml @@ -46,9 +46,8 @@ jobs: cd gcc git rev-parse HEAD - - name: Install dependencies - working-directory: riscv-gnu-toolchain - run: sudo ./.github/setup-apt.sh + - name: Prepare dependencies + uses: ./.github/actions/common/setup-dependencies - name: Clone rvv-intrinsic-doc id: update-intrinsic diff --git a/docs/rise-postcommit-cutover.md b/docs/rise-postcommit-cutover.md index 3a1ec7429..344a81867 100644 --- a/docs/rise-postcommit-cutover.md +++ b/docs/rise-postcommit-cutover.md @@ -47,6 +47,24 @@ baseline source. GitHub-hosted jobs continue to use `ubuntu-24.04`. +Provision self-hosted Ubuntu machines using a maintenance account before +starting their runner services. Run the toolchain's `.github/setup-apt.sh` and +install `python-is-python3`, `tcl-dev`, `libpixman-1-dev`, `python3-venv`, and +`python3-pyelftools`. Verify the result as the runner user with +`bash .github/actions/common/setup-dependencies/check-dependencies.sh`. +The runner account does not need sudo: self-hosted jobs check the preinstalled +tools and development packages, while GitHub-hosted jobs install dependencies +on their disposable image. + +Use an absolute work directory such as `/srv/rise-gcc-ci//_work` when +registering a runner. GCC's `pr127778.c` test at revision `78cfe216a8e` matches +`lib` in `/var/lib/...` debug strings with its unbounded `\mli` assembler +pattern, producing a false failure. Keep the account home and runner installation +separate from the work directory. If the service uses `ProtectSystem=strict`, +include the runner installation, account home and work directory in its +`ReadWritePaths`. Test a fresh build in the configured directory before enabling +the production runner label. + ## Bootstrap 1. Merge the RISE toolchain submodule update first. diff --git a/riscv-gnu-toolchain b/riscv-gnu-toolchain index 094a1ae85..12ea3d2ca 160000 --- a/riscv-gnu-toolchain +++ b/riscv-gnu-toolchain @@ -1 +1 @@ -Subproject commit 094a1ae8541c848a3785d6fae348616ae5a9d544 +Subproject commit 12ea3d2ca8a8530ffa9dc0bf583aa0f713370940 diff --git a/scripts/test_coordination_checkout.py b/scripts/test_coordination_checkout.py new file mode 100644 index 000000000..22a3d0584 --- /dev/null +++ b/scripts/test_coordination_checkout.py @@ -0,0 +1,117 @@ +"""Exercise the action against a mirror without vendor refs and local upstream.""" + +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +import yaml + + +ACTION = ( + Path(__file__).resolve().parents[1] + / ".github/actions/common/init-and-pull-gcc/action.yaml" +) +VENDOR_REF = "refs/vendors/riscv/heads/gcc-15-with-riscv-opts" +TRACKING_REF = "refs/remotes/upstream/vendors/riscv/heads/gcc-15-with-riscv-opts" + + +class CoordinationCheckoutTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.upstream = self.root / "upstream" + self.mirror = self.root / "mirror" + self.checkout = self.root / "gcc" + self.env = dict( + os.environ, + GIT_CONFIG_NOSYSTEM="1", + GIT_CONFIG_GLOBAL=os.devnull, + GIT_TERMINAL_PROMPT="0", + GIT_AUTHOR_NAME="CI test", + GIT_AUTHOR_EMAIL="ci@example.invalid", + GIT_COMMITTER_NAME="CI test", + GIT_COMMITTER_EMAIL="ci@example.invalid", + ) + self.git("init", "--initial-branch=master", str(self.upstream)) + self.git("-C", str(self.upstream), "commit", "--allow-empty", "-m", "master") + self.git("clone", "--bare", str(self.upstream), str(self.mirror)) + self.git("clone", str(self.mirror), str(self.checkout)) + self.git("-C", str(self.upstream), "commit", "--allow-empty", "-m", "vendor") + self.tip = self.git("-C", str(self.upstream), "rev-parse", "HEAD") + self.git("-C", str(self.upstream), "update-ref", VENDOR_REF, self.tip) + # Redirect only the official URL, keeping the real action commands intact. + self.env.update( + GIT_CONFIG_COUNT="1", + GIT_CONFIG_KEY_0=f"url.{self.upstream}.insteadOf", + GIT_CONFIG_VALUE_0="https://gcc.gnu.org/git/gcc.git", + ) + action = yaml.safe_load(ACTION.read_text()) + self.initial, self.retry = [ + s["run"] + for s in action["runs"]["steps"] + if "inputs.branch == 'coordination'" in s.get("if", "") + ] + # Avoid the retry delay while still executing its fetch/checkout commands. + self.retry = self.retry.replace("sleep 60", ":") + + def git(self, *args): + return subprocess.check_output( + ["git", *args], env=self.env, text=True, stderr=subprocess.DEVNULL + ).strip() + + def run_action(self, script): + return subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", script], + cwd=self.root, + env=self.env, + text=True, + capture_output=True, + ) + + def test_fetches_vendor_only_ref_without_changing_origin(self): + before = self.git("-C", str(self.checkout), "config", "--local", "--list") + result = self.run_action(self.initial) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + self.git("-C", str(self.checkout), "rev-parse", "HEAD"), self.tip + ) + self.assertEqual( + self.git("-C", str(self.checkout), "config", "--local", "--list"), before + ) + self.assertEqual( + self.git("-C", str(self.checkout), "branch", "--show-current"), "" + ) + + def test_retry_refreshes_a_cached_vendor_ref(self): + self.assertEqual(self.run_action(self.initial).returncode, 0) + self.git( + "-C", str(self.upstream), "commit", "--allow-empty", "-m", "new vendor tip" + ) + tip = self.git("-C", str(self.upstream), "rev-parse", "HEAD") + self.git("-C", str(self.upstream), "update-ref", VENDOR_REF, tip) + result = self.run_action(self.retry) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.git("-C", str(self.checkout), "rev-parse", "HEAD"), tip) + + def test_missing_upstream_ref_does_not_use_stale_cache(self): + self.git("-C", str(self.checkout), "update-ref", TRACKING_REF, "HEAD") + original = self.git("-C", str(self.checkout), "rev-parse", "HEAD") + self.git("-C", str(self.upstream), "update-ref", "-d", VENDOR_REF) + for script in (self.initial, self.retry): + with self.subTest(script=script): + result = self.run_action(script) + self.assertNotEqual(result.returncode, 0) + self.assertEqual( + self.git("-C", str(self.checkout), "rev-parse", "HEAD"), original + ) + self.assertEqual( + self.git("-C", str(self.checkout), "branch", "--show-current"), + "master", + ) + + +if __name__ == "__main__": + unittest.main()