diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b45f85..2f7799a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,8 +39,11 @@ jobs: filters: | # ci.yml is listed because the ansible jobs' cache wiring lives here, so a # change to it must re-run them — same reason the helm filter claims it. + # galaxy-build runs examples/collection-fleet against the built + # collections (galaxy/check-playbooks.sh), so the example counts too. ansible: - 'ansible/**' + - 'examples/collection-fleet/**' - '.github/workflows/ci.yml' # The chart shares the schema-key inventory and checker with molecule, # and renders monitoring/decdn-node/ through a symlink; lint-helm also @@ -425,7 +428,9 @@ jobs: - name: Script and guard tests run: make test-scripts - # Lint the workflow files themselves. + # Lint the workflow files themselves, and the one examples/collection-fleet ships + # (actionlint finds only .github/workflows/ on its own). Named paths replace its own + # discovery: every workflow here is *.yml, and a *.yaml one would go unlinted. actionlint: runs-on: ubuntu-latest timeout-minutes: 10 # ~15s in practice @@ -438,3 +443,4 @@ jobs: with: reporter: github-check fail_on_error: true + actionlint_flags: .github/workflows/*.yml examples/collection-fleet/.github/workflows/*.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a43bdeb..b9bc811 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -71,6 +71,15 @@ repos: files: ^(ansible|cloud-init)/.*\.(ya?ml)$ exclude: ^ansible/(collections|\.ansible)/ + # examples/collection-fleet's inventory and requirements.yml, under its own + # .yamllint, as the yamllint hook it ships runs them in a copy. + - id: yamllint-collection-fleet + name: yamllint (examples/collection-fleet/) + entry: yamllint -c examples/collection-fleet/.yamllint + language: python + additional_dependencies: ["yamllint==1.35.1"] + files: ^examples/collection-fleet/(inventory/.*|requirements)\.ya?ml$ + # Opt-in: full ansible-lint locally (needs `make -C ansible deps` first). # pre-commit run ansible-lint --hook-stage manual - id: ansible-lint diff --git a/AGENTS.md b/AGENTS.md index f947733..5b611eb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,7 +81,7 @@ compose/ # one-host Docker Compose path (README.md) charts/decdn-node/ # Helm chart (README.md); ci/ values mirror molecule/schema's three plays; files/monitoring -> ../../../monitoring/decdn-node (symlink); tests/render-test.sh (make lint-helm); artifacthub-repo.yml monitoring/ # Grafana dashboards + Prometheus rules, maintained here by hand: decdn-node/ (rendered by the chart), sponsord/, iroh-relay/ (+ exported-metrics.txt); README.md docs/ # cross-path operator docs: node-operators.md, publishers.md, requirements.md, lifecycle.md -examples/collection-fleet/ # consumer project for the collections; its inventory/ mirrors ansible/inventory/'s templates byte for byte except the hosts-*.yml.example comment headers (make test-scripts checks both parse to the same inventory): change both +examples/collection-fleet/ # consumer project for the collections, with its own CI, pre-commit and AGENTS.md; its inventory/ mirrors ansible/inventory/'s templates byte for byte except the hosts-*.yml.example comment headers (make test-scripts checks both parse to the same inventory): change both. release.sh moves its requirements.yml floors; its action SHAs and hook revs track this repo's (make test-scripts); galaxy/check-playbooks.sh runs it against the built collections scripts/ # upstream mirrors (sync-network-profiles.py, gen-schema-keys.py), render-compose-alloy.sh, the decdn bump (open-decdn-bump.sh, bump_decdn.py), fetch-decdn-cli.sh, release.sh (+ cliff.toml, chart-artifacthub-changes.py), molecule.sh, limit-guard.sh ``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5eb7236..a0e84b4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,9 +42,9 @@ targets, so a local pass means a CI pass. Deploy targets live in | `make lint-alloy` | renders `roles/grafana_alloy`'s templates and validates them with the **real** digest-pinned Alloy binary. The molecule stub exits 0 for everything, so this is the only gate that proves the config loads. It also checks that `compose/alloy/config.alloy` (the Compose `alloy` profile's render of the same template) is current, and runs it in Alloy: every component healthy, each daemon scraped only under the profiles that start it. `ALLOY_BIN=` skips the download. | | `make lint-compose` | reads `compose/compose.yaml` as written (`yq`) against `compose/tests/bind-sources.jq` (every bind mount's `create_host_path`), then renders it with every profile on (with its example env, without env files, with an empty `.env`) and asserts `compose/tests/invariants.jq`, `inline-env.jq` and `fail-closed.jq` | | `make lint-cloud-init` | `cloud-init schema` on `cloud-init/user-data-node.yaml` and `cloud-init/user-data-publisher.yaml`, then `cloud-init/tests/lint.py` on each: only the templates' top-level modules and no YAML anchors, no secrets (only the bootstrap's own files, once each, plain `content` only, no secret-looking keys or assignments), no hardening skip, `release` installs verified against the vendored keys with a host-generated node wallet (trust knobs only in their own group's `vars`, no moved keyring or secret path), only localhost, in `decdn_nodes` and/or `sponsord_hosts` (an origin only beside `decdn_nodes` and with a backend, every backend key only in `decdn_origin_nodes.vars`, the onramp only beside sponsord), a keyed admin account, `runcmd` exactly stage 1, shellcheck-clean scripts, and a collection lock that covers `ansible/requirements.yml` (needs `cloud-init`, `shellcheck`, `yq`). `CLOUD_INIT_FILE=` checks your own filled-in copy. | -| `make test-scripts` | `tests/scripts-test.sh`: the `ansible/Makefile` scoping guards (dry runs, and `limit-guard.sh` on a fixture inventory), the molecule driver's selection guards and locks (no containers), the split scenarios' shared inventories, the release gate, `scripts/release.sh` on a fixture repo and `pr-title.yml`'s check (the former skipped without `git-cliff` or `ssh-keygen` on PATH, failed in CI), the negative cases of `lint-compose` and `lint-cloud-init` (the latter skipped without `cloud-init` on PATH), `compose/decdn-compose`'s unit tests (`compose/tests/test_decdn_compose.py`), `scripts/bump_decdn.py`'s (`scripts/tests/test_bump_decdn.py`), that `compose/alloy/config.alloy` is a fresh render of the `grafana_alloy` template (`scripts/render-compose-alloy.sh --check`; needs ansible-core, failed in CI without it) and that Compose's `grafana/alloy` image is `grafana_alloy_version`, and the cloud-init bootstrap's contracts: `baseline-plays.sh` on fixtures and on the real playbooks (every node.yml, origin.yml and sponsord.yml play runs `baseline` tagged `baseline`), the two templates' shared stage 1, login hint and `final_message`, `bootstrap.sh`'s groups against lint.py's, and lint.py's `FORBIDDEN_VARS` against the roles' keyrings and secret paths; and the Galaxy collections' shipped playbooks: every playbook in a `playbooks.txt`, `galaxy/stage-playbooks.py` on both collections and its refusals, that it changes only role names, `decdn.node`'s `backup.yml`/`decommission.yml` against the fleet-wide node plays in both directions (needs yq, failed in CI without it), and the two roles' shared `*_backup_local_dir` default; and `examples/collection-fleet/inventory/` against `ansible/inventory/` (the same tracked files, byte for byte but for the hosts examples' comment headers; both hosts examples parse to the same inventory, needs ansible-core, failed in CI without it), and the ignore rules that keep a `secret.*` or a filled-in `hosts.yml` there out of git, with negative cases on a fixture. `UPSTREAM=` adds the sync generators' exit codes. | +| `make test-scripts` | `tests/scripts-test.sh`: the `ansible/Makefile` scoping guards (dry runs, and `limit-guard.sh` on a fixture inventory), the molecule driver's selection guards and locks (no containers), the split scenarios' shared inventories, the release gate, `scripts/release.sh` on a fixture repo and `pr-title.yml`'s check (the former skipped without `git-cliff` or `ssh-keygen` on PATH, failed in CI), the negative cases of `lint-compose` and `lint-cloud-init` (the latter skipped without `cloud-init` on PATH), `compose/decdn-compose`'s unit tests (`compose/tests/test_decdn_compose.py`), `scripts/bump_decdn.py`'s (`scripts/tests/test_bump_decdn.py`), that `compose/alloy/config.alloy` is a fresh render of the `grafana_alloy` template (`scripts/render-compose-alloy.sh --check`; needs ansible-core, failed in CI without it) and that Compose's `grafana/alloy` image is `grafana_alloy_version`, and the cloud-init bootstrap's contracts: `baseline-plays.sh` on fixtures and on the real playbooks (every node.yml, origin.yml and sponsord.yml play runs `baseline` tagged `baseline`), the two templates' shared stage 1, login hint and `final_message`, `bootstrap.sh`'s groups against lint.py's, and lint.py's `FORBIDDEN_VARS` against the roles' keyrings and secret paths; and the Galaxy collections' shipped playbooks: every playbook in a `playbooks.txt`, `galaxy/stage-playbooks.py` on both collections and its refusals, that it changes only role names, `decdn.node`'s `backup.yml`/`decommission.yml` against the fleet-wide node plays in both directions (needs yq, failed in CI without it), and the two roles' shared `*_backup_local_dir` default; and `examples/collection-fleet/inventory/` against `ansible/inventory/` (the same tracked files, byte for byte but for the hosts examples' comment headers; both hosts examples parse to the same inventory, needs ansible-core, failed in CI without it), and the ignore rules that keep a `secret.*` or a filled-in `hosts.yml` there out of git, with negative cases on a fixture; and that the example's `requirements.yml` floors are the collections' `galaxy.yml` versions and its CI and pre-commit pin this repo's action SHAs and hook revs, with negative cases. `UPSTREAM=` adds the sync generators' exit codes. | | `make security` | KICS IaC scan of `ansible/`, the rendered chart and `compose/` (digest-pinned engine, fail on HIGH) | -| `make galaxy-check` | build the `decdn.node` and `decdn.publisher` collections, run galaxy-importer's checks on each, then install each and check its playbooks by FQCN, firewall holes included (`ansible/galaxy/check-playbooks.sh`; `make -C ansible galaxy-check-node` / `galaxy-check-publisher` for one) | +| `make galaxy-check` | build the `decdn.node` and `decdn.publisher` collections, run galaxy-importer's checks on each, then install each and check its playbooks by FQCN, firewall holes included, and run `examples/collection-fleet` against it as a copy of it runs (its hosts example parses under its `ansible.cfg`, and every playbook it names ships and passes a syntax check) (`ansible/galaxy/check-playbooks.sh`; `make -C ansible galaxy-check-node` / `galaxy-check-publisher` for one) | `ansible-lint` is **not** a per-commit hook (it needs the collections installed). Run it with `make lint-ansible`, or `pre-commit run ansible-lint --hook-stage manual`. diff --git a/RELEASING.md b/RELEASING.md index c8ad899..f8be952 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -72,7 +72,12 @@ would commit, and changes no file. The script: (they are all `ci`, `test`, `style`, `build` or release commits, or unconventional ones `--allow-unconventional` leaves out), whatever the level. 3. **Sets `version:`** in `ansible/galaxy//galaxy.yml` or - `charts/decdn-node/Chart.yaml`. + `charts/decdn-node/Chart.yaml`. A collection release also raises that collection's + floor in `examples/collection-fleet/requirements.yml` to the new version, so a fleet + copied from the example starts from it (`make test-scripts` refuses a PR where they + differ). Until the tag's publish job uploads it, that floor names a version Galaxy + does not have, and a copy of the example cannot install: so a publish that fails + or waits is one to finish promptly. The chart's `appVersion` and its `artifacthub.io/images` tag already follow the role's `decdn_node_version` (`make test-scripts` refuses a PR where they differ), so a new decdn line is a chart change too: release the chart. A decdn patch release @@ -100,7 +105,7 @@ would commit, and changes no file. The script: before anything is committed. 6. **Commits** (`--execute` only) `chore(release): ` and tags ``, both signed, and pushes them with `git push --atomic origin main `: origin gets both - or neither. If a check or the commit (a pre-commit hook, say) fails, both files are + or neither. If a check or the commit (a pre-commit hook, say) fails, the files are restored and nothing is committed. If tagging or the push fails, or the script is stopped after the commit, the commit (and the tag) stay local, and it says whether origin got anything and how to undo. A refusal before step 5 never touches the diff --git a/ansible/README.md b/ansible/README.md index 7f6be87..f294600 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -131,7 +131,10 @@ make deploy INVENTORY=../../decdn-fleet/hosts.yml LIMIT= Ansible loads `group_vars/` and `host_vars/` from beside whichever `hosts.yml` you pass, so the overlay carries its own: create them next to its `hosts.yml`, starting from the public `inventory/group_vars/` and `inventory/host_vars/` templates. Git-ignore -`host_vars/*/secret.*` in the private repo, so it never tracks a per-node secret. The public +`host_vars/*/secret.*` in the private repo, so it never tracks a per-node secret (and +`group_vars/*/secret.*` for a secret a whole group shares; a `group_vars//` +directory silently replaces a `group_vars/.yml` beside it, so move that file to +`/main.yml` first). The public `inventory/group_vars/` and `inventory/host_vars/` are **not** loaded for it. (Deploying from the Galaxy collections instead? [`examples/collection-fleet/`](../examples/collection-fleet/README.md) is a ready-made private repository for that path, inventory templates and `.gitignore` included.) Add child groups diff --git a/ansible/galaxy/check-playbooks.sh b/ansible/galaxy/check-playbooks.sh index 641fb3c..6619f49 100755 --- a/ansible/galaxy/check-playbooks.sh +++ b/ansible/galaxy/check-playbooks.sh @@ -11,7 +11,12 @@ # molecule/collection runs decdn.node's); # - runs tests/firewall-holes/check.sh on its playbooks/, which runs a probe AS a # collection playbook: the firewall holes must come out as they do from -# ansible/playbooks/, so the shipped group_vars/ load and agree. +# ansible/playbooks/, so the shipped group_vars/ load and agree; +# - runs examples/collection-fleet as a private copy of it runs (its ansible.cfg, +# the collection's hosts example as inventory/hosts.yml): the inventory and its +# vars files load, and every decdn.. its README, AGENTS.md +# and CI name ships and passes a syntax check. Its tracked files only, so stage a +# new one first. # galaxy-importer cannot do the first: its ansible-lint sees one collection, outside # any collections path, so it only warns that it cannot find the roles. # The roles' own dependencies (devsec.hardening, ansible.posix) come from `make deps` @@ -51,7 +56,9 @@ export ANSIBLE_DEPRECATION_WARNINGS=0 # file from the caller. mkdir "$work/no-roles" export ANSIBLE_ROLES_PATH="$work/no-roles" -unset ANSIBLE_CONFIG +# No inventory from the caller either: the example's ansible.cfg names its own, and +# fails on one that does not parse. +unset ANSIBLE_CONFIG ANSIBLE_INVENTORY ANSIBLE_INVENTORY_UNPARSED_FAILED ANSIBLE_INVENTORY_ANY_UNPARSED_IS_FAILED inventory="$ansible_dir/tests/firewall-holes/inventory.yml" playbooks="$work/ansible_collections/decdn/$collection/playbooks" @@ -65,4 +72,43 @@ for pb in "$playbooks"/*.yml; do done ((n > 0)) || { echo "check-playbooks.sh: decdn.$collection ships no playbook" >&2; exit 1; } echo "ok decdn.$collection: $n playbooks pass a syntax check by FQCN" + +# examples/collection-fleet, as a private copy of it runs: its tracked files, its +# ansible.cfg (read from the copy's directory; ANSIBLE_COLLECTIONS_PATH stands in for +# its collections/), the collection's hosts example as inventory/hosts.yml. The +# inventory loads as its CI loads it (vars files included) and holds the hosts +# example's groups, so a cfg that stopped being read (no inventory: an implicit +# localhost, exit 0) fails here. Every decdn.. its README, +# AGENTS.md and CI name must ship, and passes a syntax check against that inventory. +example="$ansible_dir/../examples/collection-fleet" +fleet="$work/fleet" +mkdir "$fleet" +git -C "$example" ls-files -z | (cd "$example" && xargs -0 cp --parents -t "$fleet") \ + || { echo "check-playbooks.sh: could not copy examples/collection-fleet" >&2; exit 1; } +hosts_example="hosts-$collection.yml.example" +cp "$fleet/inventory/$hosts_example" "$fleet/inventory/hosts.yml" +docs=(README.md AGENTS.md .github/workflows/ci.yml) +for f in "${docs[@]}"; do + [[ -f "$fleet/$f" ]] || { echo "check-playbooks.sh: examples/collection-fleet/$f is not tracked (git add it)" >&2; exit 1; } +done +# grep exits 1 on no match: that is the check below, not a failure here. +named="$(cd "$fleet" && { grep -ohE "decdn\.$collection\.[a-z_]+" "${docs[@]}" || (($? == 1)); } | sort -u)" +[[ -n $named ]] || { echo "check-playbooks.sh: examples/collection-fleet names no decdn.$collection playbook" >&2; exit 1; } +(cd "$fleet" && ansible-inventory --list "$work/inventory.json" 2>"$work/inventory.log") \ + || { cat "$work/inventory.log" >&2; echo "check-playbooks.sh: examples/collection-fleet's $hosts_example does not load" >&2; exit 1; } +groups=(decdn_nodes) +[[ $collection == publisher ]] && groups+=(decdn_origin_nodes) +for g in "${groups[@]}"; do + grep -qF "\"$g\": {" "$work/inventory.json" \ + || { echo "check-playbooks.sh: examples/collection-fleet's inventory has no $g group: was its ansible.cfg read?" >&2; exit 1; } +done +n=0 +for fqcn in $named; do + [[ -f "$playbooks/${fqcn##*.}.yml" ]] \ + || { echo "check-playbooks.sh: examples/collection-fleet names $fqcn, which decdn.$collection does not ship" >&2; exit 1; } + (cd "$fleet" && ansible-playbook "$fqcn" --syntax-check "$work/syntax.log" 2>&1) \ + || { cat "$work/syntax.log" >&2; echo "check-playbooks.sh: $fqcn fails its syntax check in examples/collection-fleet" >&2; exit 1; } + n=$((n + 1)) +done +echo "ok examples/collection-fleet with $hosts_example: the inventory loads and the $n decdn.$collection playbooks it names pass a syntax check" "$ansible_dir/tests/firewall-holes/check.sh" "$playbooks" diff --git a/examples/collection-fleet/.github/dependabot.yml b/examples/collection-fleet/.github/dependabot.yml new file mode 100644 index 0000000..cae061a --- /dev/null +++ b/examples/collection-fleet/.github/dependabot.yml @@ -0,0 +1,36 @@ +--- +# Dependabot — keeps the SHA-pinned GitHub Actions and the pre-commit hook revs +# current. +# +# github-actions rewrites `uses: owner/repo@ # vX.Y.Z` to the new sha + comment +# pre-commit bumps the `rev:` of each repo in .pre-commit-config.yaml +# +# Ansible Galaxy (requirements.yml) is not a Dependabot ecosystem. +# +# NOT covered, bump manually: +# - the deCDN collections in requirements.yml (read each CHANGELOG first) +# - the local yamllint hook's `additional_dependencies` pin +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + commit-message: + prefix: ci # conventional commits -> "ci(deps): ..." + include: scope + groups: + actions: + patterns: ["*"] + - package-ecosystem: pre-commit + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + commit-message: + prefix: ci + include: scope + groups: + pre-commit-hooks: + patterns: ["*"] diff --git a/examples/collection-fleet/.github/workflows/ci.yml b/examples/collection-fleet/.github/workflows/ci.yml new file mode 100644 index 0000000..7767072 --- /dev/null +++ b/examples/collection-fleet/.github/workflows/ci.yml @@ -0,0 +1,105 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +# Least privilege by default; jobs widen only what they need. +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# bash -eo pipefail, not the implicit `bash -e`: a failure inside a pipe fails the step. +defaults: + run: + shell: bash + +# Third-party actions are pinned to a full commit SHA (a re-pointed tag can ship +# malicious code). The trailing comment records the human-readable version; +# .github/dependabot.yml bumps them. +jobs: + # Every pre-commit hook on every file: hygiene, markdownlint and yamllint. The same + # gate `pre-commit run --all-files` runs locally. + pre-commit: + runs-on: ubuntu-latest + timeout-minutes: 10 # hook environments are cached; ~1m cold + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Install pre-commit + run: python -m pip install --upgrade pre-commit + # Hook environments (node for markdownlint, yamllint's virtualenv) keyed on the + # config that pins them. + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/pre-commit + key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} + - name: Run every hook on every file + run: pre-commit run --all-files --show-diff-on-failure + + # The inventory and every committed vars file load, and the collections' playbooks + # load against them. The collections are installed fresh from Galaxy on every run + # (no cache): with `>=` floors in requirements.yml that is the newest release, so a + # release whose playbooks no longer load fails here. What a role makes of your + # variables is only checked by a run (--check --diff). Fails until + # inventory/hosts.yml is committed. Nothing here connects to a host or reads the + # git-ignored secret.yml files; a committed ansible-vault file would need its + # password here. + ansible: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Install ansible-core + run: python -m pip install --upgrade ansible-core + - name: Install the collections + # Galaxy resets connections now and then: retry those, and fail at once on + # anything else (a version the floors ask for that Galaxy does not have, say). + run: | + for attempt in 1 2 3; do + ansible-galaxy collection install -r requirements.yml -p collections 2>&1 | tee install.log && exit 0 + grep -qiE 'connection (reset|refused|aborted)|timed? ?out|HTTP Error 5[0-9][0-9]' install.log \ + || { echo "::error::ansible-galaxy collection install failed"; exit 1; } + ((attempt < 3)) || break + echo "::warning::ansible-galaxy hit a transient error (attempt $attempt), retrying" + sleep $((attempt * 10)) + done + echo "::error::ansible-galaxy collection install failed 3 times" + exit 1 + # The same check the pre-commit hook runs on your working tree. + - name: No vars source shadowed by another of the same name + run: scripts/check-vars-shadowing.sh + # --list, not --graph: it loads every group_vars/ and host_vars/ file too. Its + # output (your committed variables) is not printed. + - name: Load the inventory + run: ansible-inventory --list >/dev/null + # decdn.node alone (cache nodes only)? Check decdn.node.node, decdn.node.backup + # and decdn.node.decommission instead. + - name: Syntax-check the playbooks + run: | + for pb in decdn.publisher.site decdn.publisher.backup decdn.publisher.decommission; do + ansible-playbook "$pb" --syntax-check + done + + # Lint the workflow files themselves. + actionlint: + runs-on: ubuntu-latest + timeout-minutes: 10 # ~15s in practice + permissions: + contents: read + checks: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: reviewdog/action-actionlint@419a49378266e892f40c773b4f2fc6de9aa87499 # v1.80.4 + with: + reporter: github-check + fail_on_error: true diff --git a/examples/collection-fleet/.markdownlint-cli2.yaml b/examples/collection-fleet/.markdownlint-cli2.yaml new file mode 100644 index 0000000..d4f281c --- /dev/null +++ b/examples/collection-fleet/.markdownlint-cli2.yaml @@ -0,0 +1,19 @@ +# markdownlint-cli2 config — a relaxed ruleset that still catches structural issues +# (link fragments that match no heading, empty/duplicate headings, multiple H1s, …). The disabled rules are +# purely stylistic. +config: + MD013: false # line length — prose/tables run long + MD033: false # inline HTML + MD041: false # first line need not be a top-level heading + MD040: false # bare code fences (ASCII trees) are fine + MD031: false # blank lines around fences + MD022: false # blank lines around headings + MD049: false # emphasis style (underscore vs asterisk) + MD060: false # table pipe spacing/alignment + MD004: false # unordered list marker style (-, +, *) + MD024: + siblings_only: true # duplicate headings only flagged within a section + +# The installed collections' docs are excluded at the pre-commit layer too. +ignores: + - "collections/**" diff --git a/examples/collection-fleet/.pre-commit-config.yaml b/examples/collection-fleet/.pre-commit-config.yaml new file mode 100644 index 0000000..87cfae5 --- /dev/null +++ b/examples/collection-fleet/.pre-commit-config.yaml @@ -0,0 +1,68 @@ +--- +# Pre-commit hooks for the fleet inventory. +# +# pip install pre-commit && pre-commit install # one-time install +# pre-commit run --all-files # run on all files +# +# CI runs the same hooks; beside them, it loads the inventory and syntax-checks the +# collections' playbooks against it (.github/workflows/ci.yml). +minimum_pre_commit_version: "3.5.0" +default_install_hook_types: [pre-commit] + +# The installed collections, and ansible's .ansible/ working directory. +exclude: >- + (?x)^( + collections/| + \.ansible/ + ) + +repos: + # ── Generic hygiene ──────────────────────────────────────────────────────── + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] # keep markdown hard line breaks + - id: end-of-file-fixer + - id: check-merge-conflict + - id: check-added-large-files + args: [--maxkb=512] + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + - id: mixed-line-ending + args: [--fix=lf] + - id: check-yaml + args: [--unsafe] # tolerate custom/!vault tags; syntax check only + - id: check-json + + # ── Bash ─────────────────────────────────────────────────────────────────── + - repo: https://github.com/koalaman/shellcheck-precommit + rev: v0.11.0 + hooks: + - id: shellcheck + + # ── Markdown (relaxed ruleset; see .markdownlint-cli2.yaml) ──────────────── + - repo: https://github.com/DavidAnson/markdownlint-cli2 + rev: v0.23.3 + hooks: + - id: markdownlint-cli2 + + # ── Local: yamllint for the Ansible inputs (.yamllint), vars shadowing ───── + # *.yml.example templates don't match yamllint's files (they end in .example). + - repo: local + hooks: + - id: yamllint-ansible + name: yamllint (inventory/, requirements.yml) + entry: yamllint -c .yamllint + language: python + additional_dependencies: ["yamllint==1.35.1"] + files: ^(inventory/.*|requirements)\.ya?ml$ + + # On your working tree, so it sees the git-ignored secret.yml files (and their + # directories) that CI never does. + - id: vars-shadowing + name: no vars source shadowed by another of the same name + entry: scripts/check-vars-shadowing.sh + language: script + pass_filenames: false + always_run: true diff --git a/examples/collection-fleet/.yamllint b/examples/collection-fleet/.yamllint new file mode 100644 index 0000000..3c3bb6d --- /dev/null +++ b/examples/collection-fleet/.yamllint @@ -0,0 +1,18 @@ +--- +extends: default +rules: + # Compatible with ansible-lint's embedded yamllint requirements, should you add it. + comments-indentation: false + octal-values: + forbid-implicit-octal: true + forbid-explicit-octal: true + line-length: + max: 120 + truthy: + allowed-values: ["true", "false"] + comments: + min-spaces-from-content: 1 + braces: + max-spaces-inside: 1 # allow "{{ var }}" Jinja spacing +ignore: | + collections/ diff --git a/examples/collection-fleet/AGENTS.md b/examples/collection-fleet/AGENTS.md new file mode 100644 index 0000000..6e88aee --- /dev/null +++ b/examples/collection-fleet/AGENTS.md @@ -0,0 +1,93 @@ +# AGENTS.md + +Guidance for AI coding agents (Claude Code, Codex, Cursor, …) working in this fleet +repository. `CLAUDE.md` includes this file. + +## What this repo is + +A deCDN fleet as **inventory only**. Every role and playbook comes from the +[`decdn.node`](https://galaxy.ansible.com/ui/repo/published/decdn/node/) and +[`decdn.publisher`](https://galaxy.ansible.com/ui/repo/published/decdn/publisher/) +Galaxy collections (`requirements.yml`, installed into the git-ignored +`collections/`). The layout is decdn/devops's +[`examples/collection-fleet`](https://github.com/decdn/devops/tree/main/examples/collection-fleet). + +There are no roles or playbooks here. Role variables, their defaults and the role +READMEs live in the installed collections +(`collections/ansible_collections/decdn/{node,publisher}/roles/`): read them there, +not from memory. A change to a role or playbook is made upstream in +[decdn/devops](https://github.com/decdn/devops) and arrives with a collection release. + +## Hard rules + +1. **Never commit secrets.** No passwords, private keys, API tokens, RPC URLs with keys + in them or keystores in any tracked file. Inventory secrets live in the git-ignored + `secret.yml` files (`host_vars//` or `group_vars//`); host secrets are + generated on, or provisioned by the operator to, the host. `.gitignore` is a + backstop, not the design. +2. **Infrastructure only.** Protocol and economic facts trace to the + [deCDN ADRs](https://github.com/decdn/decdn/tree/main/adr). Contract addresses come + only from the collections' network profile (`decdn_network`), never copied into the + inventory or docs. +3. **Show before deploying.** Present inventory changes; a playbook run is what mutates + a host. Dry-run with `--check --diff` first, always pass `-l` (nothing here stops a + run from reaching every host the playbook targets, but for decommission's + `decdn_decommission_max_hosts` cap, and a `-l` that matches none of them exits 0 + having done nothing), and read the PLAY RECAP. + +## Gotchas + +- **A vars directory shadows the file beside it.** Ansible reads one of + `group_vars//`, `.yml`, `.yaml` and `.json` and ignores + the rest, silently. To give a group a `secret.yml`, first move its `.yml` to + `/main.yml`; never keep both. The same holds for `host_vars/`. + `scripts/check-vars-shadowing.sh` refuses such a pair (the pre-commit hook runs it on + the working tree, where the git-ignored directories exist; CI on the tracked files). +- **Host vars beat group vars.** A `host_vars//secret.yml` overrides the same + variable in a group's `secret.yml`; a secret every host shares belongs in + `group_vars/all/` (move `all.yml` to `all/main.yml` first). +- **No `ansible_user` by default.** Runs connect as the local `$USER`, the admin + account `baseline` creates. A fresh host's first run connects once as root or the + image's bootstrap user (`-u root`; `README.md` § Run and the hosts example's header + say how); root login is off after it. Never set `ansible_user` to the bootstrap user. +- **Ubuntu 25.10+ ships sudo-rs**, which hangs Ansible's become prompt, but only when + a become password is sent (`baseline_sudo_passwordless: false`; the default is + passwordless). Then set `ansible_become_exe: /usr/bin/sudo.ws` on that host + (`inventory/hosts-node.yml.example`'s header explains). +- **Never run `decdn.node.decommission` on a host that also runs sponsord, a relay or + a DNS server**: it removes the Grafana Alloy agent they share. Use + `decdn.publisher.decommission`. +- **Until you pin `==X.Y.Z`, collections float on `>=` floors.** CI installs the + newest release; a new release can move the default decdn or sponsord release line. Read each collection's + [CHANGELOG](https://github.com/decdn/devops/tree/main/ansible/galaxy) before an + `--upgrade`. + +## Commands + +```bash +ansible-galaxy collection install -r requirements.yml -p collections # once, and after a bump (--upgrade) +ansible-playbook decdn.publisher.site -l --check --diff # dry run +ansible-playbook decdn.publisher.site -l # deploy what the host's groups call for +ansible-playbook decdn.publisher.backup -l # fetched to backups/ (git-ignored) +ansible-playbook decdn.publisher.decommission -l # typed confirmation; keeps the keys +pre-commit run --all-files # the lint gate CI runs +``` + +With `decdn.node` alone (cache nodes only), use `decdn.node.node`, `decdn.node.backup` +and `decdn.node.decommission`. `README.md` maps decdn/devops's `make` targets to these +commands. + +## CI + +`.github/workflows/ci.yml`: `pre-commit` (every hook, every file), `ansible` (a fresh +Galaxy install, `scripts/check-vars-shadowing.sh`, `ansible-inventory --list`, which +loads every vars file, and `--syntax-check` of the site, backup and decommission +playbooks), and `actionlint`. None of it checks what a role makes of your variables: +only a `--check --diff` run does. A committed ansible-vault file would need its +password in CI. + +## This fleet + + diff --git a/examples/collection-fleet/CLAUDE.md b/examples/collection-fleet/CLAUDE.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/examples/collection-fleet/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/examples/collection-fleet/README.md b/examples/collection-fleet/README.md index 2c44a99..d3a4133 100644 --- a/examples/collection-fleet/README.md +++ b/examples/collection-fleet/README.md @@ -12,6 +12,11 @@ lifecycle guards included), and this repository holds only your inventory. | `ansible.cfg` | decdn/devops's `ansible/ansible.cfg`, minus `roles_path`, failing on an inventory that does not parse | | `.gitignore` | keeps the installed collections, per-host secrets and backups out of git | | `inventory/` | the inventory templates: `hosts-node.yml.example` and `hosts-publisher.yml.example`, with their `group_vars/` and `host_vars/` | +| `.github/workflows/ci.yml` | CI: the pre-commit hooks; a fresh Galaxy install that loads your inventory and syntax-checks the playbooks against it; actionlint ([Checks](#checks)) | +| `.pre-commit-config.yaml`, `.yamllint`, `.markdownlint-cli2.yaml` | the lint gate: hygiene, shellcheck, yamllint over `inventory/` and `requirements.yml`, markdownlint, and the vars-shadowing check | +| `scripts/check-vars-shadowing.sh` | refuses two vars sources of the same name in `group_vars/` or `host_vars/`, which Ansible would silently reduce to one | +| `.github/dependabot.yml` | weekly bumps of the pinned actions and hook revs (not of the collections) | +| `AGENTS.md`, `CLAUDE.md` | guidance for AI coding agents; fill in its *This fleet* section | The `inventory/` templates are decdn/devops's `ansible/inventory/` ones, with the hosts examples' headers written for this layout (decdn/devops's `make test-scripts` keeps @@ -37,6 +42,22 @@ live: uncomment the groups you run. Rename `host_vars/decdn-node-1/` (and `origi to your hosts, and keep secrets in the git-ignored `host_vars//secret.yml` (copied from its `secret.yml.example`). +A secret every host of a group shares (one `decdn_rpc_url` for every node, say) can go +once in the git-ignored `group_vars//secret.yml` instead, and one for every host +(a `grafana_alloy_api_token` the sponsord and relay hosts use too) in +`group_vars/all/secret.yml`. Move the group's file into that directory first: + +```bash +mkdir inventory/group_vars/decdn_nodes +git mv inventory/group_vars/decdn_nodes.yml inventory/group_vars/decdn_nodes/main.yml +``` + +Ansible reads a `group_vars//` directory *in place of* a `group_vars/.yml` +beside it and never says so, so keeping both silently drops every setting in the file +(the same holds for `host_vars/`, and for a `.yml` beside a `.yaml`). +`scripts/check-vars-shadowing.sh` refuses such a pair, from the pre-commit hook and +from CI. A host's own `secret.yml` still wins over the group's for the same variable. + ## Run ```bash @@ -92,8 +113,31 @@ playbook for the host's group, and read the PLAY RECAP: The Makefile's third check, an inventory that fails to parse, is `ansible.cfg`'s `unparsed_is_failed` here. +## Checks + +```bash +pip install pre-commit && pre-commit install # once: the hooks run on every commit +pre-commit run --all-files # hygiene, yamllint, markdownlint +``` + +CI (`.github/workflows/ci.yml`) runs on pull requests and pushes to `main`: the same +hooks, and beside them an `ansible` job that installs the collections fresh from +Galaxy, runs the vars-shadowing check, loads the inventory with every vars file +(`ansible-inventory --list`) and syntax-checks `decdn.publisher.site`, `.backup` and +`.decommission` against it (swap in the `decdn.node.*` ones if you installed +`decdn.node` alone). That job fails until `inventory/hosts.yml` is committed. It +installs the newest releases the floors allow, so a collection release whose playbooks +no longer load fails your next PR. What a role makes of your variables (a renamed or +newly required one) only a run shows: dry-run with `--check --diff` after an upgrade. + ## Upgrade -Raise the versions in `requirements.yml` (pin `==X.Y.Z` once the fleet runs), read -each collection's CHANGELOG, then `ansible-galaxy collection install -r -requirements.yml -p collections --upgrade` and dry-run before deploying. +The floors in `requirements.yml` are the newest tagged release of each collection when +this directory was copied, so a fresh install takes the newest anyway. (A floor is +raised when its release is tagged, a little before Galaxy publishes it: if the install +cannot find that version, lower the floor to the newest one on Galaxy.) Once the fleet runs, +pin exact versions (`==X.Y.Z`) and raise them deliberately: read each collection's +[CHANGELOG](https://github.com/decdn/devops/tree/main/ansible/galaxy) (a release can +move the default decdn or sponsord release line, or roll out host changes), then +`ansible-galaxy collection install -r requirements.yml -p collections --upgrade` and +dry-run before deploying. diff --git a/examples/collection-fleet/requirements.yml b/examples/collection-fleet/requirements.yml index f027b46..afd30c8 100644 --- a/examples/collection-fleet/requirements.yml +++ b/examples/collection-fleet/requirements.yml @@ -2,14 +2,15 @@ # The deCDN collections from Ansible Galaxy. Install into ./collections (ansible.cfg # reads it from there): # ansible-galaxy collection install -r requirements.yml -p collections -# 0.2.0 is the first release of each that ships the playbooks. Pin exact versions -# (==X.Y.Z) once the fleet runs, and bump them deliberately, reading each +# Each floor is the collection's newest release when this file was copied from +# decdn/devops (its scripts/release.sh moves them on every release). Pin exact +# versions (==X.Y.Z) once the fleet runs, and bump them deliberately, reading each # CHANGELOG: https://github.com/decdn/devops/tree/main/ansible/galaxy collections: # Node operators: cache nodes only. Delete decdn.publisher below. - name: decdn.node - version: ">=0.2.0" + version: ">=0.4.0" # Publishers: origin nodes, sponsord and its onramp, iroh relays and DNS servers # (and cache nodes). Depends on decdn.node. - name: decdn.publisher - version: ">=0.2.0" + version: ">=0.3.1" diff --git a/examples/collection-fleet/scripts/check-vars-shadowing.sh b/examples/collection-fleet/scripts/check-vars-shadowing.sh new file mode 100755 index 0000000..30fc7f5 --- /dev/null +++ b/examples/collection-fleet/scripts/check-vars-shadowing.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Fail when two vars sources in inventory/group_vars/ or inventory/host_vars/ share a +# name. Ansible loads the first of (a file or a directory), .yml, +# .yaml and .json that exists, and ignores the rest without a word: a +# group_vars/decdn_nodes/ directory (made, say, to hold a git-ignored secret.yml) +# silently drops every setting in group_vars/decdn_nodes.yml beside it. +# +# scripts/check-vars-shadowing.sh [inventory dir] (default: inventory) +# +# The pre-commit hook runs it on your working tree, where git-ignored secret.yml +# files exist; CI runs it on the tracked files. +set -euo pipefail + +inventory="${1:-inventory}" +[[ -d $inventory ]] || { echo "check-vars-shadowing: no $inventory/ directory" >&2; exit 2; } +shopt -s nullglob dotglob +rc=0 +for dir in "$inventory/group_vars" "$inventory/host_vars"; do + declare -A seen=() + for path in "$dir"/*; do + entry="${path##*/}" name="${path##*/}" + if [[ -f $path ]]; then + case $entry in *.yml | *.yaml | *.json) name="${entry%.*}" ;; esac + fi + seen[$name]+="${seen[$name]:+ }$entry" + done + for name in "${!seen[@]}"; do + read -ra entries <<<"${seen[$name]}" + ((${#entries[@]} > 1)) || continue + echo "check-vars-shadowing: $dir/ has ${entries[*]}: Ansible reads only one of them, and ignores the rest without a word. Merge them into $name/ (main.yml beside secret.yml, say)." >&2 + rc=1 + done + unset seen +done +exit "$rc" diff --git a/scripts/release.sh b/scripts/release.sh index d819e80..e4614be 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -7,10 +7,14 @@ # [auto|patch|minor|major|X.Y.Z] # [--execute] [--no-verify] [--allow-unconventional] # -# node-collection node-collection-vX.Y.Z ansible/galaxy/node/{galaxy.yml,CHANGELOG.md} -# publisher-collection publisher-collection-vX.Y.Z ansible/galaxy/publisher/{galaxy.yml,CHANGELOG.md} +# node-collection node-collection-vX.Y.Z ansible/galaxy/node/{galaxy.yml,CHANGELOG.md} + the example floor +# publisher-collection publisher-collection-vX.Y.Z ansible/galaxy/publisher/{galaxy.yml,CHANGELOG.md} + the example floor # chart decdn-node-X.Y.Z charts/decdn-node/{Chart.yaml,CHANGELOG.md} # +# A collection release also raises that collection's floor in +# examples/collection-fleet/requirements.yml to the new version, so a fleet copied +# from the example starts from it (make test-scripts checks they agree). +# # Without --execute it is a dry run: it prints the new tag and the diff it would # commit, and changes no file. The level defaults to `auto`, the bump git-cliff # derives (cliff.toml) from the conventional commits since the artifact's last tag @@ -35,7 +39,7 @@ # --execute runs scripts/check-release-version.sh on the result (and, for the chart, # scripts/chart-artifacthub-changes.py), then the artifact's make target (make # galaxy-check- or make lint-helm) unless --no-verify: a tag that fails its workflow -# burns the version. If any of that or the commit fails, both files are restored. Run +# burns the version. If any of that or the commit fails, the files are restored. Run # it on an up-to-date, clean main, with git configured to sign (user.signingkey), and # with a bypass of the main ruleset (its PR rule and its required status checks): the # release commit goes straight to main. @@ -66,6 +70,7 @@ level="${level:-auto}" repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo" +example="" case "$artifact" in node-collection|publisher-collection) collection="${artifact%-collection}" @@ -73,6 +78,7 @@ case "$artifact" in prefix="$artifact-v" manifest="$overlay/galaxy.yml" changelog="$overlay/CHANGELOG.md" + example="examples/collection-fleet/requirements.yml" workflow="release-collection.yml" check=(make -C ansible "galaxy-check-$collection") # What `galaxy/build.sh ` ships: the roles in its roles.txt, the @@ -105,16 +111,22 @@ case "$artifact" in *) usage ;; esac [[ -f "$manifest" && -f "$changelog" ]] || die "$manifest or $changelog is missing" +# What the release commit changes. +files=("$manifest" "$changelog") +if [[ -n "$example" ]]; then + [[ -f "$example" ]] || die "$example is missing" + files+=("$example") +fi tmp="$(mktemp -d)" written=0 committed=0 pushed=0 reported=0 cleanup() { - # Once --execute has written the two files, any failure before the release commit + # Once --execute has written the files, any failure before the release commit # exists puts them back. Nothing is restored before that: a refusal never touches # the operator's own edits. if ((written == 1 && committed == 0)); then - git checkout -q -- "$manifest" "$changelog" \ - || echo "release: could not restore $manifest and $changelog: run git checkout -- $manifest $changelog" >&2 + git checkout -q -- "${files[@]}" \ + || echo "release: could not restore ${files[*]}: run git checkout -- ${files[*]}" >&2 fi # Stopped (Ctrl-C, say) between the release commit and the end of the push. if ((committed == 1 && pushed == 0 && reported == 0)); then @@ -273,6 +285,20 @@ awk -v v="$version" ' rc=0 cmp -s "$manifest" "$tmp/manifest" || rc=$? ((rc == 1)) || die "could not set the version in $manifest" +if [[ -n "$example" ]]; then + # The `version:` under the collection's `- name:` entry, and only that one. + awk -v name="decdn.$collection" -v v="$version" ' + /^[[:space:]]*- name:/ { mine = ($3 == name) } + mine && /^[[:space:]]+version:/ { sub(/version:.*/, "version: \">=" v "\""); mine = 0; n++ } + {print} + END {print n + 0 > "/dev/stderr"; exit n == 1 ? 0 : 1}' "$example" > "$tmp/example" 2>"$tmp/example.n" \ + || die "$example has $(cat "$tmp/example.n") version: lines under decdn.$collection, not one" + rc=0 + cmp -s "$example" "$tmp/example" || rc=$? + ((rc != 0)) || die "$example already says decdn.$collection >=$version: was it edited by hand?" + ((rc == 1)) || die "could not compare $example with its new floor" + +fi show_diff() { # local rc=0 @@ -283,6 +309,7 @@ echo "release $artifact: ${prev:+$prefix$prev -> }$tag" echo show_diff "$manifest" "$tmp/manifest" show_diff "$changelog" "$tmp/changelog" +[[ -z "$example" ]] || show_diff "$example" "$tmp/example" echo # Until PUBLISH_ENABLED is true, a tag push only builds (RELEASING.md), but the version # is used up all the same: say so before it is. @@ -298,7 +325,7 @@ if ((execute == 0)); then echo " scripts/check-release-version.sh $tag" [[ "$artifact" == "chart" ]] && echo " scripts/chart-artifacthub-changes.py $changelog $version" ((verify == 1)) && echo " ${check[*]}" - echo " git commit -S -m 'chore(release): $tag' -- $manifest $changelog" + echo " git commit -S -m 'chore(release): $tag' -- ${files[*]}" echo " git tag -s $tag -m $tag" echo " git push --atomic origin main $tag" exit 0 @@ -308,6 +335,7 @@ fi written=1 cp "$tmp/manifest" "$manifest" cp "$tmp/changelog" "$changelog" +[[ -z "$example" ]] || cp "$tmp/example" "$example" scripts/check-release-version.sh "$tag" || die "the release gate refused $tag: nothing was committed" if [[ "$artifact" == "chart" ]]; then scripts/chart-artifacthub-changes.py "$changelog" "$version" >/dev/null \ @@ -317,7 +345,7 @@ if ((verify == 1)); then "${check[@]}" || die "${check[*]} failed: nothing was committed" fi -git commit --quiet -S -m "chore(release): $tag" -- "$manifest" "$changelog" \ +git commit --quiet -S -m "chore(release): $tag" -- "${files[@]}" \ || die "git commit failed (a pre-commit hook, or signing): nothing was committed" committed=1 git tag -s "$tag" -m "$tag" || { reported=1; die "tagging failed after the release commit. To undo: git reset --hard origin/main"; } diff --git a/tests/scripts-test.sh b/tests/scripts-test.sh index f00e6de..7e821b5 100755 --- a/tests/scripts-test.sh +++ b/tests/scripts-test.sh @@ -432,6 +432,111 @@ fleet_refuses "a filled-in hosts.yml git would track" "hosts.yml is not git-igno fleet_fixture; echo "# header only" | cat - "$fleet_inv/hosts-node.yml.example" >"$work/h" && mv "$work/h" "$fleet_inv/hosts-node.yml.example" expect 0 "fleet inventory check allows a header-only edit" fleet_check_fixture +# --- examples/collection-fleet's pins ------------------------------------------------ +# fleet_pins_check : requirements.yml's floor for each collection is that +# collection's galaxy.yml version (scripts/release.sh moves both in the release +# commit), and the example's CI and hooks pin what this repo's own do: every `uses:` +# in its ci.yml is pinned to a commit SHA this repo's ci.yml uses, and every pre-commit +# `rev:` and the yamllint pin is this repo's for the same hook repo. This repo's +# Dependabot bumps our action SHAs and hook revs (the yamllint pin is bumped by hand); +# its PR fails here until the example is moved too. +fleet_pins_check() { + local root="$1" ex="$1/examples/collection-fleet" c f floor ver line repo_url rev mine theirs n=0 + for c in node publisher; do + floor="$(awk -v name="decdn.$c" '/^[[:space:]]*- name:/ { mine = ($3 == name) } + mine && /^[[:space:]]+version:/ { gsub(/.*version:[[:space:]]*"?>=|".*/, ""); print; exit }' "$ex/requirements.yml")" + ver="$(sed -nE 's/^version:[[:space:]]*"?([^"#[:space:]]+)"?.*/\1/p' "$root/ansible/galaxy/$c/galaxy.yml")" + [[ -n $floor && -n $ver ]] || fail "could not read decdn.$c's floor ('$floor') or galaxy.yml version ('$ver')" + [[ $floor == "$ver" ]] \ + || fail "examples/collection-fleet/requirements.yml: decdn.$c >=$floor, but its galaxy.yml is $ver (scripts/release.sh moves both)" + done + while IFS= read -r line; do + [[ $line =~ @[0-9a-f]{40}$ ]] \ + || fail "examples/collection-fleet/.github/workflows/ci.yml uses $line, not pinned to a commit SHA" + grep -qF -- "uses: $line" "$root/.github/workflows/ci.yml" \ + || fail "examples/collection-fleet/.github/workflows/ci.yml uses $line, which .github/workflows/ci.yml does not: pin the same SHA" + n=$((n + 1)) + done < <(sed -nE 's/^[[:space:]]*(- )?uses:[[:space:]]*([^[:space:]#]+).*/\2/p' "$ex/.github/workflows/ci.yml") + ((n >= 3)) || fail "found $n pinned actions in examples/collection-fleet/.github/workflows/ci.yml" + # " " for each hook repo with a rev, and "yamllint ". + hook_pins() { + awk '/^[[:space:]]*- repo:/ {r=$3} /^[[:space:]]+rev:/ {print r, $2}' "$1" + sed -nE 's/.*additional_dependencies:.*"(yamllint==[^"]+)".*/yamllint \1/p' "$1" + } + mine="$(hook_pins "$root/.pre-commit-config.yaml")" + theirs="$(hook_pins "$ex/.pre-commit-config.yaml")" + # The yamllint pin is read in one spelling (a one-line, double-quoted list): one + # written another way would drop out of the comparison unseen. + for f in "$root/.pre-commit-config.yaml" "$ex/.pre-commit-config.yaml"; do + [[ "$(grep -c 'yamllint==' "$f")" == "$(hook_pins "$f" | grep -c '^yamllint ')" ]] \ + || fail "$f: a yamllint== pin not in the form additional_dependencies: [\"yamllint==X\"]" + done + grep -q '^yamllint ' <<<"$theirs" || fail "examples/collection-fleet/.pre-commit-config.yaml pins no yamllint" + n=0 + while read -r repo_url rev; do + grep -q -- "^$repo_url " <<<"$mine" \ + || fail "examples/collection-fleet/.pre-commit-config.yaml uses $repo_url, a hook repo this repo does not" + grep -qxF -- "$repo_url $rev" <<<"$mine" \ + || fail "examples/collection-fleet/.pre-commit-config.yaml pins $repo_url $rev, not this repo's $(grep -F -- "$repo_url " <<<"$mine" | cut -d' ' -f2 | xargs)" + n=$((n + 1)) + done <<<"$theirs" + ((n >= 4)) || fail "found $n hook pins in examples/collection-fleet/.pre-commit-config.yaml" + pass "examples/collection-fleet pins the collections' releases, and this repo's actions and hook revs" +} +fleet_pins_check "$repo" +pins_fixture() { + local f + rm -rf "$work/pins" && mkdir -p "$work/pins" + for f in ansible/galaxy/node/galaxy.yml ansible/galaxy/publisher/galaxy.yml .github/workflows/ci.yml \ + .pre-commit-config.yaml examples/collection-fleet/requirements.yml \ + examples/collection-fleet/.github/workflows/ci.yml examples/collection-fleet/.pre-commit-config.yaml; do + mkdir -p "$work/pins/$(dirname "$f")" && cp "$repo/$f" "$work/pins/$f" + done +} +pins_check_fixture() { (fleet_pins_check "$work/pins"); } +pins_refuses() { + expect 1 "fleet pins check refuses $1" pins_check_fixture + grep -qF -- "$2" "$work/out" || { cat "$work/out" >&2; fail "fleet pins check: $1 failed without \"$2\""; } +} +pins_fixture +expect 0 "fleet pins check passes on its fixture" pins_check_fixture +sed -i -E 's/^version:[[:space:]]*[0-9.]+/version: 99.0.0/' "$work/pins/ansible/galaxy/publisher/galaxy.yml" +pins_refuses "a floor behind its collection's release" "decdn.publisher >=" +pins_fixture; sed -i -E '0,/actions\/checkout@[0-9a-f]{40}/s//actions\/checkout@0000000000000000000000000000000000000000/' \ + "$work/pins/examples/collection-fleet/.github/workflows/ci.yml" +pins_refuses "an action pinned to another SHA" "uses actions/checkout@0000000000000000000000000000000000000000" +pins_fixture; sed -i -E '0,/actions\/checkout@[0-9a-f]{40}/s//actions\/checkout@v7/' \ + "$work/pins/examples/collection-fleet/.github/workflows/ci.yml" +pins_refuses "an action pinned to a tag" "uses actions/checkout@v7, not pinned to a commit SHA" +pins_fixture; sed -i -E 's/yamllint==[0-9.]+/yamllint==0.0.1/' "$work/pins/examples/collection-fleet/.pre-commit-config.yaml" +pins_refuses "another yamllint pin" "pins yamllint yamllint==0.0.1" +pins_fixture; sed -i -E "s/\[\"(yamllint==[0-9.]+)\"\]/['\\1']/" "$work/pins/examples/collection-fleet/.pre-commit-config.yaml" +pins_refuses "a yamllint pin in another spelling" "not in the form additional_dependencies" +pins_fixture; sed -i 's#https://github.com/koalaman/shellcheck-precommit#https://example.invalid/hooks#' \ + "$work/pins/examples/collection-fleet/.pre-commit-config.yaml" +pins_refuses "a hook repo this repo does not use" "uses https://example.invalid/hooks, a hook repo this repo does not" +pins_fixture; sed -i -E '0,/rev: v[0-9.]+/s//rev: v0.0.0/' "$work/pins/.pre-commit-config.yaml" +pins_refuses "a hook rev this repo has moved" "pins https://github.com/pre-commit/pre-commit-hooks" + +# --- examples/collection-fleet/scripts/check-vars-shadowing.sh -------------------------- +# Ansible reads one of , .yml, .yaml and .json in group_vars/ +# and host_vars/, silently. The example's pre-commit hook and CI run this check; here it +# passes on the example's own inventory and refuses each kind of pair. +shadow="$repo/examples/collection-fleet/scripts/check-vars-shadowing.sh" +expect 0 "check-vars-shadowing passes on the example's inventory" "$shadow" "$repo/examples/collection-fleet/inventory" +# shadow_refuses +shadow_refuses() { + rm -rf "$work/shadow" && cp -R "$repo/examples/collection-fleet/inventory" "$work/shadow" + case $3 in */) mkdir -p "$work/shadow/$3" ;; *) touch "$work/shadow/$3" ;; esac + expect 1 "check-vars-shadowing refuses $1" "$shadow" "$work/shadow" + grep -qF -- "$2" "$work/out" || { cat "$work/out" >&2; fail "check-vars-shadowing: $1 failed without \"$2\""; } +} +shadow_refuses "a group directory beside its file" "has decdn_nodes decdn_nodes.yml" group_vars/decdn_nodes/ +shadow_refuses "a .yaml beside a .yml" "has all.yaml all.yml" group_vars/all.yaml +shadow_refuses "a host file beside its directory" "has origin-1 origin-1.yml" host_vars/origin-1.yml +shadow_refuses "an extensionless file beside a .yml" "has all all.yml" group_vars/all +expect 2 "check-vars-shadowing refuses a missing inventory" "$shadow" "$work/no-such-inventory" + # --- release gate ---------------------------------------------------------------------- gate="$repo/scripts/check-release-version.sh" expect 2 "release gate rejects a stray argument" "$gate" decdn-node-0.1.0 notes.md @@ -917,6 +1022,15 @@ if command -v git-cliff >/dev/null && command -v ssh-keygen >/dev/null; then printf 'namespace: decdn\nname: %s\nversion: 0.0.0 # a comment\n' "$c" > "$rs/ansible/galaxy/$c/galaxy.yml" done printf 'dependencies:\n decdn.node: ">=0.0.2"\n' >> "$rs/ansible/galaxy/publisher/galaxy.yml" + # The example's floors, which each collection release moves. + rsreq="$rs/examples/collection-fleet/requirements.yml" + mkdir -p "$(dirname "$rsreq")" + printf 'collections:\n - name: decdn.node\n version: ">=0.0.0"\n - name: decdn.publisher\n version: ">=0.0.0"\n' > "$rsreq" + # : the example's floor for it. + rsfloor() { + awk -v name="decdn.$1" '/- name:/ { mine = ($3 == name) } mine && /version:/ { print $2; exit }' "$rsreq" \ + | grep -qxF "\">=$2\"" || fail "release.sh: the example's decdn.$1 floor is not >=$2: $(cat "$rsreq")" + } for log in "$chart/CHANGELOG.md" "$rs"/ansible/galaxy/{node,publisher}/CHANGELOG.md; do printf '# Changelog\n\nIntro.\n\n## [Unreleased]\n\nInitial.\n\n### Added\n\n- The first entry.\n' > "$log" done @@ -1129,10 +1243,23 @@ if command -v git-cliff >/dev/null && command -v ssh-keygen >/dev/null; then # Each collection is released on its own tags and paths: its roles.txt, its overlay. # decdn.publisher depends on decdn.node, so its first release waits for node's. rsrefused "decdn.publisher's first release before decdn.node's" "origin's highest node-collection tag is none" publisher-collection minor + : > "$RS_MAKE_LOG"; touch "$RS_MAKE_FAIL" + rsrefused "to commit a collection when the make check fails" "make -C ansible galaxy-check-node failed: nothing was committed" node-collection patch --execute + rm "$RS_MAKE_FAIL" + rsclean "with a failing collection check" + [[ -z "$(rsgit tag -l 'node-collection-*')" ]] || fail "release.sh made a node-collection tag after its check failed" + rsfloor node 0.0.0 + expect 0 "release.sh dry-runs the node collection's first release" rsrel node-collection patch + rssaid "a collection dry run" '+ version: ">=0.0.1"' + rssaid "a collection dry run" "examples/collection-fleet/requirements.yml" : > "$RS_MAKE_LOG" expect 0 "release.sh cuts the node collection's first release" rsrel node-collection patch --execute rsmade galaxy-check-node rspushed node-collection-v0.0.1 + rsfloor node 0.0.1; rsfloor publisher 0.0.0 + [[ "$(rsgit show --name-only --format= HEAD | sort | xargs)" == \ + "ansible/galaxy/node/CHANGELOG.md ansible/galaxy/node/galaxy.yml examples/collection-fleet/requirements.yml" ]] \ + || fail "release.sh's node release commit changed: $(rsgit show --name-only --format= HEAD | xargs)" # The fixture's publisher declares decdn.node >=0.0.2: a lower node release is not enough. rsrefused "decdn.publisher's first release below its decdn.node constraint" \ "needs decdn.node >=0.0.2, and origin's highest node-collection tag is 0.0.1" publisher-collection patch @@ -1141,10 +1268,16 @@ if command -v git-cliff >/dev/null && command -v ssh-keygen >/dev/null; then grep -qx 'version: 0.0.1 # a comment' "$rs/ansible/galaxy/node/galaxy.yml" \ || fail "release.sh did not set node/galaxy.yml to 0.0.1 keeping its comment: $(grep '^version' "$rs/ansible/galaxy/node/galaxy.yml")" grep -qx '## \[Unreleased\]' "$rs/ansible/galaxy/publisher/CHANGELOG.md" || fail "release.sh touched the publisher collection's changelog" + # A node floor publisher's release would not set, to see it left alone. + sed -i '0,/">=0.0.1"/s//">=0.0.9"/' "$rsreq"; rsgit commit -qam "test: a distinct node floor"; rspush : > "$RS_MAKE_LOG" expect 0 "release.sh cuts the publisher collection's first release after node's" rsrel publisher-collection patch --execute rsmade galaxy-check-publisher rspushed publisher-collection-v0.0.1 + rsfloor node 0.0.9; rsfloor publisher 0.0.1 + [[ "$(rsgit show --name-only --format= HEAD | sort | xargs)" == \ + "ansible/galaxy/publisher/CHANGELOG.md ansible/galaxy/publisher/galaxy.yml examples/collection-fleet/requirements.yml" ]] \ + || fail "release.sh's publisher release commit changed: $(rsgit show --name-only --format= HEAD | xargs)" # A later release is held to the constraint too: raised past the highest node tag, # the collection would not install from Galaxy. sed -i 's/">=0.0.1"/">=0.1.0"/' "$rs/ansible/galaxy/publisher/galaxy.yml" @@ -1191,7 +1324,18 @@ if command -v git-cliff >/dev/null && command -v ssh-keygen >/dev/null; then rssaid "after a group_vars fix, for publisher" "+- **playbooks**: A firewall hole fix" rssaid "with a feature at 0.0.x" "publisher-collection-v0.0.1 -> publisher-collection-v0.1.0" ! grep -qF "Ship the unshipped role" "$work/out" || fail "release.sh's collection section lists a build commit" - pass "release.sh releases each collection on its own tags, roles.txt, playbooks.txt and overlay, publisher after node" + # A floor already at the release, two entries, no entry, no example: each refused. + cp "$rsreq" "$work/rs-saved" + sed -i '0,/">=0.0.9"/s//">=0.0.3"/' "$rsreq"; rsgit commit -qam "test: a floor at the next release"; rspush + rsrefused "a floor already at the release" "already says decdn.node >=0.0.3" node-collection 0.0.3 + printf ' - name: decdn.node\n version: ">=0.0.1"\n' >> "$rsreq"; rsgit commit -qam "test: two node entries"; rspush + rsrefused "two entries for the collection" "has 2 version: lines under decdn.node" node-collection + cp "$work/rs-saved" "$rsreq" + sed -i '/decdn.node/,+1d' "$rsreq"; rsgit commit -qam "test: drop the node floor"; rspush + rsrefused "an example with no floor for the collection" "has 0 version: lines under decdn.node" node-collection + git -C "$rs" rm -q "$rsreq"; rsgit commit -qm "test: drop the example"; rspush + rsrefused "a repo without the example's requirements.yml" "requirements.yml is missing" node-collection + pass "release.sh releases each collection on its own tags, roles.txt, playbooks.txt and overlay, publisher after node, and moves its floor in the example" unset GIT_CONFIG_GLOBAL GIT_CONFIG_NOSYSTEM RS_MAKE_LOG RS_MAKE_FAIL elif [[ -n ${CI:-} ]]; then fail "git-cliff or ssh-keygen is not on PATH in CI; the release.sh tests would be skipped"