diff --git a/.gitignore b/.gitignore index 5ee8a47..7540787 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,6 @@ server/nginx/ppn.conf # The dashboard UI compiles into the CLI package (packages/dashboard-ui -> vite build). # Built output ships in the npm tarball via prepack, never in git. packages/cli/web/ + +# The VM branch keeps its one secret out of git +deploy/*/local.env diff --git a/README.md b/README.md index 1915971..8b5dec4 100644 --- a/README.md +++ b/README.md @@ -89,10 +89,11 @@ ppn start polkadot --fork --fresh-bite \ A `--upgrade` accepts any blob: a release asset, a PR's build artifact, a local build, and `relay=` for the relay itself. The bite warp-syncs four chains, around 20 minutes, and a -parachain's upgrade goes live an hour after the spawn, because Polkadot's upgrade delay is 600 -relay blocks. The dashboard shows the spec version flip; the log of the process applying the -upgrades is under its logs tab. If that process gave up, or you want to redo one chain, submit -the apply step yourself with no blob, and the one the bite authorized is used: +parachain's upgrade goes live a few minutes after the spawn: the bite cuts the relay's upgrade +delay to 30 blocks (live Polkadot: 600, an hour). The dashboard shows the spec version flip; +the log of the process applying the upgrades is under its logs tab. If that process gave up, +or you want to redo one chain, submit the apply step yourself with no blob, and the one the +bite authorized is used: ```bash ppn upgrade people --enact-timeout 70 # apply what the bite authorized for People diff --git a/config/versions.env b/config/versions.env index ec2ad07..0400372 100644 --- a/config/versions.env +++ b/config/versions.env @@ -50,7 +50,7 @@ ZOMBIENET_VERSION="v0.4.15" # Pinned per network as bite.doppelganger, since the build has to be able to # execute the runtimes of the network being bitten. Restoring a published bundle # (`make start FORK=1`) needs none of it: the forked network runs on the regular -# node binaries. +# node binaries, except collators, which need the fork-id fix (docs/FORK.md). # ============================================================================= # Kubo IPFS @@ -83,9 +83,16 @@ KUBO_VERSION="v0.39.0" # public `docker.io/paritytech/device-uniqueness-backend` images tagged 0.1-0.3, which were # built from older source. v0.5.0 refreshes the vendored People metadata to 3000000 (the # invite pool minted nothing against it before) and logs metadata drift at boot, which is -# the diagnostic a fork running a runtime dub has never seen needs. Both kept the asset naming (dub--.tar.gz), the roles -# PPN spawns (all-in-one plus the three single-instance workers) and every config key in -# dub.ts — checked by booting each role against this env, not read off the release notes. +# the diagnostic a fork running a runtime dub has never seen needs. +# +# v0.6.0 ships every asset twice, one build per People runtime, and names them +# dub---.tar.gz. The runtime is fixed when the binary is built: it +# selects the vendored metadata and whether the invite-tickets roles exist at all. A network +# says which it needs through `dubBuild` in its descriptor, defaulting to testnet; only the +# Polkadot fork wants `polkadot`, whose People runtime carries neither Game nor ProofOfInk. +# Naming a role the binary does not have is a boot failure, so the same field also decides +# which services and databases are generated. +# # `INVITE_INVITER_*` in upstream's .env.example is compose's host-side naming; the key the # binary reads is still INVITER_ADDRESS. v0.7.0 defaults turn-api to Cloudflare and refuses a # TURN_SECRET without TURN_PROVIDER, so dub.ts names coturn, which needs a non-empty ICE_SERVERS. @@ -94,7 +101,6 @@ KUBO_VERSION="v0.39.0" # runtime without it (paritytech/individuality-community#196). DUB_REPO="paritytech/device-uniqueness-backend-community" DUB_TAG="${DUB_TAG:-v0.8.0}" -DUB_NETWORK="${DUB_NETWORK:-testnet}" # Portable PostgreSQL for the backend services. Fetched straight from upstream # (public repo, no auth) because it is not a PPN build product. This is the diff --git a/deploy/polkadot-test/README.md b/deploy/polkadot-test/README.md new file mode 100644 index 0000000..cc098aa --- /dev/null +++ b/deploy/polkadot-test/README.md @@ -0,0 +1,40 @@ +# polkadot-test.substrate.dev + +The one machine that runs the Polkadot fork, from its own branch: main plus this directory and the +fork's engine changes, rebased onto main. The machine picks them up with one command. When +preview-net-v1 grows a fork target this directory moves there and the branch dies. + +```bash +deploy/polkadot-test/apply.sh # pull, build, fetch, ports.env, nginx, unit, restart +deploy/polkadot-test/apply.sh --no-restart # everything except touching the running network +``` + +The fetch rewrites `polkadot-parachain` in place, so it puts the stock build back, and it fails +(ETXTBSY) while the collators are running the patched one. The collators need the build with the +fork-id fix (`docs/FORK.md`), so stop the network around an apply: + +```bash +sudo systemctl stop ppn-polkadot +deploy/polkadot-test/apply.sh --no-restart +# reinstall the build with the fork-id fix +sudo systemctl start ppn-polkadot +``` + +What is where: + +| File | Role | +| --- | --- | +| `vm.env` | Domain, public IP, run user, start flags. Committed. | +| `local.env` | The dashboard actions token. Gitignored; created on first apply. | +| `apply.sh` | The command. Idempotent; installs missing packages on a fresh box. | +| `start.sh` | What the unit runs: `ppn start polkadot --fork $START_FLAGS`. | +| `ppn-polkadot.service` | systemd unit template; `__ROOT__`/`__USER__` filled by apply. | +| `nginx/` | v1's template and websocket snippet, ours to patch here. | +| `/etc/ppn/secrets.env` | Created by apply on a fresh box with `TURN_SECRET`, `JWT_ED25519_SECRET` and `PPN_PROFILE=local`, and never rotated. Optional keys from `config/secrets.env.example`, such as push notifications, are added by hand. `ports.env` names it as `PPN_SECRETS_FILE`. | + +Not done by apply, ever: the bite. That is a deliberate `make bite NETWORK=polkadot` (plus +`UPGRADES=...` for a runtime under test; see `docs/POLKADOT-FORK.md`), followed by `apply.sh` or +`systemctl start ppn-polkadot`. + +And the certificate, once: `sudo certbot certonly --standalone -d ` while nginx is +stopped; apply skips nginx until it exists and prints the command. diff --git a/deploy/polkadot-test/apply.sh b/deploy/polkadot-test/apply.sh new file mode 100755 index 0000000..12efc47 --- /dev/null +++ b/deploy/polkadot-test/apply.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# The one command on the VM. Idempotent: run it after every push to this branch. +# +# deploy/polkadot-test/apply.sh pull, build, fetch, nginx, unit, restart the network +# deploy/polkadot-test/apply.sh --no-restart same, but leave the running network alone +# deploy/polkadot-test/apply.sh --no-pull work from the checkout as it is +# +# A bite is never done here: `make bite NETWORK=polkadot` is a deliberate act. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/../.." && pwd)" +cd "$ROOT" +source "$HERE/vm.env" + +RESTART=1 +PULL=1 +for arg in "$@"; do + case "$arg" in + --no-restart) RESTART=0 ;; + --no-pull) PULL=0 ;; + *) echo "apply.sh: unknown argument '$arg'" >&2; exit 2 ;; + esac +done + +say() { echo "apply: $*"; } +changed() { ! cmp -s "$1" "$2"; } + +# ---- 1. dependencies ------------------------------------------------------------------------ +# What a fresh Ubuntu is missing for this to run at all. Node comes from NodeSource when it is +# absent or too old; pnpm through corepack; everything else from apt. +NODE_MAJOR_MIN=22 +APT_WANTED=(nginx libnginx-mod-stream certbot gettext-base jq lsof unzip curl tmux git build-essential) +APT_MISSING=() +for pkg in "${APT_WANTED[@]}"; do dpkg -s "$pkg" >/dev/null 2>&1 || APT_MISSING+=("$pkg"); done +NEED_NODE=0 +if ! command -v node >/dev/null 2>&1; then + NEED_NODE=1 +elif [[ "$(node --version | sed 's/^v\([0-9]*\).*/\1/')" -lt "$NODE_MAJOR_MIN" ]]; then + NEED_NODE=1 +fi +if [[ ${#APT_MISSING[@]} -gt 0 || "$NEED_NODE" == 1 ]]; then + say "installing: ${APT_MISSING[*]}${NEED_NODE:+ node}" + sudo apt-get update -qq + if [[ "$NEED_NODE" == 1 ]]; then + curl -fsSL "https://deb.nodesource.com/setup_${NODE_MAJOR_MIN}.x" | sudo -E bash - + sudo apt-get install -y nodejs + fi + [[ ${#APT_MISSING[@]} -gt 0 ]] && sudo apt-get install -y "${APT_MISSING[@]}" +fi +if ! command -v pnpm >/dev/null 2>&1; then + sudo corepack enable >/dev/null 2>&1 || corepack enable >/dev/null 2>&1 || sudo npm install -g pnpm +fi +if ! command -v gh >/dev/null 2>&1; then + say "WARNING: gh is not installed; \`make fetch\` needs GITHUB_TOKEN or gh auth for private release assets" +fi +sudo install -d -o "$RUN_USER" -g "$RUN_USER" "$LOG_DIR" + +# ---- 2. code ---------------------------------------------------------------------------- +# ports.env is patched below on every run, so the local copy is disposable: drop it before the +# pull, or a pull that touches the file fails. +if [[ "$PULL" == 1 ]]; then + git checkout -q -- config/ports.env 2>/dev/null || true + git pull -q --ff-only +fi +say "at $(git rev-parse --short HEAD) on $(git branch --show-current)" +pnpm install --frozen-lockfile >/dev/null +BUILD_LOG="$(mktemp)" +if ! pnpm -r build > "$BUILD_LOG" 2>&1; then cat "$BUILD_LOG"; rm -f "$BUILD_LOG"; exit 1; fi +rm -f "$BUILD_LOG" + +# ---- 3. artifacts ----------------------------------------------------------------------- +# Picks up a changed node pin or dub tag; a no-op when provenance says everything is current. +FETCH_LOG="$(mktemp)" +if ! make fetch NETWORK="$NETWORK" > "$FETCH_LOG" 2>&1; then cat "$FETCH_LOG"; rm -f "$FETCH_LOG"; exit 1; fi +grep -vE "^\s*✓|^$" "$FETCH_LOG" | grep -iE "fetch|download|missing|✗" || true +rm -f "$FETCH_LOG" + +# ---- 4. the machine's values into ports.env -------------------------------------------- +# zombienet hands custom processes no environment, so the dashboard and bootnode patching read +# these from the file. The actions token is the one secret; it lives in local.env. +LOCAL_ENV="$HERE/local.env" +if [[ ! -f "$LOCAL_ENV" ]]; then + printf 'DASHBOARD_ACTIONS_TOKEN=%s\n' "$(openssl rand -hex 24)" > "$LOCAL_ENV" + say "created $LOCAL_ENV with a new dashboard actions token" +fi +source "$LOCAL_ENV" +# TURN_SECRET and JWT_ED25519_SECRET replace public dev defaults; DUB refuses to start on a +# secrets file without the JWT seed. Created once, never rotated here. +if ! sudo test -f "$SECRETS_FILE"; then + sudo install -d -m 0750 -o root -g "$RUN_USER" "$(dirname "$SECRETS_FILE")" + printf 'TURN_SECRET=%s\nJWT_ED25519_SECRET=0x%s\n' \ + "$(openssl rand -hex 24 | tr -d '\n' | base64)" "$(openssl rand -hex 32)" \ + | sudo tee "$SECRETS_FILE" >/dev/null + sudo chown "root:$RUN_USER" "$SECRETS_FILE" && sudo chmod 0640 "$SECRETS_FILE" + say "created $SECRETS_FILE" +fi +# `ppn start` rejects a secrets file without a profile; the fork runs with the dev accounts. +if ! sudo grep -q '^PPN_PROFILE=' "$SECRETS_FILE"; then + echo 'PPN_PROFILE=local' | sudo tee -a "$SECRETS_FILE" >/dev/null + say "$SECRETS_FILE: added PPN_PROFILE=local" +fi +sed -i \ + -e "s|^BOOTNODE_HOSTNAME=.*|BOOTNODE_HOSTNAME=$DOMAIN|" \ + -e "s|^PPN_PUBLIC_URL=.*|PPN_PUBLIC_URL=https://$DOMAIN|" \ + -e "s|^DASHBOARD_HOST=.*|DASHBOARD_HOST=127.0.0.1|" \ + -e "s|^P2P_LISTEN_IP=.*|P2P_LISTEN_IP=$PUBLIC_IP|" \ + -e "s|^DASHBOARD_ACTIONS_TOKEN=.*|DASHBOARD_ACTIONS_TOKEN=${DASHBOARD_ACTIONS_TOKEN:-}|" \ + -e "s|^PPN_SECRETS_FILE=.*|PPN_SECRETS_FILE=$SECRETS_FILE|" \ + config/ports.env +say "ports.env: domain $DOMAIN, listen ip $PUBLIC_IP, secrets $SECRETS_FILE" + +# ---- 5. nginx --------------------------------------------------------------------------- +TLS_DIR="/etc/letsencrypt/live/$DOMAIN" +if [[ ! -f "$TLS_DIR/fullchain.pem" ]]; then + say "WARNING: no certificate at $TLS_DIR — nginx not touched. Issue one, then re-run:" + say " sudo systemctl stop nginx && sudo certbot certonly --standalone -d $DOMAIN --deploy-hook 'systemctl reload nginx'" +else + export PPN_NETWORK="$NETWORK" PPN_DOMAIN="$DOMAIN" PPN_TLS_DIR="$TLS_DIR" LOG_DIR DATA_DIR="$ROOT/data-fork-$NETWORK" + set -a; source config/ports.env; set +a + # Re-exported after ports.env, which must not be allowed to blank them. + export LOG_DIR DATA_DIR + ROUTED="$(mktemp)"; RENDERED="$(mktemp)" + node bin/ppn.mjs nginx-conf "$HERE/nginx/ppn.conf.template" "$ROUTED" >/dev/null + # Named explicitly so nginx's own $host and friends survive. + envsubst '$PPN_DOMAIN $PPN_TLS_DIR $DATA_DIR $LOG_DIR $RELAY_ALICE_PORT $RELAY_BOB_PORT $RELAY_CHARLIE_PORT $RELAY_DAVE_PORT $RELAY_EVE_PORT $RELAY_FERDIE_PORT $PEOPLE_PORT $ASSET_HUB_PORT $BULLETIN_PORT $WEB3_STORAGE_PORT $ETH_RPC_PORT $WEB3_STORAGE_PROVIDER_PORT $IPFS_GATEWAY_PORT $DUB_PORT $DASHBOARD_PORT $RELAY_ALICE_P2P_PORT $ASSET_HUB_P2P_PORT $PEOPLE_P2P_PORT $BULLETIN_P2P_PORT $WEB3_STORAGE_P2P_PORT $RELAY_ALICE_P2P_WSS_PORT $ASSET_HUB_P2P_WSS_PORT $PEOPLE_P2P_WSS_PORT $BULLETIN_P2P_WSS_PORT $WEB3_STORAGE_P2P_WSS_PORT' \ + < "$ROUTED" > "$RENDERED" + if LEFT=$(grep -oE '\$\{[A-Za-z_][A-Za-z0-9_]*\}' "$RENDERED" | sort -u | tr '\n' ' ') && [[ -n "$LEFT" ]]; then + echo "apply: nginx config has unsubstituted variables: $LEFT" >&2; exit 1 + fi + NGINX_CHANGED=0 + if changed "$HERE/nginx/websocket-proxy.conf" /etc/nginx/snippets/websocket-proxy.conf; then + sudo install -m 0644 "$HERE/nginx/websocket-proxy.conf" /etc/nginx/snippets/websocket-proxy.conf; NGINX_CHANGED=1 + fi + # stream{} cannot live in sites-enabled (that is inside http{}), so the template's tail + # after the marker line goes to its own file, included from nginx.conf's top level. + STREAM="$(mktemp)" + sed -n '/^# ---- stream ----$/,$p' "$RENDERED" > "$STREAM" + sed -i '/^# ---- stream ----$/,$d' "$RENDERED" + if changed "$RENDERED" /etc/nginx/sites-available/ppn.conf; then + sudo install -m 0644 "$RENDERED" /etc/nginx/sites-available/ppn.conf; NGINX_CHANGED=1 + fi + sudo install -d /etc/nginx/streams-enabled + if changed "$STREAM" /etc/nginx/streams-enabled/ppn.conf; then + sudo install -m 0644 "$STREAM" /etc/nginx/streams-enabled/ppn.conf; NGINX_CHANGED=1 + fi + if ! grep -q '^include /etc/nginx/streams-enabled/' /etc/nginx/nginx.conf; then + echo 'include /etc/nginx/streams-enabled/*.conf;' | sudo tee -a /etc/nginx/nginx.conf >/dev/null; NGINX_CHANGED=1 + fi + rm -f "$ROUTED" "$RENDERED" "$STREAM" + sudo ln -sf /etc/nginx/sites-available/ppn.conf /etc/nginx/sites-enabled/ppn.conf + sudo rm -f /etc/nginx/sites-enabled/default + if [[ "$NGINX_CHANGED" == 1 ]]; then + sudo nginx -t && sudo systemctl enable --now nginx >/dev/null && sudo systemctl reload nginx + say "nginx: config updated and reloaded" + else + sudo systemctl is-active --quiet nginx || sudo systemctl enable --now nginx + say "nginx: unchanged" + fi +fi + +# ---- 6. systemd unit -------------------------------------------------------------------- +UNIT=/etc/systemd/system/ppn-polkadot.service +RENDERED_UNIT="$(mktemp)" +sed -e "s|__ROOT__|$ROOT|g" -e "s|__USER__|$RUN_USER|g" "$HERE/ppn-polkadot.service" > "$RENDERED_UNIT" +if changed "$RENDERED_UNIT" "$UNIT"; then + sudo install -m 0644 "$RENDERED_UNIT" "$UNIT" + sudo systemctl daemon-reload + say "unit: installed" +fi +rm -f "$RENDERED_UNIT" +sudo systemctl enable ppn-polkadot.service >/dev/null 2>&1 || true + +# ---- 7. the network --------------------------------------------------------------------- +if [[ ! -f "fork-bundle-$NETWORK/manifest.json" ]]; then + say "no bundle at fork-bundle-$NETWORK — bite first: make bite NETWORK=$NETWORK" +elif [[ "$RESTART" == 1 ]]; then + # Anything running outside systemd (a `make start` in tmux) goes too: two networks on one + # port set is worse than a restart. + sudo systemctl stop ppn-polkadot.service 2>/dev/null || true + node bin/ppn.mjs kill >/dev/null 2>&1 || true + sudo systemctl start ppn-polkadot.service + say "network: restarting under systemd (journalctl -u ppn-polkadot -f)" +else + say "network: left alone (--no-restart)" +fi + +echo +say "done. dashboard https://$DOMAIN token in $LOCAL_ENV" diff --git a/deploy/polkadot-test/local.env.example b/deploy/polkadot-test/local.env.example new file mode 100644 index 0000000..99ab0d4 --- /dev/null +++ b/deploy/polkadot-test/local.env.example @@ -0,0 +1,2 @@ +# Copy to local.env (gitignored). apply.sh writes one with a random token if none exists. +DASHBOARD_ACTIONS_TOKEN= diff --git a/deploy/polkadot-test/nginx/ppn.conf.template b/deploy/polkadot-test/nginx/ppn.conf.template new file mode 100644 index 0000000..6496500 --- /dev/null +++ b/deploy/polkadot-test/nginx/ppn.conf.template @@ -0,0 +1,273 @@ +# Product Preview Network - nginx configuration +# WebSocket reverse proxy for all zombienet endpoints +# +# NOTE: This is a template. The actual config is generated during deployment +# by running: envsubst < ppn.conf.template > ppn.conf +# Port values come from config/ports.env + +# Upstream definitions for all WebSocket endpoints +# Chain routes are generated by `ppn nginx-conf` from the network descriptor — the same +# table the dashboard serves. Add a parachain to networks/.json and the next deploy +# routes it; nothing here to edit. Service blocks below stay hand-written: they carry +# bespoke CORS/body-size/header behaviour that is not mere routing. +# {{GENERATED_UPSTREAMS}} + +upstream eth_rpc { + server 127.0.0.1:${ETH_RPC_PORT}; + keepalive 32; +} + +upstream storage_provider { + server 127.0.0.1:${WEB3_STORAGE_PROVIDER_PORT}; + keepalive 32; +} + +# The identity backend. Deliberately a single upstream: several services own slices of +# /api/v1/* — identity-api and username-indexer even split /api/v1/usernames by method — and +# that split is owned upstream, now by the route table compiled into `ibv2 --role all-in-one`. +# nginx must not learn it, or every ownership change upstream becomes an nginx change here. +upstream dub { + server 127.0.0.1:${DUB_PORT}; + keepalive 32; +} + +upstream ipfs_gateway { + server 127.0.0.1:${IPFS_GATEWAY_PORT}; + keepalive 32; +} + +# WebSocket upgrade mapping +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +# HTTP server - redirect to HTTPS +server { + listen 80; + listen [::]:80; + server_name ${PPN_DOMAIN}; + + # Certbot ACME challenge location (for certificate renewals) + location /.well-known/acme-challenge/ { + root /var/www/html; + } + + # Redirect all other traffic to HTTPS + location / { + return 301 https://$host$request_uri; + } +} + +# HTTPS server - main configuration +server { + listen 443 ssl; + listen [::]:443 ssl; + server_name ${PPN_DOMAIN}; + + # Certificate directory, chosen by redeploy.sh: certbot's live path when a real + # certificate exists, otherwise the self-signed stopgap under /etc/ppn/tls. + ssl_certificate ${PPN_TLS_DIR}/fullchain.pem; + ssl_certificate_key ${PPN_TLS_DIR}/privkey.pem; + + # SSL settings (modern configuration) + ssl_protocols TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers off; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 1d; + + # {{GENERATED_LOCATIONS}} + + # Ethereum RPC (eth-rpc connected to Asset Hub) + location /eth-rpc { + proxy_pass http://eth_rpc/; + proxy_http_version 1.1; + proxy_set_header Host 127.0.0.1:${ETH_RPC_PORT}; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # CORS headers for JSON-RPC + add_header Access-Control-Allow-Origin * always; + add_header Access-Control-Allow-Methods "GET, POST, OPTIONS" always; + add_header Access-Control-Allow-Headers "Content-Type" always; + + # Handle preflight requests + if ($request_method = OPTIONS) { + return 204; + } + } + + # Identity backend (auth, username registration, search, /docs API reference) + location /dub/ { + proxy_pass http://dub/; + proxy_http_version 1.1; + proxy_set_header Host 127.0.0.1:${DUB_PORT}; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + add_header Access-Control-Allow-Origin * always; + add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" always; + add_header Access-Control-Allow-Headers "Content-Type, Authorization" always; + + if ($request_method = OPTIONS) { + return 204; + } + } + + # Web3 Storage Provider Node (HTTP API: /health, /stats, agreements) + location /web3-storage-provider/ { + proxy_pass http://storage_provider/; + proxy_http_version 1.1; + proxy_set_header Host 127.0.0.1:${WEB3_STORAGE_PROVIDER_PORT}; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Allow large file uploads (nginx defaults to 1m, which 413s anything bigger). + client_max_body_size 256m; + + # CORS headers for HTTP API. Strip the provider node's own + # Access-Control-Allow-Origin first (it ships with CorsLayer::permissive), + # otherwise browsers see "*, *" and reject the response — same trick the + # IPFS block below uses for Kubo. + proxy_hide_header Access-Control-Allow-Origin; + add_header Access-Control-Allow-Origin * always; + # Web3 Storage uploads use PUT/DELETE/HEAD and send an Authorization header, + # so the preflight must whitelist them (browsers reject otherwise). + add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, HEAD, OPTIONS" always; + add_header Access-Control-Allow-Headers "Authorization, Content-Type" always; + + # Handle preflight requests + if ($request_method = OPTIONS) { + return 204; + } + } + + # IPFS Gateway + location /ipfs/ { + proxy_pass http://ipfs_gateway/ipfs/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Increase timeouts for large files + proxy_read_timeout 300; + proxy_connect_timeout 60; + + # Strip Kubo's Access-Control-Allow-Origin before adding our own, + # otherwise browsers see "*, *" and reject the response. + proxy_hide_header Access-Control-Allow-Origin; + add_header Access-Control-Allow-Origin * always; + } + + # Chain specifications - for smoldot light clients, and for `make bite` + # against this deployment (specSource: "published" in networks/*.json). + # Serve any top-level raw chainspec JSON (not -plain, not zombie.json, not + # subdirs). Spec names follow the network's chain ids — previewnet's end in + # -local, a fork of a public network carries the source's ids — so the name + # set cannot be enumerated here. + location ~ "^/chainspecs/((?!zombie\.json$)[a-z0-9-]+(?--.tar.gz` (the tag without its leading `v`) -and extracts the single `dub` into `bin/`. PPN no longer builds it: the +`ppn fetch` downloads `dub---.tar.gz` (the tag without its leading +`v`) and extracts the single `dub` into `bin/`. Since v0.6.0 every asset ships twice, one +build per People runtime: `testnet` for previewnet and paseo-next-v2, `polkadot` for a fork +of Polkadot, whose People runtime carries neither Game nor ProofOfInk and whose build +therefore has no invite-tickets roles. A network picks its build with `dubBuild` in its +descriptor, defaulting to testnet, and `dub --help` prints the build a binary carries. PPN no longer builds it: the `build-identity-backend` job is gone from `.github/workflows/release.yml`, along with the GitHub App token it needed to clone a private repo, the pinned Rust toolchain and roughly ten minutes of every release. diff --git a/docs/FORK.md b/docs/FORK.md index 0e835a9..42911c5 100644 --- a/docs/FORK.md +++ b/docs/FORK.md @@ -63,11 +63,20 @@ The apply half is a transaction, not state, so it runs after the spawn: the `ena custom process submits `apply_authorized_upgrade` for every seeded chain, parachains first and the relay last, through the same path `ppn upgrade` uses. It waits for the chains to author, retries a chain that does not answer yet, and is a no-op on a chain already running the blob — -so a resumed fork does nothing here. Enactment itself is slow on a parachain of a real network: -the code goes live only on the relay's go-ahead, `validation_upgrade_delay` relay blocks after -the PVF pre-check, which on Polkadot is 600 blocks, an hour. The service waits up to two hours -per chain. `make runtime-upgrade NETWORK=polkadot CHAIN=people ENACT_TIMEOUT_MIN=70` with no -`WASM=` does the same for one chain by hand. +so a resumed fork does nothing here. All parachains are submitted at once and waited for +together, then the relay. + +A parachain's code goes live on the relay's go-ahead, `validation_upgrade_delay` relay blocks +after the PVF pre-check. Live Polkadot sets that to 600 blocks, an hour, and forbids a second +upgrade of the same parachain for `validation_upgrade_cooldown` = 14400, a day — sensible for a +production relay, useless for a fork whose purpose is enacting runtimes under test. A +shared-relay bite therefore patches both into `Configuration::ActiveConfig`, to 30 and 60 relay +blocks (`shared-relay.ts`), in the same rebuild that sets `num_cores`. The pre-check and +go-ahead path is unchanged; only the timers are. So from a fresh bite, all four chains are on +the new runtime within a few minutes of the spawn. A bundle bitten before this patch still +carries the live values, which is why the service waits up to two hours per chain. +`make runtime-upgrade NETWORK=polkadot CHAIN=people` with no `WASM=` does the same for one +chain by hand. Under the hood `ppn bite --upgrade =` stages the blob into the bundle under `upgrades/.wasm`, recording it in `manifest.json` as `seededUpgrades`. Add @@ -85,6 +94,47 @@ This mirrors zombie-bite's `--rc-upgrade`/`--para-upgrade` meant to be deleted along with the rest of `packages/cli/src/fork/` once PPN calls zombie-bite instead of driving doppelganger itself ([#120](https://github.com/paritytech/zombie-bite/issues/120)). +## What the bite seeds + +A fork inherits live state and no governance, so anything a running network sets by Root, by +sudo or by a transaction from an account we hold has to be written into state at import instead. +Three things are. Two are named by a descriptor field; the first is not, and everything else +here depends on it: + +- `//Alice` is endowed on the relay and on every parachain, by writing her `System::Account` + directly. Nothing below can pay a fee without it: a fork of a public network has no dev + account holding funds and no way to fund one. This is the seed with no descriptor field, + because there is nothing to decide. It happens whenever `bite.sharedRelay` is true, and that + flag says the relay is somebody else's, which is exactly the case where the funds are missing. +- `bulletinAuthorizer`, an entry in Bulletin's `AllowedAuthorizers`. Under "Bulletin content", + because a fork's relation to Bulletin's stored bytes is the same discussion. +- `attestation`, the two `AttestationAllowance` quotas the identity backend spends. Below, + "Seeding the attestation allowances". + +Add a seed and add a line here, or the next reader asking what a bite writes has to find it by +reading `paraOverrides`. + +## Seeding the attestation allowances + +Two of them, one per pallet, sharing a storage name and metering different things. +`PeopleLite::AttestationAllowance` is how many people a verifier may attest, spent by `attest`. +`DotnsGateway::AttestationAllowance` is how many names an attester may reserve, spent by +`reserve_name`. The identity backend needs both: without the first nobody becomes a lite +person, and without the second nobody gets a username. + +Both pallets that gate attestation take an origin a fork cannot raise: `PeopleLite`'s +`AttestationAllowanceManager` is `EnsureRoot`, and `DotnsGateway`'s is `RootOrWhitelist`, whose +second arm is an OpenGov track. + +The count and the pair both come from individuality-community's initial-setup, which grants +`PeopleLite` on People and `DotnsGateway` on Asset Hub by sudo, at +`PEOPLELITE_ATTESTATION_ALLOWANCE` and `DOTNSGATEWAY_ATTESTATION_ALLOWANCE`, both a million. + +```json +{ "attestation": { "attester": "5Grwva…", "count": 1000000, + "on": { "people": "PeopleLite", "asset-hub": "DotnsGateway" } } } +``` + ## What you get, and what you don't The fork resumes at the bite block and diverges from there — it is a real fork, not a mirror. It @@ -133,14 +183,18 @@ hold. Parachains are bitten first; the relay is then bitten with their heads inj `Paras::Heads`, so the relay is made to agree with wherever the parachains actually landed. **Spawn** uses the **regular** `polkadot` and `polkadot-omni-node`. Nothing about restoring a bundle -needs doppelganger, which is why `FORK=1` works with no extra binaries. +needs doppelganger, which is why `FORK=1` works with no extra binaries. The collators are the +exception: they run cumulus's minimal relay node (`--relay-chain-rpc-urls`), which upstream builds +with `fork_id: None` (`relay-chain-minimal-node/src/network.rs`). A stock collator therefore finds +no validator, and the parachains stop while the relay keeps authoring. Until upstream ships the +fix, collators need a build that has it. A bundle is: ``` fork-bundle/ manifest.json what was bitten, when, from which production version - specs/.json production's specs with bootNodes stripped + specs/.json production's specs, with a fork id and the fork's own relay bootnode overrides/*.json the storage overrides doppelganger applies snapshots/*.tgz the bitten databases (~70 MB; ~107 MB for the bundle) fork.toml generated zombienet config @@ -154,8 +208,10 @@ and per-chain flags with the genesis generator — see below. Each of these was a real failure during development; none of them announces itself clearly. -**A fork resumes only on its own database.** zombienet restores the bundle's snapshot only into -an empty base path, so a start that finds a database already there silently runs on it instead. +**A fork resumes only on its own database.** zombienet unpacks a node's `db_snapshot` over its +base path on every spawn, whether or not a database is already there, and the extraction +overwrites the files it carries while leaving the rest in place. A start that hands it the +snapshots over foreign data therefore gets a mix of the two. What that cost us once: a genesis run and a fork sharing `./data`, so the fork came up on a database belonging to a different chain. It looked healthy for a hundred blocks, then three of six validators panicked with `Trie lookup error: Database missing expected key` ~110 blocks past the @@ -165,8 +221,9 @@ Three things are in place for that. Fork mode has its own `data-fork-` never meets genesis data. The spawn stamp (`spawn.json`, written beside the chain state) records which bite the databases came from, and `ppn start --fork` compares it with the bundle's `bittenAt` before deciding: a match resumes where the fork stopped, with any runtime upgrade it -enacted still in force; anything else — a re-bitten bundle, another network's data, no stamp — -is wiped and the snapshot restored. `CLEAN=1` wipes regardless, which is how a fork is put back +enacted still in force, and regenerates the config without `db_snapshot` so there is nothing to +unpack over it; anything else — a re-bitten bundle, another network's data, no stamp — is wiped +and the snapshot restored. `CLEAN=1` wipes regardless, which is how a fork is put back at its bite block. ```bash @@ -180,11 +237,13 @@ make start FORK=1 NETWORK=polkadot FRESH_BITE=1 # new bite; the old data is wi parachain, last one wins, and all collators silently run the same chain. The symptom is parachains converging on identical block numbers — not an error. -**Spawn specs must have `bootNodes` cleared.** Otherwise the forked nodes rejoin previewnet and -follow its longer chain. This looks like success on every metric — the chain is at a plausible -height and finalizing — while not being a fork at all. The bite genuinely needs the bootnodes, so -`ppn bite` keeps the as-published specs in its work directory and puts only the -stripped copies in the bundle. +**Spawn specs carry a fork id and none of the source's bootnodes.** Otherwise the forked nodes +rejoin the source network and follow its longer chain. This looks like success on every metric (the +chain is at a plausible height and finalizing) while not being a fork at all. A fork keeps the +source's genesis hash, so without a fork id its protocol names equal the source's and it still +reaches the source's DHT. The bite genuinely needs the bootnodes, so `ppn bite` keeps the +as-published specs in its work directory and ships copies with `forkId` `ppn-fork-`, the +fork's first relay node as the relay spec's only bootnode, and no bootnodes on the parachain specs. **Collators must run via `scripts/omni-node.sh`, not the binary directly.** A collator runs a relay-chain node in the same process, configured by the args after `--`, which zombienet owns and @@ -234,6 +293,12 @@ its unincluded segment fills — exactly three blocks in. ## Bulletin content +Nothing can be stored at all until an authorizer exists. Live Polkadot Bulletin's +`AllowedAuthorizers` is empty, because authorizers arrive by governance there, and a fork has no +genesis to seed one, so `bulletinAutoAuthorize` fails `BadSigner`. The `bulletinAuthorizer` +descriptor field names the account the bite writes in; `bulletinAuthorizerInjects` in +`validators.ts` says what the entry holds and which reference it deliberately leaves out. + A fork carries chain state but not bulletin's stored bytes: those live in block bodies, and the bite is a warp sync. So a forked bulletin *lists* content it does not hold, and anything published before the bite is unreachable. Content uploaded after the fork works normally. diff --git a/docs/POLKADOT-FORK.md b/docs/POLKADOT-FORK.md index 3365f46..7263014 100644 --- a/docs/POLKADOT-FORK.md +++ b/docs/POLKADOT-FORK.md @@ -1,24 +1,26 @@ # Forking Polkadot with the fellowship runtimes -A runbook for one dedicated machine that keeps a fork of Polkadot running with a fellowship -runtime release enacted on it — currently 2.5, the release that lands Individuality on People -Polkadot and Asset Hub Polkadot. Everything here is `make` on top of what `docs/FORK.md` -explains; read that for the why. +A runbook for one dedicated machine that keeps a fork of Polkadot running, optionally with a +fellowship runtime release newer than mainnet's enacted on it. Mainnet runs v2.5.3, with the +Individuality pallets on People and Asset Hub. Everything here is `make` on top of what +`docs/FORK.md` explains; read that for the why. ## What the fork is -| Chain | Live para id | Runtime live (2026-09) | Runtime under test | -| --- | --- | --- | --- | -| Relay | | polkadot 2004000 | polkadot 2005000 (version bump only) | -| Asset Hub | 1000 | statemint 2004000 | pallet-revive 0.19.1, Individuality pallets | -| People | 1004 | people-polkadot 2004000 | Individuality pallets | -| Bulletin | 1010 | bulletin-polkadot 2004000 | version bump only | - -Six dev validators, one collator per parachain, PPN's normal ports. All six HRMP channels -between the three parachains exist on live Polkadot and survive the bite. Polkadot has no sudo, -so the runtimes are authorized at bite time and enacted after the spawn; nothing else on the -fork can dispatch root, which is why the People-side grants (attestation allowance, invites) -do not run here and Web3 Storage is not part of it. +| Chain | Live para id | Runtime live (2026-11) | +| --- | --- | --- | +| Relay | | polkadot 2005003 | +| Asset Hub | 1000 | statemint 2005003 | +| People | 1004 | people-polkadot 2005003 | +| Bulletin | 1010 | bulletin-polkadot 2005003 | + +A bite takes these runtimes as they are; `UPGRADES=` adds a runtime under test (see "Bite and +start"). Six dev validators, one collator per parachain, PPN's normal ports. All six HRMP +channels between the three parachains exist on live Polkadot and survive the bite. Polkadot has +no sudo, so a runtime under test is authorized at bite time and enacted after the spawn. Nothing +else on the fork can dispatch root, so the grant processes do not run: the bite seeds both +attestation allowances instead (`docs/FORK.md`, "What the bite seeds"), dub's `polkadot` build +has no invite roles, and Web3 Storage is not part of the fork. ## One-time setup @@ -32,49 +34,48 @@ make fetch NETWORK=polkadot # polkadot, polkadot-parachain, eth-rpc make fetch-doppelganger NETWORK=polkadot # the bite tool, into bin/polkadot/dg/ ``` +Then replace `bin/polkadot/polkadot-parachain` with a build that has the fork-id fix (see +`docs/FORK.md`). Any later `make fetch`, including the one `apply.sh` runs, restores the stock one. + ## Get the runtimes -PPN takes runtime blobs as files; fetching them is `gh`'s job. 2.5 is not a release yet, it is -the open release PR ([polkadot-fellows/runtimes#1265](https://github.com/polkadot-fellows/runtimes/pull/1265)), -whose build workflow uploads one artifact per runtime, named after it, on every push: +Only for a runtime newer than mainnet's. PPN takes runtime blobs as files; fetching them is `gh`'s +job: a release's assets, or an unreleased build from its PR's build workflow, which uploads one +artifact per runtime, named after it: ```bash -# newest "Tests" run on the release branch that actually uploaded the runtimes (not every run does) -RUN=$(for id in $(gh api "repos/polkadot-fellows/runtimes/actions/runs?branch=kiz-release-2.5&per_page=30" -q '.workflow_runs[] | select(.name=="Tests") | .id'); do - gh api repos/polkadot-fellows/runtimes/actions/runs/$id/artifacts -q '.artifacts[] | select(.name=="people-polkadot") | .id' | grep -q . && echo $id && break; done) -gh run download $RUN -R polkadot-fellows/runtimes -D runtimes/ -n asset-hub-polkadot -n people-polkadot -ls runtimes/*/ # runtimes/asset-hub-polkadot/asset_hub_polkadot_runtime.compact.compressed.wasm, … +gh release download -R polkadot-fellows/runtimes -D runtimes/ -p 'asset-hub-polkadot_*' -p 'people-polkadot_*' +gh run download -R polkadot-fellows/runtimes -D runtimes/ -n asset-hub-polkadot -n people-polkadot ``` -Re-run after the PR is pushed to and the blobs follow. Artifacts expire after 90 days; the -directory is what you keep. Once 2.5 is cut, the release assets replace the artifacts: +Artifacts expire after 90 days; the directory is what you keep. The relay and Bulletin runtimes +are in the same places (`polkadot`, `bulletin-polkadot`). + +## Bite and start ```bash -gh release download v2.5.0 -R polkadot-fellows/runtimes -D runtimes/ -p 'asset-hub-polkadot_*' -p 'people-polkadot_*' +make bite NETWORK=polkadot +make start FORK=1 NETWORK=polkadot # spawns from fork-bundle-polkadot/ ``` -The relay and bulletin runtimes are in the same places (`polkadot`, `bulletin-polkadot`) if you -want the whole release on the fork; 2.5 changes nothing in them beyond the version. - -## Bite and start +To rehearse a runtime under test, name its blobs in the bite: ```bash -make bite NETWORK=polkadot UPGRADES="asset-hub=runtimes/asset-hub-polkadot/asset_hub_polkadot_runtime.compact.compressed.wasm people=runtimes/people-polkadot/people_polkadot_runtime.compact.compressed.wasm" -make start FORK=1 NETWORK=polkadot # spawns from fork-bundle-polkadot/ +make bite NETWORK=polkadot UPGRADES="asset-hub=runtimes/ people=runtimes/" ``` -The bite is ~20 minutes: it warp-syncs all four chains and authorizes the blobs, printing per -chain what it authorized. The start then does the rest on its own: once the chains author, the -`enact-upgrades` process submits `apply_authorized_upgrade` for each seeded chain, waits for the -relay's PVF pre-check and go-ahead, and the dashboard flips it to `2005000`. Its log is under -the dashboard's logs tab. Budget about an hour per parachain: Polkadot's -`validation_upgrade_delay` is 600 relay blocks, and the go-ahead only comes after it. dub -restarts itself until People carries the Individuality pallets, so it comes up a few seconds -after the People upgrade lands. +The bite is ~20 minutes: it warp-syncs all four chains and authorizes any blobs, printing per chain +what it authorized. With `UPGRADES=`, the start then does the rest on its own: once the chains +author, the `enact-upgrades` process submits `apply_authorized_upgrade` for every seeded parachain +at once, waits for the relay's PVF pre-check and go-ahead, then does the relay, and the dashboard +flips each to its new spec version. Its log is under the dashboard's logs tab. On a bundle bitten +with the current code that is a few minutes: the bite sets the relay's `validation_upgrade_delay` +to 30 blocks and the cooldown to 60 (live Polkadot: 600 and 14400, an hour and a day). -To redo one chain by hand, `make runtime-upgrade NETWORK=polkadot CHAIN=people -ENACT_TIMEOUT_MIN=70` with no `WASM=` applies the blob the bite authorized; on a chain already -running it, it is a no-op. +To redo one chain by hand, `make runtime-upgrade NETWORK=polkadot CHAIN=people` with no `WASM=` +applies the blob the bite authorized; on a chain already running it, it is a no-op. A second +upgrade of the same chain (a newer build) is a new bite with the new blob, and the 60-block +cooldown means it can follow the first within minutes. ## Day to day @@ -83,22 +84,28 @@ running it, it is a no-op. | Stop | `make kill` | | Start again where it stopped, upgrades still enacted | `make start FORK=1 NETWORK=polkadot` | | Back to the bite block (the upgrades are re-enacted on the way up) | `make start FORK=1 NETWORK=polkadot CLEAN=1` | -| Fresh state from live Polkadot, same runtimes | the same `make bite ... UPGRADES=...` then start | -| Different runtimes (the PR was pushed to) | download again, then a new bite with the new files — the authorization is state inside the bundle | +| Fresh state from live Polkadot, same runtimes | the same `make bite` (with the same `UPGRADES=`, if any) then start | +| Different runtimes under test | download again, then a new bite with the new files — the authorization is state inside the bundle | | Throw the bundle away | `make clean-fork NETWORK=polkadot` | A start decides between resuming and wiping from the spawn stamp in `data-fork-polkadot/`: if it names the bite the bundle carries, the fork continues; if the bundle was re-bitten since, the old data goes. A start that reuses a bundle prints what that bundle has authorized. -Node logs are under `/tmp/zombie-*/`; the bite's own logs under `fork-bundle-polkadot-logs/`. +Node logs are under `data-fork-polkadot/`, one `/.log` per node; the bite's own logs +under `fork-bundle-polkadot-logs/`. ## Exposing it: nginx, TLS, systemd The engine ships no server tooling; Parity's previewnet boxes get theirs from preview-net-v1's `server/` directory and its deploy workflow, which today only know how to spawn previewnet from -genesis. Until that grows a fork target, this is the same setup done by hand, once. Ubuntu -assumed; `DOMAIN` is the name the box answers on. +genesis. Until that grows a fork target, this is the same setup done by hand, once. The fork +machine's own branch adds `deploy/polkadot-test/apply.sh`, which does all of it (see its README); +the steps below are what it automates. They predate two things it adds: the nginx `stream {}` +block for TURN over TLS, without whose `{{GENERATED_STREAMS}}` marker `ppn nginx-conf` refuses +the template, and the secrets file `/etc/ppn/secrets.env` (named by `PPN_SECRETS_FILE` in +`ports.env`), without which the box runs the public dev secrets. Ubuntu assumed; `DOMAIN` is +the name the box answers on. **DNS and packages.** Add an A record for `DOMAIN` pointing at the box (Parity's zones live in the `dns` repo). Then: @@ -197,22 +204,22 @@ sudo systemctl daemon-reload && sudo systemctl enable --now ppn-polkadot.service journalctl -u ppn-polkadot -f ``` -Run the bite (`make bite ... UPGRADES=...` above) before the first `systemctl start`; the unit +Run the bite (`make bite NETWORK=polkadot` above) before the first `systemctl start`; the unit only spawns what is already bitten, and a restart resumes it. `make kill` and `systemctl stop` are the same thing; use the latter so systemd does not restart what you stopped. -One limit of the hand-rolled version: nothing collects node logs out of `/tmp/zombie-*/` the -way v1's log collector does, and `ppn kill` removes them. Copy what you need before stopping. +One limit of the hand-rolled version: nothing collects node logs out of `data-fork-polkadot/` the +way v1's log collector does, and a wipe (`CLEAN=1`, or a start after a re-bite) removes them with +the data. Copy what you need first. ## Services on this fork -Asset Hub's eth-rpc, Bulletin's IPFS daemon, the dashboard and the identity backend (dub) -start with the chains. dub talks to the Individuality pallets, so it only does anything once -People runs 2.5; before that its chain writer errors and can be ignored. What does not run, -because each needs root on a chain that has none: the attestation allowance and invite grants -on People, HRMP opening and core assignment (both already in state), and the DotNS genesis -steps on Asset Hub. Polkadot's Asset Hub carries no DotNS deployment, so there are no products -to import into Bulletin either. +Asset Hub's eth-rpc, Bulletin's IPFS daemon, the dashboard and the identity backend (dub, its +`polkadot` build) start with the chains. What does not run, because each needs root on a chain +that has none: the attestation allowance and invite grants (the bite seeds both allowances, and +the `polkadot` build has no invite roles), HRMP opening and core assignment (both already in +state), and the DotNS genesis steps on Asset Hub (the fork inherits mainnet's dotNS). Polkadot +Bulletin stores nothing, so there are no products to import. ## What is not verified yet @@ -223,18 +230,16 @@ bite wrote the upgrade authorization as an override, which doppelganger applies the live state already has, so it never landed and the apply failed `NothingAuthorized`; and the upgrade command read `Sudo.Key` before checking whether the chain has a Sudo pallet. -- The upgrade enactment end to end on a bite made after those fixes: the apply landing, the - relay's PVF pre-check, the go-ahead after Polkadot's `validation_upgrade_delay` of 600 relay - blocks (an hour), and the chains reporting `2005000`. -- Stop and resume of a running fork. Mechanically it is only skipping the wipe, and each node - restarts on its own database, but nobody has watched six validators come back after an hour. -- Whether dub works against the fellowship's People runtime rather than previewnet's build. - dub signs with vendored metadata generated from next-people-paseo, previewnet's People; the - fellowship's people-polkadot 2005000 is built from the same pallets but is a different - runtime, and no dub release names it yet. Expect it to connect once People runs 2.5, then - either sign fine or log a metadata-drift `WARN` at boot (dub v0.5.0 does that) and fail to - encode submissions. The second outcome is a dub change, a people-polkadot entry in its known - runtimes, not an engine one. After that comes the attestation allowance and the invites, - which previewnet grants by sudo and Polkadot cannot. +- The upgrade enactment end to end was verified the same day on a re-bite: apply landing, + PVF pre-check, go-ahead 600 relay blocks later, all four chains on `2005000`, dub up. Not yet + verified: a bite made after the delay/cooldown patch enacting within minutes, and a second + upgrade of the same parachain after the shortened cooldown. +- Stop and resume of a running fork on this branch's code. A resume leaves the snapshots out of + the spawn config (`start-network.test.ts` checks it), and each node restarts on its own + database, but nobody has watched it on the box yet. +- A registration through dub on the fork. dub v0.8.0's `polkadot` build vendors people-polkadot + 2005000's metadata, which matches mainnet's 2005003 in everything dub reads, decodes and signs + with (compared with `subxt diff`, 2026-11-13). The bite seeds both attestation allowances, and + the build has no invite roles. Tell the runbook what you find. diff --git a/docs/TRINITY-STACK.md b/docs/TRINITY-STACK.md index 3e05103..52ae6a0 100644 --- a/docs/TRINITY-STACK.md +++ b/docs/TRINITY-STACK.md @@ -54,9 +54,12 @@ The descriptor is built from the dashboard model, so its URLs are the ones the d - A genesis network takes them from `bin/dotns-addresses.json`, the address list `ppn fetch` takes from the dotns release. Contracts with no key, such as the store beacons, are left out. - A fork takes the keys from its source's descriptor, at `stack.pointer`, and each address from - its own registry's `get(key)`, since the bite copied the source's registry. A key the registry - does not hold, such as a release's `multicall3`, keeps the source's address. While the source - descriptor or eth-rpc does not answer, the fork publishes no `dotns` and tries again in 30s. + its own registry's `get(key)`, since the bite copied the source's registry. When the source's + registry has no code on the fork, the registry is the one Asset Hub's + `DotnsGateway::DispatcherAddress` returns from `protocolRegistry()`. A key the registry does + not hold, such as a release's `multicall3`, keeps the source's address if that has code on + the fork, and is left out otherwise. While the source descriptor or eth-rpc does not answer, + the fork publishes no `dotns` and tries again in 30s. - `fork` and each chain's `forkedAt`: on a fork whose spawn stamp records the bite and whose network file names the source's descriptor in `stack.pointer`. `forkedAt` is the source's block, whose hash `ppn bite` reads from the source and records as `biteHashes`: the fork's diff --git a/networks/README.md b/networks/README.md index de17f68..4cab5bc 100644 --- a/networks/README.md +++ b/networks/README.md @@ -42,6 +42,7 @@ or `PPN_NETWORK=` in the environment of the scripts. | *(network overlay)* | `PPN_NETWORK_OVERLAY=` changes `extraArgs`, `rpcNodes` and `rpcNodeArgs` for one deployment without a release: `{ "previewnet": { "relay": { "extraArgs": [...] }, "parachains": { "people": { "rpcNodes": 2, "rpcNodeArgs": [...] } } } }`. Every command applies it, so the zombienet config, nginx routes and dashboard agree. An unknown network, chain or field is an error. | | `services` | Per-service switches and binaries: `false` disables a service; `{ "binary": { … } }` says what it runs. `eth-rpc` must match the revive pallet in the network's Asset Hub runtime, which is exactly why it is pinned per network rather than shared. | | `tools` | Local tools this network needs that are neither a chain nor a service — previewnet's `chain-spec-builder`, for instance, which only a genesis network uses. | +| `dubBuild` | Which dub build the network needs: `testnet` (the default) or `polkadot`. The build fixes dub's People runtime: its vendored metadata and whether the invite-tickets roles exist. Only a fork of Polkadot needs `polkadot`. It also decides which services and databases are generated, because naming a role the binary lacks is a boot failure. | | `bite.doppelganger` | `{repo, tag}` of the bite tool. Per network, because the build has to be able to execute the runtimes of the network being bitten. | | *(bin layout)* | Only node binaries are network-versioned: previewnet's live in `bin/`, every other network's in `bin//` (plus `dg/` for bites). Shared tooling with no per-network dimension — zombie-cli, ipfs, postgres, identity, design families — lives in plain `bin/` and is pinned in `config/versions.env`. Collators run through `scripts/omni-node.sh` regardless of binary; it carries the libp2p fix and picks binary/dir from `PPN_COLLATOR_BINARY`/`PPN_BIN_DIR`. | | `relay.chain` | Built-in chain name doppelganger runs with (`ZOMBIE_CHAIN`), e.g. `paseo`, `kusama`. | @@ -53,9 +54,11 @@ or `PPN_NETWORK=` in the environment of the scripts. | `parachains[].key` | One of `asset-hub`, `people`, `bulletin`, `web3-storage` — the keys PPN's port table and per-chain flags are defined for. | | `parachains[].paraId` | The network's real para id. Previewnet/paseo-next-v2 use the 1500 band; devnet and real networks the 1000 band. | | `parachains[].spec` | Basename of the parachain spec inside the bundle. | +| `parachains[].bulletinAuthorizer` | Bulletin only: an account id (32 bytes, `0x` hex) the bite writes into `TransactionStorage::AllowedAuthorizers`. A bitten Bulletin has no genesis, so without it no account can authorize storage. See `docs/FORK.md`, "Bulletin content". | | `services` | Optional `{ "": false }` map to disable custom processes that would otherwise run for the present parachains (e.g. eth-rpc for asset-hub, ipfs for bulletin). | | `dotns.resolver` | DotnsContentResolver address for `pin-bulletin-products` — per network, since every DotNS deployment has its own. Omitted → read from `bin/dotns-addresses.json` (previewnet's release artifact); still absent → the import is skipped. | | `dotns.gateway` | IPFS gateway the product bytes are fetched from. Omitted → `bite.source`, which is right where the source network's front door is also its gateway (previewnet, devnet) and wrong where it is not: paseo-next-v2's `https://dot.li` is a single-page app that returns its own HTML for every path, 200 included. | +| `attestation` | `{ attester, count, on }`: the bite grants `attester` an `AttestationAllowance` of `count` on each chain in `on`, which maps a chain key to `PeopleLite` or `DotnsGateway`. The identity backend spends both quotas, and a fork without governance cannot grant them later. See `docs/FORK.md`, "Seeding the attestation allowances". | | `stack.tld` | The dotNS TLD the network's Trinity stack descriptor names. Omitted → `genesisConfig.networkSuffix`; still absent → no descriptor is written. See `docs/TRINITY-STACK.md`. | | `stack.pointer` | `{ url, key? }` where the live network publishes its own descriptor. A fork names it as `fork.of` and takes its dotNS keys from it; omitted → the fork's descriptor carries no `fork` and no `dotns`. | | `stack.assets` | `[{ role, chain, pallet, id }]`, published as the descriptor's `assets`, such as the asset a wallet pays in. `chain` is a chain role the network runs. | @@ -87,11 +90,12 @@ One thing to know: `bite.sharedRelay` marks a network whose live relay carries parachains we do not run — Paseo, Kusama, Polkadot. Previewnet's relay is ours end to end, so it inherits state that is already correct and the bite leaves it alone. A shared relay inherits three things that are wrong for six -dev validators, and all three are silent: +dev validators, all of them silent, and upgrade timers tuned for production: | what | inherited state | why it breaks | what the bite writes | | --- | --- | --- | --- | | cores | ~18 registered parachains, so ~18 cores | the runtime splits 6 validators into 18 groups; 12 come out empty and ours sit on cores no group is assigned to, so nothing backs their blocks | ours on cores 0-N from `ParaScheduler::CoreDescriptors`, groups to match, `num_cores` cut to N (one byte of `Configuration::ActiveConfig`, so `EnabledHostFunction(EccRfc163)` survives) | +| upgrade timing | `validation_upgrade_delay` 600 and `validation_upgrade_cooldown` 14400 on Polkadot | a runtime under test goes live an hour after its apply, and the same parachain cannot be upgraded again for a day — the fork exists to do exactly that, several times | both patched in the same `ActiveConfig` rebuild, to 30 and 60 relay blocks (`FORK_VALIDATION_UPGRADE_DELAY`/`_COOLDOWN` in `shared-relay.ts`); the pre-check and go-ahead path is unchanged, only the timers | | HRMP / DMP | channel heads from a relay snapshot newer than the parachain ones | messages delivered in that window are pruned from the relay but counted in its `mqc_head`, so cumulus panics `HRMP head mismatch` and builds no block at all | every channel touching one of our paras (read off the relay's own indexes) keeps its `Hrmp::HrmpChannels` entry with `mqc_head` cleared and counters zeroed, its `HrmpChannelContents` emptied and the recipient's `HrmpChannelDigests` emptied; `Dmp::DownwardMessageQueueHeads` zeroed and `Dmp::DownwardMessageQueues` emptied per para; the parachain's own `ParachainSystem::LastDmqMqcHead` zeroed and `LastHrmpMqcHeads` emptied to match — both halves or neither. Channels stay open, which matters on a relay without sudo where nothing could reopen them | | stored-data proofs | Bulletin's `RetentionPeriod` of 201,600 against a chain at ~1.5M | `pallet-transaction-storage` asserts a periodic proof of historical data a bite does not carry, so no block can be built | `RetentionPeriod` pushed past any height we fork, applied only to a runtime that has the pallet | diff --git a/networks/polkadot.json b/networks/polkadot.json index 10aab1f..6898cd8 100644 --- a/networks/polkadot.json +++ b/networks/polkadot.json @@ -14,6 +14,8 @@ "sharedRelay": true, "_note": "No sudo: a runtime upgrade is authorized at import instead of dispatched. `ppn bite --upgrade asset-hub=` seeds System::AuthorizedUpgrade, and `ppn runtime-upgrade` submits the apply half unsigned. Belongs in zombie-bite (paritytech/zombie-bite#127) once PPN calls it instead of driving doppelganger. v0.2.3 is the first doppelganger honouring ZOMBIE_WARP_SKIP_PROOF, without which the state sync stalls partway through and never recovers." }, + "dubBuild": "polkadot", + "_dubBuild": "The Polkadot People runtime has neither Game nor ProofOfInk, so dub's polkadot build drops the invite-tickets roles. Naming a role the binary lacks is a boot failure.", "releases": { "polkadot-sdk": { "repo": "paritytech/release-automation", @@ -59,6 +61,8 @@ { "key": "bulletin", "paraId": 1010, + "bulletinAuthorizer": "0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d", + "_bulletinNote": "//Alice, the account the bite already endows on every chain. A fork has no genesis, and live Polkadot Bulletin has an empty AllowedAuthorizers because authorizers arrive by governance there, so nothing can store anything without this. paseo-next-v2 carries //Alice, //Eve and an ops key; previewnet carries only //Eve, seeded by the bulletin runtime's genesis preset (Sr25519Keyring::Eve, 100_000 transactions, 100 GiB) which is where the budget below comes from. Alice over Eve because she is funded here and Eve is not, and the authorizer and the account it authorizes may be the same.", "spec": "bulletin", "rpc": "wss://bulletin-rpc.polkadot.io", "specSource": "https://raw.githubusercontent.com/paritytech/chainspecs/main/polkadot/parachain/bulletin/chainspec.json", @@ -77,7 +81,17 @@ } } }, + "attestation": { + "attester": "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY", + "count": 1000000, + "on": { "people": "PeopleLite", "asset-hub": "DotnsGateway" }, + "_note": "//Alice, matching what the bite already endows, and 1000000 to match PEOPLELITE_ATTESTATION_ALLOWANCE and DOTNSGATEWAY_ATTESTATION_ALLOWANCE in individuality-community's initial-setup, which grants both by sudo. Two quotas share the AttestationAllowance name: PeopleLite's counts how many people a verifier may attest, DotnsGateway's how many names an attester may reserve, and the backend needs both. Managers are EnsureRoot and RootOrWhitelist; neither is reachable on a fork after the spawn. attestation.on pairs each chain with the pallet to seed there." + }, "stack": { - "tld": "dot" + "tld": "dot", + "pointer": { + "url": "https://stack.api.pcf.network/.well-known/trinity-stack.json" + }, + "_pointer": "No key. PCF signs its descriptor, but the descriptor does not carry the public key, and the engine has none from PCF to pin. Without a key the signature is not checked; a wrong key would fail every read with bad-signature and leave the fork with no dotns." } } diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index e8c1bd6..f0f05af 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -658,9 +658,10 @@ export function buildProgram(): Command { forkCmd('toml ', 'generate the zombienet config for a bundle') .addOption(new Option('--no-enable-hop', 'leave --enable-hop off the bulletin collator').env('ENABLE_HOP')) - .action((bundleDir: string, outFile: string, opts: { enableHop: boolean }) => { + .option('--resume', 'leave the snapshots out, for a fork that resumes on its own database') + .action((bundleDir: string, outFile: string, opts: { enableHop: boolean; resume?: boolean }) => { process.env.ENABLE_HOP = String(opts.enableHop); - return runFork(['toml', bundleDir, outFile]); + return runFork(['toml', bundleDir, outFile, ...(opts.resume ? ['--resume'] : [])]); }); forkCmd('products ', 'the DotNS product CIDs a fork needs') diff --git a/packages/cli/src/commands/bite.ts b/packages/cli/src/commands/bite.ts index 0dd2bea..5921841 100644 --- a/packages/cli/src/commands/bite.ts +++ b/packages/cli/src/commands/bite.ts @@ -22,6 +22,7 @@ import { specSourceUrl, asWs, readEnvFile, + forkRelayBootnode, type NetworkDef, type NetworkChain, repoRoot, @@ -458,17 +459,41 @@ async function collectSpec( } // Two copies, and only one of them ships. The as-fetched spec keeps the source's - // bootNodes, which the bite needs in order to warp-sync. What goes in the bundle has - // them stripped, because a forked node that keeps them rejoins the source network and - // follows its longer chain — which looks like success on every metric while not being a - // fork at all. + // bootNodes, which the bite needs in order to warp-sync. What goes in the bundle drops + // them, because a forked node that keeps them rejoins the source network and follows its + // longer chain, which looks like success on every metric while not being a fork at all. + // + // The relay spec ships one bootnode of its own in their place. Dropping the source's + // bootNodes without replacing them strands a fork the moment `fork_id` takes it off the + // source network's DHT: Kademlia adds peers manually, so a node with no bootnode never fills + // its routing table and authority discovery resolves no addresses. See `forkRelayBootnode`. + // Parachain specs ship none, because a fork runs a single collator per parachain and has no + // parachain peer to find. const spec = JSON.parse(fs.readFileSync(workSpec, 'utf-8')); - const bootNodes = spec.bootNodes?.length ?? 0; - spec.bootNodes = []; + const sourceBootNodes = spec.bootNodes?.length ?? 0; + spec.bootNodes = chain.paraId === null ? [forkRelayBootnode()] : []; + + // The fork id is what takes it off that DHT. Protocol names are built from the genesis hash + // plus the fork id: `block_announces_protocol_name` in `sync/src/engine.rs`, + // `kademlia_protocol_name` in `network/src/discovery.rs`. A fork keeps the source's genesis + // hash, so without a fork id its names are byte-identical to the source's and it can still + // reach the source's nodes. + // + // Only the shipped copy gets one, because the bite has to reach the real network to + // warp-sync from it. Every node in the spawned fork must then carry the same value, or the + // fork splits into groups that cannot see each other. + // + // The key is `forkId`, not `fork_id`: a chain spec is camelCase, and an unrecognised key is + // ignored rather than rejected. + spec.forkId = `ppn-fork-${net.name}`; + const shipped = path.join(out, 'specs', `${chain.spec}.json`); fs.writeFileSync(shipped, JSON.stringify(spec)); const kb = Math.round(fs.statSync(shipped).size / 1024); - console.log(` ${chain.spec}.json (${kb}K, ${bootNodes} bootNodes stripped)`); + console.log( + ` ${chain.spec}.json (${kb}K, ${sourceBootNodes} source bootNodes dropped, ` + + `${spec.bootNodes.length} added, forkId ${spec.forkId})` + ); } export async function run(args: string[], opts: BiteOptions = {}): Promise { diff --git a/packages/cli/src/commands/fetch.ts b/packages/cli/src/commands/fetch.ts index c5b3563..f16ccf3 100644 --- a/packages/cli/src/commands/fetch.ts +++ b/packages/cli/src/commands/fetch.ts @@ -354,13 +354,20 @@ export async function run(args: string[], opts: FetchOptions = {}): Promise dub-0.3.0-. - const asset = `${IDENTITY_BINARY}-${identityTag.replace(/^v/, '')}-${versions.DUB_NETWORK}-${plat.triple}.tar.gz`; + // The asset embeds the version without its leading `v`, and since v0.6.0 the build it was + // compiled for: tag v0.6.0 -> dub-0.6.0-polkadot-. The People runtime is fixed at + // build time, so the two builds are different binaries sharing one name, and net.dubBuild + // is what decides which this network needs. + const dubBuild = net.dubBuild ?? 'testnet'; + const asset = `${IDENTITY_BINARY}-${identityTag.replace(/^v/, '')}-${dubBuild}-${plat.triple}.tar.gz`; const archivePath = path.join(sharedDest, asset); // Stamped whatever happens below: writeProvenance drops entries whose file is absent, so // a failed download leaves no stamp rather than a false one. + // The build belongs in the stamp, not only the tag: bin/dub is shared across networks, so + // without it a polkadot binary left by the fork reads as current when previewnet fetches, + // and the wrong runtime's metadata is used with nothing on screen to say so. const haveDub = stamp({ kind: 'toolchain', name: IDENTITY_BINARY, repo: versions.DUB_REPO, - pinned: identityTag, resolved: identityRelease.tag, + pinned: `${identityTag} ${dubBuild}`, resolved: `${identityRelease.tag} ${dubBuild}`, file: path.join(sharedDest, IDENTITY_BINARY) }); if (haveDub) current(IDENTITY_BINARY); else if (await downloadAsset(identityRelease, asset, archivePath, token)) { @@ -371,10 +378,16 @@ export async function run(args: string[], opts: FetchOptions = {}): Promise { }, seeded.relay ); + const attestationFor = (key: string) => { + const at = NETWORK.attestation; + const pallet = at?.on?.[key]; + return at && pallet ? { attester: at.attester, count: at.count, pallet } : undefined; + }; for (const p of PARACHAINS) { await paraOverrides( p.paraId, @@ -99,6 +104,8 @@ export async function run(args: string[]): Promise { NETWORK.bite.sharedRelay, seeded[p.key], p.aura, + p.bulletinAuthorizer, + attestationFor(p.key), topology?.collators[p.key] ?? 1 ); } @@ -124,6 +131,8 @@ export async function run(args: string[]): Promise { bundleDir: path.resolve(bundleDir), enableHop: (process.env.ENABLE_HOP || 'true').toLowerCase() !== 'false', binDir: bundleNet === 'previewnet' ? path.join(WS, 'bin') : path.join(WS, 'bin', bundleNet), + // Resuming runs on the database already in the data dir, so there is nothing to restore. + restoreSnapshots: !rest.includes('--resume'), }); fs.writeFileSync(outFile, toml); const rpcPorts = [...toml.matchAll(/^rpc_port = (\d+)$/gm)].map((m) => m[1]); diff --git a/packages/cli/src/commands/service.ts b/packages/cli/src/commands/service.ts index c462997..a40e549 100644 --- a/packages/cli/src/commands/service.ts +++ b/packages/cli/src/commands/service.ts @@ -1061,7 +1061,7 @@ async function enactUpgrades(ctx: ServiceContext, _deps: ServiceDeps = {}): Prom await waitSeconds('the chains to start authoring', 60); const { run: upgrade } = await import('./upgrade.js'); - for (const chain of chains) { + const enactOne = async (chain: string): Promise => { // Retries cover the chain not answering yet, not the enactment wait, which is inside // each attempt — so the budget must exceed one full attempt. const deadline = Date.now() + 3 * 3_600_000; @@ -1069,9 +1069,11 @@ async function enactUpgrades(ctx: ServiceContext, _deps: ServiceDeps = {}): Prom try { console.log(`enact-upgrades: ${chain} (attempt ${attempt})`); // A parachain enacts on the relay's go-ahead, validation_upgrade_delay relay blocks - // after the PVF pre-check: 600 blocks, an hour, on Polkadot. Two hours leaves room. + // after the PVF pre-check. A shared-relay bite sets that to 30 blocks; a bundle bitten + // before it did carries the live value — 600 on Polkadot, an hour — so the wait stays + // generous. await upgrade([chain], { skipFunding: true, enactTimeoutMs: 2 * 3_600_000 }); - break; + return; } catch (err) { const msg = err instanceof Error ? err.message : String(err); if (Date.now() > deadline) throw new Error(`enact-upgrades: ${chain} not enacted within 3 hours — ${msg}`); @@ -1079,6 +1081,11 @@ async function enactUpgrades(ctx: ServiceContext, _deps: ServiceDeps = {}): Prom await sleep(30); } } - } + }; + // Every parachain at once: each apply is its own transaction on its own chain, and the + // go-ahead delay runs per parachain, so serial would multiply it by their number. The relay + // last and alone, because its upgrade changes what the parachains are validated against. + await Promise.all(chains.filter((c) => c !== 'relay').map(enactOne)); + if (chains.includes('relay')) await enactOne('relay'); console.log('enact-upgrades: done'); } diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index ce4f1a5..c0fcc64 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -451,6 +451,40 @@ export function forkDataVerdict(dataDir: string, network: string, manifestPath: return { action: 'resume', reason: `continuing the fork spawned ${stamp.spawnedAt} from the bite of ${bittenAt}` }; } +/** + * Settle a fork's data directory, then write the config zombie-cli spawns from. + * + * zombienet unpacks a node's db_snapshot over its base path on every spawn, whether or not a + * database is already there. So a resume regenerates the bundle's config without snapshots + * before the spawn config is copied from it; otherwise the resumed fork rewinds to the bite + * block. Only the generated config is regenerated: a --toml the caller supplied is theirs. + * See docs/FORK.md. + */ +export function prepareSpawn( + sourceToml: string, + fork: { dataDir: string; network: string; forkDir: string } | null, + facts: { workspace: string; network: string; dataDir: string }, + regenerate: (forkDir: string, outFile: string) => void = (forkDir, outFile) => + run(process.execPath, [process.argv[1], 'fork', 'toml', forkDir, outFile, '--resume']) +): string { + if (fork) { + const verdict = forkDataVerdict(fork.dataDir, fork.network, path.join(fork.forkDir, 'manifest.json')); + if (verdict.action === 'wipe') { + console.log(`fork mode: wiping ${fork.dataDir} — ${verdict.reason}`); + fs.rmSync(fork.dataDir, { recursive: true, force: true }); + } else if (verdict.action === 'resume') { + console.log(`fork mode: resuming ${fork.dataDir} — ${verdict.reason} (--clean restarts from the bite block)`); + const generated = path.join(fork.forkDir, 'fork.toml'); + if (path.resolve(sourceToml) === path.resolve(generated)) regenerate(fork.forkDir, generated); + } + } + // zombie-cli hands custom processes no environment but what their own blocks say, so this + // run's workspace, network and data directory go into each block (lib/spawn-env.ts). + const tomlFile = spawnTomlFor(sourceToml); + fs.writeFileSync(tomlFile, withCustomProcessEnv(fs.readFileSync(sourceToml, 'utf-8'), customProcessEnv(facts))); + return tomlFile; +} + /** * Is there a bundle here worth spawning from — and if not, clear what is. * @@ -555,27 +589,13 @@ export async function start(args: string[], opts: StartOptions = {}): Promise + (await s.call(ethUrl, 'eth_getCode', [address, 'latest'])) !== '0x'; + +/** + * The fork's dotNS registry: its source's when the bite copied it, otherwise the one the + * DotnsGateway dispatcher on Asset Hub reports, for a dotNS deployed on the fork after the + * bite. Undefined when there is neither. + */ +async function forkRegistry( + s: Settings, + sourceRegistry: string, + ethUrl: string, + assetHubUrl: string | undefined +): Promise { + if (await hasCode(s, ethUrl, sourceRegistry)) return sourceRegistry.toLowerCase(); + if (!assetHubUrl) return undefined; + const dispatcher = await s.call(assetHubUrl, 'state_getStorage', [DISPATCHER_ADDRESS_KEY]); + if (!dispatcher) return undefined; + // eth-rpc just answered, so a failure here is a dispatcher without `protocolRegistry()`. + const word = await s.call(ethUrl, 'eth_call', [{ to: dispatcher, data: PROTOCOL_REGISTRY }, 'latest']).catch(() => '0x'); + return addressIn(word); +} /** * A fork's dotNS addresses: each key its source's descriptor lists, with the address the fork's * own registry returns for it, since a registry rewired after the bite differs from the source. - * A key the registry does not hold, such as a release's Multicall3, keeps the source's address. - * Incomplete while the source descriptor or the fork's eth-rpc does not answer. + * A key the registry does not hold, such as a release's Multicall3, keeps the source's address + * when that has code on the fork, and is left out otherwise. Incomplete while the source + * descriptor or the fork's nodes do not answer. */ async function forkDotns( s: Settings, source: Pointer, - ethUrl: string + ethUrl: string, + assetHubUrl: string | undefined ): Promise<{ dotns?: Record; complete: boolean }> { let listed: Record | undefined; try { @@ -150,22 +184,22 @@ async function forkDotns( } catch { return { complete: false }; } - const registry = listed?.protocolRegistry; - if (!listed || !registry) return { complete: true }; + if (!listed?.protocolRegistry) return { complete: true }; - const dotns: Record = {}; - for (const [key, address] of Object.entries(listed)) { - const data = REGISTRY_GET + Buffer.from(key).toString('hex').padEnd(64, '0'); - let word: string; - try { - word = await s.call(ethUrl, 'eth_call', [{ to: registry, data }, 'latest']); - } catch { - return { complete: false }; + try { + const registry = await forkRegistry(s, listed.protocolRegistry, ethUrl, assetHubUrl); + if (!registry) return { complete: true }; + const dotns: Record = {}; + for (const [key, address] of Object.entries(listed)) { + const data = REGISTRY_GET + Buffer.from(key).toString('hex').padEnd(64, '0'); + const registered = addressIn(await s.call(ethUrl, 'eth_call', [{ to: registry, data }, 'latest'])); + if (registered) dotns[key] = registered; + else if (await hasCode(s, ethUrl, address)) dotns[key] = address.toLowerCase(); } - const registered = `0x${word.slice(-40)}`.toLowerCase(); - dotns[key] = /^0x0{40}$/.test(registered) ? address.toLowerCase() : registered; + return { dotns, complete: true }; + } catch { + return { complete: false }; } - return { dotns, complete: true }; } /** @@ -183,8 +217,9 @@ export async function writeStack( const bite = forked ? spawn.bite : null; const specs = chainSpecNames(s.dataDir, model.chains); + const nodes = chainNodes(model.chains); const chains: StackFacts['chains'] = {}; - for (const [key, node] of chainNodes(model.chains)) { + for (const [key, node] of nodes) { // The source's block, as the bite recorded it: the fork's own block at that height differs. const id = node.paraId === null ? 'relay' : String(node.paraId); const number = bite?.blocks[id]; @@ -196,9 +231,10 @@ export async function writeStack( const chainId = eth && (await ethChainId(s, eth.directUrl)); // A genesis network runs the release's dotNS; a fork runs whatever its source had at the bite. const source = ctx.net.stack?.pointer; + const assetHub = nodes.find(([key]) => key === 'asset-hub')?.[1]; const dotnsFacts = forked && source && eth - ? await forkDotns(s, source, eth.directUrl) + ? await forkDotns(s, source, eth.directUrl, assetHub && asHttp(assetHub.directUrl)) : { dotns: forked ? undefined : dotnsFromContracts(s.dotnsContracts), complete: true }; const now = s.now(); const stack = trinityStackModel(ctx.net, model, { diff --git a/packages/cli/src/fork/chains.ts b/packages/cli/src/fork/chains.ts index 5b1a7dc..9ed6605 100644 --- a/packages/cli/src/fork/chains.ts +++ b/packages/cli/src/fork/chains.ts @@ -24,6 +24,8 @@ export interface ForkParachain extends ForkChain { paraId: number; /** Curve this chain's Aura keys are on, as the descriptor declares it. */ aura?: AuraScheme; + /** Account seeded into Bulletin's AllowedAuthorizers, which a fork has no genesis to provide. */ + bulletinAuthorizer?: string; } export function parachainsOf(net: NetworkDef): ForkParachain[] { @@ -33,6 +35,7 @@ export function parachainsOf(net: NetworkDef): ForkParachain[] { endpoint: p.rpc, paraId: p.paraId, aura: p.aura, + bulletinAuthorizer: p.bulletinAuthorizer, })); } diff --git a/packages/cli/src/fork/codec.ts b/packages/cli/src/fork/codec.ts index 81e4f39..f424db1 100644 --- a/packages/cli/src/fork/codec.ts +++ b/packages/cli/src/fork/codec.ts @@ -31,6 +31,13 @@ export function u32le(n: number): string { return b.toString('hex'); } +/** A u64 as little-endian hex. */ +export function u64le(n: bigint): string { + const b = Buffer.alloc(8); + b.writeBigUInt64LE(n); + return b.toString('hex'); +} + /** Paras::Heads(paraId) storage key, matching zombie-bite's para_head_key(). */ export function parasHeadKey(paraId: number): string { const le = u32le(paraId); diff --git a/packages/cli/src/fork/overrides.ts b/packages/cli/src/fork/overrides.ts index 567186a..7272471 100644 --- a/packages/cli/src/fork/overrides.ts +++ b/packages/cli/src/fork/overrides.ts @@ -9,12 +9,15 @@ import fs from 'node:fs'; import type { StorageIndex } from './rpc.js'; import { rpc, storageIndex } from './rpc.js'; +import { hexToU8a, u8aToHex } from '@polkadot/util'; import { keyOf } from './codec.js'; import { channelsTouching, coreDescriptorsValue, coresFor, dmpWipes, + FORK_VALIDATION_UPGRADE_COOLDOWN, + FORK_VALIDATION_UPGRADE_DELAY, hrmpChannelKey, hrmpContentsWipes, hrmpParaKey, @@ -25,9 +28,11 @@ import { type PlannedPara, } from './shared-relay.js'; import { + attestationAllowanceInjects, collatorKeys, devValidators, paraCandidates, + bulletinAuthorizerInjects, paraInjects, relayCandidates, relayInjects, @@ -73,8 +78,8 @@ export function verify( continue; } try { - const decoded = reg.createType(reg.createLookupType(info.plain), '0x' + value); - if (decoded.toHex().slice(2) !== value) throw new Error('round-trip mismatch'); + const decoded = reg.createType(reg.createLookupType(info.plain), hexToU8a('0x' + value)); + if (scaleHex(decoded) !== value) throw new Error('round-trip mismatch'); kept[key] = value; } catch (e) { failures.push(`${info.label}: ${(e as Error).message}`); @@ -129,8 +134,8 @@ export function verifyInjects( continue; } try { - const decoded = reg.createType(reg.createLookupType(type), '0x' + value); - if (decoded.toHex().slice(2) !== value) throw new Error('round-trip mismatch'); + const decoded = reg.createType(reg.createLookupType(type), hexToU8a('0x' + value)); + if (scaleHex(decoded) !== value) throw new Error('round-trip mismatch'); kept[key] = value; } catch (e) { failures.push(`${info.label}: ${(e as Error).message}`); @@ -162,6 +167,18 @@ function write(outFile: string, file: OverrideFile, index?: StorageIndex): void ); } +/** + * SCALE bytes of a codec, as hex without the 0x. + * + * polkadot-js overrides `toHex` on its integer types to emit big-endian, so it does not give + * SCALE bytes. Decoding has the same trap, since a multi-byte integer read from a hex string + * comes back byte-reversed. Every encode goes through this function, including the struct ones + * `toHex` would get right. + */ +export function scaleHex(codec: { toU8a(): Uint8Array }): string { + return u8aToHex(codec.toU8a()).slice(2); +} + /** * Encode a value through the chain's own metadata. * @@ -172,63 +189,83 @@ function encode(index: StorageIndex, pallet: string, item: string, value: unknow const entry = index.byKey.get(keyOf(pallet, item)); if (!entry?.plain) throw new Error(`${pallet}::${item} is not a plain storage value here`); const type = index.reg.createLookupType(entry.plain); - return index.reg.createType(type, value).toHex().slice(2); + return scaleHex(index.reg.createType(type, value)); +} + +/** The HostConfiguration fields a shared-relay bite changes, and what they were. */ +interface HostConfigPatch { + value: string; + before: { numCores: number; validationUpgradeDelay: number; validationUpgradeCooldown: number }; } /** - * Set `scheduler_params.num_cores` in production's own HostConfiguration. + * Change three numbers in production's own HostConfiguration: `scheduler_params.num_cores`, + * `validation_upgrade_delay` and `validation_upgrade_cooldown`. * * Rebuilt from the decoded value's own fields — Codec instances, not JSON. Going through * `toJSON()` and back produces bytes that mean the same thing but are not the ones the chain * wrote, and `verify()` rightly refuses them: it requires a value to survive decode → encode * unchanged. Passing the untouched fields through as codecs keeps them byte-for-byte, so the - * only bytes that move are the ones for the number being changed. + * only bytes that move are the ones for the numbers being changed. * * That matters beyond tidiness. This struct also carries `executor_params`, whose * `EnabledHostFunction(EccRfc163)` the relay's validators need to accept People's PVFs, plus * the async-backing values production tuned. Replacing the whole value would cost all of it to - * fix one number, which is why the bite leaves this key alone on previewnet entirely. + * fix three numbers, which is why the bite leaves this key alone on previewnet entirely. */ -function patchNumCores( +function patchHostConfig( index: StorageIndex, liveHex: string, - numCores: number -): { value: string; before: number } { + want: { numCores: number; validationUpgradeDelay: number; validationUpgradeCooldown: number } +): HostConfigPatch { const entry = index.byKey.get(keyOf('Configuration', 'ActiveConfig')); if (!entry?.plain) throw new Error('Configuration::ActiveConfig is not a plain value here'); const type = index.reg.createLookupType(entry.plain); - const decoded = index.reg.createType(type, liveHex) as any; + const decoded = index.reg.createType(type, hexToU8a(liveHex)) as any; const fields = Object.fromEntries([...decoded.entries()]); const params = fields.schedulerParams; if (!params?.entries) { throw new Error('HostConfiguration has no schedulerParams — this relay runtime is not what the bite expects'); } const paramFields = Object.fromEntries([...params.entries()]); - const before = Number(paramFields.numCores?.toString()); - if (!Number.isFinite(before)) { - throw new Error('schedulerParams has no numCores — this relay runtime is not what the bite expects'); - } + const num = (v: unknown, name: string): number => { + const n = Number(v?.toString()); + if (!Number.isFinite(n)) throw new Error(`HostConfiguration has no ${name} — this relay runtime is not what the bite expects`); + return n; + }; + const before = { + numCores: num(paramFields.numCores, 'schedulerParams.numCores'), + validationUpgradeDelay: num(fields.validationUpgradeDelay, 'validationUpgradeDelay'), + validationUpgradeCooldown: num(fields.validationUpgradeCooldown, 'validationUpgradeCooldown'), + }; - const patchedParams = new (params.constructor as any)(index.reg, { - ...paramFields, - numCores: index.reg.createType('u32', numCores), - }); - const value = index.reg - .createType(type, { ...fields, schedulerParams: patchedParams }) - .toHex() - .slice(2); + const u32 = (n: number) => index.reg.createType('u32', n); + const patchedParams = new (params.constructor as any)(index.reg, { ...paramFields, numCores: u32(want.numCores) }); + const value = scaleHex( + index.reg.createType(type, { + ...fields, + schedulerParams: patchedParams, + validationUpgradeDelay: u32(want.validationUpgradeDelay), + validationUpgradeCooldown: u32(want.validationUpgradeCooldown), + }) + ); // Guards, because this rebuilds a struct whose layout is the runtime's, not ours: the result - // must differ from production only inside that one u32, and must read back as asked. + // must differ from production only inside those three u32s, and must read back as asked. const bytes = (hex: string) => hex.match(/../g) ?? []; const changed = bytes(value).filter((b, i) => b !== bytes(liveHex.slice(2))[i]).length; - if (changed === 0 || changed > 4) { - throw new Error(`patching num_cores changed ${changed} bytes of HostConfiguration; expected 1-4`); + if (changed === 0 || changed > 12) { + throw new Error(`patching HostConfiguration changed ${changed} bytes; expected 1-12`); } - const readBack = (index.reg.createType(type, '0x' + value).toJSON() as any).schedulerParams?.numCores; - if (readBack !== numCores) { - throw new Error(`num_cores read back as ${readBack}, not ${numCores}`); + const back = index.reg.createType(type, hexToU8a('0x' + value)).toJSON() as any; + const got = { + numCores: back.schedulerParams?.numCores, + validationUpgradeDelay: back.validationUpgradeDelay, + validationUpgradeCooldown: back.validationUpgradeCooldown, + }; + for (const k of Object.keys(want) as (keyof typeof want)[]) { + if (got[k] !== want[k]) throw new Error(`${k} read back as ${got[k]}, not ${want[k]}`); } return { value, before }; } @@ -263,11 +300,18 @@ async function coreLayoutCandidates( '0x' + keyOf('Configuration', 'ActiveConfig'), ]); if (!liveConfig) throw new Error('the relay has no Configuration::ActiveConfig to patch'); - const config = patchNumCores(index, liveConfig, plan.length); + const want = { + numCores: plan.length, + validationUpgradeDelay: FORK_VALIDATION_UPGRADE_DELAY, + validationUpgradeCooldown: FORK_VALIDATION_UPGRADE_COOLDOWN, + }; + const config = patchHostConfig(index, liveConfig, want); console.log( ` cores: ${paras.map((p) => `${p.key}=${coresFor(p.key, cores)}`).join(', ')} on cores 0-${plan.length - 1}, ` + - `num_cores ${config.before} -> ${plan.length}, ${validators} validators` + `num_cores ${config.before.numCores} -> ${want.numCores}, ${validators} validators, ` + + `validation_upgrade_delay ${config.before.validationUpgradeDelay} -> ${want.validationUpgradeDelay}, ` + + `cooldown ${config.before.validationUpgradeCooldown} -> ${want.validationUpgradeCooldown}` ); return { @@ -320,7 +364,7 @@ async function hrmpResets(index: StorageIndex, relayUrl: string, paraIds: number const read = (key: string) => rpc<`0x${string}` | null>(relayUrl, 'state_getStorage', ['0x' + key]); const paraList = (hex: `0x${string}` | null): number[] => - hex ? (index.reg.createType('Vec', hex).toJSON() as number[]) : []; + hex ? (index.reg.createType('Vec', hexToU8a(hex)).toJSON() as number[]) : []; const ingress = new Map(); const egress = new Map(); @@ -350,27 +394,26 @@ async function hrmpResets(index: StorageIndex, relayUrl: string, paraIds: number /** * A live `HrmpChannel` with `mqc_head` cleared and `msg_count`/`total_size` zeroed. Rebuilt - * from the decoded value's own codec fields for the same reason patchNumCores() is: the + * from the decoded value's own codec fields for the same reason patchHostConfig() is: the * untouched fields — capacities, deposits — must come out byte-for-byte as they went in. */ function resetHrmpChannel(index: StorageIndex, liveHex: string): string { const entry = index.byKey.get(keyOf('Hrmp', 'HrmpChannels')); if (!entry?.mapValue) throw new Error('Hrmp::HrmpChannels is not a map here'); const type = index.reg.createLookupType(entry.mapValue); - const decoded = index.reg.createType(type, liveHex) as any; + const decoded = index.reg.createType(type, hexToU8a(liveHex)) as any; const fields = Object.fromEntries([...decoded.entries()]); for (const f of ['mqcHead', 'msgCount', 'totalSize']) { if (!(f in fields)) throw new Error(`HrmpChannel has no ${f} field — this relay runtime is not what the reset expects`); } - return index.reg - .createType(type, { + return scaleHex( + index.reg.createType(type, { ...fields, mqcHead: index.reg.createType('Option', null), msgCount: index.reg.createType('u32', 0), totalSize: index.reg.createType('u32', 0), }) - .toHex() - .slice(2); + ); } /** A runtime to authorize at import, for a fork that has no sudo to authorize one later. */ @@ -438,6 +481,8 @@ export async function paraOverrides( sharedRelay = false, upgrade?: SeededUpgrade, scheme: AuraScheme = 'sr25519', + bulletinAuthorizer?: string, + attestation?: { attester: string; count: number; pallet: 'PeopleLite' | 'DotnsGateway' }, collatorCount = 1 ): Promise { const index = await storageIndex(paraUrl); @@ -467,6 +512,13 @@ export async function paraOverrides( // Parachains of a shared relay are live public chains: no dev account holds // funds there, so sudo is endowed at import (see sudoEndowInjects). ...(sharedRelay ? sudoEndowInjects() : {}), + // A bitten Bulletin has no authorizer: the runtime seeds one at genesis and a fork has none. + ...(bulletinAuthorizer && index.pallets.has('TransactionStorage') + ? bulletinAuthorizerInjects(bulletinAuthorizer) + : {}), + ...(attestation + ? attestationAllowanceInjects(attestation.pallet, attestation.attester, attestation.count) + : {}), ...seededUpgradeInject(index, upgrade), }; write(outFile, { overrides, injects }, index); diff --git a/packages/cli/src/fork/shared-relay.ts b/packages/cli/src/fork/shared-relay.ts index c3fc89d..a105b78 100644 --- a/packages/cli/src/fork/shared-relay.ts +++ b/packages/cli/src/fork/shared-relay.ts @@ -236,6 +236,18 @@ export function transactionStorageWipes(): Record { */ export const FULL_CORE = 57600; +/** + * How long a parachain waits for the relay's go-ahead after its new code passed pre-check, + * and how long before the same parachain may upgrade again — both in relay blocks, both in + * `Configuration::ActiveConfig`, both tuned on live networks for a caution a fork has no use + * for: Polkadot's 600 and 14400 mean an hour per upgrade and one upgrade per parachain per + * day. The fork exists to enact runtimes under test, several a day, so these drop to three + * minutes and six. Still above `minimum_validation_upgrade_delay` (20), which the runtime's + * own consistency check enforces; the pre-check and go-ahead path is exercised unchanged. + */ +export const FORK_VALIDATION_UPGRADE_DELAY = 30; +export const FORK_VALIDATION_UPGRADE_COOLDOWN = 60; + export function coreDescriptorsValue(plan: CoreAssignment[]): Record { return Object.fromEntries( plan.map(({ core, paraId }) => [ diff --git a/packages/cli/src/fork/validators.ts b/packages/cli/src/fork/validators.ts index 0a0c58a..84350da 100644 --- a/packages/cli/src/fork/validators.ts +++ b/packages/cli/src/fork/validators.ts @@ -9,8 +9,8 @@ // additionally decode-verify each value against the live runtime before writing it. import { Keyring } from '@polkadot/keyring'; -import { cryptoWaitReady } from '@polkadot/util-crypto'; -import { blake2128Concat, compactLen, keyOf, twox64Concat, u128le, u32le } from './codec.js'; +import { cryptoWaitReady, decodeAddress } from '@polkadot/util-crypto'; +import { blake2128Concat, compactLen, keyOf, twox64Concat, u128le, u32le, u64le } from './codec.js'; import { collatorNodeName, relayNodeName, type AuraScheme } from '@parity/ppn-network-config'; export const ALICE_SR = 'd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d'; @@ -156,6 +156,82 @@ export function sudoEndowInjects(): Record { return { [keyOf('System', 'Account') + blake2128Concat(alice)]: info }; } +/** + * Let one account authorize Bulletin storage, written at import. + * + * `TransactionStorage::authorize_account` takes `Authorizer`, which is Root, a sibling parachain + * in `AllowedParachainIds`, or an account in `AllowedAuthorizers`. On a fork of a chain without + * Sudo the first is unreachable and the other two are empty: live Polkadot Bulletin has no + * entry in either, because on a production chain authorizers arrive by governance. So nothing + * can store anything, and `bulletinAutoAuthorize` fails `BadSigner`. + * + * previewnet does not hit this: the bulletin runtime's own genesis preset seeds + * `Sr25519Keyring::Eve` with 100_000 transactions and 100 GiB. A fork has no genesis, so the + * same entry is written here instead, with the budget copied from that preset rather than + * invented. `valid_until: None` and `feeless: true` are what `BuildGenesisConfig` sets. + * + * This writes the map entry alone. Both paths that create one in the pallet pair the insert + * with `inc_authorizer_providers`, which keeps a `feeless` authorizer with no balance from + * being reaped between dispatches. So pass an account that holds a balance, which provides + * for itself. The reference cannot be supplied later either: both paths guard the increment + * on the entry being new, so adding the authorizer again through the pallet is a no-op. + * + * This inject is type-checkable: `TransactionStorage` exists on the chain being bitten, so + * `verifyInjects` decodes the value against real metadata. It does not check the key. + */ +export function bulletinAuthorizerInjects(accountId: string): Record { + const id = accountId.replace(/^0x/, '').toLowerCase(); + if (!/^[0-9a-f]{64}$/.test(id)) { + throw new Error(`bulletin authorizer must be a 32-byte account id, got "${accountId}"`); + } + const budget = + '01' + // quota: Some + u32le(100_000) + // transactions + u64le(100n * 1024n * 1024n * 1024n) + // bytes: 100 GiB + '00' + // valid_until: None + '01'; // feeless + return { [keyOf('TransactionStorage', 'AllowedAuthorizers') + blake2128Concat(id)]: budget }; +} + +/** + * Grant an attestation allowance, as `increase_attestation_allowance` would have. + * + * Two quotas share the name. `PeopleLite`'s is how many people a verifier may attest, spent by + * `attest`; `DotnsGateway`'s is how many names an attester may reserve, spent by `reserve_name`. + * Without the first nobody becomes a lite person, without the second nobody who did can take a + * username, and the backend needs both. + * + * `PeopleLite`'s manager is `EnsureRoot` and `DotnsGateway`'s is `RootOrWhitelist`, so on a fork + * of a chain whose Root is a referendum neither can be granted after the spawn. + * + * Both pallets store it as `StorageMap<_, Blake2_128Concat, AccountId, u32, ValueQuery>`, and + * both `increase_attestation_allowance` implementations only saturating-add, so the count is + * the whole state. + */ +export function attestationAllowanceInjects( + pallet: 'PeopleLite' | 'DotnsGateway', + attester: string, + count: number +): Record { + // One descriptor object feeds both calls, so neither message names the pallet: the count and + // the attester are the same on each, and naming one would imply a figure that is per-pallet. + if (!Number.isInteger(count) || count <= 0) { + throw new Error(`attestation allowance must be a positive integer, got ${count}`); + } + let id: string; + try { + id = Buffer.from(decodeAddress(attester)).toString('hex'); + } catch (cause) { + // decodeAddress reports the encoding it failed to read, which says nothing about where the + // value came from. This is the only place that knows it is an attester. + throw new Error(`attester "${attester}" is not an account: ${(cause as Error).message}`); + } + if (id.length !== 64) { + throw new Error(`attester must be a 32-byte account, got ${id.length / 2} bytes`); + } + return { [keyOf(pallet, 'AttestationAllowance') + blake2128Concat(id)]: u32le(count) }; +} + /** * The collator keys for a parachain, one per collator node. * diff --git a/packages/cli/tests/custom-process-env.test.ts b/packages/cli/tests/custom-process-env.test.ts index 8da27b5..2ce742c 100644 --- a/packages/cli/tests/custom-process-env.test.ts +++ b/packages/cli/tests/custom-process-env.test.ts @@ -47,7 +47,7 @@ describe('custom process environment', () => { it('reaches the identity backend\'s blocks and the TURN relay\'s, the last block in a fork config', () => { const fork = '[settings]\ntimeout = 1\n\n[relaychain]\nchain = "x"\n' + - dubCustomProcesses({ postgres: 5433, people: 10010, assetHub: 10020, gateway: 8092 }, undefined, '/opt/ppn/scripts', true); + dubCustomProcesses({ postgres: 5433, people: 10010, assetHub: 10020, gateway: 8092 }, undefined, '/opt/ppn/scripts', 'testnet', true); const parsed = toml.parse(withCustomProcessEnv(fork, facts)) as Config; assert.deepEqual(parsed.custom_processes.map((p) => p.name), [ 'dub-postgres', 'dub-api', 'device-attestation-chain-writer', 'registration-queue', 'invite-tickets-pool', 'turn', diff --git a/packages/cli/tests/fork-codec.test.ts b/packages/cli/tests/fork-codec.test.ts index aeae255..e379740 100644 --- a/packages/cli/tests/fork-codec.test.ts +++ b/packages/cli/tests/fork-codec.test.ts @@ -12,6 +12,10 @@ import { u32le, } from '../src/fork/codec.js'; import { CHAINS, PARACHAINS } from '../src/fork/chains.js'; +import { scaleHex } from '../src/fork/overrides.js'; +import { TypeRegistry } from '@polkadot/types'; +import { hexToU8a } from '@polkadot/util'; +import { blake2128Concat } from '../src/fork/codec.js'; import { VALID_PARACHAINS, paraIds } from '@parity/ppn-network-config'; describe('keyOf', () => { @@ -133,3 +137,26 @@ describe('CHAINS', () => { assert.equal(CHAINS.find((c) => c.key === 'relay')?.spec, 'paseo'); }); }); + +describe('scaleHex', () => { + // The second assertion pins polkadot-js rather than this code, on purpose: it is the premise + // `scaleHex` exists for. A failure there means `toHex` was fixed upstream and `scaleHex` is no + // longer needed. + it('encodes an integer little-endian, where toHex() does not', () => { + const value = new TypeRegistry().createType('u32', 50_000_413); + assert.equal(scaleHex(value), '1df2fa02'); + assert.equal(value.toHex(), '0x02faf21d'); + }); + + // Decoding has the same problem, and the second assertion pins polkadot-js for the same reason. + it('decodes SCALE bytes, where a hex string is read as a big-endian number', () => { + const reg = new TypeRegistry(); + assert.equal(reg.createType('u32', hexToU8a('0x01000000')).toString(), '1'); + assert.equal(reg.createType('u32', '0x01000000').toString(), '16777216'); + }); + + it('gives the storage key the runtime reads back for a u32-keyed asset', () => { + const encoded = scaleHex(new TypeRegistry().createType('u32', 50_000_413)); + assert.equal(blake2128Concat(encoded), 'f135a21cac0a18d1229a791a3532ba5a1df2fa02'); + }); +}); diff --git a/packages/cli/tests/fork-validators.test.ts b/packages/cli/tests/fork-validators.test.ts index b28dc77..caeea81 100644 --- a/packages/cli/tests/fork-validators.test.ts +++ b/packages/cli/tests/fork-validators.test.ts @@ -3,6 +3,7 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; +import { u8aToHex } from '@polkadot/util'; import fs from 'node:fs'; import path from 'node:path'; import { @@ -16,11 +17,13 @@ import { relayCandidates, relayInjects, sessionKeys, + attestationAllowanceInjects, authorizedUpgradeCandidate, + bulletinAuthorizerInjects, type DevValidator, } from '../src/fork/validators.js'; import { verify, verifyInjects } from '../src/fork/overrides.js'; -import { compactLen, keyOf, u32le } from '../src/fork/codec.js'; +import { blake2128Concat, compactLen, keyOf, u32le } from '../src/fork/codec.js'; import { PARACHAINS } from '../src/fork/chains.js'; // Captured from the override files of a bite that produced a verified working fork, before @@ -254,8 +257,11 @@ describe('verify', () => { const PLAIN = 1; const registry = { createLookupType: (id: number) => `Type${id}`, - createType: (_type: string, hex: string) => ({ - toHex: () => (hex === '0xbad' ? '0xdifferent' : hex), + // The code compares SCALE bytes, so the stub takes a Uint8Array and returns `toU8a`. + // It returns the bytes it was given, except for the candidate 'bad', which comes back + // as something else so the round-trip check fails and the failure path can be tested. + createType: (_type: string, raw: Uint8Array) => ({ + toU8a: () => (u8aToHex(raw).endsWith('bad') ? new Uint8Array([0xde, 0xad]) : raw), }), } as never; @@ -298,8 +304,11 @@ describe('verifyInjects', () => { const MAPVAL = 7; const registry = { createLookupType: (id: number) => `Type${id}`, - createType: (_type: string, hex: string) => ({ - toHex: () => (hex === '0xbad' ? '0xdifferent' : hex), + // The code compares SCALE bytes, so the stub takes a Uint8Array and returns `toU8a`. + // It returns the bytes it was given, except for the candidate 'bad', which comes back + // as something else so the round-trip check fails and the failure path can be tested. + createType: (_type: string, raw: Uint8Array) => ({ + toU8a: () => (u8aToHex(raw).endsWith('bad') ? new Uint8Array([0xde, 0xad]) : raw), }), } as never; @@ -422,3 +431,85 @@ describe('collator key curve', () => { assert.notEqual(await collatorKey(1000), await collatorKey(1000, 'ed25519')); }); }); + +// A bitten Bulletin has no authorizer at all: the runtime seeds one in its genesis preset and a +// fork has no genesis, while live Polkadot Bulletin's AllowedAuthorizers is empty because there +// authorizers arrive by governance. Without this every store fails BadSigner. +describe('the bulletin authorizer seed', () => { + const ALICE = '0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d'; + + it('writes into TransactionStorage::AllowedAuthorizers, blake2-128-concat keyed', () => { + const entry = Object.keys(bulletinAuthorizerInjects(ALICE))[0]; + assert.equal(entry.slice(0, 64), keyOf('TransactionStorage', 'AllowedAuthorizers')); + assert.ok(entry.endsWith(ALICE.slice(2))); + }); + + // AuthorizerBudget { quota: Some(Quota { transactions: u32, bytes: u64 }), valid_until: None, + // feeless: bool } — the budget the bulletin runtime's own genesis preset writes. + it('encodes the genesis presets budget: 100k transactions, 100 GiB, feeless, no expiry', () => { + const v = Object.values(bulletinAuthorizerInjects(ALICE))[0]; + assert.equal(v, '01' + 'a0860100' + '0000000019000000' + '00' + '01'); + assert.equal(v.length / 2, 15); + }); + + it('refuses anything that is not a 32-byte account id', () => { + for (const bad of ['0x1234', '', '0xd43593c7', 'nope']) { + assert.throws(() => bulletinAuthorizerInjects(bad), /32-byte account id/); + } + }); +}); + +// Two quotas share the `AttestationAllowance` name. `PeopleLite`'s counts how many people a +// verifier may attest, spent by `attest`; `DotnsGateway`'s how many names an attester may +// reserve, spent by `reserve_name`. The identity backend needs both, and on a fork neither can be +// granted after the spawn: `PeopleLite`'s manager is `EnsureRoot` and `DotnsGateway`'s is +// `RootOrWhitelist`. +describe('the attestation allowance seeds', () => { + const ALICE = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'; + // What polkadot.json grants. Multi-byte, so the literal below catches a byte-order slip. + const COUNT = 1000000; + + for (const pallet of ['PeopleLite', 'DotnsGateway'] as const) { + it(`writes one ${pallet} entry keyed by the attester`, () => { + const out = attestationAllowanceInjects(pallet, ALICE, COUNT); + assert.deepEqual(out, { + [keyOf(pallet, 'AttestationAllowance') + blake2128Concat(ALICE_SR)]: u32le(COUNT), + }); + }); + } + + // Both pallets hold it as a bare `u32` with `ValueQuery`, and both setters only + // saturating-add, so the count is the whole state the call produces. + it('encodes the count as a bare little-endian u32', () => { + const [value] = Object.values(attestationAllowanceInjects('PeopleLite', ALICE, COUNT)); + assert.equal(value, u32le(COUNT)); + assert.equal(value, '40420f00'); + }); + + it('accepts the raw account id as well as SS58', () => { + assert.deepEqual( + attestationAllowanceInjects('PeopleLite', '0x' + ALICE_SR, COUNT), + attestationAllowanceInjects('PeopleLite', ALICE, COUNT) + ); + }); + + it('refuses a count that would grant nothing', () => { + for (const bad of [0, -1, 1.5]) { + assert.throws(() => attestationAllowanceInjects('PeopleLite', ALICE, bad), /positive integer/); + } + }); + + it('refuses an attester that is not a 32-byte account', () => { + // A 20-byte address decodes, so it reaches the length check. Garbage does not: decodeAddress + // throws first, which is why that case cannot stand in for this one, and why the throw is + // wrapped in a message that names the field. + assert.throws( + () => attestationAllowanceInjects('PeopleLite', '0x' + 'ab'.repeat(20), COUNT), + /32-byte account, got 20 bytes/ + ); + assert.throws( + () => attestationAllowanceInjects('PeopleLite', 'not-an-account', COUNT), + /attester "not-an-account" is not an account: Decoding/ + ); + }); +}); diff --git a/packages/cli/tests/start-network.test.ts b/packages/cli/tests/start-network.test.ts index c753fef..8acf8cb 100644 --- a/packages/cli/tests/start-network.test.ts +++ b/packages/cli/tests/start-network.test.ts @@ -13,6 +13,7 @@ import { forkDirFor, checkPorts, forkDataVerdict, + prepareSpawn, assertBundleTopology, } from '../src/commands/start.js'; import { forkBundleName, forkBundleAsset } from '../src/lib/fork-bundle-name.js'; @@ -215,3 +216,70 @@ describe('forkDataVerdict', () => { assert.equal(forkDataVerdict(unstamped.dataDir, 'polkadot', unstamped.manifest).action, 'wipe'); }); }); + +// zombie-cli spawns from a copy of the bundle's config, and zombienet unpacks every db_snapshot +// it is given over the database already there. So a resume regenerates the bundle's config +// without snapshots before the copy is made. +describe('prepareSpawn', () => { + const BITTEN = '2026-09-01T18:30:00.000Z'; + const WITH = 'default_db_snapshot = "/b/snapshots/relay.tgz"\n\n[[parachains.collators]]\ndb_snapshot = "/b/snapshots/1000.tgz"\n'; + const WITHOUT = '\n[[parachains.collators]]\n'; + const setup = () => { + const forkDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ppn-spawn-')); + const dataDir = path.join(forkDir, 'data-fork-polkadot'); + const manifest = path.join(forkDir, 'manifest.json'); + fs.writeFileSync(manifest, JSON.stringify({ bittenAt: BITTEN })); + const generated = path.join(forkDir, 'fork.toml'); + fs.writeFileSync(generated, WITH); + return { forkDir, dataDir, manifest, generated }; + }; + const spawned = (dataDir: string, manifest: string, network = 'polkadot') => { + writeSpawnStamp(dataDir, { network, mode: 'fork', forkManifest: manifest, repoRoot: '/nowhere' }); + fs.mkdirSync(path.join(dataDir, 'alice', 'db'), { recursive: true }); + }; + // Stands in for `ppn fork toml --resume`, whose output fork-toml.test.ts covers. + const regenerated: string[] = []; + const regenerate = (_forkDir: string, outFile: string) => { + regenerated.push(outFile); + fs.writeFileSync(outFile, WITHOUT); + }; + const facts = (dataDir: string) => ({ workspace: '/ws', network: 'polkadot', dataDir }); + + it('spawns a resumed fork with no db_snapshot', () => { + const { forkDir, dataDir, manifest, generated } = setup(); + spawned(dataDir, manifest); + regenerated.length = 0; + const spawn = prepareSpawn(generated, { dataDir, network: 'polkadot', forkDir }, facts(dataDir), regenerate); + assert.deepEqual(regenerated, [generated]); + assert.doesNotMatch(fs.readFileSync(spawn, 'utf-8'), /db_snapshot/); + }); + + it('keeps the snapshots on a fresh start', () => { + const { forkDir, dataDir, generated } = setup(); + regenerated.length = 0; + const spawn = prepareSpawn(generated, { dataDir, network: 'polkadot', forkDir }, facts(dataDir), regenerate); + assert.deepEqual(regenerated, []); + assert.match(fs.readFileSync(spawn, 'utf-8'), /default_db_snapshot[\s\S]*\ndb_snapshot/); + }); + + it('keeps the snapshots when it wipes the data', () => { + const { forkDir, dataDir, manifest, generated } = setup(); + spawned(dataDir, manifest, 'kusama'); + regenerated.length = 0; + const spawn = prepareSpawn(generated, { dataDir, network: 'polkadot', forkDir }, facts(dataDir), regenerate); + assert.deepEqual(regenerated, []); + assert.equal(fs.existsSync(dataDir), false); + assert.match(fs.readFileSync(spawn, 'utf-8'), /default_db_snapshot[\s\S]*\ndb_snapshot/); + }); + + it('leaves a config the caller supplied as it is', () => { + const { forkDir, dataDir, manifest } = setup(); + spawned(dataDir, manifest); + const supplied = path.join(forkDir, 'mine.toml'); + fs.writeFileSync(supplied, WITH); + regenerated.length = 0; + const spawn = prepareSpawn(supplied, { dataDir, network: 'polkadot', forkDir }, facts(dataDir), regenerate); + assert.deepEqual(regenerated, []); + assert.match(fs.readFileSync(spawn, 'utf-8'), /db_snapshot/); + }); +}); diff --git a/packages/cli/tests/trinity-stack.test.ts b/packages/cli/tests/trinity-stack.test.ts index a088fc6..ecdc428 100644 --- a/packages/cli/tests/trinity-stack.test.ts +++ b/packages/cli/tests/trinity-stack.test.ts @@ -160,44 +160,85 @@ describe('writeStack — what it writes', () => { // The bite copies the source's registry, so the fork has the source's keys; a key rewired on // either side since then is what the fork's own registry now returns. - it("publishes its source's dotNS keys with the addresses the fork's registry returns", async () => { - const registry = '0xd19e3d0c97cf501125a04a97405e3e6592fa846e'; - const sourceDir = dataDir; + const sourceRegistry = '0xd19e3d0c97cf501125a04a97405e3e6592fa846e'; + const multicall3 = '0xb4468000abd87d3c56cbfbd153161223d7b109e5'; + /** A source descriptor listing three dotNS keys, and a fresh fork data directory to write into. */ + const forkOfSource = async (): Promise => { await writeStack( { net }, settings({ dotnsContracts: { - DotnsProtocolRegistry: registry, + DotnsProtocolRegistry: sourceRegistry, DotnsRegistrar: '0x4f06e818ba3d987704fd91cf3d868e4b019106ab', - Multicall3: '0xb4468000abd87d3c56cbfbd153161223d7b109e5', + Multicall3: multicall3, }, }) ); - const sourceDocument = fs.readFileSync(path.join(sourceDir, 'trinity-stack.json')); - + const sourceDocument = fs.readFileSync(path.join(dataDir, 'trinity-stack.json')); dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ppn-stack-')); writeSpecs('127.0.0.1'); stamp({ mode: 'fork', bite: bite({ hashes }) }); - const rewiredRegistrar = '0x' + '12'.repeat(20); - const forkRegistry: Record = { protocolRegistry: registry, registrar: rewiredRegistrar }; - const call: Settings['call'] = async (url, method, params = []) => { + return async (url) => + url === net.stack!.pointer!.url ? new Response(sourceDocument) : new Response(null, { status: 404 }); + }; + const word = (address: string) => '0x' + address.slice(2).padStart(64, '0'); + /** The fork's nodes: contracts with code, each registry's entries, and the dispatcher Asset Hub stores. */ + const forkChain = (contracts: string[], registries: Record>, dispatcher?: { address: string; registry: string }): Settings['call'] => + async (url, method, params = []) => { + if (method === 'eth_getCode') return (contracts.includes(params[0] as string) ? '0x6080' : '0x') as never; + if (method === 'state_getStorage') return (dispatcher?.address ?? null) as never; if (method !== 'eth_call') return fakeNodes(url, method, params); - const { data } = params[0] as { data: string }; + const { to, data } = params[0] as { to: string; data: string }; + if (to === dispatcher?.address) return word(dispatcher.registry) as never; + if (!registries[to]) return '0x' as never; const key = Buffer.from(data.slice(10), 'hex').toString().replace(/\0+$/, ''); - return ('0x' + (forkRegistry[key] ?? '0x' + '00'.repeat(20)).slice(2).padStart(64, '0')) as never; + return word(registries[to][key] ?? '0x' + '00'.repeat(20)) as never; }; - const fetch: Settings['fetch'] = async (url) => - url === net.stack!.pointer!.url ? new Response(sourceDocument) : new Response(null, { status: 404 }); + + it("publishes its source's dotNS keys with the addresses the fork's registry returns", async () => { + const fetch = await forkOfSource(); + const rewiredRegistrar = '0x' + '12'.repeat(20); + const call = forkChain([sourceRegistry, multicall3], { + [sourceRegistry]: { protocolRegistry: sourceRegistry, registrar: rewiredRegistrar }, + }); const { written, complete } = await writeStack({ net }, settings({ call, fetch })); assert.equal(complete, true); assert.deepEqual(open(read('trinity-stack.json'), written.pointer).descriptor.dotns, { - multicall3: '0xb4468000abd87d3c56cbfbd153161223d7b109e5', - protocolRegistry: registry, + multicall3, + protocolRegistry: sourceRegistry, registrar: rewiredRegistrar, }); }); + // dotNS deployed on the fork after the bite: the source's contracts do not exist here, and + // the registry is found through the dispatcher the chain's DotnsGateway calls into. + it("takes the registry from Asset Hub's dotNS dispatcher when the source's is not on the fork", async () => { + const fetch = await forkOfSource(); + const ownRegistry = '0x' + '34'.repeat(20); + const ownRegistrar = '0x' + '56'.repeat(20); + const call = forkChain( + [ownRegistry], + { [ownRegistry]: { protocolRegistry: ownRegistry, registrar: ownRegistrar } }, + { address: '0x' + '78'.repeat(20), registry: ownRegistry } + ); + + const { written, complete } = await writeStack({ net }, settings({ call, fetch })); + assert.equal(complete, true); + // Multicall3 is neither registered nor deployed at the source's address, so it is left out. + assert.deepEqual(open(read('trinity-stack.json'), written.pointer).descriptor.dotns, { + protocolRegistry: ownRegistry, + registrar: ownRegistrar, + }); + }); + + it("publishes no dotNS when neither the source's registry nor a dispatcher is on the fork", async () => { + const fetch = await forkOfSource(); + const { written, complete } = await writeStack({ net }, settings({ call: forkChain([], {}), fetch })); + assert.equal(complete, true); + assert.equal(open(read('trinity-stack.json'), written.pointer).descriptor.dotns, undefined); + }); + it('publishes a fork without dotNS while its source descriptor does not answer, and tries again', async () => { stamp({ mode: 'fork', bite: bite({ hashes }) }); const { written, complete } = await writeStack({ net }, settings()); diff --git a/packages/network-config/src/dub.ts b/packages/network-config/src/dub.ts index d582245..2b32a73 100644 --- a/packages/network-config/src/dub.ts +++ b/packages/network-config/src/dub.ts @@ -1,3 +1,5 @@ +import type { DubBuild } from './networks.js'; + // Device uniqueness backend wiring for the generated zombienet config. // // device-uniqueness-backend v0.2.0 replaced its ten binaries with one, selected by `--role`, @@ -81,7 +83,15 @@ export interface DubService { * old names as the no-downtime path — whereas renaming it would hand every existing cluster a * choice between an authentication failure and losing the rows already in it. */ -export const DUB_DATABASES = ['identity', 'username_indexer', 'invite_tickets'] as const; +/** + * The databases each build owns. `invite_tickets` belongs to the invite-tickets service, which + * the polkadot build does not have, so that build never creates it. + */ +export function dubDatabases(build: DubBuild): readonly string[] { + return build === 'polkadot' + ? ['identity', 'username_indexer'] + : ['identity', 'username_indexer', 'invite_tickets']; +} /** * Postgres connection string for one service's database. @@ -125,10 +135,12 @@ export function peopleRpcUrl(peoplePort: number): string { export function dubServices( ports: DubPorts, attesterAccount: string = ALICE_SS58, + build: DubBuild = 'testnet', inviterAccount: string = BOB_SS58 ): DubService[] { const rpc = peopleRpcUrl(ports.people); - const db = (name: (typeof DUB_DATABASES)[number]) => dubDatabaseUrl(name, ports.postgres); + const db = (name: string) => dubDatabaseUrl(name, ports.postgres); + const tickets = build !== 'polkadot'; // Every role reads its own database key, and all-in-one reads all of them, so the whole set // is given to every process. Cheaper than three near-identical env blocks, and it means @@ -140,7 +152,7 @@ export function dubServices( const databases = { DEVICE_ATTESTATION_DATABASE_URL: db('identity'), INDEXER_DATABASE_URL: db('username_indexer'), - INVITE_TICKETS_DATABASE_URL: db('invite_tickets'), + ...(tickets ? { INVITE_TICKETS_DATABASE_URL: db('invite_tickets') } : {}), }; // The Prometheus exporter binds before config validation, so leaving it on means five @@ -148,6 +160,10 @@ export function dubServices( // scrapes them locally. const common = { ...databases, + // service.sh waits for every database to exist before starting its role, and waiting for + // one this build never creates costs its whole timeout at each start. Same list the + // dub-postgres process creates from, for the same reason: they cannot disagree. + DUB_DATABASES: dubDatabases(build).join(' '), METRICS_ENABLED: 'false', RUST_LOG: 'info', PEOPLE_RPC_URL: rpc, @@ -159,8 +175,8 @@ export function dubServices( // Public counterpart of INVITER_SIGNER_SURI, which service.sh supplies. Equal to that // signing key's own account on purpose: the ticket services wrap a batch in // Proxy.proxy(real = INVITER_ADDRESS) only when the two differ. See BOB_SS58 for why it - // is not the attester. - INVITER_ADDRESS: inviterAccount, + // is not the attester. Absent on the polkadot build, which has no ticket services. + ...(tickets ? { INVITER_ADDRESS: inviterAccount } : {}), }; return [ @@ -194,7 +210,11 @@ export function dubServices( role: 'registration-queue', env: { ...common }, }, - { name: 'invite-tickets-pool', role: 'invite-tickets-pool', env: { ...common } }, + // Naming a role the binary does not have is a boot failure, not a no-op, so this is + // omitted rather than left idle on the polkadot build. + ...(tickets + ? [{ name: 'invite-tickets-pool', role: 'invite-tickets-pool', env: { ...common } }] + : []), ]; } @@ -228,6 +248,7 @@ export function dubCustomProcesses( // Genesis mode emits the {{SCRIPTS}} placeholder that zombie-compat expands; // fork mode writes a standalone TOML with absolute paths already resolved. scriptsDir: string = '{{SCRIPTS}}', + build: DubBuild = 'testnet', // The TURN/STUN relay whose credentials turn-api mints (docs/TURN.md). turn: boolean = true ): string { @@ -235,7 +256,7 @@ export function dubCustomProcesses( // zombienet validates custom_process args as CLI arguments — each must parse // as a flag or --key=value, so bare positionals are rejected outright. - const blocks = dubServices(ports, attesterAccount).map( + const blocks = dubServices(ports, attesterAccount, build).map( (svc) => ` [[custom_processes]] name = "${svc.name}" @@ -253,9 +274,13 @@ command = "${scriptsDir}/turn.sh" ` : ''; + // postgres.sh creates one database per service, and which services exist depends on the + // build. Passed rather than duplicated in the script, so the two cannot disagree about + // whether `invite_tickets` should be there. return ` [[custom_processes]] name = "dub-postgres" command = "${scriptsDir}/dub/postgres.sh" +${tomlEnv({ DUB_DATABASES: dubDatabases(build).join(' ') })} ${blocks.join('')}${relay}`; } diff --git a/packages/network-config/src/fork-toml.ts b/packages/network-config/src/fork-toml.ts index 5535ba4..d14432f 100644 --- a/packages/network-config/src/fork-toml.ts +++ b/packages/network-config/src/fork-toml.ts @@ -226,6 +226,7 @@ function parachainSection( scriptsDir: string, enableHop: boolean, net: NetworkDef, + restoreSnapshots: boolean, binDirOverride?: string ): string { const chain = manifest.chains[key]; @@ -265,7 +266,7 @@ function parachainSection( // restoring the same snapshot. Only the first holds the documented ports; the rest take // whatever zombienet hands out, and drop a fixed listen address that would collide. const count = manifest.topology?.collators[key] ?? 1; - const launch = `command = "${scriptsDir}/omni-node.sh"\ndb_snapshot = "${bundleDir}/snapshots/${paraId}.tgz"${envLine}`; + const launch = `command = "${scriptsDir}/omni-node.sh"\n${restoreSnapshots ? `db_snapshot = "${bundleDir}/snapshots/${paraId}.tgz"` : ''}${envLine}`; const collators = Array.from({ length: count }, (_, i) => i === 0 ? ` @@ -360,10 +361,16 @@ export interface GenerateForkTomlOptions { enableHop?: boolean; /** Overrides /bin — non-previewnet binaries live in bin/. */ binDir?: string; + /** + * Emit the bundle's snapshots as `db_snapshot`. Off when resuming a fork: zombienet unpacks a + * configured snapshot over the node's base path on every spawn, whether or not a database is + * already there, so a resumed fork that kept them would rewind to the bite block. + */ + restoreSnapshots?: boolean; } export function generateForkToml(options: GenerateForkTomlOptions): string { - const { repoDir, bundleDir, enableHop = true } = options; + const { repoDir, bundleDir, enableHop = true, restoreSnapshots = true } = options; const binDir = options.binDir ?? `${repoDir}/bin`; const scriptsDir = options.scriptsDir ?? `${repoDir}/scripts`; @@ -389,6 +396,7 @@ export function generateForkToml(options: GenerateForkTomlOptions): string { }, undefined, scriptsDir, + net.dubBuild, net.services['turn'] !== false ) : ''; @@ -409,11 +417,12 @@ node_spawn_timeout = 400 chain = "${chainIdOf(relaySpecPath)}" chain_spec_path = "${relaySpecPath}" default_command = "${binDir}/${net.relay.binary.name}" -default_db_snapshot = "${bundleDir}/snapshots/relay.tgz" -default_args = ${tomlArgs(relayArgs)} +${restoreSnapshots ? `default_db_snapshot = "${bundleDir}/snapshots/relay.tgz"\n` : ''}default_args = ${tomlArgs(relayArgs)} ${relayNodes(manifest.topology?.validators ?? net.relay.validators)} ${parachains - .map((p) => parachainSection(p, manifest, bundleDir, scriptsDir, enableHop, net, options.binDir)) + .map((p) => + parachainSection(p, manifest, bundleDir, scriptsDir, enableHop, net, restoreSnapshots, options.binDir) + ) .join('')} ${customProcesses(scriptsDir, net, present, manifest)}${identity}`; diff --git a/packages/network-config/src/index.ts b/packages/network-config/src/index.ts index 7a4b9d2..5ee9ab6 100644 --- a/packages/network-config/src/index.ts +++ b/packages/network-config/src/index.ts @@ -28,6 +28,7 @@ export { CHAIN_ARGS, PORTS, P2P_PORTS, + forkRelayBootnode, paraIds, RELAY_BASE_PORT, VALIDATORS, diff --git a/packages/network-config/src/networks.ts b/packages/network-config/src/networks.ts index 65e9f01..50ebdfc 100644 --- a/packages/network-config/src/networks.ts +++ b/packages/network-config/src/networks.ts @@ -26,6 +26,13 @@ import path from 'node:path'; import type { Asset, Coinage, Pointer } from '@parity/trinity-stack'; import { defaultHome, packageRoot, networksDirs } from './repo-root.js'; +/** + * Which People runtime a dub binary was built against. Since v0.6.0 every asset ships twice + * and the runtime is fixed at build time: it selects the vendored metadata and whether the + * invite-tickets roles exist. `dub --help` prints the build a binary carries. + */ +export type DubBuild = 'testnet' | 'polkadot'; + /** The parachain keys PPN knows: what its port and per-chain flag tables are keyed by. */ export type Parachain = 'asset-hub' | 'people' | 'bulletin' | 'web3-storage'; export type ChainKey = 'relay' | Parachain; @@ -128,6 +135,13 @@ export interface NetworkParachain { rpcNodes: number; /** Extra args for the RPC nodes, applied over the chain's own args. */ rpcNodeArgs: string[]; + /** + * Account written into this chain's `TransactionStorage::AllowedAuthorizers` at bite time, so + * something can authorize Bulletin storage. Only a fork needs it: the bulletin runtime seeds an + * authorizer in its genesis preset, and a fork has no genesis, so a bitten Bulletin has none and + * every store fails. A 32-byte account id, hex, `0x`-prefixed. + */ + bulletinAuthorizer?: string; } /** RPC node i of a parachain listens on the chain's port + i, below the next chain's port. */ @@ -199,6 +213,16 @@ export interface NetworkDef { */ sharedRelay?: boolean; }; + /** + * Which dub build this network needs, `testnet` (the default) or `polkadot`. + * + * Since dub v0.6.0 every asset ships twice, one per People runtime, and the runtime is + * fixed at build time: it selects the vendored metadata and whether the invite-tickets + * roles exist. A fork of Polkadot needs `polkadot`, whose People runtime carries neither + * Game nor ProofOfInk; everything else needs `testnet`. Naming a role the binary does not + * have is a boot failure, so this decides the services as well as the download. + */ + dubBuild?: DubBuild; /** Named releases every binary/runtime reference points into. */ releases: Record; /** Genesis-wide chain-spec settings. Present on a genesis network. */ @@ -227,6 +251,25 @@ export interface NetworkDef { */ pinProducts?: boolean; }; + /** + * Attester granted an `AttestationAllowance` on each pallet named in `on`, at bite time. + * + * The two quotas meter different things: `PeopleLite`'s how many people a verifier may attest, + * `DotnsGateway`'s how many names an attester may reserve. Without the first nobody becomes a + * lite person, without the second nobody who did can take a username, and the identity backend + * needs both. + * + * `PeopleLite`'s manager is `EnsureRoot` and `DotnsGateway`'s is `RootOrWhitelist`, so a fork + * without governance cannot grant either afterwards. + * + * `on` names the pallet to seed on each chain. + */ + attestation?: { + attester: string; + count: number; + /** Chain key -> the pallet that gates attestation there. */ + on: Record; + }; /** What the network's Trinity stack descriptor says that no other field does. */ stack?: NetworkStack; /** Every `_todo` note found in the file — non-empty means the descriptor is a stub. */ @@ -304,6 +347,10 @@ export function loadDescriptor(name: string): NetworkDef { bad('relay needs chain, spec, rpc and specSource'); } if (!raw.bite?.source) bad('bite.source is required'); + // A typo here would otherwise surface as a 404 on an asset name, long after generate. + if (raw.dubBuild !== undefined && raw.dubBuild !== 'testnet' && raw.dubBuild !== 'polkadot') { + bad(`dubBuild is "${String(raw.dubBuild)}", expected "testnet" or "polkadot"`); + } if (!Array.isArray(raw.parachains) || raw.parachains.length === 0) { bad('at least one parachain is required'); } @@ -383,6 +430,9 @@ export function loadDescriptor(name: string): NetworkDef { if (p.aura !== undefined && p.aura !== 'sr25519' && p.aura !== 'ed25519') { bad(`parachain ${p.key}: aura must be "sr25519" or "ed25519"`); } + if (p.bulletinAuthorizer !== undefined && !/^0x[0-9a-fA-F]{64}$/.test(p.bulletinAuthorizer)) { + bad(`parachain ${p.key}: bulletinAuthorizer must be a 0x-prefixed 32-byte account id`); + } checkBinary(`parachains.${p.key}`, p.binary); checkRuntime(`parachains.${p.key}`, p.runtime); if (seen.has(p.key)) bad(`parachain ${p.key} listed twice`); @@ -391,6 +441,30 @@ export function loadDescriptor(name: string): NetworkDef { const ids = raw.parachains.map((p: NetworkParachain) => p.paraId); if (new Set(ids).size !== ids.length) bad('duplicate para ids'); + // Every seed names the chains it writes to, and a chain this network does not have is seeded + // nowhere and reported nowhere. `what` is the field, so each caller's error names itself. + const namesOurChains = (what: string, keys: string[]): void => { + for (const k of keys) { + if (!seen.has(k)) bad(`${what} names "${k}", which this network does not have`); + } + }; + + if (raw.attestation) { + const at = raw.attestation; + if (!at.attester) bad('attestation needs an attester account'); + if (!Number.isInteger(at.count) || at.count <= 0) { + bad('attestation.count must be a positive integer'); + } + const on = Object.entries(at.on ?? {}); + if (on.length === 0) bad('attestation.on must name at least one chain'); + namesOurChains('attestation.on', on.map(([k]) => k)); + for (const [k, pallet] of on) { + if (pallet !== 'PeopleLite' && pallet !== 'DotnsGateway') { + bad(`attestation.on.${k} must be PeopleLite or DotnsGateway, got "${pallet}"`); + } + } + } + // A genesis network builds its chain specs locally, so every chain needs a runtime to // build from and a spec to build into, and the network needs the settings they share. if (raw.genesis === true) { @@ -475,7 +549,9 @@ export function loadDescriptor(name: string): NetworkDef { })), services, tools, + dubBuild: raw.dubBuild, dotns: raw.dotns, + attestation: raw.attestation, stack, todos, }; diff --git a/packages/network-config/src/toml-generator.ts b/packages/network-config/src/toml-generator.ts index af92339..c604bbe 100644 --- a/packages/network-config/src/toml-generator.ts +++ b/packages/network-config/src/toml-generator.ts @@ -194,6 +194,33 @@ const P2P_PORTS: Record = { bulletin: requiredPort('BULLETIN_P2P_PORT'), 'web3-storage': requiredPort('WEB3_STORAGE_P2P_PORT'), }; +/** + * libp2p identity of the first relay node. zombienet derives each well-known node's key from + * its name, so this is the same in every spawn; re-derive it with `system_localPeerId` on + * `RELAY_ALICE_PORT` if the pinned zombie-cli ever changes that derivation. + */ +const RELAY_BOOTNODE_PEER_ID = '12D3KooWQCkBm1BYtkHpocxCwMgR8yjitEeHGx8spzcDLGt2gkBm'; + +/** + * The bootnode a bitten relay spec ships, so a fork can find its own nodes. + * + * A fork keeps the source chain's genesis hash, so `fork_id` is what keeps it off the source + * network's protocols, and off the source network's DHT with them. Kademlia inserts peers + * manually (`BucketInserts::Manual` in `discovery.rs`), so connecting to a peer never adds it + * to the routing table: the table fills from bootnodes or from identify, and a bitten spec + * that ships neither leaves every node's table empty and authority discovery resolving no + * addresses. + * + * Only the first relay node can be named ahead of the spawn, because it is the only one whose + * p2p port the generator pins; the rest take ephemeral ports that change every time. One entry + * is enough, since Kademlia walks out from it to the rest. Every node then bootstraps through + * that one, which the fork already depends on anyway: it is the node every collator reaches the + * relay through. + */ +function forkRelayBootnode(): string { + return `/ip4/127.0.0.1/tcp/${P2P_PORTS.relay}/ws/p2p/${RELAY_BOOTNODE_PEER_ID}`; +} + // The interface the webrtc-direct collators bind. Loopback on a laptop; a server sets its // public IP so browser peers can reach the collators at all — webrtc-direct advertises what // it is bound to, so a 127.0.0.1 listener is unreachable from anywhere else. The environment @@ -579,12 +606,18 @@ command = "{{SCRIPTS}}/grant-invites.sh" ` )}${section( has('people') && serviceEnabled('dub'), - dubCustomProcesses({ - postgres: requiredPort('DUB_POSTGRES_PORT'), - people: PORTS.people, - assetHub: PORTS['asset-hub'], - gateway: requiredPort('DUB_PORT'), - }, undefined, undefined, serviceEnabled('turn')) + dubCustomProcesses( + { + postgres: requiredPort('DUB_POSTGRES_PORT'), + people: PORTS.people, + assetHub: PORTS['asset-hub'], + gateway: requiredPort('DUB_PORT'), + }, + undefined, + undefined, + genesisNet().dubBuild, + serviceEnabled('turn') + ) )} ${section(serviceEnabled('patch-bootnodes'), ` [[custom_processes]] @@ -623,6 +656,7 @@ export { VALID_PARACHAINS, CHAIN_ARGS, P2P_PORTS, + forkRelayBootnode, POPULAR_LOG_TARGETS, LOG_LEVELS, // Shared with fork-toml.ts so that fork mode and genesis mode cannot drift apart. diff --git a/packages/network-config/tests/dub.test.ts b/packages/network-config/tests/dub.test.ts index 8006bec..1914a13 100644 --- a/packages/network-config/tests/dub.test.ts +++ b/packages/network-config/tests/dub.test.ts @@ -9,6 +9,7 @@ import { BOB_SS58, dubCustomProcesses, dubDatabaseUrl, + dubDatabases, dubServices, peopleRpcUrl, } from '../src/dub.js'; @@ -206,7 +207,7 @@ describe('dubCustomProcesses', () => { // was, along with a standing risk of drifting from it. it('runs the TURN relay beside the API, unless switched off', () => { assert.match(toml, /name = "turn"\ncommand = "\{\{SCRIPTS\}\}\/turn\.sh"/); - assert.ok(!dubCustomProcesses(PORTS, undefined, undefined, false).includes('name = "turn"')); + assert.ok(!dubCustomProcesses(PORTS, undefined, undefined, 'testnet', false).includes('name = "turn"')); }); it('runs no hand-written gateway', () => { @@ -215,3 +216,59 @@ describe('dubCustomProcesses', () => { assert.match(toml, /name = "dub-api"/); }); }); + +describe('dub builds', () => { + // The runtime a dub binary was built against decides whether the invite-tickets roles + // exist. Naming one the binary does not have fails its role gate at boot, so a polkadot + // build must not be handed that process. + it('drops the invite-tickets role on the polkadot build and keeps it on testnet', () => { + assert.deepEqual( + dubServices(PORTS, undefined, 'polkadot').map((s) => s.role), + ['all-in-one', 'device-attestation-chain-writer', 'registration-queue'] + ); + assert.ok( + dubServices(PORTS, undefined, 'testnet').some((s) => s.role === 'invite-tickets-pool') + ); + }); + + // The keys go with the service. INVITER_ADDRESS is half of a pair with the signing key + // service.sh supplies, and nothing reads either once the role is gone. + it('omits the invite-tickets keys on the polkadot build', () => { + for (const svc of dubServices(PORTS, undefined, 'polkadot')) { + assert.ok(!('INVITER_ADDRESS' in svc.env), `${svc.name} sets INVITER_ADDRESS`); + assert.ok(!('INVITE_TICKETS_DATABASE_URL' in svc.env), `${svc.name} sets the tickets DB`); + } + const testnet = dubServices(PORTS, undefined, 'testnet')[0]; + assert.equal(testnet.env.INVITER_ADDRESS, BOB_SS58); + }); + + // v0.6.0 requires it on the chain writer, which 0.5.0 read optionally, and it is the one + // key whose absence aborts that role at startup rather than at first use. + it('gives every role the Asset Hub RPC, on both builds', () => { + for (const build of ['testnet', 'polkadot'] as const) { + for (const svc of dubServices(PORTS, undefined, build)) { + assert.equal(svc.env.ASSET_HUB_RPC_URL, `ws://127.0.0.1:${PORTS.assetHub}`); + } + } + }); + + // postgres.sh creates exactly these, and reads the list out of the generated TOML so the + // two cannot disagree. + it('creates the tickets database only where the service exists', () => { + assert.deepEqual(dubDatabases('polkadot'), ['identity', 'username_indexer']); + assert.deepEqual(dubDatabases('testnet'), ['identity', 'username_indexer', 'invite_tickets']); + assert.match(dubCustomProcesses(PORTS, undefined, '{{SCRIPTS}}', 'polkadot'), + /DUB_DATABASES", value = "identity username_indexer"/); + }); + + // service.sh blocks on each of these existing before it starts its role, so a role given a + // longer list than postgres.sh creates waits out its whole timeout at every start. + it('gives the services the same list postgres.sh creates', () => { + for (const build of ['testnet', 'polkadot'] as const) { + const expected = dubDatabases(build).join(' '); + for (const svc of dubServices(PORTS, undefined, build)) { + assert.equal(svc.env.DUB_DATABASES, expected, `${svc.name} on ${build}`); + } + } + }); +}); diff --git a/packages/network-config/tests/fork-toml.test.ts b/packages/network-config/tests/fork-toml.test.ts index 1efa8b6..51bf0af 100644 --- a/packages/network-config/tests/fork-toml.test.ts +++ b/packages/network-config/tests/fork-toml.test.ts @@ -247,6 +247,20 @@ describe('generateForkToml — collators', () => { assert.match(toml, /chain_spec_path = ".*\/specs\/individuality\.json"/); }); + // zombienet unpacks a db_snapshot over the node's base path on every spawn, whether or not a + // database is already there, so a resumed fork given one rewinds to the bite block. The resume + // config is therefore the normal one minus those lines, and only those lines. + it('omits every snapshot when resuming, and changes nothing else', () => { + const dir = makeBundle(); + const restored = generateForkToml({ repoDir: REPO, bundleDir: dir }); + const resumed = generateForkToml({ repoDir: REPO, bundleDir: dir, restoreSnapshots: false }); + assert.ok(!resumed.includes('db_snapshot'), 'a resumed fork must not be handed a snapshot'); + assert.equal( + resumed, + restored.replace(/^default_db_snapshot = .*\n/m, '').replace(/^db_snapshot = .*$/gm, '') + ); + }); + // A warp-synced relay cannot serve history to an embedded relay node, which then sits at #0. it('points every collator at the relay over RPC', () => { for (const id of Object.values(paraIds())) { @@ -446,6 +460,28 @@ describe('generateForkToml — non-previewnet bundles', () => { ); }); + // dub ships one build per People runtime, and the polkadot one has no invite-tickets roles: + // naming a role the binary lacks is a boot failure. Which build a network wants is a + // descriptor field, so this asserts on the generated toml. Calling dubCustomProcesses with + // the build passed in would test the function and miss what reaches it, which is where the + // field went missing once already. + it('leaves the invite-tickets service out of a polkadot fork', () => { + const toml = generateForkToml({ + repoDir: REPO, + bundleDir: makeNetBundle('polkadot', { + relay: { paraId: null, spec: 'polkadot', specId: 'polkadot' }, + 'asset-hub': { paraId: 1000, spec: 'asset-hub', specId: 'asset-hub' }, + people: { paraId: 1004, spec: 'people', specId: 'people' }, + bulletin: { paraId: 1010, spec: 'bulletin', specId: 'bulletin' }, + }), + }); + for (const absent of ['invite-tickets', 'INVITE_TICKETS_DATABASE_URL', 'INVITER_ADDRESS']) { + assert.ok(!toml.includes(absent), `${absent} must not reach a polkadot fork`); + } + assert.ok(toml.includes('name = "dub-api"'), 'the rest of the backend still runs'); + assert.ok(devnetToml().includes('invite-tickets'), 'every other network keeps the role'); + }); + it('picks the descriptor collator binary via the wrapper env', () => { const toml = generateForkToml({ repoDir: REPO, diff --git a/packages/network-config/tests/networks.test.ts b/packages/network-config/tests/networks.test.ts index 8cf3839..fd78e80 100644 --- a/packages/network-config/tests/networks.test.ts +++ b/packages/network-config/tests/networks.test.ts @@ -14,7 +14,12 @@ import { asHttp, asWs, DEFAULT_NETWORK, + loadDescriptor, } from '../src/networks.js'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { repoRoot } from '../src/repo-root.js'; import { loadNetwork } from '../src/load.js'; import { pinsProducts } from '../src/fork-toml.js'; import { paraIds, VALID_PARACHAINS } from '../src/toml-generator.js'; @@ -194,6 +199,15 @@ describe('network descriptors', () => { assert.ok(net.parachains.every((p) => p.binary.name === 'polkadot-parachain')); }); + // Every consumer of the field reads it off the loaded descriptor: the fetcher picks the + // asset by it and dubCustomProcesses picks the roles by it, and both fall back to `testnet`. + // A descriptor that parses the field and then drops it is a fork running the wrong binary + // with the wrong roles, reported nowhere. + it('carries dubBuild through from the descriptor', () => { + assert.equal(loadNetwork('polkadot').dubBuild, 'polkadot'); + assert.equal(loadNetwork('previewnet').dubBuild, undefined); + }); + it('pins the bite tool per network, since it must run that network\'s runtimes', () => { for (const name of listNetworks()) { const dg = loadNetwork(name).bite.doppelganger; @@ -201,10 +215,86 @@ describe('network descriptors', () => { } }); + // A copy of a real descriptor in a scratch PPN_HOME, so a check runs against a file that is + // otherwise valid. `where` is a parachain key, or "root" to patch the network itself. + const make = (patch: Record, where: string): (() => void) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ppn-desc-')); + fs.mkdirSync(path.join(dir, 'networks')); + const d = JSON.parse(fs.readFileSync(path.join(repoRoot(), 'networks', 'polkadot.json'), 'utf-8')); + d.name = 'scratchnet'; + if (where === 'root') Object.assign(d, patch); + else Object.assign(d.parachains.find((p: { key: string }) => p.key === where), patch); + fs.writeFileSync(path.join(dir, 'networks', 'scratchnet.json'), JSON.stringify(d)); + return () => { + const prev = process.env.PPN_HOME; + process.env.PPN_HOME = dir; + try { + loadDescriptor('scratchnet'); + } finally { + if (prev === undefined) delete process.env.PPN_HOME; + else process.env.PPN_HOME = prev; + } + }; + }; + const withNetwork = (patch: Record): (() => void) => make(patch, 'root'); + it('rejects unknown networks with the known list', () => { assert.throws(() => loadNetwork('mainnet'), /unknown network "mainnet" — known: .*previewnet/); assert.throws(() => loadNetwork('../etc/passwd'), /invalid network name/); }); + + describe('the attestation descriptor', () => { + const AT = { + attester: '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY', + count: 1000000, + on: { people: 'PeopleLite' }, + }; + + it('scopes each attestation pallet to a chain the network has', () => { + const net = loadNetwork('polkadot'); + assert.deepEqual(net.attestation?.on, { people: 'PeopleLite', 'asset-hub': 'DotnsGateway' }); + const keys = net.parachains.map((p) => p.key); + for (const k of Object.keys(net.attestation?.on ?? {})) { + assert.ok(keys.includes(k), `attestation.on names "${k}", not a chain of this network`); + } + }); + + it('refuses a chain this network does not have', () => { + assert.throws( + withNetwork({ attestation: { ...AT, on: { 'web3-storage': 'PeopleLite' } } }), + /attestation.on names "web3-storage", which this network does not have/ + ); + }); + + it('refuses a pallet that gates no attestation', () => { + assert.throws( + withNetwork({ attestation: { ...AT, on: { people: 'Nfts' } } }), + /attestation.on.people must be PeopleLite or DotnsGateway, got "Nfts"/ + ); + }); + + it('refuses a count that would grant nothing', () => { + for (const count of [0, -1, 1.5]) { + assert.throws(withNetwork({ attestation: { ...AT, count } }), /must be a positive integer/); + } + }); + + // Without this the descriptor passes and the failure surfaces at bite time instead, as a + // decodeAddress throw from attestationAllowanceInjects. + it('refuses an attester that is not there', () => { + assert.throws( + withNetwork({ attestation: { ...AT, attester: '' } }), + /attestation needs an attester account/ + ); + }); + + it('refuses naming no chain at all', () => { + assert.throws( + withNetwork({ attestation: { ...AT, on: {} } }), + /attestation.on must name at least one chain/ + ); + }); + }); }); describe('network selection and URL helpers', () => { diff --git a/packages/network-config/tests/toml-generator.test.ts b/packages/network-config/tests/toml-generator.test.ts index 59f6dae..fc39f50 100644 --- a/packages/network-config/tests/toml-generator.test.ts +++ b/packages/network-config/tests/toml-generator.test.ts @@ -9,6 +9,7 @@ import { VALID_PARACHAINS, CHAIN_ARGS, P2P_PORTS, + forkRelayBootnode, POPULAR_LOG_TARGETS, LOG_LEVELS, } from '../src/toml-generator.js'; @@ -478,3 +479,15 @@ describe('exports', () => { assert.deepEqual(LOG_LEVELS, ['trace', 'debug', 'info', 'warn', 'error']); }); }); + +// The bootnode a bitten relay spec ships. Its peer id cannot be checked without a spawn, so +// what is pinned here is the port. With another chain's port the relay nodes never reach the +// bootnode, their routing tables stay empty, and nothing says so. +describe('the fork relay bootnode', () => { + it('dials the relay on the port the generator pins', () => { + assert.equal( + forkRelayBootnode(), + `/ip4/127.0.0.1/tcp/${P2P_PORTS.relay}/ws/p2p/12D3KooWQCkBm1BYtkHpocxCwMgR8yjitEeHGx8spzcDLGt2gkBm` + ); + }); +}); diff --git a/scripts/dub/postgres.sh b/scripts/dub/postgres.sh index b98cc46..ebb8a02 100755 --- a/scripts/dub/postgres.sh +++ b/scripts/dub/postgres.sh @@ -33,7 +33,18 @@ PGDATA="${DUB_POSTGRES_DATA_DIR:-${DATA_DIR:-$PPN_WS/data}/identity-pgdata}" # DEVICE_ATTESTATION_DATABASE_URL in v0.3.0, but nothing upstream reads the database's name, and # renaming it would strand every cluster already initialised under the old one. `dim_tickets` is # gone with the service that owned it — an existing cluster keeps the database, unread. -DATABASES=(identity username_indexer invite_tickets) +# Supplied by the generated TOML (see dubDatabases in packages/network-config/src/dub.ts), +# because which databases exist depends on the dub build: the polkadot one has no +# invite-tickets service and so never owns `invite_tickets`. No default: one that is right +# for a testnet cluster is silently wrong for a polkadot one, and the cost lands later. +if [[ -z "${DUB_DATABASES:-}" ]]; then + echo "Error: DUB_DATABASES is not set." >&2 + echo " The generated TOML supplies it; which databases exist depends on the dub" >&2 + echo " build, so there is no default that is right for both. Run through a" >&2 + echo " generated network, or set it: DUB_DATABASES=\"identity username_indexer\"" >&2 + exit 1 +fi +read -r -a DATABASES <<< "$DUB_DATABASES" PG_USER=identity diff --git a/scripts/dub/service.sh b/scripts/dub/service.sh index d929ece..15d10a2 100755 --- a/scripts/dub/service.sh +++ b/scripts/dub/service.sh @@ -175,6 +175,21 @@ fi WAIT_HOST="${WAIT_TARGET%:*}" WAIT_PORT="${WAIT_TARGET##*:}" +# Which databases the wait below expects, from the generated TOML rather than a list of its +# own: the polkadot dub build has no invite-tickets service and so never owns +# `invite_tickets`, and waiting for one that will never exist burns that loop's full 120 +# seconds at every start and then proceeds anyway. That is the failure this list prevents, so +# an unset value is fatal rather than defaulted, and checked here so it costs no waiting. +if [[ -z "${DUB_DATABASES:-}" ]]; then + echo "Error: DUB_DATABASES is not set." >&2 + echo " The generated TOML supplies it; which databases exist depends on the dub" >&2 + echo " build, so there is no default that is right for both. Run through a" >&2 + echo " generated network, or set it: DUB_DATABASES=\"identity username_indexer\"" >&2 + exit 1 +fi +read -r -a _dbs <<< "$DUB_DATABASES" +DB_WAIT_LIST="${_dbs[*]}" + echo "[$ROLE] waiting for $WAIT_TARGET..." for _ in $(seq 1 120); do if "$BIN_DIR/postgres-dist/bin/pg_isready" -h "$WAIT_HOST" -p "$WAIT_PORT" \ @@ -191,7 +206,7 @@ done # the registration test to give up while the chain half had already succeeded. for _ in $(seq 1 120); do missing="" - for db in identity username_indexer invite_tickets; do + for db in $DB_WAIT_LIST; do "$BIN_DIR/postgres-dist/bin/psql" -h "$WAIT_HOST" -p "$WAIT_PORT" -U identity \ -d "$db" -tAc 'select 1' >/dev/null 2>&1 || missing="$missing $db" done diff --git a/tests/scripts/dub-ready.ts b/tests/scripts/dub-ready.ts index f35a857..1a08325 100644 --- a/tests/scripts/dub-ready.ts +++ b/tests/scripts/dub-ready.ts @@ -11,7 +11,7 @@ // tests can use it without the chain tooling being set up. /** Every HTTP surface of the backend, on one origin (`dub --role all-in-one`). */ -export const DUB_BASE = "http://127.0.0.1:8092"; +export const DUB_BASE = process.env.DUB_BASE || "http://127.0.0.1:8092"; /** * Why a response is not ready yet, or null when it is. diff --git a/tests/scripts/test-turn.ts b/tests/scripts/test-turn.ts index d411d39..9124d83 100644 --- a/tests/scripts/test-turn.ts +++ b/tests/scripts/test-turn.ts @@ -15,7 +15,8 @@ import { dubToken } from "./dub-auth"; const SUCCESS = 1 as const; const FAILURE = 0 as const; -const HOST = "127.0.0.1"; +// TURN_HOST and DUB_BASE point the test at a deployed relay instead of the local one. +const HOST = process.env.TURN_HOST || "127.0.0.1"; const PORT = 3478; const EXPECTED_SERVERS = [ `stun:${HOST}:${PORT}`, @@ -170,7 +171,8 @@ export async function run(nodeName: string, _networkInfo: NetworkInfo, _args: st return FAILURE; } const creds = (await res.json()) as { servers: string[]; username: string; password: string; ttl: number }; - if (JSON.stringify(creds.servers) !== JSON.stringify(EXPECTED_SERVERS)) { + // Behind https a fourth, turns:, follows the three plain servers. + if (JSON.stringify(creds.servers.slice(0, 3)) !== JSON.stringify(EXPECTED_SERVERS)) { console.error(`[TEST] FAIL ICE servers ${JSON.stringify(creds.servers)}, expected ${JSON.stringify(EXPECTED_SERVERS)}`); return FAILURE; } diff --git a/zombienet-configs/local-dev.toml b/zombienet-configs/local-dev.toml index ab4df9a..ec8c60a 100644 --- a/zombienet-configs/local-dev.toml +++ b/zombienet-configs/local-dev.toml @@ -182,6 +182,9 @@ command = "{{SCRIPTS}}/grant-invites.sh" [[custom_processes]] name = "dub-postgres" command = "{{SCRIPTS}}/dub/postgres.sh" +env = [ + { name = "DUB_DATABASES", value = "identity username_indexer invite_tickets" }, +] [[custom_processes]] name = "dub-api" @@ -191,6 +194,7 @@ env = [ { name = "DEVICE_ATTESTATION_DATABASE_URL", value = "postgres://identity@127.0.0.1:5433/identity" }, { name = "INDEXER_DATABASE_URL", value = "postgres://identity@127.0.0.1:5433/username_indexer" }, { name = "INVITE_TICKETS_DATABASE_URL", value = "postgres://identity@127.0.0.1:5433/invite_tickets" }, + { name = "DUB_DATABASES", value = "identity username_indexer invite_tickets" }, { name = "METRICS_ENABLED", value = "false" }, { name = "RUST_LOG", value = "info" }, { name = "PEOPLE_RPC_URL", value = "ws://127.0.0.1:10010" }, @@ -211,6 +215,7 @@ env = [ { name = "DEVICE_ATTESTATION_DATABASE_URL", value = "postgres://identity@127.0.0.1:5433/identity" }, { name = "INDEXER_DATABASE_URL", value = "postgres://identity@127.0.0.1:5433/username_indexer" }, { name = "INVITE_TICKETS_DATABASE_URL", value = "postgres://identity@127.0.0.1:5433/invite_tickets" }, + { name = "DUB_DATABASES", value = "identity username_indexer invite_tickets" }, { name = "METRICS_ENABLED", value = "false" }, { name = "RUST_LOG", value = "info" }, { name = "PEOPLE_RPC_URL", value = "ws://127.0.0.1:10010" }, @@ -228,6 +233,7 @@ env = [ { name = "DEVICE_ATTESTATION_DATABASE_URL", value = "postgres://identity@127.0.0.1:5433/identity" }, { name = "INDEXER_DATABASE_URL", value = "postgres://identity@127.0.0.1:5433/username_indexer" }, { name = "INVITE_TICKETS_DATABASE_URL", value = "postgres://identity@127.0.0.1:5433/invite_tickets" }, + { name = "DUB_DATABASES", value = "identity username_indexer invite_tickets" }, { name = "METRICS_ENABLED", value = "false" }, { name = "RUST_LOG", value = "info" }, { name = "PEOPLE_RPC_URL", value = "ws://127.0.0.1:10010" }, @@ -245,6 +251,7 @@ env = [ { name = "DEVICE_ATTESTATION_DATABASE_URL", value = "postgres://identity@127.0.0.1:5433/identity" }, { name = "INDEXER_DATABASE_URL", value = "postgres://identity@127.0.0.1:5433/username_indexer" }, { name = "INVITE_TICKETS_DATABASE_URL", value = "postgres://identity@127.0.0.1:5433/invite_tickets" }, + { name = "DUB_DATABASES", value = "identity username_indexer invite_tickets" }, { name = "METRICS_ENABLED", value = "false" }, { name = "RUST_LOG", value = "info" }, { name = "PEOPLE_RPC_URL", value = "ws://127.0.0.1:10010" },