From 6167ae23620b69520498c9dd7c3b33bf8c61366b Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:16:10 +0000 Subject: [PATCH 01/19] Add a per-deployment network overlay with RPC nodes per parachain --- .github/workflows/zombienet-tests.yml | 29 +++- config/ports.env | 3 +- networks/README.md | 4 +- packages/network-config/src/args.ts | 11 ++ .../network-config/src/dashboard-model.ts | 10 +- packages/network-config/src/fork-toml.ts | 15 +- packages/network-config/src/load.ts | 10 +- packages/network-config/src/networks.ts | 43 ++++-- packages/network-config/src/nginx-routes.ts | 6 +- packages/network-config/src/overlay.ts | 72 +++++++++ packages/network-config/src/toml-generator.ts | 140 +++++++++--------- .../network-config/tests/fork-toml.test.ts | 18 +++ packages/network-config/tests/overlay.test.ts | 115 ++++++++++++++ 13 files changed, 381 insertions(+), 95 deletions(-) create mode 100644 packages/network-config/src/args.ts create mode 100644 packages/network-config/src/overlay.ts create mode 100644 packages/network-config/tests/overlay.test.ts diff --git a/.github/workflows/zombienet-tests.yml b/.github/workflows/zombienet-tests.yml index 4c6c149..1e0aec7 100644 --- a/.github/workflows/zombienet-tests.yml +++ b/.github/workflows/zombienet-tests.yml @@ -341,11 +341,16 @@ jobs: # environment into custom_processes; that file exists for exactly this. rm -rf ${{ env.ZOMBIE_DIR }} data/identity-pgdata mkdir -p ${{ env.ZOMBIE_DIR }} - echo "DUB_POSTGRES_DATA_DIR=${{ env.ZOMBIE_DIR }}/identity-pgdata" > config/ports.local.env + # One People RPC node through the network overlay, so the spawn covers what a + # deployment's overlay generates. A separate file keeps local-dev.toml as checked in. + OVERLAY="$RUNNER_TEMP/network-overlay.json" + echo '{"previewnet":{"parachains":{"people":{"rpcNodes":1}}}}' > "$OVERLAY" + PPN_NETWORK_OVERLAY="$OVERLAY" node bin/ppn.mjs genesis-toml > zombienet-configs/ci.toml + printf 'DUB_POSTGRES_DATA_DIR=%s\nPPN_NETWORK_OVERLAY=%s\n' "${{ env.ZOMBIE_DIR }}/identity-pgdata" "$OVERLAY" > config/ports.local.env # Spawn network in background, store state in ZOMBIE_DIR BIN="${{ env.BIN }}" SCRIPTS="${{ env.SCRIPTS }}" \ ${{ env.BIN }}/zombie-cli spawn -p native -d ${{ env.ZOMBIE_DIR }} \ - zombienet-configs/local-dev.toml & + zombienet-configs/ci.toml & # Wait for network to be ready (check for zombie.json) echo "Waiting for network to start..." @@ -360,6 +365,26 @@ jobs: # Additional wait for nodes to be ready sleep 30 + - name: Verify the People RPC node serves the chain + run: | + source ./config/ports.env + RPC_PORT=$((PEOPLE_PORT + 1)) + rpc() { + curl -sf -m 5 -H 'Content-Type: application/json' \ + -d "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"$1\",\"params\":[]}" \ + "http://127.0.0.1:$RPC_PORT" | jq -c .result + } + for _ in $(seq 1 60); do + height=$(rpc chain_getHeader | jq -r '.number // empty' 2>/dev/null || true) + if [ -n "$height" ] && [ $((height)) -gt 0 ]; then break; fi + sleep 10 + done + echo "people-rpc1 at block $((height))" + [ $((height)) -gt 0 ] || { echo "ERROR: people-rpc1 on $RPC_PORT has no blocks"; exit 1; } + roles=$(rpc system_nodeRoles) + echo "people-rpc1 roles: $roles" + [ "$roles" = '["Full"]' ] || { echo "ERROR: people-rpc1 should be a full node, is $roles"; exit 1; } + - name: Verify raw chainspecs have bootNodes run: | echo "Checking raw chainspecs in ${{ env.ZOMBIE_DIR }}..." diff --git a/config/ports.env b/config/ports.env index 787cba8..9fef1f7 100644 --- a/config/ports.env +++ b/config/ports.env @@ -10,7 +10,8 @@ RELAY_DAVE_PORT=10003 RELAY_EVE_PORT=10004 RELAY_FERDIE_PORT=10005 -# Parachain collator RPC ports +# Parachain collator RPC ports. A chain's RPC nodes (rpcNodes in the descriptor or the +# network overlay) take the next ports up: 10011, 10012, ... for People. PEOPLE_PORT=10010 ASSET_HUB_PORT=10020 BULLETIN_PORT=10030 diff --git a/networks/README.md b/networks/README.md index 9608756..c6087c2 100644 --- a/networks/README.md +++ b/networks/README.md @@ -37,7 +37,9 @@ or `PPN_NETWORK=` in the environment of the scripts. | `relay.genesisSpec`, `parachains[].genesisSpec` | `{ "chainId", "name", "preset", "file" }` — the chain spec a genesis network builds for this chain. `ppn generate` passes them to `chain-spec-builder` (`-i`, `-n`, `named-preset`, output filename) and the genesis TOML generator points zombienet at `chainId` + `file`. **Required on a genesis network**, absent on fork-only ones (a fork uses the specs inside its bundle). | | `genesisConfig` | `{ "chainType", "properties", "validatorNameSuffix", "networkSuffix" }` — settings every chain in a genesis network shares: `chain-spec-builder -t`, `--properties`, and the relay node-name suffix zombienet derives validator keys from (`alice-paseo-validator`). Required on a genesis network. `networkSuffix` is optional: the namespace product contexts derive in, written into the genesis of the chains whose runtime carries the pallet holding it (People and Asset Hub). At most 16 bytes. Leave it out to keep whatever the runtime presets ship — a fork needs nothing here, since its runtime already has the value set. | | *(para ids)* | `parachains[].paraId` is the single source: the genesis TOML, the fork TOML, `ppn generate` and `assign-cores.sh` all read it from here. `config/ports.env` deliberately holds no para ids — those belong to the network, not the machine. | -| `relay.extraArgs`, `parachains[].extraArgs` | Extra node flags for that chain, appended to the shared per-key flag table (existing flags win over duplicates). | +| `relay.extraArgs`, `parachains[].extraArgs` | Extra node flags for that chain. A flag the shared per-key table also sets takes this value (`--rpc-max-connections=5000` replaces the table's 500); the rest are appended. | +| `parachains[].rpcNodes`, `parachains[].rpcNodeArgs` | Non-authoring full nodes (0..9) that serve the chain's public path instead of the collator, on the chain's port + 1, + 2, …, with nginx balancing across them. They run the collator's flags without the authoring ones, with `rpcNodeArgs` over them. | +| *(network overlay)* | `PPN_NETWORK_OVERLAY=` changes `extraArgs`, `rpcNodes` and `rpcNodeArgs` for one deployment without a release: `{ "previewnet": { "relay": { "extraArgs": [...] }, "parachains": { "people": { "rpcNodes": 2, "rpcNodeArgs": [...] } } } }`. Every command applies it, so the zombienet config, nginx routes and dashboard agree. An unknown network, chain or field is an error. | | `services` | Per-service switches and binaries: `false` disables a service; `{ "binary": { … } }` says what it runs. `eth-rpc` must match the revive pallet in the network's Asset Hub runtime, which is exactly why it is pinned per network rather than shared. | | `tools` | Local tools this network needs that are neither a chain nor a service — previewnet's `chain-spec-builder`, for instance, which only a genesis network uses. | | `bite.doppelganger` | `{repo, tag}` of the bite tool. Per network, because the build has to be able to execute the runtimes of the network being bitten. | diff --git a/packages/network-config/src/args.ts b/packages/network-config/src/args.ts new file mode 100644 index 0000000..ab2a5a3 --- /dev/null +++ b/packages/network-config/src/args.ts @@ -0,0 +1,11 @@ +/** The flag an arg sets: `--rpc-port` for `--rpc-port=9944`, `-lsync` for `-lsync=debug`. */ +export function flagOf(arg: string): string { + return arg.split('=')[0]; +} + +/** `args` with each flag `extra` also sets taking extra's value, and extra's other flags appended. */ +export function overrideArgs(args: string[], extra: string[]): string[] { + const byFlag = new Map(extra.map((a) => [flagOf(a), a])); + const present = new Set(args.map(flagOf)); + return [...args.map((a) => byFlag.get(flagOf(a)) ?? a), ...extra.filter((a) => !present.has(flagOf(a)))]; +} diff --git a/packages/network-config/src/dashboard-model.ts b/packages/network-config/src/dashboard-model.ts index be7bcf2..2eb0ec1 100644 --- a/packages/network-config/src/dashboard-model.ts +++ b/packages/network-config/src/dashboard-model.ts @@ -9,7 +9,7 @@ // which is the one thing that differs per environment (127.0.0.1 vs a domain). import type { NetworkDef, Parachain } from './networks.js'; -import { PORTS, RELAY_BASE_PORT, VALIDATORS, requiredPort } from './toml-generator.js'; +import { PORTS, RELAY_BASE_PORT, VALIDATORS, requiredPort, rpcNodePorts } from './toml-generator.js'; import { iceServers, runsTurnRelay, type IceServer } from './turn.js'; /** Bumped when the shape changes incompatibly. Renderers support N and N-1. */ @@ -50,6 +50,8 @@ export interface DashboardEndpoint { export interface DashboardChain extends DashboardEndpoint { paraId: number | null; + /** Every local port the path balances across: the chain's RPC nodes, else `port` alone. */ + upstreamPorts: number[]; } /** The TURN/STUN relay: what a WebRTC client puts in `iceServers`, and where to get credentials. */ @@ -115,6 +117,7 @@ export function dashboardModel(net: NetworkDef, baseUrl: string): DashboardModel paraId: null, path, port, + upstreamPorts: [port], protocol: 'ws', url, directUrl: `ws://127.0.0.1:${port}`, @@ -126,7 +129,9 @@ export function dashboardModel(net: NetworkDef, baseUrl: string): DashboardModel for (const p of net.parachains) { const path = `/${p.key}`; - const port = PORTS[p.key as Parachain]; + const rpcPorts = rpcNodePorts(p); + const upstreamPorts = rpcPorts.length ? rpcPorts : [PORTS[p.key as Parachain]]; + const port = upstreamPorts[0]; const url = wsUrl(base, path); chains.push({ id: p.key, @@ -134,6 +139,7 @@ export function dashboardModel(net: NetworkDef, baseUrl: string): DashboardModel paraId: p.paraId, path, port, + upstreamPorts, protocol: 'ws', url, directUrl: `ws://127.0.0.1:${port}`, diff --git a/packages/network-config/src/fork-toml.ts b/packages/network-config/src/fork-toml.ts index 6a15f53..514704c 100644 --- a/packages/network-config/src/fork-toml.ts +++ b/packages/network-config/src/fork-toml.ts @@ -48,12 +48,14 @@ import { addMissing, tomlArgs, requiredPort, + rpcNodeBlocks, } from './toml-generator.js'; import type { Parachain } from './types.js'; import type { ForkManifest } from './bundle.js'; import { DEFAULT_NETWORK, type NetworkDef } from './networks.js'; import { loadNetwork } from './load.js'; import { dubCustomProcesses } from './dub.js'; +import { overrideArgs } from './args.js'; // --------------------------------------------------------------------------- // Bundle manifest — built by ./fork/manifest.ts, extended with biteBlocks by bite.sh @@ -229,7 +231,7 @@ function parachainSection( const paraId = chain.paraId as number; const specPath = `${bundleDir}/specs/${chain.spec}.json`; const args = addMissing( - addMissing( + overrideArgs( [ `--relay-chain-rpc-urls=ws://127.0.0.1:${RELAY_BASE_PORT}`, ...buildArgs(key, undefined, key === 'bulletin' && enableHop), @@ -261,6 +263,7 @@ function parachainSection( // restoring the same snapshot. Only the first holds the documented ports; the rest take // whatever zombienet hands out, and drop a fixed listen address that would collide. const count = manifest.topology?.collators[key] ?? 1; + const launch = `command = "${scriptsDir}/omni-node.sh"\ndb_snapshot = "${bundleDir}/snapshots/${paraId}.tgz"${envLine}`; const collators = Array.from({ length: count }, (_, i) => i === 0 ? ` @@ -268,15 +271,13 @@ function parachainSection( name = "${collatorNodeName(paraId, i)}" rpc_port = ${PORTS[key]} p2p_port = ${P2P_PORTS[key]} -command = "${scriptsDir}/omni-node.sh" -db_snapshot = "${bundleDir}/snapshots/${paraId}.tgz"${envLine} +${launch} args = ${tomlArgs(args)} ` : ` [[parachains.collators]] name = "${collatorNodeName(paraId, i)}" -command = "${scriptsDir}/omni-node.sh" -db_snapshot = "${bundleDir}/snapshots/${paraId}.tgz"${envLine} +${launch} args = ${tomlArgs(args.filter((a) => !a.startsWith('--listen-addr=')))} ` ); @@ -288,7 +289,7 @@ id = ${paraId} cumulus_based = true chain = "${chainIdOf(specPath)}" chain_spec_path = "${specPath}" -${collators.join('')}`; +${collators.join('')}${rpcNodeBlocks(desc, args, launch)}`; } /** @@ -369,7 +370,7 @@ export function generateForkToml(options: GenerateForkTomlOptions): string { const present = new Set(parachains); const relaySpecPath = `${bundleDir}/specs/${manifest.chains.relay.spec}.json`; const relayArgs = addMissing( - addMissing(buildArgs('relay', FORK_RELAY_LOGS), net.relay.extraArgs), + overrideArgs(buildArgs('relay', FORK_RELAY_LOGS), net.relay.extraArgs), FORK_RELAY_ARGS ); diff --git a/packages/network-config/src/load.ts b/packages/network-config/src/load.ts index b47e4bd..a38fba9 100644 --- a/packages/network-config/src/load.ts +++ b/packages/network-config/src/load.ts @@ -6,14 +6,18 @@ import { loadDescriptor, currentNetworkName, type NetworkDef } from './networks.js'; import { applyOverrides, effectiveOverrides, type OverrideSet } from './overrides.js'; +import { applyOverlay } from './overlay.js'; /** * A network as it will actually run: the descriptor, with any --release/--binary overrides - * applied. Every command goes through here, so none of them can disagree about which binary - * comes from where. `extra` carries flag overrides; the environment is always consulted. + * and the PPN_NETWORK_OVERLAY file applied. Every command goes through here, so none of them + * can disagree about which binary comes from where or which nodes run. `extra` carries flag + * overrides; the environment is always consulted. */ export function loadNetwork(name: string, extra?: OverrideSet): NetworkDef { - return applyOverrides(loadDescriptor(name), effectiveOverrides(extra)); + const net = applyOverrides(loadDescriptor(name), effectiveOverrides(extra)); + const overlay = process.env.PPN_NETWORK_OVERLAY; + return overlay ? applyOverlay(net, overlay) : net; } /** The descriptor exactly as checked in, overrides ignored. For drift checks and tests. */ diff --git a/packages/network-config/src/networks.ts b/packages/network-config/src/networks.ts index 6418581..28d0689 100644 --- a/packages/network-config/src/networks.ts +++ b/packages/network-config/src/networks.ts @@ -117,16 +117,25 @@ export interface NetworkParachain { runtime?: RuntimeRef; /** The chain spec built for this chain at genesis. Genesis networks only. */ genesisSpec?: GenesisSpec; - /** Extra node args for this chain, appended to the shared per-key flag table. */ + /** Extra node args for this chain; a flag the shared per-key table also sets takes this value. */ extraArgs: string[]; /** * Curve this chain's Aura keys are on. Nearly every chain is sr25519; Polkadot's Asset Hub * is ed25519, and the collator authors nothing at all if the fork guesses wrong. */ aura?: AuraScheme; + /** Non-authoring full nodes that serve the chain's public RPC path instead of the collator. */ + rpcNodes: number; + /** Extra args for the RPC nodes, applied over the chain's own args. */ + rpcNodeArgs: string[]; } -export interface NetworkRelay extends Omit { +/** RPC node i of a parachain listens on the chain's port + i, below the next chain's port. */ +export const MAX_RPC_NODES = 9; + +type ChainFields = Omit; + +export interface NetworkRelay extends ChainFields { /** Built-in chain name doppelganger runs with (ZOMBIE_CHAIN), e.g. 'paseo'. */ chain: string; /** Dev-key validators the fork runs with, max 6 (alice…ferdie). */ @@ -136,7 +145,7 @@ export interface NetworkRelay extends Omit { } /** A chain of either kind, as networkChains() yields them. */ -export type NetworkChain = Omit & { +export type NetworkChain = ChainFields & { key: ChainKey; paraId: number | null; }; @@ -357,13 +366,7 @@ export function loadDescriptor(name: string): NetworkDef { runtimeFile.set(ref.file, at); return ref; }; - const checkArgs = (at: string, args: unknown): string[] => { - if (args === undefined) return []; - if (!Array.isArray(args) || args.some((x) => typeof x !== 'string' || !x.startsWith('-'))) { - bad(`${at}.extraArgs must be an array of "--flag" strings`); - } - return args as string[]; - }; + const checkArgs = (at: string, args: unknown): string[] => parseArgs(`${at}.extraArgs`, args, bad); checkBinary('relay', raw.relay.binary); checkRuntime('relay', raw.relay.runtime); @@ -467,6 +470,8 @@ export function loadDescriptor(name: string): NetworkDef { parachains: raw.parachains.map((p: NetworkParachain) => ({ ...p, extraArgs: checkArgs(`parachains.${p.key}`, p.extraArgs), + rpcNodes: parseRpcNodes(`parachains.${p.key}.rpcNodes`, p.rpcNodes, bad), + rpcNodeArgs: parseArgs(`parachains.${p.key}.rpcNodeArgs`, p.rpcNodeArgs, bad), })), services, tools, @@ -476,6 +481,24 @@ export function loadDescriptor(name: string): NetworkDef { }; } +/** `args` as node flags, or `bad` when it is not a list of "--flag" strings. Absent is none. */ +export function parseArgs(at: string, args: unknown, bad: (why: string) => never): string[] { + if (args === undefined) return []; + if (!Array.isArray(args) || args.some((x) => typeof x !== 'string' || !x.startsWith('-'))) { + bad(`${at} must be an array of "--flag" strings`); + } + return args as string[]; +} + +/** `count` as an RPC node count, or `bad` when it is not 0..MAX_RPC_NODES. Absent is 0. */ +export function parseRpcNodes(at: string, count: unknown, bad: (why: string) => never): number { + if (count === undefined) return 0; + if (!Number.isInteger(count) || (count as number) < 0 || (count as number) > MAX_RPC_NODES) { + bad(`${at} must be an integer 0..${MAX_RPC_NODES}`); + } + return count as number; +} + /** PPN_NETWORK, or previewnet. */ export function currentNetworkName(): string { return process.env.PPN_NETWORK || DEFAULT_NETWORK; diff --git a/packages/network-config/src/nginx-routes.ts b/packages/network-config/src/nginx-routes.ts index c716928..4d2e802 100644 --- a/packages/network-config/src/nginx-routes.ts +++ b/packages/network-config/src/nginx-routes.ts @@ -26,8 +26,10 @@ export function nginxRoutes(model: DashboardModel): { upstreams: string; locatio const upstreams = entries .map( - (e) => `upstream ${sanitize(e.id)} { - server 127.0.0.1:${e.port}; + // least_conn: a WebSocket stays open for the whole session, so round-robin by request + // would leave the nodes as unevenly loaded as the sessions happen to last. + (e) => `upstream ${sanitize(e.id)} {${e.upstreamPorts.length > 1 ? '\n least_conn;' : ''} +${e.upstreamPorts.map((port) => ` server 127.0.0.1:${port};`).join('\n')} keepalive 32; }` ) diff --git a/packages/network-config/src/overlay.ts b/packages/network-config/src/overlay.ts new file mode 100644 index 0000000..a68df0b --- /dev/null +++ b/packages/network-config/src/overlay.ts @@ -0,0 +1,72 @@ +// A deployment's own shape for a network, without a release of the engine: RPC nodes in front of +// a chain, or different flags on its nodes. Keyed by network, so loading another network (as +// `ppn networks` does) is unaffected by a section written for this one. +// +// PPN_NETWORK_OVERLAY=/etc/ppn/network-overlay.json +// { "previewnet": { +// "relay": { "extraArgs": ["--rpc-max-connections=2000"] }, +// "parachains": { "people": { "rpcNodes": 2, "rpcNodeArgs": ["--rpc-max-connections=5000"] } } } } + +import fs from 'node:fs'; +import { overrideArgs } from './args.js'; +import { listNetworks, parseArgs, parseRpcNodes, type NetworkDef } from './networks.js'; + +const RELAY_FIELDS = ['extraArgs']; +const PARACHAIN_FIELDS = ['extraArgs', 'rpcNodes', 'rpcNodeArgs']; + +/** + * `net` with the overlay file's section for it applied: its extraArgs and rpcNodeArgs override + * the descriptor's flag by flag, and rpcNodes replaces the count. Throws on an unreadable file, + * an unknown network, chain or field, or a value the descriptor itself would reject. + */ +export function applyOverlay(net: NetworkDef, file: string): NetworkDef { + const bad = (why: string): never => { + throw new Error(`${file}: ${why}`); + }; + let raw: Record; + try { + raw = JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch (error) { + return bad(`cannot read the network overlay: ${(error as Error).message}`); + } + const object = (at: string, value: unknown, fields?: string[]): Record => { + if (!value || typeof value !== 'object' || Array.isArray(value)) bad(`${at || 'the file'} must be a JSON object`); + const unknown = fields ? Object.keys(value as object).filter((k) => !fields.includes(k)) : []; + if (unknown.length) bad(`${at} has unknown fields ${unknown.join(', ')} (allowed: ${fields!.join(', ')})`); + return value as Record; + }; + + object('', raw); + const networks = listNetworks(); + const strangers = Object.keys(raw).filter((name) => !networks.includes(name)); + if (strangers.length) bad(`unknown networks ${strangers.join(', ')} (known: ${networks.join(', ')})`); + if (raw[net.name] === undefined) return net; + + const section = object(net.name, raw[net.name], ['relay', 'parachains']); + const relay = object(`${net.name}.relay`, section.relay ?? {}, RELAY_FIELDS); + const parachains = object(`${net.name}.parachains`, section.parachains ?? {}); + const declared = net.parachains.map((p) => p.key); + const undeclared = Object.keys(parachains).filter((key) => !declared.includes(key as never)); + if (undeclared.length) { + bad(`${net.name} has no parachains ${undeclared.join(', ')} (declared: ${declared.join(', ')})`); + } + + return { + ...net, + relay: { + ...net.relay, + extraArgs: overrideArgs(net.relay.extraArgs, parseArgs(`${net.name}.relay.extraArgs`, relay.extraArgs, bad)), + }, + parachains: net.parachains.map((p) => { + if (parachains[p.key] === undefined) return p; + const at = `${net.name}.parachains.${p.key}`; + const chain = object(at, parachains[p.key], PARACHAIN_FIELDS); + return { + ...p, + extraArgs: overrideArgs(p.extraArgs, parseArgs(`${at}.extraArgs`, chain.extraArgs, bad)), + rpcNodes: chain.rpcNodes === undefined ? p.rpcNodes : parseRpcNodes(`${at}.rpcNodes`, chain.rpcNodes, bad), + rpcNodeArgs: overrideArgs(p.rpcNodeArgs, parseArgs(`${at}.rpcNodeArgs`, chain.rpcNodeArgs, bad)), + }; + }), + }; +} diff --git a/packages/network-config/src/toml-generator.ts b/packages/network-config/src/toml-generator.ts index b2c9c4a..bd403f6 100644 --- a/packages/network-config/src/toml-generator.ts +++ b/packages/network-config/src/toml-generator.ts @@ -14,9 +14,10 @@ import path from 'node:path'; import { repoRoot } from './repo-root.js'; import { PARACHAIN_KEYS } from './types.js'; import type { Parachain, ChainKey, ChainDef, LogLevel, GenerateTomlOptions } from './types.js'; -import { DEFAULT_NETWORK, type NetworkDef } from './networks.js'; +import { DEFAULT_NETWORK, type NetworkDef, type NetworkParachain } from './networks.js'; import { loadNetwork } from './load.js'; import { dubCustomProcesses } from './dub.js'; +import { flagOf, overrideArgs } from './args.js'; const VALIDATORS = ['alice', 'bob', 'charlie', 'dave', 'eve', 'ferdie'] as const; @@ -127,6 +128,36 @@ const PORTS: Record = { bulletin: requiredPort('BULLETIN_PORT'), 'web3-storage': requiredPort('WEB3_STORAGE_PORT'), }; + +/** RPC ports of a parachain's RPC nodes: the chain's own port + 1..rpcNodes. */ +function rpcNodePorts(p: NetworkParachain): number[] { + return Array.from({ length: p.rpcNodes }, (_, i) => PORTS[p.key] + i + 1); +} + +// Authoring flags a full node does not take, and the collator's fixed listen address. +const COLLATOR_ONLY_FLAGS = new Set(['--collator', '--force-authoring', '--authoring', '--listen-addr']); + +/** + * The `[[parachains.collators]]` blocks of a chain's RPC nodes: non-validators running the + * collator's args without its authoring flags, with the chain's rpcNodeArgs over them. `launch` + * holds the lines each block shares with the collator (command, env, snapshot). + */ +function rpcNodeBlocks(p: NetworkParachain, collatorArgs: string[], launch: string): string { + const args = overrideArgs(collatorArgs.filter((a) => !COLLATOR_ONLY_FLAGS.has(flagOf(a))), p.rpcNodeArgs); + return rpcNodePorts(p) + .map( + (port, i) => ` +[[parachains.collators]] +name = "${p.key}-rpc${i + 1}" +validator = false +rpc_port = ${port} +${launch} +args = ${tomlArgs(args)} +` + ) + .join(''); +} + // Para ids belong to the network, not the machine, so they come from the descriptor — // ports.env keeps only what is local (which port each chain listens on). const paraIds = memo((): Record => { @@ -255,12 +286,10 @@ function buildArgs(chainKey: ChainKey, logTargets?: Record, en return args; } -// Append only flags the base list has not already set, so the shared per-key table -// wins over descriptor extras and fork defaults rather than being specified twice. +/** `args` plus the flags of `extra` it does not set already, so defaults never override. */ function addMissing(args: string[], extra: string[]): string[] { - const flag = (a: string) => a.split('=')[0]; - const present = new Set(args.map(flag)); - return [...args, ...extra.filter((a) => !present.has(flag(a)))]; + const present = new Set(args.map(flagOf)); + return [...args, ...extra.filter((a) => !present.has(flagOf(a)))]; } // The genesis collator command is always the omni-node.sh wrapper (it carries the @@ -278,13 +307,32 @@ function genesisChainArgs(key: ChainKey, logTargets?: Record, key === 'relay' ? genesisNet().relay.extraArgs : genesisNet().parachains.find((p) => p.key === key)?.extraArgs ?? []; - return addMissing(buildArgs(key, logTargets, enableHop), extras); + return overrideArgs(buildArgs(key, logTargets, enableHop), extras); } function tomlArgs(args: string[]): string { return '[' + args.map((a) => `"${a}"`).join(', ') + ']'; } +function genesisParachainSection(key: Parachain, title: string, args: string[]): string { + const launch = `command = "{{SCRIPTS}}/omni-node.sh"${genesisCollatorEnv(key)}`; + return ` +## ${title} +[[parachains]] +id = ${paraIds()[key]} +cumulus_based = true +chain = "${genesisSpecOf(key).chainId}" +chain_spec_path = "{{BIN}}/${genesisSpecOf(key).file}" + +[[parachains.collators]] +name = "${key}-collator1" +rpc_port = ${PORTS[key]} +p2p_port = ${P2P_PORTS[key]} +${launch} +args = ${tomlArgs(args)} +${rpcNodeBlocks(genesisChainOf(key) as NetworkParachain, args, launch)}`; +} + function validatorNodes(count: number): string { return VALIDATORS.slice(0, count) .map( @@ -433,72 +481,28 @@ default_args = ${relayArgs} ${validatorNodes(validatorCount)} ${section( has('asset-hub'), - ` -## Asset Hub Paseo Next (Parachain ${paraIds()['asset-hub']}) - Elastic Scaling with 2-second blocks -[[parachains]] -id = ${paraIds()['asset-hub']} -cumulus_based = true -chain = "${genesisSpecOf('asset-hub').chainId}" -chain_spec_path = "{{BIN}}/${genesisSpecOf('asset-hub').file}" - -[[parachains.collators]] -name = "asset-hub-collator1" -rpc_port = ${PORTS['asset-hub']} -p2p_port = ${P2P_PORTS['asset-hub']}${genesisCollatorEnv('asset-hub')} -command = "{{SCRIPTS}}/omni-node.sh" -args = ${tomlArgs(genesisChainArgs('asset-hub', lt['asset-hub']))} -` + genesisParachainSection( + 'asset-hub', + `Asset Hub Paseo Next (Parachain ${paraIds()['asset-hub']}) - Elastic Scaling with 2-second blocks`, + genesisChainArgs('asset-hub', lt['asset-hub']) + ) )}${section( has('people'), - ` -## People Chain (Parachain ${paraIds().people}) -[[parachains]] -id = ${paraIds().people} -cumulus_based = true -chain = "${genesisSpecOf('people').chainId}" -chain_spec_path = "{{BIN}}/${genesisSpecOf('people').file}" - -[[parachains.collators]] -name = "people-collator1" -rpc_port = ${PORTS.people} -p2p_port = ${P2P_PORTS.people}${genesisCollatorEnv('people')} -command = "{{SCRIPTS}}/omni-node.sh" -args = ${tomlArgs(genesisChainArgs('people', lt.people))} -` + genesisParachainSection('people', `People Chain (Parachain ${paraIds().people})`, genesisChainArgs('people', lt.people)) )}${section( has('bulletin'), - ` -## Bulletin Chain (Parachain ${paraIds().bulletin}) -[[parachains]] -id = ${paraIds().bulletin} -cumulus_based = true -chain = "${genesisSpecOf('bulletin').chainId}" -chain_spec_path = "{{BIN}}/${genesisSpecOf('bulletin').file}" - -[[parachains.collators]] -name = "bulletin-collator1" -rpc_port = ${PORTS.bulletin} -p2p_port = ${P2P_PORTS.bulletin}${genesisCollatorEnv('bulletin')} -command = "{{SCRIPTS}}/omni-node.sh" -args = ${tomlArgs(genesisChainArgs('bulletin', lt.bulletin, options.enableHop))} -` + genesisParachainSection( + 'bulletin', + `Bulletin Chain (Parachain ${paraIds().bulletin})`, + genesisChainArgs('bulletin', lt.bulletin, options.enableHop) + ) )}${section( has('web3-storage'), - ` -## Web3 Storage Chain (Parachain ${paraIds()['web3-storage']}) -[[parachains]] -id = ${paraIds()['web3-storage']} -cumulus_based = true -chain = "${genesisSpecOf('web3-storage').chainId}" -chain_spec_path = "{{BIN}}/${genesisSpecOf('web3-storage').file}" - -[[parachains.collators]] -name = "web3-storage-collator1" -rpc_port = ${PORTS['web3-storage']} -p2p_port = ${P2P_PORTS['web3-storage']}${genesisCollatorEnv('web3-storage')} -command = "{{SCRIPTS}}/omni-node.sh" -args = ${tomlArgs(genesisChainArgs('web3-storage', lt['web3-storage']))} -` + genesisParachainSection( + 'web3-storage', + `Web3 Storage Chain (Parachain ${paraIds()['web3-storage']})`, + genesisChainArgs('web3-storage', lt['web3-storage']) + ) )}${section(both('people', 'bulletin'), hrmpPair(paraIds().people, paraIds().bulletin))} ${section(both('people', 'asset-hub'), hrmpPair(paraIds().people, paraIds()['asset-hub']))} ${section( @@ -620,4 +624,6 @@ export { addMissing, tomlArgs, requiredPort, + rpcNodePorts, + rpcNodeBlocks, }; diff --git a/packages/network-config/tests/fork-toml.test.ts b/packages/network-config/tests/fork-toml.test.ts index 326f3f0..1efa8b6 100644 --- a/packages/network-config/tests/fork-toml.test.ts +++ b/packages/network-config/tests/fork-toml.test.ts @@ -630,3 +630,21 @@ describe('generateForkToml — provenance', () => { assert.ok(toml.endsWith('\n')); }); }); + +describe('generateForkToml — RPC nodes from the network overlay', () => { + it('restores each RPC node from the chain snapshot, on the ports after the collator', () => { + const overlay = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-')), 'overlay.json'); + fs.writeFileSync(overlay, JSON.stringify({ previewnet: { parachains: { people: { rpcNodes: 1 } } } })); + process.env.PPN_NETWORK_OVERLAY = overlay; + try { + const block = generate().split('name = "people-rpc1"')[1].split('[[')[0]; + assert.match(block, /validator = false/); + assert.match(block, new RegExp(`rpc_port = ${PORTS.people + 1}`)); + assert.match(block, new RegExp(`db_snapshot = ".*/snapshots/${paraIds().people}\\.tgz"`)); + assert.match(block, /--relay-chain-rpc-urls=/); + assert.doesNotMatch(block, /--force-authoring/); + } finally { + delete process.env.PPN_NETWORK_OVERLAY; + } + }); +}); diff --git a/packages/network-config/tests/overlay.test.ts b/packages/network-config/tests/overlay.test.ts new file mode 100644 index 0000000..466d15b --- /dev/null +++ b/packages/network-config/tests/overlay.test.ts @@ -0,0 +1,115 @@ +// A deployment reshapes a network through PPN_NETWORK_OVERLAY instead of an engine release: +// RPC nodes in front of a chain that players hammer, and flags such as the connection limit. +// What matters is that every consumer of the network agrees on the result, so these check the +// generated zombienet config, the nginx routes and the dashboard together, and that a typo in +// the file fails loudly instead of deploying the descriptor's defaults. + +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-')); +const write = (name: string, body: unknown): string => { + const file = path.join(dir, name); + fs.writeFileSync(file, JSON.stringify(body)); + return file; +}; + +// Set before anything loads the network: the genesis generator reads it once per process. +process.env.PPN_NETWORK_OVERLAY = write('people.json', { + previewnet: { + parachains: { + people: { + rpcNodes: 2, + extraArgs: ['--rpc-max-connections=50'], + rpcNodeArgs: ['--rpc-max-connections=5000', '--rpc-message-buffer-capacity-per-connection=512'], + }, + }, + }, + // Another network's section must not affect previewnet, nor fail on previewnet's chains. + devnet: { relay: { extraArgs: ['--rpc-max-connections=7'] } }, +}); + +const { generateToml, VALID_PARACHAINS, PORTS } = await import('../src/toml-generator.js'); +const { loadNetwork, loadDescriptorOnly } = await import('../src/load.js'); +const { applyOverlay } = await import('../src/overlay.js'); +const { dashboardModel } = await import('../src/dashboard-model.js'); +const { nginxRoutes } = await import('../src/nginx-routes.js'); + +/** name -> the TOML lines of that node's block. */ +function nodeBlocks(toml: string): Map { + const blocks = new Map(); + for (const block of toml.split(/^\[\[/m).slice(1)) { + const name = /^name = "([^"]+)"/m.exec(block)?.[1]; + if (name) blocks.set(name, block.split('\n')); + } + return blocks; +} + +const argsOf = (lines: string[]): string[] => + [...(lines.find((l) => l.startsWith('args = ')) ?? '').matchAll(/"([^"]+)"/g)].map((m) => m[1]); + +describe('network overlay: RPC nodes in front of People', () => { + const nodes = nodeBlocks(generateToml([...VALID_PARACHAINS])); + const collator = nodes.get('people-collator1')!; + const rpc = [nodes.get('people-rpc1')!, nodes.get('people-rpc2')!]; + + it('spawns full nodes on the ports after the collator, which keeps authoring', () => { + for (const [i, block] of rpc.entries()) { + assert.ok(block.includes('validator = false')); + assert.ok(block.includes(`rpc_port = ${PORTS.people + i + 1}`)); + assert.ok(!argsOf(block).some((a) => a.startsWith('--force-authoring') || a.startsWith('--authoring'))); + assert.ok(argsOf(block).includes('--enable-statement-store')); + } + assert.ok(argsOf(collator).includes('--force-authoring')); + }); + + // The engine's own table sets --rpc-max-connections=500; dropping the deployment's value + // in favour of it is what capped a game at a few hundred players. + it('gives the deployment the last word on a flag the engine also sets', () => { + assert.ok(argsOf(collator).includes('--rpc-max-connections=50')); + assert.ok(!argsOf(collator).includes('--rpc-max-connections=500')); + for (const block of rpc) { + assert.ok(argsOf(block).includes('--rpc-max-connections=5000')); + assert.ok(argsOf(block).includes('--rpc-message-buffer-capacity-per-connection=512')); + } + }); + + it('routes the public People path across the RPC nodes and not the collator', () => { + const model = dashboardModel(loadNetwork('previewnet'), 'https://example.test'); + const people = model.chains.find((c) => c.id === 'people')!; + assert.deepEqual(people.upstreamPorts, [PORTS.people + 1, PORTS.people + 2]); + assert.equal(people.port, PORTS.people + 1); + + const upstream = nginxRoutes(model).upstreams.split('\n\n').find((u) => u.startsWith('upstream people '))!; + assert.match(upstream, /least_conn;/); + assert.match(upstream, new RegExp(`server 127\\.0\\.0\\.1:${PORTS.people + 2};`)); + assert.doesNotMatch(upstream, new RegExp(`:${PORTS.people};`)); + }); + + it('leaves the other network untouched', () => { + const devnet = loadNetwork('devnet'); + assert.deepEqual(devnet.relay.extraArgs, ['--rpc-max-connections=7']); + assert.ok(devnet.parachains.every((p) => p.rpcNodes === 0)); + }); +}); + +describe('network overlay: a mistake fails instead of deploying defaults', () => { + const net = loadDescriptorOnly('previewnet'); + const rejects = (body: unknown, message: RegExp) => + assert.throws(() => applyOverlay(net, write('bad.json', body)), message); + + it('rejects an unknown network', () => rejects({ previewnett: {} }, /unknown networks previewnett/)); + it('rejects a chain the network does not have', () => + rejects({ previewnet: { parachains: { coretime: {} } } }, /no parachains coretime/)); + it('rejects an unknown field', () => + rejects({ previewnet: { parachains: { people: { rpcNode: 2 } } } }, /unknown fields rpcNode/)); + it('rejects more RPC nodes than the port gap holds', () => + rejects({ previewnet: { parachains: { people: { rpcNodes: 10 } } } }, /rpcNodes must be an integer 0\.\.9/)); + it('rejects args that are not flags', () => + rejects({ previewnet: { relay: { extraArgs: ['rpc-max-connections=1'] } } }, /must be an array of "--flag" strings/)); + it('rejects a file that is not there', () => + assert.throws(() => applyOverlay(net, path.join(dir, 'missing.json')), /cannot read the network overlay/)); +}); From 87e516c157852ef21e8f5cab8d1699c276fbed3a Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:16:49 +0000 Subject: [PATCH 02/19] Hand PPN_NETWORK_OVERLAY to custom processes through ports.local.env --- packages/cli/src/commands/start.ts | 1 + packages/cli/src/lib/spawn-env.ts | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 3758d00..b82dd48 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -605,6 +605,7 @@ export async function start(args: string[], opts: StartOptions = {}): Promise Date: Thu, 12 Nov 2026 23:17:20 +0000 Subject: [PATCH 03/19] Require a server to state PPN_NETWORK for its custom processes --- packages/cli/src/lib/spawn-env.ts | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/packages/cli/src/lib/spawn-env.ts b/packages/cli/src/lib/spawn-env.ts index ff5faa7..ca48fdd 100644 --- a/packages/cli/src/lib/spawn-env.ts +++ b/packages/cli/src/lib/spawn-env.ts @@ -156,18 +156,4 @@ export function spawnEnvKeys(): string[] { export const WAIVED: Record = { // A server persists by definition; ephemeral is a laptop mode. Nothing reads it there. EPHEMERAL: 'ephemeral is a local-only mode; a deployment always persists', - - // KNOWN GAP, deliberately recorded rather than silently waived. - // - // `ppn start` sets it so every service resolves the network the user asked for. - // On a server nothing sets it, and currentNetworkName() (network-config/src/networks.ts) - // falls back to previewnet. Staging and prod run previewnet, so this is *accidentally* - // correct today and breaks nothing — but the first non-previewnet deployment would have - // every service resolve previewnet's chains, ports and bin directory instead. - // - // Not fixed here because the fix belongs with `ppn start --server` (see docs/DASHBOARD.md, - // "Why a server does not run `ppn start`"), which makes one path the sole author of all of - // this. Until then it is written down, and the conformance test keeps it from being - // forgotten. - PPN_NETWORK: 'KNOWN GAP: falls back to previewnet, which is what servers run today', }; From 6233f4d97cd4852c8bdbf322f99c1d741a77a4bb Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:17:32 +0000 Subject: [PATCH 04/19] Assign a fixed Prometheus port to every node with a fixed RPC port --- .github/workflows/zombienet-tests.yml | 4 +++- config/ports.env | 4 ++++ packages/network-config/src/fork-toml.ts | 3 +++ packages/network-config/src/toml-generator.ts | 9 +++++++++ packages/network-config/tests/toml-generator.test.ts | 9 +++++++++ zombienet-configs/local-dev.toml | 10 ++++++++++ 6 files changed, 38 insertions(+), 1 deletion(-) diff --git a/.github/workflows/zombienet-tests.yml b/.github/workflows/zombienet-tests.yml index 1e0aec7..a94399b 100644 --- a/.github/workflows/zombienet-tests.yml +++ b/.github/workflows/zombienet-tests.yml @@ -365,7 +365,7 @@ jobs: # Additional wait for nodes to be ready sleep 30 - - name: Verify the People RPC node serves the chain + - name: Verify the People RPC node serves the chain and its metrics run: | source ./config/ports.env RPC_PORT=$((PEOPLE_PORT + 1)) @@ -384,6 +384,8 @@ jobs: roles=$(rpc system_nodeRoles) echo "people-rpc1 roles: $roles" [ "$roles" = '["Full"]' ] || { echo "ERROR: people-rpc1 should be a full node, is $roles"; exit 1; } + curl -sf -m 5 "http://127.0.0.1:$((RPC_PORT + PROMETHEUS_PORT_OFFSET))/metrics" | grep -q '^substrate_block_height' \ + || { echo "ERROR: no metrics on $((RPC_PORT + PROMETHEUS_PORT_OFFSET))"; exit 1; } - name: Verify raw chainspecs have bootNodes run: | diff --git a/config/ports.env b/config/ports.env index 9fef1f7..b445948 100644 --- a/config/ports.env +++ b/config/ports.env @@ -17,6 +17,10 @@ ASSET_HUB_PORT=10020 BULLETIN_PORT=10030 WEB3_STORAGE_PORT=10040 +# Each node's Prometheus port is its RPC port plus this: 20000 for alice, 20010 for the People +# collator, 20011 for its first RPC node. +PROMETHEUS_PORT_OFFSET=10000 + # Parachain IDs are NOT here: they belong to the network, not the machine, and two # networks use different ones (previewnet 1500+, devnet 1000+). Each chain declares # its own in networks/.json — see networks/README.md. This file holds only what diff --git a/packages/network-config/src/fork-toml.ts b/packages/network-config/src/fork-toml.ts index 514704c..5535ba4 100644 --- a/packages/network-config/src/fork-toml.ts +++ b/packages/network-config/src/fork-toml.ts @@ -48,6 +48,7 @@ import { addMissing, tomlArgs, requiredPort, + prometheusPort, rpcNodeBlocks, } from './toml-generator.js'; import type { Parachain } from './types.js'; @@ -213,6 +214,7 @@ function relayNodes(count: number): string { name = "${relayNodeName(i)}" validator = true rpc_port = ${RELAY_BASE_PORT + i}${i === 0 ? `\np2p_port = ${P2P_PORTS.relay}` : ''} +prometheus_port = ${prometheusPort(RELAY_BASE_PORT + i)} env = [{ name = "ZOMBIE_DISPUTE_CANDIDATE_LIFETIME_AFTER_FINALIZATION", value = "1" }]` ).join('\n'); } @@ -271,6 +273,7 @@ function parachainSection( name = "${collatorNodeName(paraId, i)}" rpc_port = ${PORTS[key]} p2p_port = ${P2P_PORTS[key]} +prometheus_port = ${prometheusPort(PORTS[key])} ${launch} args = ${tomlArgs(args)} ` diff --git a/packages/network-config/src/toml-generator.ts b/packages/network-config/src/toml-generator.ts index bd403f6..43546d8 100644 --- a/packages/network-config/src/toml-generator.ts +++ b/packages/network-config/src/toml-generator.ts @@ -129,6 +129,11 @@ const PORTS: Record = { 'web3-storage': requiredPort('WEB3_STORAGE_PORT'), }; +/** The Prometheus port of a node whose RPC port is `rpcPort`. */ +function prometheusPort(rpcPort: number): number { + return rpcPort + requiredPort('PROMETHEUS_PORT_OFFSET'); +} + /** RPC ports of a parachain's RPC nodes: the chain's own port + 1..rpcNodes. */ function rpcNodePorts(p: NetworkParachain): number[] { return Array.from({ length: p.rpcNodes }, (_, i) => PORTS[p.key] + i + 1); @@ -151,6 +156,7 @@ function rpcNodeBlocks(p: NetworkParachain, collatorArgs: string[], launch: stri name = "${p.key}-rpc${i + 1}" validator = false rpc_port = ${port} +prometheus_port = ${prometheusPort(port)} ${launch} args = ${tomlArgs(args)} ` @@ -328,6 +334,7 @@ chain_spec_path = "{{BIN}}/${genesisSpecOf(key).file}" name = "${key}-collator1" rpc_port = ${PORTS[key]} p2p_port = ${P2P_PORTS[key]} +prometheus_port = ${prometheusPort(PORTS[key])} ${launch} args = ${tomlArgs(args)} ${rpcNodeBlocks(genesisChainOf(key) as NetworkParachain, args, launch)}`; @@ -341,6 +348,7 @@ function validatorNodes(count: number): string { name = "${v}-${genesisConfig().validatorNameSuffix}" validator = true rpc_port = ${RELAY_BASE_PORT + i}${i === 0 ? `\np2p_port = ${P2P_PORTS.relay}` : ''} +prometheus_port = ${prometheusPort(RELAY_BASE_PORT + i)} balance = 1000000000000000` ) .join('\n'); @@ -624,6 +632,7 @@ export { addMissing, tomlArgs, requiredPort, + prometheusPort, rpcNodePorts, rpcNodeBlocks, }; diff --git a/packages/network-config/tests/toml-generator.test.ts b/packages/network-config/tests/toml-generator.test.ts index 4328786..59f6dae 100644 --- a/packages/network-config/tests/toml-generator.test.ts +++ b/packages/network-config/tests/toml-generator.test.ts @@ -234,6 +234,15 @@ describe('generateToml', () => { assert.ok(toml.includes(`p2p_port = ${P2P_PORTS.bulletin}`)); // bulletin }); + // Metrics are scraped from fixed ports, so every node with a fixed RPC port needs one too, + // and no two may collide. + it('gives every node a Prometheus port at its RPC port plus the offset', () => { + const toml = generateToml([...VALID_PARACHAINS]); + const rpc = [...toml.matchAll(/^rpc_port = (\d+)$/gm)].map((m) => Number(m[1])); + const prometheus = [...toml.matchAll(/^prometheus_port = (\d+)$/gm)].map((m) => Number(m[1])); + assert.deepEqual(prometheus, rpc.map((port) => port + 10000)); + }); + it('does not produce triple blank lines', () => { const combos: Parachain[][] = [ ['asset-hub'], diff --git a/zombienet-configs/local-dev.toml b/zombienet-configs/local-dev.toml index 2b55865..ab4df9a 100644 --- a/zombienet-configs/local-dev.toml +++ b/zombienet-configs/local-dev.toml @@ -21,36 +21,42 @@ name = "alice-paseo-validator" validator = true rpc_port = 10000 p2p_port = 30334 +prometheus_port = 20000 balance = 1000000000000000 [[relaychain.nodes]] name = "bob-paseo-validator" validator = true rpc_port = 10001 +prometheus_port = 20001 balance = 1000000000000000 [[relaychain.nodes]] name = "charlie-paseo-validator" validator = true rpc_port = 10002 +prometheus_port = 20002 balance = 1000000000000000 [[relaychain.nodes]] name = "dave-paseo-validator" validator = true rpc_port = 10003 +prometheus_port = 20003 balance = 1000000000000000 [[relaychain.nodes]] name = "eve-paseo-validator" validator = true rpc_port = 10004 +prometheus_port = 20004 balance = 1000000000000000 [[relaychain.nodes]] name = "ferdie-paseo-validator" validator = true rpc_port = 10005 +prometheus_port = 20005 balance = 1000000000000000 ## Asset Hub Paseo Next (Parachain 1500) - Elastic Scaling with 2-second blocks @@ -64,6 +70,7 @@ chain_spec_path = "{{BIN}}/asset_hub.json" name = "asset-hub-collator1" rpc_port = 10020 p2p_port = 30335 +prometheus_port = 20020 command = "{{SCRIPTS}}/omni-node.sh" args = ["-lxcm=trace", "--rpc-max-connections=500", "--force-authoring", "--authoring=slot-based", "--unsafe-rpc-external", "--rpc-max-response-size=50", "--listen-addr=/ip4/127.0.0.1/udp/30335/webrtc-direct"] @@ -78,6 +85,7 @@ chain_spec_path = "{{BIN}}/individuality.json" name = "people-collator1" rpc_port = 10010 p2p_port = 30336 +prometheus_port = 20010 command = "{{SCRIPTS}}/omni-node.sh" args = ["-lparachain=debug,xcm=trace", "--rpc-max-connections=500", "--force-authoring", "--enable-statement-store", "--network-backend=libp2p", "--authoring=slot-based", "--unsafe-rpc-external", "--rpc-max-response-size=50"] @@ -92,6 +100,7 @@ chain_spec_path = "{{BIN}}/bulletin.json" name = "bulletin-collator1" rpc_port = 10030 p2p_port = 30333 +prometheus_port = 20030 command = "{{SCRIPTS}}/omni-node.sh" args = ["-lparachain=debug,xcm=trace,sub-libp2p::bitswap=trace,runtime::transaction-storage=trace", "--rpc-max-connections=500", "--force-authoring", "--ipfs-server", "--authoring=slot-based", "--unsafe-rpc-external", "--rpc-max-response-size=50", "--listen-addr=/ip4/127.0.0.1/udp/30333/webrtc-direct", "--enable-hop", "--hop-retention-secs=3600", "--hop-promotion-buffer-secs=1800", "--hop-check-interval=60"] @@ -106,6 +115,7 @@ chain_spec_path = "{{BIN}}/web3-storage.json" name = "web3-storage-collator1" rpc_port = 10040 p2p_port = 30337 +prometheus_port = 20040 command = "{{SCRIPTS}}/omni-node.sh" args = ["-lruntime=info", "--collator", "--rpc-max-connections=500", "--unsafe-rpc-external", "--rpc-max-response-size=50", "--listen-addr=/ip4/127.0.0.1/udp/30337/webrtc-direct"] From 1a0236df0777832cf3df78dde40136ed8f52fae1 Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:41:22 +0000 Subject: [PATCH 05/19] Spawn an overlay's RPC nodes from a local genesis start --- .gitignore | 1 + packages/cli/src/commands/start.ts | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2b99af1..5ee8a47 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ fork-bundle*/ fork.toml # What `ppn start` spawns: the config above with each custom process's environment (lib/spawn-env.ts) *.spawn.toml +*.overlay.toml # Working notes, kept locally rather than in the repo docs/issues-drafts/ diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index b82dd48..0c27471 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -321,7 +321,15 @@ async function ensureDeps(netDef: NetworkDef, opts: StartOptions, binDir: string if (!opts.fork) { const { run: generate } = await import('./generate.js'); await generate([binDir], { ifNeeded: true }); - return opts.toml ?? path.join(REPO, 'zombienet-configs', 'local-dev.toml'); + if (opts.toml) return opts.toml; + // The checked-in config has no overlay in it, so an overlay's nodes need one generated. + if (!process.env.PPN_NETWORK_OVERLAY) return path.join(REPO, 'zombienet-configs', 'local-dev.toml'); + const overlaid = path.join(WS, 'zombienet-configs', 'local-dev.overlay.toml'); + fs.mkdirSync(path.dirname(overlaid), { recursive: true }); + const generated = spawnSync(nodeBin, [ppn, 'genesis-toml'], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'inherit'] }); + if (generated.status !== 0) throw new Error('ppn genesis-toml failed'); + fs.writeFileSync(overlaid, generated.stdout); + return overlaid; } const forkDir = forkDirFor(netDef.name); From 3088f1a4c0b4f439c9c3b3e1b9f77c506f25b3f5 Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:41:22 +0000 Subject: [PATCH 06/19] Exclude a shell's network overlay from the checked-in config --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8d42e2d..46a7097 100644 --- a/Makefile +++ b/Makefile @@ -240,7 +240,7 @@ build build-spawner: # Regenerate local-dev.toml from @ppn/network-config (dev only). # Run this after editing the generator — do not edit local-dev.toml directly. generate-toml: build-spawner - @node $(CURDIR)/bin/ppn.mjs genesis-toml > $(CONFIG_DIR)/local-dev.toml + @PPN_NETWORK_OVERLAY= node $(CURDIR)/bin/ppn.mjs genesis-toml > $(CONFIG_DIR)/local-dev.toml @echo "✓ local-dev.toml regenerated" # ============================================================================= From 6abbfeeb012afff662bf76869b9a20432f465604 Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:41:22 +0000 Subject: [PATCH 07/19] Reject RPC node ports that ports.env assigns to another node --- packages/network-config/src/toml-generator.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/network-config/src/toml-generator.ts b/packages/network-config/src/toml-generator.ts index 43546d8..0215320 100644 --- a/packages/network-config/src/toml-generator.ts +++ b/packages/network-config/src/toml-generator.ts @@ -134,9 +134,15 @@ function prometheusPort(rpcPort: number): number { return rpcPort + requiredPort('PROMETHEUS_PORT_OFFSET'); } -/** RPC ports of a parachain's RPC nodes: the chain's own port + 1..rpcNodes. */ +/** RPC ports of a parachain's RPC nodes: the chain's own port + 1..rpcNodes. Throws on a port another chain uses. */ function rpcNodePorts(p: NetworkParachain): number[] { - return Array.from({ length: p.rpcNodes }, (_, i) => PORTS[p.key] + i + 1); + const ports = Array.from({ length: p.rpcNodes }, (_, i) => PORTS[p.key] + i + 1); + const used = [...Object.values(PORTS), ...VALIDATORS.map((_, i) => RELAY_BASE_PORT + i)]; + const clash = ports.find((port) => used.includes(port)); + if (clash !== undefined) { + throw new Error(`${p.rpcNodes} RPC nodes for ${p.key} would take port ${clash}, which ports.env gives another node`); + } + return ports; } // Authoring flags a full node does not take, and the collator's fixed listen address. From 53e4d2a1e1dd835fa62aa22a814324f0643f4113 Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:41:22 +0000 Subject: [PATCH 08/19] Leave RPC nodes out of the genesis collator invulnerables --- packages/network-config/src/toml-generator.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/network-config/src/toml-generator.ts b/packages/network-config/src/toml-generator.ts index 0215320..f0b7829 100644 --- a/packages/network-config/src/toml-generator.ts +++ b/packages/network-config/src/toml-generator.ts @@ -161,6 +161,7 @@ function rpcNodeBlocks(p: NetworkParachain, collatorArgs: string[], launch: stri [[parachains.collators]] name = "${p.key}-rpc${i + 1}" validator = false +invulnerable = false rpc_port = ${port} prometheus_port = ${prometheusPort(port)} ${launch} From 32fadd15c1f3db6a1f286a5c6a3dc9cb6acdb7b6 Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:41:22 +0000 Subject: [PATCH 09/19] Cover RPC node Prometheus ports and fractional counts in overlay tests --- packages/network-config/tests/overlay.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/network-config/tests/overlay.test.ts b/packages/network-config/tests/overlay.test.ts index 466d15b..a9d18f4 100644 --- a/packages/network-config/tests/overlay.test.ts +++ b/packages/network-config/tests/overlay.test.ts @@ -60,14 +60,14 @@ describe('network overlay: RPC nodes in front of People', () => { for (const [i, block] of rpc.entries()) { assert.ok(block.includes('validator = false')); assert.ok(block.includes(`rpc_port = ${PORTS.people + i + 1}`)); + assert.ok(block.includes(`prometheus_port = ${PORTS.people + i + 1 + 10000}`)); assert.ok(!argsOf(block).some((a) => a.startsWith('--force-authoring') || a.startsWith('--authoring'))); assert.ok(argsOf(block).includes('--enable-statement-store')); } assert.ok(argsOf(collator).includes('--force-authoring')); }); - // The engine's own table sets --rpc-max-connections=500; dropping the deployment's value - // in favour of it is what capped a game at a few hundred players. + // A deployment has to be able to raise a limit the engine's table sets. it('gives the deployment the last word on a flag the engine also sets', () => { assert.ok(argsOf(collator).includes('--rpc-max-connections=50')); assert.ok(!argsOf(collator).includes('--rpc-max-connections=500')); @@ -89,10 +89,11 @@ describe('network overlay: RPC nodes in front of People', () => { assert.doesNotMatch(upstream, new RegExp(`:${PORTS.people};`)); }); - it('leaves the other network untouched', () => { + it('applies each network its own section only', () => { const devnet = loadNetwork('devnet'); assert.deepEqual(devnet.relay.extraArgs, ['--rpc-max-connections=7']); assert.ok(devnet.parachains.every((p) => p.rpcNodes === 0)); + assert.deepEqual(loadNetwork('previewnet').relay.extraArgs, []); }); }); @@ -108,6 +109,8 @@ describe('network overlay: a mistake fails instead of deploying defaults', () => rejects({ previewnet: { parachains: { people: { rpcNode: 2 } } } }, /unknown fields rpcNode/)); it('rejects more RPC nodes than the port gap holds', () => rejects({ previewnet: { parachains: { people: { rpcNodes: 10 } } } }, /rpcNodes must be an integer 0\.\.9/)); + it('rejects a fractional RPC node count', () => + rejects({ previewnet: { parachains: { people: { rpcNodes: 1.5 } } } }, /rpcNodes must be an integer/)); it('rejects args that are not flags', () => rejects({ previewnet: { relay: { extraArgs: ['rpc-max-connections=1'] } } }, /must be an array of "--flag" strings/)); it('rejects a file that is not there', () => From 6170e42eb1d3cc9b733eaa2b6cba8dd4b138cb15 Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:41:37 +0000 Subject: [PATCH 10/19] Let a -l arg replace the default log targets instead of joining them --- networks/README.md | 2 +- packages/network-config/src/args.ts | 4 ++-- packages/network-config/tests/overlay.test.ts | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/networks/README.md b/networks/README.md index c6087c2..de17f68 100644 --- a/networks/README.md +++ b/networks/README.md @@ -37,7 +37,7 @@ or `PPN_NETWORK=` in the environment of the scripts. | `relay.genesisSpec`, `parachains[].genesisSpec` | `{ "chainId", "name", "preset", "file" }` — the chain spec a genesis network builds for this chain. `ppn generate` passes them to `chain-spec-builder` (`-i`, `-n`, `named-preset`, output filename) and the genesis TOML generator points zombienet at `chainId` + `file`. **Required on a genesis network**, absent on fork-only ones (a fork uses the specs inside its bundle). | | `genesisConfig` | `{ "chainType", "properties", "validatorNameSuffix", "networkSuffix" }` — settings every chain in a genesis network shares: `chain-spec-builder -t`, `--properties`, and the relay node-name suffix zombienet derives validator keys from (`alice-paseo-validator`). Required on a genesis network. `networkSuffix` is optional: the namespace product contexts derive in, written into the genesis of the chains whose runtime carries the pallet holding it (People and Asset Hub). At most 16 bytes. Leave it out to keep whatever the runtime presets ship — a fork needs nothing here, since its runtime already has the value set. | | *(para ids)* | `parachains[].paraId` is the single source: the genesis TOML, the fork TOML, `ppn generate` and `assign-cores.sh` all read it from here. `config/ports.env` deliberately holds no para ids — those belong to the network, not the machine. | -| `relay.extraArgs`, `parachains[].extraArgs` | Extra node flags for that chain. A flag the shared per-key table also sets takes this value (`--rpc-max-connections=5000` replaces the table's 500); the rest are appended. | +| `relay.extraArgs`, `parachains[].extraArgs` | Extra node flags for that chain. A flag the shared per-key table also sets takes this value (`--rpc-max-connections=5000` replaces the table's 500), and a `-l` arg replaces the default log targets whole; the rest are appended. | | `parachains[].rpcNodes`, `parachains[].rpcNodeArgs` | Non-authoring full nodes (0..9) that serve the chain's public path instead of the collator, on the chain's port + 1, + 2, …, with nginx balancing across them. They run the collator's flags without the authoring ones, with `rpcNodeArgs` over them. | | *(network overlay)* | `PPN_NETWORK_OVERLAY=` changes `extraArgs`, `rpcNodes` and `rpcNodeArgs` for one deployment without a release: `{ "previewnet": { "relay": { "extraArgs": [...] }, "parachains": { "people": { "rpcNodes": 2, "rpcNodeArgs": [...] } } } }`. Every command applies it, so the zombienet config, nginx routes and dashboard agree. An unknown network, chain or field is an error. | | `services` | Per-service switches and binaries: `false` disables a service; `{ "binary": { … } }` says what it runs. `eth-rpc` must match the revive pallet in the network's Asset Hub runtime, which is exactly why it is pinned per network rather than shared. | diff --git a/packages/network-config/src/args.ts b/packages/network-config/src/args.ts index ab2a5a3..8a4fb85 100644 --- a/packages/network-config/src/args.ts +++ b/packages/network-config/src/args.ts @@ -1,6 +1,6 @@ -/** The flag an arg sets: `--rpc-port` for `--rpc-port=9944`, `-lsync` for `-lsync=debug`. */ +/** The flag an arg sets: `--rpc-port` for `--rpc-port=9944`, and `-l` for every `-l…`/`--log` arg. */ export function flagOf(arg: string): string { - return arg.split('=')[0]; + return /^(-l|--log(=|$))/.test(arg) ? '-l' : arg.split('=')[0]; } /** `args` with each flag `extra` also sets taking extra's value, and extra's other flags appended. */ diff --git a/packages/network-config/tests/overlay.test.ts b/packages/network-config/tests/overlay.test.ts index a9d18f4..2444d74 100644 --- a/packages/network-config/tests/overlay.test.ts +++ b/packages/network-config/tests/overlay.test.ts @@ -23,7 +23,7 @@ process.env.PPN_NETWORK_OVERLAY = write('people.json', { parachains: { people: { rpcNodes: 2, - extraArgs: ['--rpc-max-connections=50'], + extraArgs: ['--rpc-max-connections=50', '-lsync=debug'], rpcNodeArgs: ['--rpc-max-connections=5000', '--rpc-message-buffer-capacity-per-connection=512'], }, }, @@ -71,6 +71,7 @@ describe('network overlay: RPC nodes in front of People', () => { it('gives the deployment the last word on a flag the engine also sets', () => { assert.ok(argsOf(collator).includes('--rpc-max-connections=50')); assert.ok(!argsOf(collator).includes('--rpc-max-connections=500')); + assert.deepEqual(argsOf(collator).filter((a) => a.startsWith('-l')), ['-lsync=debug']); for (const block of rpc) { assert.ok(argsOf(block).includes('--rpc-max-connections=5000')); assert.ok(argsOf(block).includes('--rpc-message-buffer-capacity-per-connection=512')); From 06a4867218ef5e4c1ef6c9e433991c12d9ddbcb6 Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:45:44 +0000 Subject: [PATCH 11/19] Generate a start's overlay config in-process through genesis-toml --- packages/cli/src/commands/genesis-toml.ts | 7 ++++++- packages/cli/src/commands/start.ts | 5 ++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/commands/genesis-toml.ts b/packages/cli/src/commands/genesis-toml.ts index a11a25b..180a8e6 100644 --- a/packages/cli/src/commands/genesis-toml.ts +++ b/packages/cli/src/commands/genesis-toml.ts @@ -17,7 +17,12 @@ export async function run(_args?: string[]): Promise { console.error(`ppn genesis-toml: ${net.name} is fork-only — it has no genesis config`); process.exit(1); } + process.stdout.write(genesisToml()); +} +/** The genesis zombienet config, with PPN_PARACHAINS and ENABLE_HOP applied. */ +export function genesisToml(): string { + const net = loadCurrentNetwork(); const parachains = (process.env.PPN_PARACHAINS || net.parachains.map((p) => p.key).join(',')) .split(',') .map((s: string) => s.trim()) @@ -26,5 +31,5 @@ export async function run(_args?: string[]): Promise { const header = '# DO NOT EDIT — regenerate with: make generate-toml\n# Source: packages/network-config/src/toml-generator.ts\n\n'; - process.stdout.write(header + generateToml(parachains, { enableHop })); + return header + generateToml(parachains, { enableHop }); } diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 0c27471..0d6a4dc 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -326,9 +326,8 @@ async function ensureDeps(netDef: NetworkDef, opts: StartOptions, binDir: string if (!process.env.PPN_NETWORK_OVERLAY) return path.join(REPO, 'zombienet-configs', 'local-dev.toml'); const overlaid = path.join(WS, 'zombienet-configs', 'local-dev.overlay.toml'); fs.mkdirSync(path.dirname(overlaid), { recursive: true }); - const generated = spawnSync(nodeBin, [ppn, 'genesis-toml'], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'inherit'] }); - if (generated.status !== 0) throw new Error('ppn genesis-toml failed'); - fs.writeFileSync(overlaid, generated.stdout); + const { genesisToml } = await import('./genesis-toml.js'); + fs.writeFileSync(overlaid, genesisToml()); return overlaid; } From 08826754e4b9eee4a992a1377c802a7fbc492dc0 Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:45:44 +0000 Subject: [PATCH 12/19] Put both arg-merging rules in args.ts --- packages/network-config/src/args.ts | 6 ++++++ packages/network-config/src/toml-generator.ts | 8 +------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/network-config/src/args.ts b/packages/network-config/src/args.ts index 8a4fb85..fcd2a7f 100644 --- a/packages/network-config/src/args.ts +++ b/packages/network-config/src/args.ts @@ -3,6 +3,12 @@ export function flagOf(arg: string): string { return /^(-l|--log(=|$))/.test(arg) ? '-l' : arg.split('=')[0]; } +/** `args` plus the flags of `extra` it does not set already, so defaults never override. */ +export function addMissing(args: string[], extra: string[]): string[] { + const present = new Set(args.map(flagOf)); + return [...args, ...extra.filter((a) => !present.has(flagOf(a)))]; +} + /** `args` with each flag `extra` also sets taking extra's value, and extra's other flags appended. */ export function overrideArgs(args: string[], extra: string[]): string[] { const byFlag = new Map(extra.map((a) => [flagOf(a), a])); diff --git a/packages/network-config/src/toml-generator.ts b/packages/network-config/src/toml-generator.ts index f0b7829..0e0f03d 100644 --- a/packages/network-config/src/toml-generator.ts +++ b/packages/network-config/src/toml-generator.ts @@ -17,7 +17,7 @@ import type { Parachain, ChainKey, ChainDef, LogLevel, GenerateTomlOptions } fro import { DEFAULT_NETWORK, type NetworkDef, type NetworkParachain } from './networks.js'; import { loadNetwork } from './load.js'; import { dubCustomProcesses } from './dub.js'; -import { flagOf, overrideArgs } from './args.js'; +import { addMissing, flagOf, overrideArgs } from './args.js'; const VALIDATORS = ['alice', 'bob', 'charlie', 'dave', 'eve', 'ferdie'] as const; @@ -299,12 +299,6 @@ function buildArgs(chainKey: ChainKey, logTargets?: Record, en return args; } -/** `args` plus the flags of `extra` it does not set already, so defaults never override. */ -function addMissing(args: string[], extra: string[]): string[] { - const present = new Set(args.map(flagOf)); - return [...args, ...extra.filter((a) => !present.has(flagOf(a)))]; -} - // The genesis collator command is always the omni-node.sh wrapper (it carries the // libp2p fix any cumulus collator needs); which binary it execs comes from the // descriptor via env. Empty for the default binary, keeping previewnet's TOML stable. From 629863f56f4d0c0c5886ff35a88f4220e071ddad Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:45:44 +0000 Subject: [PATCH 13/19] Drop a needless cast in the overlay parser --- packages/network-config/src/overlay.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/network-config/src/overlay.ts b/packages/network-config/src/overlay.ts index a68df0b..40332e8 100644 --- a/packages/network-config/src/overlay.ts +++ b/packages/network-config/src/overlay.ts @@ -45,8 +45,8 @@ export function applyOverlay(net: NetworkDef, file: string): NetworkDef { const section = object(net.name, raw[net.name], ['relay', 'parachains']); const relay = object(`${net.name}.relay`, section.relay ?? {}, RELAY_FIELDS); const parachains = object(`${net.name}.parachains`, section.parachains ?? {}); - const declared = net.parachains.map((p) => p.key); - const undeclared = Object.keys(parachains).filter((key) => !declared.includes(key as never)); + const declared: string[] = net.parachains.map((p) => p.key); + const undeclared = Object.keys(parachains).filter((key) => !declared.includes(key)); if (undeclared.length) { bad(`${net.name} has no parachains ${undeclared.join(', ')} (declared: ${declared.join(', ')})`); } From 13854344164febe8acac3869f3b33e3c4cd0cb3e Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:49:57 +0000 Subject: [PATCH 14/19] Use full names and plain words in the overlay code --- packages/cli/src/commands/start.ts | 8 +++--- packages/network-config/src/args.ts | 6 ++--- packages/network-config/src/networks.ts | 2 +- packages/network-config/src/nginx-routes.ts | 4 +-- packages/network-config/src/overlay.ts | 26 +++++++++---------- packages/network-config/src/toml-generator.ts | 23 +++++++++------- 6 files changed, 36 insertions(+), 33 deletions(-) diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 0d6a4dc..febbf91 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -324,11 +324,11 @@ async function ensureDeps(netDef: NetworkDef, opts: StartOptions, binDir: string if (opts.toml) return opts.toml; // The checked-in config has no overlay in it, so an overlay's nodes need one generated. if (!process.env.PPN_NETWORK_OVERLAY) return path.join(REPO, 'zombienet-configs', 'local-dev.toml'); - const overlaid = path.join(WS, 'zombienet-configs', 'local-dev.overlay.toml'); - fs.mkdirSync(path.dirname(overlaid), { recursive: true }); + const overlayToml = path.join(WS, 'zombienet-configs', 'local-dev.overlay.toml'); + fs.mkdirSync(path.dirname(overlayToml), { recursive: true }); const { genesisToml } = await import('./genesis-toml.js'); - fs.writeFileSync(overlaid, genesisToml()); - return overlaid; + fs.writeFileSync(overlayToml, genesisToml()); + return overlayToml; } const forkDir = forkDirFor(netDef.name); diff --git a/packages/network-config/src/args.ts b/packages/network-config/src/args.ts index fcd2a7f..32a6114 100644 --- a/packages/network-config/src/args.ts +++ b/packages/network-config/src/args.ts @@ -6,12 +6,12 @@ export function flagOf(arg: string): string { /** `args` plus the flags of `extra` it does not set already, so defaults never override. */ export function addMissing(args: string[], extra: string[]): string[] { const present = new Set(args.map(flagOf)); - return [...args, ...extra.filter((a) => !present.has(flagOf(a)))]; + return [...args, ...extra.filter((arg) => !present.has(flagOf(arg)))]; } /** `args` with each flag `extra` also sets taking extra's value, and extra's other flags appended. */ export function overrideArgs(args: string[], extra: string[]): string[] { - const byFlag = new Map(extra.map((a) => [flagOf(a), a])); + const byFlag = new Map(extra.map((arg) => [flagOf(arg), arg])); const present = new Set(args.map(flagOf)); - return [...args.map((a) => byFlag.get(flagOf(a)) ?? a), ...extra.filter((a) => !present.has(flagOf(a)))]; + return [...args.map((arg) => byFlag.get(flagOf(arg)) ?? arg), ...extra.filter((arg) => !present.has(flagOf(arg)))]; } diff --git a/packages/network-config/src/networks.ts b/packages/network-config/src/networks.ts index 28d0689..65e9f01 100644 --- a/packages/network-config/src/networks.ts +++ b/packages/network-config/src/networks.ts @@ -484,7 +484,7 @@ export function loadDescriptor(name: string): NetworkDef { /** `args` as node flags, or `bad` when it is not a list of "--flag" strings. Absent is none. */ export function parseArgs(at: string, args: unknown, bad: (why: string) => never): string[] { if (args === undefined) return []; - if (!Array.isArray(args) || args.some((x) => typeof x !== 'string' || !x.startsWith('-'))) { + if (!Array.isArray(args) || args.some((arg) => typeof arg !== 'string' || !arg.startsWith('-'))) { bad(`${at} must be an array of "--flag" strings`); } return args as string[]; diff --git a/packages/network-config/src/nginx-routes.ts b/packages/network-config/src/nginx-routes.ts index 4d2e802..86bd0cd 100644 --- a/packages/network-config/src/nginx-routes.ts +++ b/packages/network-config/src/nginx-routes.ts @@ -28,8 +28,8 @@ export function nginxRoutes(model: DashboardModel): { upstreams: string; locatio .map( // least_conn: a WebSocket stays open for the whole session, so round-robin by request // would leave the nodes as unevenly loaded as the sessions happen to last. - (e) => `upstream ${sanitize(e.id)} {${e.upstreamPorts.length > 1 ? '\n least_conn;' : ''} -${e.upstreamPorts.map((port) => ` server 127.0.0.1:${port};`).join('\n')} + (entry) => `upstream ${sanitize(entry.id)} {${entry.upstreamPorts.length > 1 ? '\n least_conn;' : ''} +${entry.upstreamPorts.map((port) => ` server 127.0.0.1:${port};`).join('\n')} keepalive 32; }` ) diff --git a/packages/network-config/src/overlay.ts b/packages/network-config/src/overlay.ts index 40332e8..d89c83f 100644 --- a/packages/network-config/src/overlay.ts +++ b/packages/network-config/src/overlay.ts @@ -31,7 +31,7 @@ export function applyOverlay(net: NetworkDef, file: string): NetworkDef { } const object = (at: string, value: unknown, fields?: string[]): Record => { if (!value || typeof value !== 'object' || Array.isArray(value)) bad(`${at || 'the file'} must be a JSON object`); - const unknown = fields ? Object.keys(value as object).filter((k) => !fields.includes(k)) : []; + const unknown = fields ? Object.keys(value as object).filter((field) => !fields.includes(field)) : []; if (unknown.length) bad(`${at} has unknown fields ${unknown.join(', ')} (allowed: ${fields!.join(', ')})`); return value as Record; }; @@ -45,10 +45,10 @@ export function applyOverlay(net: NetworkDef, file: string): NetworkDef { const section = object(net.name, raw[net.name], ['relay', 'parachains']); const relay = object(`${net.name}.relay`, section.relay ?? {}, RELAY_FIELDS); const parachains = object(`${net.name}.parachains`, section.parachains ?? {}); - const declared: string[] = net.parachains.map((p) => p.key); - const undeclared = Object.keys(parachains).filter((key) => !declared.includes(key)); - if (undeclared.length) { - bad(`${net.name} has no parachains ${undeclared.join(', ')} (declared: ${declared.join(', ')})`); + const parachainKeys: string[] = net.parachains.map((parachain) => parachain.key); + const unknownKeys = Object.keys(parachains).filter((key) => !parachainKeys.includes(key)); + if (unknownKeys.length) { + bad(`${net.name} has no parachains ${unknownKeys.join(', ')} (declared: ${parachainKeys.join(', ')})`); } return { @@ -57,15 +57,15 @@ export function applyOverlay(net: NetworkDef, file: string): NetworkDef { ...net.relay, extraArgs: overrideArgs(net.relay.extraArgs, parseArgs(`${net.name}.relay.extraArgs`, relay.extraArgs, bad)), }, - parachains: net.parachains.map((p) => { - if (parachains[p.key] === undefined) return p; - const at = `${net.name}.parachains.${p.key}`; - const chain = object(at, parachains[p.key], PARACHAIN_FIELDS); + parachains: net.parachains.map((parachain) => { + if (parachains[parachain.key] === undefined) return parachain; + const at = `${net.name}.parachains.${parachain.key}`; + const chain = object(at, parachains[parachain.key], PARACHAIN_FIELDS); return { - ...p, - extraArgs: overrideArgs(p.extraArgs, parseArgs(`${at}.extraArgs`, chain.extraArgs, bad)), - rpcNodes: chain.rpcNodes === undefined ? p.rpcNodes : parseRpcNodes(`${at}.rpcNodes`, chain.rpcNodes, bad), - rpcNodeArgs: overrideArgs(p.rpcNodeArgs, parseArgs(`${at}.rpcNodeArgs`, chain.rpcNodeArgs, bad)), + ...parachain, + extraArgs: overrideArgs(parachain.extraArgs, parseArgs(`${at}.extraArgs`, chain.extraArgs, bad)), + rpcNodes: chain.rpcNodes === undefined ? parachain.rpcNodes : parseRpcNodes(`${at}.rpcNodes`, chain.rpcNodes, bad), + rpcNodeArgs: overrideArgs(parachain.rpcNodeArgs, parseArgs(`${at}.rpcNodeArgs`, chain.rpcNodeArgs, bad)), }; }), }; diff --git a/packages/network-config/src/toml-generator.ts b/packages/network-config/src/toml-generator.ts index 0e0f03d..af92339 100644 --- a/packages/network-config/src/toml-generator.ts +++ b/packages/network-config/src/toml-generator.ts @@ -135,12 +135,14 @@ function prometheusPort(rpcPort: number): number { } /** RPC ports of a parachain's RPC nodes: the chain's own port + 1..rpcNodes. Throws on a port another chain uses. */ -function rpcNodePorts(p: NetworkParachain): number[] { - const ports = Array.from({ length: p.rpcNodes }, (_, i) => PORTS[p.key] + i + 1); - const used = [...Object.values(PORTS), ...VALIDATORS.map((_, i) => RELAY_BASE_PORT + i)]; +function rpcNodePorts(parachain: NetworkParachain): number[] { + const ports = Array.from({ length: parachain.rpcNodes }, (_, index) => PORTS[parachain.key] + index + 1); + const used = [...Object.values(PORTS), ...VALIDATORS.map((_, index) => RELAY_BASE_PORT + index)]; const clash = ports.find((port) => used.includes(port)); if (clash !== undefined) { - throw new Error(`${p.rpcNodes} RPC nodes for ${p.key} would take port ${clash}, which ports.env gives another node`); + throw new Error( + `${parachain.rpcNodes} RPC nodes for ${parachain.key} would take port ${clash}, which ports.env assigns to another node` + ); } return ports; } @@ -151,15 +153,16 @@ const COLLATOR_ONLY_FLAGS = new Set(['--collator', '--force-authoring', '--autho /** * The `[[parachains.collators]]` blocks of a chain's RPC nodes: non-validators running the * collator's args without its authoring flags, with the chain's rpcNodeArgs over them. `launch` - * holds the lines each block shares with the collator (command, env, snapshot). + * is the lines each block shares with the collator (command, env, snapshot). */ -function rpcNodeBlocks(p: NetworkParachain, collatorArgs: string[], launch: string): string { - const args = overrideArgs(collatorArgs.filter((a) => !COLLATOR_ONLY_FLAGS.has(flagOf(a))), p.rpcNodeArgs); - return rpcNodePorts(p) +function rpcNodeBlocks(parachain: NetworkParachain, collatorArgs: string[], launch: string): string { + const nodeArgs = collatorArgs.filter((arg) => !COLLATOR_ONLY_FLAGS.has(flagOf(arg))); + const args = overrideArgs(nodeArgs, parachain.rpcNodeArgs); + return rpcNodePorts(parachain) .map( - (port, i) => ` + (port, index) => ` [[parachains.collators]] -name = "${p.key}-rpc${i + 1}" +name = "${parachain.key}-rpc${index + 1}" validator = false invulnerable = false rpc_port = ${port} From c1cdb47d2bb909b13a51374124294ef4c350b08e Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:55:14 +0000 Subject: [PATCH 15/19] Generate a start's overlay config from every parachain, and reject --toml with it --- packages/cli/src/commands/genesis-toml.ts | 7 +------ packages/cli/src/commands/start.ts | 8 ++++---- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/commands/genesis-toml.ts b/packages/cli/src/commands/genesis-toml.ts index 180a8e6..a11a25b 100644 --- a/packages/cli/src/commands/genesis-toml.ts +++ b/packages/cli/src/commands/genesis-toml.ts @@ -17,12 +17,7 @@ export async function run(_args?: string[]): Promise { console.error(`ppn genesis-toml: ${net.name} is fork-only — it has no genesis config`); process.exit(1); } - process.stdout.write(genesisToml()); -} -/** The genesis zombienet config, with PPN_PARACHAINS and ENABLE_HOP applied. */ -export function genesisToml(): string { - const net = loadCurrentNetwork(); const parachains = (process.env.PPN_PARACHAINS || net.parachains.map((p) => p.key).join(',')) .split(',') .map((s: string) => s.trim()) @@ -31,5 +26,5 @@ export function genesisToml(): string { const header = '# DO NOT EDIT — regenerate with: make generate-toml\n# Source: packages/network-config/src/toml-generator.ts\n\n'; - return header + generateToml(parachains, { enableHop }); + process.stdout.write(header + generateToml(parachains, { enableHop })); } diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index febbf91..32995cc 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -15,6 +15,7 @@ import { spawn, spawnSync } from 'node:child_process'; import { loadNetwork, currentNetworkName, + generateToml, readEnvFile, repoRoot, runsTurnRelay, @@ -321,13 +322,12 @@ async function ensureDeps(netDef: NetworkDef, opts: StartOptions, binDir: string if (!opts.fork) { const { run: generate } = await import('./generate.js'); await generate([binDir], { ifNeeded: true }); - if (opts.toml) return opts.toml; + if (!process.env.PPN_NETWORK_OVERLAY) return opts.toml ?? path.join(REPO, 'zombienet-configs', 'local-dev.toml'); // The checked-in config has no overlay in it, so an overlay's nodes need one generated. - if (!process.env.PPN_NETWORK_OVERLAY) return path.join(REPO, 'zombienet-configs', 'local-dev.toml'); + if (opts.toml) throw new Error('--toml spawns that config as it is, without the PPN_NETWORK_OVERLAY nodes; drop one'); const overlayToml = path.join(WS, 'zombienet-configs', 'local-dev.overlay.toml'); fs.mkdirSync(path.dirname(overlayToml), { recursive: true }); - const { genesisToml } = await import('./genesis-toml.js'); - fs.writeFileSync(overlayToml, genesisToml()); + fs.writeFileSync(overlayToml, generateToml(netDef.parachains.map((parachain) => parachain.key), { enableHop: true })); return overlayToml; } From b7b426c4a2c13c96404746b4eb2003e04f3c7960 Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:55:14 +0000 Subject: [PATCH 16/19] Pass --toml from make start only for a fork --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 46a7097..03f23f9 100644 --- a/Makefile +++ b/Makefile @@ -150,8 +150,7 @@ else start: build-spawner @node $(CURDIR)/bin/ppn.mjs start $(NETWORK) \ --data-dir "$(DATA_DIR)" \ - --toml "$(TOML_FILE)" \ - $(if $(filter 1,$(FORK)),--fork,) \ + $(if $(filter 1,$(FORK)),--fork --toml "$(TOML_FILE)",) \ $(if $(filter 1,$(CLEAN)),--clean,) \ $(if $(filter 1,$(EPHEMERAL)),--ephemeral,) \ $(if $(filter 1,$(REGENERATE)),--regenerate,) \ From e36329e3386fdec210fb521f5ef80782ba3b7f86 Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:55:14 +0000 Subject: [PATCH 17/19] Append repeatable node flags instead of replacing them --- packages/network-config/src/args.ts | 10 ++++++++-- packages/network-config/tests/overlay.test.ts | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/network-config/src/args.ts b/packages/network-config/src/args.ts index 32a6114..7b0bce1 100644 --- a/packages/network-config/src/args.ts +++ b/packages/network-config/src/args.ts @@ -9,9 +9,15 @@ export function addMissing(args: string[], extra: string[]): string[] { return [...args, ...extra.filter((arg) => !present.has(flagOf(arg)))]; } +// Flags a node takes more than once: an extra one adds to the node's, never replaces them. +const REPEATABLE_FLAGS = new Set(['--listen-addr', '--public-addr', '--bootnodes', '--reserved-nodes']); + /** `args` with each flag `extra` also sets taking extra's value, and extra's other flags appended. */ export function overrideArgs(args: string[], extra: string[]): string[] { - const byFlag = new Map(extra.map((arg) => [flagOf(arg), arg])); + const byFlag = new Map(extra.filter((arg) => !REPEATABLE_FLAGS.has(flagOf(arg))).map((arg) => [flagOf(arg), arg])); const present = new Set(args.map(flagOf)); - return [...args.map((arg) => byFlag.get(flagOf(arg)) ?? arg), ...extra.filter((arg) => !present.has(flagOf(arg)))]; + return [ + ...args.map((arg) => byFlag.get(flagOf(arg)) ?? arg), + ...extra.filter((arg) => !byFlag.has(flagOf(arg)) || !present.has(flagOf(arg))), + ]; } diff --git a/packages/network-config/tests/overlay.test.ts b/packages/network-config/tests/overlay.test.ts index 2444d74..4de6d86 100644 --- a/packages/network-config/tests/overlay.test.ts +++ b/packages/network-config/tests/overlay.test.ts @@ -90,6 +90,16 @@ describe('network overlay: RPC nodes in front of People', () => { assert.doesNotMatch(upstream, new RegExp(`:${PORTS.people};`)); }); + // A node listens on, and dials, every address it is given. + it('adds a repeatable flag to the node\'s own instead of replacing them', async () => { + const { overrideArgs } = await import('../src/args.js'); + assert.deepEqual(overrideArgs(['--listen-addr=/a', '--rpc-port=1'], ['--listen-addr=/b', '--rpc-port=2']), [ + '--listen-addr=/a', + '--rpc-port=2', + '--listen-addr=/b', + ]); + }); + it('applies each network its own section only', () => { const devnet = loadNetwork('devnet'); assert.deepEqual(devnet.relay.extraArgs, ['--rpc-max-connections=7']); From 4cea1c1d172e61cf7d70842eac34fd6e6fa6c63a Mon Sep 17 00:00:00 2001 From: Maksym H Date: Thu, 12 Nov 2026 23:55:14 +0000 Subject: [PATCH 18/19] List RPC node logs with the chains in the dashboard --- packages/cli/src/commands/dashboard.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/commands/dashboard.ts b/packages/cli/src/commands/dashboard.ts index 27ad2cc..a1597fb 100644 --- a/packages/cli/src/commands/dashboard.ts +++ b/packages/cli/src/commands/dashboard.ts @@ -300,7 +300,7 @@ export async function dashboard(ctx: ServiceContext): Promise { // well-known validators (alice…ferdie) and Collator- (see docs/FORK.md). const WELL_KNOWN = new Set(['alice', 'bob', 'charlie', 'dave', 'eve', 'ferdie']); const chainish = (n: string) => - /-validator\d*$|-collator\d*$|^Collator-/.test(n) || WELL_KNOWN.has(n) || n === 'doppelganger'; + /-validator\d*$|-collator\d*$|-rpc\d+$|^Collator-/.test(n) || WELL_KNOWN.has(n) || n === 'doppelganger'; const serviceIds = new Set([ ...model.services.map((s) => s.id), ...(model.ice ? [model.ice.id] : []), From 40b487f92437ee688f9978c7073173aa2421b939 Mon Sep 17 00:00:00 2001 From: Maksym H Date: Fri, 13 Nov 2026 00:02:15 +0000 Subject: [PATCH 19/19] Let ppn start pick the config in both modes, rejecting --toml with an overlay --- Makefile | 8 +------- packages/cli/src/commands/start.ts | 5 ++++- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 03f23f9..20f6f77 100644 --- a/Makefile +++ b/Makefile @@ -89,12 +89,6 @@ UPGRADE_FLAGS := $(foreach u,$(UPGRADES),--upgrade $(u)) # bite time like UPGRADES, so a published bundle needs FRESH_BITE=1 to take them. TOPOLOGY_FLAGS := $(foreach c,$(CORES),--cores $(c)) $(foreach c,$(COLLATORS),--collators $(c)) -ifeq ($(FORK),1) -TOML_FILE := $(FORK_TOML) -else -TOML_FILE := $(CONFIG_DIR)/local-dev.toml -endif - # Docker settings DOCKER_IMAGE ?= paritytech/previewnet-engine:latest DOCKER_NAME ?= ppn @@ -150,7 +144,7 @@ else start: build-spawner @node $(CURDIR)/bin/ppn.mjs start $(NETWORK) \ --data-dir "$(DATA_DIR)" \ - $(if $(filter 1,$(FORK)),--fork --toml "$(TOML_FILE)",) \ + $(if $(filter 1,$(FORK)),--fork,) \ $(if $(filter 1,$(CLEAN)),--clean,) \ $(if $(filter 1,$(EPHEMERAL)),--ephemeral,) \ $(if $(filter 1,$(REGENERATE)),--regenerate,) \ diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 32995cc..ce4f1a5 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -319,12 +319,15 @@ async function ensureDeps(netDef: NetworkDef, opts: StartOptions, binDir: string const dotCli = path.join(REPO, 'scripts', 'ensure-dot-cli.sh'); if (fs.existsSync(dotCli)) run('bash', [dotCli]); + if (opts.toml && process.env.PPN_NETWORK_OVERLAY) { + throw new Error('--toml spawns that config as it is, without the PPN_NETWORK_OVERLAY nodes; drop one'); + } + if (!opts.fork) { const { run: generate } = await import('./generate.js'); await generate([binDir], { ifNeeded: true }); if (!process.env.PPN_NETWORK_OVERLAY) return opts.toml ?? path.join(REPO, 'zombienet-configs', 'local-dev.toml'); // The checked-in config has no overlay in it, so an overlay's nodes need one generated. - if (opts.toml) throw new Error('--toml spawns that config as it is, without the PPN_NETWORK_OVERLAY nodes; drop one'); const overlayToml = path.join(WS, 'zombienet-configs', 'local-dev.overlay.toml'); fs.mkdirSync(path.dirname(overlayToml), { recursive: true }); fs.writeFileSync(overlayToml, generateToml(netDef.parachains.map((parachain) => parachain.key), { enableHop: true }));