From 549335119b163acc3f9b0a8c00c9246f23ad3f37 Mon Sep 17 00:00:00 2001 From: Andrzej Sulkowski Date: Mon, 28 Sep 2026 13:33:38 +0300 Subject: [PATCH 01/27] Bite args to set parachain cores and collators (#39) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds bite-time `--core`s and `--collators` options for running forked parachains with a custom topology: ```bash make start FORK=1 FRESH_BITE=1 CORES=people=3 COLLATORS=people=5 ``` This will support individuality-community’s future stress test gate, which will test People with three cores, and multiple collators. ### What changes - ppn bite writes the requested collator authorities and core assignments into the fork state. - The relay validator set grows as needed to ensure every core has a backing group, up to 10 validators. - manifest.json records the topology, and the generated fork TOML starts the matching validators and collators. - ppn start rejects bundles whose topology does not match the requested flags and prints the required re-bite command. - Validator keys are now derived from node names rather than hard-coded. The existing six remain unchanged except for Charlie and Dave’s BEEFY keys, which now match their actual zombienet keystores. ### Validation Tested a live previewnet fork with CORES=people=3 COLLATORS=people=5: - 8 relay cores and 8 validators - People assigned to three cores - Five collators each authored 12 of 60 sampled blocks - 2.03-second average People block time - ppn fork verify passed Unit coverage was added for topology parsing, core planning, key derivation, fork TOML generation, and bundle compatibility checks. All tests pass except dashboard API -> streams a log over SSE, which also fails on main. ### Follow-ups - Add cores/collators inputs and FRESH_BITE=1 to the [polkadot-pop-e2e](https://github.com/paritytech/polkadot-pop-e2e) release gate. --- CLAUDE.md | 4 + Makefile | 9 +- docs/FORK.md | 31 +++- packages/cli/src/cli.ts | 25 +++- packages/cli/src/commands/bite.ts | 9 ++ packages/cli/src/commands/fork.ts | 29 ++-- packages/cli/src/commands/start.ts | 35 ++++- packages/cli/src/fork/overrides.ts | 108 +++++++++----- packages/cli/src/fork/shared-relay.ts | 10 +- packages/cli/src/fork/topology.ts | 70 +++++++++ packages/cli/src/fork/validators.ts | 136 ++++++++++++------ .../cli/tests/fixtures/overrides.golden.json | 6 +- packages/cli/tests/fork-shared-relay.test.ts | 23 ++- packages/cli/tests/fork-topology.test.ts | 69 +++++++++ packages/cli/tests/fork-validators.test.ts | 115 +++++++++++++-- packages/cli/tests/start-network.test.ts | 51 ++++++- packages/network-config/src/bundle.ts | 18 +++ packages/network-config/src/fork-toml.ts | 54 ++++--- packages/network-config/src/index.ts | 2 + packages/network-config/src/toml-generator.ts | 20 +++ .../network-config/tests/fork-toml.test.ts | 52 +++++++ 21 files changed, 730 insertions(+), 146 deletions(-) create mode 100644 packages/cli/src/fork/topology.ts create mode 100644 packages/cli/tests/fork-topology.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 05e9a83..7741c3d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,8 +64,12 @@ make start FORK=1 NETWORK=devnet # Fork a different network (networks/*.json) make start FORK=1 FRESH_BITE=1 # Fork, biting production now instead of using a bundle make start FORK=1 CLEAN=1 # Fork, back at its bite block (a stopped fork otherwise resumes) make bite NETWORK=polkadot UPGRADES="people=" # Bite with a runtime authorized at import (no-sudo networks) +make start FORK=1 FRESH_BITE=1 CORES=people=3 COLLATORS=people=5 # Bite with People on 3 cores and 5 collators ``` +`CORES`/`COLLATORS` are bite-time: the authority sets and the core layout are state inside the +bundle, so they need a fresh bite, and the relay grows to one validator per core (max 10). + Fork mode continues from a real block rather than resetting to genesis, so contracts, registrations and balances are already present. A fork that is started again resumes where it stopped, as long as the data came from the bundle being spawned. See `docs/FORK.md`. diff --git a/Makefile b/Makefile index d805f7e..b05c9a9 100644 --- a/Makefile +++ b/Makefile @@ -85,6 +85,9 @@ FORK_TOML := $(FORK_DIR)/fork.toml # Runtimes a bite authorizes at import, for a network without sudo: UPGRADES="= ..." # becomes one `--upgrade` per entry on `ppn bite` (or `ppn start` when it bites). UPGRADE_FLAGS := $(foreach u,$(UPGRADES),--upgrade $(u)) +# How a bite lays the network out: CORES="= ..." COLLATORS="= ...". Fixed at +# bite time like UPGRADES, so a published bundle needs FRESH_BITE=1 to take them. +TOPOLOGY_FLAGS := $(foreach c,$(CORES),--cores $(c)) $(foreach c,$(COLLATORS),--collators $(c)) ifeq ($(FORK),1) TOML_FILE := $(FORK_TOML) @@ -153,7 +156,7 @@ start: build-spawner $(if $(filter 1,$(EPHEMERAL)),--ephemeral,) \ $(if $(filter 1,$(REGENERATE)),--regenerate,) \ $(if $(filter 1,$(FRESH_BITE)),--fresh-bite,) \ - $(UPGRADE_FLAGS) + $(UPGRADE_FLAGS) $(TOPOLOGY_FLAGS) kill: @node $(CURDIR)/bin/ppn.mjs kill @@ -185,7 +188,7 @@ show-network: build-spawner # build: fork.toml is generated by @ppn/network-config, which shares its # ports and per-chain flags with the genesis config generator. bite: build-spawner ensure-binaries - @node $(CURDIR)/bin/ppn.mjs bite "$(FORK_DIR)" $(UPGRADE_FLAGS) + @node $(CURDIR)/bin/ppn.mjs bite "$(FORK_DIR)" $(UPGRADE_FLAGS) $(TOPOLOGY_FLAGS) @node $(CURDIR)/bin/ppn.mjs fork toml "$(FORK_DIR)" "$(FORK_TOML)" # Make sure a bundle + config exist before a FORK=1 start. @@ -345,6 +348,8 @@ help: @echo " Only previewnet starts from genesis; the rest are fork-only." @echo " FRESH_BITE=1 With FORK=1: bite the live network now, not a published bundle" @echo " UPGRADES=... With a bite: authorize runtimes at import, \"= ...\" (no-sudo networks)" + @echo " CORES=... With a bite: cores per parachain, \"= ...\" (validators grow to match)" + @echo " COLLATORS=... With a bite: collators per parachain, \"= ...\"" @echo "" @echo "Examples:" @echo " make start Start with persistence (default)" diff --git a/docs/FORK.md b/docs/FORK.md index e93b46d..5d92eba 100644 --- a/docs/FORK.md +++ b/docs/FORK.md @@ -89,7 +89,31 @@ instead of driving doppelganger itself ([#120](https://github.com/paritytech/zom The fork resumes at the bite block and diverges from there — it is a real fork, not a mirror. It runs six relay validators and one collator per parachain, on PPN's normal ports, with Asset Hub's -2-second elastic scaling intact. +2-second elastic scaling intact — unless the bite was asked for a different layout, below. + +### Cores and collators + +```bash +make start FORK=1 FRESH_BITE=1 CORES=people=3 COLLATORS=people=5 +make bite CORES="people=3 asset-hub=1" COLLATORS=people=5 +``` + +`CORES==` lays a parachain out on that many cores (Asset Hub defaults to three, the +rest to one); `COLLATORS==` installs that many collators as its authorities. Both are +**bite-time** settings: the collator authority sets and the relay's core layout are state inside +the snapshots, so a published bundle — bitten with the defaults — cannot take them. `ppn start` +refuses a bundle whose recorded layout differs from what it was asked for and says how to re-bite. +The bundle's `manifest.json` carries the layout under `topology`, and the fork TOML follows it. + +The relay grows to one validator per core, because a core with no validator group backs nothing: +previewnet with `people=3` is 3 + 3 + 1 + 1 = 8 cores, so 8 validators. Past the well-known six +they are named `Validator-7`, `Validator-8`, … and their keys derived from those names, the same +way zombienet does it; the cap is 10, where the relay RPC ports (`10000 + i`) would run into +People's. Extra collators are `Collator--2` … and restore the same snapshot as the first; +only the first holds the documented ports. + +What the parachain does with its cores is the runtime's business: People authors at 2 seconds on +three cores only if its runtime is built for that velocity. The bite provides the cores. **There is no block history before the bite point.** Warp sync delivers finality proofs plus the state at the target, not the chain. Block *numbers* continue from the bite block, but querying an @@ -187,10 +211,13 @@ adds (`--relay-chain-rpc-urls`, `--discover-local`, `--allow-private-ip`, `--sta keys its chains with the same names as the `Parachain` type, so a bundle manifest is checked against the descriptor with no mapping table in between. -**Relay nodes must be named `alice`…`eve`, collators `Collator-`.** zombienet maps the +**Relay nodes must be named `alice`…`ferdie`, collators `Collator-`.** zombienet maps the well-known names to the well-known dev keys, which is exactly the authority set the bite installs. PPN's genesis-mode names (`alice-paseo-validator`, …) get generated keys instead, and the network cannot author. This is why fork mode has its own config rather than overlaying `local-dev.toml`. +Any node past those (`Validator-7`, `Collator-1502-2`) gets keys zombienet derives from `//`, +and the bite derives the same — `relayNodeName()`/`collatorNodeName()` in `toml-generator.ts` are +the one place both read the name from. **Every relay node needs `ZOMBIE_DISPUTE_CANDIDATE_LIFETIME_AFTER_FINALIZATION=1`.** A warp-synced database has no ancestry before the bite block, so the dispute scrape fails, diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index c5dcb34..071b1a5 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -319,6 +319,8 @@ export function buildProgram(): Command { 'with a bite: authorize a runtime at import, for a fork without sudo (see `ppn bite`)' ) .option('--upgrade-same-spec', 'with --upgrade: authorize a runtime whose spec_version is not bumped') + .option('--cores ', 'with a bite: lay a parachain out on this many cores (see `ppn bite`)') + .option('--collators ', 'with a bite: install this many collators on a parachain') .option('--data-dir ', 'where chain state goes; default data/ (suffixed per network and mode)') .option('--toml ', 'use this zombienet config instead of the generated one') // Tri-state on purpose: unset leaves the decision to the descriptor's dotns.pinProducts, @@ -346,6 +348,8 @@ export function buildProgram(): Command { freshBite: Boolean(opts.freshBite), upgrades: opts.upgrade as string[] | undefined, upgradeSameSpec: Boolean(opts.upgradeSameSpec), + cores: opts.cores as string[] | undefined, + collators: opts.collators as string[] | undefined, dataDir: opts.dataDir as string | undefined, toml: opts.toml as string | undefined, }); @@ -451,16 +455,24 @@ export function buildProgram(): Command { '--upgrade-same-spec', 'authorize even a runtime whose spec_version is not bumped (replaying production\'s own)' ) + .option( + '--cores ', + 'lay a parachain out on this many cores (the relay gets a validator per core):\n' + + ' --cores people=3' + ) + .option('--collators ', 'install this many collators as a parachain\'s authorities') .action( async ( outDir: string | undefined, - opts: { source?: string; upgrade?: string[]; upgradeSameSpec?: boolean } + opts: { source?: string; upgrade?: string[]; upgradeSameSpec?: boolean; cores?: string[]; collators?: string[] } ) => { const { run } = await import('./commands/bite.js'); await run(outDir ? [outDir] : [], { source: opts.source, upgrades: opts.upgrade, upgradeCheckVersion: !opts.upgradeSameSpec, + cores: opts.cores, + collators: opts.collators, }); } ); @@ -588,8 +600,15 @@ export function buildProgram(): Command { forkCmd('overrides ', 'build the storage overrides the bite applies') // `ppn bite` passes what it staged; a hand-run needs neither the flag nor a value. .option('--upgrades ', 'runtimes to authorize at import, as {"":{codeHash,checkVersion}}') - .action((outDir: string, baseUrl: string, opts: { upgrades?: string }) => - runFork(['overrides', outDir, baseUrl, ...(opts.upgrades ? ['--upgrades', opts.upgrades] : [])]) + .option('--topology ', 'cores and collators the bite was asked for, as {validators,cores,collators}') + .action((outDir: string, baseUrl: string, opts: { upgrades?: string; topology?: string }) => + runFork([ + 'overrides', + outDir, + baseUrl, + ...(opts.upgrades ? ['--upgrades', opts.upgrades] : []), + ...(opts.topology ? ['--topology', opts.topology] : []), + ]) ); forkCmd('head-env ', 'the parachain heads to inject into the relay bite') diff --git a/packages/cli/src/commands/bite.ts b/packages/cli/src/commands/bite.ts index 99427b9..1b17d4f 100644 --- a/packages/cli/src/commands/bite.ts +++ b/packages/cli/src/commands/bite.ts @@ -31,6 +31,7 @@ import { blake2AsHex } from '@polkadot/util-crypto'; import { githubToken, downloadUrl } from '../lib/github.js'; import { extractTarGz, extractZip, withTempDir } from '../lib/archive.js'; import { forkBundleName, forkBundleAsset } from '../lib/fork-bundle-name.js'; +import { resolveTopology, topologyFlags } from '../fork/topology.js'; const REPO = repoRoot(); /** Mutable state — binaries, chain data, bundles — lives in the workspace, not the package. */ @@ -209,6 +210,10 @@ export interface BiteOptions { upgrades?: string[]; /** false authorizes a runtime whose spec_version is not bumped. Default true. */ upgradeCheckVersion?: boolean; + /** `=`, cores to lay the parachain out on. See fork/topology.ts. */ + cores?: string[]; + /** `=`, collators to install as the parachain's authorities. */ + collators?: string[]; } /** @@ -511,6 +516,8 @@ export async function run(args: string[], opts: BiteOptions = {}): Promise // Before the first network call: a mistyped chain or a missing file should fail now, not // after minutes of syncing. const staged = stageUpgrades(opts.upgrades, chains, out, opts.upgradeCheckVersion ?? true); + const topology = resolveTopology(opts, net); + if (topology) console.log(` topology: ${topologyFlags(topology)} → ${topology.validators} validators`); console.log(`=== 1/6 collecting chain specs (network: ${net.name}) ===`); for (const chain of chains) await collectSpec(chain, net, out, baseUrl, dgDir); @@ -529,6 +536,7 @@ export async function run(args: string[], opts: BiteOptions = {}): Promise path.join(out, 'overrides'), baseUrl, ...(Object.keys(staged).length ? ['--upgrades', JSON.stringify(staged)] : []), + ...(topology ? ['--topology', JSON.stringify(topology)] : []), ]); console.log('=== 4/6 biting parachains (parallel) ==='); @@ -662,6 +670,7 @@ export async function run(args: string[], opts: BiteOptions = {}): Promise biteBlocks, snapshotBytes, ...(Object.keys(staged).length ? { seededUpgrades: staged } : {}), + ...(topology ? { topology } : {}), }, null, 2 diff --git a/packages/cli/src/commands/fork.ts b/packages/cli/src/commands/fork.ts index 505dabd..2c322e3 100644 --- a/packages/cli/src/commands/fork.ts +++ b/packages/cli/src/commands/fork.ts @@ -6,7 +6,7 @@ // need no compiled code and `make bite` should not have to build first to read them. // // ppn fork manifest -// ppn fork overrides +// ppn fork overrides [--upgrades ] [--topology ] // ppn fork head-env // ppn fork toml // ppn fork products @@ -22,7 +22,7 @@ import { PORTS, RELAY_BASE_PORT, repoRoot, workspaceRoot} from '@parity/ppn-network-config'; import { writeManifest, headEnvLines } from '../fork/manifest.js'; import { relayOverrides, paraOverrides } from '../fork/overrides.js'; -import { generateForkToml, type ForkManifest } from '@parity/ppn-network-config'; +import { generateForkToml, type ForkManifest, type ForkTopology } from '@parity/ppn-network-config'; import { scanProducts } from '../fork/products.js'; import { targetsFromManifest, @@ -68,23 +68,27 @@ export async function run(args: string[]): Promise { need(baseUrl, 'baseUrl'); // Runtimes `ppn bite --upgrade` staged, keyed by chain. Passed as JSON rather than // read off disk so this subcommand stays a pure function of its arguments. - const flag = rest.indexOf('--upgrades'); + const flagValue = (flag: string) => { + const at = rest.indexOf(flag); + return at === -1 ? undefined : need(rest[at + 1], flag); + }; const seeded: Record = - flag === -1 ? {} : JSON.parse(need(rest[flag + 1], '--upgrades')); + JSON.parse(flagValue('--upgrades') ?? '{}'); + // What `ppn bite --cores/--collators` settled on (fork/topology.ts); absent = defaults. + const topology: ForkTopology | undefined = JSON.parse(flagValue('--topology') ?? 'null') ?? undefined; fs.mkdirSync(outDir, { recursive: true }); const relay = CHAINS.find((c) => c.key === 'relay')!; // A shared relay carries parachains this network does not run, and its inherited core // layout and messaging state are then wrong for us — see fork/shared-relay.ts. - const shared = NETWORK.bite.sharedRelay - ? { - paras: PARACHAINS.map((p) => ({ key: p.key, paraId: p.paraId })), - validators: NETWORK.relay.validators, - } - : undefined; await relayOverrides( endpointOf(relay, NETWORK, baseUrl), `${outDir}/rc_overrides.json`, - shared, + { + paras: PARACHAINS.map((p) => ({ key: p.key, paraId: p.paraId })), + validators: topology?.validators ?? NETWORK.relay.validators, + cores: topology?.cores, + sharedRelay: NETWORK.bite.sharedRelay === true, + }, seeded.relay ); for (const p of PARACHAINS) { @@ -94,7 +98,8 @@ export async function run(args: string[]): Promise { `${outDir}/${p.paraId}_overrides.json`, NETWORK.bite.sharedRelay, seeded[p.key], - p.aura + p.aura, + topology?.collators[p.key] ?? 1 ); } return; diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index adc374b..db12f7f 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -18,11 +18,14 @@ import { readEnvFile, repoRoot, workspaceRoot, + type ForkManifest, + type ForkTopology, type NetworkDef, } from '@parity/ppn-network-config'; import { readSpawnStamp, writeSpawnStamp, SPAWN_FILE } from '../lib/spawn-stamp.js'; import { localEnvContent, childEnv } from '../lib/spawn-env.js'; import { forkBundleName } from '../lib/fork-bundle-name.js'; +import { resolveTopology, sameTopology, topologyFlags } from '../fork/topology.js'; const REPO = repoRoot(); /** Mutable state — binaries, chain data, bundles — lives in the workspace, not the package. */ @@ -73,6 +76,10 @@ export interface StartOptions { upgrades?: string[]; /** With a bite: authorize a runtime whose spec_version is not bumped. */ upgradeSameSpec?: boolean; + /** With a bite: `=` cores per parachain. See `ppn bite --cores`. */ + cores?: string[]; + /** With a bite: `=` collators per parachain. See `ppn bite --collators`. */ + collators?: string[]; /** Override the data directory. */ dataDir?: string; /** Override the zombienet config, bypassing the generated one. */ @@ -211,7 +218,13 @@ async function ensureDeps(netDef: NetworkDef, opts: StartOptions, binDir: string const forkDir = forkDirFor(netDef.name); const forkToml = path.join(forkDir, 'fork.toml'); - const biteOpts = { upgrades: opts.upgrades, upgradeCheckVersion: !opts.upgradeSameSpec }; + const biteOpts = { + upgrades: opts.upgrades, + upgradeCheckVersion: !opts.upgradeSameSpec, + cores: opts.cores, + collators: opts.collators, + }; + const asked = resolveTopology(opts, netDef); if (opts.freshBite) { console.log('biting the source network now (--fresh-bite)'); const { run: bite } = await import('./bite.js'); @@ -259,11 +272,31 @@ async function ensureDeps(netDef: NetworkDef, opts: StartOptions, binDir: string await bite([forkDir], biteOpts); } } + assertBundleTopology(forkDir, netDef.name, asked); run(nodeBin, [ppn, 'fork', 'toml', forkDir, forkToml]); console.log(`✓ fork config: ${forkToml}`); return opts.toml ?? forkToml; } +/** + * Refuse a bundle bitten with a different layout than the run asks for. + * + * `--cores`/`--collators` are bite-time: the collator authority sets and the relay's core + * layout are state inside the snapshots, and the fork TOML follows the manifest, so a spawn + * cannot take a layout its bundle was not bitten with. Asking for none matches only a bundle + * bitten with the defaults. + */ +export function assertBundleTopology(forkDir: string, network: string, asked: ForkTopology | undefined): void { + const m = JSON.parse(fs.readFileSync(path.join(forkDir, 'manifest.json'), 'utf-8')) as ForkManifest; + if (m.topology) console.log(` bitten with ${topologyFlags(m.topology)} (${m.topology.validators} validators)`); + if (sameTopology(asked, m.topology)) return; + throw new Error( + `--cores/--collators change how the bite lays the network out, and the bundle in ${forkDir} was bitten` + + (m.topology ? ` with ${topologyFlags(m.topology)}.` : ' with the defaults.') + + `\n Re-bite with it: ppn start ${network} --fork --fresh-bite${asked ? ' ' + topologyFlags(asked) : ''}` + ); +} + export type ForkDataVerdict = | { action: 'fresh'; reason: string } | { action: 'resume'; reason: string } diff --git a/packages/cli/src/fork/overrides.ts b/packages/cli/src/fork/overrides.ts index 00fa64e..dff4e94 100644 --- a/packages/cli/src/fork/overrides.ts +++ b/packages/cli/src/fork/overrides.ts @@ -13,6 +13,7 @@ import { keyOf } from './codec.js'; import { channelsTouching, coreDescriptorsValue, + coresFor, dmpWipes, hrmpChannelKey, hrmpContentsWipes, @@ -24,7 +25,8 @@ import { type PlannedPara, } from './shared-relay.js'; import { - collatorKey, + collatorKeys, + devValidators, paraCandidates, paraInjects, relayCandidates, @@ -231,19 +233,30 @@ function patchNumCores( return { value, before }; } +/** What a relay bite is asked to lay out — see relayOverrides(). */ +export interface RelayLayout { + paras: PlannedPara[]; + validators: number; + /** Cores per parachain key where they differ from coresFor()'s default. */ + cores?: Record; + /** The relay carries parachains we do not run: reset its messaging state too. */ + sharedRelay: boolean; +} + /** - * The extra overrides a shared relay needs: our parachains registered alone, laid out on cores - * from 0 with validator groups to match, `num_cores` cut to fit, and the inherited messaging - * state cleared. See ./shared-relay.ts for why each one is here. + * Our parachains laid out on cores from 0 with validator groups to match and `num_cores` cut + * to fit. A shared relay always needs this (see ./shared-relay.ts); previewnet's own relay + * only when the bite asks for a core count production does not have. */ -async function sharedRelayCandidates( +async function coreLayoutCandidates( index: StorageIndex, relayUrl: string, - paras: PlannedPara[], - validators: number -): Promise<{ overrides: Record; injects: Record }> { - const plan = planCores(paras); - const paraIds = paras.map((p) => p.paraId); + { paras, validators, cores }: RelayLayout +): Promise> { + const plan = planCores(paras, cores); + if (plan.length > validators) { + throw new Error(`${plan.length} cores need at least ${plan.length} validators, and the bite has ${validators}`); + } // Read production's own HostConfiguration and change exactly one field in it. const liveConfig = await rpc<`0x${string}` | null>(relayUrl, 'state_getStorage', [ @@ -253,26 +266,35 @@ async function sharedRelayCandidates( const config = patchNumCores(index, liveConfig, plan.length); console.log( - ` shared relay: ${paraIds.join(', ')} on cores 0-${plan.length - 1}, ` + - `num_cores ${config.before} -> ${plan.length}` + ` cores: ${paras.map((p) => `${p.key}=${coresFor(p.key, cores)}`).join(', ')} on cores 0-${plan.length - 1}, ` + + `num_cores ${config.before} -> ${plan.length}, ${validators} validators` ); return { - overrides: { - [keyOf('ParaScheduler', 'ValidatorGroups')]: validatorGroups(plan.length, validators), - [keyOf('ParaScheduler', 'CoreDescriptors')]: encode( - index, - 'ParaScheduler', - 'CoreDescriptors', - coreDescriptorsValue(plan) - ), - [keyOf('Configuration', 'ActiveConfig')]: config.value, - }, - // Storage map entries, so injects. - injects: { ...dmpWipes(paraIds), ...(await hrmpResets(index, relayUrl, paraIds)) }, + [keyOf('ParaScheduler', 'ValidatorGroups')]: validatorGroups(plan.length, validators), + [keyOf('ParaScheduler', 'CoreDescriptors')]: encode( + index, + 'ParaScheduler', + 'CoreDescriptors', + coreDescriptorsValue(plan) + ), + [keyOf('Configuration', 'ActiveConfig')]: config.value, }; } +/** + * The inherited messaging state a shared relay carries, cleared. See ./shared-relay.ts for + * why. Storage map entries, so injects. + */ +async function sharedRelayInjects( + index: StorageIndex, + relayUrl: string, + paras: PlannedPara[] +): Promise> { + const paraIds = paras.map((p) => p.paraId); + return { ...dmpWipes(paraIds), ...(await hrmpResets(index, relayUrl, paraIds)) }; +} + /** * Reset every HRMP channel touching one of our parachains, keeping it open. * @@ -362,18 +384,22 @@ export interface SeededUpgrade { export async function relayOverrides( relayUrl: string, outFile: string, - shared?: { paras: PlannedPara[]; validators: number }, + layout: RelayLayout, upgrade?: SeededUpgrade ): Promise { const index = await storageIndex(relayUrl); + const validators = await devValidators(layout.validators); console.log('relay:'); - // Order matters: the shared-relay values replace the ones relayCandidates() sets for a relay - // that is ours (ValidatorGroups above all), so they come second. - const extra = shared - ? await sharedRelayCandidates(index, relayUrl, shared.paras, shared.validators) - : { overrides: {}, injects: {} }; - const candidates = { ...relayCandidates(), ...extra.overrides }; + // Production's own core layout is kept unless it has to move: a shared relay's is wrong for + // us, and a bite asked for cores is asking for a layout production does not have. Order + // matters: these values replace the ones relayCandidates() sets (ValidatorGroups above all), + // so they come second. + const relaid = layout.sharedRelay || Object.keys(layout.cores ?? {}).length > 0; + const candidates = { + ...relayCandidates(validators), + ...(relaid ? await coreLayoutCandidates(index, relayUrl, layout) : {}), + }; const overrides = report('relay overrides', verify(index, candidates), candidates); write(outFile, { @@ -381,9 +407,9 @@ export async function relayOverrides( // On a shared relay the dev accounts hold nothing — endow sudo at import so its // first transaction (a runtime upgrade's fees) is payable. injects: { - ...relayInjects(), - ...(shared ? sudoEndowInjects() : {}), - ...extra.injects, + ...relayInjects(validators), + ...(layout.sharedRelay ? sudoEndowInjects() : {}), + ...(layout.sharedRelay ? await sharedRelayInjects(index, relayUrl, layout.paras) : {}), ...seededUpgradeInject(index, upgrade), }, }, index); @@ -411,10 +437,11 @@ export async function paraOverrides( outFile: string, sharedRelay = false, upgrade?: SeededUpgrade, - scheme: AuraScheme = 'sr25519' + scheme: AuraScheme = 'sr25519', + collatorCount = 1 ): Promise { const index = await storageIndex(paraUrl); - const collator = await collatorKey(paraId, scheme); + const collators = await collatorKeys(paraId, collatorCount, scheme); // On a shared relay the inherited messaging state is reset on both sides at once — see // ./shared-relay.ts. On our own relay it is preserved on both sides, which is what keeps // previewnet's HRMP channels and its XCM tests working. @@ -424,17 +451,20 @@ export async function paraOverrides( // proof check panics "Storage proof must be checked once in the block" and the chain // wedges one block in — reproduced on the published previewnet bundle 2026-08-19. const candidates = { - ...paraCandidates(collator), + ...paraCandidates(collators), ...(sharedRelay ? paraMessagingWipes() : {}), ...(index.pallets.has('TransactionStorage') ? transactionStorageWipes() : {}), }; - console.log(`para ${paraId} (collator //Collator-${paraId} ${scheme} = ${collator.slice(0, 16)}…):`); + console.log( + `para ${paraId} (${collators.length} collator${collators.length === 1 ? '' : 's'} from //Collator-${paraId} ` + + `${scheme} = ${collators.map((c) => c.slice(0, 16) + '…').join(', ')}):` + ); const overrides = report(`para ${paraId} overrides`, verify(index, candidates), candidates); // A parachain without a Session pallet manages authorities through Aura alone. const injects = { - ...(index.pallets.has('Session') ? paraInjects(collator) : {}), + ...(index.pallets.has('Session') ? paraInjects(collators) : {}), // Parachains of a shared relay are live public chains: no dev account holds // funds there, so sudo is endowed at import (see sudoEndowInjects). ...(sharedRelay ? sudoEndowInjects() : {}), diff --git a/packages/cli/src/fork/shared-relay.ts b/packages/cli/src/fork/shared-relay.ts index 1e898eb..da41685 100644 --- a/packages/cli/src/fork/shared-relay.ts +++ b/packages/cli/src/fork/shared-relay.ts @@ -53,10 +53,10 @@ export interface PlannedPara { * * Asset Hub keeps three: that is what gives it 2-second blocks through elastic scaling, and it * is the one place where the number is a property of the network rather than a default. The - * rest get one each. + * rest get one each — unless the bite was asked for more (`ppn bite --cores people=3`). */ -export function coresFor(key: string): number { - return key === 'asset-hub' ? 3 : 1; +export function coresFor(key: string, cores: Record = {}): number { + return cores[key] ?? (key === 'asset-hub' ? 3 : 1); } /** @@ -66,11 +66,11 @@ export function coresFor(key: string): number { * fills groups in order, so with `num_cores` set to exactly what we plan here, every core in * the plan is staffed and none of the plan depends on rotation luck. */ -export function planCores(paras: PlannedPara[]): CoreAssignment[] { +export function planCores(paras: PlannedPara[], cores: Record = {}): CoreAssignment[] { const plan: CoreAssignment[] = []; let core = 0; for (const para of paras) { - for (let i = 0; i < coresFor(para.key); i++) plan.push({ core: core++, paraId: para.paraId }); + for (let i = 0; i < coresFor(para.key, cores); i++) plan.push({ core: core++, paraId: para.paraId }); } return plan; } diff --git a/packages/cli/src/fork/topology.ts b/packages/cli/src/fork/topology.ts new file mode 100644 index 0000000..6be5041 --- /dev/null +++ b/packages/cli/src/fork/topology.ts @@ -0,0 +1,70 @@ +// What `ppn bite --cores --collators ` asks for, checked against the network +// and settled into the ForkTopology the bundle records. +// +// Settled at bite time, not at spawn: the collator authority sets and the relay's core layout +// are state inside the snapshots (fork/validators.ts, fork/shared-relay.ts), so the fork TOML +// can only follow what the bundle says. A bundle bitten with the defaults carries no topology. + +import type { ForkTopology, NetworkDef } from '@parity/ppn-network-config'; +import { planCores } from './shared-relay.js'; +import { MAX_VALIDATORS } from './validators.js'; + +function parseCounts(flag: string, specs: string[] | undefined, net: NetworkDef): Record { + const counts: Record = {}; + const known = net.parachains.map((p) => p.key); + for (const spec of specs ?? []) { + const at = spec.indexOf('='); + if (at < 1) throw new Error(`${flag} wants =, got "${spec}"`); + const key = spec.slice(0, at); + const count = Number(spec.slice(at + 1)); + if (!known.includes(key as never)) { + throw new Error(`${flag} names "${key}", which ${net.name} does not run (${known.join(', ')})`); + } + if (!Number.isInteger(count) || count < 1) { + throw new Error(`${flag} ${key}: a count is a whole number of at least 1, not "${spec.slice(at + 1)}"`); + } + counts[key] = count; + } + return counts; +} + +/** + * The topology a bite was asked for, or undefined when it was asked for nothing. + * + * The validator count is not asked for: every core needs a validator group of its own or nothing + * backs the blocks on it, so it is the descriptor's count or the number of cores, whichever is + * larger. That is capped where the relay ports run out (MAX_VALIDATORS). + */ +export function resolveTopology( + opts: { cores?: string[]; collators?: string[] }, + net: NetworkDef +): ForkTopology | undefined { + const cores = parseCounts('--cores', opts.cores, net); + const collators = parseCounts('--collators', opts.collators, net); + if (!Object.keys(cores).length && !Object.keys(collators).length) return undefined; + + const totalCores = planCores(net.parachains, cores).length; + const validators = Math.max(net.relay.validators, totalCores); + if (validators > MAX_VALIDATORS) { + throw new Error( + `--cores adds up to ${totalCores} cores, which need ${totalCores} validators; a fork runs at most ${MAX_VALIDATORS}` + ); + } + return { validators, cores, collators }; +} + +/** Whether a bundle was bitten with this topology — both absent counts as the same. */ +export function sameTopology(a: ForkTopology | undefined, b: ForkTopology | undefined): boolean { + const norm = (t: ForkTopology | undefined) => + JSON.stringify(t ? { v: t.validators, c: sorted(t.cores), k: sorted(t.collators) } : null); + const sorted = (r: Record) => Object.fromEntries(Object.entries(r).sort()); + return norm(a) === norm(b); +} + +/** The flags that ask for a topology, for an error message that says how to re-bite. */ +export function topologyFlags(t: ForkTopology): string { + return [ + ...Object.entries(t.cores).map(([k, n]) => `--cores ${k}=${n}`), + ...Object.entries(t.collators).map(([k, n]) => `--collators ${k}=${n}`), + ].join(' '); +} diff --git a/packages/cli/src/fork/validators.ts b/packages/cli/src/fork/validators.ts index 2b0b24b..0a0c58a 100644 --- a/packages/cli/src/fork/validators.ts +++ b/packages/cli/src/fork/validators.ts @@ -11,7 +11,7 @@ import { Keyring } from '@polkadot/keyring'; import { cryptoWaitReady } from '@polkadot/util-crypto'; import { blake2128Concat, compactLen, keyOf, twox64Concat, u128le, u32le } from './codec.js'; -import type { AuraScheme } from '@parity/ppn-network-config'; +import { collatorNodeName, relayNodeName, type AuraScheme } from '@parity/ppn-network-config'; export const ALICE_SR = 'd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d'; @@ -25,26 +25,51 @@ export interface DevValidator { authorityDiscovery: string; } -const mk = (stash: string, babe: string, grandpa: string, beefy: string): DevValidator => ({ - stash, babe, grandpa, beefy, - paraValidator: babe, paraAssignment: babe, authorityDiscovery: babe, -}); +/** Relay RPC ports are `RELAY_BASE_PORT + i`, and the eleventh would land on People's. */ +export const MAX_VALIDATORS = 10; /** - * Well-known dev keys, verbatim from zombie-bite src/utils.rs. + * The names the relay's dev keys derive from, in the order the authority set lists them. * - * Order matters: it must match get_validator_keys() — ALICE, BOB, CHARLIE, DAVE, FERDIE, - * EVE (note FERDIE before EVE) — because ActiveValidatorIndices and ValidatorGroups index - * into it positionally. + * The first six are zombie-bite's get_validator_keys() order — ALICE, BOB, CHARLIE, DAVE, + * FERDIE, EVE (note FERDIE before EVE) — because ActiveValidatorIndices and ValidatorGroups + * index into the list positionally, and a bundle bitten with the old hard-coded keys had that + * order. The names beyond them are whatever the fork TOML calls the extra nodes: zombienet + * derives a non-well-known node's keys from `//`, exactly as done here. */ -export const VALIDATORS: DevValidator[] = [ - mk('be5ddb1579b72e84524fc29e78609e3caf42e85aa118ebfe0b0ad404b5bdd25f', ALICE_SR, '88dc3417d5058ec4b4503e0c12ea1a0a89be200fe98922423d4334014fa6b0ee', '020a1091341fe5664bfa1782d5e04779689068c916b04cb365ec3153755684d9a1'), - mk('fe65717dad0447d715f660a0a58411de509b42e6efb8375f562f58a554d5860e', '8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48', 'd17c2d7823ebf260fd138f2d7e27d114c0145d968b5ff5006125f2414fadae69', '0390084fdbf27d2b79d26a4f13f0ccd982cb755a661969143c37cbc49ef5b91f27'), - mk('1e07379407fecc4b89eb7dbd287c2c781cfb1907a96947a3eb18e4f8e7198625', '90b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe22', '439660b36c6c03afafca027b910b4fecf99801834c62a5e6006f27d978de234f', '020e7446f3910e15fed2b2db1e71a01c57f3dd85cc2e65f30680220e09f8bbbc79'), - mk('e860f1b1c7227f7c22602f53f15af80747814dffd839719731ee3bba6edc126c', '306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20', '5e639b43e0052c47447dac87d6fd2b6ec50bdd4d0f614e4299c665249bbd09d9', '0227e2b139697b04eb01f4eef7e8f3724431b795c45ce6ef7b8e23a4e93f4abd26'), - mk('101191192fc877c24d725b337120fa3edc63d227bbc92705db1e2cb65f56981a', '1cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c', '568cb4a574c6d178feb39c27dfc8b3f789e5f5423e19c71633c748b9acf086b5', '0291f1217d5a04cb83312ee3d88a6e6b33284e053e6ccfc3a90339a0299d12967c'), - mk('8ac59e11963af19174d0b94d5d78041c233f55d2e19324665bafdfb62925af2d', 'e659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4e', '1dfe3e22cc0d45c70779c1095f7489a8ef3cf52d62fbd8c2fa38c9f1723502b5', '031d10105e323c4afce225208f71a6441ee327a65b9e646e772500c74d31f669aa'), -]; +function validatorNames(count: number): string[] { + const wellKnown = ['Alice', 'Bob', 'Charlie', 'Dave', 'Ferdie', 'Eve']; + return Array.from({ length: count }, (_, i) => wellKnown[i] ?? relayNodeName(i)); +} + +/** + * The dev keys for `count` relay validators, derived the way zombienet writes them into a + * node's keystore: stash `////stash`, BABE and the parachain keys sr25519 `//`, + * GRANDPA the ed25519 key of that seed and BEEFY its ecdsa key. For the first six this is + * byte-identical to the keys zombie-bite hard-codes in src/utils.rs (pinned by test), except + * Charlie's and Dave's BEEFY keys, which zombie-bite has as bytes no seed derives to; the + * keystore holds the derived ones, so those are what the session keys name here. + */ +export async function devValidators(count: number): Promise { + if (!Number.isInteger(count) || count < 1 || count > MAX_VALIDATORS) { + throw new Error(`a fork runs 1..${MAX_VALIDATORS} relay validators, not ${count}`); + } + await cryptoWaitReady(); + const hex = (scheme: 'sr25519' | 'ed25519' | 'ecdsa', suri: string) => + Buffer.from(new Keyring({ type: scheme }).addFromUri(suri).publicKey).toString('hex'); + return validatorNames(count).map((name) => { + const babe = hex('sr25519', `//${name}`); + return { + stash: hex('sr25519', `//${name}//stash`), + babe, + grandpa: hex('ed25519', `//${name}`), + beefy: hex('ecdsa', `//${name}`), + paraValidator: babe, + paraAssignment: babe, + authorityDiscovery: babe, + }; + }); +} export const sessionKeys = (v: DevValidator): string => v.grandpa + v.babe + v.paraValidator + v.paraAssignment + v.authorityDiscovery + v.beefy; @@ -63,9 +88,9 @@ export const sessionKeys = (v: DevValidator): string => * Hrmp::* / Dmp::* keeps the four HRMP channels * Paras::Parachains keeps all four parachains registered */ -export function relayCandidates(): Record { - const len = compactLen(VALIDATORS.length); - const each = (f: (v: DevValidator) => string) => VALIDATORS.map(f).join(''); +export function relayCandidates(validators: DevValidator[]): Record { + const len = compactLen(validators.length); + const each = (f: (v: DevValidator) => string) => validators.map(f).join(''); return { [keyOf('Session', 'Validators')]: len + each((v) => v.stash), @@ -74,7 +99,7 @@ export function relayCandidates(): Record { [keyOf('Babe', 'NextAuthorities')]: len + each((v) => v.babe + '0100000000000000'), [keyOf('Grandpa', 'Authorities')]: len + each((v) => v.grandpa + '0100000000000000'), [keyOf('Staking', 'Invulnerables')]: len + each((v) => v.stash), - [keyOf('ParasShared', 'ActiveValidatorIndices')]: len + VALIDATORS.map((_, i) => u32le(i)).join(''), + [keyOf('ParasShared', 'ActiveValidatorIndices')]: len + validators.map((_, i) => u32le(i)).join(''), [keyOf('ParasShared', 'ActiveValidatorKeys')]: len + each((v) => v.paraValidator), [keyOf('AuthorityDiscovery', 'Keys')]: len + each((v) => v.authorityDiscovery), [keyOf('AuthorityDiscovery', 'NextKeys')]: len + each((v) => v.authorityDiscovery), @@ -82,7 +107,7 @@ export function relayCandidates(): Record { // Each group carries its own compact length. Verified against production, whose real // value is `18` + 6x(`04` + u32) = [[0],[1],[2],[3],[4],[5]]. [keyOf('ParaScheduler', 'ValidatorGroups')]: - len + VALIDATORS.map((_, i) => compactLen(1) + u32le(i)).join(''), + len + validators.map((_, i) => compactLen(1) + u32le(i)).join(''), // Without this the relay takes the validator set Asset Hub elects. None of those accounts // holds session keys on a fork, so `pallet_session` queues an empty set and announces the // next BABE epoch with no authorities: nobody can claim a slot, no block enacts the next @@ -96,11 +121,11 @@ export function relayCandidates(): Record { } /** Session::NextKeys is a map, so our validators' entries are injects, not overrides. */ -export function relayInjects(): Record { +export function relayInjects(validators: DevValidator[]): Record { // twox128(":UsePreviousValidators:") const injects: Record = { c57d82d01f0fc18afc048ca20ac460dd: '01' }; const nextKeys = keyOf('Session', 'NextKeys'); - for (const v of VALIDATORS) { + for (const v of validators) { injects[nextKeys + twox64Concat(v.stash)] = sessionKeys(v); } return injects; @@ -132,19 +157,32 @@ export function sudoEndowInjects(): Record { } /** - * The collator key for a parachain. + * The collator keys for a parachain, one per collator node. * - * zombie-bite derives it from the seed "//Collator-" and names the collator + * zombie-bite derives the first from the seed "//Collator-" and names the collator * Collator-; zombienet derives non-well-known node keys the same way, so the two - * agree. Verified byte-identical against a real `bite -r paseo` run. + * agree. Verified byte-identical against a real `bite -r paseo` run. Further collators follow + * the node names collatorNodeName() gives them, for the same reason. * * The curve is the chain's, not ours: see auraScheme(). The same seed on the other curve * is a different key, and zombienet already writes it — under `gran`, not `aura`. */ -export async function collatorKey(paraId: number, scheme: AuraScheme = 'sr25519'): Promise { +export async function collatorKeys( + paraId: number, + count: number, + scheme: AuraScheme = 'sr25519' +): Promise { + if (!Number.isInteger(count) || count < 1) throw new Error(`a parachain runs at least one collator, not ${count}`); await cryptoWaitReady(); - const pair = new Keyring({ type: scheme }).addFromUri(`//Collator-${paraId}`); - return Buffer.from(pair.publicKey).toString('hex'); + const keyring = new Keyring({ type: scheme }); + return Array.from({ length: count }, (_, i) => + Buffer.from(keyring.addFromUri(`//${collatorNodeName(paraId, i)}`).publicKey).toString('hex') + ); +} + +/** The one collator every bite before collator counts installed. */ +export async function collatorKey(paraId: number, scheme: AuraScheme = 'sr25519'): Promise { + return (await collatorKeys(paraId, 1, scheme))[0]; } /** @@ -174,26 +212,36 @@ export function authorizedUpgradeCandidate(codeHash: string, checkVersion: boole return { [keyOf('System', 'AuthorizedUpgrade')]: hash + (checkVersion ? '01' : '00') }; } -export function paraCandidates(collator: string): Record { - const one = compactLen(1); +/** + * Parachain storage values that install the collators as its authority set. One key serves as + * both the account and the Aura key: the seed is the same and the account is the sr25519 public + * key, which is what CollatorSelection and Session hold. + */ +export function paraCandidates(collators: string[]): Record { + const len = compactLen(collators.length); + const all = collators.join(''); return { - [keyOf('CollatorSelection', 'DesiredCandidates')]: '01000000', - [keyOf('CollatorSelection', 'Invulnerables')]: one + collator, - [keyOf('AuraExt', 'Authorities')]: one + collator, - [keyOf('Aura', 'Authorities')]: one + collator, - [keyOf('Session', 'Validators')]: one + collator, - [keyOf('Session', 'QueuedKeys')]: one + collator + collator, + [keyOf('CollatorSelection', 'DesiredCandidates')]: u32le(collators.length), + [keyOf('CollatorSelection', 'Invulnerables')]: len + all, + [keyOf('AuraExt', 'Authorities')]: len + all, + [keyOf('Aura', 'Authorities')]: len + all, + [keyOf('Session', 'Validators')]: len + all, + [keyOf('Session', 'QueuedKeys')]: len + collators.map((c) => c + c).join(''), [keyOf('Sudo', 'Key')]: ALICE_SR, // ParachainSystem::LastDmqMqcHead is left alone: zeroing it (as zombie-bite does) // desyncs the parachain from the relay's preserved Dmp state. }; } -export function paraInjects(collator: string): Record { - return { - [keyOf('Session', 'NextKeys') + twox64Concat(collator)]: collator, +export function paraInjects(collators: string[]): Record { + const injects: Record = {}; + for (const collator of collators) { + injects[keyOf('Session', 'NextKeys') + twox64Concat(collator)] = collator; // Session::KeyOwner(("aura", collatorKey)) - ['cec5070d609dd3497f72bde07fc96ba0726380404683fc89e8233450c8aa1950eab3d4a1675d3d746175726180' + - collator]: collator, - }; + injects[ + 'cec5070d609dd3497f72bde07fc96ba0726380404683fc89e8233450c8aa1950eab3d4a1675d3d746175726180' + + collator + ] = collator; + } + return injects; } diff --git a/packages/cli/tests/fixtures/overrides.golden.json b/packages/cli/tests/fixtures/overrides.golden.json index 5303ae2..c974f2d 100644 --- a/packages/cli/tests/fixtures/overrides.golden.json +++ b/packages/cli/tests/fixtures/overrides.golden.json @@ -62,7 +62,7 @@ "relay": { "overrides": { "cec5070d609dd3497f72bde07fc96ba088dcde934c658227ee1dfafcd6e16903": "18be5ddb1579b72e84524fc29e78609e3caf42e85aa118ebfe0b0ad404b5bdd25ffe65717dad0447d715f660a0a58411de509b42e6efb8375f562f58a554d5860e1e07379407fecc4b89eb7dbd287c2c781cfb1907a96947a3eb18e4f8e7198625e860f1b1c7227f7c22602f53f15af80747814dffd839719731ee3bba6edc126c101191192fc877c24d725b337120fa3edc63d227bbc92705db1e2cb65f56981a8ac59e11963af19174d0b94d5d78041c233f55d2e19324665bafdfb62925af2d", - "cec5070d609dd3497f72bde07fc96ba0e0cdd062e6eaf24295ad4ccfc41d4609": "18be5ddb1579b72e84524fc29e78609e3caf42e85aa118ebfe0b0ad404b5bdd25f88dc3417d5058ec4b4503e0c12ea1a0a89be200fe98922423d4334014fa6b0eed43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27dd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27dd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27dd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d020a1091341fe5664bfa1782d5e04779689068c916b04cb365ec3153755684d9a1fe65717dad0447d715f660a0a58411de509b42e6efb8375f562f58a554d5860ed17c2d7823ebf260fd138f2d7e27d114c0145d968b5ff5006125f2414fadae698eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a488eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a488eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a488eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a480390084fdbf27d2b79d26a4f13f0ccd982cb755a661969143c37cbc49ef5b91f271e07379407fecc4b89eb7dbd287c2c781cfb1907a96947a3eb18e4f8e7198625439660b36c6c03afafca027b910b4fecf99801834c62a5e6006f27d978de234f90b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe2290b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe2290b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe2290b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe22020e7446f3910e15fed2b2db1e71a01c57f3dd85cc2e65f30680220e09f8bbbc79e860f1b1c7227f7c22602f53f15af80747814dffd839719731ee3bba6edc126c5e639b43e0052c47447dac87d6fd2b6ec50bdd4d0f614e4299c665249bbd09d9306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc200227e2b139697b04eb01f4eef7e8f3724431b795c45ce6ef7b8e23a4e93f4abd26101191192fc877c24d725b337120fa3edc63d227bbc92705db1e2cb65f56981a568cb4a574c6d178feb39c27dfc8b3f789e5f5423e19c71633c748b9acf086b51cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c1cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c1cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c1cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c0291f1217d5a04cb83312ee3d88a6e6b33284e053e6ccfc3a90339a0299d12967c8ac59e11963af19174d0b94d5d78041c233f55d2e19324665bafdfb62925af2d1dfe3e22cc0d45c70779c1095f7489a8ef3cf52d62fbd8c2fa38c9f1723502b5e659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4ee659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4ee659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4ee659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4e031d10105e323c4afce225208f71a6441ee327a65b9e646e772500c74d31f669aa", + "cec5070d609dd3497f72bde07fc96ba0e0cdd062e6eaf24295ad4ccfc41d4609": "18be5ddb1579b72e84524fc29e78609e3caf42e85aa118ebfe0b0ad404b5bdd25f88dc3417d5058ec4b4503e0c12ea1a0a89be200fe98922423d4334014fa6b0eed43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27dd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27dd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27dd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d020a1091341fe5664bfa1782d5e04779689068c916b04cb365ec3153755684d9a1fe65717dad0447d715f660a0a58411de509b42e6efb8375f562f58a554d5860ed17c2d7823ebf260fd138f2d7e27d114c0145d968b5ff5006125f2414fadae698eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a488eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a488eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a488eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a480390084fdbf27d2b79d26a4f13f0ccd982cb755a661969143c37cbc49ef5b91f271e07379407fecc4b89eb7dbd287c2c781cfb1907a96947a3eb18e4f8e7198625439660b36c6c03afafca027b910b4fecf99801834c62a5e6006f27d978de234f90b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe2290b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe2290b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe2290b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe220389411795514af1627765eceffcbd002719f031604fadd7d188e2dc585b4e1afbe860f1b1c7227f7c22602f53f15af80747814dffd839719731ee3bba6edc126c5e639b43e0052c47447dac87d6fd2b6ec50bdd4d0f614e4299c665249bbd09d9306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc2003bc9d0ca094bd5b8b3225d7651eac5d18c1c04bf8ae8f8b263eebca4e1410ed0c101191192fc877c24d725b337120fa3edc63d227bbc92705db1e2cb65f56981a568cb4a574c6d178feb39c27dfc8b3f789e5f5423e19c71633c748b9acf086b51cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c1cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c1cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c1cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c0291f1217d5a04cb83312ee3d88a6e6b33284e053e6ccfc3a90339a0299d12967c8ac59e11963af19174d0b94d5d78041c233f55d2e19324665bafdfb62925af2d1dfe3e22cc0d45c70779c1095f7489a8ef3cf52d62fbd8c2fa38c9f1723502b5e659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4ee659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4ee659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4ee659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4e031d10105e323c4afce225208f71a6441ee327a65b9e646e772500c74d31f669aa", "1cb6f36e027abb2091cfb5110ab5087f5e0621c4869aa60c02be9adcc98a0d1d": "18d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d01000000000000008eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48010000000000000090b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe220100000000000000306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc2001000000000000001cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c0100000000000000e659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4e0100000000000000", "1cb6f36e027abb2091cfb5110ab5087faacf00b9b41fda7a9268821c2a2b3e4c": "18d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d01000000000000008eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48010000000000000090b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe220100000000000000306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc2001000000000000001cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c0100000000000000e659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4e0100000000000000", "5f9cc45b7a00c5899361e1c6099678dc5e0621c4869aa60c02be9adcc98a0d1d": "1888dc3417d5058ec4b4503e0c12ea1a0a89be200fe98922423d4334014fa6b0ee0100000000000000d17c2d7823ebf260fd138f2d7e27d114c0145d968b5ff5006125f2414fadae690100000000000000439660b36c6c03afafca027b910b4fecf99801834c62a5e6006f27d978de234f01000000000000005e639b43e0052c47447dac87d6fd2b6ec50bdd4d0f614e4299c665249bbd09d90100000000000000568cb4a574c6d178feb39c27dfc8b3f789e5f5423e19c71633c748b9acf086b501000000000000001dfe3e22cc0d45c70779c1095f7489a8ef3cf52d62fbd8c2fa38c9f1723502b50100000000000000", @@ -79,8 +79,8 @@ "c57d82d01f0fc18afc048ca20ac460dd": "01", "cec5070d609dd3497f72bde07fc96ba04c014e6bf8b8c2c011e7290b85696bb3e535263148daaf49be5ddb1579b72e84524fc29e78609e3caf42e85aa118ebfe0b0ad404b5bdd25f": "88dc3417d5058ec4b4503e0c12ea1a0a89be200fe98922423d4334014fa6b0eed43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27dd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27dd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27dd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d020a1091341fe5664bfa1782d5e04779689068c916b04cb365ec3153755684d9a1", "cec5070d609dd3497f72bde07fc96ba04c014e6bf8b8c2c011e7290b85696bb30e5be00fbc2e15b5fe65717dad0447d715f660a0a58411de509b42e6efb8375f562f58a554d5860e": "d17c2d7823ebf260fd138f2d7e27d114c0145d968b5ff5006125f2414fadae698eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a488eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a488eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a488eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a480390084fdbf27d2b79d26a4f13f0ccd982cb755a661969143c37cbc49ef5b91f27", - "cec5070d609dd3497f72bde07fc96ba04c014e6bf8b8c2c011e7290b85696bb3ce4f6702f7c0a2951e07379407fecc4b89eb7dbd287c2c781cfb1907a96947a3eb18e4f8e7198625": "439660b36c6c03afafca027b910b4fecf99801834c62a5e6006f27d978de234f90b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe2290b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe2290b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe2290b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe22020e7446f3910e15fed2b2db1e71a01c57f3dd85cc2e65f30680220e09f8bbbc79", - "cec5070d609dd3497f72bde07fc96ba04c014e6bf8b8c2c011e7290b85696bb33d336178353c677be860f1b1c7227f7c22602f53f15af80747814dffd839719731ee3bba6edc126c": "5e639b43e0052c47447dac87d6fd2b6ec50bdd4d0f614e4299c665249bbd09d9306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc200227e2b139697b04eb01f4eef7e8f3724431b795c45ce6ef7b8e23a4e93f4abd26", + "cec5070d609dd3497f72bde07fc96ba04c014e6bf8b8c2c011e7290b85696bb3ce4f6702f7c0a2951e07379407fecc4b89eb7dbd287c2c781cfb1907a96947a3eb18e4f8e7198625": "439660b36c6c03afafca027b910b4fecf99801834c62a5e6006f27d978de234f90b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe2290b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe2290b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe2290b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe220389411795514af1627765eceffcbd002719f031604fadd7d188e2dc585b4e1afb", + "cec5070d609dd3497f72bde07fc96ba04c014e6bf8b8c2c011e7290b85696bb33d336178353c677be860f1b1c7227f7c22602f53f15af80747814dffd839719731ee3bba6edc126c": "5e639b43e0052c47447dac87d6fd2b6ec50bdd4d0f614e4299c665249bbd09d9306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc2003bc9d0ca094bd5b8b3225d7651eac5d18c1c04bf8ae8f8b263eebca4e1410ed0c", "cec5070d609dd3497f72bde07fc96ba04c014e6bf8b8c2c011e7290b85696bb33973b022fa2007af101191192fc877c24d725b337120fa3edc63d227bbc92705db1e2cb65f56981a": "568cb4a574c6d178feb39c27dfc8b3f789e5f5423e19c71633c748b9acf086b51cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c1cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c1cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c1cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c0291f1217d5a04cb83312ee3d88a6e6b33284e053e6ccfc3a90339a0299d12967c", "cec5070d609dd3497f72bde07fc96ba04c014e6bf8b8c2c011e7290b85696bb36dd53641586aad2e8ac59e11963af19174d0b94d5d78041c233f55d2e19324665bafdfb62925af2d": "1dfe3e22cc0d45c70779c1095f7489a8ef3cf52d62fbd8c2fa38c9f1723502b5e659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4ee659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4ee659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4ee659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4e031d10105e323c4afce225208f71a6441ee327a65b9e646e772500c74d31f669aa" } diff --git a/packages/cli/tests/fork-shared-relay.test.ts b/packages/cli/tests/fork-shared-relay.test.ts index 97f9b46..2f45e02 100644 --- a/packages/cli/tests/fork-shared-relay.test.ts +++ b/packages/cli/tests/fork-shared-relay.test.ts @@ -51,11 +51,28 @@ describe('planCores', () => { ]); }); - // 5 cores + 1 = 6 validators, which is exactly the number of dev keys a fork runs. If this - // ever exceeds them, groups start coming out empty again and blocks stop being backed. - it('plans no more cores than six validators can staff', () => { + // 5 cores + 1 = 6 validators, which is exactly the number of dev keys a fork runs by + // default. If this ever exceeds them, groups start coming out empty again and blocks stop + // being backed — which is why a bite asked for more cores grows the validator set to match. + it('plans no more cores than six validators can staff, by default', () => { assert.ok(planCores(PNV2).length < 6, 'a fork has six dev validators'); }); + + // `ppn bite --cores people=3`: the asked-for count replaces the default for that chain only. + it('lays a parachain out on as many cores as the bite asked for', () => { + assert.equal(coresFor('people', { people: 3 }), 3); + assert.equal(coresFor('asset-hub', { people: 3 }), 3, 'the default stays for the others'); + assert.equal(coresFor('asset-hub', { 'asset-hub': 1 }), 1, 'and can be cut too'); + assert.deepEqual(planCores(PNV2, { people: 3 }), [ + { core: 0, paraId: 1500 }, + { core: 1, paraId: 1500 }, + { core: 2, paraId: 1500 }, + { core: 3, paraId: 1502 }, + { core: 4, paraId: 1502 }, + { core: 5, paraId: 1502 }, + { core: 6, paraId: 1501 }, + ]); + }); }); describe('validatorGroups', () => { diff --git a/packages/cli/tests/fork-topology.test.ts b/packages/cli/tests/fork-topology.test.ts new file mode 100644 index 0000000..84f9296 --- /dev/null +++ b/packages/cli/tests/fork-topology.test.ts @@ -0,0 +1,69 @@ +// Tests for packages/cli/src/fork/topology.ts — what `--cores`/`--collators` settle into. +// Run with: tsx --test packages/cli/tests/fork-topology.test.ts + +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { loadNetwork } from '@parity/ppn-network-config'; +import { resolveTopology, sameTopology, topologyFlags } from '../src/fork/topology.js'; + +const previewnet = loadNetwork('previewnet'); + +describe('resolveTopology', () => { + it('is nothing when nothing was asked for', () => { + assert.equal(resolveTopology({}, previewnet), undefined); + assert.equal(resolveTopology({ cores: [], collators: [] }, previewnet), undefined); + }); + + // previewnet: Asset Hub 3 + People 3 + Bulletin 1 + Web3 Storage 1 = 8 cores, and every + // core needs a validator group of its own or nothing backs the blocks on it. + it('grows the validator set to one per core', () => { + assert.deepEqual(resolveTopology({ cores: ['people=3'], collators: ['people=5'] }, previewnet), { + validators: 8, + cores: { people: 3 }, + collators: { people: 5 }, + }); + }); + + it('keeps the descriptor validators when the cores fit inside them', () => { + assert.equal(resolveTopology({ collators: ['people=5'] }, previewnet)?.validators, previewnet.relay.validators); + }); + + it('refuses a chain the network does not run, a bad count, and a malformed spec', () => { + assert.throws(() => resolveTopology({ cores: ['kitchen-sink=3'] }, previewnet), /does not run/); + assert.throws(() => resolveTopology({ cores: ['people=0'] }, previewnet), /at least 1/); + assert.throws(() => resolveTopology({ collators: ['people=two'] }, previewnet), /at least 1/); + assert.throws(() => resolveTopology({ cores: ['people'] }, previewnet), /=/); + }); + + // The eleventh relay RPC port is People's. + it('refuses more cores than the relay has validator ports for', () => { + assert.throws(() => resolveTopology({ cores: ['people=6'] }, previewnet), /at most 10/); + }); +}); + +describe('sameTopology', () => { + const t = { validators: 8, cores: { people: 3 }, collators: { people: 5 } }; + + it('treats two default bites as the same', () => { + assert.ok(sameTopology(undefined, undefined)); + }); + + it('tells a default bundle from an asked-for one, either way round', () => { + assert.ok(!sameTopology(t, undefined)); + assert.ok(!sameTopology(undefined, t)); + }); + + it('compares what was asked for, not key order', () => { + assert.ok(sameTopology(t, { validators: 8, collators: { people: 5 }, cores: { people: 3 } })); + assert.ok(!sameTopology(t, { ...t, collators: { people: 4 } })); + }); +}); + +describe('topologyFlags', () => { + it('spells the flags that would ask for it again', () => { + assert.equal( + topologyFlags({ validators: 8, cores: { people: 3 }, collators: { people: 5 } }), + '--cores people=3 --collators people=5' + ); + }); +}); diff --git a/packages/cli/tests/fork-validators.test.ts b/packages/cli/tests/fork-validators.test.ts index e84fcd2..b28dc77 100644 --- a/packages/cli/tests/fork-validators.test.ts +++ b/packages/cli/tests/fork-validators.test.ts @@ -7,14 +7,17 @@ import fs from 'node:fs'; import path from 'node:path'; import { ALICE_SR, - VALIDATORS, + MAX_VALIDATORS, collatorKey, + collatorKeys, + devValidators, paraCandidates, paraInjects, relayCandidates, relayInjects, sessionKeys, authorizedUpgradeCandidate, + type DevValidator, } from '../src/fork/validators.js'; import { verify, verifyInjects } from '../src/fork/overrides.js'; import { compactLen, keyOf, u32le } from '../src/fork/codec.js'; @@ -27,14 +30,36 @@ const GOLDEN = JSON.parse( fs.readFileSync(path.join(import.meta.dirname, 'fixtures/overrides.golden.json'), 'utf-8') ) as Record; injects: Record }>; +// Well-known dev keys as zombie-bite src/utils.rs hard-codes them, in get_validator_keys() +// order. The keys used to be hard-coded from this list; now they are derived from the names, +// and the derivation must land on these exact bytes or every bundle's authority set silently +// changes. One deliberate difference: zombie-bite's BEEFY keys for Charlie and Dave +// (020e7446…, 0227e2b1…) derive from no seed at all, while zombienet writes the ecdsa key of +// `//Charlie` (0389…) and `//Dave` (03bc…) into the keystore — so those two are pinned to what +// the nodes actually hold. +const mk = (stash: string, babe: string, grandpa: string, beefy: string): DevValidator => ({ + stash, babe, grandpa, beefy, + paraValidator: babe, paraAssignment: babe, authorityDiscovery: babe, +}); +const ZOMBIE_BITE_KEYS: DevValidator[] = [ + mk('be5ddb1579b72e84524fc29e78609e3caf42e85aa118ebfe0b0ad404b5bdd25f', ALICE_SR, '88dc3417d5058ec4b4503e0c12ea1a0a89be200fe98922423d4334014fa6b0ee', '020a1091341fe5664bfa1782d5e04779689068c916b04cb365ec3153755684d9a1'), + mk('fe65717dad0447d715f660a0a58411de509b42e6efb8375f562f58a554d5860e', '8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48', 'd17c2d7823ebf260fd138f2d7e27d114c0145d968b5ff5006125f2414fadae69', '0390084fdbf27d2b79d26a4f13f0ccd982cb755a661969143c37cbc49ef5b91f27'), + mk('1e07379407fecc4b89eb7dbd287c2c781cfb1907a96947a3eb18e4f8e7198625', '90b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe22', '439660b36c6c03afafca027b910b4fecf99801834c62a5e6006f27d978de234f', '0389411795514af1627765eceffcbd002719f031604fadd7d188e2dc585b4e1afb'), + mk('e860f1b1c7227f7c22602f53f15af80747814dffd839719731ee3bba6edc126c', '306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20', '5e639b43e0052c47447dac87d6fd2b6ec50bdd4d0f614e4299c665249bbd09d9', '03bc9d0ca094bd5b8b3225d7651eac5d18c1c04bf8ae8f8b263eebca4e1410ed0c'), + mk('101191192fc877c24d725b337120fa3edc63d227bbc92705db1e2cb65f56981a', '1cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c', '568cb4a574c6d178feb39c27dfc8b3f789e5f5423e19c71633c748b9acf086b5', '0291f1217d5a04cb83312ee3d88a6e6b33284e053e6ccfc3a90339a0299d12967c'), + mk('8ac59e11963af19174d0b94d5d78041c233f55d2e19324665bafdfb62925af2d', 'e659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4e', '1dfe3e22cc0d45c70779c1095f7489a8ef3cf52d62fbd8c2fa38c9f1723502b5', '031d10105e323c4afce225208f71a6441ee327a65b9e646e772500c74d31f669aa'), +]; + +const VALIDATORS = await devValidators(6); + describe('the dev authority set', () => { - it('has six validators', () => { - assert.equal(VALIDATORS.length, 6); + it('derives the six well-known keys byte-for-byte as zombie-bite hard-codes them', () => { + assert.deepEqual(VALIDATORS, ZOMBIE_BITE_KEYS); }); it('starts with Alice, whose babe key is also the sudo key', () => { assert.equal(VALIDATORS[0].babe, ALICE_SR); - assert.equal(relayCandidates()[keyOf('Sudo', 'Key')], ALICE_SR); + assert.equal(relayCandidates(VALIDATORS)[keyOf('Sudo', 'Key')], ALICE_SR); }); // ActiveValidatorIndices and ValidatorGroups index into this list positionally, so the @@ -46,13 +71,35 @@ describe('the dev authority set', () => { assert.equal(VALIDATORS[5].babe, ferdie, 'index 5 should be Ferdie'); }); - it('has no duplicate keys', () => { + // Past the well-known six, a validator is named for its position and zombienet derives its + // keystore from that name — the keys here must be the same derivation, or the node holds + // keys no authority entry mentions and never authors. + it('extends past the well-known six with keys derived from the node names', async () => { + const eight = await devValidators(8); + assert.deepEqual(eight.slice(0, 6), ZOMBIE_BITE_KEYS, 'the first six do not move'); + const { Keyring } = await import('@polkadot/keyring'); + const hex = (k: Uint8Array) => Buffer.from(k).toString('hex'); + assert.equal(eight[6].babe, hex(new Keyring({ type: 'sr25519' }).addFromUri('//Validator-7').publicKey)); + assert.equal(eight[6].stash, hex(new Keyring({ type: 'sr25519' }).addFromUri('//Validator-7//stash').publicKey)); + assert.equal(eight[6].grandpa, hex(new Keyring({ type: 'ed25519' }).addFromUri('//Validator-7').publicKey)); + assert.equal(eight[6].beefy, hex(new Keyring({ type: 'ecdsa' }).addFromUri('//Validator-7').publicKey)); + assert.equal(eight[7].babe, hex(new Keyring({ type: 'sr25519' }).addFromUri('//Validator-8').publicKey)); + }); + + it('has no duplicate keys, however many', async () => { + const all = await devValidators(MAX_VALIDATORS); for (const field of ['stash', 'babe', 'grandpa', 'beefy'] as const) { - const values = VALIDATORS.map((v) => v[field]); - assert.equal(new Set(values).size, 6, `duplicate ${field}`); + const values = all.map((v) => v[field]); + assert.equal(new Set(values).size, MAX_VALIDATORS, `duplicate ${field}`); } }); + // The eleventh relay RPC port is People's. + it('refuses more validators than the relay has ports for', async () => { + await assert.rejects(devValidators(MAX_VALIDATORS + 1), /at most|1\.\.10/); + await assert.rejects(devValidators(0), /1\.\.10/); + }); + it('composes session keys as grandpa+babe+paraValidator+paraAssignment+discovery+beefy', () => { const v = VALIDATORS[0]; assert.equal(sessionKeys(v), v.grandpa + v.babe + v.babe + v.babe + v.babe + v.beefy); @@ -62,17 +109,28 @@ describe('the dev authority set', () => { }); describe('relay overrides', () => { - const candidates = relayCandidates(); + const candidates = relayCandidates(VALIDATORS); it('matches the values a verified bite produced', () => { for (const [key, value] of Object.entries(GOLDEN.relay.overrides)) { assert.equal(candidates[key], value, `override ${key} changed`); } for (const [key, value] of Object.entries(GOLDEN.relay.injects)) { - assert.equal(relayInjects()[key], value, `inject ${key} changed`); + assert.equal(relayInjects(VALIDATORS)[key], value, `inject ${key} changed`); } }); + it('lists every validator it is given, in order', async () => { + const eight = await devValidators(8); + const c = relayCandidates(eight); + assert.equal( + c[keyOf('ParasShared', 'ActiveValidatorIndices')], + compactLen(8) + [0, 1, 2, 3, 4, 5, 6, 7].map(u32le).join('') + ); + assert.equal(c[keyOf('Session', 'Validators')], compactLen(8) + eight.map((v) => v.stash).join('')); + assert.equal(Object.keys(relayInjects(eight)).length, 9); + }); + // The bug this guards: without the inner compact length each group encoded as empty, the // value decoded as [[], …], and cores would have been silently mis-assigned. it('gives every validator group its own compact length', () => { @@ -118,7 +176,7 @@ describe('relay overrides', () => { }); it('injects session keys for all six validators plus the UsePreviousValidators flag', () => { - const injects = relayInjects(); + const injects = relayInjects(VALIDATORS); assert.equal(injects['c57d82d01f0fc18afc048ca20ac460dd'], '01'); assert.equal(Object.keys(injects).length, 7); }); @@ -136,8 +194,8 @@ describe('para overrides', () => { it('matches the values a verified bite produced, for every parachain', async () => { for (const { paraId } of PARACHAINS) { const collator = await collatorKey(paraId); - const candidates = paraCandidates(collator); - const injects = paraInjects(collator); + const candidates = paraCandidates([collator]); + const injects = paraInjects([collator]); for (const [key, value] of Object.entries(GOLDEN[paraId].overrides)) { assert.equal(candidates[key], value, `para ${paraId} override ${key} changed`); } @@ -147,18 +205,45 @@ describe('para overrides', () => { } }); - it('installs exactly one collator as the authority', async () => { + it('installs exactly one collator as the authority by default', async () => { const collator = await collatorKey(1500); - const candidates = paraCandidates(collator); + const candidates = paraCandidates([collator]); assert.equal(candidates[keyOf('CollatorSelection', 'DesiredCandidates')], '01000000'); for (const [pallet, item] of [['Aura', 'Authorities'], ['AuraExt', 'Authorities'], ['Session', 'Validators']]) { assert.equal(candidates[keyOf(pallet, item)], compactLen(1) + collator, `${pallet}::${item}`); } }); + // The first key is the one every existing bundle was bitten with; the rest follow the node + // names the fork TOML gives the extra collators, which is where zombienet derives them from. + it('derives further collator keys from Collator--, keeping the first', async () => { + const five = await collatorKeys(1502, 5); + assert.equal(five.length, 5); + assert.equal(five[0], await collatorKey(1502)); + assert.equal(new Set(five).size, 5, 'duplicate collator keys'); + const { Keyring } = await import('@polkadot/keyring'); + assert.equal( + five[1], + Buffer.from(new Keyring({ type: 'sr25519' }).addFromUri('//Collator-1502-2').publicKey).toString('hex') + ); + await assert.rejects(collatorKeys(1502, 0), /at least one collator/); + }); + + it('installs every collator it is given as an authority', async () => { + const five = await collatorKeys(1502, 5); + const candidates = paraCandidates(five); + assert.equal(candidates[keyOf('CollatorSelection', 'DesiredCandidates')], u32le(5)); + for (const [pallet, item] of [['Aura', 'Authorities'], ['AuraExt', 'Authorities'], ['Session', 'Validators'], ['CollatorSelection', 'Invulnerables']]) { + assert.equal(candidates[keyOf(pallet, item)], compactLen(5) + five.join(''), `${pallet}::${item}`); + } + assert.equal(candidates[keyOf('Session', 'QueuedKeys')], compactLen(5) + five.map((c) => c + c).join('')); + // One NextKeys entry and one KeyOwner entry per collator. + assert.equal(Object.keys(paraInjects(five)).length, 10); + }); + // Zeroing it, as zombie-bite does, desyncs the parachain from the relay's preserved Dmp. it('leaves ParachainSystem::LastDmqMqcHead alone', async () => { - const candidates = paraCandidates(await collatorKey(1500)); + const candidates = paraCandidates([await collatorKey(1500)]); assert.equal(candidates[keyOf('ParachainSystem', 'LastDmqMqcHead')], undefined); }); }); diff --git a/packages/cli/tests/start-network.test.ts b/packages/cli/tests/start-network.test.ts index 4697559..c753fef 100644 --- a/packages/cli/tests/start-network.test.ts +++ b/packages/cli/tests/start-network.test.ts @@ -7,7 +7,14 @@ import net from 'node:net'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { dataDirFor, binDirFor, forkDirFor, checkPorts, forkDataVerdict } from '../src/commands/start.js'; +import { + dataDirFor, + binDirFor, + forkDirFor, + checkPorts, + forkDataVerdict, + assertBundleTopology, +} from '../src/commands/start.js'; import { forkBundleName, forkBundleAsset } from '../src/lib/fork-bundle-name.js'; import { writeSpawnStamp, SPAWN_FILE } from '../src/lib/spawn-stamp.js'; @@ -108,6 +115,48 @@ describe('checkPorts', () => { }); }); +// `--cores`/`--collators` are fixed at bite time, so a spawn can only refuse a bundle bitten +// differently. The check runs once a bundle is in place however it got there — on disk, +// downloaded, or just bitten — because a downloaded nightly bundle carries the defaults, and +// spawning it for a run that asked for a layout used to ignore the flags without a word. +describe('assertBundleTopology', () => { + const PEOPLE = { validators: 8, cores: { people: 3 }, collators: { people: 5 } }; + const bundle = (topology?: typeof PEOPLE) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ppn-topology-')); + fs.writeFileSync( + path.join(dir, 'manifest.json'), + JSON.stringify({ bittenAt: '2026-09-01T18:30:00.000Z', biteBlocks: { relay: 1 }, ...(topology ? { topology } : {}) }) + ); + return dir; + }; + + it('passes a default bundle for a run that asks for nothing', () => { + assert.doesNotThrow(() => assertBundleTopology(bundle(), 'previewnet', undefined)); + }); + + it('passes a bundle bitten with exactly what the run asks for', () => { + assert.doesNotThrow(() => assertBundleTopology(bundle(PEOPLE), 'previewnet', PEOPLE)); + }); + + // The downloaded-nightly case: flags given, bundle bitten with the defaults. + it('refuses a default bundle for a run that asks for a layout, saying how to re-bite', () => { + assert.throws( + () => assertBundleTopology(bundle(), 'previewnet', PEOPLE), + (err: Error) => + err.message.includes('was bitten with the defaults') && + err.message.includes('ppn start previewnet --fork --fresh-bite --cores people=3 --collators people=5') + ); + }); + + it('refuses a laid-out bundle for a run that asks for another layout, or for none', () => { + assert.throws( + () => assertBundleTopology(bundle(PEOPLE), 'previewnet', { ...PEOPLE, collators: { people: 4 } }), + /bitten with --cores people=3 --collators people=5/ + ); + assert.throws(() => assertBundleTopology(bundle(PEOPLE), 'previewnet', undefined), /--fresh-bite$/m); + }); +}); + // A fork start used to wipe its data every time, so a stopped fork came back at the bite // block and an enacted upgrade had to be re-applied. Now it resumes — but only when the // spawn stamp says the databases came from the very bundle being spawned. Resuming on diff --git a/packages/network-config/src/bundle.ts b/packages/network-config/src/bundle.ts index cdef679..aaa48f2 100644 --- a/packages/network-config/src/bundle.ts +++ b/packages/network-config/src/bundle.ts @@ -16,6 +16,22 @@ export interface ForkManifestChain { genesis: string; } +/** + * How many nodes and cores a bite laid the network out with, when `ppn bite --cores/--collators` + * asked for something other than the default (one collator per parachain, three cores for Asset + * Hub and one for everything else, the descriptor's validators). Fixed at bite time: the + * authority sets and the core layout are state inside the snapshots, so a spawn can only follow + * what the bundle says. + */ +export interface ForkTopology { + /** Relay validators — at least one per core, so this grows with `cores`. */ + validators: number; + /** Cores per parachain key, only the ones that differ from the default. */ + cores: Record; + /** Collators per parachain key, only the ones that differ from one. */ + collators: Record; +} + export interface ForkManifest { bittenAt: string; source: string; @@ -35,5 +51,7 @@ export interface ForkManifest { * `ppn runtime-upgrade` submits the apply half, which needs no privilege. */ seededUpgrades?: Record; + /** Absent when the bite was asked for nothing beyond the defaults. */ + topology?: ForkTopology; } diff --git a/packages/network-config/src/fork-toml.ts b/packages/network-config/src/fork-toml.ts index f456edb..5f59855 100644 --- a/packages/network-config/src/fork-toml.ts +++ b/packages/network-config/src/fork-toml.ts @@ -39,7 +39,8 @@ import fs from 'node:fs'; import { - VALIDATORS, + relayNodeName, + collatorNodeName, PORTS, P2P_PORTS, RELAY_BASE_PORT, @@ -58,7 +59,7 @@ import { dubCustomProcesses } from './dub.js'; // Bundle manifest — built by ./fork/manifest.ts, extended with biteBlocks by bite.sh // --------------------------------------------------------------------------- -export type { ForkManifest, ForkManifestChain } from './bundle.js'; +export type { ForkManifest, ForkManifestChain, ForkTopology } from './bundle.js'; // Services, data loading, and spec publishing — see note 4 above. // @@ -192,11 +193,14 @@ function chainIdOf(specPath: string): string { return JSON.parse(fs.readFileSync(specPath, 'utf-8')).id; } +// As many as the bite installed keys for: the well-known six, then names zombienet derives +// keys from (see relayNodeName). One RPC port each, counting up from the relay base port. function relayNodes(count: number): string { - return VALIDATORS.slice(0, count).map( - (name, i) => ` + return Array.from( + { length: count }, + (_, i) => ` [[relaychain.nodes]] -name = "${name}" +name = "${relayNodeName(i)}" validator = true rpc_port = ${RELAY_BASE_PORT + i}${i === 0 ? `\np2p_port = ${P2P_PORTS.relay}` : ''} env = [{ name = "ZOMBIE_DISPUTE_CANDIDATE_LIFETIME_AFTER_FINALIZATION", value = "1" }]` @@ -245,6 +249,30 @@ function parachainSection( } const envLine = env.length ? `\nenv = [${env.join(', ')}]` : ''; + // As many collators as the bite installed as authorities (manifest.topology), every one + // restoring the same snapshot. Only the first holds the documented ports; the rest take + // whatever zombienet hands out, and drop a fixed listen address that would collide. + const count = manifest.topology?.collators[key] ?? 1; + const collators = Array.from({ length: count }, (_, i) => + i === 0 + ? ` +[[parachains.collators]] +name = "${collatorNodeName(paraId, i)}" +rpc_port = ${PORTS[key]} +p2p_port = ${P2P_PORTS[key]} +command = "${scriptsDir}/omni-node.sh" +db_snapshot = "${bundleDir}/snapshots/${paraId}.tgz"${envLine} +args = ${tomlArgs(args)} +` + : ` +[[parachains.collators]] +name = "${collatorNodeName(paraId, i)}" +command = "${scriptsDir}/omni-node.sh" +db_snapshot = "${bundleDir}/snapshots/${paraId}.tgz"${envLine} +args = ${tomlArgs(args.filter((a) => !a.startsWith('--listen-addr=')))} +` + ); + return ` ## ${chain.specName} (${paraId}) — bitten at block ${manifest.biteBlocks[paraId]} [[parachains]] @@ -252,15 +280,7 @@ id = ${paraId} cumulus_based = true chain = "${chainIdOf(specPath)}" chain_spec_path = "${specPath}" - -[[parachains.collators]] -name = "Collator-${paraId}" -rpc_port = ${PORTS[key]} -p2p_port = ${P2P_PORTS[key]} -command = "${scriptsDir}/omni-node.sh" -db_snapshot = "${bundleDir}/snapshots/${paraId}.tgz"${envLine} -args = ${tomlArgs(args)} -`; +${collators.join('')}`; } /** @@ -362,7 +382,9 @@ export function generateForkToml(options: GenerateForkTomlOptions): string { # DO NOT EDIT — regenerate with: ppn fork toml # Forked ${net.name}, bitten ${manifest.bittenAt} from ${manifest.source} # Production node version at bite time: ${manifest.nodeVersion} -# Bite blocks: ${JSON.stringify(manifest.biteBlocks)} +# Bite blocks: ${JSON.stringify(manifest.biteBlocks)}${ + manifest.topology ? `\n# Topology: ${JSON.stringify(manifest.topology)}` : '' + } [settings] timeout = 1200 @@ -374,7 +396,7 @@ chain_spec_path = "${relaySpecPath}" default_command = "${binDir}/${net.relay.binary.name}" default_db_snapshot = "${bundleDir}/snapshots/relay.tgz" default_args = ${tomlArgs(relayArgs)} -${relayNodes(net.relay.validators)} +${relayNodes(manifest.topology?.validators ?? net.relay.validators)} ${parachains .map((p) => parachainSection(p, manifest, bundleDir, scriptsDir, enableHop, net, options.binDir)) .join('')} diff --git a/packages/network-config/src/index.ts b/packages/network-config/src/index.ts index a98235c..a577324 100644 --- a/packages/network-config/src/index.ts +++ b/packages/network-config/src/index.ts @@ -31,6 +31,8 @@ export { paraIds, RELAY_BASE_PORT, VALIDATORS, + relayNodeName, + collatorNodeName, POPULAR_LOG_TARGETS, LOG_LEVELS, buildArgs, diff --git a/packages/network-config/src/toml-generator.ts b/packages/network-config/src/toml-generator.ts index 1f5f6a8..7bca602 100644 --- a/packages/network-config/src/toml-generator.ts +++ b/packages/network-config/src/toml-generator.ts @@ -20,6 +20,24 @@ import { dubCustomProcesses } from './dub.js'; const VALIDATORS = ['alice', 'bob', 'charlie', 'dave', 'eve', 'ferdie'] as const; +/** + * The name of the i-th relay node in fork mode. The first six are the well-known names, which + * zombienet maps to the well-known dev keys; past them zombienet derives a node's keys from + * `//` — the same rule that makes `Collator-` work — so any name serves, as long + * as the bite installs the keys derived from it (cli fork/validators.ts). + */ +function relayNodeName(i: number): string { + return VALIDATORS[i] ?? `Validator-${i + 1}`; +} + +/** + * The name of the i-th collator of a parachain. The first keeps the name every bite has used, + * so a bundle bitten before collator counts existed still spawns; the rest are numbered from 2. + */ +function collatorNodeName(paraId: number, i: number): string { + return i === 0 ? `Collator-${paraId}` : `Collator-${paraId}-${i + 1}`; +} + const VALID_PARACHAINS: Parachain[] = PARACHAIN_KEYS; // Genesis is previewnet-only by design (see networks/README.md), but everything about @@ -580,6 +598,8 @@ export { LOG_LEVELS, // Shared with fork-toml.ts so that fork mode and genesis mode cannot drift apart. VALIDATORS, + relayNodeName, + collatorNodeName, PORTS, paraIds, RELAY_BASE_PORT, diff --git a/packages/network-config/tests/fork-toml.test.ts b/packages/network-config/tests/fork-toml.test.ts index ff22b2e..5301956 100644 --- a/packages/network-config/tests/fork-toml.test.ts +++ b/packages/network-config/tests/fork-toml.test.ts @@ -171,6 +171,58 @@ describe('generateForkToml — relay chain', () => { }); }); +// `ppn bite --cores people=3 --collators people=5` writes the layout it installed into the +// manifest, and the spawn has to match it: the bite installed keys for exactly these nodes. +describe('generateForkToml — topology from the bundle', () => { + const topology = { validators: 8, cores: { people: 3 }, collators: { people: 5 } }; + const toml = generate((m) => (m.topology = topology)); + + it('runs as many relay validators as the bite installed keys for', () => { + assert.equal(toml.split('[[relaychain.nodes]]').length - 1, 8); + for (const [i, name] of ['alice', 'bob', 'charlie', 'dave', 'eve', 'ferdie', 'Validator-7', 'Validator-8'].entries()) { + assert.ok(toml.includes(`name = "${name}"\nvalidator = true\nrpc_port = ${RELAY_BASE_PORT + i}`), + `${name} missing or on the wrong port`); + } + assert.equal(toml.split('ZOMBIE_DISPUTE_CANDIDATE_LIFETIME_AFTER_FINALIZATION').length - 1, 8); + }); + + it('runs as many collators as the bite installed as authorities, all off the one snapshot', () => { + const people = paraIds().people; + const names = [...toml.matchAll(/name = "(Collator-[\d-]+)"/g)].map((m) => m[1]); + assert.deepEqual( + names.filter((n) => n.startsWith(`Collator-${people}`)), + [`Collator-${people}`, `Collator-${people}-2`, `Collator-${people}-3`, `Collator-${people}-4`, `Collator-${people}-5`] + ); + assert.equal(toml.split(`db_snapshot = "`).length - 1, 1 + 4 + 4, 'relay default + 4 first collators + 4 extra'); + assert.equal(toml.split(`snapshots/${people}.tgz`).length - 1, 5); + // The other chains keep the one collator every bundle has had. + assert.equal(names.filter((n) => n.startsWith(`Collator-${paraIds().bulletin}`)).length, 1); + }); + + it('gives the documented ports to the first collator only', () => { + assert.equal(toml.split(`rpc_port = ${PORTS.people}`).length - 1, 1); + assert.equal(toml.split(`p2p_port = ${P2P_PORTS.people}`).length - 1, 1); + const extra = toml.split(`name = "Collator-${paraIds().people}-2"`)[1].split('[[')[0]; + assert.doesNotMatch(extra, /rpc_port|p2p_port/); + }); + + // Asset Hub binds a fixed webrtc listen address; a second collator on the same port would + // fail to bind, so the extras go without and let zombienet pick. + it('drops a fixed listen address from the extra collators', () => { + const ah = generate((m) => (m.topology = { validators: 6, cores: {}, collators: { 'asset-hub': 2 } })); + const first = collatorArgs(ah, paraIds()['asset-hub']); + assert.ok(first.some((a) => a.startsWith('--listen-addr=')), 'the first keeps it'); + const extra = ah.split(`name = "Collator-${paraIds()['asset-hub']}-2"`)[1].split('[[')[0]; + assert.doesNotMatch(extra, /--listen-addr=/); + assert.match(extra, /--relay-chain-rpc-urls=/, 'every other flag stays'); + }); + + it('records the topology in the header', () => { + assert.ok(toml.includes(`# Topology: ${JSON.stringify(topology)}`)); + assert.ok(!generate().includes('# Topology:'), 'no line when the bundle has none'); + }); +}); + describe('generateForkToml — collators', () => { const toml = generate(); From 79bf6a7c79070c9d6f754a0bb903efb55444ff11 Mon Sep 17 00:00:00 2001 From: Mak <1177472+mordamax@users.noreply.github.com> Date: Mon, 28 Sep 2026 12:18:11 +0100 Subject: [PATCH 02/27] Run a TURN/STUN relay for DUB (#38) DUB's turn-api minted TURN credentials with nothing to relay them and an empty server list. This runs eturnal next to DUB and wires the two together, so p2p clients get working STUN/TURN servers from `POST /api/v1/turn/issue`. - New `turn` process: eturnal from eturnal.net on Linux, from the processone Homebrew tap on macOS - DUB gets `TURN_SECRET` and `ICE_SERVERS` from the same code that configures eturnal. A deployment must set its own `TURN_SECRET` (base64 of a printable string) - `/api/network` and the dashboard list the ICE servers, with a STUN health probe - `ppn nginx-conf` emits a TLS stream for `turns:` at a new `{{GENERATED_STREAMS}}` marker, so the deploy template needs that marker in its `stream {}` block - `11-turn.zndsl` checks STUN, credential issuance and an authenticated Allocate against the relay Details in `docs/TURN.md` --- .github/workflows/zombienet-tests.yml | 8 + CLAUDE.md | 7 +- Makefile | 6 + config/ports.env | 9 + config/versions.env | 3 + docs/DEVICE-UNIQUENESS-BACKEND.md | 19 +- docs/TURN.md | 77 +++++++ packages/cli/src/commands/dashboard.ts | 43 +++- packages/cli/src/commands/fetch.ts | 37 ++++ packages/cli/src/commands/nginx-conf.ts | 20 +- packages/cli/src/commands/service.ts | 2 + packages/cli/src/commands/turn.ts | 104 +++++++++ packages/cli/src/lib/eturnal.ts | 38 ++++ packages/dashboard-ui/src/lib/Ice.svelte | 43 ++++ packages/dashboard-ui/src/lib/Overview.svelte | 2 + packages/dashboard-ui/src/lib/api.ts | 14 ++ .../network-config/src/dashboard-model.ts | 19 +- packages/network-config/src/dub.ts | 21 +- packages/network-config/src/fork-toml.ts | 3 +- packages/network-config/src/index.ts | 13 +- packages/network-config/src/toml-generator.ts | 2 +- packages/network-config/src/turn.ts | 153 +++++++++++++ .../tests/dashboard-model.test.ts | 33 ++- packages/network-config/tests/dub.test.ts | 5 + packages/network-config/tests/turn.test.ts | 65 ++++++ scripts/dub/service.sh | 11 +- scripts/run-tests.sh | 1 + scripts/turn.sh | 17 ++ tests/11-turn.zndsl | 22 ++ tests/scripts/dub-auth.ts | 45 ++++ tests/scripts/test-dub-registration.ts | 35 +-- tests/scripts/test-turn.ts | 201 ++++++++++++++++++ zombienet-configs/local-dev.toml | 4 + 33 files changed, 1022 insertions(+), 60 deletions(-) create mode 100644 docs/TURN.md create mode 100644 packages/cli/src/commands/turn.ts create mode 100644 packages/cli/src/lib/eturnal.ts create mode 100644 packages/dashboard-ui/src/lib/Ice.svelte create mode 100644 packages/network-config/src/turn.ts create mode 100644 packages/network-config/tests/turn.test.ts create mode 100755 scripts/turn.sh create mode 100644 tests/11-turn.zndsl create mode 100644 tests/scripts/dub-auth.ts create mode 100644 tests/scripts/test-turn.ts diff --git a/.github/workflows/zombienet-tests.yml b/.github/workflows/zombienet-tests.yml index 48c65e0..ac9249b 100644 --- a/.github/workflows/zombienet-tests.yml +++ b/.github/workflows/zombienet-tests.yml @@ -462,6 +462,14 @@ jobs: ./tests/10-network-suffix.zndsl \ ${{ env.ZOMBIE_DIR }}/zombie.json + - name: Run TURN Relay Tests + if: ${{ github.event.inputs.test_file == 'all' || github.event.inputs.test_file == '11-turn.zndsl' || github.event.inputs.test_file == '' }} + run: | + node bin/ppn.mjs zombie-compat ${{ env.ZOMBIE_DIR }}/zombie.json + BIN="${{ env.BIN }}" zombienet -p native test \ + ./tests/11-turn.zndsl \ + ${{ env.ZOMBIE_DIR }}/zombie.json + - name: Run Runtime Upgrade Tests if: ${{ github.event.inputs.test_file == 'all' || github.event.inputs.test_file == '13-runtime-upgrade.zndsl' || github.event.inputs.test_file == '' }} run: | diff --git a/CLAUDE.md b/CLAUDE.md index 7741c3d..7f145b0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -90,6 +90,7 @@ prints what resolves. - **GitHub auth**: Required for fetch - `gh auth login` or set `GITHUB_TOKEN` - **Node.js 24+**: Required for tests and startup helper scripts +- **eturnal (macOS only)**: `brew tap processone/eturnal https://github.com/processone/eturnal && brew install processone/eturnal/eturnal`. Linux gets it from `make fetch` (see `docs/TURN.md`) **Mac Users (Apple Silicon)**: Disable IPv6 due to [known bug](https://github.com/paritytech/polkadot-sdk/issues/8918): @@ -133,7 +134,8 @@ docs/ ├── FORK.md # Fork mode: spawn from production state ├── POLKADOT-FORK.md # Runbook: fork Polkadot with the fellowship runtimes on a dedicated machine ├── PROFILES.md # dev/prod profiles: funded accounts, sudo, signing keys -└── RUNTIME-UPGRADE.md # Upgrade the runtime of a running chain +├── RUNTIME-UPGRADE.md # Upgrade the runtime of a running chain +└── TURN.md # TURN/STUN relay (eturnal) behind DUB's turn-api bin/ # Output directory (gitignored except ppn.mjs) ├── polkadot # Relay chain binary (+ execute/prepare PVF workers) @@ -154,6 +156,7 @@ scripts/ # Shell launchers zombienet execs; the logic lives ├── ipfs-daemon.sh # IPFS daemon launcher ├── ipfs-swarm.sh # IPFS swarm connection ├── dashboard.sh # Launcher over `ppn service dashboard` +├── turn.sh # Launcher over `ppn service turn`: eturnal TURN/STUN relay ├── storage-provider-node.sh # Launcher over `ppn service storage-provider-node` ├── set-dispatcher-address.sh # Launcher over `ppn service set-dispatcher-address` ├── pin-design-families.sh # Pins design families to IPFS @@ -185,6 +188,7 @@ tests/ ├── 08-dub.zndsl # device-uniqueness-backend smoke tests ├── 09-dub-registration.zndsl # dub registration flow ├── 10-network-suffix.zndsl # Genesis set the product-context namespace on both chains +├── 11-turn.zndsl # TURN relay: STUN, DUB credentials, authenticated Allocate ├── 13-runtime-upgrade.zndsl # Live runtime upgrade on Asset Hub └── scripts/ # Custom TS test scripts the .zndsl suites call @@ -224,6 +228,7 @@ zombienet-configs/ | Web3 Storage Provider | `http://127.0.0.1:3333` | HTTP API connected to Web3 Storage Chain | | Device Uniqueness Backend | `http://127.0.0.1:8092` | Auth, usernames, tickets, TURN, notify (one origin, all services) | | DUB API Reference | `http://127.0.0.1:8092/docs` | Generated OpenAPI reference | +| TURN/STUN (eturnal) | `stun:127.0.0.1:3478`, `turn:127.0.0.1:3478` | Relay for p2p clients; credentials from DUB `/api/v1/turn/issue` (see `docs/TURN.md`) | ## Elastic Scaling (Asset Hub) diff --git a/Makefile b/Makefile index b05c9a9..437b508 100644 --- a/Makefile +++ b/Makefile @@ -309,6 +309,12 @@ doctor: else \ echo "✗ GitHub auth (run: gh auth login)"; \ fi + @if [ "$$(uname)" = Darwin ]; then \ + brew list --versions eturnal 2>/dev/null | sed 's/^/✓ /' | grep . \ + || echo "✗ eturnal (brew tap processone/eturnal https://github.com/processone/eturnal && brew install processone/eturnal/eturnal)"; \ + else \ + [ -x "$(CURDIR)/bin/eturnal/bin/eturnalctl" ] && echo "✓ eturnal" || echo "✗ eturnal (run: make fetch)"; \ + fi @command -v docker >/dev/null && echo "✓ docker" || echo "- docker (optional, for DOCKER=1 mode)" @echo "" diff --git a/config/ports.env b/config/ports.env index 1556085..37ef59c 100644 --- a/config/ports.env +++ b/config/ports.env @@ -93,6 +93,15 @@ DUB_POSTGRES_PORT=5433 # table it compiles in, which is why there is one port here and not one per service — and # why PPN no longer runs a gateway of its own to reassemble them. DUB_PORT=8092 +# TURN/STUN (eturnal, docs/TURN.md). TURN_PORT takes UDP and TCP straight from clients on +# P2P_LISTEN_IP: STUN has to see the client's own address, so it cannot sit behind nginx. +# nginx terminates TLS on TURN_TLS_PORT and forwards to TURN_PROXY_PORT with the PROXY +# protocol. The relay range must be open in the firewall alongside TURN_PORT. +TURN_PORT=3478 +TURN_TLS_PORT=5349 +TURN_PROXY_PORT=3479 +TURN_RELAY_MIN_PORT=49152 +TURN_RELAY_MAX_PORT=49407 # The dashboard: the network's own status UI and API (docs/DASHBOARD.md) DASHBOARD_PORT=8090 diff --git a/config/versions.env b/config/versions.env index 40aa941..bc94174 100644 --- a/config/versions.env +++ b/config/versions.env @@ -97,6 +97,9 @@ DUB_TAG="${DUB_TAG:-v0.5.0}" # version device-uniqueness-backend's own compose file pins. POSTGRES_REPO="theseus-rs/postgresql-binaries" POSTGRES_VERSION="16.14.0" +# TURN/STUN relay for DUB's turn-api (docs/TURN.md). eturnal.net publishes self-contained +# Linux tarballs; macOS has no binary, so there it is the processone Homebrew tap, same version. +ETURNAL_VERSION="1.12.3" # ============================================================================= # DotNS - Naming Service Genesis Contracts (deployed on Asset Hub from genesis) diff --git a/docs/DEVICE-UNIQUENESS-BACKEND.md b/docs/DEVICE-UNIQUENESS-BACKEND.md index add0221..7567363 100644 --- a/docs/DEVICE-UNIQUENESS-BACKEND.md +++ b/docs/DEVICE-UNIQUENESS-BACKEND.md @@ -48,14 +48,14 @@ The five surfaces behind the one port, all live: | `/api/v1/usernames` (POST, `/available`, `/payment-status`), `/api/v1/attester`, `/.well-known/jwks.json` | device-attestation-api | challenge → attestation → JWT → registration | | `/api/v1/usernames/search` (GET) | username-indexer | finalized-chain projection + prefix search | | `/api/v1/invitation-ticket/claim` | invite-tickets-api | | -| `/api/v1/turn/issue` | turn-api | short-lived TURN credentials; needs `TURN_SECRET` (base64) + `TURN_REALM` | +| `/api/v1/turn/issue` | turn-api | short-lived credentials for the eturnal relay, with `ICE_SERVERS`; see [TURN.md](TURN.md) | | `/api/v1/notify` | notify-relay | pushes report provider failure until APNs/FCM credentials are set | | `/readyz`, `/livez`, `/healthcheck`, `/docs` | all-in-one | `/readyz` aggregates every merged service's dependencies | -Two of these work without operator credentials only in the local sense: `turn-api` mints -credentials against a dev secret nothing outside this machine honours, and `notify-relay` -accepts requests and reports a provider failure. They are up, and they answer — that is -enough for a client to develop against, and not enough to deliver a push. +`notify-relay` works without operator credentials only in the local sense: it accepts +requests and reports a provider failure. That is enough for a client to develop against, and +not enough to deliver a push. `turn-api` is backed by a real relay, eturnal (see +[TURN.md](TURN.md)). ### Two signing accounts, not one @@ -259,8 +259,9 @@ TURN_REALM=previewnet.local And the secrets, in `scripts/dub/service.sh` rather than the TOML (see `docs/PROFILES.md`): `CHAIN_WRITER_SIGNER_SURI`, `INVITER_SIGNER_SURI`, -`JWT_ED25519_SECRET` with `JWT_ED25519_PUBLIC_KEY` derived from it, and `TURN_SECRET` — -which must be **base64**; hex is rejected with `invalid base64 encoding`. +`JWT_ED25519_SECRET` with `JWT_ED25519_PUBLIC_KEY` derived from it, and `TURN_SECRET` with +`ICE_SERVERS`, which `ppn service dub-turn-env` resolves together with the eturnal config +(see [TURN.md](TURN.md)). The rest stay dormant under the dev defaults: `PAYMENT_MASTER_ACCOUNT` / `PAYMENT_AMOUNT_PLANCK` are only read when `PAYMENT_LANE_ENABLED=true` (default false, @@ -415,8 +416,8 @@ namespaced URL. Scratch databases for the suites would be additional. a deliberate edit. - The repo is private, so `ppn fetch` needs a token that can read it. Nothing else PPN fetches has that requirement any more. -- `turn-api` and `notify-relay` are up but not usable for real delivery: no coturn, no - APNs/FCM credentials. Requests are accepted and answered; a push does not arrive. +- `notify-relay` is up but not usable for real delivery: no APNs/FCM credentials. Requests + are accepted and answered; a push does not arrive. - **A killed cluster leaks a shared-memory segment, and macOS allows 32.** Postgres takes a SysV segment at startup and releases it on shutdown, so anything that SIGKILLs it — a crash, or a `ppn kill` before the SIGTERM it now sends first — leaves one behind. Once diff --git a/docs/TURN.md b/docs/TURN.md new file mode 100644 index 0000000..744b213 --- /dev/null +++ b/docs/TURN.md @@ -0,0 +1,77 @@ +# TURN / STUN + +Peer-to-peer clients (WebRTC) need a relay when two peers cannot reach each other directly. +PPN runs [eturnal](https://eturnal.net) for that, next to the device-uniqueness backend (DUB), +whose `turn-api` mints the short-lived credentials the relay accepts. + +## How a client uses it + +1. Sign in to DUB and hold its JWT. +2. `POST /dub/api/v1/turn/issue` with `Authorization: Bearer `. The response is + `{ servers, username, password, ttl }`. +3. Pass `servers` as the ICE server URLs, with `username` and `password` as the credential. + +Clients never configure a password themselves. The dashboard lists the same servers under +**TURN / STUN**, and so does `/api/network` under `ice`. + +| URL | Where it goes | +| --- | --- | +| `stun::3478` | eturnal, UDP, direct | +| `turn::3478?transport=udp` | eturnal, UDP, direct | +| `turn::3478?transport=tcp` | eturnal, TCP, direct | +| `turns::5349?transport=tcp` | nginx (TLS) → eturnal on `127.0.0.1:3479`, PROXY protocol. Only listed behind https | + +`` is the hostname of `PPN_PUBLIC_URL`, or `127.0.0.1` locally. Ports come from +`config/ports.env` (`TURN_*`). + +STUN and plain TURN cannot go through nginx: STUN answers with the address it sees the +request come from, and behind a proxy that address is the proxy's own. nginx only handles TLS, +and passes the client's address on with the PROXY protocol. + +## Installing eturnal + +- **Linux**: `ppn fetch` downloads the self-contained release from eturnal.net + (`ETURNAL_VERSION` in `config/versions.env`) into `bin/eturnal/`. +- **macOS**: eturnal.net publishes no macOS build. Use the processone tap: + + ```sh + brew tap processone/eturnal https://github.com/processone/eturnal + brew install processone/eturnal/eturnal + ``` + + `ppn fetch` and `make doctor` fail until it is installed. + +## The shared secret + +`TURN_SECRET` is base64, which is what turn-api reads. turn-api signs with the decoded +bytes, while eturnal takes its secret as a string, so the decoded value must be printable ASCII +or the two derive different passwords. Both `ppn service turn` and `dub-api` read the secret +through `turnSecret()` in `packages/network-config/src/turn.ts`, and both refuse a value that +breaks this rule. + +Locally the secret defaults to a public value. A deployment (one that sets `PPN_SECRETS_FILE`) +must set its own there: + +```sh +TURN_SECRET=$(openssl rand -hex 24 | tr -d '\n' | base64) +``` + +The secret never goes into `eturnal.yml`. eturnal reads it from `ETURNAL_SECRET`. + +## Deploying + +- `P2P_LISTEN_IP` is the address eturnal binds and relays from, just as it is for the + webrtc-direct collators. On a server it must be the public IP that `PPN_PUBLIC_URL`'s host + resolves to. +- Open `TURN_PORT` (UDP and TCP), `TURN_TLS_PORT` (TCP) and + `TURN_RELAY_MIN_PORT`–`TURN_RELAY_MAX_PORT` (UDP) in the firewall. +- `ppn nginx-conf` emits the TLS stream block at a ` # {{GENERATED_STREAMS}}` marker. That + marker belongs inside the template's `stream {}` block, which must set `ssl_certificate` and + `ssl_certificate_key` for the domain at that level. When the network runs the relay, the + marker is required. + +Under `DOCKER=1` the relay runs but cannot be reached from the host: it binds the +container's loopback, and the TURN ports are not published. + +A descriptor can switch the relay off with `"services": { "turn": false }`. DUB still runs, +and its credentials then point at nothing. diff --git a/packages/cli/src/commands/dashboard.ts b/packages/cli/src/commands/dashboard.ts index 8745606..c30738e 100644 --- a/packages/cli/src/commands/dashboard.ts +++ b/packages/cli/src/commands/dashboard.ts @@ -9,6 +9,8 @@ // Never in the production data path: on servers nginx routes the chains directly and only // `/` is proxied here, so this process crashing costs the dashboard, not the network. +import crypto from 'node:crypto'; +import dgram from 'node:dgram'; import fs from 'node:fs'; import http from 'node:http'; import net from 'node:net'; @@ -37,6 +39,27 @@ interface HealthEntry { checkedAt: string; } +/** A STUN Binding request (RFC 5389); resolves on a matching success response. */ +function stunBinding(host: string, port: number): Promise { + const sock = dgram.createSocket(host.includes(':') ? 'udp6' : 'udp4'); + const txid = crypto.randomBytes(12); + const request = Buffer.concat([Buffer.from([0x00, 0x01, 0x00, 0x00, 0x21, 0x12, 0xa4, 0x42]), txid]); + return new Promise((resolve, reject) => { + const done = (err?: Error) => { + clearTimeout(timer); + sock.close(); + if (err) reject(err); + else resolve(); + }; + const timer = setTimeout(() => done(new Error('no STUN response')), 3000); + sock.on('message', (msg) => { + if (msg.length >= 20 && msg.readUInt16BE(0) === 0x0101 && msg.subarray(8, 20).equals(txid)) done(); + }); + sock.on('error', (err) => done(err)); + sock.send(request, port, host); + }); +} + /** * One JSON-RPC batch per chain per tick: head, runtime version, client version. A batch * rather than three requests because it is one round trip and one connection, and the probe @@ -189,8 +212,23 @@ export async function dashboard(ctx: ServiceContext): Promise { }) ); }; - probe(); - const probeTimer = setInterval(probe, 5000); + // The relay listens on P2P_LISTEN_IP, which is a public address on a server. + const ice = model.ice; + const stunHost = process.env.P2P_LISTEN_IP || ctx.ports.P2P_LISTEN_IP || '127.0.0.1'; + const stunPort = ice?.servers.find((s) => s.id === 'stun')?.port; + const probeAll = async () => { + await probe(); + if (!ice || !stunPort) return; + const at = new Date().toISOString(); + try { + await stunBinding(stunHost, stunPort); + health.set(ice.id, { id: ice.id, status: 'ok', checkedAt: at }); + } catch (err) { + health.set(ice.id, { id: ice.id, status: 'down', error: (err as Error).message, checkedAt: at }); + } + }; + probeAll(); + const probeTimer = setInterval(probeAll, 5000); probeTimer.unref(); // ---- logs: whitelist is what exists on disk as DATA_DIR//.log ---- @@ -281,6 +319,7 @@ export async function dashboard(ctx: ServiceContext): Promise { /-validator\d*$|-collator\d*$|^Collator-/.test(n) || WELL_KNOWN.has(n) || n === 'doppelganger'; const serviceIds = new Set([ ...model.services.map((s) => s.id), + ...(model.ice ? [model.ice.id] : []), 'dashboard', 'ipfs-swarm', 'dub-api', 'dub-postgres', 'device-attestation-chain-writer', 'registration-queue', 'invite-tickets-pool', ]); diff --git a/packages/cli/src/commands/fetch.ts b/packages/cli/src/commands/fetch.ts index 8f5f8c2..7ba946f 100644 --- a/packages/cli/src/commands/fetch.ts +++ b/packages/cli/src/commands/fetch.ts @@ -22,6 +22,7 @@ import path from 'node:path'; import { execFileSync } from 'node:child_process'; import { loadCurrentNetwork, + runsTurnRelay, networkBinaries, networkRuntimes, readEnvFile, @@ -40,6 +41,7 @@ import { } from '../lib/github.js'; import { extractTarGz, extractZip, findFile, withTempDir } from '../lib/archive.js'; import { writeProvenance, readProvenance, reusable, type StampInput } from '../lib/provenance.js'; +import { ETURNAL_BREW_INSTALL, brewEturnalVersion, eturnalCtl, eturnalDist } from '../lib/eturnal.js'; /** * A `file:` pin is a path, not a repo: `--binary polkadot=file:/build/polkadot`, or @@ -483,6 +485,40 @@ export async function run(args: string[], opts: FetchOptions = {}): Promise { @@ -558,6 +594,7 @@ function whatIsMissing( if (gone(sharedDest, 'zombie-cli')) out.push('zombie-cli'); // One binary now, so its absence is one entry rather than four. if (gone(sharedDest, IDENTITY_BINARY)) out.push(IDENTITY_BINARY); + if (runsTurnRelay(net) && !eturnalCtl(sharedDest)) out.push('eturnal'); if (!fs.existsSync(path.join(WS, 'design-families'))) out.push('design-families/'); return out; } diff --git a/packages/cli/src/commands/nginx-conf.ts b/packages/cli/src/commands/nginx-conf.ts index 2fa8a6f..c967c0b 100644 --- a/packages/cli/src/commands/nginx-conf.ts +++ b/packages/cli/src/commands/nginx-conf.ts @@ -1,16 +1,18 @@ // `ppn nginx-conf