diff --git a/AGENTS.md b/AGENTS.md index 843abea..6335f0f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -142,7 +142,7 @@ test-scripts` fails while the committed file differs from a fresh render, and `m lint-alloy` validates and runs it. **Pinned upstream releases.** One decdn/decdn release line (`decdn_node_version`, today -`0.1`) and one decdn/sponsord release line (`sponsord_version` = `sponsord_onramp_version`, +`0.2`) and one decdn/sponsord release line (`sponsord_version` = `sponsord_onramp_version`, today `0.1`; decdn/sponsord tags its whole workspace `vX.Y.Z`, there are no per-crate tags) are pinned across every path. A line, `X.Y`, is what the role defaults, the onramp's installer pin (`sponsord_onramp_decdn_release`, `vX.Y`) and the chart's @@ -255,7 +255,13 @@ the pinned `iroh-relay` (re-capture it when bumping `iroh_relay_version`). the installed binary does not know is a startup crash-loop. Two guards: the role runs `decdn config validate` against the real binary after templating, and the `molecule/schema` scenario checks the rendered key set against a committed inventory of - upstream field names. Re-sync both when bumping the pinned decdn version. + upstream field names (`gen-schema-keys.py`, which refuses a config struct its + `STRUCT_PATHS` does not map, so a section upstream adds fails the regeneration instead + of dropping out). Re-sync both when bumping the pinned decdn version. `[self_update]` + (decdn v0.2) is CLI-only, the daemon ignores it: the role and `decdn-compose init` + render `check = false`, and `decdn-compose cli` sets `DECDN_SELF_UPDATE_CHECK=0`, so a + managed host never hears "run `decdn self-update`", which would replace the CLI outside + the signed-release install and fail the role's version check. - **`ansible/roles/grafana_alloy`** — opt-in Grafana Cloud observability for that node, one mirrored flag (`decdn_grafana_cloud_enabled`, `false` by default) driving a diff --git a/ansible/inventory/group_vars/decdn_nodes.yml b/ansible/inventory/group_vars/decdn_nodes.yml index c77132b..67d0aad 100644 --- a/ansible/inventory/group_vars/decdn_nodes.yml +++ b/ansible/inventory/group_vars/decdn_nodes.yml @@ -26,9 +26,10 @@ # --- Cache sizing ----------------------------------------------------------------- # Size to each host's disk and to the largest object you expect to serve; see # https://github.com/decdn/devops/blob/main/docs/requirements.md for the disk formula. -# A node rejects any blob larger than decdn_max_blob_size_mb with BlobTooLarge, and -# decdn_cache_size_mb must be at least that. Override per host in host_vars when -# disks differ. +# decdn_max_blob_size_mb caps the bytes held for one blob: an own-origin blob above it +# is refused with BlobTooLarge, and a pull-through serves ranges of a larger one while +# the bytes held fit. decdn_cache_size_mb must be at least that. Override per host in +# host_vars when disks differ. # decdn_max_blob_size_mb: 1024 # role default # decdn_cache_size_mb: 10240 # role default (10 GiB) diff --git a/ansible/inventory/group_vars/sponsord_onramp_hosts.yml b/ansible/inventory/group_vars/sponsord_onramp_hosts.yml index 7b1d476..8ab8a89 100644 --- a/ansible/inventory/group_vars/sponsord_onramp_hosts.yml +++ b/ansible/inventory/group_vars/sponsord_onramp_hosts.yml @@ -36,6 +36,6 @@ # role's sponsord_onramp_decdn_release line (decdn/decdn, the decdn_node role's default # line; a host's decdn_node_version does not move it), with the SHA-256 of its # verified SHA256SUMS, on each deploy. To serve one exact release, set its tag and digest: -# sponsord_onramp_decdn_release: v0.1.0 +# sponsord_onramp_decdn_release: v0.2.0 # sponsord_onramp_decdn_sums_sha256: "…" # CapacityBond and SlashJudge come from sponsord_network (group_vars/sponsord_hosts.yml). diff --git a/ansible/inventory/host_vars/decdn-node-1/main.yml b/ansible/inventory/host_vars/decdn-node-1/main.yml index 4ec5209..fd155bc 100644 --- a/ansible/inventory/host_vars/decdn-node-1/main.yml +++ b/ansible/inventory/host_vars/decdn-node-1/main.yml @@ -24,13 +24,13 @@ # line the role pins ("X.Y": its newest release, installed by each deploy); set an # exact one only to hold a node on it. decdn_node_install_method: release -# decdn_node_version: "0.1.0" +# decdn_node_version: "0.2.0" # # Alternative: build on the node from a git ref (any tag, branch or SHA), as an # unprivileged build user with a pinned toolchain (decdn_node_version is not checked # in this mode): # decdn_node_install_method: source -# decdn_node_source_ref: v0.1.0 +# decdn_node_source_ref: v0.2.0 # Alternative: copy binaries built on this control machine # (`cargo build --release -p decdn-node -p decdn-cli` in a decdn/decdn checkout): # decdn_node_install_method: manual diff --git a/ansible/inventory/host_vars/origin-1/main.yml b/ansible/inventory/host_vars/origin-1/main.yml index 1b0b93e..0fe88bd 100644 --- a/ansible/inventory/host_vars/origin-1/main.yml +++ b/ansible/inventory/host_vars/origin-1/main.yml @@ -26,13 +26,13 @@ # line the role pins ("X.Y": its newest release, installed by each deploy); set an # exact one only to hold a node on it. decdn_node_install_method: release -# decdn_node_version: "0.1.0" +# decdn_node_version: "0.2.0" # # Alternative: build on the node from a git ref (any tag, branch or SHA), as an # unprivileged build user with a pinned toolchain (decdn_node_version is not checked # in this mode): # decdn_node_install_method: source -# decdn_node_source_ref: v0.1.0 +# decdn_node_source_ref: v0.2.0 # Alternative: copy binaries built on this control machine # (`cargo build --release -p decdn-node -p decdn-cli` in a decdn/decdn checkout): # decdn_node_install_method: manual diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml index 988f237..945549f 100644 --- a/ansible/molecule/default/verify.yml +++ b/ansible/molecule/default/verify.yml @@ -230,6 +230,10 @@ ("load_shed", shed.get("per_client_serve_cap") == 16), ("receipts", d.get("receipts", {}).get("retained_files") == 8), ("content.denied_hashes", d.get("content", {}).get("denied_hashes") == ["c" * 64]), + # The role default: the CLI's self-update notice is off on a managed + # host (a bare TOML bool, not a string). + ("self_update.check (role default false must emit)", + d.get("self_update", {}).get("check") is False), # Rate-limit tables render from a fixed key list: the keys converge set # must be present with the right types, and the ones it did NOT set must # be ABSENT (rendering a default would silently pin it). diff --git a/ansible/molecule/schema/files/gen-schema-keys.py b/ansible/molecule/schema/files/gen-schema-keys.py index 3681cc9..6161599 100755 --- a/ansible/molecule/schema/files/gen-schema-keys.py +++ b/ansible/molecule/schema/files/gen-schema-keys.py @@ -29,6 +29,8 @@ "FileConfig": None, "IdentityConfig": "identity", "ClientConfig": "client", + # CLI-only (the daemon ignores it): the `decdn` self-update notice knob. + "SelfUpdateConfig": "self_update", "NetworkConfig": "network", "DiscoveryConfig": "network.discovery", # Keyed by NodeId, so the concrete hop is a wildcard. @@ -64,24 +66,31 @@ ORIGIN_ENUM_FIELDS = ["kind", "url", "decompress", "path"] CREDENTIAL_FIELDS = ["source", "profile"] -FIELD_RE = re.compile(r"^\s+pub\s+([a-z_0-9]+)\s*:") -TYPE_RE = re.compile(r"^pub\s+(?:struct|enum)\s+([A-Za-z0-9_]+)") +FIELD_RE = re.compile(r"^\s+pub\s+([a-z_0-9]+)\s*:(.*)$") +TYPE_RE = re.compile(r"^pub\s+(struct|enum)\s+([A-Za-z0-9_]+)") -def fields_by_struct(paths): - """Map every `pub struct`/`pub enum` name to its own `pub field:` names.""" - out, current = {}, None +def parse(paths): + """Map every `pub struct`/`pub enum` name to its own `pub field:` names, to the + type names those fields mention (`Option` -> SelfUpdateConfig), + and collect which of the names are structs (an enum is a leaf value or one of the + tagged origin shapes named above, never a section of its own).""" + fields, refs, structs, current = {}, {}, set(), None for path in paths: for line in Path(path).read_text().splitlines(): type_match = TYPE_RE.match(line) if type_match: - current = type_match.group(1) - out.setdefault(current, []) + current = type_match.group(2) + fields.setdefault(current, []) + refs.setdefault(current, set()) + if type_match.group(1) == "struct": + structs.add(current) continue field_match = FIELD_RE.match(line) if field_match and current: - out[current].append(field_match.group(1)) - return out + fields[current].append(field_match.group(1)) + refs[current].update(re.findall(r"\b[A-Z][A-Za-z0-9_]*\b", field_match.group(2))) + return fields, refs, structs def main(): @@ -97,7 +106,7 @@ def main(): if missing: sys.exit(f"not a decdn checkout -- missing: {', '.join(missing)}") - by_struct = fields_by_struct(sources) + by_struct, refs, structs = parse(sources) unknown = sorted( name for name in STRUCT_PATHS if name not in by_struct @@ -108,6 +117,26 @@ def main(): f"(renamed or removed upstream?): {', '.join(unknown)}" ) + # The inverse: a section upstream ADDED would otherwise drop out of the + # inventory in silence (v0.2's [self_update] did, until this check). Every + # struct a mapped struct (FileConfig included) names in a field type must be + # mapped itself, or be the S3 origin shape the constants above spell out. + handled = set(STRUCT_PATHS) | {"S3OriginConfig"} + unmapped = sorted( + { + ref + for struct in STRUCT_PATHS + for ref in refs[struct] + if ref in structs and ref not in handled + } + ) + if unmapped: + sys.exit( + "upstream config structs that no STRUCT_PATHS entry maps (a new section?): " + f"{', '.join(unmapped)}. Map each to its TOML path, then decide whether " + "node.toml.j2 (and the chart's values.config) should render it." + ) + paths = set() for struct, prefix in STRUCT_PATHS.items(): if prefix is None: diff --git a/ansible/molecule/schema/files/schema-keys.txt b/ansible/molecule/schema/files/schema-keys.txt index d13423b..beb40cf 100644 --- a/ansible/molecule/schema/files/schema-keys.txt +++ b/ansible/molecule/schema/files/schema-keys.txt @@ -13,7 +13,7 @@ # a leaf-name inventory cannot tell the two apart. See gen-schema-keys.py for # the struct -> path mapping and molecule/schema/README.md for why this exists. # -# 156 paths. +# 157 paths. blockchain.buyer_max_approve blockchain.buyer_working_deposit_micro_usdc blockchain.capacity_bond_address @@ -170,3 +170,4 @@ security.max_concurrent_handlers security.max_tracked_sources security.per_source_burst security.per_source_rate_per_sec +self_update.check diff --git a/ansible/molecule/schema/verify.yml b/ansible/molecule/schema/verify.yml index 3d609e5..92edafa 100644 --- a/ansible/molecule/schema/verify.yml +++ b/ansible/molecule/schema/verify.yml @@ -124,7 +124,8 @@ payment probe.rate_limit receipts - security' + security + self_update' if [ "$tables" != "$(echo "$expected" | sed 's/^ *//')" ]; then echo "rendered table set changed. got:" >&2; echo "$tables" >&2 echo "expected:" >&2; echo "$expected" | sed 's/^ *//' >&2 diff --git a/ansible/roles/decdn_node/README.md b/ansible/roles/decdn_node/README.md index 43bd291..8da036f 100644 --- a/ansible/roles/decdn_node/README.md +++ b/ansible/roles/decdn_node/README.md @@ -315,7 +315,9 @@ asserts miss. See `defaults/main.yml` for every knob's upstream default, unit an - **Cache sizing + eviction** — `decdn_cache_size_mb`, `decdn_disk_headroom_mb` (free disk the eviction driver keeps unused on the `cache_dir` volume; daemon default `8192`, `0` opts out of the disk clamp), `decdn_max_blob_size_mb` - (defaults to `cache_size_mb` upstream; must be `<=` it), `decdn_max_rate_per_mb` + (the bytes held for one blob: an own-origin blob above it is refused, a pull-through + serves ranges of a larger one while the bytes held fit; daemon default + `min(51200, cache_size_mb)`; must be `<=` it), `decdn_max_rate_per_mb` (buyer-side rate ceiling), `decdn_gc_interval_sec`, `decdn_fs_rescan_interval_sec`, `decdn_max_probe_holds`, `decdn_stake_lane_reserved_holds`, `decdn_pinned_hashes` (64-char **lowercase-hex** BLAKE3), `decdn_cache_user_agent`, the @@ -366,6 +368,12 @@ asserts miss. See `defaults/main.yml` for every knob's upstream default, unit an `decdn_client_region_allowlist` (a `[client]` knob) narrows which regions this node discovers/probes **as a client** to a list of ISO 3166-1 alpha-2 codes; `[]` ⇒ no filter. +- **`decdn` CLI self-update notice** — `decdn_self_update_check` (`[self_update] + check`, decdn v0.2; CLI-only, the daemon ignores it) defaults to `false`: the + once-a-day "a newer release exists, run `decdn self-update`" line the CLI prints + after a command run at a terminal is wrong advice on a host this role manages (see + [On-chain onboarding](#on-chain-onboarding)). `true` turns it on; `""`/`null` + omits the section, which an exact `decdn_node_version` below `0.2` needs. - **Abuse limits + load shedding** — the `decdn_security_*` family, `decdn_load_shed_*` (`policy` is `resource-pressure`|`always-admit`; the low-water serve mark must be `<=` the high), and the `decdn_dht_rate_limit` / @@ -560,6 +568,13 @@ sudo systemd-run --pty --wait --collect -p User=decdn \ --keystore-password-file /etc/decdn/keystore.password ``` +Never run `decdn self-update` (decdn v0.2) on a host this role manages. Under plain +`sudo` it replaces `/usr/local/bin/decdn` outside the role's signed-release install, +and the next deploy fails its version check instead of repairing it. The role takes +the pinned line's newest release on every run; to move, change `decdn_node_version`. +The CLI's daily notice suggesting it is off in the rendered `node.toml` +(`decdn_self_update_check`), which these recipes pass as `--config`. + Exiting is the reverse, in order: `decdn node deregister` (leaves the active set and clears the declared tier — the bond stays deposited and **fully slashable**), then `decdn node unbond --all` (starts the unbonding window; the node is INACTIVE @@ -747,7 +762,9 @@ The steps: `ansible/molecule/schema/files/gen-schema-keys.py > ansible/molecule/schema/files/schema-keys.txt` (run on a checkout of the tag). If the schema keys changed, re-sync `templates/node.toml.j2` (and the chart's - `values.config`). + `values.config`). The generator refuses a config struct its `STRUCT_PATHS` does + not map (a section upstream added): map it to its TOML path, then decide whether + the role renders it (v0.2's `[self_update]`, CLI-only, is rendered off). 3. Compose's onramp installer pin: `ONRAMP_DECDN_RELEASE` (`v`) and `ONRAMP_DECDN_SUMS_SHA256` (`sha256sum SHA256SUMS` once `SHA256SUMS.asc` verifies) in `compose/sponsord-onramp.env.example`. The role reads both from the @@ -767,6 +784,13 @@ The steps: for a new line AGENTS.md's "Pinned upstream releases". Other docs name the pin, never its number. The changelog entry is the PR title (`feat`, `fix` or `chore`, not `ci` or `build`, which the changelogs leave out; RELEASING.md). +8. The monitoring assets, by hand (nothing in CI checks the node's metric names): + diff upstream's `adr/appendix-observability.md` and the `INBOUND_FAILURE_REASONS` + list in `crates/node/src/metrics.rs` between the two tags, fix any renamed or + removed `decdn_*` name in `monitoring/decdn-node/`, add a new inbound failure + reason to both "Unattributed stream failures" panels, and chart or alert on new + series worth it (`.claude/skills/grafana-dashboards/SKILL.md`). Republish the + Grafana Cloud copies after the merge. `make test-scripts` fails while the role, the onramp's decdn line, the chart's `appVersion` and its `artifacthub.io/images` image or AGENTS.md's pinned line diff --git a/ansible/roles/decdn_node/defaults/main.yml b/ansible/roles/decdn_node/defaults/main.yml index 14cfa96..76582bc 100644 --- a/ansible/roles/decdn_node/defaults/main.yml +++ b/ansible/roles/decdn_node/defaults/main.yml @@ -64,7 +64,7 @@ decdn_rustup_sha256: # default is the line this repo is synced against; a new line also moves the # generated mirrors, the onramp's decdn pin and the chart's appVersion (README.md, # "Bumping the decdn release"). -decdn_node_version: "0.1" +decdn_node_version: "0.2" # Release target triple. Derived from the host's gathered architecture (x86_64 or # aarch64, see vars/main/arch.yml), so arm64 hosts (Graviton, Ampere, Hetzner CAX) # get the aarch64 tarball with no inventory change. Needs facts: a play with @@ -224,12 +224,16 @@ decdn_cache_size_mb: 10240 # 10 GB # against ANY process, not just this cache (#1930). "" => daemon dflt 8192 (8 GiB); # 0 opts out of the disk clamp so only cache_size_mb binds. decdn_disk_headroom_mb: "" -# NOTE: this is a deliberate OVERRIDE, not the daemon default. Upstream defaults -# max_blob_size_mb to cache_size_mb; 1 GB caps any single blob at a tenth of the -# cache so one large object cannot evict everything else. Must be <= cache_size_mb. -# DO NOT set 0: despite what upstream's own `config init` template says, there is -# no zero special-case in the cache engine — the admit gate is a plain -# `advertised > max_blob_bytes`, so 0 rejects every blob with a size hint. +# Cap on the bytes the node HOLDS for one blob (decdn v0.2): a blob from this node's +# own origin above it is refused (BlobTooLarge); a pull-through serves a range at any +# offset of a larger blob while the bytes held for it fit, and stops the stream once +# they do not (decdn_serve_stream_midstream_blob_too_large_total). A deliberate +# OVERRIDE of the daemon default, min(51200, cache_size_mb): 1 GB caps one blob at a +# tenth of the default cache so one large object cannot evict everything else. Must +# be <= cache_size_mb. DO NOT set 0: despite what upstream's own `config init` +# template says, there is no zero special-case in the cache engine — the admit gate +# is a plain `advertised > max_blob_bytes`, so 0 refuses every own-origin blob with a +# size hint. decdn_max_blob_size_mb: 1024 # 1 GB # Buyer-side ABSOLUTE per-MB rate ceiling for paid pulls (#1375) — what this node # will accept a provider to quote on a cache-miss pull, on top of the always-applied @@ -387,6 +391,17 @@ decdn_content_denied_origins: [] # 0x publisher addresses (checksum-agnostic; # node considered). ISO 3166-1 alpha-2 codes, e.g. ["US", "DE"]. decdn_client_region_allowlist: [] +# --- `decdn` CLI self-update notice ([self_update]) ---------------------------- +# Since decdn v0.2 the `decdn` CLI checks once a day, after a command run at a +# terminal, for a newer signed release and prints "run `decdn self-update`". The +# daemon ignores the section. Off here: this role owns the binaries (the newest +# release of the pinned line, verified against the release's signed SHA256SUMS), and +# a `decdn self-update` run as root would replace /usr/local/bin/decdn behind its +# back and fail the next deploy's version check. Move decdn_node_version instead. +# true turns the notice back on; "" or null omits the section (needed for an exact +# decdn_node_version below 0.2, whose binaries refuse the key). +decdn_self_update_check: false + # --- Readiness probe (advisory) ----------------------------------------------- # After start, the role probes http://127.0.0.1:/metrics as a # best-effort readiness signal. A probe TIMEOUT only WARNS (it does NOT fail the diff --git a/ansible/roles/decdn_node/tasks/main.yml b/ansible/roles/decdn_node/tasks/main.yml index e73c206..351ee58 100644 --- a/ansible/roles/decdn_node/tasks/main.yml +++ b/ansible/roles/decdn_node/tasks/main.yml @@ -753,6 +753,7 @@ - {name: decdn_node_to_node_pull_through_enabled, value: "{{ decdn_node_to_node_pull_through_enabled }}"} - {name: decdn_relay_foreign_namespaces, value: "{{ decdn_relay_foreign_namespaces }}"} - {name: decdn_circuit_breaker_enabled, value: "{{ decdn_circuit_breaker_enabled }}"} + - {name: decdn_self_update_check, value: "{{ decdn_self_update_check }}"} loop_control: label: "{{ item.name }}" @@ -978,7 +979,8 @@ # Origin-probe memo TTLs must be ordered negative <= fault <= positive. - _eff_probe_neg | int <= _eff_probe_fault | int - _eff_probe_fault | int <= _eff_probe_pos | int - # A single blob may not exceed the whole cache (0 = unlimited, so exempt). + # The bytes held for one blob may not exceed the whole cache (0 is exempt + # here; defaults/main.yml says why never to set it). - _eff_max_blob | int == 0 or (_eff_max_blob | int <= decdn_cache_size_mb | int) # Load-shed low-water mark may not exceed the high-water mark. - _eff_shed_low | int <= _eff_shed_high | int @@ -988,7 +990,7 @@ decdn_eviction_high_water_pct (dflt 90) MINUS 5 — the gap is structural; decdn_origin_probe_negative_ttl_sec (dflt 2) <= decdn_origin_probe_fault_ttl_sec (dflt 5) <= decdn_origin_probe_ttl_sec (dflt 15); - decdn_max_blob_size_mb must be <= decdn_cache_size_mb (0 = unlimited); + decdn_max_blob_size_mb must be <= decdn_cache_size_mb; decdn_load_shed_max_concurrent_serves_low (dflt 192) must be <= decdn_load_shed_max_concurrent_serves_high (dflt 256). vars: @@ -1005,10 +1007,11 @@ _eff_probe_fault: >- {{ decdn_origin_probe_fault_ttl_sec if decdn_origin_probe_fault_ttl_sec not in ['', none] else 5 }} - # max_blob_size_mb defaults to cache_size_mb upstream (#1685), so an unset side - # trivially satisfies the constraint. + # max_blob_size_mb defaults to min(51200, cache_size_mb) upstream, so an unset + # side trivially satisfies the constraint. _eff_max_blob: >- - {{ decdn_max_blob_size_mb if decdn_max_blob_size_mb not in ['', none] else decdn_cache_size_mb }} + {{ decdn_max_blob_size_mb if decdn_max_blob_size_mb not in ['', none] + else [51200, decdn_cache_size_mb | int] | min }} _eff_shed_high: >- {{ decdn_load_shed_max_concurrent_serves_high if decdn_load_shed_max_concurrent_serves_high not in ['', none] else 256 }} diff --git a/ansible/roles/decdn_node/templates/node.toml.j2 b/ansible/roles/decdn_node/templates/node.toml.j2 index 1162fff..29d47c5 100644 --- a/ansible/roles/decdn_node/templates/node.toml.j2 +++ b/ansible/roles/decdn_node/templates/node.toml.j2 @@ -440,3 +440,8 @@ denied_origins = [{% for a in decdn_content_denied_origins %}"{{ a }}"{% if not [client] region_allowlist = [{% for r in decdn_client_region_allowlist %}"{{ r }}"{% if not loop.last %}, {% endif %}{% endfor %}] {% endif %} +{% if decdn_self_update_check not in ["", none] %} + +[self_update] +check = {{ decdn_self_update_check | bool | lower }} +{% endif %} diff --git a/ansible/roles/decdn_node/vars/main/networks.yml b/ansible/roles/decdn_node/vars/main/networks.yml index 49a770a..e968723 100644 --- a/ansible/roles/decdn_node/vars/main/networks.yml +++ b/ansible/roles/decdn_node/vars/main/networks.yml @@ -1,6 +1,6 @@ --- # GENERATED by scripts/sync-network-profiles.py -- do not hand-edit. -# Source: decdn/decdn @ 538bf319631cb37854819e43998e4a73da06fd5c +# Source: decdn/decdn @ 5c830ea2dae90c99410097192b558b3ac970ccc4 # crates/cli/src/known_chains.rs (KNOWN_CHAINS, KnownChain::addresses) # + contracts/deployments/.json # diff --git a/ansible/roles/sponsord/vars/main/networks.yml b/ansible/roles/sponsord/vars/main/networks.yml index b0a0405..770e4d7 100644 --- a/ansible/roles/sponsord/vars/main/networks.yml +++ b/ansible/roles/sponsord/vars/main/networks.yml @@ -1,6 +1,6 @@ --- # GENERATED by scripts/sync-network-profiles.py -- do not hand-edit. -# Source: decdn/decdn @ 538bf319631cb37854819e43998e4a73da06fd5c +# Source: decdn/decdn @ 5c830ea2dae90c99410097192b558b3ac970ccc4 # crates/cli/src/known_chains.rs (KNOWN_CHAINS, KnownChain::addresses) # + contracts/deployments/.json # diff --git a/ansible/roles/sponsord_onramp/defaults/main.yml b/ansible/roles/sponsord_onramp/defaults/main.yml index d47dc8b..bbd89a3 100644 --- a/ansible/roles/sponsord_onramp/defaults/main.yml +++ b/ansible/roles/sponsord_onramp/defaults/main.yml @@ -146,7 +146,7 @@ sponsord_onramp_gate_template_src: "" # verified against sponsord_onramp_decdn_keyring first. An exact tag, "vX.Y.Z", # needs that digest set beside it (`sha256sum SHA256SUMS` after verifying its .asc); # preflight refuses a tag without one and a line with one. -sponsord_onramp_decdn_release: v0.1 # decdn/decdn line or tag +sponsord_onramp_decdn_release: v0.2 # decdn/decdn line or tag sponsord_onramp_decdn_sums_sha256: "" # with a tag only sponsord_onramp_decdn_release_index: "https://api.github.com/repos/decdn/decdn/releases?per_page=100" # Where the role reads a line's SHA256SUMS: the installers' own download location diff --git a/ansible/roles/sponsord_onramp/vars/main/networks.yml b/ansible/roles/sponsord_onramp/vars/main/networks.yml index 63f107b..9a228c7 100644 --- a/ansible/roles/sponsord_onramp/vars/main/networks.yml +++ b/ansible/roles/sponsord_onramp/vars/main/networks.yml @@ -1,6 +1,6 @@ --- # GENERATED by scripts/sync-network-profiles.py -- do not hand-edit. -# Source: decdn/decdn @ 538bf319631cb37854819e43998e4a73da06fd5c +# Source: decdn/decdn @ 5c830ea2dae90c99410097192b558b3ac970ccc4 # crates/cli/src/known_chains.rs (KNOWN_CHAINS, KnownChain::addresses) # + contracts/deployments/.json # diff --git a/charts/decdn-node/Chart.yaml b/charts/decdn-node/Chart.yaml index 634540d..8a95378 100644 --- a/charts/decdn-node/Chart.yaml +++ b/charts/decdn-node/Chart.yaml @@ -15,7 +15,7 @@ version: 0.2.0 # restarted pod runs the newest). Keep it equal to the decdn_node role's # decdn_node_version, and the tag of the one image in the artifacthub.io/images # annotation below (make test-scripts checks both). -appVersion: "0.1" +appVersion: "0.2" kubeVersion: ">=1.25.0-0" home: https://decdn.org icon: https://decdn.org/icon.svg @@ -39,7 +39,7 @@ annotations: artifacthub.io/license: MIT artifacthub.io/images: | - name: decdn-node - image: ghcr.io/decdn/decdn-node:0.1 + image: ghcr.io/decdn/decdn-node:0.2 artifacthub.io/links: | - name: Chart source url: https://github.com/decdn/devops/tree/main/charts/decdn-node diff --git a/charts/decdn-node/README.md b/charts/decdn-node/README.md index aba2fd5..36dd978 100644 --- a/charts/decdn-node/README.md +++ b/charts/decdn-node/README.md @@ -193,9 +193,11 @@ documents each knob. Any key you do not set takes the daemon default, and a `nul Chart defaults that differ from the daemon's are the same as the role's: -- `cache.max_blob_size_mb: 1024`: a deliberate cap at one tenth of the default cache (the - daemon default is `min(51200, cache_size_mb)`). It must be at least 1: the cache engine's - admit gate has no zero special-case, despite the CLI help, so `0` rejects every blob. +- `cache.max_blob_size_mb: 1024`: the bytes held for one blob (an own-origin blob above it + is refused; a pull-through serves ranges of a larger one while the bytes held fit), a + deliberate cap at one tenth of the default cache (the daemon default is + `min(51200, cache_size_mb)`). It must be at least 1: the cache engine's admit gate has + no zero special-case, despite the CLI help, so `0` refuses every own-origin blob. - `observability.log_format: json` (the daemon default is `pretty`). - `cache.cache_size_mb: 10240` and `payment.rate_per_mb: 10` are pinned explicitly, at the daemon's own values. diff --git a/charts/decdn-node/values.yaml b/charts/decdn-node/values.yaml index c4ff8f7..de4b6a3 100644 --- a/charts/decdn-node/values.yaml +++ b/charts/decdn-node/values.yaml @@ -14,8 +14,8 @@ image: tag: "" # sha256:... — takes precedence over tag when set. Prefer it: a tag can be moved. # Take it from the release's signed decdn-node-image-digest.txt (v0.0.1: - # image-digest.txt), e.g. for v0.1.0 - # sha256:848532722735b4a115aca9606230d951c25f2dab06a7345305ecf67c7a0244b5 + # image-digest.txt), e.g. for v0.2.0 + # sha256:03a9f2e1167db31bb9a95af0b6672916300f86191a41db75db4c568c94899e60 digest: "" # "" picks Always for a release-line tag (X.Y, the default), so a restarted pod # pulls the newest release in it rather than whatever the node cached first, and @@ -198,9 +198,12 @@ config: rate_per_mb: 10 # USDC base units (6 decimals) cache: cache_size_mb: 10240 # 10 GB + # The bytes held for one blob (an own-origin blob above it is refused; a + # pull-through serves ranges of a larger one while the bytes held fit). # Deliberate override (daemon default = min(51200, cache_size_mb)): caps one # blob at a tenth of the default cache. Must be >= 1 — the cache engine's admit - # gate has no zero special-case (despite the CLI help), so 0 rejects every blob. + # gate has no zero special-case (despite the CLI help), so 0 refuses every + # own-origin blob. max_blob_size_mb: 1024 observability: log_level: info diff --git a/cloud-init/user-data-publisher.yaml b/cloud-init/user-data-publisher.yaml index a8b6876..999c023 100644 --- a/cloud-init/user-data-publisher.yaml +++ b/cloud-init/user-data-publisher.yaml @@ -143,7 +143,7 @@ write_files: # the digest read from it. To serve one exact release, set its tag with the # SHA-256 of its SHA256SUMS (`sha256sum SHA256SUMS` once its SHA256SUMS.asc # verifies), e.g.: - # sponsord_onramp_decdn_release: "v0.1.0" + # sponsord_onramp_decdn_release: "v0.2.0" # sponsord_onramp_decdn_sums_sha256: "<64 lowercase hex>" # What an interactive login prints while the bootstrap is unfinished. It is written diff --git a/compose/.env.example b/compose/.env.example index 0cbea2f..0739c94 100644 --- a/compose/.env.example +++ b/compose/.env.example @@ -29,8 +29,8 @@ COMPOSE_PROFILES=node # Compose names one release of it (README.md, the node's "Operate"). # --- node ----------------------------------------------------------------------- -# decdn/decdn v0.1.0 -DECDN_IMAGE_DIGEST=sha256:848532722735b4a115aca9606230d951c25f2dab06a7345305ecf67c7a0244b5 +# decdn/decdn v0.2.0 +DECDN_IMAGE_DIGEST=sha256:03a9f2e1167db31bb9a95af0b6672916300f86191a41db75db4c568c94899e60 # DECDN_IMAGE_REPO=ghcr.io/decdn/decdn-node # uid/gid of the host's dedicated `decdn` system account, which owns /var/lib/decdn diff --git a/compose/README.md b/compose/README.md index 564052d..5e2ec02 100644 --- a/compose/README.md +++ b/compose/README.md @@ -122,7 +122,10 @@ take a newer one. names the public RPC `config init` wrote, so `cli` hands the CLI a copy whose `rpc_url` reads your endpoint from `decdn.env` (`"${DECDN_RPC_URL}"`, which the config loader expands): it never appears on a command line, in a sudo log or in shell history. - The node serves paid traffic only after that. + The node serves paid traffic only after that. The CLI's daily "a newer release + exists, run `decdn self-update`" notice (decdn v0.2) is off in the `node.toml` + that `init` writes and under `cli`: the image's binary is read-only, and the pin + moves with `DECDN_IMAGE_DIGEST` ([Operate](#operate)). ### How it is laid out diff --git a/compose/decdn-compose b/compose/decdn-compose index 7c22d26..02d4d28 100755 --- a/compose/decdn-compose +++ b/compose/decdn-compose @@ -776,11 +776,28 @@ def edit_node_toml(region: str, cache_node: bool) -> None: text = re.sub(r"(?m)^\[cache\]$", "[cache]\nnode_to_node_pull_through_enabled = true", text, count=1) else: text = text.rstrip("\n") + "\n\n[cache]\nnode_to_node_pull_through_enabled = true\n" + # Since decdn v0.2, `config init` ends with a [self_update] table whose `check` + # is commented: the CLI's daily "run `decdn self-update`" notice, which the + # daemon ignores. Off here, in the file the daemon and `cli` share: the image's + # binary is read-only and the pin moves with DECDN_IMAGE_DIGEST, so the notice + # has nowhere to go (`cli` also sets DECDN_SELF_UPDATE_CHECK=0, for a file + # edited by hand). An older `config init` without the table gets it appended. + if not re.search(r"(?m)^check\s*=", text): + if re.search(r"(?m)^# check = .*$", text): + text = re.sub(r"(?m)^# check = .*$", "check = false", text, count=1) + elif re.search(r"(?m)^\[self_update\]$", text): + text = re.sub(r"(?m)^\[self_update\]$", "[self_update]\ncheck = false", text, count=1) + else: + text = text.rstrip("\n") + "\n\n[self_update]\ncheck = false\n" try: parsed = tomllib.loads(text) except tomllib.TOMLDecodeError as e: raise Refused(f"editing {NODE_TOML} left invalid TOML ({e}); it was not written") from e - want = {("identity", "data_dir"): str(VAR_DECDN), ("identity", "region"): region} + want = { + ("identity", "data_dir"): str(VAR_DECDN), + ("identity", "region"): region, + ("self_update", "check"): False, + } if cache_node: want[("cache", "node_to_node_pull_through_enabled")] = True wrong = [f"{t}.{k}" for (t, k), v in want.items() if parsed.get(t, {}).get(k) != v] @@ -2157,7 +2174,7 @@ def cmd_cli(args: argparse.Namespace) -> None: cp = decdn_cli( project, cli_args, - cli_env={"DECDN_KEYSTORE_PASSWORD_FILE": str(NODE_PASSWORD)}, + cli_env=CLI_ENV, network="host", tty=True, check=False, @@ -2168,6 +2185,10 @@ def cmd_cli(args: argparse.Namespace) -> None: CLI_CONFIG = "/run/decdn-cli/node.toml" +# What `cli` hands the CLI besides the node's own environment: the keystore +# password, and no self-update notice (see edit_node_toml) even when node.toml was +# edited by hand. +CLI_ENV = {"DECDN_KEYSTORE_PASSWORD_FILE": str(NODE_PASSWORD), "DECDN_SELF_UPDATE_CHECK": "0"} def with_env_rpc_url(text: str) -> str: diff --git a/compose/sponsord-onramp.env.example b/compose/sponsord-onramp.env.example index 8145d66..9b288fa 100644 --- a/compose/sponsord-onramp.env.example +++ b/compose/sponsord-onramp.env.example @@ -22,9 +22,9 @@ ONRAMP_TURNSTILE_SITEKEY= # SHA-256 of that release's SHA256SUMS file (`sha256sum SHA256SUMS` once its .asc # verifies). The Ansible role serves the newest release of its decdn line; Compose # has no resolver, so this names one, the newest when this file last moved: -# decdn/decdn v0.1.0. -ONRAMP_DECDN_RELEASE=v0.1.0 -ONRAMP_DECDN_SUMS_SHA256=c88ad5479f31bb0857db2c0205c54b240cbceb33a96724e4f8a288b6d866ab66 +# decdn/decdn v0.2.0. +ONRAMP_DECDN_RELEASE=v0.2.0 +ONRAMP_DECDN_SUMS_SHA256=f4f0a16f1a3896c729bba21ece023f00cae89bea51369047fa7a03f6c7f088d4 # --- Client address --------------------------------------------------------------- # The header the proxy puts the client address in; the onramp rate limits by it. diff --git a/compose/tests/test_decdn_compose.py b/compose/tests/test_decdn_compose.py index 40a3f5b..ea161d2 100644 --- a/compose/tests/test_decdn_compose.py +++ b/compose/tests/test_decdn_compose.py @@ -415,9 +415,37 @@ def test_cache_node(self) -> None: self.edit(True), '# deCDN node configuration\n\n[identity]\ndata_dir = "/var/lib/decdn"\nregion = "DE"\n\n' "[network]\n# bind_port = 4433\n\n[cache]\n" - 'node_to_node_pull_through_enabled = true\n# region = "us-east-1"\n', + 'node_to_node_pull_through_enabled = true\n# region = "us-east-1"\n' + "\n[self_update]\ncheck = false\n", ) + def test_turns_the_self_update_notice_off_in_place(self) -> None: + # The shape `decdn config init` writes since v0.2: the table is there, its + # knob commented. The edit uncomments it as false, once (idempotent). + p = self.tmp / "node.toml" + p.write_text( + self.SAMPLE + '\n[client]\n# region_allowlist = ["US", "DE"]\n\n[self_update]\n' + "# Once a day, after a command's own work, `decdn` checks for a newer signed\n# check = true\n" + ) + with mock.patch.object(dc, "NODE_TOML", p): + dc.edit_node_toml("DE", cache_node=False) + dc.edit_node_toml("DE", cache_node=False) + text = p.read_text() + self.assertTrue(text.endswith("[self_update]\n# Once a day, after a command's own work, `decdn` checks for a newer signed\ncheck = false\n")) + self.assertEqual(text.count("check = false"), 1) + self.assertIs(dc.tomllib.loads(text)["self_update"]["check"], False) + + def test_a_bare_self_update_table_gets_the_knob(self) -> None: + p = self.tmp / "node.toml" + p.write_text(self.SAMPLE + "\n[self_update]\n") + with mock.patch.object(dc, "NODE_TOML", p): + dc.edit_node_toml("DE", cache_node=False) + self.assertIs(dc.tomllib.loads(p.read_text())["self_update"]["check"], False) + + def test_cli_runs_without_the_self_update_notice(self) -> None: + self.assertEqual(dc.CLI_ENV["DECDN_SELF_UPDATE_CHECK"], "0") + self.assertEqual(dc.CLI_ENV["DECDN_KEYSTORE_PASSWORD_FILE"], str(dc.NODE_PASSWORD)) + def test_appends_a_missing_cache_table(self) -> None: p = self.tmp / "node.toml" p.write_text('[identity]\n# data_dir = "~/.decdn"\n# region = "US"\n') diff --git a/docs/lifecycle.md b/docs/lifecycle.md index 4a9159c..1f03b53 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -257,6 +257,14 @@ sudo systemd-run --pty --wait --collect -p User=decdn \ --keystore-password-file /etc/decdn/keystore.password ``` +Since decdn v0.2 the CLI can end a command run at a terminal with a one-line notice +that a newer release exists and that `decdn self-update` installs it. Never run that +on a role-managed host: under plain `sudo` it replaces `/usr/local/bin/decdn` outside +the role's signed-release install, and the next deploy fails its version check. The +role moves the binaries itself (`decdn_node_version`, [node-operators.md § Day +2](node-operators.md#day-2)), and renders the notice off in `node.toml` +(`decdn_self_update_check`), which these recipes pass as `--config`. + ## sponsord (the onboarding sponsor) sponsord keeps no state. What makes it *that* sponsor are its credentials under diff --git a/docs/node-operators.md b/docs/node-operators.md index e0d35d5..b4f9e52 100644 --- a/docs/node-operators.md +++ b/docs/node-operators.md @@ -73,7 +73,9 @@ path. This repository stops at host preparation and startup. - Upgrades: every path pins a decdn/decdn release line (`X.Y`), not one release. On Ansible and cloud-init each deploy installs the line's newest release and restarts - the node onto it (set `decdn_node_version` to an exact `"X.Y.Z"` to hold a host). + the node onto it (set `decdn_node_version` to an exact `"X.Y.Z"` to hold a host); + never `decdn self-update` there, which would replace the CLI outside that install + and fail the next deploy's version check (the role renders its notice off). The chart's default image tag is the line's, pulled `Always`, so `kubectl rollout restart` picks a new release up (set `image.tag` or `image.digest` to hold it). Compose pins a digest: take a newer one as its README's diff --git a/docs/requirements.md b/docs/requirements.md index 4dd5fcc..f9937de 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -114,8 +114,10 @@ cache_size_mb the cache (role default 10 GiB) + an fs origin, if the node holds one ``` -`max_blob_size_mb` must be ≤ `cache_size_mb`: the node rejects any blob larger than it -(`BlobTooLarge`), so size it to the largest single object you expect to serve (role +`max_blob_size_mb` must be ≤ `cache_size_mb`: it caps the bytes the node holds for one +blob. A blob from the node's own origin above it is refused (`BlobTooLarge`); a +pull-through serves ranges of a larger blob while the bytes held for it fit, and stops +once they do not. Size it to the largest single object you expect to serve whole (role default 1 GiB). Put `/var/lib/decdn` on the fast disk. The Ansible `source` install method adds, outside the data volume, a Rust toolchain diff --git a/examples/collection-fleet/inventory/group_vars/decdn_nodes.yml b/examples/collection-fleet/inventory/group_vars/decdn_nodes.yml index c77132b..67d0aad 100644 --- a/examples/collection-fleet/inventory/group_vars/decdn_nodes.yml +++ b/examples/collection-fleet/inventory/group_vars/decdn_nodes.yml @@ -26,9 +26,10 @@ # --- Cache sizing ----------------------------------------------------------------- # Size to each host's disk and to the largest object you expect to serve; see # https://github.com/decdn/devops/blob/main/docs/requirements.md for the disk formula. -# A node rejects any blob larger than decdn_max_blob_size_mb with BlobTooLarge, and -# decdn_cache_size_mb must be at least that. Override per host in host_vars when -# disks differ. +# decdn_max_blob_size_mb caps the bytes held for one blob: an own-origin blob above it +# is refused with BlobTooLarge, and a pull-through serves ranges of a larger one while +# the bytes held fit. decdn_cache_size_mb must be at least that. Override per host in +# host_vars when disks differ. # decdn_max_blob_size_mb: 1024 # role default # decdn_cache_size_mb: 10240 # role default (10 GiB) diff --git a/examples/collection-fleet/inventory/group_vars/sponsord_onramp_hosts.yml b/examples/collection-fleet/inventory/group_vars/sponsord_onramp_hosts.yml index 7b1d476..8ab8a89 100644 --- a/examples/collection-fleet/inventory/group_vars/sponsord_onramp_hosts.yml +++ b/examples/collection-fleet/inventory/group_vars/sponsord_onramp_hosts.yml @@ -36,6 +36,6 @@ # role's sponsord_onramp_decdn_release line (decdn/decdn, the decdn_node role's default # line; a host's decdn_node_version does not move it), with the SHA-256 of its # verified SHA256SUMS, on each deploy. To serve one exact release, set its tag and digest: -# sponsord_onramp_decdn_release: v0.1.0 +# sponsord_onramp_decdn_release: v0.2.0 # sponsord_onramp_decdn_sums_sha256: "…" # CapacityBond and SlashJudge come from sponsord_network (group_vars/sponsord_hosts.yml). diff --git a/examples/collection-fleet/inventory/host_vars/decdn-node-1/main.yml b/examples/collection-fleet/inventory/host_vars/decdn-node-1/main.yml index 4ec5209..fd155bc 100644 --- a/examples/collection-fleet/inventory/host_vars/decdn-node-1/main.yml +++ b/examples/collection-fleet/inventory/host_vars/decdn-node-1/main.yml @@ -24,13 +24,13 @@ # line the role pins ("X.Y": its newest release, installed by each deploy); set an # exact one only to hold a node on it. decdn_node_install_method: release -# decdn_node_version: "0.1.0" +# decdn_node_version: "0.2.0" # # Alternative: build on the node from a git ref (any tag, branch or SHA), as an # unprivileged build user with a pinned toolchain (decdn_node_version is not checked # in this mode): # decdn_node_install_method: source -# decdn_node_source_ref: v0.1.0 +# decdn_node_source_ref: v0.2.0 # Alternative: copy binaries built on this control machine # (`cargo build --release -p decdn-node -p decdn-cli` in a decdn/decdn checkout): # decdn_node_install_method: manual diff --git a/examples/collection-fleet/inventory/host_vars/origin-1/main.yml b/examples/collection-fleet/inventory/host_vars/origin-1/main.yml index 1b0b93e..0fe88bd 100644 --- a/examples/collection-fleet/inventory/host_vars/origin-1/main.yml +++ b/examples/collection-fleet/inventory/host_vars/origin-1/main.yml @@ -26,13 +26,13 @@ # line the role pins ("X.Y": its newest release, installed by each deploy); set an # exact one only to hold a node on it. decdn_node_install_method: release -# decdn_node_version: "0.1.0" +# decdn_node_version: "0.2.0" # # Alternative: build on the node from a git ref (any tag, branch or SHA), as an # unprivileged build user with a pinned toolchain (decdn_node_version is not checked # in this mode): # decdn_node_install_method: source -# decdn_node_source_ref: v0.1.0 +# decdn_node_source_ref: v0.2.0 # Alternative: copy binaries built on this control machine # (`cargo build --release -p decdn-node -p decdn-cli` in a decdn/decdn checkout): # decdn_node_install_method: manual diff --git a/monitoring/decdn-node/dashboard-chain.json b/monitoring/decdn-node/dashboard-chain.json index b2b29d0..347a607 100644 --- a/monitoring/decdn-node/dashboard-chain.json +++ b/monitoring/decdn-node/dashboard-chain.json @@ -3061,6 +3061,113 @@ }, "id": 25 }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Unresolved on-chain holds", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max by (instance) (decdn_buyer_pool_open_unresolved{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A", + "legendFormat": "{{instance}} openPool" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max by (instance) (decdn_buyer_topup_unresolved{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "B", + "legendFormat": "{{instance}} topUp" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none", + "min": 0, + "max": 1, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 38, + "description": "1 while this node holds its slot for an openPool or a topUp whose outcome the chain has not shown (the topUp hold is decdn v0.2): the submit failed in transport or the receipt could not be read, and the transaction may still mine, so the node sends no other transaction of that kind until a receipt arrives, the account's nonce passes it, or a re-send shows the pool closed. A hold that stays means the RPC cannot answer, a pending transaction holds the nonce or an earlier one, or every re-send at that nonce fails (the wallet cannot fund the amount or the gas); the daemon's repeating warning names the last failure in its `last` field. Do not restart a node holding a topUp slot: the restart forgets the hold, and the node may escrow a second topUp into its own pool. DecdnBuyerTopupUnresolved fires after 15m." + }, { "datasource": { "type": "loki", @@ -3099,7 +3206,7 @@ "h": 9, "w": 24, "x": 0, - "y": 83 + "y": 90 }, "id": 26 }, @@ -3109,7 +3216,7 @@ "h": 1, "w": 24, "x": 0, - "y": 92 + "y": 99 }, "id": 33, "panels": [], @@ -3207,7 +3314,7 @@ "h": 8, "w": 12, "x": 0, - "y": 93 + "y": 100 }, "id": 34 }, @@ -3337,7 +3444,7 @@ "h": 8, "w": 12, "x": 12, - "y": 93 + "y": 100 }, "id": 35 }, @@ -3432,7 +3539,7 @@ "h": 8, "w": 12, "x": 0, - "y": 101 + "y": 108 }, "id": 36 }, @@ -3527,7 +3634,7 @@ "h": 8, "w": 12, "x": 12, - "y": 101 + "y": 108 }, "id": 37 } diff --git a/monitoring/decdn-node/dashboard-delivery.json b/monitoring/decdn-node/dashboard-delivery.json index 8c459c0..5cd74d9 100644 --- a/monitoring/decdn-node/dashboard-delivery.json +++ b/monitoring/decdn-node/dashboard-delivery.json @@ -397,7 +397,7 @@ }, "title": "Serve refusals by reason", "type": "timeseries", - "description": "Each reason signs one of three wire classes (ADR 005 § Open-time refusal classes), so the per-reason split survives only here. NotFound: cache miss, unknown lane, owner mismatch, pool unconfirmed, signer floor at cap, chain stale, both load sheds and the pull-loop guard. Declined: evicted since probe, internal error, range not satisfiable, blob too large, foreign declined and the three takedown refusals (hash denied, chain hash denied, origin denied). Unfunded: insufficient deposit, signer cap exhausted and pool closing, spoken only to a requester that proved its lane or signer binding; any other requester gets NotFound. stream cap full and bad binding are resets with no signed response (RATE_LIMITED and MALFORMED_MESSAGE). insufficient deposit is fed by three separate guards (the pre-spend miss floor, the window tier's ceiling and the direct-serve ceiling) and cannot attribute a spike to one — the node's throttled warn! line does. Sustained insufficient deposit means either clients running dry, or this node's chain watcher lagging an on-chain top-up and turning funded clients away.", + "description": "Each reason signs one of three wire classes (ADR 005 § Open-time refusal classes), so the per-reason split survives only here. NotFound: cache miss, unknown lane, owner mismatch, pool unconfirmed, signer floor at cap, chain stale, both load sheds and the pull-loop guard. Declined: evicted since probe, internal error, range not satisfiable, blob too large, foreign declined and the three takedown refusals (hash denied, chain hash denied, origin denied). Unfunded: insufficient deposit, signer cap exhausted and pool closing, spoken only to a requester that proved its lane or signer binding; any other requester gets NotFound. stream cap full and bad binding are resets with no signed response (RATE_LIMITED and MALFORMED_MESSAGE). insufficient deposit is fed by three separate guards (the pre-spend miss floor, the window tier's ceiling and the direct-serve ceiling) and cannot attribute a spike to one — the node's throttled warn! line does. Sustained insufficient deposit means either clients running dry, or this node's chain watcher lagging an on-chain top-up and turning funded clients away. `blob too large (midstream)` is a pull-through the node stopped mid-delivery with a Declined refusal because its fill reached the max_blob_size cap on bytes held for one blob (decdn v0.2): the bytes that landed were served and the requester asks another node. A rising value means clients ask this node for more of a blob than its cap lets it hold.", "targets": [ { "datasource": { @@ -498,6 +498,15 @@ "refId": "T", "legendFormat": "blob too large" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_midstream_blob_too_large_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "W", + "legendFormat": "blob too large (midstream)" + }, { "datasource": { "type": "prometheus", @@ -3366,14 +3375,14 @@ }, "title": "Unattributed stream failures", "type": "timeseries", - "description": "Seller-leg stream failures that no reason counter claims: rate(decdn_streams_failed_total{direction=\"inbound\"}) minus the siblings that each end exactly one inbound stream — every serve_stream_rejected_* reason (the stream-cap reset included), the unreadable request, the two midstream exhaustion stops, the takedown termination, the rejected voucher, the spent proof budget, the declined quote (a requester that leaves before any voucher credits a byte, which includes the header handshake a downstream miss pull does not adopt), the mid-stream client abandon, and the node fault. decdn_serve_economics_refused_total is deliberately NOT subtracted: it is a buyer-leg gate that declines to buy from a candidate upstream, and ends no inbound stream. Expected 0. Any positive rate is a failure no reason claims, which is a bug: file it. The node crate's INBOUND_FAILURE_REASONS list and a test hold this expression to the code.", + "description": "Seller-leg stream failures that no reason counter claims: rate(decdn_streams_failed_total{direction=\"inbound\"}) minus the siblings that each end exactly one inbound stream — every serve_stream_rejected_* reason (the stream-cap reset included), the unreadable request, the three midstream stops (pool and signer cap exhausted, and the blob cap on bytes held reached), the takedown termination, the rejected voucher, the spent proof budget, the declined quote (a requester that leaves before any voucher credits a byte, which includes the header handshake a downstream miss pull does not adopt), the mid-stream client abandon, and the node fault. decdn_serve_economics_refused_total is deliberately NOT subtracted: it is a buyer-leg gate that declines to buy from a candidate upstream, and ends no inbound stream. Expected 0. Any positive rate is a failure no reason claims, which is a bug: file it. The node crate's INBOUND_FAILURE_REASONS list and a test hold this expression to the code.", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(decdn_streams_failed_total{direction=\"inbound\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_bad_binding_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_blob_too_large_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_cache_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_chain_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_chain_stale_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_evicted_since_probe_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_foreign_declined_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_insufficient_deposit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_internal_error_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_load_shed_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_load_shed_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_origin_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_owner_mismatch_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_pool_closing_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_pool_unconfirmed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_pull_loop_guard_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_range_not_satisfiable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_signer_floor_at_cap_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_stream_cap_full_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_unknown_lane_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_request_unreadable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_midstream_pool_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_midstream_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_terminated_takedown_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_voucher_rejected_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_proof_budget_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_client_declined_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_client_abandoned_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_node_fault_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "expr": "sum(rate(decdn_streams_failed_total{direction=\"inbound\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_bad_binding_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_blob_too_large_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_cache_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_chain_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_chain_stale_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_evicted_since_probe_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_foreign_declined_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_insufficient_deposit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_internal_error_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_load_shed_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_load_shed_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_origin_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_owner_mismatch_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_pool_closing_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_pool_unconfirmed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_pull_loop_guard_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_range_not_satisfiable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_signer_floor_at_cap_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_stream_cap_full_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_unknown_lane_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_request_unreadable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_midstream_pool_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_midstream_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_midstream_blob_too_large_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_terminated_takedown_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_voucher_rejected_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_proof_budget_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_client_declined_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_client_abandoned_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_node_fault_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", "refId": "A", "legendFormat": "no reason counter" } diff --git a/monitoring/decdn-node/dashboard-node.json b/monitoring/decdn-node/dashboard-node.json index 7d67533..766dcc3 100644 --- a/monitoring/decdn-node/dashboard-node.json +++ b/monitoring/decdn-node/dashboard-node.json @@ -2776,6 +2776,15 @@ "expr": "sum(rate(decdn_dht_lookup_round_ceiling_total{instance=~\"$node\"}[$__rate_interval]))", "refId": "H", "legendFormat": "lookup round ceiling" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_lookup_dial_timeouts_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "I", + "legendFormat": "lookup dial timeouts" } ], "fieldConfig": { diff --git a/monitoring/decdn-node/grafana-dashboard.json b/monitoring/decdn-node/grafana-dashboard.json index 0407a03..ca96be4 100644 --- a/monitoring/decdn-node/grafana-dashboard.json +++ b/monitoring/decdn-node/grafana-dashboard.json @@ -864,7 +864,7 @@ }, "title": "Serve refusals by reason", "type": "timeseries", - "description": "Every refusal reason the serve leg records. The wire carries only three classes (NotFound, Declined, Unfunded), each covering many of these reasons, so a client cannot tell them apart. These counters are the only place the cause survives.", + "description": "Every refusal reason the serve leg records. The wire carries only three classes (NotFound, Declined, Unfunded), each covering many of these reasons, so a client cannot tell them apart. These counters are the only place the cause survives. `blob too large (midstream)` is a pull-through the node stopped mid-delivery with a Declined refusal because its fill reached the max_blob_size cap on bytes held for one blob (decdn v0.2): the bytes that landed were served and the requester asks another node. A rising value means clients ask this node for more of a blob than its cap lets it hold.", "targets": [ { "datasource": { @@ -965,6 +965,15 @@ "refId": "T", "legendFormat": "blob too large" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_midstream_blob_too_large_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "W", + "legendFormat": "blob too large (midstream)" + }, { "datasource": { "type": "prometheus", @@ -2313,14 +2322,14 @@ }, "title": "Unattributed stream failures", "type": "timeseries", - "description": "Seller-leg stream failures that no reason counter claims: rate(decdn_streams_failed_total{direction=\"inbound\"}) minus the siblings that each end exactly one inbound stream — every serve_stream_rejected_* reason (the stream-cap reset included), the unreadable request, the two midstream exhaustion stops, the takedown termination, the rejected voucher, the spent proof budget, the declined quote (a requester that leaves before any voucher credits a byte, which includes the header handshake a downstream miss pull does not adopt), the mid-stream client abandon, and the node fault. decdn_serve_economics_refused_total is deliberately NOT subtracted: it is a buyer-leg gate that declines to buy from a candidate upstream, and ends no inbound stream. Expected 0. Any positive rate is a failure no reason claims, which is a bug: file it. The node crate's INBOUND_FAILURE_REASONS list and a test hold this expression to the code.", + "description": "Seller-leg stream failures that no reason counter claims: rate(decdn_streams_failed_total{direction=\"inbound\"}) minus the siblings that each end exactly one inbound stream — every serve_stream_rejected_* reason (the stream-cap reset included), the unreadable request, the three midstream stops (pool and signer cap exhausted, and the blob cap on bytes held reached), the takedown termination, the rejected voucher, the spent proof budget, the declined quote (a requester that leaves before any voucher credits a byte, which includes the header handshake a downstream miss pull does not adopt), the mid-stream client abandon, and the node fault. decdn_serve_economics_refused_total is deliberately NOT subtracted: it is a buyer-leg gate that declines to buy from a candidate upstream, and ends no inbound stream. Expected 0. Any positive rate is a failure no reason claims, which is a bug: file it. The node crate's INBOUND_FAILURE_REASONS list and a test hold this expression to the code.", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(decdn_streams_failed_total{direction=\"inbound\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_bad_binding_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_blob_too_large_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_cache_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_chain_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_chain_stale_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_evicted_since_probe_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_foreign_declined_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_insufficient_deposit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_internal_error_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_load_shed_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_load_shed_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_origin_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_owner_mismatch_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_pool_closing_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_pool_unconfirmed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_pull_loop_guard_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_range_not_satisfiable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_signer_floor_at_cap_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_stream_cap_full_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_unknown_lane_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_request_unreadable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_midstream_pool_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_midstream_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_terminated_takedown_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_voucher_rejected_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_proof_budget_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_client_declined_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_client_abandoned_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_node_fault_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "expr": "sum(rate(decdn_streams_failed_total{direction=\"inbound\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_bad_binding_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_blob_too_large_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_cache_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_chain_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_chain_stale_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_evicted_since_probe_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_foreign_declined_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_insufficient_deposit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_internal_error_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_load_shed_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_load_shed_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_origin_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_owner_mismatch_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_pool_closing_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_pool_unconfirmed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_pull_loop_guard_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_range_not_satisfiable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_signer_floor_at_cap_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_stream_cap_full_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_unknown_lane_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_request_unreadable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_midstream_pool_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_midstream_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_midstream_blob_too_large_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_terminated_takedown_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_voucher_rejected_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_proof_budget_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_client_declined_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_client_abandoned_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_node_fault_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", "refId": "A", "legendFormat": "no reason counter" } diff --git a/monitoring/decdn-node/prometheus-alerts.yml b/monitoring/decdn-node/prometheus-alerts.yml index 23d4589..d9e73c4 100644 --- a/monitoring/decdn-node/prometheus-alerts.yml +++ b/monitoring/decdn-node/prometheus-alerts.yml @@ -1026,6 +1026,33 @@ groups: again. Compare decdn_buyer_wallet_usdc with the pools the operator address owns on chain. + # The gauge (decdn v0.2) is 1 while the node holds its top-up slot for a + # topUp whose outcome the chain has not shown: the submit failed in + # transport or the receipt could not be read, and the transaction may + # still mine, so no other topUp goes out meanwhile. A healthy settle + # clears it within a few blocks; 15m of hold is a stuck chain path. + - alert: DecdnBuyerTopupUnresolved + expr: decdn_buyer_topup_unresolved == 1 + for: 15m + labels: + component: payments + severity: warning + annotations: + summary: "An unconfirmed topUp holds the top-up slot on {{ $labels.instance }}" + description: | + The node sent a topUp whose outcome the chain has not shown for + 15 minutes, and holds its top-up slot until it does: a low-water + refill does nothing meanwhile, and a pull's funding recovery step + fails at once (decdn_node_pull_recovery_step_refused_total). + One of three causes: the RPC cannot answer, a pending transaction + holds the topUp's nonce or an earlier one, or every re-send at + that nonce fails because the wallet cannot fund the topUp or its + gas. The daemon's repeating warning names the last failure in its + `last` field. Check the RPC provider, then the wallet's pending + transactions and balance (decdn_buyer_wallet_usdc). Do not + restart the node to clear it: a restart forgets the hold, and the + node may escrow a second topUp into its own pool. + - alert: DecdnBuyerLaneSeedFailing expr: rate(decdn_buyer_lane_seed_failures_total[15m]) > 0 for: 30m diff --git a/monitoring/decdn-node/prometheus-alerts_test.yml b/monitoring/decdn-node/prometheus-alerts_test.yml index 11375ec..8a8bb76 100644 --- a/monitoring/decdn-node/prometheus-alerts_test.yml +++ b/monitoring/decdn-node/prometheus-alerts_test.yml @@ -120,3 +120,32 @@ tests: - eval_time: 1h20m alertname: DecdnCapacityBondRegistryResyncFailing exp_alerts: [] + + # Buyer top-up hold: a reads 1 from the start and fires at exactly the 15m for. + # b holds for 9 minutes (6m to 14m) and clears, inside the for, so it never + # fires. c's hold starts at 20m and fires at 35m; a is still firing then. + - interval: 1m + input_series: + - series: 'decdn_buyer_topup_unresolved{job="decdn-node", instance="a"}' + values: '1x60' + - series: 'decdn_buyer_topup_unresolved{job="decdn-node", instance="b"}' + values: '0x5 1x9 0x45' + - series: 'decdn_buyer_topup_unresolved{job="decdn-node", instance="c"}' + values: '0x19 1x40' + alert_rule_test: + - eval_time: 14m + alertname: DecdnBuyerTopupUnresolved + exp_alerts: [] + - eval_time: 15m + alertname: DecdnBuyerTopupUnresolved + exp_alerts: + - exp_labels: {job: decdn-node, instance: a, component: payments, severity: warning} + - eval_time: 34m + alertname: DecdnBuyerTopupUnresolved + exp_alerts: + - exp_labels: {job: decdn-node, instance: a, component: payments, severity: warning} + - eval_time: 35m + alertname: DecdnBuyerTopupUnresolved + exp_alerts: + - exp_labels: {job: decdn-node, instance: a, component: payments, severity: warning} + - exp_labels: {job: decdn-node, instance: c, component: payments, severity: warning}