From 96134c76ce02c812f9d0cda9e7a1acb00326b8cd Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Wed, 21 Oct 2026 18:19:51 +0300 Subject: [PATCH 1/2] fix(monitoring): probe timeout, dial timeouts, no-hash causes and maybe-broadcast alert - dashboard-node: the probe-collection text names the 1 s probe timeout (decdn#2533); the DHT panel adds decdn_dht_lookup_dial_timeouts_total beside the round timeouts (decdn#2532). - Buyer-pool unresolved panel and DecdnBuyerTopupUnresolved comment name the already known / nonce too low / underpriced no-hash answers. - New DecdnRedemptionMaybeBroadcast on decdn_onchain_tx_maybe_broadcast_total (decdn#2574), with a promtool test; the outcome panel adds the series and DecdnRedemptionFailing points at the new rule. Closes #163, closes #164, closes #169, closes #170 Co-Authored-By: Claude Opus 5.5 --- .claude/skills/grafana-dashboards/SKILL.md | 2 +- monitoring/decdn-node/dashboard-chain.json | 19 ++++++--- monitoring/decdn-node/dashboard-node.json | 15 +++++-- monitoring/decdn-node/prometheus-alerts.yml | 40 ++++++++++++++++--- .../decdn-node/prometheus-alerts_test.yml | 19 +++++++++ 5 files changed, 81 insertions(+), 14 deletions(-) diff --git a/.claude/skills/grafana-dashboards/SKILL.md b/.claude/skills/grafana-dashboards/SKILL.md index a159126..03a7ed1 100644 --- a/.claude/skills/grafana-dashboards/SKILL.md +++ b/.claude/skills/grafana-dashboards/SKILL.md @@ -39,7 +39,7 @@ the iroh relay's in `monitoring/iroh-relay/`): process, iroh transport, DHT and probe, logs, traces. - `prometheus-alerts.yml` — three groups: `decdn-slash-safety`, `decdn-liveness`, `decdn-delivery`. Every rule carries a `component` label. Only rules with a matching - section in upstream's `docs/runbook.md` (20 of 51) also carry a `runbook_url`. + section in upstream's `docs/runbook.md` (20 of 53) also carry a `runbook_url`. `DecdnNodeDown` and `DecdnCapacityBondRegistryResyncFailing` have promtool unit tests in `prometheus-alerts_test.yml` (the chart's `.helmignore` keeps that file out of the package): keep them in step. diff --git a/monitoring/decdn-node/dashboard-chain.json b/monitoring/decdn-node/dashboard-chain.json index 347a607..91aacef 100644 --- a/monitoring/decdn-node/dashboard-chain.json +++ b/monitoring/decdn-node/dashboard-chain.json @@ -2438,7 +2438,7 @@ }, "title": "On-chain transaction outcomes", "type": "timeseries", - "description": "Every node transaction counts once by outcome. A receipt wait that fails or times out falls back to fetching the receipt by hash, and a receipt found there counts as landed or reverted. receipt failed and timeout are the transactions still unconfirmed after that fetch; they can still mine, and their tx hash is in the warn! line. receipt recovered is not an outcome: it counts the waits the fetch by hash resolved, which points at a lagging RPC provider.", + "description": "Every node transaction counts once by outcome. A receipt wait that fails or times out falls back to fetching the receipt by hash, and a receipt found there counts as landed or reverted. receipt failed and timeout are the transactions still unconfirmed after that fetch; they can still mine, and their tx hash is in the warn! line. maybe broadcast is a send that failed with no tx hash (a timeout, a reset, an HTTP 5xx, or an `already known`, `nonce too low` or `replacement transaction underpriced` answer): it can be in a mempool or have mined, and the claims stay for the next sweep (decdn after v0.2; DecdnRedemptionMaybeBroadcast). receipt recovered is not an outcome: it counts the waits the fetch by hash resolved, which points at a lagging RPC provider.", "targets": [ { "datasource": { @@ -2472,8 +2472,17 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(decdn_onchain_tx_receipt_failed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "expr": "sum(rate(decdn_onchain_tx_maybe_broadcast_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", "refId": "D", + "legendFormat": "maybe broadcast" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_onchain_tx_receipt_failed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", "legendFormat": "receipt failed" }, { @@ -2482,7 +2491,7 @@ "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(decdn_onchain_tx_timeout_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", - "refId": "E", + "refId": "F", "legendFormat": "timeout" }, { @@ -2491,7 +2500,7 @@ "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(decdn_onchain_tx_receipt_recovered_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", - "refId": "F", + "refId": "G", "legendFormat": "receipt recovered" } ], @@ -3166,7 +3175,7 @@ "y": 83 }, "id": 38, - "description": "1 while this node holds its slot for an openPool or a topUp whose outcome the chain has not shown (the topUp hold is decdn v0.2): the submit failed in transport or the receipt could not be read, and the transaction may still mine, so the node sends no other transaction of that kind until a receipt arrives, the account's nonce passes it, or a re-send shows the pool closed. A hold that stays means the RPC cannot answer, a pending transaction holds the nonce or an earlier one, or every re-send at that nonce fails (the wallet cannot fund the amount or the gas); the daemon's repeating warning names the last failure in its `last` field. Do not restart a node holding a topUp slot: the restart forgets the hold, and the node may escrow a second topUp into its own pool. DecdnBuyerTopupUnresolved fires after 15m." + "description": "1 while this node holds its slot for an openPool or a topUp whose outcome the chain has not shown (the topUp hold is decdn v0.2): the submit got no tx hash or the receipt could not be read, and the transaction may still mine. A submit gets no tx hash when the transport fails, or when the RPC node already holds the transaction (`already known`), counts its nonce as spent (`nonce too low`) or holds another transaction of this key at that nonce (`replacement transaction underpriced`); each can answer a gateway that retried a send another backend accepted. Meanwhile the node sends no other transaction of that kind until a receipt arrives, the account's nonce passes it, or a re-send shows the pool closed. A hold that stays means the RPC cannot answer, a pending transaction holds the nonce or an earlier one, or every re-send at that nonce fails (the wallet cannot fund the amount or the gas); the daemon's repeating warning names the last failure in its `last` field. Do not restart a node holding a topUp slot: the restart forgets the hold, and the node may escrow a second topUp into its own pool. DecdnBuyerTopupUnresolved fires after 15m." }, { "datasource": { diff --git a/monitoring/decdn-node/dashboard-node.json b/monitoring/decdn-node/dashboard-node.json index 766dcc3..b223eca 100644 --- a/monitoring/decdn-node/dashboard-node.json +++ b/monitoring/decdn-node/dashboard-node.json @@ -3421,8 +3421,17 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(decdn_dht_bucket_refresh_failures_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum(rate(decdn_dht_lookup_dial_timeouts_total{instance=~\"$node\"}[$__rate_interval]))", "refId": "D", + "legendFormat": "lookup dial timeouts" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_bucket_refresh_failures_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "E", "legendFormat": "bucket refresh failures" }, { @@ -3431,7 +3440,7 @@ "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(decdn_dht_bootstrap_find_node_failures_total{instance=~\"$node\"}[$__rate_interval]))", - "refId": "E", + "refId": "F", "legendFormat": "bootstrap failures" } ], @@ -3730,7 +3739,7 @@ "y": 92 }, "id": 48, - "description": "p50, p95 and p99 duration of one probe collection window on the cache-miss path, from the start of the concurrent probes (dials included) to the end of collection (ADR 001 §Probe response collection). Collection stops early once enough holders answer, or when every probe resolves; the 500 ms probe timeout bounds each probe. A p95 near 0.5 s means rounds wait on a slow or unreachable peer before enough holders answer. A probe-cache hit sends no probe and is not in these series." + "description": "p50, p95 and p99 duration of one probe collection window on the cache-miss path, from the start of the concurrent probes (dials included) to the end of collection (ADR 001 §Probe response collection). Collection stops early once enough holders answer, or when every probe resolves; the 1 s probe timeout bounds each probe. A p95 near 1 s means rounds wait on a slow or unreachable peer before enough holders answer. A probe-cache hit sends no probe and is not in these series." }, { "collapsed": false, diff --git a/monitoring/decdn-node/prometheus-alerts.yml b/monitoring/decdn-node/prometheus-alerts.yml index d9e73c4..63d9cd9 100644 --- a/monitoring/decdn-node/prometheus-alerts.yml +++ b/monitoring/decdn-node/prometheus-alerts.yml @@ -846,12 +846,37 @@ groups: Redemption steps keep failing: a redeemMany that reverted or was refused at send, or a lane the redeemer could not plan or load. An unconfirmed receipt does not count here; DecdnOnchainTxUnconfirmed - covers it. + covers it. Nor does a send that failed with no tx hash (decdn after + v0.2); DecdnRedemptionMaybeBroadcast covers it. The claims stay and retry, but unredeemed revenue accrues and a closing pool's grace window can run out. Read the redeemMany warn! lines (each carries the tx hash when one was issued) and check the RPC provider and the operator key's gas balance. + # The counter ships in the first decdn release after v0.2; on an older + # node the series is absent and the rule stays silent. Not folded into + # DecdnOnchainTxUnconfirmed (for: 0m): a node's seller and buyer share one + # key, so a harmless nonce collision between them gives these same answers. + - alert: DecdnRedemptionMaybeBroadcast + expr: increase(decdn_onchain_tx_maybe_broadcast_total[30m]) > 2 + for: 30m + labels: + component: payments + severity: warning + annotations: + summary: "redeemMany sends failing with no tx hash on {{ $labels.instance }}" + description: | + redeemMany sends keep failing with no tx hash: the request timed + out, was reset or got an HTTP 5xx, or the RPC answered `already + known`, `nonce too low` or `replacement transaction underpriced`. + Each may still have broadcast; the claims stay, and the next sweep + skips lanes the chain shows paid. A steady rate stops redemptions + while unredeemed revenue accrues and a closing pool's grace window + runs. Check the RPC's eth_sendRawTransaction health (reads can + still work), compare `cast nonce --block pending` with + `--block latest` for a lagging backend, and read the + "redeemMany send failed with no tx hash" warn lines. + - alert: DecdnWatermarkReconcileFailing expr: increase(decdn_redemption_reconcile_failures_total[30m]) > 2 for: 30m @@ -1027,10 +1052,15 @@ groups: operator address owns on chain. # The gauge (decdn v0.2) is 1 while the node holds its top-up slot for a - # topUp whose outcome the chain has not shown: the submit failed in - # transport or the receipt could not be read, and the transaction may - # still mine, so no other topUp goes out meanwhile. A healthy settle - # clears it within a few blocks; 15m of hold is a stuck chain path. + # topUp whose outcome the chain has not shown: the submit got no tx hash + # or the receipt could not be read, and the transaction may still mine, + # so no other topUp goes out meanwhile. A submit gets no tx hash when the + # transport fails, or when the RPC node already holds the transaction + # (`already known`), counts its nonce as spent (`nonce too low`) or holds + # another transaction of this key at that nonce (`replacement transaction + # underpriced`), as when a seller redemption took the topUp's nonce. A + # healthy settle clears it within a few blocks; 15m of hold is a stuck + # chain path. - alert: DecdnBuyerTopupUnresolved expr: decdn_buyer_topup_unresolved == 1 for: 15m diff --git a/monitoring/decdn-node/prometheus-alerts_test.yml b/monitoring/decdn-node/prometheus-alerts_test.yml index 8a8bb76..0541d4c 100644 --- a/monitoring/decdn-node/prometheus-alerts_test.yml +++ b/monitoring/decdn-node/prometheus-alerts_test.yml @@ -149,3 +149,22 @@ tests: exp_alerts: - exp_labels: {job: decdn-node, instance: a, component: payments, severity: warning} - exp_labels: {job: decdn-node, instance: c, component: payments, severity: warning} + + # Maybe-broadcast. a adds one at 20m and one at 60m, never more than one in a + # 30m window, and never fires. b adds one a minute, goes pending once its 30m + # increase passes 2 and fires after the 30m for. A node older than the counter + # has no series, and nothing fires for it. + - interval: 1m + input_series: + - series: 'decdn_onchain_tx_maybe_broadcast_total{job="decdn-node", instance="a"}' + values: '0x19 1x39 2x20' + - series: 'decdn_onchain_tx_maybe_broadcast_total{job="decdn-node", instance="b"}' + values: '0+1x70' + alert_rule_test: + - eval_time: 10m + alertname: DecdnRedemptionMaybeBroadcast + exp_alerts: [] + - eval_time: 70m + alertname: DecdnRedemptionMaybeBroadcast + exp_alerts: + - exp_labels: {job: decdn-node, instance: b, component: payments, severity: warning} From d693a92b3aee723a6b0fecffd3ea3c10ab10198b Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Wed, 21 Oct 2026 18:31:25 +0300 Subject: [PATCH 2/2] fix(monitoring): address review of the batch fix - Buyer-pool texts and the topUp rule comment: the three no-hash answers apply from the first decdn release after v0.2.0 (decdn#2560/#2563/#2571 are not in v0.2.0); the panel names the seller-redemption nonce case. - Drop the duplicate dial-timeout series from "DHT republish" and describe it in "DHT discovery and publishing". - Redemption failures panel, outcome panel and the grace-close alert point at maybe broadcast / DecdnRedemptionMaybeBroadcast; the rule's description names the seller/buyer collision and the extrapolated threshold. - Stronger promtool case: pins the for, the threshold on a running counter and a one-off burst; tested-rule lists updated; SKILL.md rule 4 carves out the unreleased counter until the pin leaves 0.2. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/grafana-dashboards/SKILL.md | 11 +++-- monitoring/README.md | 2 +- monitoring/decdn-node/dashboard-chain.json | 6 +-- monitoring/decdn-node/dashboard-node.json | 11 +---- monitoring/decdn-node/prometheus-alerts.yml | 33 +++++++++------ .../decdn-node/prometheus-alerts_test.yml | 41 +++++++++++++++---- 6 files changed, 67 insertions(+), 37 deletions(-) diff --git a/.claude/skills/grafana-dashboards/SKILL.md b/.claude/skills/grafana-dashboards/SKILL.md index 03a7ed1..708edca 100644 --- a/.claude/skills/grafana-dashboards/SKILL.md +++ b/.claude/skills/grafana-dashboards/SKILL.md @@ -40,8 +40,8 @@ the iroh relay's in `monitoring/iroh-relay/`): - `prometheus-alerts.yml` — three groups: `decdn-slash-safety`, `decdn-liveness`, `decdn-delivery`. Every rule carries a `component` label. Only rules with a matching section in upstream's `docs/runbook.md` (20 of 53) also carry a `runbook_url`. - `DecdnNodeDown` and `DecdnCapacityBondRegistryResyncFailing` have promtool unit - tests in `prometheus-alerts_test.yml` (the chart's `.helmignore` keeps that file + `DecdnNodeDown`, `DecdnCapacityBondRegistryResyncFailing`, `DecdnBuyerTopupUnresolved` + and `DecdnRedemptionMaybeBroadcast` have promtool unit tests in `prometheus-alerts_test.yml` (the chart's `.helmignore` keeps that file out of the package): keep them in step. - `monitoring/sponsord/` — the onboarding sponsor's own pair: `dashboard-sponsord.json` (uid @@ -123,7 +123,12 @@ region) emits nothing — read a stray hit with that in mind. `decdn_health` is Do not write a `by (reason)` query against a series that has no such label. 3. A name that resolves can still sit at a permanent zero because nothing increments it. 4. A panel or alert that names a *new* metric needs the metric in upstream - `crates/node/src/metrics.rs` first, in a released or pinned `decdn-node`. + `crates/node/src/metrics.rs` first, in a released or pinned `decdn-node`. The one + exception: `decdn_onchain_tx_maybe_broadcast_total` (decdn/decdn#2574) lands ahead of + the release after v0.2.0, because its absence keeps `DecdnRedemptionMaybeBroadcast` + silent and the hole it covers would otherwise go unalerted (#170). The name check + prints it against a v0.2 node; drop this exception once `decdn_node_version` moves + past 0.2. 5. The two "Unattributed stream failures" panels (overview and delivery) subtract every inbound failure reason from `decdn_streams_failed_total{direction="inbound"}`. When upstream adds a reason to `INBOUND_FAILURE_REASONS` in `crates/node/src/metrics.rs`, diff --git a/monitoring/README.md b/monitoring/README.md index a4dc2e1..2df2700 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -15,7 +15,7 @@ this repo's MIT license. sponsord's were written here. | `decdn-node/dashboard-chain.json` | Chain, payments and slash safety (`uid: decdn-chain`): watcher liveness, chain RPC, registries, payments. | | `decdn-node/dashboard-node.json` | Single-node drilldown (`uid: decdn-node`): host, process, iroh transport, DHT and probe, logs, traces. | | `decdn-node/prometheus-alerts.yml` | Rule groups `decdn-slash-safety`, `decdn-liveness`, `decdn-delivery`. A rule with a matching runbook section carries a `runbook_url` into upstream's `docs/runbook.md`. | -| `decdn-node/prometheus-alerts_test.yml` | promtool unit tests for `DecdnNodeDown` and `DecdnCapacityBondRegistryResyncFailing` (run by `make lint-helm`; the chart's `.helmignore` keeps them out of the package). | +| `decdn-node/prometheus-alerts_test.yml` | promtool unit tests for `DecdnNodeDown`, `DecdnCapacityBondRegistryResyncFailing`, `DecdnBuyerTopupUnresolved` and `DecdnRedemptionMaybeBroadcast` (run by `make lint-helm`; the chart's `.helmignore` keeps them out of the package). | | `sponsord/` | The onboarding sponsor's dashboard and alert rules ([below](#sponsord)). Not rendered by the chart. | | `iroh-relay/` | The iroh relay's dashboard and alert rules ([below](#iroh-relay)). Not rendered by the chart. | diff --git a/monitoring/decdn-node/dashboard-chain.json b/monitoring/decdn-node/dashboard-chain.json index 91aacef..ff782ae 100644 --- a/monitoring/decdn-node/dashboard-chain.json +++ b/monitoring/decdn-node/dashboard-chain.json @@ -2087,7 +2087,7 @@ }, "title": "Redemption failures", "type": "timeseries", - "description": "Redemption steps that failed: a redeemMany that reverted or was refused at send, or a lane the redeemer could not plan. An unconfirmed receipt does not count here; the On-chain transaction outcomes panel shows it as receipt failed or timeout.", + "description": "Redemption steps that failed: a redeemMany that reverted or was refused at send, or a lane the redeemer could not plan or load. An unconfirmed receipt does not count here; the On-chain transaction outcomes panel shows it as receipt failed or timeout. From the first decdn release after v0.2.0, neither does a send that failed with no tx hash; that panel shows it as maybe broadcast.", "targets": [ { "datasource": { @@ -2438,7 +2438,7 @@ }, "title": "On-chain transaction outcomes", "type": "timeseries", - "description": "Every node transaction counts once by outcome. A receipt wait that fails or times out falls back to fetching the receipt by hash, and a receipt found there counts as landed or reverted. receipt failed and timeout are the transactions still unconfirmed after that fetch; they can still mine, and their tx hash is in the warn! line. maybe broadcast is a send that failed with no tx hash (a timeout, a reset, an HTTP 5xx, or an `already known`, `nonce too low` or `replacement transaction underpriced` answer): it can be in a mempool or have mined, and the claims stay for the next sweep (decdn after v0.2; DecdnRedemptionMaybeBroadcast). receipt recovered is not an outcome: it counts the waits the fetch by hash resolved, which points at a lagging RPC provider.", + "description": "Every settlement redeemMany counts once by outcome (openPool, topUp and reclaim do not count here). A receipt wait that fails or times out falls back to fetching the receipt by hash, and a receipt found there counts as landed or reverted. receipt failed and timeout are the transactions still unconfirmed after that fetch; they can still mine, and their tx hash is in the warn! line. maybe broadcast is a send that failed with no tx hash (a timeout, a reset, an HTTP 5xx, or an `already known`, `nonce too low` or `replacement transaction underpriced` answer): it can be in a mempool or have mined, and the claims stay for the next sweep; or it collided at one nonce with this node's buyer service, which shares the key, and broadcast nothing. It is counted from the first decdn release after v0.2.0, and DecdnRedemptionMaybeBroadcast alerts on it. receipt recovered is not an outcome: it counts the waits the fetch by hash resolved, which points at a lagging RPC provider.", "targets": [ { "datasource": { @@ -3175,7 +3175,7 @@ "y": 83 }, "id": 38, - "description": "1 while this node holds its slot for an openPool or a topUp whose outcome the chain has not shown (the topUp hold is decdn v0.2): the submit got no tx hash or the receipt could not be read, and the transaction may still mine. A submit gets no tx hash when the transport fails, or when the RPC node already holds the transaction (`already known`), counts its nonce as spent (`nonce too low`) or holds another transaction of this key at that nonce (`replacement transaction underpriced`); each can answer a gateway that retried a send another backend accepted. Meanwhile the node sends no other transaction of that kind until a receipt arrives, the account's nonce passes it, or a re-send shows the pool closed. A hold that stays means the RPC cannot answer, a pending transaction holds the nonce or an earlier one, or every re-send at that nonce fails (the wallet cannot fund the amount or the gas); the daemon's repeating warning names the last failure in its `last` field. Do not restart a node holding a topUp slot: the restart forgets the hold, and the node may escrow a second topUp into its own pool. DecdnBuyerTopupUnresolved fires after 15m." + "description": "1 while this node holds its slot for an openPool or a topUp whose outcome the chain has not shown (the topUp hold is decdn v0.2): the submit got no tx hash or the receipt could not be read, and the transaction may still mine. A submit gets no tx hash when the transport fails and, from the first decdn release after v0.2.0, also when the RPC node already holds the transaction (`already known`), counts its nonce as spent (`nonce too low`) or holds another transaction of this key at that nonce (`replacement transaction underpriced`). Each of those can come back when a gateway retries a send that another backend already accepted, or when this node's own seller redemption took that nonce; the node then settles the nonce against the chain, and the hold ends once the redemption mines. Meanwhile the node sends no other transaction of that kind until a receipt arrives, the account's nonce passes it, or a re-send shows the pool closed. A hold that stays means the RPC cannot answer, a pending transaction holds the nonce or an earlier one, or every re-send at that nonce fails (the wallet cannot fund the amount or the gas); the daemon's repeating warning names the last failure in its `last` field. Do not restart a node holding a topUp slot: the restart forgets the hold, and the node may escrow a second topUp into its own pool. DecdnBuyerTopupUnresolved fires after 15m." }, { "datasource": { diff --git a/monitoring/decdn-node/dashboard-node.json b/monitoring/decdn-node/dashboard-node.json index b223eca..0403ae7 100644 --- a/monitoring/decdn-node/dashboard-node.json +++ b/monitoring/decdn-node/dashboard-node.json @@ -2776,15 +2776,6 @@ "expr": "sum(rate(decdn_dht_lookup_round_ceiling_total{instance=~\"$node\"}[$__rate_interval]))", "refId": "H", "legendFormat": "lookup round ceiling" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(decdn_dht_lookup_dial_timeouts_total{instance=~\"$node\"}[$__rate_interval]))", - "refId": "I", - "legendFormat": "lookup dial timeouts" } ], "fieldConfig": { @@ -3525,7 +3516,7 @@ "y": 85 }, "id": 45, - "description": "Lookups this node ran and STORE records peers accepted from it. A round timeout is a lookup round whose peers did not all answer in time. A refresh failure is a FindNode error, an invalid peer key or a panicked refresh task; a bootstrap failure is a FindNode against a seed that failed." + "description": "Lookups this node ran and STORE records peers accepted from it. A round timeout is a lookup round whose peers did not all answer in time. A dial timeout is a lookup FIND_VALUE RPC whose QUIC dial missed the 2 s dial budget; the lookup then skips that peer for a cooldown, so a steady rate of about one per cooldown names an offline bonded peer, and round timeouts should stay near zero (ADR 022 § Routing Table). A refresh failure is a FindNode error, an invalid peer key or a panicked refresh task; a bootstrap failure is a FindNode against a seed that failed." }, { "datasource": { diff --git a/monitoring/decdn-node/prometheus-alerts.yml b/monitoring/decdn-node/prometheus-alerts.yml index 63d9cd9..d402f4b 100644 --- a/monitoring/decdn-node/prometheus-alerts.yml +++ b/monitoring/decdn-node/prometheus-alerts.yml @@ -846,17 +846,19 @@ groups: Redemption steps keep failing: a redeemMany that reverted or was refused at send, or a lane the redeemer could not plan or load. An unconfirmed receipt does not count here; DecdnOnchainTxUnconfirmed - covers it. Nor does a send that failed with no tx hash (decdn after - v0.2); DecdnRedemptionMaybeBroadcast covers it. + covers it. From the first decdn release after v0.2.0, nor does a send + that failed with no tx hash; DecdnRedemptionMaybeBroadcast covers it. The claims stay and retry, but unredeemed revenue accrues and a closing pool's grace window can run out. Read the redeemMany warn! lines (each carries the tx hash when one was issued) and check the RPC provider and the operator key's gas balance. - # The counter ships in the first decdn release after v0.2; on an older + # The counter ships in the first decdn release after v0.2.0; on an older # node the series is absent and the rule stays silent. Not folded into # DecdnOnchainTxUnconfirmed (for: 0m): a node's seller and buyer share one # key, so a harmless nonce collision between them gives these same answers. + # As with DecdnRedemptionFailing, increase() extrapolation reads two events + # in the window at about 2.05, so two failed sends per ~30m, sustained, fire. - alert: DecdnRedemptionMaybeBroadcast expr: increase(decdn_onchain_tx_maybe_broadcast_total[30m]) > 2 for: 30m @@ -870,12 +872,15 @@ groups: out, was reset or got an HTTP 5xx, or the RPC answered `already known`, `nonce too low` or `replacement transaction underpriced`. Each may still have broadcast; the claims stay, and the next sweep - skips lanes the chain shows paid. A steady rate stops redemptions - while unredeemed revenue accrues and a closing pool's grace window - runs. Check the RPC's eth_sendRawTransaction health (reads can - still work), compare `cast nonce --block pending` with - `--block latest` for a lagging backend, and read the - "redeemMany send failed with no tx hash" warn lines. + skips lanes the chain shows paid. Or this node's seller and buyer + services, which share one key, collided at a nonce, and nothing + was broadcast. If decdn_onchain_tx_landed_total is flat and + decdn_unredeemed_usdc keeps rising, redemptions are not landing and + a closing pool's grace window can run out. Check the RPC's + eth_sendRawTransaction health (reads can still work), compare + `cast nonce --block pending` with `--block latest` for + a lagging backend, and read the "redeemMany send failed with no tx + hash" warn lines. - alert: DecdnWatermarkReconcileFailing expr: increase(decdn_redemption_reconcile_failures_total[30m]) > 2 @@ -996,7 +1001,8 @@ groups: vouchers on it. The redeemer must land them before the grace window ends or the revenue is lost. Check that redeemMany is landing (decdn_onchain_tx_landed_total, DecdnRedemptionFailing, - DecdnOnchainTxUnconfirmed) and that the operator key has gas. + DecdnRedemptionMaybeBroadcast, DecdnOnchainTxUnconfirmed) and that + the operator key has gas. - alert: DecdnBuyerWalletUnfunded # Scoped on `> 0`, not on the series existing: every metric is registered @@ -1055,9 +1061,10 @@ groups: # topUp whose outcome the chain has not shown: the submit got no tx hash # or the receipt could not be read, and the transaction may still mine, # so no other topUp goes out meanwhile. A submit gets no tx hash when the - # transport fails, or when the RPC node already holds the transaction - # (`already known`), counts its nonce as spent (`nonce too low`) or holds - # another transaction of this key at that nonce (`replacement transaction + # transport fails and, from the first decdn release after v0.2.0, also + # when the RPC node already holds the transaction (`already known`), + # counts its nonce as spent (`nonce too low`) or holds another + # transaction of this key at that nonce (`replacement transaction # underpriced`), as when a seller redemption took the topUp's nonce. A # healthy settle clears it within a few blocks; 15m of hold is a stuck # chain path. diff --git a/monitoring/decdn-node/prometheus-alerts_test.yml b/monitoring/decdn-node/prometheus-alerts_test.yml index 0541d4c..a3237ea 100644 --- a/monitoring/decdn-node/prometheus-alerts_test.yml +++ b/monitoring/decdn-node/prometheus-alerts_test.yml @@ -1,8 +1,9 @@ # promtool unit tests for prometheus-alerts.yml beside it, run by the chart's # tests/render-test.sh (`make lint-helm`). DecdnNodeDown's absence arm only shows -# itself on a series that stops, and DecdnCapacityBondRegistryResyncFailing's -# threshold only on `increase` extrapolation, neither of which promtool check can -# catch. The test clock starts at 0. The chart's .helmignore keeps this file out of +# itself on a series that stops, and the thresholds of +# DecdnCapacityBondRegistryResyncFailing and DecdnRedemptionMaybeBroadcast only on +# `increase` extrapolation, none of which promtool check can catch; +# DecdnBuyerTopupUnresolved pins its for. The test clock starts at 0. The chart's .helmignore keeps this file out of # the package. # # It does not run in place: render-test.sh copies it into a scratch directory as @@ -151,20 +152,46 @@ tests: - exp_labels: {job: decdn-node, instance: c, component: payments, severity: warning} # Maybe-broadcast. a adds one at 20m and one at 60m, never more than one in a - # 30m window, and never fires. b adds one a minute, goes pending once its 30m - # increase passes 2 and fires after the 30m for. A node older than the counter - # has no series, and nothing fires for it. + # 30m window, and never fires. b adds one a minute from 0: its increase reads 2.5 + # at 2m30s (no extrapolation below a zero first sample), so it goes pending then + # and fires at exactly the 30m for. c adds one every 14m on a running counter: + # two in a window read about 2.07 with extrapolation, so it goes pending at 28m + # and fires at 58m. d adds 10 at once at 20m and stays quiet: pending until the + # jump leaves the window (48m30s), short of the for, so a one-off burst of any + # size never fires. - interval: 1m input_series: - series: 'decdn_onchain_tx_maybe_broadcast_total{job="decdn-node", instance="a"}' values: '0x19 1x39 2x20' - series: 'decdn_onchain_tx_maybe_broadcast_total{job="decdn-node", instance="b"}' values: '0+1x70' + - series: 'decdn_onchain_tx_maybe_broadcast_total{job="decdn-node", instance="c"}' + values: '5x13 6x13 7x13 8x13 9x13 10x13' + - series: 'decdn_onchain_tx_maybe_broadcast_total{job="decdn-node", instance="d"}' + values: '5x19 15x60' alert_rule_test: - - eval_time: 10m + - eval_time: 32m alertname: DecdnRedemptionMaybeBroadcast exp_alerts: [] + - eval_time: 32m30s + alertname: DecdnRedemptionMaybeBroadcast + exp_alerts: + - exp_labels: {job: decdn-node, instance: b, component: payments, severity: warning} + - eval_time: 48m30s + alertname: DecdnRedemptionMaybeBroadcast + exp_alerts: + - exp_labels: {job: decdn-node, instance: b, component: payments, severity: warning} + - eval_time: 57m30s + alertname: DecdnRedemptionMaybeBroadcast + exp_alerts: + - exp_labels: {job: decdn-node, instance: b, component: payments, severity: warning} + - eval_time: 58m + alertname: DecdnRedemptionMaybeBroadcast + exp_alerts: + - exp_labels: {job: decdn-node, instance: b, component: payments, severity: warning} + - exp_labels: {job: decdn-node, instance: c, component: payments, severity: warning} - eval_time: 70m alertname: DecdnRedemptionMaybeBroadcast exp_alerts: - exp_labels: {job: decdn-node, instance: b, component: payments, severity: warning} + - exp_labels: {job: decdn-node, instance: c, component: payments, severity: warning}