feat(decdn_node): follow decdn v0.2 (self-update off, schema keys, new metrics)

AGENTS.md

@@ -142,7 +142,7 @@ test-scripts` fails while the committed file differs from a fresh render, and `m
 lint-alloy` validates and runs it.
 
 **Pinned upstream releases.** One decdn/decdn release line (`decdn_node_version`, today
-`0.1`) and one decdn/sponsord release line (`sponsord_version` = `sponsord_onramp_version`,
+`0.2`) and one decdn/sponsord release line (`sponsord_version` = `sponsord_onramp_version`,
 today `0.1`; decdn/sponsord tags its whole workspace `vX.Y.Z`, there are no per-crate
 tags) are pinned across every path. A line, `X.Y`, is what the role defaults, the
 onramp's installer pin (`sponsord_onramp_decdn_release`, `vX.Y`) and the chart's
@@ -255,7 +255,13 @@ the pinned `iroh-relay` (re-capture it when bumping `iroh_relay_version`).
   the installed binary does not know is a startup crash-loop. Two guards: the role runs
   `decdn config validate` against the real binary after templating, and the
   `molecule/schema` scenario checks the rendered key set against a committed inventory of
-  upstream field names. Re-sync both when bumping the pinned decdn version.
+  upstream field names (`gen-schema-keys.py`, which refuses a config struct its
+  `STRUCT_PATHS` does not map, so a section upstream adds fails the regeneration instead
+  of dropping out). Re-sync both when bumping the pinned decdn version. `[self_update]`
+  (decdn v0.2) is CLI-only, the daemon ignores it: the role and `decdn-compose init`
+  render `check = false`, and `decdn-compose cli` sets `DECDN_SELF_UPDATE_CHECK=0`, so a
+  managed host never hears "run `decdn self-update`", which would replace the CLI outside
+  the signed-release install and fail the role's version check.
 
 - **`ansible/roles/grafana_alloy`** — opt-in Grafana Cloud observability for that node,
   one mirrored flag (`decdn_grafana_cloud_enabled`, `false` by default) driving a

ansible/inventory/group_vars/decdn_nodes.yml

@@ -26,9 +26,10 @@
 # --- Cache sizing -----------------------------------------------------------------
 # Size to each host's disk and to the largest object you expect to serve; see
 # https://github.com/decdn/devops/blob/main/docs/requirements.md for the disk formula.
-# A node rejects any blob larger than decdn_max_blob_size_mb with BlobTooLarge, and
-# decdn_cache_size_mb must be at least that. Override per host in host_vars when
-# disks differ.
+# decdn_max_blob_size_mb caps the bytes held for one blob: an own-origin blob above it
+# is refused with BlobTooLarge, and a pull-through serves ranges of a larger one while
+# the bytes held fit. decdn_cache_size_mb must be at least that. Override per host in
+# host_vars when disks differ.
 # decdn_max_blob_size_mb: 1024      # role default
 # decdn_cache_size_mb: 10240        # role default (10 GiB)
 

ansible/inventory/group_vars/sponsord_onramp_hosts.yml

@@ -36,6 +36,6 @@
 # role's sponsord_onramp_decdn_release line (decdn/decdn, the decdn_node role's default
 # line; a host's decdn_node_version does not move it), with the SHA-256 of its
 # verified SHA256SUMS, on each deploy. To serve one exact release, set its tag and digest:
-# sponsord_onramp_decdn_release: v0.1.0
+# sponsord_onramp_decdn_release: v0.2.0
 # sponsord_onramp_decdn_sums_sha256: "…"
 # CapacityBond and SlashJudge come from sponsord_network (group_vars/sponsord_hosts.yml).

ansible/inventory/host_vars/decdn-node-1/main.yml

@@ -24,13 +24,13 @@
 # line the role pins ("X.Y": its newest release, installed by each deploy); set an
 # exact one only to hold a node on it.
 decdn_node_install_method: release
-# decdn_node_version: "0.1.0"
+# decdn_node_version: "0.2.0"
 #
 # Alternative: build on the node from a git ref (any tag, branch or SHA), as an
 # unprivileged build user with a pinned toolchain (decdn_node_version is not checked
 # in this mode):
 #     decdn_node_install_method: source
-#     decdn_node_source_ref: v0.1.0
+#     decdn_node_source_ref: v0.2.0
 # Alternative: copy binaries built on this control machine
 # (`cargo build --release -p decdn-node -p decdn-cli` in a decdn/decdn checkout):
 #     decdn_node_install_method: manual

ansible/inventory/host_vars/origin-1/main.yml

@@ -26,13 +26,13 @@
 # line the role pins ("X.Y": its newest release, installed by each deploy); set an
 # exact one only to hold a node on it.
 decdn_node_install_method: release
-# decdn_node_version: "0.1.0"
+# decdn_node_version: "0.2.0"
 #
 # Alternative: build on the node from a git ref (any tag, branch or SHA), as an
 # unprivileged build user with a pinned toolchain (decdn_node_version is not checked
 # in this mode):
 #     decdn_node_install_method: source
-#     decdn_node_source_ref: v0.1.0
+#     decdn_node_source_ref: v0.2.0
 # Alternative: copy binaries built on this control machine
 # (`cargo build --release -p decdn-node -p decdn-cli` in a decdn/decdn checkout):
 #     decdn_node_install_method: manual

ansible/molecule/default/verify.yml

@@ -230,6 +230,10 @@
               ("load_shed", shed.get("per_client_serve_cap") == 16),
               ("receipts", d.get("receipts", {}).get("retained_files") == 8),
               ("content.denied_hashes", d.get("content", {}).get("denied_hashes") == ["c" * 64]),
+              # The role default: the CLI's self-update notice is off on a managed
+              # host (a bare TOML bool, not a string).
+              ("self_update.check (role default false must emit)",
+               d.get("self_update", {}).get("check") is False),
               # Rate-limit tables render from a fixed key list: the keys converge set
               # must be present with the right types, and the ones it did NOT set must
               # be ABSENT (rendering a default would silently pin it).

ansible/molecule/schema/files/gen-schema-keys.py

@@ -29,6 +29,8 @@
     "FileConfig": None,
     "IdentityConfig": "identity",
     "ClientConfig": "client",
+    # CLI-only (the daemon ignores it): the `decdn` self-update notice knob.
+    "SelfUpdateConfig": "self_update",
     "NetworkConfig": "network",
     "DiscoveryConfig": "network.discovery",
     # Keyed by NodeId, so the concrete hop is a wildcard.
@@ -64,24 +66,31 @@
 ORIGIN_ENUM_FIELDS = ["kind", "url", "decompress", "path"]
 CREDENTIAL_FIELDS = ["source", "profile"]
 
-FIELD_RE = re.compile(r"^\s+pub\s+([a-z_0-9]+)\s*:")
-TYPE_RE = re.compile(r"^pub\s+(?:struct|enum)\s+([A-Za-z0-9_]+)")
+FIELD_RE = re.compile(r"^\s+pub\s+([a-z_0-9]+)\s*:(.*)$")
+TYPE_RE = re.compile(r"^pub\s+(struct|enum)\s+([A-Za-z0-9_]+)")
 
 
-def fields_by_struct(paths):
-    """Map every `pub struct`/`pub enum` name to its own `pub field:` names."""
-    out, current = {}, None
+def parse(paths):
+    """Map every `pub struct`/`pub enum` name to its own `pub field:` names, to the
+    type names those fields mention (`Option<SelfUpdateConfig>` -> SelfUpdateConfig),
+    and collect which of the names are structs (an enum is a leaf value or one of the
+    tagged origin shapes named above, never a section of its own)."""
+    fields, refs, structs, current = {}, {}, set(), None
     for path in paths:
         for line in Path(path).read_text().splitlines():
             type_match = TYPE_RE.match(line)
             if type_match:
-                current = type_match.group(1)
-                out.setdefault(current, [])
+                current = type_match.group(2)
+                fields.setdefault(current, [])
+                refs.setdefault(current, set())
+                if type_match.group(1) == "struct":
+                    structs.add(current)
                 continue
             field_match = FIELD_RE.match(line)
             if field_match and current:
-                out[current].append(field_match.group(1))
-    return out
+                fields[current].append(field_match.group(1))
+                refs[current].update(re.findall(r"\b[A-Z][A-Za-z0-9_]*\b", field_match.group(2)))
+    return fields, refs, structs
 
 
 def main():
@@ -97,7 +106,7 @@ def main():
     if missing:
         sys.exit(f"not a decdn checkout -- missing: {', '.join(missing)}")
 
-    by_struct = fields_by_struct(sources)
+    by_struct, refs, structs = parse(sources)
 
     unknown = sorted(
         name for name in STRUCT_PATHS if name not in by_struct
@@ -108,6 +117,26 @@ def main():
             f"(renamed or removed upstream?): {', '.join(unknown)}"
         )
 
+    # The inverse: a section upstream ADDED would otherwise drop out of the
+    # inventory in silence (v0.2's [self_update] did, until this check). Every
+    # struct a mapped struct (FileConfig included) names in a field type must be
+    # mapped itself, or be the S3 origin shape the constants above spell out.
+    handled = set(STRUCT_PATHS) | {"S3OriginConfig"}
+    unmapped = sorted(
+        {
+            ref
+            for struct in STRUCT_PATHS
+            for ref in refs[struct]
+            if ref in structs and ref not in handled
+        }
+    )
+    if unmapped:
+        sys.exit(
+            "upstream config structs that no STRUCT_PATHS entry maps (a new section?): "
+            f"{', '.join(unmapped)}. Map each to its TOML path, then decide whether "
+            "node.toml.j2 (and the chart's values.config) should render it."
+        )
+
     paths = set()
     for struct, prefix in STRUCT_PATHS.items():
         if prefix is None:

ansible/molecule/schema/files/schema-keys.txt

@@ -13,7 +13,7 @@
 # a leaf-name inventory cannot tell the two apart. See gen-schema-keys.py for
 # the struct -> path mapping and molecule/schema/README.md for why this exists.
 #
-# 156 paths.
+# 157 paths.
 blockchain.buyer_max_approve
 blockchain.buyer_working_deposit_micro_usdc
 blockchain.capacity_bond_address
@@ -170,3 +170,4 @@ security.max_concurrent_handlers
 security.max_tracked_sources
 security.per_source_burst
 security.per_source_rate_per_sec
+self_update.check

ansible/molecule/schema/verify.yml

@@ -124,7 +124,8 @@
           payment
           probe.rate_limit
           receipts
-          security'
+          security
+          self_update'
           if [ "$tables" != "$(echo "$expected" | sed 's/^ *//')" ]; then
             echo "rendered table set changed. got:" >&2; echo "$tables" >&2
             echo "expected:" >&2; echo "$expected" | sed 's/^ *//' >&2

ansible/roles/decdn_node/README.md

@@ -315,7 +315,9 @@ asserts miss. See `defaults/main.yml` for every knob's upstream default, unit an
 - **Cache sizing + eviction** — `decdn_cache_size_mb`, `decdn_disk_headroom_mb`
   (free disk the eviction driver keeps unused on the `cache_dir` volume; daemon
   default `8192`, `0` opts out of the disk clamp), `decdn_max_blob_size_mb`
-  (defaults to `cache_size_mb` upstream; must be `<=` it), `decdn_max_rate_per_mb`
+  (the bytes held for one blob: an own-origin blob above it is refused, a pull-through
+  serves ranges of a larger one while the bytes held fit; daemon default
+  `min(51200, cache_size_mb)`; must be `<=` it), `decdn_max_rate_per_mb`
   (buyer-side rate ceiling), `decdn_gc_interval_sec`, `decdn_fs_rescan_interval_sec`,
   `decdn_max_probe_holds`, `decdn_stake_lane_reserved_holds`, `decdn_pinned_hashes`
   (64-char **lowercase-hex** BLAKE3), `decdn_cache_user_agent`, the
@@ -366,6 +368,12 @@ asserts miss. See `defaults/main.yml` for every knob's upstream default, unit an
   `decdn_client_region_allowlist` (a `[client]` knob) narrows which regions this
   node discovers/probes **as a client** to a list of ISO 3166-1 alpha-2 codes;
   `[]` ⇒ no filter.
+- **`decdn` CLI self-update notice** — `decdn_self_update_check` (`[self_update]
+  check`, decdn v0.2; CLI-only, the daemon ignores it) defaults to `false`: the
+  once-a-day "a newer release exists, run `decdn self-update`" line the CLI prints
+  after a command run at a terminal is wrong advice on a host this role manages (see
+  [On-chain onboarding](#on-chain-onboarding)). `true` turns it on; `""`/`null`
+  omits the section, which an exact `decdn_node_version` below `0.2` needs.
 - **Abuse limits + load shedding** — the `decdn_security_*` family,
   `decdn_load_shed_*` (`policy` is `resource-pressure`|`always-admit`; the low-water
   serve mark must be `<=` the high), and the `decdn_dht_rate_limit` /
@@ -560,6 +568,13 @@ sudo systemd-run --pty --wait --collect -p User=decdn \
   --keystore-password-file /etc/decdn/keystore.password
 ```
 
+Never run `decdn self-update` (decdn v0.2) on a host this role manages. Under plain
+`sudo` it replaces `/usr/local/bin/decdn` outside the role's signed-release install,
+and the next deploy fails its version check instead of repairing it. The role takes
+the pinned line's newest release on every run; to move, change `decdn_node_version`.
+The CLI's daily notice suggesting it is off in the rendered `node.toml`
+(`decdn_self_update_check`), which these recipes pass as `--config`.
+
 Exiting is the reverse, in order: `decdn node deregister` (leaves the active set
 and clears the declared tier — the bond stays deposited and **fully slashable**),
 then `decdn node unbond --all` (starts the unbonding window; the node is INACTIVE
@@ -747,7 +762,9 @@ The steps:
    `ansible/molecule/schema/files/gen-schema-keys.py <checkout> >
    ansible/molecule/schema/files/schema-keys.txt` (run on a checkout of the tag).
    If the schema keys changed, re-sync `templates/node.toml.j2` (and the chart's
-   `values.config`).
+   `values.config`). The generator refuses a config struct its `STRUCT_PATHS` does
+   not map (a section upstream added): map it to its TOML path, then decide whether
+   the role renders it (v0.2's `[self_update]`, CLI-only, is rendered off).
 3. Compose's onramp installer pin: `ONRAMP_DECDN_RELEASE` (`v<version>`) and
    `ONRAMP_DECDN_SUMS_SHA256` (`sha256sum SHA256SUMS` once `SHA256SUMS.asc`
    verifies) in `compose/sponsord-onramp.env.example`. The role reads both from the
@@ -767,6 +784,13 @@ The steps:
    for a new line AGENTS.md's "Pinned upstream releases". Other docs name the pin,
    never its number. The changelog entry is the PR title (`feat`, `fix` or `chore`,
    not `ci` or `build`, which the changelogs leave out; RELEASING.md).
+8. The monitoring assets, by hand (nothing in CI checks the node's metric names):
+   diff upstream's `adr/appendix-observability.md` and the `INBOUND_FAILURE_REASONS`
+   list in `crates/node/src/metrics.rs` between the two tags, fix any renamed or
+   removed `decdn_*` name in `monitoring/decdn-node/`, add a new inbound failure
+   reason to both "Unattributed stream failures" panels, and chart or alert on new
+   series worth it (`.claude/skills/grafana-dashboards/SKILL.md`). Republish the
+   Grafana Cloud copies after the merge.
 
 `make test-scripts` fails while the role, the onramp's decdn line, the chart's
 `appVersion` and its `artifacthub.io/images` image or AGENTS.md's pinned line

ansible/roles/decdn_node/defaults/main.yml

@@ -64,7 +64,7 @@ decdn_rustup_sha256:
 # default is the line this repo is synced against; a new line also moves the
 # generated mirrors, the onramp's decdn pin and the chart's appVersion (README.md,
 # "Bumping the decdn release").
-decdn_node_version: "0.1"
+decdn_node_version: "0.2"
 # Release target triple. Derived from the host's gathered architecture (x86_64 or
 # aarch64, see vars/main/arch.yml), so arm64 hosts (Graviton, Ampere, Hetzner CAX)
 # get the aarch64 tarball with no inventory change. Needs facts: a play with
@@ -224,12 +224,16 @@ decdn_cache_size_mb: 10240      # 10 GB
 # against ANY process, not just this cache (#1930). "" => daemon dflt 8192 (8 GiB);
 # 0 opts out of the disk clamp so only cache_size_mb binds.
 decdn_disk_headroom_mb: ""
-# NOTE: this is a deliberate OVERRIDE, not the daemon default. Upstream defaults
-# max_blob_size_mb to cache_size_mb; 1 GB caps any single blob at a tenth of the
-# cache so one large object cannot evict everything else. Must be <= cache_size_mb.
-# DO NOT set 0: despite what upstream's own `config init` template says, there is
-# no zero special-case in the cache engine — the admit gate is a plain
-# `advertised > max_blob_bytes`, so 0 rejects every blob with a size hint.
+# Cap on the bytes the node HOLDS for one blob (decdn v0.2): a blob from this node's
+# own origin above it is refused (BlobTooLarge); a pull-through serves a range at any
+# offset of a larger blob while the bytes held for it fit, and stops the stream once
+# they do not (decdn_serve_stream_midstream_blob_too_large_total). A deliberate
+# OVERRIDE of the daemon default, min(51200, cache_size_mb): 1 GB caps one blob at a
+# tenth of the default cache so one large object cannot evict everything else. Must
+# be <= cache_size_mb. DO NOT set 0: despite what upstream's own `config init`
+# template says, there is no zero special-case in the cache engine — the admit gate
+# is a plain `advertised > max_blob_bytes`, so 0 refuses every own-origin blob with a
+# size hint.
 decdn_max_blob_size_mb: 1024    # 1 GB
 # Buyer-side ABSOLUTE per-MB rate ceiling for paid pulls (#1375) — what this node
 # will accept a provider to quote on a cache-miss pull, on top of the always-applied
@@ -387,6 +391,17 @@ decdn_content_denied_origins: []    # 0x publisher addresses (checksum-agnostic;
 # node considered). ISO 3166-1 alpha-2 codes, e.g. ["US", "DE"].
 decdn_client_region_allowlist: []
 
+# --- `decdn` CLI self-update notice ([self_update]) ----------------------------
+# Since decdn v0.2 the `decdn` CLI checks once a day, after a command run at a
+# terminal, for a newer signed release and prints "run `decdn self-update`". The
+# daemon ignores the section. Off here: this role owns the binaries (the newest
+# release of the pinned line, verified against the release's signed SHA256SUMS), and
+# a `decdn self-update` run as root would replace /usr/local/bin/decdn behind its
+# back and fail the next deploy's version check. Move decdn_node_version instead.
+# true turns the notice back on; "" or null omits the section (needed for an exact
+# decdn_node_version below 0.2, whose binaries refuse the key).
+decdn_self_update_check: false
+
 # --- Readiness probe (advisory) -----------------------------------------------
 # After start, the role probes http://127.0.0.1:<metrics_port>/metrics as a
 # best-effort readiness signal. A probe TIMEOUT only WARNS (it does NOT fail the

ansible/roles/decdn_node/tasks/main.yml

@@ -753,6 +753,7 @@
     - {name: decdn_node_to_node_pull_through_enabled, value: "{{ decdn_node_to_node_pull_through_enabled }}"}
     - {name: decdn_relay_foreign_namespaces, value: "{{ decdn_relay_foreign_namespaces }}"}
     - {name: decdn_circuit_breaker_enabled, value: "{{ decdn_circuit_breaker_enabled }}"}
+    - {name: decdn_self_update_check, value: "{{ decdn_self_update_check }}"}
   loop_control:
     label: "{{ item.name }}"
 
@@ -978,7 +979,8 @@
       # Origin-probe memo TTLs must be ordered negative <= fault <= positive.
       - _eff_probe_neg | int <= _eff_probe_fault | int
       - _eff_probe_fault | int <= _eff_probe_pos | int
-      # A single blob may not exceed the whole cache (0 = unlimited, so exempt).
+      # The bytes held for one blob may not exceed the whole cache (0 is exempt
+      # here; defaults/main.yml says why never to set it).
       - _eff_max_blob | int == 0 or (_eff_max_blob | int <= decdn_cache_size_mb | int)
       # Load-shed low-water mark may not exceed the high-water mark.
       - _eff_shed_low | int <= _eff_shed_high | int
@@ -988,7 +990,7 @@
       decdn_eviction_high_water_pct (dflt 90) MINUS 5 — the gap is structural;
       decdn_origin_probe_negative_ttl_sec (dflt 2) <= decdn_origin_probe_fault_ttl_sec
       (dflt 5) <= decdn_origin_probe_ttl_sec (dflt 15);
-      decdn_max_blob_size_mb must be <= decdn_cache_size_mb (0 = unlimited);
+      decdn_max_blob_size_mb must be <= decdn_cache_size_mb;
       decdn_load_shed_max_concurrent_serves_low (dflt 192) must be <=
       decdn_load_shed_max_concurrent_serves_high (dflt 256).
   vars:
@@ -1005,10 +1007,11 @@
     _eff_probe_fault: >-
       {{ decdn_origin_probe_fault_ttl_sec
          if decdn_origin_probe_fault_ttl_sec not in ['', none] else 5 }}
-    # max_blob_size_mb defaults to cache_size_mb upstream (#1685), so an unset side
-    # trivially satisfies the constraint.
+    # max_blob_size_mb defaults to min(51200, cache_size_mb) upstream, so an unset
+    # side trivially satisfies the constraint.
     _eff_max_blob: >-
-      {{ decdn_max_blob_size_mb if decdn_max_blob_size_mb not in ['', none] else decdn_cache_size_mb }}
+      {{ decdn_max_blob_size_mb if decdn_max_blob_size_mb not in ['', none]
+         else [51200, decdn_cache_size_mb | int] | min }}
     _eff_shed_high: >-
       {{ decdn_load_shed_max_concurrent_serves_high
          if decdn_load_shed_max_concurrent_serves_high not in ['', none] else 256 }}