From 79d40674f10a7b2506523c76b5ebffaac2c5fc13 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Tue, 9 Jun 2026 08:53:45 +0200 Subject: [PATCH 01/47] feat(validator-status): persist f_activation_eligibility_epoch Adds the f_activation_eligibility_epoch column to t_validator_last_status so the pre-activation queue (f_status = 0) can be split into the deposit-lifecycle states aligned with the Beacon API RFC: pending_balance (eligibility = FAR_FUTURE_EPOCH) and pending_lookahead (eligibility set). The coarse f_status (0-3) encoding is left unchanged. Migration 037 uses ADD COLUMN IF NOT EXISTS so it is idempotent on databases where the earlier (reverted) #268 migration already added the column. docs/tables.md documents the derived vocabulary, including deposit_inqueue, which is materialised downstream from t_deposit_requests because pre-registration deposits have no validator index yet. --- docs/tables.md | 19 +++++++++++++++ pkg/analyzer/process_state.go | 23 +++++++++--------- ..._add_activation_eligibility_epoch.down.sql | 3 +++ ...37_add_activation_eligibility_epoch.up.sql | 13 ++++++++++ pkg/db/validator_last_status.go | 20 +++++++++------- pkg/spec/validator_last_status.go | 24 ++++++++++--------- 6 files changed, 72 insertions(+), 30 deletions(-) create mode 100644 pkg/db/migrations/000037_add_activation_eligibility_epoch.down.sql create mode 100644 pkg/db/migrations/000037_add_activation_eligibility_epoch.up.sql diff --git a/docs/tables.md b/docs/tables.md index 081e7d03..fc327468 100644 --- a/docs/tables.md +++ b/docs/tables.md @@ -156,12 +156,31 @@ Config: `engine = MergeTree ORDER BY f_val_idx` | f_status | uint8 | status (see status table) | | f_slashed | bool | whether the validator has ever been slashed or not | | f_activation_epoch | uint64 | epoch at which the validator was activated | +| f_activation_eligibility_epoch | uint64 | epoch at which the validator became eligible for activation; `FAR_FUTURE_EPOCH` (`18446744073709551615`) means not yet eligible. Combined with `f_activation_epoch`, this splits the pre-activation queue (`f_status = 0`) into the two deposit-lifecycle states described below. | | f_withdrawal_epoch | uint64 | epoch at which the validator can withdraw funds | | f_exit_epoch | uint64 | epoch at which the validator exited the network | | f_public_key | string | public key of the validator | | f_withdrawal_prefix | uint8 | withdrawal prefix of the validator's credentials | | f_withdrawal_credentials | text | withdrawal credentials of the validator (see below) | +## Appendix: deposit-lifecycle status vocabulary + +`f_status` keeps its coarse, stable encoding (`0 in_activation_queue · 1 active · 2 exited · 3 slashed`). +Consumers that want the finer, Beacon-API-aligned deposit-lifecycle vocabulary derive it from +`f_status` + `f_activation_eligibility_epoch` (and, for pre-registration deposits, from +`t_deposit_requests`). This is the terminology proposed upstream for the `ValidatorStatus` enum +(EIP-6110/EIP-7251 deposit pipeline): + +| Derived status | Condition | Meaning | +| -------------- | --------- | ------- | +| `deposit_inqueue` | pubkey present in `t_deposit_requests`, absent from `t_validator_last_status` (not yet a registered validator) | Deposit waiting in `state.pending_deposits`; the real churn-limited queue | +| `pending_balance` | `f_status = 0` and `f_activation_eligibility_epoch = FAR_FUTURE_EPOCH` | Validator created, effective balance < 32 ETH (not yet eligible) | +| `pending_lookahead` | `f_status = 0` and `f_activation_eligibility_epoch != FAR_FUTURE_EPOCH` | Eligible; fixed `1 + MAX_SEED_LOOKAHEAD` delay (not a queue) | +| `active` / `exited` / `slashed` | `f_status` 1 / 2 / 3 | unchanged | + +`deposit_inqueue` is not a `t_validator_last_status` row because the deposit has no validator index +yet; it is materialised by downstream consumers (e.g. bindeth) joining `t_deposit_requests`. + ## Appendix: Reference for `f_withdrawal_prefix` The `f_withdrawal_prefix` column indicates the type of withdrawal credentials associated with the validator. The possible values are: diff --git a/pkg/analyzer/process_state.go b/pkg/analyzer/process_state.go index ca4e6fa9..1e0d9385 100644 --- a/pkg/analyzer/process_state.go +++ b/pkg/analyzer/process_state.go @@ -234,17 +234,18 @@ func (s *ChainAnalyzer) processValLastStatus(bundle metrics.StateMetrics) { for i, validator := range nextState.Validators { valIdx := phase0.ValidatorIndex(i) newVal := spec.ValidatorLastStatus{ - ValIdx: valIdx, - Epoch: nextState.Epoch, - CurrentBalance: nextState.Balances[valIdx], - EffectiveBalance: validator.EffectiveBalance, - CurrentStatus: nextState.GetValStatus(valIdx), - Slashed: validator.Slashed, - ActivationEpoch: validator.ActivationEpoch, - WithdrawalEpoch: validator.WithdrawableEpoch, - ExitEpoch: validator.ExitEpoch, - PublicKey: validator.PublicKey, - WithdrawalCredentials: validator.WithdrawalCredentials, + ValIdx: valIdx, + Epoch: nextState.Epoch, + CurrentBalance: nextState.Balances[valIdx], + EffectiveBalance: validator.EffectiveBalance, + CurrentStatus: nextState.GetValStatus(valIdx), + Slashed: validator.Slashed, + ActivationEpoch: validator.ActivationEpoch, + ActivationEligibilityEpoch: validator.ActivationEligibilityEpoch, + WithdrawalEpoch: validator.WithdrawableEpoch, + ExitEpoch: validator.ExitEpoch, + PublicKey: validator.PublicKey, + WithdrawalCredentials: validator.WithdrawalCredentials, } valStatusArr = append(valStatusArr, newVal) } diff --git a/pkg/db/migrations/000037_add_activation_eligibility_epoch.down.sql b/pkg/db/migrations/000037_add_activation_eligibility_epoch.down.sql new file mode 100644 index 00000000..640e19bd --- /dev/null +++ b/pkg/db/migrations/000037_add_activation_eligibility_epoch.down.sql @@ -0,0 +1,3 @@ +-- Remove f_activation_eligibility_epoch column from t_validator_last_status. +ALTER TABLE t_validator_last_status +DROP COLUMN IF EXISTS f_activation_eligibility_epoch; diff --git a/pkg/db/migrations/000037_add_activation_eligibility_epoch.up.sql b/pkg/db/migrations/000037_add_activation_eligibility_epoch.up.sql new file mode 100644 index 00000000..1592cd77 --- /dev/null +++ b/pkg/db/migrations/000037_add_activation_eligibility_epoch.up.sql @@ -0,0 +1,13 @@ +-- Add f_activation_eligibility_epoch column to t_validator_last_status. +-- Default is FAR_FUTURE_EPOCH (2^64 - 1) so historical rows that pre-date +-- this migration read as "not yet eligible" — which matches the consensus +-- spec semantics for an unset eligibility epoch. goteth rewrites every row +-- on each finalized epoch, so the default is corrected for live validators +-- on the next analyzer pass. +-- +-- IF NOT EXISTS: the column may already be present on databases where the +-- earlier #268 migration was applied before it was reverted in code; this +-- keeps the migration idempotent. ADD COLUMN with a DEFAULT is a metadata-only +-- operation in ClickHouse (no data rewrite). +ALTER TABLE t_validator_last_status +ADD COLUMN IF NOT EXISTS f_activation_eligibility_epoch UInt64 DEFAULT 18446744073709551615 AFTER f_activation_epoch; diff --git a/pkg/db/validator_last_status.go b/pkg/db/validator_last_status.go index 420008f2..02496915 100644 --- a/pkg/db/validator_last_status.go +++ b/pkg/db/validator_last_status.go @@ -17,6 +17,7 @@ var ( f_status, f_slashed, f_activation_epoch, + f_activation_eligibility_epoch, f_withdrawal_epoch, f_exit_epoch, f_public_key, @@ -36,14 +37,15 @@ func valStatusInput(validatorStatuses []spec.ValidatorLastStatus) proto.Input { f_epoch proto.ColUInt64 f_balance_eth proto.ColFloat32 f_effective_balance proto.ColUInt64 - f_status proto.ColUInt8 - f_slashed proto.ColBool - f_activation_epoch proto.ColUInt64 - f_withdrawal_epoch proto.ColUInt64 - f_exit_epoch proto.ColUInt64 - f_public_key proto.ColStr - f_withdrawal_prefix proto.ColUInt8 - f_withdrawal_credentials proto.ColStr + f_status proto.ColUInt8 + f_slashed proto.ColBool + f_activation_epoch proto.ColUInt64 + f_activation_eligibility_epoch proto.ColUInt64 + f_withdrawal_epoch proto.ColUInt64 + f_exit_epoch proto.ColUInt64 + f_public_key proto.ColStr + f_withdrawal_prefix proto.ColUInt8 + f_withdrawal_credentials proto.ColStr ) for _, status := range validatorStatuses { @@ -54,6 +56,7 @@ func valStatusInput(validatorStatuses []spec.ValidatorLastStatus) proto.Input { f_status.Append(uint8(status.CurrentStatus)) f_slashed.Append(status.Slashed) f_activation_epoch.Append(uint64(status.ActivationEpoch)) + f_activation_eligibility_epoch.Append(uint64(status.ActivationEligibilityEpoch)) f_withdrawal_epoch.Append(uint64(status.WithdrawalEpoch)) f_exit_epoch.Append(uint64(status.ExitEpoch)) f_public_key.Append(status.PublicKey.String()) @@ -69,6 +72,7 @@ func valStatusInput(validatorStatuses []spec.ValidatorLastStatus) proto.Input { {Name: "f_status", Data: f_status}, {Name: "f_slashed", Data: f_slashed}, {Name: "f_activation_epoch", Data: f_activation_epoch}, + {Name: "f_activation_eligibility_epoch", Data: f_activation_eligibility_epoch}, {Name: "f_withdrawal_epoch", Data: f_withdrawal_epoch}, {Name: "f_exit_epoch", Data: f_exit_epoch}, {Name: "f_public_key", Data: f_public_key}, diff --git a/pkg/spec/validator_last_status.go b/pkg/spec/validator_last_status.go index e4e3a202..13bee5db 100644 --- a/pkg/spec/validator_last_status.go +++ b/pkg/spec/validator_last_status.go @@ -7,17 +7,18 @@ import ( ) type ValidatorLastStatus struct { - ValIdx phase0.ValidatorIndex - Epoch phase0.Epoch - CurrentBalance phase0.Gwei - EffectiveBalance phase0.Gwei - CurrentStatus ValidatorStatus - Slashed bool - ActivationEpoch phase0.Epoch - WithdrawalEpoch phase0.Epoch - ExitEpoch phase0.Epoch - PublicKey phase0.BLSPubKey - WithdrawalCredentials []byte + ValIdx phase0.ValidatorIndex + Epoch phase0.Epoch + CurrentBalance phase0.Gwei + EffectiveBalance phase0.Gwei + CurrentStatus ValidatorStatus + Slashed bool + ActivationEpoch phase0.Epoch + ActivationEligibilityEpoch phase0.Epoch + WithdrawalEpoch phase0.Epoch + ExitEpoch phase0.Epoch + PublicKey phase0.BLSPubKey + WithdrawalCredentials []byte } func (f ValidatorLastStatus) ToArray() []interface{} { @@ -29,6 +30,7 @@ func (f ValidatorLastStatus) ToArray() []interface{} { resultArgs = append(resultArgs, f.CurrentStatus) resultArgs = append(resultArgs, f.Slashed) resultArgs = append(resultArgs, f.ActivationEpoch) + resultArgs = append(resultArgs, f.ActivationEligibilityEpoch) resultArgs = append(resultArgs, f.WithdrawalEpoch) resultArgs = append(resultArgs, f.ExitEpoch) resultArgs = append(resultArgs, f.PublicKey.String()) From 771eb30a3b9792976a2d57739fc70fdd89f5443d Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Tue, 9 Jun 2026 09:33:30 +0200 Subject: [PATCH 02/47] docs: clarify deposit-lifecycle vocabulary is proposed, not current Beacon API --- docs/tables.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/tables.md b/docs/tables.md index fc327468..5dc7d4c2 100644 --- a/docs/tables.md +++ b/docs/tables.md @@ -166,10 +166,11 @@ Config: `engine = MergeTree ORDER BY f_val_idx` ## Appendix: deposit-lifecycle status vocabulary `f_status` keeps its coarse, stable encoding (`0 in_activation_queue · 1 active · 2 exited · 3 slashed`). -Consumers that want the finer, Beacon-API-aligned deposit-lifecycle vocabulary derive it from +Consumers that want the finer deposit-lifecycle vocabulary derive it from `f_status` + `f_activation_eligibility_epoch` (and, for pre-registration deposits, from -`t_deposit_requests`). This is the terminology proposed upstream for the `ValidatorStatus` enum -(EIP-6110/EIP-7251 deposit pipeline): +`t_deposit_requests`). This is a proposed vocabulary the current Beacon API `ValidatorStatus` +enum does not provide — its `pending_initialized` / `pending_queued` predate the post-Electra +deposit pipeline (EIP-6110/EIP-7251) and there is no deposit-queue state: | Derived status | Condition | Meaning | | -------------- | --------- | ------- | From ab60df09c4c83f4737bbc02e4974c04ee639230f Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Fri, 12 Jun 2026 10:33:06 +0200 Subject: [PATCH 03/47] docs: rename pending_balance -> pending_minBalance in tables.md Align the derived deposit-lifecycle vocabulary with the agreed naming (pending_minBalance) used in the beacon-APIs proposal. --- docs/tables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tables.md b/docs/tables.md index 5dc7d4c2..c61127d7 100644 --- a/docs/tables.md +++ b/docs/tables.md @@ -175,7 +175,7 @@ deposit pipeline (EIP-6110/EIP-7251) and there is no deposit-queue state: | Derived status | Condition | Meaning | | -------------- | --------- | ------- | | `deposit_inqueue` | pubkey present in `t_deposit_requests`, absent from `t_validator_last_status` (not yet a registered validator) | Deposit waiting in `state.pending_deposits`; the real churn-limited queue | -| `pending_balance` | `f_status = 0` and `f_activation_eligibility_epoch = FAR_FUTURE_EPOCH` | Validator created, effective balance < 32 ETH (not yet eligible) | +| `pending_minBalance` | `f_status = 0` and `f_activation_eligibility_epoch = FAR_FUTURE_EPOCH` | Validator created, effective balance < 32 ETH (not yet eligible) | | `pending_lookahead` | `f_status = 0` and `f_activation_eligibility_epoch != FAR_FUTURE_EPOCH` | Eligible; fixed `1 + MAX_SEED_LOOKAHEAD` delay (not a queue) | | `active` / `exited` / `slashed` | `f_status` 1 / 2 / 3 | unchanged | From 3a2d502b722f8b343151ea0fa09408e486fa6c97 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Mon, 15 Jun 2026 14:15:12 +0200 Subject: [PATCH 04/47] fix(db): remove semicolon inside migration 000037 comment The comment 'reverted in code; this' contained a semicolon. With x-multi-statement=true the ClickHouse golang-migrate driver splits the file on ';', so the leading comment became a standalone statement and the migration failed on a fresh database with 'Empty query' (code 62), leaving the schema dirty at version 37. Replace the ';' with a comma so the file is a single statement again. --- .../migrations/000037_add_activation_eligibility_epoch.up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/db/migrations/000037_add_activation_eligibility_epoch.up.sql b/pkg/db/migrations/000037_add_activation_eligibility_epoch.up.sql index 1592cd77..11cf3171 100644 --- a/pkg/db/migrations/000037_add_activation_eligibility_epoch.up.sql +++ b/pkg/db/migrations/000037_add_activation_eligibility_epoch.up.sql @@ -6,7 +6,7 @@ -- on the next analyzer pass. -- -- IF NOT EXISTS: the column may already be present on databases where the --- earlier #268 migration was applied before it was reverted in code; this +-- earlier #268 migration was applied before it was reverted in code, this -- keeps the migration idempotent. ADD COLUMN with a DEFAULT is a metadata-only -- operation in ClickHouse (no data rewrite). ALTER TABLE t_validator_last_status From dadcd7b33a179eb73f0c07cc10a3f94fa04ed803 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Mon, 13 Jul 2026 11:52:01 +0200 Subject: [PATCH 05/47] feat(relay): add ETHGas relay to mainnet and hoodi relay lists Track blocks delivered by the ETHGas relay. Pubkeys taken from the official docs (docs.ethgas.com, Technical Integration); both data APIs verified to serve proposer_payload_delivered bid traces at head. --- pkg/relay/constants.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/relay/constants.go b/pkg/relay/constants.go index 88ad4a6f..659ecb7b 100644 --- a/pkg/relay/constants.go +++ b/pkg/relay/constants.go @@ -9,6 +9,7 @@ const ( mainnetTitanRegionalRelay string = "https://0x8c4ed5e24fe5c6ae21018437bde147693f68cda427cd1122cf20819c30eda7ed74f72dece09bb313f2a1855595ab677d@regional.titanrelay.xyz" mainnetFlashbotsRelay string = "https://0xac6e77dfe25ecd6110b8e780608cce0dab71fdd5ebea22a16c0205200f2f8e2e3ad3b71d3499c54ad14d6c21b41a37ae@boost-relay.flashbots.net" mainnetUltraSoundRelay string = "https://0xa1559ace749633b997cb3fdacffb890aeebdb0f5a3b6aaa7eeeaf1a38af0a8fe88b9e4b1f61f236d2e64d95733327a62@relay.ultrasound.money" + mainnetEthgasRelay string = "https://0x88ef3061f598101ca713d556cf757763d9be93d33c3092d3ab6334a36855b6b4a4020528dd533a62d25ea6648251e62e@relay.ethgas.com" ) var mainnetRelayList []string = []string{ @@ -20,6 +21,7 @@ var mainnetRelayList []string = []string{ mainnetTitanRegionalRelay, mainnetFlashbotsRelay, mainnetUltraSoundRelay, + mainnetEthgasRelay, } const ( @@ -44,6 +46,7 @@ const ( hoodiUltraSoundRelay string = "https://0xb1559beef7b5ba3127485bbbb090362d9f497ba64e177ee2c8e7db74746306efad687f2cf8574e38d70067d40ef136dc@relay-hoodi.ultrasound.money" hoodiTitanRelay string = "https://0xaa58208899c6105603b74396734a6263cc7d947f444f396a90f7b7d3e65d102aec7e5e5291b27e08d02c50a050825c2f@hoodi.titanrelay.xyz" hoodiAestusRelay string = "https://0x98f0ef62f00780cf8eb06701a7d22725b9437d4768bb19b363e882ae87129945ec206ec2dc16933f31d983f8225772b6@hoodi.aestus.live" + hoodiEthgasRelay string = "https://0xb20c3fe59db9c3655088839ef3d972878d182eb745afd8abb1dd2abf6c14f93cd5934ed4446a5fe1ba039e2bc0cf1011@hoodi-relay.ethgas.com" ) var hoodiRelayList []string = []string{ @@ -52,6 +55,7 @@ var hoodiRelayList []string = []string{ hoodiUltraSoundRelay, hoodiTitanRelay, hoodiAestusRelay, + hoodiEthgasRelay, } const ( From b064dfde1c50be953e951da314d680a6e6a58d51 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Wed, 22 Jul 2026 14:08:00 +0200 Subject: [PATCH 06/47] Update maintainer information in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 15819635..5b54ddae 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ If you wish to migrate your existing database, please follow [this](https://miga # Maintainers -@santi1234567 +@Zyra-V21 # Contributing From b96a86b4ed5fe2f0ee9f7293c90c9aa3e8587cf9 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Thu, 23 Jul 2026 10:55:15 +0200 Subject: [PATCH 07/47] fix(db): make pool summary aggregation consistent and refreshable - Group validators without a pool tag under a synthetic 'unknown' pool instead of dropping them, so the sum across pools always matches the network totals in t_epoch_metrics_summary (refs #277) - Guard the missed-blocks condition against ClickHouse LEFT JOIN default values, which counted validator index 0 as a missed proposal on every epoch it did not propose (refs #254) - Add RefreshPoolSummaryRange (delete-first recompute in epoch batches) and a cheap order-independent fingerprint of t_eth2_pubkeys to detect pool re-tagging --- pkg/db/pools_summary.go | 135 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 126 insertions(+), 9 deletions(-) diff --git a/pkg/db/pools_summary.go b/pkg/db/pools_summary.go index 1d55da22..8e2aff11 100644 --- a/pkg/db/pools_summary.go +++ b/pkg/db/pools_summary.go @@ -7,13 +7,30 @@ import ( "github.com/attestantio/go-eth2-client/spec/phase0" ) +const ( + // UnknownPoolName groups validators that have no entry (or an empty pool + // name) in t_eth2_pubkeys at aggregation time. Keeping them in a synthetic + // pool instead of dropping them guarantees that the sum across pools + // always matches the network totals in t_epoch_metrics_summary. + UnknownPoolName = "unknown" + + // poolSummaryRefreshBatchEpochs is the number of epochs recomputed per + // INSERT when refreshing a range. Bounds the memory used by the + // aggregation while keeping the refresh fast (a batch takes seconds). + poolSummaryRefreshBatchEpochs = 250 +) + var ( poolsTables = "t_pool_summary" - insertPoolSummary = ` - INSERT INTO %s - SELECT - t_eth2_pubkeys.f_pool_name, f_epoch, + // The missed-blocks condition re-checks that the joined proposer duty row + // really belongs to this validator and epoch: ClickHouse fills the right + // side of a non-matched LEFT JOIN with type defaults (f_val_idx = 0, + // f_proposed = false), which validator index 0 satisfies spuriously and + // gets a fake missed block on every epoch it does not propose. + poolSummarySelect = ` + SELECT + if(t_eth2_pubkeys.f_pool_name = '', '%s', t_eth2_pubkeys.f_pool_name) as f_pool_name, f_epoch, SUM(CASE WHEN (f_reward <= f_max_reward) THEN f_reward ELSE 0 END) as aggregated_rewards, SUM(CASE WHEN (f_reward <= f_max_reward) THEN f_max_reward ELSE 0 END) as aggregated_max_rewards, SUM(f_effective_balance) as aggregated_effective_balance, @@ -25,7 +42,9 @@ var ( COUNT(*) as count_expected_attestations, SUM(CASE WHEN f_attestation_included = TRUE THEN 1 ELSE 0 END) as count_included_attestations, SUM(CASE WHEN t_proposer_duties.f_proposed = TRUE THEN 1 ELSE 0 END) as proposed_blocks_performance, - SUM(CASE WHEN t_proposer_duties.f_proposed = FALSE and t_validator_rewards_summary.f_val_idx = t_proposer_duties.f_val_idx THEN 1 ELSE 0 END) as missed_blocks_performance, + SUM(CASE WHEN t_proposer_duties.f_proposed = FALSE + AND t_validator_rewards_summary.f_val_idx = t_proposer_duties.f_val_idx + AND t_validator_rewards_summary.f_epoch = toUInt64(t_proposer_duties.f_proposer_slot/32) THEN 1 ELSE 0 END) as missed_blocks_performance, count(distinct(t_validator_rewards_summary.f_val_idx)) as number_active_vals, count(CASE WHEN f_withdrawal_prefix = 2 THEN 1 ELSE null END) as number_compounding_vals, AVG(f_inclusion_delay) as avg_inclusion_delay @@ -33,15 +52,31 @@ var ( LEFT JOIN t_eth2_pubkeys final ON t_validator_rewards_summary.f_val_idx = t_eth2_pubkeys.f_val_idx LEFT JOIN t_proposer_duties final - ON t_validator_rewards_summary.f_val_idx = t_proposer_duties.f_val_idx + ON t_validator_rewards_summary.f_val_idx = t_proposer_duties.f_val_idx AND t_validator_rewards_summary.f_epoch = toUInt64(t_proposer_duties.f_proposer_slot/32) - WHERE f_epoch = $1 AND f_status = 1 AND f_pool_name != '' - GROUP BY t_eth2_pubkeys.f_pool_name, f_epoch` + WHERE %s AND f_status = 1 + GROUP BY if(t_eth2_pubkeys.f_pool_name = '', '%s', t_eth2_pubkeys.f_pool_name), f_epoch` + + deletePoolSummaryRangeQuery = ` + DELETE FROM %s + WHERE f_epoch >= $1 AND f_epoch <= $2; + ` + + // Order-independent fingerprint of the validator->pool mapping, used to + // detect pool re-tagging so recent pool summaries can be refreshed. + selectPoolsFingerprintQuery = ` + SELECT concat(toString(count()), '-', toString(sum(cityHash64(f_val_idx, f_pool_name)))) as f_fingerprint + FROM t_eth2_pubkeys FINAL` ) +func insertPoolSummaryQuery(whereClause string) string { + selectStmt := fmt.Sprintf(poolSummarySelect, UnknownPoolName, whereClause, UnknownPoolName) + return fmt.Sprintf("INSERT INTO %s%s", poolsTables, selectStmt) +} + func (p *DBService) InsertPoolSummary(epoch phase0.Epoch) error { - query := fmt.Sprintf(insertPoolSummary, poolsTables) + query := insertPoolSummaryQuery("f_epoch = $1") var err error startTime := time.Now() @@ -56,3 +91,85 @@ func (p *DBService) InsertPoolSummary(epoch phase0.Epoch) error { return err } + +func (p *DBService) insertPoolSummaryRange(fromEpoch phase0.Epoch, toEpoch phase0.Epoch) error { + + query := insertPoolSummaryQuery("f_epoch >= $1 AND f_epoch <= $2") + var err error + + p.highMu.Lock() + err = p.highLevelClient.Exec(p.ctx, query, fromEpoch, toEpoch) + p.highMu.Unlock() + + return err +} + +func (p *DBService) DeletePoolSummaryRange(fromEpoch phase0.Epoch, toEpoch phase0.Epoch) error { + + deleteObj := DeletableObject{ + query: deletePoolSummaryRangeQuery, + table: poolsTables, + args: []any{fromEpoch, toEpoch}, + } + + err := p.Delete(deleteObj) + if err != nil { + log.Errorf("error deleting pool summaries: %s", err.Error()) + } + + return err +} + +// RefreshPoolSummaryRange recomputes t_pool_summary for [fromEpoch, toEpoch] +// from the current contents of t_validator_rewards_summary and +// t_eth2_pubkeys. Existing rows in the range are deleted first: the table is +// a ReplacingMergeTree keyed on (f_epoch, f_pool_name), so re-inserting after +// a pool was renamed or split would leave ghost rows under the old pool name +// and double count its validators. +func (p *DBService) RefreshPoolSummaryRange(fromEpoch phase0.Epoch, toEpoch phase0.Epoch) error { + + if toEpoch < fromEpoch { + return fmt.Errorf("invalid pool summary refresh range: %d-%d", fromEpoch, toEpoch) + } + + startTime := time.Now() + + err := p.DeletePoolSummaryRange(fromEpoch, toEpoch) + if err != nil { + return err + } + + for batchStart := fromEpoch; batchStart <= toEpoch; batchStart += poolSummaryRefreshBatchEpochs { + batchEnd := batchStart + poolSummaryRefreshBatchEpochs - 1 + if batchEnd > toEpoch { + batchEnd = toEpoch + } + err = p.insertPoolSummaryRange(batchStart, batchEnd) + if err != nil { + return fmt.Errorf("refreshing pool summaries %d-%d: %w", batchStart, batchEnd, err) + } + log.Debugf("pool summaries refreshed for epochs %d-%d", batchStart, batchEnd) + } + + log.Infof("pool summaries refreshed for epochs %d-%d, %f seconds", + fromEpoch, toEpoch, time.Since(startTime).Seconds()) + + return nil +} + +// RetrievePoolsFingerprint returns a cheap fingerprint of the current +// validator->pool mapping. It changes whenever t_eth2_pubkeys gains rows or +// any validator is re-tagged, and is used to trigger pool summary refreshes. +func (p *DBService) RetrievePoolsFingerprint() (string, error) { + + var dest []struct { + F_fingerprint string `ch:"f_fingerprint"` + } + + err := p.highSelect(selectPoolsFingerprintQuery, &dest) + + if len(dest) > 0 { + return dest[0].F_fingerprint, err + } + return "", err +} From d176ba63a3077c10dda186432695dc78f4e92fea Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Thu, 23 Jul 2026 10:55:15 +0200 Subject: [PATCH 08/47] feat(val-window): refresh recent pool summaries when pool tags change - New --pool-reroll-epochs flag (POOL_REROLL_EPOCHS, default 0, disabled): on finalized checkpoints, if the t_eth2_pubkeys fingerprint changed, recompute t_pool_summary for the trailing N epochs with delete-first, clamped to the rewards retention window so summaries beyond it are never wiped - New pool-summary-refill command for manual repairs after re-taggings or rewards backfills Refs #254, #277 --- cmd/pool_summary_refill_cmd.go | 71 +++++++++++++++ cmd/validator_window_cmd.go | 6 ++ main.go | 1 + pkg/config/defaults.go | 1 + pkg/config/validator_window.go | 6 ++ pkg/validator_window/window.go | 162 ++++++++++++++++++++++++--------- 6 files changed, 205 insertions(+), 42 deletions(-) create mode 100644 cmd/pool_summary_refill_cmd.go diff --git a/cmd/pool_summary_refill_cmd.go b/cmd/pool_summary_refill_cmd.go new file mode 100644 index 00000000..cd520cc3 --- /dev/null +++ b/cmd/pool_summary_refill_cmd.go @@ -0,0 +1,71 @@ +package cmd + +import ( + "fmt" + + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/migalabs/goteth/pkg/db" + "github.com/migalabs/goteth/pkg/utils" + "github.com/sirupsen/logrus" + cli "github.com/urfave/cli/v2" +) + +var PoolSummaryRefillCommand = &cli.Command{ + Name: "pool-summary-refill", + Usage: "Recomputes t_pool_summary for an epoch range from the current rewards and pool tags. Deletes the range first so pool renames do not leave ghost rows. Use after pool re-tagging or rewards backfills.", + Action: LaunchPoolSummaryRefill, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "log-level", + Usage: "Log level: debug, warn, info, error", + EnvVars: []string{"ANALYZER_LOG_LEVEL"}, + DefaultText: "info", + }, + &cli.StringFlag{ + Name: "db-url", + Usage: "Database where the pool summaries are persisted", + EnvVars: []string{"ANALYZER_DB_URL"}, + DefaultText: "clickhouse://username:password@localhost:9000/goteth?x-multi-statement=true&max_memory_usage=10000000000", + }, + &cli.Uint64Flag{ + Name: "from-epoch", + Usage: "First epoch of the range to recompute (inclusive)", + Required: true, + }, + &cli.Uint64Flag{ + Name: "to-epoch", + Usage: "Last epoch of the range to recompute (inclusive)", + Required: true, + }, + }, +} + +func LaunchPoolSummaryRefill(c *cli.Context) error { + + if c.IsSet("log-level") { + logrus.SetLevel(utils.ParseLogLevel(c.String("log-level"))) + } + + dbUrl := c.String("db-url") + if dbUrl == "" { + return fmt.Errorf("db-url is required") + } + + fromEpoch := phase0.Epoch(c.Uint64("from-epoch")) + toEpoch := phase0.Epoch(c.Uint64("to-epoch")) + if toEpoch < fromEpoch { + return fmt.Errorf("to-epoch (%d) must be greater or equal than from-epoch (%d)", toEpoch, fromEpoch) + } + + dbClient, err := db.New(c.Context, dbUrl) + if err != nil { + return err + } + err = dbClient.Connect() + if err != nil { + return err + } + defer dbClient.Finish() + + return dbClient.RefreshPoolSummaryRange(fromEpoch, toEpoch) +} diff --git a/cmd/validator_window_cmd.go b/cmd/validator_window_cmd.go index d74cddac..22b70e79 100644 --- a/cmd/validator_window_cmd.go +++ b/cmd/validator_window_cmd.go @@ -43,6 +43,12 @@ var ValidatorWindowCommand = &cli.Command{ EnvVars: []string{"DELETE_CADENCE_EPOCHS"}, DefaultText: "32", }, + &cli.IntFlag{ + Name: "pool-reroll-epochs", + Usage: "Refresh t_pool_summary for the last N epochs whenever the validator->pool mapping in t_eth2_pubkeys changes, so late-tagged validators and pool renames propagate to recent history. Set to 0 to disable.", + EnvVars: []string{"POOL_REROLL_EPOCHS"}, + DefaultText: "0", + }, &cli.StringFlag{ Name: "bn-endpoint", Usage: "Beacon node endpoint (to request the Beacon States and Blocks)", diff --git a/main.go b/main.go index 11720a26..1d22c725 100644 --- a/main.go +++ b/main.go @@ -47,6 +47,7 @@ func main() { Commands: []*cli.Command{ cmd.BlocksCommand, cmd.ValidatorWindowCommand, + cmd.PoolSummaryRefillCommand, }, } diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index ab7af74c..73da9262 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -15,6 +15,7 @@ var ( DefaultPrometheusPort int = 9080 DefaultValidatorWindowEpochs int = 100 DefaultValidatorWindowDeleteCadence int = 32 + DefaultValidatorWindowPoolReroll int = 0 DefaultMaxRequestRetries int = 3 DefaultBeaconContractAddress string = "mainnet" ) diff --git a/pkg/config/validator_window.go b/pkg/config/validator_window.go index 65483987..55b0b894 100644 --- a/pkg/config/validator_window.go +++ b/pkg/config/validator_window.go @@ -9,6 +9,7 @@ type ValidatorWindowConfig struct { DBUrl string `json:"db-url"` NumEpochs int `json:"num-epochs"` DeleteCadenceEpochs int `json:"delete-cadence-epochs"` + PoolRerollEpochs int `json:"pool-reroll-epochs"` BnEndpoint string `json:"bn-endpoint"` MaxRequestRetries int `json:"max-request-retries"` } @@ -21,6 +22,7 @@ func NewValidatorWindowConfig() *ValidatorWindowConfig { DBUrl: DefaultDBUrl, NumEpochs: DefaultValidatorWindowEpochs, DeleteCadenceEpochs: DefaultValidatorWindowDeleteCadence, + PoolRerollEpochs: DefaultValidatorWindowPoolReroll, BnEndpoint: DefaultBnEndpoint, MaxRequestRetries: DefaultMaxRequestRetries, } @@ -44,6 +46,10 @@ func (c *ValidatorWindowConfig) Apply(ctx *cli.Context) { if ctx.IsSet("delete-cadence-epochs") { c.DeleteCadenceEpochs = ctx.Int("delete-cadence-epochs") } + // pool summary reroll window (0 disables the automatic refresh) + if ctx.IsSet("pool-reroll-epochs") { + c.PoolRerollEpochs = ctx.Int("pool-reroll-epochs") + } // cl url if ctx.IsSet("bn-endpoint") { c.BnEndpoint = ctx.String("bn-endpoint") diff --git a/pkg/validator_window/window.go b/pkg/validator_window/window.go index 7dfa2b14..69c9811e 100644 --- a/pkg/validator_window/window.go +++ b/pkg/validator_window/window.go @@ -22,15 +22,17 @@ var ( ) type ValidatorWindowRunner struct { - ctx context.Context - dbClient *db.DBService // client to communicate with psql - eventsObj events.Events // object to receive signals from beacon node (needed to trigger the deletes) - stop bool // used to know if the tool should stop - windowEpochSize int // number of epochs of data to maintain in the database - deleteCadenceEpochs int // only emit DELETE once the boundary has advanced by this many epochs - lastDeletedBoundary phase0.Epoch // last boundary we successfully sent a DELETE for - deleteFiredOnce bool // becomes true after the first DELETE so the cadence check kicks in - routineSyncGroup sync.WaitGroup // to check if the routine is running + ctx context.Context + dbClient *db.DBService // client to communicate with psql + eventsObj events.Events // object to receive signals from beacon node (needed to trigger the deletes) + stop bool // used to know if the tool should stop + windowEpochSize int // number of epochs of data to maintain in the database + deleteCadenceEpochs int // only emit DELETE once the boundary has advanced by this many epochs + lastDeletedBoundary phase0.Epoch // last boundary we successfully sent a DELETE for + deleteFiredOnce bool // becomes true after the first DELETE so the cadence check kicks in + poolRerollEpochs int // refresh t_pool_summary for the last N epochs when the pool mapping changes (0 disables) + lastPoolsFingerprint string // last observed fingerprint of t_eth2_pubkeys + routineSyncGroup sync.WaitGroup // to check if the routine is running } func NewValidatorWindow( @@ -73,6 +75,7 @@ func NewValidatorWindow( eventsObj: events.NewEventsObj(pCtx, cli), windowEpochSize: iConfig.NumEpochs, deleteCadenceEpochs: cadence, + poolRerollEpochs: iConfig.PoolRerollEpochs, routineSyncGroup: sync.WaitGroup{}, }, nil } @@ -97,42 +100,12 @@ func (s *ValidatorWindowRunner) Run() { return } - if dbHeadEpoch < phase0.Epoch(s.windowEpochSize) { - log.Infof("database head epoch: %d is less than window epoch size: %d", dbHeadEpoch, s.windowEpochSize) - continue - } - windowLowerEpochBoundary := dbHeadEpoch - phase0.Epoch(s.windowEpochSize) - - // Each DELETE FROM t_validator_rewards_summary becomes a ClickHouse - // mutation that rewrites every part holding in-window epochs. On - // networks with ~1M validators that mutation takes minutes per fire; - // firing on every finalized checkpoint (~6:24 min) saturates the - // merge thread and stalls the head. Only emit the DELETE once the - // boundary has advanced by deleteCadenceEpochs since the last fire. - if s.deleteFiredOnce { - if windowLowerEpochBoundary <= s.lastDeletedBoundary { - log.Debugf("skipping delete: boundary %d not advanced past last fire %d", - windowLowerEpochBoundary, s.lastDeletedBoundary) - continue - } - advanced := windowLowerEpochBoundary - s.lastDeletedBoundary - if advanced < phase0.Epoch(s.deleteCadenceEpochs) { - log.Debugf("skipping delete: boundary advanced %d/%d epochs since last fire (boundary=%d)", - advanced, s.deleteCadenceEpochs, windowLowerEpochBoundary) - continue - } - } - - log.Infof("database head epoch: %d", dbHeadEpoch) - log.Infof("deleting validator rewards from %d epoch backwards", windowLowerEpochBoundary) - err = s.dbClient.DeleteValidatorRewardsUntil(windowLowerEpochBoundary) - if err != nil { - log.Errorf("could not delete the validators: %s", err) + if !s.runRetention(dbHeadEpoch) { s.EndProcesses() return } - s.lastDeletedBoundary = windowLowerEpochBoundary - s.deleteFiredOnce = true + + s.maybeRerollPoolSummaries(dbHeadEpoch) case <-ticker.C: if s.stop { return @@ -141,6 +114,111 @@ func (s *ValidatorWindowRunner) Run() { } } +// runRetention applies the sliding-window delete on the rewards table. +// Returns false on a fatal error that should stop the runner. +func (s *ValidatorWindowRunner) runRetention(dbHeadEpoch phase0.Epoch) bool { + + if dbHeadEpoch < phase0.Epoch(s.windowEpochSize) { + log.Infof("database head epoch: %d is less than window epoch size: %d", dbHeadEpoch, s.windowEpochSize) + return true + } + windowLowerEpochBoundary := dbHeadEpoch - phase0.Epoch(s.windowEpochSize) + + // Each DELETE FROM t_validator_rewards_summary becomes a ClickHouse + // mutation that rewrites every part holding in-window epochs. On + // networks with ~1M validators that mutation takes minutes per fire; + // firing on every finalized checkpoint (~6:24 min) saturates the + // merge thread and stalls the head. Only emit the DELETE once the + // boundary has advanced by deleteCadenceEpochs since the last fire. + if s.deleteFiredOnce { + if windowLowerEpochBoundary <= s.lastDeletedBoundary { + log.Debugf("skipping delete: boundary %d not advanced past last fire %d", + windowLowerEpochBoundary, s.lastDeletedBoundary) + return true + } + advanced := windowLowerEpochBoundary - s.lastDeletedBoundary + if advanced < phase0.Epoch(s.deleteCadenceEpochs) { + log.Debugf("skipping delete: boundary advanced %d/%d epochs since last fire (boundary=%d)", + advanced, s.deleteCadenceEpochs, windowLowerEpochBoundary) + return true + } + } + + log.Infof("database head epoch: %d", dbHeadEpoch) + log.Infof("deleting validator rewards from %d epoch backwards", windowLowerEpochBoundary) + err := s.dbClient.DeleteValidatorRewardsUntil(windowLowerEpochBoundary) + if err != nil { + log.Errorf("could not delete the validators: %s", err) + return false + } + s.lastDeletedBoundary = windowLowerEpochBoundary + s.deleteFiredOnce = true + return true +} + +// maybeRerollPoolSummaries refreshes t_pool_summary for the trailing +// poolRerollEpochs epochs whenever the validator->pool mapping changes, so +// late-tagged validators and pool renames propagate into recent history +// instead of freezing the tagging snapshot taken at ingestion time. +func (s *ValidatorWindowRunner) maybeRerollPoolSummaries(dbHeadEpoch phase0.Epoch) { + + if s.poolRerollEpochs <= 0 { + return + } + + fingerprint, err := s.dbClient.RetrievePoolsFingerprint() + if err != nil { + log.Errorf("could not retrieve pools fingerprint: %s", err) + return + } + if fingerprint == "" { + return + } + if s.lastPoolsFingerprint == "" { + // first observation after startup: record without refreshing so a + // restart alone does not trigger a reroll + s.lastPoolsFingerprint = fingerprint + return + } + if fingerprint == s.lastPoolsFingerprint { + return + } + + // leave a margin below the database head: the analyzer may still be + // writing the most recent epochs + const headMarginEpochs = 3 + if dbHeadEpoch <= headMarginEpochs { + return + } + toEpoch := dbHeadEpoch - headMarginEpochs + + fromEpoch := phase0.Epoch(0) + if toEpoch > phase0.Epoch(s.poolRerollEpochs) { + fromEpoch = toEpoch - phase0.Epoch(s.poolRerollEpochs) + 1 + } + // never refresh below the rewards retention window: those epochs have no + // source rows anymore, so delete+reinsert there would wipe their summaries + if dbHeadEpoch > phase0.Epoch(s.windowEpochSize) { + rewardsFloor := dbHeadEpoch - phase0.Epoch(s.windowEpochSize) + 1 + if fromEpoch < rewardsFloor { + fromEpoch = rewardsFloor + } + } + if toEpoch < fromEpoch { + return + } + + log.Infof("pool mapping changed, refreshing pool summaries for epochs %d-%d", fromEpoch, toEpoch) + err = s.dbClient.RefreshPoolSummaryRange(fromEpoch, toEpoch) + if err != nil { + // keep the old fingerprint so the refresh is retried on the next + // finalized checkpoint + log.Errorf("could not refresh pool summaries: %s", err) + return + } + s.lastPoolsFingerprint = fingerprint +} + func (s *ValidatorWindowRunner) Close() { s.stop = true s.routineSyncGroup.Wait() From a39dba2d670e885dae8e7b0c45be29b248291e5a Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Fri, 24 Jul 2026 11:46:05 +0200 Subject: [PATCH 09/47] fix(val-window): skip pool reroll while the pubkeys table is mid-sync The scripts that refresh t_eth2_pubkeys typically do TRUNCATE + INSERT, leaving the table empty or partial for seconds to minutes. A finalized checkpoint landing inside that window would observe a collapsed fingerprint and reroll the trailing epochs with most validators grouped under 'unknown', until the next checkpoint repaired it. Guard: if the mapping is empty, or its row count dropped by more than half since the last stable observation, skip the reroll (and keep the previous fingerprint) so it fires exactly once when the sync finishes. Refs #277 --- pkg/validator_window/window.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/pkg/validator_window/window.go b/pkg/validator_window/window.go index 69c9811e..bd3dfc78 100644 --- a/pkg/validator_window/window.go +++ b/pkg/validator_window/window.go @@ -2,6 +2,8 @@ package validatorwindow import ( "context" + "strconv" + "strings" "sync" "time" @@ -174,6 +176,23 @@ func (s *ValidatorWindowRunner) maybeRerollPoolSummaries(dbHeadEpoch phase0.Epoc if fingerprint == "" { return } + // Sync scripts commonly refresh t_eth2_pubkeys with TRUNCATE + INSERT, + // so a checkpoint landing mid-sync would observe an empty or partial + // mapping and reroll recent history with most validators as 'unknown'. + // Skip (without recording the fingerprint) until the mapping recovers. + newCount := fingerprintRowCount(fingerprint) + if newCount == 0 { + log.Warnf("pool mapping is empty, skipping pool summary reroll (pubkeys sync in progress?)") + return + } + if s.lastPoolsFingerprint != "" { + lastCount := fingerprintRowCount(s.lastPoolsFingerprint) + if lastCount > 0 && newCount < lastCount/2 { + log.Warnf("pool mapping shrank from %d to %d entries, skipping pool summary reroll until it recovers (pubkeys sync in progress?)", + lastCount, newCount) + return + } + } if s.lastPoolsFingerprint == "" { // first observation after startup: record without refreshing so a // restart alone does not trigger a reroll @@ -219,6 +238,20 @@ func (s *ValidatorWindowRunner) maybeRerollPoolSummaries(dbHeadEpoch phase0.Epoc s.lastPoolsFingerprint = fingerprint } +// fingerprintRowCount extracts the row count from a pools fingerprint, +// which is formatted as "-". +func fingerprintRowCount(fingerprint string) uint64 { + parts := strings.SplitN(fingerprint, "-", 2) + if len(parts) < 1 { + return 0 + } + count, err := strconv.ParseUint(parts[0], 10, 64) + if err != nil { + return 0 + } + return count +} + func (s *ValidatorWindowRunner) Close() { s.stop = true s.routineSyncGroup.Wait() From 55acccee02cb81a168511399ab2efffcd35c8c0e Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Fri, 24 Jul 2026 18:58:08 +0200 Subject: [PATCH 10/47] fix(blobs): record p2p arrival timings again via data_column_sidecar (PeerDAS) Backport of the goteth-v4 fix to v3. Closes #280. Blob arrival timings stopped being recorded at the Fulu fork and nobody noticed: t_blob_sidecars_events took its last row at slot 13162611 (2025-12-03) on every v3 deployment. Under PeerDAS blobs are no longer gossiped whole: they are erasure-coded into columns and each node custodies a subset, so beacon nodes emit data_column_sidecar and never blob_sidecar again. The old topic is still accepted (HTTP 200) but silently never fires, so goteth kept logging 'subscribed to blob_sidecar events' with zero errors while recording nothing. Verified on a synced Lighthouse v8.2.1: 128 data_column_sidecar events vs 0 blob_sidecar events over the same 15s window. Replaces the dead subscription with data_column_sidecar into a new t_data_column_sidecars_events table (one row per column arrival, up to 128 per slot), same shape and column names as v4 so bindeth needs no changes beyond its already-merged #352. The channel is buffered at 1024 because the handler does a blocking send and PeerDAS bursts one event per column instead of <=6 per block. Migration 000038 (renumbered from v4's 000039: v3 migrations end at 000037) renames t_blob_sidecars_events to t_blob_sidecars_events_pre_fulu rather than dropping it, preserving the pre-Fulu history. Deploy notes: - bindeth must already include PR #352 when this migration runs, otherwise its blob endpoints break on the renamed table. - Subscribing to data_column_sidecar on a pre-Fulu beacon node may fail at startup; all current deployments (mainnet, hoodi, sepolia) are post-Fulu. --- cmd/blocks_cmd.go | 2 +- docs/tables.md | 30 ++++++---- pkg/analyzer/routines.go | 6 +- pkg/db/blob_events.go | 57 ++++++++++--------- ...000038_data_column_sidecar_events.down.sql | 2 + .../000038_data_column_sidecar_events.up.sql | 31 ++++++++++ pkg/db/prometheus_metrics.go | 2 +- pkg/db/service.go | 2 +- pkg/events/blobs.go | 30 ++++++---- pkg/events/standard.go | 15 +++-- pkg/spec/blob.go | 11 +++- 11 files changed, 124 insertions(+), 64 deletions(-) create mode 100644 pkg/db/migrations/000038_data_column_sidecar_events.down.sql create mode 100644 pkg/db/migrations/000038_data_column_sidecar_events.up.sql diff --git a/cmd/blocks_cmd.go b/cmd/blocks_cmd.go index aff6fdf7..1fcf44dd 100644 --- a/cmd/blocks_cmd.go +++ b/cmd/blocks_cmd.go @@ -81,7 +81,7 @@ var BlocksCommand = &cli.Command{ }, &cli.StringFlag{ Name: "metrics", - Usage: "Metrics to be persisted to the database: epoch,block,rewards,transactions,api_rewards,blob_sidecars", + Usage: "Metrics to be persisted to the database: epoch,block,rewards,transactions,api_rewards,blob_sidecars. blob_sidecars covers blob metadata (t_blob_sidecars); blob p2p arrival timings are always recorded in head mode via the data_column_sidecar event into t_data_column_sidecars_events (the post-Fulu replacement of the dead blob_sidecar event)", EnvVars: []string{"ANALYZER_METRICS"}, DefaultText: "epoch,block", }, diff --git a/docs/tables.md b/docs/tables.md index c61127d7..825660a1 100644 --- a/docs/tables.md +++ b/docs/tables.md @@ -327,18 +327,24 @@ Config: `engine = ReplacingMergeTree ORDER BY f_slot, f_index` | f_kzg_proof | string | kzg proof of the blob | | f_ending_0s | uint64 | amount of consecutive 0s at the end of the blob bytes | -# Blob Sidecars Events (`t_blob_sidecars_events`) - -Config: `engine = ReplacingMergeTree ORDER BY f_arrival_timestamp_ms, f_blob_hash, f_slot` - -| Column Name | Type of Data | Description | | | -| ---------------------- | ------------ | ------------------------------------------------- | --- | --- | -| f_arrival_timestamp_ms | uint64 | timestamp at which goteth received the blob event | -| f_blob_hash | string | hash of the blob | -| f_slot | uint64 | slot at which the blob was sent | -| f_block_root | string | block root hash | -| f_index | uint8 | index of the blob | -| f_kzg_commitment | string | kzg commitment of the blob | +# Data Column Sidecars Events (`t_data_column_sidecars_events`) + +Config: `engine = ReplacingMergeTree PARTITION BY intDiv(f_slot, 100000) ORDER BY (f_slot, f_column_index, f_block_root)` + +P2P arrival timings for blob data. Since the Fulu fork (PeerDAS) blobs are no longer gossiped whole: they are erasure-coded into columns and each node custodies a subset, so beacon nodes emit `data_column_sidecar` instead of the now-dead `blob_sidecar` event. One row per **column** arrival (up to 128 per slot), which also exposes the reconstruction curve of each blob rather than a single arrival timestamp. + +Blob *metadata* is unaffected and still comes from the block itself (`t_blob_sidecars`). + +| Column Name | Type of Data | Description | | | +| ---------------------- | ------------- | -------------------------------------------------------------- | --- | --- | +| f_arrival_timestamp_ms | uint64 | timestamp at which goteth received the data column event | +| f_slot | uint64 | slot the data column belongs to | +| f_block_root | string | block root hash | +| f_column_index | uint16 | index of the data column (0..127) | +| f_kzg_commitments | array(string) | kzg commitments of the blobs this column carries a piece of | +| f_num_commitments | uint16 | number of commitments (denormalised from the array for queries)| + +> `t_blob_sidecars_events` is the pre-Fulu predecessor of this table (one row per whole blob). It can no longer receive rows because the `blob_sidecar` event is never emitted post-Fulu; migration `000038` renames it to `t_blob_sidecars_events_pre_fulu` to preserve its history. # Block Rewards (`t_block_rewards`) diff --git a/pkg/analyzer/routines.go b/pkg/analyzer/routines.go index ffc3dac8..1b966d20 100644 --- a/pkg/analyzer/routines.go +++ b/pkg/analyzer/routines.go @@ -78,7 +78,7 @@ func (s *ChainAnalyzer) runHead() { s.eventsObj.SubscribeToHeadEvents() s.eventsObj.SubscribeToFinalizedCheckpointEvents() s.eventsObj.SubscribeToReorgsEvents() - s.eventsObj.SubscribeToBlobSidecarsEvents() + s.eventsObj.SubscribeToDataColumnSidecarsEvents() ticker := time.NewTicker(utils.RoutineFlushTimeout) // loop over the list of slots that we need to analyze @@ -117,8 +117,8 @@ func (s *ChainAnalyzer) runHead() { s.dbClient.PersistReorgs([]v1.ChainReorgEvent{newReorg}) go s.HandleReorg(newReorg) - case newBlobSidecarEvent := <-s.eventsObj.BlobSidecarChan: - s.dbClient.PersistBlobSidecarsEvents([]spec.BlobSideCarEventWraper{newBlobSidecarEvent}) + case newDataColumnEvent := <-s.eventsObj.DataColumnSidecarChan: + s.dbClient.PersistDataColumnSidecarsEvents([]spec.DataColumnSidecarEventWrapper{newDataColumnEvent}) case <-s.ctx.Done(): log.Info("context has died, closing block requester routine") diff --git a/pkg/db/blob_events.go b/pkg/db/blob_events.go index a9d700c8..ddd09cc7 100644 --- a/pkg/db/blob_events.go +++ b/pkg/db/blob_events.go @@ -6,55 +6,60 @@ import ( ) var ( - blobEventsTable = "t_blob_sidecars_events" - insertBlobSideCarsEventsQuery = ` + dataColumnEventsTable = "t_data_column_sidecars_events" + insertDataColumnSidecarsEventsQuery = ` INSERT INTO %s ( f_arrival_timestamp_ms, - f_blob_hash, f_slot, f_block_root, - f_index, - f_kzg_commitment) + f_column_index, + f_kzg_commitments, + f_num_commitments) VALUES` ) -func blobSidecarsEventInput(blobSidecarsEvents []spec.BlobSideCarEventWraper) proto.Input { +func dataColumnSidecarsEventInput(events []spec.DataColumnSidecarEventWrapper) proto.Input { // one object per column var ( f_arrival_timestamp_ms proto.ColUInt64 - f_blob_hash proto.ColStr f_slot proto.ColUInt64 f_block_root proto.ColStr - f_index proto.ColUInt8 - f_kzg_commitment proto.ColStr + f_column_index proto.ColUInt16 + f_kzg_commitments = proto.NewArray[string](new(proto.ColStr)) + f_num_commitments proto.ColUInt16 ) - for _, blobSidecar := range blobSidecarsEvents { + for _, ev := range events { + commitments := make([]string, 0, len(ev.DataColumnSidecarEvent.KZGCommitments)) + for _, c := range ev.DataColumnSidecarEvent.KZGCommitments { + commitments = append(commitments, c.String()) + } - f_arrival_timestamp_ms.Append(uint64(blobSidecar.Timestamp.UnixMilli())) - f_blob_hash.Append(blobSidecar.BlobSidecarEvent.VersionedHash.String()) - f_slot.Append(uint64(blobSidecar.BlobSidecarEvent.Slot)) - f_block_root.Append(blobSidecar.BlobSidecarEvent.BlockRoot.String()) - f_index.Append(uint8(blobSidecar.BlobSidecarEvent.Index)) - f_kzg_commitment.Append(blobSidecar.BlobSidecarEvent.KZGCommitment.String()) + f_arrival_timestamp_ms.Append(uint64(ev.Timestamp.UnixMilli())) + f_slot.Append(uint64(ev.DataColumnSidecarEvent.Slot)) + f_block_root.Append(ev.DataColumnSidecarEvent.BlockRoot.String()) + f_column_index.Append(uint16(ev.DataColumnSidecarEvent.Index)) + f_kzg_commitments.Append(commitments) + // Denormalised so "how many blobs did this block carry" does not have to + // unnest the array on a table that takes ~128 rows per slot. + f_num_commitments.Append(uint16(len(commitments))) } return proto.Input{ - {Name: "f_arrival_timestamp_ms", Data: f_arrival_timestamp_ms}, - {Name: "f_blob_hash", Data: f_blob_hash}, {Name: "f_slot", Data: f_slot}, {Name: "f_block_root", Data: f_block_root}, - {Name: "f_index", Data: f_index}, - {Name: "f_kzg_commitment", Data: f_kzg_commitment}, + {Name: "f_column_index", Data: f_column_index}, + {Name: "f_kzg_commitments", Data: f_kzg_commitments}, + {Name: "f_num_commitments", Data: f_num_commitments}, } } -func (p *DBService) PersistBlobSidecarsEvents(data []spec.BlobSideCarEventWraper) error { - persistObj := PersistableObject[spec.BlobSideCarEventWraper]{ - input: blobSidecarsEventInput, - table: blobEventsTable, - query: insertBlobSideCarsEventsQuery, +func (p *DBService) PersistDataColumnSidecarsEvents(data []spec.DataColumnSidecarEventWrapper) error { + persistObj := PersistableObject[spec.DataColumnSidecarEventWrapper]{ + input: dataColumnSidecarsEventInput, + table: dataColumnEventsTable, + query: insertDataColumnSidecarsEventsQuery, } for _, item := range data { @@ -63,7 +68,7 @@ func (p *DBService) PersistBlobSidecarsEvents(data []spec.BlobSideCarEventWraper err := p.Persist(persistObj.ExportPersist()) if err != nil { - log.Errorf("error persisting blob events: %s", err.Error()) + log.Errorf("error persisting data column events: %s", err.Error()) } return err } diff --git a/pkg/db/migrations/000038_data_column_sidecar_events.down.sql b/pkg/db/migrations/000038_data_column_sidecar_events.down.sql new file mode 100644 index 00000000..80941637 --- /dev/null +++ b/pkg/db/migrations/000038_data_column_sidecar_events.down.sql @@ -0,0 +1,2 @@ +RENAME TABLE t_blob_sidecars_events_pre_fulu TO t_blob_sidecars_events; +DROP TABLE IF EXISTS t_data_column_sidecars_events; diff --git a/pkg/db/migrations/000038_data_column_sidecar_events.up.sql b/pkg/db/migrations/000038_data_column_sidecar_events.up.sql new file mode 100644 index 00000000..1d4376f5 --- /dev/null +++ b/pkg/db/migrations/000038_data_column_sidecar_events.up.sql @@ -0,0 +1,31 @@ +-- PeerDAS (Fulu) replaced blob-level p2p propagation with column-level propagation: +-- a node no longer receives whole blobs, it receives DataColumnSidecars. The +-- beacon API's `blob_sidecar` event stopped firing at the Fulu fork, so +-- t_blob_sidecars_events silently stopped filling (last row 2025-12-03) while +-- every other blob table kept working. This table is its replacement: one row per +-- column arrival, which is the finest-grained propagation signal available now +-- (and strictly richer than the old one: it exposes the reconstruction curve of +-- each blob, not just a single arrival timestamp). +-- +-- Note: blob *metadata* still comes from the block itself (t_blob_sidecars) and is +-- unaffected. Only the p2p arrival timing moved here. +CREATE TABLE t_data_column_sidecars_events ( + f_arrival_timestamp_ms UInt64 CODEC(DoubleDelta, ZSTD(3)), + f_slot UInt64 CODEC(DoubleDelta, ZSTD(3)), + f_block_root String CODEC(ZSTD(3)), + f_column_index UInt16 CODEC(T64, ZSTD(3)), + f_kzg_commitments Array(String) CODEC(ZSTD(3)), + f_num_commitments UInt16 CODEC(T64, ZSTD(3)) + ) + ENGINE = ReplacingMergeTree() + PARTITION BY intDiv(f_slot, 100000) + ORDER BY (f_slot, f_column_index, f_block_root); + +-- The blob_sidecar event no longer exists post-Fulu, so this table can never +-- receive another row. Its historical rows (pre-2025-12-03) are preserved by +-- renaming rather than dropping. +-- +-- Deploy note: any bindeth serving this database must already include bindeth +-- PR #352 (which re-points the blob queries to t_data_column_sidecars_events), +-- otherwise its blob endpoints break on the renamed table. +RENAME TABLE t_blob_sidecars_events TO t_blob_sidecars_events_pre_fulu; diff --git a/pkg/db/prometheus_metrics.go b/pkg/db/prometheus_metrics.go index b937ca3a..43d19a73 100644 --- a/pkg/db/prometheus_metrics.go +++ b/pkg/db/prometheus_metrics.go @@ -76,7 +76,7 @@ func (r *DBService) initMonitorMetrics() { tablesArr := []string{ blobsTable, - blobEventsTable, + dataColumnEventsTable, blockRewardsTable, blocksTable, epochsTable, diff --git a/pkg/db/service.go b/pkg/db/service.go index a08bf0a7..0ee311ae 100644 --- a/pkg/db/service.go +++ b/pkg/db/service.go @@ -145,7 +145,7 @@ type PersistableObject[ spec.Withdrawal | HeadEvent | spec.AgnosticBlobSidecar | - spec.BlobSideCarEventWraper | + spec.DataColumnSidecarEventWrapper | BlockReward | spec.AgnosticSlashing | spec.BLSToExecutionChange | diff --git a/pkg/events/blobs.go b/pkg/events/blobs.go index d6a3cc54..3be68dec 100644 --- a/pkg/events/blobs.go +++ b/pkg/events/blobs.go @@ -8,28 +8,34 @@ import ( "github.com/migalabs/goteth/pkg/spec" ) -func (e *Events) SubscribeToBlobSidecarsEvents() { - // subscribe to head event +// SubscribeToDataColumnSidecarsEvents subscribes to the p2p arrival of data columns. +// +// This replaces the old `blob_sidecar` subscription. Since the Fulu fork (PeerDAS) +// blobs are erasure-coded into columns and gossiped per column, so beacon nodes no +// longer emit `blob_sidecar` at all: the topic is still accepted (HTTP 200) but +// never fires. That is why blob arrival timings silently stopped being recorded at +// the fork while every other blob table kept working. +func (e *Events) SubscribeToDataColumnSidecarsEvents() { err := e.cli.Api.Events(e.ctx, ð2api.EventsOpts{ - Topics: []string{"blob_sidecar"}, - Handler: e.HandleBlobSidecarEvent, - }) // every reorg + Topics: []string{"data_column_sidecar"}, + Handler: e.HandleDataColumnSidecarEvent, + }) if err != nil { - log.Panicf("failed to subscribe to blob_sidecar events: %s", err) + log.Panicf("failed to subscribe to data_column_sidecar events: %s", err) } - log.Infof("subscribed to blob_sidecar events") + log.Infof("subscribed to data_column_sidecar events") } -func (e *Events) HandleBlobSidecarEvent(event *apiv1.Event) { +func (e *Events) HandleDataColumnSidecarEvent(event *apiv1.Event) { timestamp := time.Now() if event.Data == nil { return } - data := spec.BlobSideCarEventWraper{ - Timestamp: timestamp, - BlobSidecarEvent: *event.Data.(*apiv1.BlobSidecarEvent), + data := spec.DataColumnSidecarEventWrapper{ + Timestamp: timestamp, + DataColumnSidecarEvent: *event.Data.(*apiv1.DataColumnSidecarEvent), } - e.BlobSidecarChan <- data + e.DataColumnSidecarChan <- data } diff --git a/pkg/events/standard.go b/pkg/events/standard.go index 315e07f4..88ab1c46 100644 --- a/pkg/events/standard.go +++ b/pkg/events/standard.go @@ -22,10 +22,10 @@ type Events struct { SubscribedHead bool HeadChan chan db.HeadEvent - SubscribedFinalized bool - FinalizedChan chan apiv1.FinalizedCheckpointEvent - ReorgChan chan apiv1.ChainReorgEvent - BlobSidecarChan chan spec.BlobSideCarEventWraper + SubscribedFinalized bool + FinalizedChan chan apiv1.FinalizedCheckpointEvent + ReorgChan chan apiv1.ChainReorgEvent + DataColumnSidecarChan chan spec.DataColumnSidecarEventWrapper } func NewEventsObj(iCtx context.Context, iCli *clientapi.APIClient) Events { @@ -37,6 +37,11 @@ func NewEventsObj(iCtx context.Context, iCli *clientapi.APIClient) Events { SubscribedFinalized: false, FinalizedChan: make(chan apiv1.FinalizedCheckpointEvent), ReorgChan: make(chan apiv1.ChainReorgEvent), - BlobSidecarChan: make(chan spec.BlobSideCarEventWraper), + // Buffered like HeadChan: HandleDataColumnSidecarEvent does a blocking send, so + // a full channel stalls the SSE reader and with it head ingestion. Sized much + // larger than the old blob channel on purpose: PeerDAS emits one event per data + // *column*, so a single block can produce up to 128 events (NUMBER_OF_COLUMNS) + // instead of the <=6 blob events it used to, all arriving in a burst. + DataColumnSidecarChan: make(chan spec.DataColumnSidecarEventWrapper, 1024), } } diff --git a/pkg/spec/blob.go b/pkg/spec/blob.go index 79f5ec94..81c3292b 100644 --- a/pkg/spec/blob.go +++ b/pkg/spec/blob.go @@ -61,9 +61,14 @@ func (b *AgnosticBlobSidecar) GetTxHash(txs []AgnosticTransaction) { } } -type BlobSideCarEventWraper struct { - Timestamp time.Time - BlobSidecarEvent api.BlobSidecarEvent +// DataColumnSidecarEventWrapper carries a data-column arrival plus the local time +// we observed it. Since the Fulu fork (PeerDAS) blobs are no longer gossiped whole: +// they are erasure-coded into columns and each node custodies a subset, so +// `data_column_sidecar` replaced the now-dead `blob_sidecar` event as the p2p +// propagation signal. +type DataColumnSidecarEventWrapper struct { + Timestamp time.Time + DataColumnSidecarEvent api.DataColumnSidecarEvent } func KZGCommitmentToVersionedHash(input deneb.KZGCommitment) string { From 2a8aef321049a73541961b4f17cc50bcd1c46233 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Tue, 11 Aug 2026 14:45:47 +0200 Subject: [PATCH 11/47] fix(db): make migration 000038 retry-safe with IF NOT EXISTS ClickHouse DDL is not transactional: if the RENAME right after the CREATE fails, golang-migrate marks the migration dirty, and on retry after a manual force the unguarded CREATE dies with 'table already exists'. IF NOT EXISTS makes the retry reach the RENAME instead, consistent with the ADD COLUMN IF NOT EXISTS guard migration 000037 uses for the same reason. Requested in the #281 review. --- pkg/db/migrations/000038_data_column_sidecar_events.up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/db/migrations/000038_data_column_sidecar_events.up.sql b/pkg/db/migrations/000038_data_column_sidecar_events.up.sql index 1d4376f5..d6c7b32d 100644 --- a/pkg/db/migrations/000038_data_column_sidecar_events.up.sql +++ b/pkg/db/migrations/000038_data_column_sidecar_events.up.sql @@ -9,7 +9,7 @@ -- -- Note: blob *metadata* still comes from the block itself (t_blob_sidecars) and is -- unaffected. Only the p2p arrival timing moved here. -CREATE TABLE t_data_column_sidecars_events ( +CREATE TABLE IF NOT EXISTS t_data_column_sidecars_events ( f_arrival_timestamp_ms UInt64 CODEC(DoubleDelta, ZSTD(3)), f_slot UInt64 CODEC(DoubleDelta, ZSTD(3)), f_block_root String CODEC(ZSTD(3)), From 9072b7d2fc5381c2405438985cb9c2e62aae2ab9 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Tue, 11 Aug 2026 14:45:47 +0200 Subject: [PATCH 12/47] fix(events): log and skip instead of crashing on data_column_sidecar subscribe failure Requested in the #281 review: pointing goteth at a beacon node where the subscription cannot be established should not take the whole process down, it should just run without blob arrival timings. Note on scope: go-eth2-client v0.27.2 returns errors from Events() synchronously only for client-side conditions (inactive client, topic not in its supported list, missing handler); a node that rejects the topic surfaces asynchronously inside the library's own goroutine, which logs and retries every second and never hit the old Panicf. The change makes the client-side conditions degrade gracefully too. The head, finalized and reorg subscriptions keep their panic: without those topics the analyzer cannot make progress at all. --- pkg/events/blobs.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/events/blobs.go b/pkg/events/blobs.go index 3be68dec..7c0dcfc6 100644 --- a/pkg/events/blobs.go +++ b/pkg/events/blobs.go @@ -15,13 +15,20 @@ import ( // longer emit `blob_sidecar` at all: the topic is still accepted (HTTP 200) but // never fires. That is why blob arrival timings silently stopped being recorded at // the fork while every other blob table kept working. +// +// A subscription failure is logged and skipped rather than crashing the process: +// the analyzer can keep indexing everything else without this data source. Note +// that Events() only returns errors synchronously for client-side conditions; +// node-side rejections surface asynchronously inside go-eth2-client's own +// goroutine, which logs and retries every second. func (e *Events) SubscribeToDataColumnSidecarsEvents() { err := e.cli.Api.Events(e.ctx, ð2api.EventsOpts{ Topics: []string{"data_column_sidecar"}, Handler: e.HandleDataColumnSidecarEvent, }) if err != nil { - log.Panicf("failed to subscribe to data_column_sidecar events: %s", err) + log.Errorf("failed to subscribe to data_column_sidecar events, blob arrival timings will not be recorded: %s", err) + return } log.Infof("subscribed to data_column_sidecar events") } From 761372c286310f7b296c2647e343392c7c056fd6 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Tue, 11 Aug 2026 14:48:52 +0200 Subject: [PATCH 13/47] perf(analyzer): batch data_column_sidecar persistence off the head loop Each data column arrival was persisted individually and synchronously inside runHead's select loop. Under PeerDAS a full-custody node emits up to 128 of these events per slot, so a single slot triggered ~128 serialized, mutex-guarded ClickHouse INSERTs on the same loop that reads head events and dispatches block downloads; a slow write could stall block following. The pre-Fulu blob_sidecar path produced at most 6-9. runHead no longer touches DataColumnSidecarChan. A dedicated persister goroutine drains the channel and feeds PersistDataColumnSidecarsEvents (which always accepted a slice and batches rows into one INSERT) in batches: immediately at 128 events (one full-custody per-slot burst) and on a 1s ticker otherwise. On shutdown it drains the channel and does a best-effort final flush before DBService.Finish() runs; it registers on wgMainRoutine so the shutdown sequence waits for it. Previously up to 1024 queued events were silently dropped on shutdown. The batching core is a standalone function with an injected tick channel, covered by deterministic unit tests (no beacon node or database needed). Closes #282 --- pkg/analyzer/chain_analyzer.go | 4 + pkg/analyzer/routines.go | 101 +++++++++++++++++++- pkg/analyzer/routines_test.go | 166 +++++++++++++++++++++++++++++++++ 3 files changed, 268 insertions(+), 3 deletions(-) create mode 100644 pkg/analyzer/routines_test.go diff --git a/pkg/analyzer/chain_analyzer.go b/pkg/analyzer/chain_analyzer.go index 212fcf9d..169eb68c 100644 --- a/pkg/analyzer/chain_analyzer.go +++ b/pkg/analyzer/chain_analyzer.go @@ -215,6 +215,10 @@ func (s *ChainAnalyzer) Run() { // Block requester in finalized slots, not used for now s.wgMainRoutine.Add(1) go s.runHead() + + // Persists data_column_sidecar arrivals in batches, off the head loop (#282). + s.wgMainRoutine.Add(1) + go s.runDataColumnSidecarsPersister() } s.PromMetrics.Start() diff --git a/pkg/analyzer/routines.go b/pkg/analyzer/routines.go index 1b966d20..020a0e0e 100644 --- a/pkg/analyzer/routines.go +++ b/pkg/analyzer/routines.go @@ -117,9 +117,6 @@ func (s *ChainAnalyzer) runHead() { s.dbClient.PersistReorgs([]v1.ChainReorgEvent{newReorg}) go s.HandleReorg(newReorg) - case newDataColumnEvent := <-s.eventsObj.DataColumnSidecarChan: - s.dbClient.PersistDataColumnSidecarsEvents([]spec.DataColumnSidecarEventWrapper{newDataColumnEvent}) - case <-s.ctx.Done(): log.Info("context has died, closing block requester routine") return @@ -134,6 +131,104 @@ func (s *ChainAnalyzer) runHead() { } } +// dataColumnMaxBatch flushes as soon as one full-custody per-slot burst +// (NUMBER_OF_COLUMNS = 128) has accumulated, without waiting for the ticker. +const dataColumnMaxBatch = 128 + +// runDataColumnSidecarsPersister drains DataColumnSidecarChan and persists the +// events in batches, off the head loop. Under PeerDAS a full-custody node emits +// up to 128 data_column_sidecar events per slot; persisting them one by one from +// runHead meant one synchronous ClickHouse INSERT per event on the same loop +// that dispatches block downloads (#282). +func (s *ChainAnalyzer) runDataColumnSidecarsPersister() { + defer s.wgMainRoutine.Done() + log.Info("launching data column sidecars persister routine") + + ticker := time.NewTicker(utils.RoutineFlushTimeout) + defer ticker.Stop() + + batchDataColumnSidecarEvents( + s.eventsObj.DataColumnSidecarChan, + ticker.C, + s.ctx.Done(), + func() bool { return s.stop }, + dataColumnMaxBatch, + func(batch []spec.DataColumnSidecarEventWrapper) { + if err := s.dbClient.PersistDataColumnSidecarsEvents(batch); err != nil { + log.Errorf("error persisting data column sidecar events batch: %s", err) + } + }, + ) + + log.Info("data column sidecars persister routine finished") +} + +// batchDataColumnSidecarEvents accumulates events from the channel and flushes +// them in batches: immediately when maxBatch is reached, and on every tick when +// the buffer is non-empty. It returns when done fires or when stopped() reports +// true on a tick; in both cases it first drains whatever sits in the channel and +// does a best-effort final flush (on the done path the write can fail if the DB +// context is already cancelled, which the flush callback logs). +func batchDataColumnSidecarEvents( + events <-chan spec.DataColumnSidecarEventWrapper, + tick <-chan time.Time, + done <-chan struct{}, + stopped func() bool, + maxBatch int, + flush func([]spec.DataColumnSidecarEventWrapper), +) { + buffer := make([]spec.DataColumnSidecarEventWrapper, 0, maxBatch) + + // Reusing the backing array is safe: the persist path copies every value + // into its insert columns before returning. + flushBuffer := func() { + if len(buffer) == 0 { + return + } + flush(buffer) + buffer = buffer[:0] + } + + // drain moves everything currently sitting in the channel into the buffer, + // flushing at maxBatch so the final batches stay bounded. + drain := func() { + for { + select { + case ev := <-events: + buffer = append(buffer, ev) + if len(buffer) >= maxBatch { + flushBuffer() + } + default: + return + } + } + } + + for { + select { + case ev := <-events: + buffer = append(buffer, ev) + if len(buffer) >= maxBatch { + flushBuffer() + } + + case <-tick: + flushBuffer() + if stopped() { + drain() + flushBuffer() + return + } + + case <-done: + drain() + flushBuffer() + return + } + } +} + func (s *ChainAnalyzer) fillToHead() phase0.Slot { // ------ fill from last epoch in database to current head ------- diff --git a/pkg/analyzer/routines_test.go b/pkg/analyzer/routines_test.go new file mode 100644 index 00000000..6a517ae5 --- /dev/null +++ b/pkg/analyzer/routines_test.go @@ -0,0 +1,166 @@ +package analyzer + +import ( + "sync" + "testing" + "time" + + "github.com/migalabs/goteth/pkg/spec" +) + +// flushRecorder is a thread-safe fake flush target for +// batchDataColumnSidecarEvents. It records call and row counts only; the +// batch slice itself is reused by the batcher and must not be retained. +type flushRecorder struct { + mu sync.Mutex + calls int + rows int +} + +func (r *flushRecorder) flush(batch []spec.DataColumnSidecarEventWrapper) { + r.mu.Lock() + defer r.mu.Unlock() + r.calls++ + r.rows += len(batch) +} + +func (r *flushRecorder) snapshot() (calls int, rows int) { + r.mu.Lock() + defer r.mu.Unlock() + return r.calls, r.rows +} + +// waitFor polls cond until it is true or the timeout expires. +func waitFor(t *testing.T, timeout time.Duration, cond func() bool, msg string) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(2 * time.Millisecond) + } + t.Fatalf("timed out waiting for: %s", msg) +} + +type batcherHarness struct { + events chan spec.DataColumnSidecarEventWrapper + tick chan time.Time + done chan struct{} + stopped func() bool + rec *flushRecorder + finished chan struct{} +} + +func startBatcher(maxBatch int, stopped func() bool) *batcherHarness { + h := &batcherHarness{ + events: make(chan spec.DataColumnSidecarEventWrapper, 1024), + tick: make(chan time.Time), + done: make(chan struct{}), + stopped: stopped, + rec: &flushRecorder{}, + finished: make(chan struct{}), + } + go func() { + batchDataColumnSidecarEvents(h.events, h.tick, h.done, h.stopped, maxBatch, h.rec.flush) + close(h.finished) + }() + return h +} + +func (h *batcherHarness) sendEvents(n int) { + for i := 0; i < n; i++ { + h.events <- spec.DataColumnSidecarEventWrapper{Timestamp: time.Now()} + } +} + +func (h *batcherHarness) waitFinished(t *testing.T) { + t.Helper() + select { + case <-h.finished: + case <-time.After(2 * time.Second): + t.Fatal("batcher did not return") + } +} + +// A burst larger than maxBatch flushes in maxBatch-sized batches without any +// tick, and the tick then flushes the remainder: far fewer flushes than events. +func TestBatchDataColumnEventsFlushesAtMaxBatch(t *testing.T) { + h := startBatcher(128, func() bool { return false }) + h.sendEvents(300) + + // The two full 128-event batches flush without any tick. + waitFor(t, time.Second, func() bool { _, rows := h.rec.snapshot(); return rows == 256 }, + "two maxBatch flushes (256 rows)") + + // Wait until the loop has consumed the whole burst, then tick to flush the + // remaining 44 (sending the tick earlier could reach the select while part + // of the burst still sits in the channel, splitting the last flush). + waitFor(t, time.Second, func() bool { return len(h.events) == 0 }, "channel drained") + h.tick <- time.Time{} + waitFor(t, time.Second, func() bool { _, rows := h.rec.snapshot(); return rows == 300 }, + "tick flush of the remainder (300 rows total)") + + calls, rows := h.rec.snapshot() + if rows != 300 { + t.Fatalf("expected 300 rows flushed, got %d", rows) + } + if calls != 3 { + t.Fatalf("expected 3 flush calls (128+128+44), got %d", calls) + } + + close(h.done) + h.waitFinished(t) +} + +// A partial buffer is flushed once per tick, not once per event. +func TestBatchDataColumnEventsTickFlushesPartialBuffer(t *testing.T) { + h := startBatcher(128, func() bool { return false }) + h.sendEvents(5) + + // Once the channel is empty every sent event is either appended already or + // being appended by the current loop iteration; the blocking tick send below + // is only received after that iteration finishes. + waitFor(t, time.Second, func() bool { return len(h.events) == 0 }, "channel drained") + h.tick <- time.Time{} + + waitFor(t, time.Second, func() bool { calls, rows := h.rec.snapshot(); return calls == 1 && rows == 5 }, + "single flush of 5 rows") + + close(h.done) + h.waitFinished(t) +} + +// When stopped() reports true on a tick, pending channel events are drained and +// flushed before returning. +func TestBatchDataColumnEventsStopDrainsChannel(t *testing.T) { + var mu sync.Mutex + stop := false + setStop := func(v bool) { mu.Lock(); stop = v; mu.Unlock() } + getStop := func() bool { mu.Lock(); defer mu.Unlock(); return stop } + + h := startBatcher(128, getStop) + h.sendEvents(10) + setStop(true) + h.tick <- time.Time{} + + h.waitFinished(t) + _, rows := h.rec.snapshot() + if rows != 10 { + t.Fatalf("expected all 10 rows flushed on stop, got %d", rows) + } +} + +// When done fires, pending channel events are drained and flushed before +// returning. +func TestBatchDataColumnEventsDoneDrainsChannel(t *testing.T) { + h := startBatcher(128, func() bool { return false }) + h.sendEvents(7) + close(h.done) + + h.waitFinished(t) + _, rows := h.rec.snapshot() + if rows != 7 { + t.Fatalf("expected all 7 rows flushed on done, got %d", rows) + } +} From e7b924b04407d9b3461aca373881c8edfd02a805 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Tue, 11 Aug 2026 14:59:49 +0200 Subject: [PATCH 14/47] fix(analyzer): make the stop flag an atomic.Bool The flag was a plain bool written from Close(), Run() and two download error paths, and read from four routines plus the new data column persister, all without synchronization. The race was benign in practice (stale bool reads) but is exactly what the race detector flags, and the persister added one more reader. atomic.Bool keeps the semantics and removes the race; ChainAnalyzer is only ever used through a pointer, so the non-copyable field is safe. --- pkg/analyzer/chain_analyzer.go | 7 ++++--- pkg/analyzer/download.go | 4 ++-- pkg/analyzer/process_state.go | 2 +- pkg/analyzer/routines.go | 10 +++++----- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/analyzer/chain_analyzer.go b/pkg/analyzer/chain_analyzer.go index 169eb68c..034780be 100644 --- a/pkg/analyzer/chain_analyzer.go +++ b/pkg/analyzer/chain_analyzer.go @@ -3,6 +3,7 @@ package analyzer import ( "context" "sync" + "sync/atomic" "time" "github.com/attestantio/go-eth2-client/spec/phase0" @@ -42,7 +43,7 @@ type ChainAnalyzer struct { // Control Variables wgMainRoutine *sync.WaitGroup // wait group for main routine (either historical or head) wgDownload *sync.WaitGroup // wait group for download routine - stop bool // flag to notify all routine to finish + stop atomic.Bool // flag to notify all routine to finish (read and written from several goroutines) routineClosed chan struct{} // signal that everything was closed succesfully downloadMode string // whether to download historical blocks (defined by user) or follow chain head rewardsAggregationEpochs int // number of epochs to aggregate rewards @@ -224,7 +225,7 @@ func (s *ChainAnalyzer) Run() { s.PromMetrics.Start() s.wgMainRoutine.Wait() - s.stop = true + s.stop.Store(true) log.Infof("main routine finished, waiting for downloader...") s.wgDownload.Wait() @@ -242,7 +243,7 @@ func (s *ChainAnalyzer) Run() { func (s *ChainAnalyzer) Close() { log.Info("Sudden closed detected, closing StateAnalyzer") - s.stop = true + s.stop.Store(true) s.cancel() <-s.routineClosed // Wait for services to stop before returning } diff --git a/pkg/analyzer/download.go b/pkg/analyzer/download.go index b4275df6..6dbf2d97 100644 --- a/pkg/analyzer/download.go +++ b/pkg/analyzer/download.go @@ -22,7 +22,7 @@ func (s *ChainAnalyzer) DownloadBlock(slot phase0.Slot) { newBlock, err := s.cli.RequestBeaconBlock(slot) if err != nil { log.Errorf("block error at slot %d: %s", slot, err) - s.stop = true + s.stop.Store(true) s.cancel() return } @@ -63,7 +63,7 @@ func (s *ChainAnalyzer) DownloadState(slot phase0.Slot) { if err != nil { log.Errorf("unable to retrieve beacon state from the beacon node, closing requester routine. %s", err.Error()) - s.stop = true + s.stop.Store(true) s.cancel() return } diff --git a/pkg/analyzer/process_state.go b/pkg/analyzer/process_state.go index 00aaaa25..42be2e5c 100644 --- a/pkg/analyzer/process_state.go +++ b/pkg/analyzer/process_state.go @@ -73,7 +73,7 @@ func (s *ChainAnalyzer) ProcessStateTransitionMetrics(epoch phase0.Epoch) { if err != nil { s.processerBook.FreePage(routineKey) log.Errorf("could not parse bundle metrics at epoch: %s", err) - s.stop = true + s.stop.Store(true) s.cancel() return } diff --git a/pkg/analyzer/routines.go b/pkg/analyzer/routines.go index 020a0e0e..3dd3e366 100644 --- a/pkg/analyzer/routines.go +++ b/pkg/analyzer/routines.go @@ -37,7 +37,7 @@ downloadRoutine: go s.ProcessStateTransitionMetrics(phase0.Epoch(downloadSlot / spec.SlotsPerEpoch)) } case <-ticker.C: // every certain amount of time check if need to finish - if s.stop && len(s.downloadTaskChan) == 0 && s.cli.ActiveReqNum() == 0 && s.processerBook.ActivePages() == 0 { + if s.stop.Load() && len(s.downloadTaskChan) == 0 && s.cli.ActiveReqNum() == 0 && s.processerBook.ActivePages() == 0 { break downloadRoutine } } @@ -122,7 +122,7 @@ func (s *ChainAnalyzer) runHead() { return case <-ticker.C: - if s.stop { + if s.stop.Load() { log.Info("sudden shutdown detected, block downloader routine") return } @@ -151,7 +151,7 @@ func (s *ChainAnalyzer) runDataColumnSidecarsPersister() { s.eventsObj.DataColumnSidecarChan, ticker.C, s.ctx.Done(), - func() bool { return s.stop }, + func() bool { return s.stop.Load() }, dataColumnMaxBatch, func(batch []spec.DataColumnSidecarEventWrapper) { if err := s.dbClient.PersistDataColumnSidecarsEvents(batch); err != nil { @@ -291,7 +291,7 @@ func (s *ChainAnalyzer) fillToHead() phase0.Slot { // runHistorical again — and since the chain keeps advancing, // handoffThreshold is always exceeded and the loop spins // indefinitely on shutdown. - if s.stop { + if s.stop.Load() { return headSlot } @@ -313,7 +313,7 @@ func (s *ChainAnalyzer) runHistorical(init phase0.Slot, end phase0.Slot) { i := init for i <= end { - if s.stop { + if s.stop.Load() { log.Info("sudden shutdown detected, block downloader routine") return } From 6d8209a96f14ae8fd29ce036c5319a9c9b3fae4b Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Tue, 11 Aug 2026 14:59:49 +0200 Subject: [PATCH 15/47] feat(metrics): expose data_column_sidecar subscription state as a gauge The subscribe path now logs and skips on failure instead of crashing, which reintroduces a silent-loss risk: the analyzer would run without recording blob arrival timings and only a startup log line would tell. goteth_analyzer_data_column_subscription_active makes that state visible to operators (1 while the subscription is established, 0 otherwise, so it can be alerted on). The flag lives behind a pointer so the Events struct stays copyable. --- pkg/analyzer/prometheus_metrics.go | 40 ++++++++++++++++++++++++++++++ pkg/events/blobs.go | 1 + pkg/events/standard.go | 8 ++++++ 3 files changed, 49 insertions(+) diff --git a/pkg/analyzer/prometheus_metrics.go b/pkg/analyzer/prometheus_metrics.go index fbfdf013..ac8a1b03 100644 --- a/pkg/analyzer/prometheus_metrics.go +++ b/pkg/analyzer/prometheus_metrics.go @@ -25,6 +25,12 @@ var ( Name: "block_queue_length", Help: "The number of blocks int the history queue", }) + DataColumnSubscriptionActive = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: strings.ToLower(utils.CliName), + Subsystem: modName, + Name: "data_column_subscription_active", + Help: "1 while the data_column_sidecar SSE subscription is established; blob arrival timings are not recorded while 0", + }) ) func (c *ChainAnalyzer) GetPrometheusMetrics() *metrics.MetricsModule { @@ -36,6 +42,7 @@ func (c *ChainAnalyzer) GetPrometheusMetrics() *metrics.MetricsModule { metricsMod.AddIndvMetric(c.getStateHistoryLength()) metricsMod.AddIndvMetric(c.getBlockHistoryLength()) + metricsMod.AddIndvMetric(c.getDataColumnSubscriptionActive()) return metricsMod } @@ -66,6 +73,39 @@ func (p *ChainAnalyzer) getStateHistoryLength() *metrics.IndvMetrics { return indvMetr } +// getDataColumnSubscriptionActive makes a silent data_column_sidecar +// subscription failure visible: the subscribe path logs and skips instead of +// crashing, so without this gauge an operator could miss that blob arrival +// timings stopped being recorded (the exact silent-loss failure mode of #280). +func (p *ChainAnalyzer) getDataColumnSubscriptionActive() *metrics.IndvMetrics { + + initFn := func() error { + prometheus.MustRegister(DataColumnSubscriptionActive) + return nil + } + + updateFn := func() (interface{}, error) { + active := 0 + if p.eventsObj.SubscribedDataColumns.Load() { + active = 1 + } + DataColumnSubscriptionActive.Set(float64(active)) + return active, nil + } + + indvMetr, err := metrics.NewIndvMetrics( + "data_column_subscription_active", + initFn, + updateFn, + ) + if err != nil { + log.Error(errors.Wrap(err, "unable to init data_column_subscription_active")) + return nil + } + + return indvMetr +} + func (p *ChainAnalyzer) getBlockHistoryLength() *metrics.IndvMetrics { initFn := func() error { diff --git a/pkg/events/blobs.go b/pkg/events/blobs.go index 7c0dcfc6..9c48e37d 100644 --- a/pkg/events/blobs.go +++ b/pkg/events/blobs.go @@ -30,6 +30,7 @@ func (e *Events) SubscribeToDataColumnSidecarsEvents() { log.Errorf("failed to subscribe to data_column_sidecar events, blob arrival timings will not be recorded: %s", err) return } + e.SubscribedDataColumns.Store(true) log.Infof("subscribed to data_column_sidecar events") } diff --git a/pkg/events/standard.go b/pkg/events/standard.go index 88ab1c46..55b3c0d1 100644 --- a/pkg/events/standard.go +++ b/pkg/events/standard.go @@ -2,6 +2,7 @@ package events import ( "context" + "sync/atomic" apiv1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/migalabs/goteth/pkg/clientapi" @@ -26,6 +27,12 @@ type Events struct { FinalizedChan chan apiv1.FinalizedCheckpointEvent ReorgChan chan apiv1.ChainReorgEvent DataColumnSidecarChan chan spec.DataColumnSidecarEventWrapper + + // SubscribedDataColumns turns true once the data_column_sidecar subscription + // is established; while false, blob arrival timings are not being recorded. + // Pointer so Events stays copyable (atomic values must not be copied); read + // by the prometheus updater goroutine. + SubscribedDataColumns *atomic.Bool } func NewEventsObj(iCtx context.Context, iCli *clientapi.APIClient) Events { @@ -43,5 +50,6 @@ func NewEventsObj(iCtx context.Context, iCli *clientapi.APIClient) Events { // *column*, so a single block can produce up to 128 events (NUMBER_OF_COLUMNS) // instead of the <=6 blob events it used to, all arriving in a burst. DataColumnSidecarChan: make(chan spec.DataColumnSidecarEventWrapper, 1024), + SubscribedDataColumns: new(atomic.Bool), } } From bfb2cc85c64be218b4ae313cc60c344a99e1efd5 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Fri, 28 Aug 2026 13:06:47 +0200 Subject: [PATCH 16/47] fix(electra): recompute deposit counters when the state object was replaced f_deposits_num is written as exactly 0 for some epochs while t_deposits holds the corresponding rows. Never slightly wrong, always exactly zero. The counters are accumulated on one state and read from another, an epoch later: processPendingDeposits() state := p.baseMetrics.NextState state.DepositsNum += 1 ExportToEpoch() DepositsNum: int(s.CurrentState.DepositsNum) The value written into the epoch row for E-1 was accumulated while that object was NextState, during the processing of E-1. It survives only if the same object is still in the cache when E is processed. Three things replace or reset it in between: - AdvanceFinalized deletes and re-downloads a state whose root changed - ensureDependencyStates re-downloads states evicted by CleanUpTo - RefreshBlocks resets the counters outright, and post-Electra AddBlocks does not refill them, since deposits arrive through the pending queue rather than block bodies A replaced object carries zeros and nothing recomputes them, so the epoch row reports no deposits while the detail rows exist. Measured on mainnet: 252 epochs over 450421-470670, 1,653 deposits and roughly 113,000 ETH absent from the summaries, against 10,029 epochs written correctly. 97.5% right, and wrong in exactly the way a lost accumulator is wrong. The same reading explains why the consolidation counters fail on a different set of epochs, overlapping the deposit failures in only 18 of 37 cases: RefreshBlocks resets the deposit counters and not the consolidation ones, so the two have different wipe conditions. Fix: record on the state whether its deposit counters were ever accumulated, and recompute them from the state itself when they were not. What processPendingDepositsFor derives depends only on the state handed to it, so a state that came back from the beacon node reaches the same numbers the original run reached. It must run at most once per object, since the counters accumulate with += rather than being assigned, and the new flag is what enforces that. The recomputation runs before the existing map clears, so its contribution to DepositedAmounts is discarded exactly as the surrounding code expects and the reward path repopulates that map afterwards; only the two counters survive. Deposit rows are persisted from NextState.DepositsProcessed, so recomputing for CurrentState cannot duplicate them. The common case costs a boolean check. Seven tests. One found a hole in the first version of this fix: RefreshBlocks zeroed the counters without clearing the flag, so a refreshed state still claimed it had been accumulated and the recomputation never fired. Others cover the claim the fix rests on, that a re-downloaded state recomputes to the numbers the original reached, keep "no deposits this epoch" distinguishable from "never counted", and pin the doubling that makes the flag necessary. Pre-existing failures in pkg/analyzer are unchanged, 13 before and 13 after; they need a live beacon endpoint. Closes #287 --- pkg/spec/deposit_counters_test.go | 50 ++++++++++ pkg/spec/metrics/deposit_recompute_test.go | 104 +++++++++++++++++++++ pkg/spec/metrics/state_electra.go | 36 ++++++- pkg/spec/state.go | 24 +++-- 4 files changed, 205 insertions(+), 9 deletions(-) create mode 100644 pkg/spec/deposit_counters_test.go create mode 100644 pkg/spec/metrics/deposit_recompute_test.go diff --git a/pkg/spec/deposit_counters_test.go b/pkg/spec/deposit_counters_test.go new file mode 100644 index 00000000..e4cd51c6 --- /dev/null +++ b/pkg/spec/deposit_counters_test.go @@ -0,0 +1,50 @@ +package spec + +import "testing" + +// The deposit counters on a state are accumulated while it is NextState and +// read one epoch later, while it is CurrentState. Anything that replaces or +// resets the object in between leaves them at zero, and the epoch row is then +// written with no deposits while t_deposits holds the rows +// (migalabs/goteth#287). + +func TestRefreshBlocksClearsTheDepositAccumulationFlag(t *testing.T) { + state := &AgnosticState{ + DepositsNum: 4, + TotalDepositsAmount: 128_000_000_000, + PendingDepositsProcessed: true, + } + + state.RefreshBlocks(nil) + + if state.DepositsNum != 0 { + t.Fatalf("RefreshBlocks left DepositsNum at %d; the fixture assumes it resets", + state.DepositsNum) + } + if state.PendingDepositsProcessed { + t.Error("the counters were reset but the state still claims they were accumulated, " + + "so nothing would recompute them and the epoch row would report zero deposits") + } +} + +// A freshly downloaded state has never been accumulated, which is what marks it +// for recomputation rather than being taken at face value. +func TestAFreshStateIsNotMarkedAccumulated(t *testing.T) { + if (&AgnosticState{}).PendingDepositsProcessed { + t.Error("a new state claims its deposit counters were accumulated") + } +} + +// Zero deposits is a legitimate outcome, and must stay distinguishable from +// "never counted": most epochs genuinely have none. +func TestZeroDepositsIsDistinctFromNeverCounted(t *testing.T) { + counted := &AgnosticState{DepositsNum: 0, PendingDepositsProcessed: true} + never := &AgnosticState{DepositsNum: 0} + + if !counted.PendingDepositsProcessed { + t.Error("an epoch with genuinely no deposits should not be recomputed") + } + if never.PendingDepositsProcessed { + t.Error("a state that was never counted must be recomputed") + } +} diff --git a/pkg/spec/metrics/deposit_recompute_test.go b/pkg/spec/metrics/deposit_recompute_test.go new file mode 100644 index 00000000..5227a0d1 --- /dev/null +++ b/pkg/spec/metrics/deposit_recompute_test.go @@ -0,0 +1,104 @@ +package metrics + +import ( + "testing" + + "github.com/attestantio/go-eth2-client/spec/electra" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/migalabs/goteth/pkg/spec" +) + +// The deposit counters are accumulated while a state is NextState and read an +// epoch later, while it is CurrentState. A state that was re-downloaded or +// refreshed in between carries zeros, and the epoch row is then written with no +// deposits while t_deposits holds the rows (migalabs/goteth#287). + +const depositAmount = phase0.Gwei(32_000_000_000) + +// stateWithOnePendingDeposit builds the smallest state that processes a single +// deposit: the queue is finalized, the Eth1 bridge check is satisfied, and the +// balance available for processing covers the amount. +func stateWithOnePendingDeposit() *spec.AgnosticState { + var pubkey phase0.BLSPubKey + pubkey[0] = 0x01 + + return &spec.AgnosticState{ + Epoch: 10, + Validators: []*phase0.Validator{{PublicKey: pubkey, ExitEpoch: phase0.Epoch(spec.FarFutureEpoch), WithdrawableEpoch: phase0.Epoch(spec.FarFutureEpoch)}}, + PendingDeposits: []*electra.PendingDeposit{{Pubkey: pubkey, Amount: depositAmount, Slot: 0}}, + DepositBalanceToConsume: depositAmount * 4, + Eth1DepositIndex: 1, + DepositRequestsStartIndex: 0, + DepositedAmounts: make(map[phase0.ValidatorIndex]phase0.Gwei), + } +} + +// The fix rests on this: a state that lost its counters can have them derived +// again from the state alone, reaching the numbers the original run reached. +func TestRecomputingAReplacedStateReproducesTheCounters(t *testing.T) { + p := &ElectraMetrics{} + + accumulated := stateWithOnePendingDeposit() + p.processPendingDepositsFor(accumulated) + + // The same state as it comes back from the beacon node, counters at zero. + redownloaded := stateWithOnePendingDeposit() + p.processPendingDepositsFor(redownloaded) + + if accumulated.DepositsNum == 0 { + t.Fatal("the fixture processed no deposits, so it cannot show anything") + } + if redownloaded.DepositsNum != accumulated.DepositsNum { + t.Errorf("recomputed %d deposits, the original run reached %d", + redownloaded.DepositsNum, accumulated.DepositsNum) + } + if redownloaded.TotalDepositsAmount != accumulated.TotalDepositsAmount { + t.Errorf("recomputed %d gwei, the original run reached %d", + redownloaded.TotalDepositsAmount, accumulated.TotalDepositsAmount) + } +} + +func TestProcessingMarksTheStateAccumulated(t *testing.T) { + p := &ElectraMetrics{} + state := stateWithOnePendingDeposit() + + p.processPendingDepositsFor(state) + + if !state.PendingDepositsProcessed { + t.Error("the state was processed but not marked, so it would be recomputed again") + } +} + +// An epoch with an empty queue must still be marked, otherwise every epoch +// without deposits, which is most of them, is recomputed on every pass. +func TestAnEmptyQueueStillMarksTheState(t *testing.T) { + p := &ElectraMetrics{} + state := &spec.AgnosticState{Epoch: 10, DepositedAmounts: make(map[phase0.ValidatorIndex]phase0.Gwei)} + + p.processPendingDepositsFor(state) + + if !state.PendingDepositsProcessed { + t.Error("a state with no pending deposits was left unmarked") + } + if state.DepositsNum != 0 { + t.Errorf("an empty queue produced %d deposits", state.DepositsNum) + } +} + +// Why PendingDepositsProcessed has to gate the call: the counters accumulate +// with +=, so a second run on the same object doubles them. This pins the +// hazard the flag exists to prevent. +func TestRunningTwiceOnOneStateDoublesTheCounters(t *testing.T) { + p := &ElectraMetrics{} + state := stateWithOnePendingDeposit() + + p.processPendingDepositsFor(state) + once := state.DepositsNum + p.processPendingDepositsFor(state) + + if state.DepositsNum != once*2 { + t.Errorf("expected a second run to double %d to %d, got %d; if this no longer "+ + "holds, revisit whether PendingDepositsProcessed still needs to gate the call", + once, once*2, state.DepositsNum) + } +} diff --git a/pkg/spec/metrics/state_electra.go b/pkg/spec/metrics/state_electra.go index e6df3ab6..4d27eb8e 100644 --- a/pkg/spec/metrics/state_electra.go +++ b/pkg/spec/metrics/state_electra.go @@ -52,6 +52,21 @@ func (p *ElectraMetrics) PreProcessBundle() error { return err } p.processPendingDeposits() + + // The epoch row for CurrentState.Epoch reads its deposit counters from + // CurrentState, but they were accumulated an epoch earlier, while that + // object was NextState. Reorg handling deletes and re-downloads states + // (AdvanceFinalized, ensureDependencyStates) and RefreshBlocks zeroes + // these counters outright, so the object serving as CurrentState here is + // not always the one that was accumulated. When it is not, the counters + // read as exactly zero while t_deposits holds the rows. + // + // Recompute from the state itself, which is deterministic and costs + // nothing on the common path where the object survived (#287). + if !p.baseMetrics.CurrentState.PendingDepositsProcessed { + p.processPendingDepositsFor(p.baseMetrics.CurrentState) + } + // FIX: Clear state maps before processing (state objects are reused between iterations) p.baseMetrics.CurrentState.ConsolidatedAmounts = make(map[phase0.ValidatorIndex]phase0.Gwei) p.baseMetrics.CurrentState.ConsolidatedOutAmounts = make(map[phase0.ValidatorIndex]phase0.Gwei) @@ -862,8 +877,24 @@ func (p *ElectraMetrics) ProcessSlashings() { // https://github.com/ethereum/consensus-specs/blob/dev/specs/electra/beacon-chain.md#new-process_pending_deposits func (p *ElectraMetrics) processPendingDeposits() { - nextEpoch := p.baseMetrics.NextState.Epoch + 1 - state := p.baseMetrics.NextState + p.processPendingDepositsFor(p.baseMetrics.NextState) +} + +// processPendingDepositsFor walks a state's pending deposit queue, accumulating +// its deposit counters and rebuilding its list of processed deposits. What it +// derives depends only on the state passed in, so running it on a state that +// was re-downloaded reproduces the numbers the original accumulation reached. +// +// It must run at most once per state object. The counters are accumulated with +// +=, not assigned, so a second run doubles them; PendingDepositsProcessed +// records that a state has been through it and is what keeps that from +// happening. +// +// It is normally run while the state is NextState. The epoch row, however, +// reads the counters from CurrentState, one epoch later. See PreProcessBundle +// for why that gap has to be closed. +func (p *ElectraMetrics) processPendingDepositsFor(state *spec.AgnosticState) { + nextEpoch := state.Epoch + 1 availableForProcessing := state.DepositBalanceToConsume + phase0.Gwei(getActivationExitChurnLimit(state)) processedAmount := phase0.Gwei(0) nextDepositIndex := uint64(0) @@ -935,4 +966,5 @@ func (p *ElectraMetrics) processPendingDeposits() { state.TotalDepositsAmount += deposit.Amount } state.DepositsProcessed = processedDeposits + state.PendingDepositsProcessed = true } diff --git a/pkg/spec/state.go b/pkg/spec/state.go index c427e3ae..58fe731f 100644 --- a/pkg/spec/state.go +++ b/pkg/spec/state.go @@ -42,13 +42,19 @@ type AgnosticState struct { Deposits []phase0.Gwei // one per validator index DepositsNum uint64 // number of deposits TotalDepositsAmount phase0.Gwei // total amount of deposits - CurrentJustifiedCheckpoint phase0.Checkpoint // the latest justified checkpoint - CurrentFinalizedCheckpoint phase0.Checkpoint // the latest finalized checkpoint - LatestBlockHeader *phase0.BeaconBlockHeader - SyncCommitteeParticipation uint64 // Tracks sync committee participation - NewProposerSlashings int // number of new proposer slashings - NewAttesterSlashings int // number of new attester slashings - Slashings []AgnosticSlashing + // PendingDepositsProcessed records whether the two counters above were + // accumulated for this state object. They are filled while the state is + // NextState and read one epoch later while it is CurrentState, so a state + // that was re-downloaded or refreshed in between carries zeros that nothing + // else recomputes (migalabs/goteth#287). + PendingDepositsProcessed bool + CurrentJustifiedCheckpoint phase0.Checkpoint // the latest justified checkpoint + CurrentFinalizedCheckpoint phase0.Checkpoint // the latest finalized checkpoint + LatestBlockHeader *phase0.BeaconBlockHeader + SyncCommitteeParticipation uint64 // Tracks sync committee participation + NewProposerSlashings int // number of new proposer slashings + NewAttesterSlashings int // number of new attester slashings + Slashings []AgnosticSlashing // Electra ConsolidationRequests []ConsolidationRequest WithdrawalRequests []WithdrawalRequest @@ -140,6 +146,10 @@ func (p *AgnosticState) RefreshBlocks(blockList []*AgnosticBlock) { p.TotalWithdrawalsAmount = 0 p.DepositsNum = 0 p.TotalDepositsAmount = 0 + // Reset the flag with the counters it describes. Leaving it set would let a + // refreshed state claim numbers it no longer has, and nothing would + // recompute them. + p.PendingDepositsProcessed = false p.NumAttestations = 0 p.SyncCommitteeParticipation = 0 p.AddBlocks(blockList) From 7f17394b579a7c0f08d1f82d6c5e225e54eacbd9 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Fri, 28 Aug 2026 13:18:49 +0200 Subject: [PATCH 17/47] Attribute blob sidecars by position instead of by hash A blob's versioned hash is sha256 of its KZG commitment and nothing else, so two transactions carrying byte-identical blobs produce the same hash. GetTxHash matched sidecars on that hash and did not stop at the first match, so every copy was credited to whichever transaction was compared last, leaving the others with no blobs at all. Mainnet slot 14414626 has three consecutive transactions each carrying the same near-empty blob, 130,816 of its 131,072 bytes being trailing zeros. All three blobs landed on the last of them. Over 136,175 mainnet slots this affects 252, and every one of them contains a repeated blob hash: no slot mismatches without one, and no slot with one is correct. A block's blob commitments are the concatenation of each type-3 transaction's blob hashes in transaction order, so the sidecar holding index i belongs to the transaction owning the i-th entry. That mapping is exact and unaffected by repeated content. Nothing is assigned until the whole block checks out: the sidecars must account for exactly the blobs the transactions committed to, each index must appear once, and every hash must agree with the one committed at that position. A transaction whose receipt never arrived is dropped while parsing, which breaks the first of those, so the mapping is left alone and the slot logged rather than sliding every blob onto its neighbour. Nine cases cover it, including the slot above, one transaction carrying the same blob twice, sidecars and transactions arriving out of order, and each way the block can fail to line up. Verified against the old algorithm on the same fixture: it reproduces the production rows exactly, so the test fails without this change. Pre-existing failures in pkg/analyzer are unchanged, 13 before and 13 after; they need a live beacon endpoint. Closes #286 --- pkg/analyzer/process_block.go | 4 +- pkg/spec/blob.go | 76 +++++++++++-- pkg/spec/blob_test.go | 207 ++++++++++++++++++++++++++++++++++ 3 files changed, 278 insertions(+), 9 deletions(-) create mode 100644 pkg/spec/blob_test.go diff --git a/pkg/analyzer/process_block.go b/pkg/analyzer/process_block.go index 3f2154a3..e5e47d15 100644 --- a/pkg/analyzer/process_block.go +++ b/pkg/analyzer/process_block.go @@ -227,8 +227,8 @@ func (s *ChainAnalyzer) processBlobSidecars(block *spec.AgnosticBlock, txs []spe } if len(blobs) > 0 { if len(txs) > 0 { - for _, blob := range blobs { - blob.GetTxHash(txs) + if err := spec.AssignTxHashes(blobs, txs); err != nil { + log.Errorf("slot %d: could not attribute blob sidecars to transactions: %s", block.Slot, err) } } s.dbClient.PersistBlobSidecars(blobs) diff --git a/pkg/spec/blob.go b/pkg/spec/blob.go index 81c3292b..72138ea1 100644 --- a/pkg/spec/blob.go +++ b/pkg/spec/blob.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "sort" "time" api "github.com/attestantio/go-eth2-client/api/v1" @@ -45,20 +46,81 @@ func NewAgnosticBlobFromAPI(slot phase0.Slot, blob deneb.BlobSidecar) (*Agnostic }, nil } -func (b *AgnosticBlobSidecar) GetTxHash(txs []AgnosticTransaction) { +// AssignTxHashes maps every blob sidecar of a block to the transaction that +// carried it. +// +// A block's blob commitments are the concatenation of each type-3 transaction's +// blob hashes taken in transaction order, so the sidecar holding index i belongs +// to the transaction that owns the i-th entry of that concatenation. +// +// Position is the only dependable key. A versioned hash is derived from nothing +// but the blob's KZG commitment, so two transactions carrying byte-identical +// blobs share one hash and cannot be told apart by it; matching on the hash +// awards every copy to whichever of them is compared last, and leaves the +// others with no blobs at all. +// +// Nothing is assigned until the whole block has been checked: the sidecars must +// account for exactly the blobs the transactions committed to, each index must +// appear once, and every hash must agree with the one committed at that +// position. Any disagreement means these sidecars are not the ones this block's +// transactions committed to, or that a transaction went missing because its +// receipt never arrived, and the positions are then meaningless. In that case +// the mapping is left untouched and an error returned, rather than shifting +// every blob onto the wrong transaction. +func AssignTxHashes(blobs []*AgnosticBlobSidecar, txs []AgnosticTransaction) error { + + carriers := make([]AgnosticTransaction, 0, len(blobs)) for _, tx := range txs { - if tx.BlobHashes == nil { + if len(tx.BlobHashes) == 0 { continue // this tx does not reference any blobs } + carriers = append(carriers, tx) + } - for _, txBlobHash := range tx.BlobHashes { - if txBlobHash.String() == b.BlobHash { - // we found it - b.TxHash = common.Hash(tx.Hash) - } + sort.Slice(carriers, func(i, j int) bool { return carriers[i].TxIdx < carriers[j].TxIdx }) + + // The block's commitments, flattened in transaction order: one entry per + // blob, holding the transaction that carried it and the hash it committed. + type commitment struct { + txHash common.Hash + blobHash string + } + committed := make([]commitment, 0, len(blobs)) + for _, tx := range carriers { + for _, blobHash := range tx.BlobHashes { + committed = append(committed, commitment{common.Hash(tx.Hash), blobHash.String()}) } } + + if len(committed) != len(blobs) { + return fmt.Errorf("%d blob sidecars but %d blobs committed by %d transactions", + len(blobs), len(committed), len(carriers)) + } + + seen := make([]bool, len(committed)) + for _, blob := range blobs { + index := int(blob.Index) + if index >= len(committed) { + return fmt.Errorf("blob index %d is past the %d blobs this block committed", + blob.Index, len(committed)) + } + if seen[index] { + return fmt.Errorf("two sidecars claim blob index %d", blob.Index) + } + seen[index] = true + + if committed[index].blobHash != blob.BlobHash { + return fmt.Errorf("blob %d does not match the hash committed at that position", + blob.Index) + } + } + + for _, blob := range blobs { + blob.TxHash = committed[blob.Index].txHash + } + + return nil } // DataColumnSidecarEventWrapper carries a data-column arrival plus the local time diff --git a/pkg/spec/blob_test.go b/pkg/spec/blob_test.go new file mode 100644 index 00000000..10c8a20e --- /dev/null +++ b/pkg/spec/blob_test.go @@ -0,0 +1,207 @@ +package spec_test + +import ( + "fmt" + "testing" + + "github.com/attestantio/go-eth2-client/spec/deneb" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethereum/go-ethereum/common" + "github.com/migalabs/goteth/pkg/spec" +) + +// blobHash builds a distinguishable versioned hash. Passing the same seed twice +// produces the same hash, which is what happens on chain when two transactions +// carry byte-identical blobs. +func blobHash(seed byte) common.Hash { + var h common.Hash + h[0] = 0x01 + h[31] = seed + return h +} + +func carrier(txIdx uint64, txSeed byte, blobSeeds ...byte) spec.AgnosticTransaction { + var txHash phase0.Hash32 + txHash[31] = txSeed + + hashes := make([]common.Hash, 0, len(blobSeeds)) + for _, seed := range blobSeeds { + hashes = append(hashes, blobHash(seed)) + } + return spec.AgnosticTransaction{TxIdx: txIdx, Hash: txHash, BlobHashes: hashes} +} + +// sidecars builds the block's sidecar list, indexed in block order. +func sidecars(blobSeeds ...byte) []*spec.AgnosticBlobSidecar { + blobs := make([]*spec.AgnosticBlobSidecar, 0, len(blobSeeds)) + for i, seed := range blobSeeds { + blobs = append(blobs, &spec.AgnosticBlobSidecar{ + Index: deneb.BlobIndex(i), + BlobHash: blobHash(seed).String(), + }) + } + return blobs +} + +func attributed(blobs []*spec.AgnosticBlobSidecar) []string { + got := make([]string, 0, len(blobs)) + for _, b := range blobs { + got = append(got, fmt.Sprintf("%d:%s", b.Index, b.TxHash.String())) + } + return got +} + +func expectedAttribution(pairs ...uint64) []string { + want := make([]string, 0, len(pairs)) + for i := 0; i < len(pairs); i += 2 { + var txHash phase0.Hash32 + txHash[31] = byte(pairs[i+1]) + want = append(want, fmt.Sprintf("%d:%s", pairs[i], common.Hash(txHash).String())) + } + return want +} + +func TestAssignTxHashes(t *testing.T) { + tests := []struct { + name string + blobs []*spec.AgnosticBlobSidecar + txs []spec.AgnosticTransaction + want []string + wantErr bool + }{ + { + // Mainnet slot 14414626. Three consecutive transactions each carried + // the same near-empty blob, so all three shared one versioned hash. + // Matching on that hash collapsed every copy onto transaction 132. + name: "transactions carrying identical blobs keep their own", + blobs: sidecars(1, 2, 3, 4, 5, 9, 9, 9, 6, 7), + txs: []spec.AgnosticTransaction{ + carrier(9, 0xaa, 1, 2, 3, 4, 5), + carrier(130, 0xbb, 9), + carrier(131, 0xcc, 9), + carrier(132, 0xdd, 9), + carrier(201, 0xee, 6), + carrier(471, 0xff, 7), + }, + want: expectedAttribution( + 0, 0xaa, 1, 0xaa, 2, 0xaa, 3, 0xaa, 4, 0xaa, + 5, 0xbb, 6, 0xcc, 7, 0xdd, + 8, 0xee, 9, 0xff), + }, + { + name: "one transaction carrying the same blob twice", + blobs: sidecars(4, 4), + txs: []spec.AgnosticTransaction{carrier(3, 0xaa, 4, 4)}, + want: expectedAttribution(0, 0xaa, 1, 0xaa), + }, + { + name: "transactions without blobs are ignored", + blobs: sidecars(1, 2), + txs: []spec.AgnosticTransaction{ + {TxIdx: 0}, + carrier(1, 0xaa, 1), + {TxIdx: 2}, + carrier(3, 0xbb, 2), + }, + want: expectedAttribution(0, 0xaa, 1, 0xbb), + }, + { + // A transaction whose receipt never arrived is dropped while parsing, + // so the sidecars outnumber the blobs the transactions account for. + // Attributing them positionally would shift every blob onto the wrong + // transaction, so nothing is attributed at all. + name: "a missing transaction leaves the mapping untouched", + blobs: sidecars(1, 2, 3), + txs: []spec.AgnosticTransaction{ + carrier(1, 0xaa, 1), + carrier(9, 0xbb, 3), + }, + want: expectedAttribution(0, 0, 1, 0, 2, 0), + wantErr: true, + }, + { + // Nothing else in the block tells these apart, so an index the + // block never committed has to be refused rather than folded onto + // whichever transaction happens to sit nearby. + name: "a sidecar claiming an index past the block's blobs", + blobs: []*spec.AgnosticBlobSidecar{ + {Index: 0, BlobHash: blobHash(1).String()}, + {Index: 7, BlobHash: blobHash(2).String()}, + }, + txs: []spec.AgnosticTransaction{ + carrier(1, 0xaa, 1), + carrier(2, 0xbb, 2), + }, + want: expectedAttribution(0, 0, 7, 0), + wantErr: true, + }, + { + name: "two sidecars claiming the same index", + blobs: []*spec.AgnosticBlobSidecar{ + {Index: 0, BlobHash: blobHash(1).String()}, + {Index: 0, BlobHash: blobHash(1).String()}, + }, + txs: []spec.AgnosticTransaction{ + carrier(1, 0xaa, 1), + carrier(2, 0xbb, 1), + }, + want: expectedAttribution(0, 0, 0, 0), + wantErr: true, + }, + { + // Sidecars need not arrive in index order. + name: "sidecars given out of order", + blobs: []*spec.AgnosticBlobSidecar{ + {Index: 2, BlobHash: blobHash(3).String()}, + {Index: 0, BlobHash: blobHash(1).String()}, + {Index: 1, BlobHash: blobHash(2).String()}, + }, + txs: []spec.AgnosticTransaction{ + carrier(1, 0xaa, 1, 2), + carrier(2, 0xbb, 3), + }, + want: expectedAttribution(2, 0xbb, 0, 0xaa, 1, 0xaa), + }, + { + // Transaction order decides the mapping, not the order the caller + // happened to hand them over in. + name: "transactions given out of order", + blobs: sidecars(1, 2, 3), + txs: []spec.AgnosticTransaction{ + carrier(9, 0xbb, 3), + carrier(1, 0xaa, 1, 2), + }, + want: expectedAttribution(0, 0xaa, 1, 0xaa, 2, 0xbb), + }, + { + name: "sidecars that the transactions did not commit to", + blobs: sidecars(1, 8), + txs: []spec.AgnosticTransaction{ + carrier(1, 0xaa, 1), + carrier(2, 0xbb, 2), + }, + want: expectedAttribution(0, 0, 1, 0), + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := spec.AssignTxHashes(test.blobs, test.txs) + + if test.wantErr && err == nil { + t.Fatal("expected an error, got none") + } + if !test.wantErr && err != nil { + t.Fatalf("unexpected error: %s", err) + } + + got := attributed(test.blobs) + for i := range test.want { + if got[i] != test.want[i] { + t.Errorf("blob %d: got %s, want %s", i, got[i], test.want[i]) + } + } + }) + } +} From b7889c62b9423597db0519427d5d1e26948ed59c Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Fri, 28 Aug 2026 12:46:50 +0200 Subject: [PATCH 18/47] fix(reorg): stop AdvanceFinalized orphaning rows it never rewrites DeleteStateMetrics(E) removed six things while ProcessStateTransitionMetrics(E) writes back three. The other three belong to later epochs: written by processing E deleted by DeleteStateMetrics(E) epoch_metrics(E-1) epoch_metrics(E-1) rewritten proposer_duties(E) epoch_metrics(E) orphaned rewards(E) proposer_duties(E) rewritten rewards(E) rewritten rewards(E+1) orphaned rewards(E+2) orphaned epoch_metrics(E) is written by processing E+1 (ExportToEpoch uses CurrentState.Epoch), rewards(E+1) by E+1 and rewards(E+2) by E+2. The loop does reprocess those when the state at E was replaced, but only if it reaches them: epochs at or past finalizedEpoch are skipped, and the flags that would have marked them do not survive the call, so the next invocation no longer knows they are stale. Their rows stay deleted for good. Six such holes were live on mainnet when this was written, every one matching the signature exactly: epoch_metrics absent at E with E-1 and E+1 present, and t_validator_rewards_summary empty at E+1 and E+2 while E holds ~902k rows. The oldest had survived 774 epochs, so it is not indexing lag. New ones appear about every day and a half, and the two production deployments do not agree on which epochs are affected, both consistent with a boundary race rather than deterministic logic. Two changes: Delete only what this epoch's processing writes back. Rows belonging to E+1 and E+2 are deleted and rewritten when those epochs are processed. The invariant is one-directional: everything deleted is rewritten by the same call. Processing also writes slashings, deposits, consolidations, requests, last status, block rewards and pool metrics, none of which were deleted before this change either. Carry the epochs whose derived rows went stale but that this invocation could not reach, so a later one rewrites them. Without this the narrower delete would leave them holding pre-reorg values, which is worse than a hole: wrong data that looks present. What gets carried is keyed on the state at E having been replaced, not on its blocks having changed. Both a state root mismatch, which deletes and re-downloads the state, and a block change, which refreshes the state's derived fields, rewrite state E, and it is state E that epoch_metrics(E), rewards(E+1) and rewards(E+2) derive from. Keying on blocks alone would leave the root mismatch case holding stale values. The block case is recorded where it is detected rather than after the state root check, since that check can fail and skip the rest of the iteration. Epochs reprocessed only because a predecessor changed are not carried: their own state is untouched, so nothing downstream of them went stale, and propagating from them would cascade forward without end. The carried set lives in memory and is guarded by advanceFinalizedMu, which AdvanceFinalized already holds for its whole body. It does not survive a restart: an epoch carried but not yet reached when the process stops keeps its pre-reorg values instead of being rewritten. Making that durable needs somewhere to persist it and is left out of this change. Seven tests cover the carry-over rule against the real functions: that a dependent below the boundary is not carried, that a state replaced without any block change still is, that a carried epoch is consumed exactly once, and that consuming one does not cascade. They exercise the rule directly; which set AdvanceFinalized hands it is a single line that no test reaches, since that function needs a live beacon endpoint. The three realigned lines in chain_analyzer.go are gofmt: the new field is longer than the column the block was aligned to. That file was already unformatted on dev. Pre-existing test failures in pkg/analyzer are unchanged, 13 before and 13 after; they need a live beacon endpoint. Closes #285 --- pkg/analyzer/chain_analyzer.go | 8 ++- pkg/analyzer/reorg.go | 69 ++++++++++++++++++ pkg/analyzer/reorg_pending_test.go | 108 +++++++++++++++++++++++++++++ pkg/db/epoch_metrics.go | 76 ++++++++++---------- 4 files changed, 217 insertions(+), 44 deletions(-) create mode 100644 pkg/analyzer/reorg_pending_test.go diff --git a/pkg/analyzer/chain_analyzer.go b/pkg/analyzer/chain_analyzer.go index 034780be..8a7e8a66 100644 --- a/pkg/analyzer/chain_analyzer.go +++ b/pkg/analyzer/chain_analyzer.go @@ -52,12 +52,13 @@ type ChainAnalyzer struct { metrics db.DBMetrics // what metrics to be downloaded / processed processerBook *utils.RoutineBook // defines slot to process new metrics into the database, good for monitoring - downloadCache ChainCache // store the blocks and states downloaded + downloadCache ChainCache // store the blocks and states downloaded validatorsRewardsAggregations map[phase0.ValidatorIndex]*spec.ValidatorRewardsAggregation validatorsRewardsAggregationsMu sync.Mutex - advanceFinalizedMu sync.Mutex // serializes AdvanceFinalized so concurrent invocations cannot race CleanUpTo against in-flight Wait()s + advanceFinalizedMu sync.Mutex // serializes AdvanceFinalized so concurrent invocations cannot race CleanUpTo against in-flight Wait()s + pendingReprocess map[uint64]bool // epochs whose derived rows went stale when an earlier epoch was rewritten, and that AdvanceFinalized could not reach in that invocation (#285). Guarded by advanceFinalizedMu. aggregatedEpochsInWindow map[phase0.Epoch]bool // set of unique epochs aggregated in current window; prevents double-counting on reprocessing (#255) - epochBoundaryStateRoots sync.Map // slot -> phase0.Root, caches state roots from Head SSE events at epoch boundaries + epochBoundaryStateRoots sync.Map // slot -> phase0.Root, caches state roots from Head SSE events at epoch boundaries initTime time.Time PromMetrics *prom_metrics.PrometheusMetrics // metrics to be stored to prometheus @@ -180,6 +181,7 @@ func NewChainAnalyzer( downloadCache: NewQueue(), validatorsRewardsAggregations: make(map[phase0.ValidatorIndex]*spec.ValidatorRewardsAggregation), aggregatedEpochsInWindow: make(map[phase0.Epoch]bool), + pendingReprocess: make(map[uint64]bool), processerBook: utils.NewRoutineBook(32, "processer"), // one whole epoch wgMainRoutine: &sync.WaitGroup{}, wgDownload: &sync.WaitGroup{}, diff --git a/pkg/analyzer/reorg.go b/pkg/analyzer/reorg.go index 6990fbf6..deb778d7 100644 --- a/pkg/analyzer/reorg.go +++ b/pkg/analyzer/reorg.go @@ -36,6 +36,10 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { advance := false epochsWithChangedBlocks := make(map[uint64]bool) + // Epochs whose state object this invocation replaced, by re-download or by + // refreshing its blocks. The rows derived from those states are what goes + // stale, so this, not epochsWithChangedBlocks, is what gets carried. + epochsWithChangedState := make(map[uint64]bool) for _, epoch := range stateKeys { if epoch >= uint64(finalizedEpoch) { @@ -72,6 +76,10 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { if blocksChanged { epochsWithChangedBlocks[epoch] = true + // Refreshing the blocks below rewrites the derived fields of state + // epoch, so record it here rather than after the state root check: + // that check can fail and skip the rest of this iteration. + epochsWithChangedState[epoch] = true // Refresh the state's Blocks array so it points to the newly // downloaded block objects instead of the stale pre-reorg ones. if err := s.downloadCache.RefreshStateBlocks(s.ctx, epoch); err != nil { @@ -100,6 +108,10 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { // nextState (E) — proposer duties, epoch metrics // If blocks changed in any of those epochs, the derived metrics // for epoch E are stale and must be recomputed. + if stateRootChanged { + epochsWithChangedState[epoch] = true + } + needsReprocess := stateRootChanged || blocksChanged if epoch >= 1 && epochsWithChangedBlocks[epoch-1] { needsReprocess = true @@ -107,6 +119,9 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { if epoch >= 2 && epochsWithChangedBlocks[epoch-2] { needsReprocess = true } + if s.consumeCarriedEpoch(epoch) { + needsReprocess = true + } if needsReprocess { if stateRootChanged { @@ -136,6 +151,8 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { } } + s.carryStaleDependents(epochsWithChangedState, uint64(finalizedEpoch)) + s.downloadCache.CleanUpTo(newFinalizedSlot) if advance { @@ -143,6 +160,58 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { } } +// consumeCarriedEpoch reports whether this epoch was carried over by an earlier +// invocation that rewrote one of its predecessors, and clears the debt. +// +// The debt is cleared whether or not the rest of the iteration succeeds. An +// epoch that keeps failing would otherwise be retried on every finalized event +// for the lifetime of the process. +// +// Callers must hold advanceFinalizedMu. +func (s *ChainAnalyzer) consumeCarriedEpoch(epoch uint64) bool { + if !s.pendingReprocess[epoch] { + return false + } + delete(s.pendingReprocess, epoch) + log.Infof("reprocessing epoch %d carried over from an earlier AdvanceFinalized", epoch) + return true +} + +// carryStaleDependents records the epochs whose derived rows this invocation +// made stale but could not rewrite itself. +// +// Replacing the state at epoch E leaves rows belonging to later epochs stale: +// the epoch row for E is written by processing E+1 (ExportToEpoch reads +// CurrentState), and the validator rewards for E+1 and E+2 by processing E+1 +// and E+2, both of which read state E. AdvanceFinalized reprocesses those when +// its loop reaches them, but it skips everything at or past the finalized +// boundary, and the flags that would have marked them do not survive the call, +// so the next invocation no longer knows they are stale. Carrying them is what +// makes a later invocation rewrite them (#285). +// +// The caller passes the epochs whose state object it replaced, whether by +// re-downloading it after a state root mismatch or by refreshing its blocks. +// Both rewrite state E, and it is state E the stale rows derive from. Epochs +// reprocessed only because a predecessor changed do not belong here: their own +// state is untouched, so nothing downstream of them went stale. +// +// Only dependents at or past the boundary are carried. Anything below it the +// loop already reprocessed, and carrying it would repeat the work. +// +// Callers must hold advanceFinalizedMu. +func (s *ChainAnalyzer) carryStaleDependents(changed map[uint64]bool, finalizedEpoch uint64) { + if s.pendingReprocess == nil { + s.pendingReprocess = make(map[uint64]bool) + } + for epoch := range changed { + for _, dependent := range []uint64{epoch + 1, epoch + 2} { + if dependent >= finalizedEpoch { + s.pendingReprocess[dependent] = true + } + } + } +} + // ensureDependencyStates checks that the states required by // ProcessStateTransitionMetrics (epoch, epoch-1, epoch-2) are present in the // in-memory cache. Any missing state is re-downloaded from the beacon node, diff --git a/pkg/analyzer/reorg_pending_test.go b/pkg/analyzer/reorg_pending_test.go new file mode 100644 index 00000000..8c5a733a --- /dev/null +++ b/pkg/analyzer/reorg_pending_test.go @@ -0,0 +1,108 @@ +package analyzer + +import "testing" + +// AdvanceFinalized rewrites epoch E, which leaves rows derived from E stale: +// the epoch row for E is written by processing E+1, and the validator rewards +// for E+1 and E+2 by processing E+1 and E+2. Epochs at or past the finalized +// boundary are skipped, and the in-function flags that would have marked them +// do not survive the call. Without carrying that debt, nothing ever rewrites +// them (migalabs/goteth#285). + +func TestDependentsBelowTheBoundaryAreNotCarried(t *testing.T) { + // The loop reached them itself, so there is no debt. + s := &ChainAnalyzer{} + + s.carryStaleDependents(map[uint64]bool{100: true}, 200) + + if len(s.pendingReprocess) != 0 { + t.Errorf("carried %v; those epochs were processed in the same invocation", s.pendingReprocess) + } +} + +func TestDependentsAtTheBoundaryAreCarried(t *testing.T) { + // Blocks changed at 199 while 200 is the first unfinalized epoch, so 200 + // and 201 were skipped and their rows are stale. + s := &ChainAnalyzer{} + + s.carryStaleDependents(map[uint64]bool{199: true}, 200) + + for _, epoch := range []uint64{200, 201} { + if !s.pendingReprocess[epoch] { + t.Errorf("epoch %d was not carried; nothing would ever rewrite its rows", epoch) + } + } +} + +func TestOnlyTheUnreachableDependentIsCarried(t *testing.T) { + // 199 is below the boundary and was processed; 200 was not. + s := &ChainAnalyzer{} + + s.carryStaleDependents(map[uint64]bool{198: true}, 200) + + if s.pendingReprocess[199] { + t.Error("199 is below the boundary and was already processed; carrying it repeats work") + } + if !s.pendingReprocess[200] { + t.Error("200 is past the boundary and its rows are stale; it must be carried") + } +} + +// A carried epoch must be reprocessed once and then forgotten. Leaving the debt +// in place would retry it on every finalized event for the life of the process. +func TestACarriedEpochIsConsumedExactlyOnce(t *testing.T) { + s := &ChainAnalyzer{} + s.carryStaleDependents(map[uint64]bool{199: true}, 200) + + if !s.consumeCarriedEpoch(200) { + t.Fatal("the carried epoch was not reported as needing reprocessing") + } + if s.consumeCarriedEpoch(200) { + t.Error("the same epoch was reported twice; the debt was never cleared") + } +} + +func TestAnEpochThatWasNeverCarriedIsNotReprocessed(t *testing.T) { + s := &ChainAnalyzer{} + + if s.consumeCarriedEpoch(200) { + t.Error("an epoch nothing made stale was queued for reprocessing") + } +} + +// A state root mismatch makes AdvanceFinalized delete and re-download the state +// for that epoch, even when none of its blocks changed. The rows derived from +// that state go stale exactly as they would after a block change, so the carry +// keys on the state having been replaced rather than on the blocks. +func TestAStateReplacedWithoutBlockChangesIsStillCarried(t *testing.T) { + s := &ChainAnalyzer{} + + // What AdvanceFinalized records for an epoch where only the root differed. + s.carryStaleDependents(map[uint64]bool{199: true}, 200) + + for _, epoch := range []uint64{200, 201} { + if !s.pendingReprocess[epoch] { + t.Errorf("epoch %d was not carried after the state at 199 was replaced; "+ + "its rows keep pre-reorg values and nothing rewrites them", epoch) + } + } +} + +// Consuming a carried epoch must not enqueue its own dependents: it is being +// reprocessed because a predecessor changed, not because its own blocks did. +// Propagating from it would cascade forward with no end. +func TestConsumingACarriedEpochDoesNotCascade(t *testing.T) { + s := &ChainAnalyzer{} + s.carryStaleDependents(map[uint64]bool{199: true}, 200) + + s.consumeCarriedEpoch(200) + s.consumeCarriedEpoch(201) + + // The consumed epochs are not added to epochsWithChangedBlocks, so the + // next invocation carries nothing on their behalf. + s.carryStaleDependents(map[uint64]bool{}, 202) + + if len(s.pendingReprocess) != 0 { + t.Errorf("consuming carried epochs left %v queued", s.pendingReprocess) + } +} diff --git a/pkg/db/epoch_metrics.go b/pkg/db/epoch_metrics.go index ce55b546..834863df 100644 --- a/pkg/db/epoch_metrics.go +++ b/pkg/db/epoch_metrics.go @@ -202,64 +202,58 @@ func (p *DBService) RetrieveLastEpoch() (phase0.Epoch, error) { } -// delete metrics that use the state at epoch x +// DeleteStateMetrics removes rows ahead of ProcessStateTransitionMetrics(epoch) +// rewriting them. Every row it deletes must be one that the same call writes +// back; deleting anything else leaves a hole nothing fills. +// +// ProcessStateTransitionMetrics(E) writes the epoch row for E-1 (ExportToEpoch +// uses CurrentState.Epoch), the proposer duties for E and the validator rewards +// for E (both use NextState.Epoch). It writes nothing for E's own epoch row, +// nor for the rewards of E+1 and E+2: those belong to the processing of E+1 and +// E+2. +// +// The reverse does not hold, and is not required here. That call also writes +// slashings, deposits, consolidations, withdrawal and deposit requests, +// validator last status, block rewards and pool metrics, none of which are +// deleted first. Those are left as they were before this change. +// +// This used to delete those three as well. When the changed epoch sat close to +// the finalized boundary, AdvanceFinalized skipped E+1 and E+2 (they are not +// below finalizedEpoch yet) and nothing ever rewrote them, leaving permanent +// holes: epoch_metrics(E) absent with E-1 and E+1 present, and +// t_validator_rewards_summary empty for E+1 and E+2. Six such holes were live +// on mainnet, all matching that signature exactly (migalabs/goteth#285). +// +// Rows belonging to E+1 and E+2 are deleted and rewritten when those epochs are +// processed, which AdvanceFinalized now guarantees by carrying them over to a +// later invocation if it cannot reach them in this one. func (s *DBService) DeleteStateMetrics(epoch phase0.Epoch) error { - var err error - - // epochs are written at currentState using current state and nextState - err = s.Delete(DeletableObject{ + // The epoch row written by this epoch's processing is the one for epoch-1. + if err := s.Delete(DeletableObject{ query: deleteEpochsQuery, table: epochsTable, args: []any{epoch - 1}, - }) // when deleteState -> nextState - - if err != nil { - return err - } - err = s.Delete(DeletableObject{ - query: deleteEpochsQuery, - table: epochsTable, - args: []any{epoch}, - }) // when deleteState -> currentState - if err != nil { + }); err != nil { return err } - // proposer duties are writter using nextState - err = s.Delete(DeletableObject{ + // Proposer duties are written using nextState, so they belong to epoch. + if err := s.Delete(DeletableObject{ query: deleteProposerDutiesQuery, table: proposerDutiesTable, args: []any{epoch}, - }) - if err != nil { + }); err != nil { return err } - // valRewards are written at nextState using prevState, currentState and nextState - err = s.Delete(DeletableObject{ - query: deleteValidatorRewardsInEpochQuery, - table: valRewardsTable, - args: []any{epoch + 2}, - }) // when deleteState -> prevState - if err != nil { - return err - } - err = s.Delete(DeletableObject{ - query: deleteValidatorRewardsInEpochQuery, - table: valRewardsTable, - args: []any{epoch + 1}, - }) // when deleteState -> currentState - if err != nil { - return err - } - err = s.Delete(DeletableObject{ + // Validator rewards are written at nextState, so they belong to epoch. + if err := s.Delete(DeletableObject{ query: deleteValidatorRewardsInEpochQuery, table: valRewardsTable, args: []any{epoch}, - }) // when deleteState -> nextState - if err != nil { + }); err != nil { return err } - return nil + return nil } From 8d40fd25bfa8f39605da00fd7141d9e5206816d3 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Tue, 1 Sep 2026 10:33:34 +0200 Subject: [PATCH 19/47] fix(electra): make the deposit recompute idempotent instead of guarded Addresses the review on #289. The previous version guarded one call site: PreProcessBundle checked PendingDepositsProcessed before recomputing from CurrentState. The other path, processPendingDeposits -> processPendingDepositsFor(NextState), still ran unconditionally on every ProcessStateTransitionMetrics call, so an epoch reprocessed against a state still held in cache accumulated its counters a second time and doubled them. A dependency-triggered pass in AdvanceFinalized, a carried reprocess, or a RefreshStateBlocks failure that left the flag unset all reach that path. The guard now lives in processPendingDepositsFor, where every path meets, and the caller-side condition is gone. Guarding at a call site only ever protects that call site. TestRunningTwiceOnOneStateDoublesTheCounters asserted the doubling as intended behaviour. It pinned the bug. Replaced with tests that assert idempotence for the scalar counters, for the per-validator amounts, and for the NextState path specifically. Found while writing those: RefreshBlocks zeroes DepositsNum and TotalDepositsAmount but left DepositedAmounts populated, so a refreshed state recomputed to correct scalars and doubled per-validator amounts. Its own doc says it "resets the accumulators first so that calling it on a state whose blocks were already set does not double-count values", and DepositedAmounts is an accumulator it missed. Reset it there, with a test that fails without the line. Five mutations, all caught. --- pkg/spec/metrics/deposit_recompute_test.go | 119 +++++++++++++++++++-- pkg/spec/metrics/state_electra.go | 28 +++-- pkg/spec/state.go | 6 ++ 3 files changed, 136 insertions(+), 17 deletions(-) diff --git a/pkg/spec/metrics/deposit_recompute_test.go b/pkg/spec/metrics/deposit_recompute_test.go index 5227a0d1..50468e7c 100644 --- a/pkg/spec/metrics/deposit_recompute_test.go +++ b/pkg/spec/metrics/deposit_recompute_test.go @@ -88,17 +88,124 @@ func TestAnEmptyQueueStillMarksTheState(t *testing.T) { // Why PendingDepositsProcessed has to gate the call: the counters accumulate // with +=, so a second run on the same object doubles them. This pins the // hazard the flag exists to prevent. -func TestRunningTwiceOnOneStateDoublesTheCounters(t *testing.T) { +func TestRunningTwiceOnOneStateLeavesTheCountersUnchanged(t *testing.T) { + // The counters accumulate with +=, so a second pass over the same state + // object would double them. The same object legitimately reaches this + // function twice - once as NextState during its own epoch, once as + // CurrentState an epoch later - so idempotence is what makes the second + // call safe rather than a guard at one call site. p := &ElectraMetrics{} state := stateWithOnePendingDeposit() p.processPendingDepositsFor(state) - once := state.DepositsNum + num, amount, deposits := state.DepositsNum, state.TotalDepositsAmount, len(state.DepositsProcessed) + + p.processPendingDepositsFor(state) + + if state.DepositsNum != num { + t.Errorf("DepositsNum went from %d to %d on a second run", num, state.DepositsNum) + } + if state.TotalDepositsAmount != amount { + t.Errorf("TotalDepositsAmount went from %d to %d on a second run", amount, state.TotalDepositsAmount) + } + if len(state.DepositsProcessed) != deposits { + t.Errorf("DepositsProcessed went from %d to %d entries on a second run", deposits, len(state.DepositsProcessed)) + } +} + +func TestPerValidatorAmountsDoNotDoubleOnASecondRun(t *testing.T) { + // DepositedAmounts is accumulated per validator with += as well, and it + // feeds the per-validator deposit rows rather than the epoch counters, so + // it needs its own assertion. + p := &ElectraMetrics{} + state := stateWithOnePendingDeposit() + + p.processPendingDepositsFor(state) + before := make(map[phase0.ValidatorIndex]phase0.Gwei, len(state.DepositedAmounts)) + for idx, amount := range state.DepositedAmounts { + before[idx] = amount + } + if len(before) == 0 { + t.Fatal("no per-validator amounts were recorded; the fixture no longer exercises this") + } + + p.processPendingDepositsFor(state) + + for idx, amount := range before { + if state.DepositedAmounts[idx] != amount { + t.Errorf("validator %d went from %d to %d on a second run", idx, amount, state.DepositedAmounts[idx]) + } + } +} + +func TestTheNextStatePathIsGuardedToo(t *testing.T) { + // The failure the guard-at-one-call-site version missed: processPendingDeposits + // runs unconditionally on every ProcessStateTransitionMetrics call, so an + // epoch reprocessed against a state still held in cache - a dependency pass + // in AdvanceFinalized, a carried reprocess, a refresh that failed - went + // through this path a second time and doubled the counters. + state := stateWithOnePendingDeposit() + p := &ElectraMetrics{} + p.baseMetrics.NextState = state + + p.processPendingDeposits() + num := state.DepositsNum + + p.processPendingDeposits() + + if state.DepositsNum != num { + t.Errorf("reprocessing through the NextState path doubled %d to %d", num, state.DepositsNum) + } +} + +func TestRefreshingAStateAllowsItToBeCountedAgain(t *testing.T) { + // Idempotence must not become "counted once, ever". RefreshBlocks zeroes + // the counters, so it clears the flag, and the next pass has to refill them + // or the epoch row reports no deposits (#287 in the other direction). + p := &ElectraMetrics{} + state := stateWithOnePendingDeposit() + + p.processPendingDepositsFor(state) + num := state.DepositsNum + if num == 0 { + t.Fatal("fixture processed no deposits") + } + + state.RefreshBlocks(nil) + if state.PendingDepositsProcessed { + t.Fatal("RefreshBlocks left the state marked as already accumulated") + } + + p.processPendingDepositsFor(state) + if state.DepositsNum != num { + t.Errorf("after a refresh the counters came back as %d, want %d", state.DepositsNum, num) + } +} + +func TestARefreshResetsThePerValidatorAmountsToo(t *testing.T) { + // RefreshBlocks promises to reset the accumulators so that recalculating + // does not double-count. DepositedAmounts accumulates with += and was left + // populated, so the scalar counters came back correct while the + // per-validator amounts came back doubled. + p := &ElectraMetrics{} + state := stateWithOnePendingDeposit() + + p.processPendingDepositsFor(state) + want := make(map[phase0.ValidatorIndex]phase0.Gwei, len(state.DepositedAmounts)) + for idx, amount := range state.DepositedAmounts { + want[idx] = amount + } + if len(want) == 0 { + t.Fatal("no per-validator amounts were recorded; the fixture no longer exercises this") + } + + state.RefreshBlocks(nil) p.processPendingDepositsFor(state) - if state.DepositsNum != once*2 { - t.Errorf("expected a second run to double %d to %d, got %d; if this no longer "+ - "holds, revisit whether PendingDepositsProcessed still needs to gate the call", - once, once*2, state.DepositsNum) + for idx, amount := range want { + if state.DepositedAmounts[idx] != amount { + t.Errorf("validator %d: %d after a refresh and recompute, want %d", + idx, state.DepositedAmounts[idx], amount) + } } } diff --git a/pkg/spec/metrics/state_electra.go b/pkg/spec/metrics/state_electra.go index 4d27eb8e..a58e4e7d 100644 --- a/pkg/spec/metrics/state_electra.go +++ b/pkg/spec/metrics/state_electra.go @@ -62,10 +62,10 @@ func (p *ElectraMetrics) PreProcessBundle() error { // read as exactly zero while t_deposits holds the rows. // // Recompute from the state itself, which is deterministic and costs - // nothing on the common path where the object survived (#287). - if !p.baseMetrics.CurrentState.PendingDepositsProcessed { - p.processPendingDepositsFor(p.baseMetrics.CurrentState) - } + // nothing on the common path where the object survived (#287): the + // function returns immediately for a state that has already been + // through it. + p.processPendingDepositsFor(p.baseMetrics.CurrentState) // FIX: Clear state maps before processing (state objects are reused between iterations) p.baseMetrics.CurrentState.ConsolidatedAmounts = make(map[phase0.ValidatorIndex]phase0.Gwei) @@ -885,15 +885,21 @@ func (p *ElectraMetrics) processPendingDeposits() { // derives depends only on the state passed in, so running it on a state that // was re-downloaded reproduces the numbers the original accumulation reached. // -// It must run at most once per state object. The counters are accumulated with -// +=, not assigned, so a second run doubles them; PendingDepositsProcessed -// records that a state has been through it and is what keeps that from -// happening. +// It is idempotent, and that is load-bearing rather than a nicety. The counters +// are accumulated with +=, not assigned, so a second run would double them, and +// the same state object legitimately reaches this function twice: once as +// NextState during its own epoch, and again as CurrentState an epoch later. +// Reprocessing adds more ways in - a dependency-triggered pass in +// AdvanceFinalized, a carried reprocess, a state retained in cache rather than +// re-downloaded. Guarding at one call site only protects that call site, so the +// guard lives here, where every path meets. // -// It is normally run while the state is NextState. The epoch row, however, -// reads the counters from CurrentState, one epoch later. See PreProcessBundle -// for why that gap has to be closed. +// RefreshBlocks clears the flag, because it zeroes the counters this sets. func (p *ElectraMetrics) processPendingDepositsFor(state *spec.AgnosticState) { + if state.PendingDepositsProcessed { + return + } + nextEpoch := state.Epoch + 1 availableForProcessing := state.DepositBalanceToConsume + phase0.Gwei(getActivationExitChurnLimit(state)) processedAmount := phase0.Gwei(0) diff --git a/pkg/spec/state.go b/pkg/spec/state.go index 58fe731f..d02d7f09 100644 --- a/pkg/spec/state.go +++ b/pkg/spec/state.go @@ -150,6 +150,12 @@ func (p *AgnosticState) RefreshBlocks(blockList []*AgnosticBlock) { // refreshed state claim numbers it no longer has, and nothing would // recompute them. p.PendingDepositsProcessed = false + // DepositedAmounts accumulates per validator with +=, so it is one of the + // accumulators this function promises to reset. Zeroing DepositsNum while + // leaving this populated meant the next pass added a second time and the + // per-validator amounts doubled, even though the scalar counters looked + // right. + p.DepositedAmounts = make(map[phase0.ValidatorIndex]phase0.Gwei) p.NumAttestations = 0 p.SyncCommitteeParticipation = 0 p.AddBlocks(blockList) From df5bc904462d2083506cab9ba8bf0bd9f8cd5eb7 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Tue, 1 Sep 2026 11:02:30 +0200 Subject: [PATCH 20/47] fix(blobs): attribute from the block body instead of the receipt-parsed transactions Addresses the review on #290. AssignTxHashes took AgnosticTransactions, which ParseTransactionsFromBlock builds by matching transactions against EL receipts and which only contains the ones that matched. A receipt that had not arrived therefore dropped a blob carrier from the list, the block then appeared to commit fewer blobs than there were sidecars, the consistency check refused the whole slot, and every blob in it was persisted with an empty tx_hash. The code this replaced would still have attributed the blobs of the transactions that were present, so this was a regression. Partial receipt fetches are common enough to have their own recovery path in recoverBlockReceipts, so attribution must not depend on them. Everything the mapping needs is in the block body: each type-3 transaction's hash and the blob hashes it committed. AssignTxHashes now takes block.ExecutionPayload.Transactions and decodes them itself, so receipts are not an input at all. That field is populated from block..Message.Body.ExecutionPayload.Transactions for every fork that can carry blobs, Fulu included. The block's transaction order is authoritative, which also removes the sort the previous version needed to put receipt-parsed transactions back in order. The strict checks stay, and now mean what they say: a mismatch is genuinely a block whose sidecars are not the ones it committed to, not a slow execution client. processBlobSidecars loses its transactions parameter, along with the len(txs) > 0 guard that existed only because AgnosticTransactions could legitimately be empty. Also fixes a panic that predates this branch. Index is a uint64 from the beacon API, and the bounds check narrowed it to int before comparing. Any index at or above 2^63 wraps negative, passes the "past the end" check because a negative is below the length, and then panics indexing the seen array, taking block processing down with it. The comparison now happens before the narrowing. An undecodable transaction aborts attribution for the slot rather than being skipped. Skipping one would leave every later position shifted with no way to tell, since the undecodable transaction may itself have been a blob carrier. Tests are rebuilt on encoded transactions rather than synthetic structs, so each hash is derived from its own transaction instead of chosen. They are unsigned, so the hashes are not ones that would appear on chain; attribution does not depend on that, only on the hashes being derived and distinct. Added the case the review asked for, and three more found while auditing: a block that commits more blobs than arrived, which the index-bound check cannot see and which would otherwise publish a slot quietly short a blob; the wrapping index above; and an assertion that the error counts blob-carrying transactions rather than all transactions. Seven mutations, all caught. --- pkg/analyzer/process_block.go | 13 +- pkg/spec/blob.go | 61 ++++--- pkg/spec/blob_test.go | 308 +++++++++++++++++++++++----------- 3 files changed, 253 insertions(+), 129 deletions(-) diff --git a/pkg/analyzer/process_block.go b/pkg/analyzer/process_block.go index e5e47d15..3774372d 100644 --- a/pkg/analyzer/process_block.go +++ b/pkg/analyzer/process_block.go @@ -63,7 +63,7 @@ func (s *ChainAnalyzer) ProcessETH1Data(block *spec.AgnosticBlock) { } if block.HardForkVersion >= eth2_client_spec.DataVersionDeneb && s.metrics.BlobSidecars { - s.processBlobSidecars(block, block.ExecutionPayload.AgnosticTransactions) + s.processBlobSidecars(block) } } @@ -212,7 +212,7 @@ func (s *ChainAnalyzer) recoverBlockReceipts(block *spec.AgnosticBlock) { log.Infof("slot %d: recovered %d transaction receipts for fee calculation", block.Slot, len(txs)) } -func (s *ChainAnalyzer) processBlobSidecars(block *spec.AgnosticBlock, txs []spec.AgnosticTransaction) { +func (s *ChainAnalyzer) processBlobSidecars(block *spec.AgnosticBlock) { var blobs []*spec.AgnosticBlobSidecar var err error @@ -226,10 +226,11 @@ func (s *ChainAnalyzer) processBlobSidecars(block *spec.AgnosticBlock, txs []spe log.Errorf("could not download blobs for slot %d: %s", block.Slot, err) } if len(blobs) > 0 { - if len(txs) > 0 { - if err := spec.AssignTxHashes(blobs, txs); err != nil { - log.Errorf("slot %d: could not attribute blob sidecars to transactions: %s", block.Slot, err) - } + // The block's own transaction list, not AgnosticTransactions: the + // latter is filtered by which EL receipts arrived, and a missing + // receipt would cost this slot every one of its blob attributions. + if err := spec.AssignTxHashes(blobs, block.ExecutionPayload.Transactions); err != nil { + log.Errorf("slot %d: could not attribute blob sidecars to transactions: %s", block.Slot, err) } s.dbClient.PersistBlobSidecars(blobs) } diff --git a/pkg/spec/blob.go b/pkg/spec/blob.go index 72138ea1..d3b29eb1 100644 --- a/pkg/spec/blob.go +++ b/pkg/spec/blob.go @@ -4,13 +4,14 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - "sort" "time" api "github.com/attestantio/go-eth2-client/api/v1" + "github.com/attestantio/go-eth2-client/spec/bellatrix" "github.com/attestantio/go-eth2-client/spec/deneb" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/migalabs/goteth/pkg/utils" ) @@ -59,26 +60,25 @@ func NewAgnosticBlobFromAPI(slot phase0.Slot, blob deneb.BlobSidecar) (*Agnostic // awards every copy to whichever of them is compared last, and leaves the // others with no blobs at all. // +// It reads the block's own transaction list rather than the parsed +// AgnosticTransactions, and that is deliberate. Everything needed here - each +// type-3 transaction's hash and the blob hashes it committed - is in the block +// body. AgnosticTransactions are built by matching transactions to EL receipts +// and only include the ones that matched, so a receipt that has not arrived +// drops a blob carrier from the list. Attribution would then see fewer blobs +// committed than sidecars delivered and refuse the whole slot, leaving every +// blob in it with no transaction. Partial receipt fetches happen often enough +// to have their own recovery path (recoverBlockReceipts), so this must not +// depend on them. +// // Nothing is assigned until the whole block has been checked: the sidecars must // account for exactly the blobs the transactions committed to, each index must // appear once, and every hash must agree with the one committed at that -// position. Any disagreement means these sidecars are not the ones this block's -// transactions committed to, or that a transaction went missing because its -// receipt never arrived, and the positions are then meaningless. In that case -// the mapping is left untouched and an error returned, rather than shifting -// every blob onto the wrong transaction. -func AssignTxHashes(blobs []*AgnosticBlobSidecar, txs []AgnosticTransaction) error { - - carriers := make([]AgnosticTransaction, 0, len(blobs)) - - for _, tx := range txs { - if len(tx.BlobHashes) == 0 { - continue // this tx does not reference any blobs - } - carriers = append(carriers, tx) - } - - sort.Slice(carriers, func(i, j int) bool { return carriers[i].TxIdx < carriers[j].TxIdx }) +// position. Because the input no longer depends on receipts, a disagreement now +// means the sidecars are genuinely not the ones this block committed to, and +// leaving the mapping untouched is the right answer rather than a symptom of a +// slow EL. +func AssignTxHashes(blobs []*AgnosticBlobSidecar, blockTxs []bellatrix.Transaction) error { // The block's commitments, flattened in transaction order: one entry per // blob, holding the transaction that carried it and the hash it committed. @@ -86,25 +86,38 @@ func AssignTxHashes(blobs []*AgnosticBlobSidecar, txs []AgnosticTransaction) err txHash common.Hash blobHash string } + committed := make([]commitment, 0, len(blobs)) - for _, tx := range carriers { - for _, blobHash := range tx.BlobHashes { - committed = append(committed, commitment{common.Hash(tx.Hash), blobHash.String()}) + carriers := 0 + for idx, raw := range blockTxs { + var tx types.Transaction + if err := tx.UnmarshalBinary(raw); err != nil { + return fmt.Errorf("transaction %d could not be decoded: %w", idx, err) + } + if tx.Type() != types.BlobTxType { + continue + } + carriers++ + for _, blobHash := range tx.BlobHashes() { + committed = append(committed, commitment{tx.Hash(), blobHash.String()}) } } if len(committed) != len(blobs) { return fmt.Errorf("%d blob sidecars but %d blobs committed by %d transactions", - len(blobs), len(committed), len(carriers)) + len(blobs), len(committed), carriers) } seen := make([]bool, len(committed)) for _, blob := range blobs { - index := int(blob.Index) - if index >= len(committed) { + // Compared before narrowing, not after. Index is a uint64, and + // converting one at or above 2^63 to int wraps it negative, which slips + // past a "too large" check and then panics indexing seen. + if uint64(blob.Index) >= uint64(len(committed)) { return fmt.Errorf("blob index %d is past the %d blobs this block committed", blob.Index, len(committed)) } + index := int(blob.Index) if seen[index] { return fmt.Errorf("two sidecars claim blob index %d", blob.Index) } diff --git a/pkg/spec/blob_test.go b/pkg/spec/blob_test.go index 10c8a20e..f687a45c 100644 --- a/pkg/spec/blob_test.go +++ b/pkg/spec/blob_test.go @@ -2,11 +2,14 @@ package spec_test import ( "fmt" + "strings" "testing" + "github.com/attestantio/go-eth2-client/spec/bellatrix" "github.com/attestantio/go-eth2-client/spec/deneb" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/holiman/uint256" "github.com/migalabs/goteth/pkg/spec" ) @@ -20,15 +23,56 @@ func blobHash(seed byte) common.Hash { return h } -func carrier(txIdx uint64, txSeed byte, blobSeeds ...byte) spec.AgnosticTransaction { - var txHash phase0.Hash32 - txHash[31] = txSeed +// blobTx encodes a type-3 transaction committing the given blob hashes, in the +// form a block's execution payload carries, and returns its hash. +// +// The transaction is unsigned, so the hash is not one that would appear on +// chain - a signature changes it. That does not matter here: what attribution +// needs is that each hash is derived from its own transaction rather than +// chosen by the test, and that two transactions differ. The nonce provides the +// second part, so two transactions committing identical blobs still hash +// differently, which is the situation that produced the bug. +func blobTx(t *testing.T, nonce uint64, blobSeeds ...byte) (bellatrix.Transaction, common.Hash) { + t.Helper() hashes := make([]common.Hash, 0, len(blobSeeds)) for _, seed := range blobSeeds { hashes = append(hashes, blobHash(seed)) } - return spec.AgnosticTransaction{TxIdx: txIdx, Hash: txHash, BlobHashes: hashes} + tx := types.NewTx(&types.BlobTx{ + ChainID: uint256.NewInt(1), + Nonce: nonce, + GasTipCap: uint256.NewInt(1), + GasFeeCap: uint256.NewInt(1), + Gas: 21000, + Value: uint256.NewInt(0), + BlobFeeCap: uint256.NewInt(1), + BlobHashes: hashes, + }) + raw, err := tx.MarshalBinary() + if err != nil { + t.Fatalf("encoding blob transaction: %s", err) + } + return bellatrix.Transaction(raw), tx.Hash() +} + +// plainTx encodes a transaction that carries no blobs. +func plainTx(t *testing.T, nonce uint64) bellatrix.Transaction { + t.Helper() + + tx := types.NewTx(&types.DynamicFeeTx{ + ChainID: uint256.NewInt(1).ToBig(), + Nonce: nonce, + GasTipCap: uint256.NewInt(1).ToBig(), + GasFeeCap: uint256.NewInt(1).ToBig(), + Gas: 21000, + Value: uint256.NewInt(0).ToBig(), + }) + raw, err := tx.MarshalBinary() + if err != nil { + t.Fatalf("encoding plain transaction: %s", err) + } + return bellatrix.Transaction(raw) } // sidecars builds the block's sidecar list, indexed in block order. @@ -51,12 +95,21 @@ func attributed(blobs []*spec.AgnosticBlobSidecar) []string { return got } -func expectedAttribution(pairs ...uint64) []string { - want := make([]string, 0, len(pairs)) - for i := 0; i < len(pairs); i += 2 { - var txHash phase0.Hash32 - txHash[31] = byte(pairs[i+1]) - want = append(want, fmt.Sprintf("%d:%s", pairs[i], common.Hash(txHash).String())) +// expect pairs each sidecar, in the order they were handed over, with the +// transaction hash it should end up carrying. +func expect(blobs []*spec.AgnosticBlobSidecar, hashes ...common.Hash) []string { + want := make([]string, 0, len(blobs)) + for i, b := range blobs { + want = append(want, fmt.Sprintf("%d:%s", b.Index, hashes[i].String())) + } + return want +} + +// unattributed is the expectation when nothing should be assigned at all. +func unattributed(blobs []*spec.AgnosticBlobSidecar) []string { + want := make([]string, 0, len(blobs)) + for _, b := range blobs { + want = append(want, fmt.Sprintf("%d:%s", b.Index, common.Hash{}.String())) } return want } @@ -64,130 +117,170 @@ func expectedAttribution(pairs ...uint64) []string { func TestAssignTxHashes(t *testing.T) { tests := []struct { name string - blobs []*spec.AgnosticBlobSidecar - txs []spec.AgnosticTransaction - want []string + build func(t *testing.T) ([]*spec.AgnosticBlobSidecar, []bellatrix.Transaction, []string) wantErr bool }{ { // Mainnet slot 14414626. Three consecutive transactions each carried // the same near-empty blob, so all three shared one versioned hash. - // Matching on that hash collapsed every copy onto transaction 132. - name: "transactions carrying identical blobs keep their own", - blobs: sidecars(1, 2, 3, 4, 5, 9, 9, 9, 6, 7), - txs: []spec.AgnosticTransaction{ - carrier(9, 0xaa, 1, 2, 3, 4, 5), - carrier(130, 0xbb, 9), - carrier(131, 0xcc, 9), - carrier(132, 0xdd, 9), - carrier(201, 0xee, 6), - carrier(471, 0xff, 7), + // Matching on that hash collapsed every copy onto the last of them. + name: "transactions carrying identical blobs keep their own", + build: func(t *testing.T) ([]*spec.AgnosticBlobSidecar, []bellatrix.Transaction, []string) { + t0, h0 := blobTx(t, 0, 1, 2, 3, 4, 5) + t1, h1 := blobTx(t, 1, 9) + t2, h2 := blobTx(t, 2, 9) + t3, h3 := blobTx(t, 3, 9) + t4, h4 := blobTx(t, 4, 6) + t5, h5 := blobTx(t, 5, 7) + + blobs := sidecars(1, 2, 3, 4, 5, 9, 9, 9, 6, 7) + txs := []bellatrix.Transaction{t0, t1, t2, t3, t4, t5} + want := expect(blobs, h0, h0, h0, h0, h0, h1, h2, h3, h4, h5) + return blobs, txs, want }, - want: expectedAttribution( - 0, 0xaa, 1, 0xaa, 2, 0xaa, 3, 0xaa, 4, 0xaa, - 5, 0xbb, 6, 0xcc, 7, 0xdd, - 8, 0xee, 9, 0xff), }, { - name: "one transaction carrying the same blob twice", - blobs: sidecars(4, 4), - txs: []spec.AgnosticTransaction{carrier(3, 0xaa, 4, 4)}, - want: expectedAttribution(0, 0xaa, 1, 0xaa), + // The review case: a blob transaction whose EL receipt has not + // arrived. Attribution reads the block body, so the receipt is + // irrelevant and every blob in the slot is still attributed. + name: "a blob transaction whose receipt never arrived", + build: func(t *testing.T) ([]*spec.AgnosticBlobSidecar, []bellatrix.Transaction, []string) { + t0, h0 := blobTx(t, 0, 1) + t1, h1 := blobTx(t, 1, 2) + t2, h2 := blobTx(t, 2, 3) + + blobs := sidecars(1, 2, 3) + // All three are in the block. Only one produced a receipt, so + // AgnosticTransactions would have held a single entry. + txs := []bellatrix.Transaction{t0, t1, t2} + return blobs, txs, expect(blobs, h0, h1, h2) + }, }, { - name: "transactions without blobs are ignored", - blobs: sidecars(1, 2), - txs: []spec.AgnosticTransaction{ - {TxIdx: 0}, - carrier(1, 0xaa, 1), - {TxIdx: 2}, - carrier(3, 0xbb, 2), + name: "one transaction carrying the same blob twice", + build: func(t *testing.T) ([]*spec.AgnosticBlobSidecar, []bellatrix.Transaction, []string) { + tx, h := blobTx(t, 0, 4, 4) + blobs := sidecars(4, 4) + return blobs, []bellatrix.Transaction{tx}, expect(blobs, h, h) }, - want: expectedAttribution(0, 0xaa, 1, 0xbb), }, { - // A transaction whose receipt never arrived is dropped while parsing, - // so the sidecars outnumber the blobs the transactions account for. - // Attributing them positionally would shift every blob onto the wrong - // transaction, so nothing is attributed at all. - name: "a missing transaction leaves the mapping untouched", - blobs: sidecars(1, 2, 3), - txs: []spec.AgnosticTransaction{ - carrier(1, 0xaa, 1), - carrier(9, 0xbb, 3), + name: "transactions without blobs are ignored", + build: func(t *testing.T) ([]*spec.AgnosticBlobSidecar, []bellatrix.Transaction, []string) { + t1, h1 := blobTx(t, 1, 1) + t3, h3 := blobTx(t, 3, 2) + blobs := sidecars(1, 2) + txs := []bellatrix.Transaction{plainTx(t, 0), t1, plainTx(t, 2), t3} + return blobs, txs, expect(blobs, h1, h3) }, - want: expectedAttribution(0, 0, 1, 0, 2, 0), - wantErr: true, }, { - // Nothing else in the block tells these apart, so an index the - // block never committed has to be refused rather than folded onto - // whichever transaction happens to sit nearby. - name: "a sidecar claiming an index past the block's blobs", - blobs: []*spec.AgnosticBlobSidecar{ - {Index: 0, BlobHash: blobHash(1).String()}, - {Index: 7, BlobHash: blobHash(2).String()}, + // Sidecars need not arrive in index order; the index decides, not + // the position in the slice. + name: "sidecars given out of order", + build: func(t *testing.T) ([]*spec.AgnosticBlobSidecar, []bellatrix.Transaction, []string) { + t0, h0 := blobTx(t, 0, 1, 2) + t1, h1 := blobTx(t, 1, 3) + blobs := []*spec.AgnosticBlobSidecar{ + {Index: 2, BlobHash: blobHash(3).String()}, + {Index: 0, BlobHash: blobHash(1).String()}, + {Index: 1, BlobHash: blobHash(2).String()}, + } + return blobs, []bellatrix.Transaction{t0, t1}, expect(blobs, h1, h0, h0) }, - txs: []spec.AgnosticTransaction{ - carrier(1, 0xaa, 1), - carrier(2, 0xbb, 2), + }, + { + // More sidecars than the block committed to. Attributing them + // positionally would shift every blob onto the wrong transaction. + name: "more sidecars than the block committed", + build: func(t *testing.T) ([]*spec.AgnosticBlobSidecar, []bellatrix.Transaction, []string) { + t0, _ := blobTx(t, 0, 1) + blobs := sidecars(1, 2, 3) + return blobs, []bellatrix.Transaction{t0}, unattributed(blobs) }, - want: expectedAttribution(0, 0, 7, 0), wantErr: true, }, { - name: "two sidecars claiming the same index", - blobs: []*spec.AgnosticBlobSidecar{ - {Index: 0, BlobHash: blobHash(1).String()}, - {Index: 0, BlobHash: blobHash(1).String()}, + // The other direction, and the one the index-bound check cannot + // see: every sidecar present is valid and in range, but the block + // committed a blob that never arrived. Attributing the ones that + // did would publish a slot that is quietly short a blob. + name: "fewer sidecars than the block committed", + build: func(t *testing.T) ([]*spec.AgnosticBlobSidecar, []bellatrix.Transaction, []string) { + t0, _ := blobTx(t, 0, 1, 2, 3) + blobs := sidecars(1, 2) + return blobs, []bellatrix.Transaction{t0}, unattributed(blobs) }, - txs: []spec.AgnosticTransaction{ - carrier(1, 0xaa, 1), - carrier(2, 0xbb, 1), + wantErr: true, + }, + { + name: "a sidecar claiming an index past the block's blobs", + build: func(t *testing.T) ([]*spec.AgnosticBlobSidecar, []bellatrix.Transaction, []string) { + t0, _ := blobTx(t, 0, 1) + t1, _ := blobTx(t, 1, 2) + blobs := []*spec.AgnosticBlobSidecar{ + {Index: 0, BlobHash: blobHash(1).String()}, + {Index: 7, BlobHash: blobHash(2).String()}, + } + return blobs, []bellatrix.Transaction{t0, t1}, unattributed(blobs) }, - want: expectedAttribution(0, 0, 0, 0), wantErr: true, }, { - // Sidecars need not arrive in index order. - name: "sidecars given out of order", - blobs: []*spec.AgnosticBlobSidecar{ - {Index: 2, BlobHash: blobHash(3).String()}, - {Index: 0, BlobHash: blobHash(1).String()}, - {Index: 1, BlobHash: blobHash(2).String()}, + // Index is a uint64 from the beacon API. Narrowing one at or above + // 2^63 to int wraps it negative, which passes a "too large" check + // and then panics on the bounds array, taking block processing with + // it. + name: "a sidecar with an index that wraps when narrowed", + build: func(t *testing.T) ([]*spec.AgnosticBlobSidecar, []bellatrix.Transaction, []string) { + t0, _ := blobTx(t, 0, 1) + blobs := []*spec.AgnosticBlobSidecar{ + {Index: deneb.BlobIndex(1 << 63), BlobHash: blobHash(1).String()}, + } + return blobs, []bellatrix.Transaction{t0}, unattributed(blobs) }, - txs: []spec.AgnosticTransaction{ - carrier(1, 0xaa, 1, 2), - carrier(2, 0xbb, 3), + wantErr: true, + }, + { + name: "two sidecars claiming the same index", + build: func(t *testing.T) ([]*spec.AgnosticBlobSidecar, []bellatrix.Transaction, []string) { + t0, _ := blobTx(t, 0, 1) + t1, _ := blobTx(t, 1, 1) + blobs := []*spec.AgnosticBlobSidecar{ + {Index: 0, BlobHash: blobHash(1).String()}, + {Index: 0, BlobHash: blobHash(1).String()}, + } + return blobs, []bellatrix.Transaction{t0, t1}, unattributed(blobs) }, - want: expectedAttribution(2, 0xbb, 0, 0xaa, 1, 0xaa), + wantErr: true, }, { - // Transaction order decides the mapping, not the order the caller - // happened to hand them over in. - name: "transactions given out of order", - blobs: sidecars(1, 2, 3), - txs: []spec.AgnosticTransaction{ - carrier(9, 0xbb, 3), - carrier(1, 0xaa, 1, 2), + name: "sidecars that the transactions did not commit to", + build: func(t *testing.T) ([]*spec.AgnosticBlobSidecar, []bellatrix.Transaction, []string) { + t0, _ := blobTx(t, 0, 1) + t1, _ := blobTx(t, 1, 2) + blobs := sidecars(1, 8) + return blobs, []bellatrix.Transaction{t0, t1}, unattributed(blobs) }, - want: expectedAttribution(0, 0xaa, 1, 0xaa, 2, 0xbb), + wantErr: true, }, { - name: "sidecars that the transactions did not commit to", - blobs: sidecars(1, 8), - txs: []spec.AgnosticTransaction{ - carrier(1, 0xaa, 1), - carrier(2, 0xbb, 2), + // A payload entry that is not a valid transaction is a decoding + // failure, not a mismatch, and must not be silently skipped. + name: "an undecodable transaction", + build: func(t *testing.T) ([]*spec.AgnosticBlobSidecar, []bellatrix.Transaction, []string) { + blobs := sidecars(1) + return blobs, []bellatrix.Transaction{{0xff, 0xff, 0xff}}, unattributed(blobs) }, - want: expectedAttribution(0, 0, 1, 0), wantErr: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - err := spec.AssignTxHashes(test.blobs, test.txs) + blobs, txs, want := test.build(t) + + err := spec.AssignTxHashes(blobs, txs) if test.wantErr && err == nil { t.Fatal("expected an error, got none") @@ -196,12 +289,29 @@ func TestAssignTxHashes(t *testing.T) { t.Fatalf("unexpected error: %s", err) } - got := attributed(test.blobs) - for i := range test.want { - if got[i] != test.want[i] { - t.Errorf("blob %d: got %s, want %s", i, got[i], test.want[i]) + got := attributed(blobs) + for i := range want { + if got[i] != want[i] { + t.Errorf("blob %d: got %s, want %s", i, got[i], want[i]) } } }) } } + +func TestAssignTxHashesCountsOnlyBlobCarriersInItsError(t *testing.T) { + // The error is the only thing anyone sees when a slot goes unattributed, so + // the transaction count in it has to mean "transactions that committed + // blobs", not "transactions in the block". Counting plain transactions here + // would send whoever reads the log looking in the wrong place. + t1, _ := blobTx(t, 1, 1) + txs := []bellatrix.Transaction{plainTx(t, 0), t1, plainTx(t, 2), plainTx(t, 3)} + + err := spec.AssignTxHashes(sidecars(1, 2), txs) + if err == nil { + t.Fatal("expected a mismatch error, got none") + } + if !strings.Contains(err.Error(), "by 1 transactions") { + t.Errorf("error should report 1 blob-carrying transaction, got: %s", err) + } +} From e36ae45a9d35338597cc11bbb51bfd0b98e82260 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Tue, 1 Sep 2026 11:26:55 +0200 Subject: [PATCH 21/47] fix(blobs): name both hashes when a sidecar does not match its position From the review on #290. Every other check in AssignTxHashes reports the values it compared; the hash comparison reported only the index. That is the one most likely to need reading against the chain, because when several blobs in a block share a versioned hash "does not match" alone does not say which sidecar arrived where it should not have. The test asserts the order as well as the presence of both hashes. Naming them the wrong way round is worse than naming neither: it reads plausibly and sends whoever is debugging after the wrong sidecar. Mutation testing found exactly that gap - swapping the two arguments passed a test that only checked both strings appeared. --- pkg/spec/blob.go | 9 +++++++-- pkg/spec/blob_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/pkg/spec/blob.go b/pkg/spec/blob.go index d3b29eb1..1841dbd9 100644 --- a/pkg/spec/blob.go +++ b/pkg/spec/blob.go @@ -124,8 +124,13 @@ func AssignTxHashes(blobs []*AgnosticBlobSidecar, blockTxs []bellatrix.Transacti seen[index] = true if committed[index].blobHash != blob.BlobHash { - return fmt.Errorf("blob %d does not match the hash committed at that position", - blob.Index) + // Both hashes belong in the message. Every other check here reports + // the numbers it compared, and this is the one most likely to need + // reading against the chain: when several blobs in a block share a + // versioned hash, "does not match" alone gives nobody enough to + // tell which sidecar arrived where it should not have. + return fmt.Errorf("blob %d carries hash %s but this block committed %s at that position", + blob.Index, blob.BlobHash, committed[index].blobHash) } } diff --git a/pkg/spec/blob_test.go b/pkg/spec/blob_test.go index f687a45c..d8a1a601 100644 --- a/pkg/spec/blob_test.go +++ b/pkg/spec/blob_test.go @@ -315,3 +315,34 @@ func TestAssignTxHashesCountsOnlyBlobCarriersInItsError(t *testing.T) { t.Errorf("error should report 1 blob-carrying transaction, got: %s", err) } } + +func TestAssignTxHashesReportsBothHashesOnMismatch(t *testing.T) { + // The error is all anyone gets when a slot goes unattributed. Naming only + // the index leaves them unable to tell which sidecar landed where it should + // not have, which matters most in exactly the case this function exists + // for: several blobs in one block sharing a versioned hash. + tx, _ := blobTx(t, 0, 1) + arrived, committed := blobHash(8).String(), blobHash(1).String() + + err := spec.AssignTxHashes( + []*spec.AgnosticBlobSidecar{{Index: 0, BlobHash: arrived}}, + []bellatrix.Transaction{tx}, + ) + if err == nil { + t.Fatal("expected a hash mismatch error, got none") + } + msg := err.Error() + gotAt, wantAt := strings.Index(msg, arrived), strings.Index(msg, committed) + if gotAt < 0 { + t.Errorf("error omits the hash that arrived (%s): %s", arrived, msg) + } + if wantAt < 0 { + t.Errorf("error omits the hash the block committed (%s): %s", committed, msg) + } + // Naming both but the wrong way round is worse than naming neither: it + // reads plausibly and sends the reader looking for the wrong sidecar. The + // hash that arrived is described first, the one the block committed second. + if gotAt >= 0 && wantAt >= 0 && gotAt > wantAt { + t.Errorf("the hashes are the wrong way round; the arrived hash should come first: %s", msg) + } +} From ef0525adcab06073680305e2b4698ed716a391e6 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Tue, 1 Sep 2026 12:11:01 +0200 Subject: [PATCH 22/47] fix(reorg): rewrite reorged epoch metrics after the walk, lowest first Addresses the HandleReorg half of the review on #288. DeleteStateMetrics is shared by AdvanceFinalized and HandleReorg, and only the first was taught the narrower rules. HandleReorg walks slots from the head downwards, so it reaches a higher epoch before the lower ones it derives from. ProcessStateTransitionMetrics(E) reads the states for E, E-1 and E-2 out of the cache, so rewriting E during the walk computed it from whichever of those the walk had not replaced yet: the stale pre-reorg copies. Under the previous wide delete those rows were removed and left as a visible hole. Under the narrow one they stay, holding values derived from stale state, and AdvanceFinalized will not correct them later because by then the state roots already match the finalized node. That trades a detectable gap for a silent wrong value, which is the worse of the two. The walk now only records which epochs had their state replaced, and the rewrites happen after it, so every rewrite reads states that have all been brought up to date. They run in ascending epoch order, which matters separately: ProcessStateTransitionMetrics already waits for epoch-1 before starting, because ProcessAttestations writes ManualReward onto block objects the following epoch reads back (#242). Running E before E-1 satisfied that barrier trivially, E-1 not having started, and then let E-1 mutate blocks E had already read. The rewrite loop takes its delete and process as parameters so the order can be asserted without a database or a beacon node. It also deletes each epoch immediately before rewriting it rather than clearing everything up front, which would leave every epoch after the first deleted and not yet written for as long as the rewrites take. ProcessStateTransitionMetrics is synchronous - it is only launched with go from the live path in routines.go - so the sequential calls here genuinely serialise and the ordering does what it claims. Five mutations, all caught. --- pkg/analyzer/reorg.go | 66 ++++++++++++- pkg/analyzer/reorg_order_test.go | 161 +++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 4 deletions(-) create mode 100644 pkg/analyzer/reorg_order_test.go diff --git a/pkg/analyzer/reorg.go b/pkg/analyzer/reorg.go index deb778d7..e639bf9a 100644 --- a/pkg/analyzer/reorg.go +++ b/pkg/analyzer/reorg.go @@ -255,6 +255,10 @@ func (s *ChainAnalyzer) HandleReorg(newReorg v1.ChainReorgEvent) { cacheHeadBlock := s.downloadCache.GetHeadBlock() i := cacheHeadBlock.Slot + // Epochs whose state the reorg replaced, collected here and rewritten after + // the walk. See rewriteStateMetrics for why not during it. + replaced := make([]phase0.Epoch, 0) + for reorgedSlots <= depth { // for every slot in the reorg block, err := s.downloadCache.BlockHistory.Wait(s.ctx, SlotTo[uint64](i)) // first check that it was already in the cache @@ -304,14 +308,68 @@ func (s *ChainAnalyzer) HandleReorg(newReorg v1.ChainReorgEvent) { return } + // Only note it here. Rewriting the metrics now would read the two + // preceding states, which sit at lower slots this walk has not + // reached yet and so are still the pre-reorg copies. if newState.StateRoot != oldState.StateRoot { - s.dbClient.DeleteStateMetrics(epoch) - log.Infof("rewriting metrics for epoch %d", epoch) - // write epoch metrics - s.ProcessStateTransitionMetrics(epoch) + replaced = append(replaced, epoch) } } i -= 1 } + s.rewriteStateMetrics(replaced) +} + +// rewriteStateMetrics rewrites the epoch metrics for every state the reorg +// replaced, once every replacement has been downloaded and in ascending epoch +// order. +// +// Both conditions matter, for different reasons. +// +// The walk above runs backwards, so it reaches a higher epoch before the lower +// ones it derives from. ProcessStateTransitionMetrics(E) reads the states for +// E, E-1 and E-2 out of the cache, so rewriting E during the walk computes it +// from whichever of those the walk has not replaced yet: the stale pre-reorg +// copies. Deferring every rewrite until the walk has finished means each one +// reads states that have all been brought up to date. +// +// Ascending order is the second half. ProcessStateTransitionMetrics already +// waits for epoch-1 to finish before starting, because ProcessAttestations +// writes ManualReward onto block objects that the following epoch reads back +// (goteth#242). Running E before E-1 satisfies that barrier trivially - E-1 has +// not started - and then lets E-1 mutate blocks E has already read. +func (s *ChainAnalyzer) rewriteStateMetrics(epochs []phase0.Epoch) { + rewriteInOrder(epochs, s.dbClient.DeleteStateMetrics, s.ProcessStateTransitionMetrics) +} + +// rewriteInOrder deletes and rewrites each epoch's metrics, lowest first, and +// deletes an epoch's rows immediately before rewriting them rather than +// clearing everything up front. Taking the two apart would leave every epoch +// after the first deleted and not yet written, which is the orphaning this +// branch exists to stop, for as long as the rewrites take. +// +// The delete and the process are passed in so the order can be asserted without +// a database or a beacon node. +func rewriteInOrder(epochs []phase0.Epoch, del func(phase0.Epoch) error, process func(phase0.Epoch)) { + for _, epoch := range ascendingEpochs(epochs) { + if err := del(epoch); err != nil { + log.Errorf("could not delete metrics for epoch %d before rewriting them: %s", epoch, err) + } + log.Infof("rewriting metrics for epoch %d", epoch) + process(epoch) + } +} + +// ascendingEpochs returns the epochs in the order they must be processed. The +// reorg walk collects them from the head downwards, which is the reverse of the +// order each epoch's inputs become correct in. +// +// It copies rather than sorting in place: the caller's slice records what the +// walk found, and reordering that under it would make the two disagree. +func ascendingEpochs(epochs []phase0.Epoch) []phase0.Epoch { + ordered := make([]phase0.Epoch, len(epochs)) + copy(ordered, epochs) + sort.Slice(ordered, func(i, j int) bool { return ordered[i] < ordered[j] }) + return ordered } diff --git a/pkg/analyzer/reorg_order_test.go b/pkg/analyzer/reorg_order_test.go new file mode 100644 index 00000000..1e7cbabb --- /dev/null +++ b/pkg/analyzer/reorg_order_test.go @@ -0,0 +1,161 @@ +package analyzer + +import ( + "errors" + "testing" + + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +var errDeleteFailed = errors.New("delete failed") + +// HandleReorg walks slots from the head downwards, so it meets a higher epoch +// before the lower ones that epoch derives from. Rewriting in that order +// computes each epoch from predecessors the walk has not corrected yet, so the +// epochs are collected during the walk and ordered before anything is written. + +func TestCollectedEpochsAreRewrittenLowestFirst(t *testing.T) { + // The order a backward walk produces. + walked := []phase0.Epoch{102, 101, 100} + + got := ascendingEpochs(walked) + + want := []phase0.Epoch{100, 101, 102} + for i := range want { + if got[i] != want[i] { + t.Fatalf("processing order %v, want %v", got, want) + } + } +} + +func TestOrderingLeavesTheCollectedSliceAlone(t *testing.T) { + // The caller's slice records what the walk found. Sorting it in place would + // make the record disagree with the walk. + walked := []phase0.Epoch{102, 101, 100} + + ascendingEpochs(walked) + + for i, want := range []phase0.Epoch{102, 101, 100} { + if walked[i] != want { + t.Errorf("the collected slice was reordered: %v", walked) + break + } + } +} + +func TestOrderingHandlesNothingToRewrite(t *testing.T) { + // A reorg that replaced no state at all is the common case: block roots + // changed, state roots did not. + if got := ascendingEpochs(nil); len(got) != 0 { + t.Errorf("got %v, want nothing to rewrite", got) + } + if got := ascendingEpochs([]phase0.Epoch{}); len(got) != 0 { + t.Errorf("got %v, want nothing to rewrite", got) + } +} + +func TestOrderingKeepsEveryEpochItWasGiven(t *testing.T) { + // Dropping one would leave an epoch deleted-but-not-rewritten, which is the + // orphaning this branch exists to stop. + walked := []phase0.Epoch{5, 3, 4, 1, 2} + + got := ascendingEpochs(walked) + + if len(got) != len(walked) { + t.Fatalf("got %d epochs, want %d: %v", len(got), len(walked), got) + } + seen := make(map[phase0.Epoch]bool, len(got)) + for _, e := range got { + seen[e] = true + } + for _, e := range walked { + if !seen[e] { + t.Errorf("epoch %d was collected but would never be rewritten", e) + } + } +} + +func TestOrderingIsAscendingWhateverTheInput(t *testing.T) { + // A reorg spanning several epochs can meet them in any order once the walk + // crosses boundaries, so the ordering must not assume the input is reversed. + for _, input := range [][]phase0.Epoch{ + {1, 2, 3}, + {3, 1, 2}, + {7}, + {9, 9, 8}, + } { + got := ascendingEpochs(input) + for i := 1; i < len(got); i++ { + if got[i-1] > got[i] { + t.Errorf("input %v produced %v, which is not ascending", input, got) + break + } + } + } +} + +// record captures the sequence of operations rewriteInOrder performs, so the +// order and the pairing can be asserted without a database. +type record struct{ steps []string } + +func (r *record) del(e phase0.Epoch) error { + r.steps = append(r.steps, "delete "+epochStr(e)) + return nil +} + +func (r *record) process(e phase0.Epoch) { + r.steps = append(r.steps, "process "+epochStr(e)) +} + +func epochStr(e phase0.Epoch) string { + return string(rune('0' + int(e))) +} + +func TestEachEpochIsDeletedImmediatelyBeforeItIsRewritten(t *testing.T) { + // Deleting everything up front would leave every epoch after the first + // deleted and not yet rewritten for as long as the rewrites take, which is + // the orphaning this branch exists to stop. + var r record + + rewriteInOrder([]phase0.Epoch{3, 1, 2}, r.del, r.process) + + want := []string{ + "delete 1", "process 1", + "delete 2", "process 2", + "delete 3", "process 3", + } + if len(r.steps) != len(want) { + t.Fatalf("got %v, want %v", r.steps, want) + } + for i := range want { + if r.steps[i] != want[i] { + t.Fatalf("got %v, want %v", r.steps, want) + } + } +} + +func TestARewriteStillHappensWhenTheDeleteFails(t *testing.T) { + // A failed delete must not skip the rewrite. Skipping would leave the + // pre-reorg rows in place with nothing to correct them, which is the silent + // wrong value this whole change is about. + var processed []phase0.Epoch + failing := func(phase0.Epoch) error { return errDeleteFailed } + + rewriteInOrder([]phase0.Epoch{4}, failing, func(e phase0.Epoch) { + processed = append(processed, e) + }) + + if len(processed) != 1 || processed[0] != 4 { + t.Errorf("epoch 4 was not rewritten after its delete failed: %v", processed) + } +} + +func TestNothingIsTouchedWhenNoStateWasReplaced(t *testing.T) { + var r record + + rewriteInOrder(nil, r.del, r.process) + + if len(r.steps) != 0 { + t.Errorf("a reorg that replaced no state still touched the database: %v", r.steps) + } +} From 4de43bd83a1e6d49c9afbbd171a9c64df87dbe96 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Tue, 1 Sep 2026 14:07:46 +0200 Subject: [PATCH 23/47] fix(reorg): guard the genesis epoch row, and carry an epoch whose own check failed Both from the review comments on #288. DeleteStateMetrics(0) computed epoch-1 on an unsigned type, wrapping to the maximum uint64 and addressing a row that cannot exist. Harmless today only because nothing matches it, which is not a property worth relying on. Reachable from a low initSlot or a fresh testnet, where HandleReorg derives epoch from slot/32 and a reorg near genesis gives epoch 0. The guard is scoped to the epoch row rather than the whole function. Processing epoch 0 writes no epoch row - there is nothing before genesis - but it does write the proposer duties and validator rewards for epoch 0, so those deletes still have to run. Returning early would have swapped an underflow for a hole. Second, AdvanceFinalized gave up on an epoch when it could not fetch the finalized state root, and the continue happened before the epoch was recorded or its carried debt consumed. Nothing else covers that case: carryStaleDependents carries an epoch's dependents, never an epoch whose own check failed, and CleanUpTo can evict the state before another invocation looks again. Rewriting anyway was suggested and is not safe: the rows would derive from a state that may still be the pre-reorg copy, which is the silent wrong value this branch exists to remove. The epoch is carried instead, so a later invocation decides with a root it could actually fetch. That debt re-arms for as long as the fetch keeps failing, which is the point. Three mutations, all caught. The test for the wrapped row is written through a variable on purpose: the compiler rejects the constant form as an overflow, which is exactly why this went unnoticed, since the real code wraps a variable silently. --- pkg/analyzer/process_state.go | 2 +- pkg/analyzer/reorg.go | 28 ++++++++++++++--- pkg/analyzer/reorg_pending_test.go | 41 +++++++++++++++++++++++++ pkg/db/block_rewards.go | 2 +- pkg/db/epoch_metrics.go | 29 ++++++++++++++---- pkg/db/epoch_metrics_test.go | 48 ++++++++++++++++++++++++++++++ pkg/db/validator_last_status.go | 8 ++--- 7 files changed, 142 insertions(+), 16 deletions(-) create mode 100644 pkg/db/epoch_metrics_test.go diff --git a/pkg/analyzer/process_state.go b/pkg/analyzer/process_state.go index 42be2e5c..eab4000d 100644 --- a/pkg/analyzer/process_state.go +++ b/pkg/analyzer/process_state.go @@ -414,7 +414,7 @@ func (s *ChainAnalyzer) getSingleBlockRewards( RewardFees: rewardFees, BurntFees: burntFees, Relays: relayAddresses, - BidCommission: bidCommission, + BidCommission: bidCommission, BuilderPubkeys: builderPubkeys, } } diff --git a/pkg/analyzer/reorg.go b/pkg/analyzer/reorg.go index e639bf9a..adffc399 100644 --- a/pkg/analyzer/reorg.go +++ b/pkg/analyzer/reorg.go @@ -96,6 +96,18 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { finalizedStateRoot, err := s.cli.RequestStateRoot(phase0.Slot(cacheState.Slot)) if err != nil { log.Errorf("could not get state root at slot %d: %s", cacheState.Slot, err) + // Whether this epoch's state is stale is now unknown, and both + // obvious answers are wrong. Rewriting anyway would derive the rows + // from a state that may still be the pre-reorg copy. Skipping + // silently drops the epoch: nothing carries one whose own check + // failed, blocksChanged does not survive the call, and CleanUpTo + // may evict the state before the next invocation ever looks again. + // + // Carry it instead, so a later invocation decides with a root it + // could actually fetch. Unlike a debt consumed on arrival, this one + // re-arms itself for as long as the fetch keeps failing, which is + // the point: the epoch stays owed until something can answer for it. + s.carryEpoch(epoch) continue } @@ -200,18 +212,26 @@ func (s *ChainAnalyzer) consumeCarriedEpoch(epoch uint64) bool { // // Callers must hold advanceFinalizedMu. func (s *ChainAnalyzer) carryStaleDependents(changed map[uint64]bool, finalizedEpoch uint64) { - if s.pendingReprocess == nil { - s.pendingReprocess = make(map[uint64]bool) - } for epoch := range changed { for _, dependent := range []uint64{epoch + 1, epoch + 2} { if dependent >= finalizedEpoch { - s.pendingReprocess[dependent] = true + s.carryEpoch(dependent) } } } } +// carryEpoch records that an epoch still owes a reprocess, so a later +// AdvanceFinalized picks it up. +// +// Callers must hold advanceFinalizedMu. +func (s *ChainAnalyzer) carryEpoch(epoch uint64) { + if s.pendingReprocess == nil { + s.pendingReprocess = make(map[uint64]bool) + } + s.pendingReprocess[epoch] = true +} + // ensureDependencyStates checks that the states required by // ProcessStateTransitionMetrics (epoch, epoch-1, epoch-2) are present in the // in-memory cache. Any missing state is re-downloaded from the beacon node, diff --git a/pkg/analyzer/reorg_pending_test.go b/pkg/analyzer/reorg_pending_test.go index 8c5a733a..3bae5025 100644 --- a/pkg/analyzer/reorg_pending_test.go +++ b/pkg/analyzer/reorg_pending_test.go @@ -106,3 +106,44 @@ func TestConsumingACarriedEpochDoesNotCascade(t *testing.T) { t.Errorf("consuming carried epochs left %v queued", s.pendingReprocess) } } + +func TestAnEpochWhoseOwnCheckFailedIsCarried(t *testing.T) { + // AdvanceFinalized gives up on an epoch when it cannot fetch the finalized + // state root, which leaves it unable to say whether that epoch's state is + // stale. Nothing else would carry it: carryStaleDependents only carries an + // epoch's dependents, never an epoch whose own check failed, and CleanUpTo + // can evict the state before another invocation looks again. + s := &ChainAnalyzer{} + + s.carryEpoch(500) + + if !s.consumeCarriedEpoch(500) { + t.Error("an epoch whose state root could not be fetched was not carried") + } +} + +func TestCarryingAnEpochTwiceOwesItOnce(t *testing.T) { + // The fetch can keep failing across invocations, re-arming the debt each + // time. Consuming it once must clear it. + s := &ChainAnalyzer{} + + s.carryEpoch(500) + s.carryEpoch(500) + + if !s.consumeCarriedEpoch(500) { + t.Fatal("epoch 500 was not carried") + } + if s.consumeCarriedEpoch(500) { + t.Error("epoch 500 was still owed after being reprocessed once") + } +} + +func TestCarryingOneEpochLeavesOthersAlone(t *testing.T) { + s := &ChainAnalyzer{} + + s.carryEpoch(500) + + if s.consumeCarriedEpoch(501) { + t.Error("carrying epoch 500 also marked 501 as owed") + } +} diff --git a/pkg/db/block_rewards.go b/pkg/db/block_rewards.go index 7d25c985..b0015993 100644 --- a/pkg/db/block_rewards.go +++ b/pkg/db/block_rewards.go @@ -91,5 +91,5 @@ type BlockReward struct { BurntFees *big.Int // Wei Relays []string BuilderPubkeys []string - BidCommission *big.Int // Wei + BidCommission *big.Int // Wei } diff --git a/pkg/db/epoch_metrics.go b/pkg/db/epoch_metrics.go index 834863df..ea967d06 100644 --- a/pkg/db/epoch_metrics.go +++ b/pkg/db/epoch_metrics.go @@ -227,14 +227,31 @@ func (p *DBService) RetrieveLastEpoch() (phase0.Epoch, error) { // Rows belonging to E+1 and E+2 are deleted and rewritten when those epochs are // processed, which AdvanceFinalized now guarantees by carrying them over to a // later invocation if it cannot reach them in this one. +// EpochRowWrittenBy returns the epoch row that processing the given epoch +// writes, and whether there is one at all. +// +// Processing epoch E writes the row for E-1. Processing epoch 0 writes no epoch +// row: there is nothing before genesis. phase0.Epoch is unsigned, so computing +// 0-1 wraps to the maximum uint64 and would address a row that cannot exist - +// harmless today only because nothing matches it, which is not a property worth +// relying on. +func EpochRowWrittenBy(epoch phase0.Epoch) (phase0.Epoch, bool) { + if epoch == 0 { + return 0, false + } + return epoch - 1, true +} + func (s *DBService) DeleteStateMetrics(epoch phase0.Epoch) error { // The epoch row written by this epoch's processing is the one for epoch-1. - if err := s.Delete(DeletableObject{ - query: deleteEpochsQuery, - table: epochsTable, - args: []any{epoch - 1}, - }); err != nil { - return err + if row, ok := EpochRowWrittenBy(epoch); ok { + if err := s.Delete(DeletableObject{ + query: deleteEpochsQuery, + table: epochsTable, + args: []any{row}, + }); err != nil { + return err + } } // Proposer duties are written using nextState, so they belong to epoch. diff --git a/pkg/db/epoch_metrics_test.go b/pkg/db/epoch_metrics_test.go new file mode 100644 index 00000000..9465efb4 --- /dev/null +++ b/pkg/db/epoch_metrics_test.go @@ -0,0 +1,48 @@ +package db + +import ( + "testing" + + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +// Processing epoch E writes the epoch row for E-1, which is what +// DeleteStateMetrics clears before the rewrite. Epoch 0 is the exception: +// there is no epoch before genesis. + +func TestProcessingAnEpochWritesThePreviousEpochsRow(t *testing.T) { + row, ok := EpochRowWrittenBy(10) + + if !ok { + t.Fatal("processing epoch 10 should write an epoch row") + } + if row != 9 { + t.Errorf("epoch row = %d, want 9", row) + } +} + +func TestGenesisWritesNoEpochRow(t *testing.T) { + // phase0.Epoch is unsigned, so 0-1 wraps to the maximum uint64 and would + // address a row that cannot exist. Nothing matches it today, which is why + // this never surfaced, but that is not a property to rely on. + row, ok := EpochRowWrittenBy(0) + + if ok { + t.Errorf("processing epoch 0 claimed to write epoch row %d", row) + } +} + +func TestNoEpochEverAddressesTheWrappedRow(t *testing.T) { + // Written through a variable on purpose. The compiler rejects the constant + // form as an overflow, which is exactly why this went unnoticed in the + // original code: there the epoch is a variable, so it wraps silently. + var zero phase0.Epoch + wrapped := zero - 1 + + for _, epoch := range []phase0.Epoch{0, 1, 2, 100, 471766} { + row, ok := EpochRowWrittenBy(epoch) + if ok && row == wrapped { + t.Errorf("epoch %d addresses the wrapped row %d", epoch, row) + } + } +} diff --git a/pkg/db/validator_last_status.go b/pkg/db/validator_last_status.go index 02496915..560c2101 100644 --- a/pkg/db/validator_last_status.go +++ b/pkg/db/validator_last_status.go @@ -33,10 +33,10 @@ var ( func valStatusInput(validatorStatuses []spec.ValidatorLastStatus) proto.Input { // one object per column var ( - f_val_idx proto.ColUInt64 - f_epoch proto.ColUInt64 - f_balance_eth proto.ColFloat32 - f_effective_balance proto.ColUInt64 + f_val_idx proto.ColUInt64 + f_epoch proto.ColUInt64 + f_balance_eth proto.ColFloat32 + f_effective_balance proto.ColUInt64 f_status proto.ColUInt8 f_slashed proto.ColBool f_activation_epoch proto.ColUInt64 From 8bbfc205ec7d6d5e194c0ec41fabb35f89c3bd94 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Tue, 1 Sep 2026 14:12:59 +0200 Subject: [PATCH 24/47] revert(reorg): the carry cannot cover an epoch whose own check failed The previous commit carried an epoch forward when its finalized state root could not be fetched. That does not work, and auditing it against the mechanism's own invariant is what showed why. carryStaleDependents carries only dependents at or past finalizedEpoch. That condition is load-bearing, not an optimisation. The loop skips epochs at or past the boundary, so it only ever visits epochs below it, and CleanUpTo then evicts every state below the finalized slot. An epoch carried from below the boundary is gone from StateHistory before any later invocation could reach it: the loop never visits it, consumeCarriedEpoch never fires, and the debt sits unpayable forever. A failed state root fetch happens below the boundary by definition, so the carry added for it was inert. It recorded a debt nothing could pay and made the code read as though the case were handled. Reverted to the plain continue, with a comment saying what is actually wrong and why neither obvious fix works: rewriting anyway derives the rows from a possibly pre-reorg state, and carrying cannot reach the epoch. A real fix needs the loop to revisit an epoch whose state has been evicted, which is a larger change and belongs with the other narrow edges in the tracking issue. The genesis guard from that commit is unaffected and stays. TestDependentsBelowTheBoundaryAreNotCarried already encoded this invariant; its comment now gives the eviction reason rather than only noting the loop reached them. --- pkg/analyzer/reorg.go | 45 +++++++++++++----------------- pkg/analyzer/reorg_pending_test.go | 45 ++---------------------------- 2 files changed, 22 insertions(+), 68 deletions(-) diff --git a/pkg/analyzer/reorg.go b/pkg/analyzer/reorg.go index adffc399..33445257 100644 --- a/pkg/analyzer/reorg.go +++ b/pkg/analyzer/reorg.go @@ -96,18 +96,13 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { finalizedStateRoot, err := s.cli.RequestStateRoot(phase0.Slot(cacheState.Slot)) if err != nil { log.Errorf("could not get state root at slot %d: %s", cacheState.Slot, err) - // Whether this epoch's state is stale is now unknown, and both - // obvious answers are wrong. Rewriting anyway would derive the rows - // from a state that may still be the pre-reorg copy. Skipping - // silently drops the epoch: nothing carries one whose own check - // failed, blocksChanged does not survive the call, and CleanUpTo - // may evict the state before the next invocation ever looks again. - // - // Carry it instead, so a later invocation decides with a root it - // could actually fetch. Unlike a debt consumed on arrival, this one - // re-arms itself for as long as the fetch keeps failing, which is - // the point: the epoch stays owed until something can answer for it. - s.carryEpoch(epoch) + // This epoch is now unresolved: its state may be stale and nothing + // here can tell. Rewriting anyway would derive the rows from a + // state that may still be the pre-reorg copy, and carrying it does + // not help either - see carryStaleDependents for why a debt below + // the finalized boundary can never be paid. Fixing this needs the + // loop to be able to revisit an epoch whose state has been evicted, + // which is a larger change than this one. continue } @@ -207,31 +202,29 @@ func (s *ChainAnalyzer) consumeCarriedEpoch(epoch uint64) bool { // reprocessed only because a predecessor changed do not belong here: their own // state is untouched, so nothing downstream of them went stale. // -// Only dependents at or past the boundary are carried. Anything below it the -// loop already reprocessed, and carrying it would repeat the work. +// Only dependents at or past the boundary are carried, and that condition is +// load-bearing rather than an optimisation. The loop skips epochs at or past +// finalizedEpoch, so it only ever visits epochs below it, and CleanUpTo then +// evicts every state below the finalized slot. An epoch carried from below the +// boundary is therefore gone from StateHistory before any later invocation +// could reach it: the loop never visits it, consumeCarriedEpoch never fires, +// and the debt sits unpayable. Only epochs at or past the boundary survive the +// eviction and become reachable once the boundary advances. // // Callers must hold advanceFinalizedMu. func (s *ChainAnalyzer) carryStaleDependents(changed map[uint64]bool, finalizedEpoch uint64) { + if s.pendingReprocess == nil { + s.pendingReprocess = make(map[uint64]bool) + } for epoch := range changed { for _, dependent := range []uint64{epoch + 1, epoch + 2} { if dependent >= finalizedEpoch { - s.carryEpoch(dependent) + s.pendingReprocess[dependent] = true } } } } -// carryEpoch records that an epoch still owes a reprocess, so a later -// AdvanceFinalized picks it up. -// -// Callers must hold advanceFinalizedMu. -func (s *ChainAnalyzer) carryEpoch(epoch uint64) { - if s.pendingReprocess == nil { - s.pendingReprocess = make(map[uint64]bool) - } - s.pendingReprocess[epoch] = true -} - // ensureDependencyStates checks that the states required by // ProcessStateTransitionMetrics (epoch, epoch-1, epoch-2) are present in the // in-memory cache. Any missing state is re-downloaded from the beacon node, diff --git a/pkg/analyzer/reorg_pending_test.go b/pkg/analyzer/reorg_pending_test.go index 3bae5025..09a0aa01 100644 --- a/pkg/analyzer/reorg_pending_test.go +++ b/pkg/analyzer/reorg_pending_test.go @@ -10,7 +10,9 @@ import "testing" // them (migalabs/goteth#285). func TestDependentsBelowTheBoundaryAreNotCarried(t *testing.T) { - // The loop reached them itself, so there is no debt. + // The loop reached them itself, so there is no debt - and a debt below the + // boundary could not be paid anyway: CleanUpTo evicts those states at the + // end of the invocation, so no later loop ever visits them again. s := &ChainAnalyzer{} s.carryStaleDependents(map[uint64]bool{100: true}, 200) @@ -106,44 +108,3 @@ func TestConsumingACarriedEpochDoesNotCascade(t *testing.T) { t.Errorf("consuming carried epochs left %v queued", s.pendingReprocess) } } - -func TestAnEpochWhoseOwnCheckFailedIsCarried(t *testing.T) { - // AdvanceFinalized gives up on an epoch when it cannot fetch the finalized - // state root, which leaves it unable to say whether that epoch's state is - // stale. Nothing else would carry it: carryStaleDependents only carries an - // epoch's dependents, never an epoch whose own check failed, and CleanUpTo - // can evict the state before another invocation looks again. - s := &ChainAnalyzer{} - - s.carryEpoch(500) - - if !s.consumeCarriedEpoch(500) { - t.Error("an epoch whose state root could not be fetched was not carried") - } -} - -func TestCarryingAnEpochTwiceOwesItOnce(t *testing.T) { - // The fetch can keep failing across invocations, re-arming the debt each - // time. Consuming it once must clear it. - s := &ChainAnalyzer{} - - s.carryEpoch(500) - s.carryEpoch(500) - - if !s.consumeCarriedEpoch(500) { - t.Fatal("epoch 500 was not carried") - } - if s.consumeCarriedEpoch(500) { - t.Error("epoch 500 was still owed after being reprocessed once") - } -} - -func TestCarryingOneEpochLeavesOthersAlone(t *testing.T) { - s := &ChainAnalyzer{} - - s.carryEpoch(500) - - if s.consumeCarriedEpoch(501) { - t.Error("carrying epoch 500 also marked 501 as owed") - } -} From df48665e2ddd1b825696f67ae04519a9713f81fe Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Tue, 1 Sep 2026 14:15:14 +0200 Subject: [PATCH 25/47] fix(reorg): make sure the states are still cached before rewriting Deferring the rewrites until after the backward walk widened the gap between downloading a state and using it, from a few statements to the length of the whole walk. HandleReorg holds no lock and AdvanceFinalized calls CleanUpTo, so a state downloaded early in the walk can be evicted before the rewrite reaches it. Either outcome of that is bad. ProcessStateTransitionMetrics blocks forever inside StateHistory.Wait for a missing state, holding processerBook slots until the pool leaks (#245), or its guard sees a missing state and writes nothing at all - after DeleteStateMetrics has already run. ensureDependencyStates re-downloads whatever is missing, and is what AdvanceFinalized already uses before its own reprocess for the same reason. It runs before the delete rather than before the process, so the second case cannot leave a hole: nothing is deleted until the states needed to rewrite it are present. It is a no-op when the states are cached, which is the normal case, since reorgs are above the finalized boundary CleanUpTo evicts below. --- pkg/analyzer/reorg.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pkg/analyzer/reorg.go b/pkg/analyzer/reorg.go index 33445257..044090c1 100644 --- a/pkg/analyzer/reorg.go +++ b/pkg/analyzer/reorg.go @@ -353,7 +353,21 @@ func (s *ChainAnalyzer) HandleReorg(newReorg v1.ChainReorgEvent) { // (goteth#242). Running E before E-1 satisfies that barrier trivially - E-1 has // not started - and then lets E-1 mutate blocks E has already read. func (s *ChainAnalyzer) rewriteStateMetrics(epochs []phase0.Epoch) { - rewriteInOrder(epochs, s.dbClient.DeleteStateMetrics, s.ProcessStateTransitionMetrics) + // Make sure the states are present before deleting anything. Deferring the + // rewrites until after the walk widened the gap between downloading a state + // and using it, and HandleReorg holds no lock: a concurrent + // AdvanceFinalized can CleanUpTo in between and evict one. Missing states + // would make ProcessStateTransitionMetrics either block forever inside + // StateHistory.Wait, holding processerBook slots (#245), or write nothing + // at all - after the rows had already been deleted. + // + // Ensuring before the delete rather than before the process is what keeps + // that last case from turning into a hole. + ensureThenDelete := func(epoch phase0.Epoch) error { + s.ensureDependencyStates(uint64(epoch)) + return s.dbClient.DeleteStateMetrics(epoch) + } + rewriteInOrder(epochs, ensureThenDelete, s.ProcessStateTransitionMetrics) } // rewriteInOrder deletes and rewrites each epoch's metrics, lowest first, and From 017b5abb3ba93191401a29fec685045866bf94c6 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Tue, 1 Sep 2026 14:16:01 +0200 Subject: [PATCH 26/47] chore: keep unrelated files out of this branch Running gofmt over whole directories and staging with add -A pulled three files into 4de43bd that this branch has no business touching: pkg/analyzer/process_state.go, pkg/db/block_rewards.go and pkg/db/validator_last_status.go. All three are whitespace only, and all three are unformatted on dev already, so the change is real but belongs in its own commit rather than hidden inside a reorg fix. Restored to dev's version. The diff is now confined to the reorg and delete paths under review. --- pkg/analyzer/process_state.go | 2 +- pkg/db/block_rewards.go | 2 +- pkg/db/validator_last_status.go | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/analyzer/process_state.go b/pkg/analyzer/process_state.go index eab4000d..42be2e5c 100644 --- a/pkg/analyzer/process_state.go +++ b/pkg/analyzer/process_state.go @@ -414,7 +414,7 @@ func (s *ChainAnalyzer) getSingleBlockRewards( RewardFees: rewardFees, BurntFees: burntFees, Relays: relayAddresses, - BidCommission: bidCommission, + BidCommission: bidCommission, BuilderPubkeys: builderPubkeys, } } diff --git a/pkg/db/block_rewards.go b/pkg/db/block_rewards.go index b0015993..7d25c985 100644 --- a/pkg/db/block_rewards.go +++ b/pkg/db/block_rewards.go @@ -91,5 +91,5 @@ type BlockReward struct { BurntFees *big.Int // Wei Relays []string BuilderPubkeys []string - BidCommission *big.Int // Wei + BidCommission *big.Int // Wei } diff --git a/pkg/db/validator_last_status.go b/pkg/db/validator_last_status.go index 560c2101..02496915 100644 --- a/pkg/db/validator_last_status.go +++ b/pkg/db/validator_last_status.go @@ -33,10 +33,10 @@ var ( func valStatusInput(validatorStatuses []spec.ValidatorLastStatus) proto.Input { // one object per column var ( - f_val_idx proto.ColUInt64 - f_epoch proto.ColUInt64 - f_balance_eth proto.ColFloat32 - f_effective_balance proto.ColUInt64 + f_val_idx proto.ColUInt64 + f_epoch proto.ColUInt64 + f_balance_eth proto.ColFloat32 + f_effective_balance proto.ColUInt64 f_status proto.ColUInt8 f_slashed proto.ColBool f_activation_epoch proto.ColUInt64 From b22e83b16d322c57386435f0ccbe2a8a786e6b47 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Thu, 3 Sep 2026 12:28:28 +0200 Subject: [PATCH 27/47] fix: apply the deposit-counter pattern to consolidation counters ConsolidationsProcessed and ConsolidationsProcessedAmount have the same shape as the deposit counters fixed in the previous commit: appended to and accumulated with += while the state is NextState, read from CurrentState an epoch later by standard.go. They had both of the same failure modes. They double when an epoch is reprocessed against a retained NextState, because processPendingConsolidations ran unconditionally, and they read zero when the CurrentState object was re-downloaded in between, while t_consolidations still holds the rows. RefreshBlocks reset neither. Adds PendingConsolidationsProcessed, an early return in processPendingConsolidationsFor, a CurrentState recompute in PreProcessBundle next to the deposit one, and the resets in RefreshBlocks. The recompute touches CurrentState only, which feeds the epoch row; t_consolidations is persisted from NextState, so no row is written twice. Eight tests, each verified to fail without the part of the fix it covers, including one at the PreProcessBundle level that pins the call site rather than the function. --- .../metrics/consolidation_recompute_test.go | 251 ++++++++++++++++++ pkg/spec/metrics/state_electra.go | 28 +- pkg/spec/state.go | 34 ++- 3 files changed, 301 insertions(+), 12 deletions(-) create mode 100644 pkg/spec/metrics/consolidation_recompute_test.go diff --git a/pkg/spec/metrics/consolidation_recompute_test.go b/pkg/spec/metrics/consolidation_recompute_test.go new file mode 100644 index 00000000..95d1ab06 --- /dev/null +++ b/pkg/spec/metrics/consolidation_recompute_test.go @@ -0,0 +1,251 @@ +package metrics + +import ( + "testing" + + "github.com/attestantio/go-eth2-client/spec/electra" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/migalabs/goteth/pkg/spec" +) + +// The consolidation counters have the same shape as the deposit ones fixed in +// migalabs/goteth#287: ConsolidationsProcessed is appended to and +// ConsolidationsProcessedAmount accumulated with +=, both while the state is +// NextState, and both read from CurrentState an epoch later by standard.go. A +// state re-downloaded in between reports zero while t_consolidations holds the +// rows; a state reprocessed against a retained object reports double. + +const consolidatedBalance = phase0.Gwei(32_000_000_000) + +// eth1Credentials returns a 0x01 withdrawal credential. The prefix byte is read +// unconditionally by processExcessActiveBalanceRestructuring, and keeping both +// states on the same prefix means no switch-to-compounding is detected. +func eth1Credentials() []byte { + credentials := make([]byte, 32) + credentials[0] = spec.Eth1AddressWithdrawalPrefix + return credentials +} + +// stateWithOnePendingConsolidation builds the smallest state that processes a +// single consolidation: the source is unslashed and already withdrawable, so +// neither the skip nor the break branch is taken. +func stateWithOnePendingConsolidation() *spec.AgnosticState { + return &spec.AgnosticState{ + Epoch: 10, + Validators: []*phase0.Validator{ + {EffectiveBalance: consolidatedBalance, WithdrawableEpoch: 5, WithdrawalCredentials: eth1Credentials()}, + {EffectiveBalance: consolidatedBalance, WithdrawableEpoch: phase0.Epoch(spec.FarFutureEpoch), WithdrawalCredentials: eth1Credentials()}, + }, + Balances: []phase0.Gwei{consolidatedBalance, consolidatedBalance}, + PendingConsolidations: []*electra.PendingConsolidation{{SourceIndex: 0, TargetIndex: 1}}, + ConsolidationsProcessed: make([]spec.ConsolidationProcessed, 0), + ConsolidatedAmounts: make(map[phase0.ValidatorIndex]phase0.Gwei), + } +} + +// The fix rests on this: a state that lost its counters can have them derived +// again from the state alone, reaching the numbers the original run reached. +func TestRecomputingAReplacedStateReproducesTheConsolidationCounters(t *testing.T) { + p := &ElectraMetrics{} + + accumulated := stateWithOnePendingConsolidation() + p.processPendingConsolidationsFor(accumulated) + + // The same state as it comes back from the beacon node, counters at zero. + redownloaded := stateWithOnePendingConsolidation() + p.processPendingConsolidationsFor(redownloaded) + + if len(accumulated.ConsolidationsProcessed) == 0 { + t.Fatal("the fixture processed no consolidations, so it cannot show anything") + } + if len(redownloaded.ConsolidationsProcessed) != len(accumulated.ConsolidationsProcessed) { + t.Errorf("recomputed %d consolidations, the original run reached %d", + len(redownloaded.ConsolidationsProcessed), len(accumulated.ConsolidationsProcessed)) + } + if redownloaded.ConsolidationsProcessedAmount != accumulated.ConsolidationsProcessedAmount { + t.Errorf("recomputed %d gwei, the original run reached %d", + redownloaded.ConsolidationsProcessedAmount, accumulated.ConsolidationsProcessedAmount) + } +} + +func TestProcessingMarksTheStateConsolidated(t *testing.T) { + p := &ElectraMetrics{} + state := stateWithOnePendingConsolidation() + + p.processPendingConsolidationsFor(state) + + if !state.PendingConsolidationsProcessed { + t.Error("the state was processed but not marked, so it would be recomputed again") + } +} + +// An epoch with an empty queue must still be marked, otherwise every epoch +// without consolidations, which is most of them, is recomputed on every pass. +func TestAnEmptyConsolidationQueueStillMarksTheState(t *testing.T) { + p := &ElectraMetrics{} + state := &spec.AgnosticState{Epoch: 10, ConsolidatedAmounts: make(map[phase0.ValidatorIndex]phase0.Gwei)} + + p.processPendingConsolidationsFor(state) + + if !state.PendingConsolidationsProcessed { + t.Error("a state with no pending consolidations was left unmarked") + } + if len(state.ConsolidationsProcessed) != 0 { + t.Errorf("an empty queue produced %d consolidations", len(state.ConsolidationsProcessed)) + } +} + +// Why the flag has to gate the call: the counters accumulate with += and +// append, so a second run on the same object doubles them. This pins the +// hazard the flag exists to prevent. +func TestRunningConsolidationsTwiceLeavesTheCountersUnchanged(t *testing.T) { + p := &ElectraMetrics{} + state := stateWithOnePendingConsolidation() + + p.processPendingConsolidationsFor(state) + num, amount := len(state.ConsolidationsProcessed), state.ConsolidationsProcessedAmount + + p.processPendingConsolidationsFor(state) + + if len(state.ConsolidationsProcessed) != num { + t.Errorf("ConsolidationsProcessed went from %d to %d entries on a second run", + num, len(state.ConsolidationsProcessed)) + } + if state.ConsolidationsProcessedAmount != amount { + t.Errorf("ConsolidationsProcessedAmount went from %d to %d on a second run", + amount, state.ConsolidationsProcessedAmount) + } +} + +func TestPerValidatorConsolidatedAmountsDoNotDoubleOnASecondRun(t *testing.T) { + // ConsolidatedAmounts is accumulated per validator with += as well. It is + // rebuilt from scratch by processConsolidationsForRewardCalculation on the + // PreProcessBundle path, but this function is what populates it on the + // NextState path, so it needs its own assertion. + p := &ElectraMetrics{} + state := stateWithOnePendingConsolidation() + + p.processPendingConsolidationsFor(state) + before := make(map[phase0.ValidatorIndex]phase0.Gwei, len(state.ConsolidatedAmounts)) + for idx, amount := range state.ConsolidatedAmounts { + before[idx] = amount + } + if len(before) == 0 { + t.Fatal("no per-validator amounts were recorded; the fixture no longer exercises this") + } + + p.processPendingConsolidationsFor(state) + + for idx, amount := range before { + if state.ConsolidatedAmounts[idx] != amount { + t.Errorf("validator %d went from %d to %d on a second run", idx, amount, state.ConsolidatedAmounts[idx]) + } + } +} + +func TestTheNextStateConsolidationPathIsGuardedToo(t *testing.T) { + // processPendingConsolidations runs on every ProcessStateTransitionMetrics + // call, so an epoch reprocessed against a state still held in cache - a + // dependency pass in AdvanceFinalized, a carried reprocess - went through + // this path a second time and doubled the counters. + state := stateWithOnePendingConsolidation() + p := &ElectraMetrics{} + p.baseMetrics.NextState = state + + p.processPendingConsolidations() + amount := state.ConsolidationsProcessedAmount + + p.processPendingConsolidations() + + if state.ConsolidationsProcessedAmount != amount { + t.Errorf("reprocessing through the NextState path doubled %d to %d", amount, state.ConsolidationsProcessedAmount) + } +} + +func TestRefreshingAStateAllowsConsolidationsToBeCountedAgain(t *testing.T) { + // Idempotence must not become "counted once, ever". RefreshBlocks zeroes + // the counters, so it clears the flag, and the next pass has to refill them + // or the epoch row reports no consolidations. + p := &ElectraMetrics{} + state := stateWithOnePendingConsolidation() + + p.processPendingConsolidationsFor(state) + amount := state.ConsolidationsProcessedAmount + if amount == 0 { + t.Fatal("fixture processed no consolidations") + } + + state.RefreshBlocks(nil) + if state.PendingConsolidationsProcessed { + t.Fatal("RefreshBlocks left the state marked as already accumulated") + } + + p.processPendingConsolidationsFor(state) + if state.ConsolidationsProcessedAmount != amount { + t.Errorf("after a refresh the counters came back as %d, want %d", state.ConsolidationsProcessedAmount, amount) + } +} + +func TestARefreshClearsTheConsolidationEntries(t *testing.T) { + // RefreshBlocks promises to reset the accumulators. It zeroed the deposit + // counters but left ConsolidationsProcessed and ConsolidationsProcessedAmount + // populated, so a recompute after a refresh appended a second set of rows on + // top of the first. + p := &ElectraMetrics{} + state := stateWithOnePendingConsolidation() + + p.processPendingConsolidationsFor(state) + num, amount := len(state.ConsolidationsProcessed), state.ConsolidationsProcessedAmount + if num == 0 { + t.Fatal("fixture processed no consolidations") + } + + state.RefreshBlocks(nil) + p.processPendingConsolidationsFor(state) + + if len(state.ConsolidationsProcessed) != num { + t.Errorf("after a refresh and recompute there are %d entries, want %d", len(state.ConsolidationsProcessed), num) + } + if state.ConsolidationsProcessedAmount != amount { + t.Errorf("after a refresh and recompute the amount is %d, want %d", state.ConsolidationsProcessedAmount, amount) + } +} + +// The tests above pin the function's own behaviour. This one pins the wiring: +// that PreProcessBundle recomputes against CurrentState, which is the object +// standard.go reads the epoch row from. Without that call site the function is +// correct and the epoch row is still written with zeros. +func TestPreProcessBundleFillsAReDownloadedCurrentState(t *testing.T) { + // CurrentState as it comes back from the beacon node after a reorg: the + // pending queue is there, the counters accumulated an epoch earlier are not. + currentState := stateWithOnePendingConsolidation() + currentState.StateRoot = phase0.Root{0x01} // not empty, or the bundle is a no-op + currentState.Blocks = nil // short-circuits ProcessAttestations + currentState.DepositedAmounts = make(map[phase0.ValidatorIndex]phase0.Gwei) + currentState.ConsolidatedOutAmounts = make(map[phase0.ValidatorIndex]phase0.Gwei) + + nextState := stateWithOnePendingConsolidation() + nextState.Epoch = currentState.Epoch + 1 + nextState.ConsolidatedOutAmounts = make(map[phase0.ValidatorIndex]phase0.Gwei) + nextState.DepositedAmounts = make(map[phase0.ValidatorIndex]phase0.Gwei) + + p := &ElectraMetrics{} + p.baseMetrics.CurrentState = currentState + p.baseMetrics.NextState = nextState + // PrevState carries an empty state root, so the block-reward tail of the + // bundle, which needs a far larger fixture, does not run. It still has to be + // non-nil: EmptyStateRoot has a value receiver. + p.baseMetrics.PrevState = &spec.AgnosticState{} + + if err := p.PreProcessBundle(); err != nil { + t.Fatalf("PreProcessBundle: %v", err) + } + + if currentState.ConsolidationsProcessedAmount == 0 { + t.Error("CurrentState came out of the bundle with no consolidated amount; " + + "the epoch row would be written as zero while t_consolidations holds the rows") + } + if len(currentState.ConsolidationsProcessed) == 0 { + t.Error("CurrentState came out of the bundle with no consolidation entries") + } +} diff --git a/pkg/spec/metrics/state_electra.go b/pkg/spec/metrics/state_electra.go index a58e4e7d..7fb52b6f 100644 --- a/pkg/spec/metrics/state_electra.go +++ b/pkg/spec/metrics/state_electra.go @@ -67,6 +67,16 @@ func (p *ElectraMetrics) PreProcessBundle() error { // through it. p.processPendingDepositsFor(p.baseMetrics.CurrentState) + // The consolidation counters have the same lifecycle and the same + // failure: ConsolidationsProcessedNum and ConsolidationsProcessedAmount + // are read from CurrentState in standard.go but accumulated an epoch + // earlier against NextState, so a re-downloaded object reports zero + // while t_consolidations holds the rows. This runs before the + // ConsolidatedAmounts maps are cleared below, so only the epoch-row + // counters survive it; the per-validator reward maps are rebuilt by + // processConsolidationsForRewardCalculation as before. + p.processPendingConsolidationsFor(p.baseMetrics.CurrentState) + // FIX: Clear state maps before processing (state objects are reused between iterations) p.baseMetrics.CurrentState.ConsolidatedAmounts = make(map[phase0.ValidatorIndex]phase0.Gwei) p.baseMetrics.CurrentState.ConsolidatedOutAmounts = make(map[phase0.ValidatorIndex]phase0.Gwei) @@ -75,7 +85,7 @@ func (p *ElectraMetrics) PreProcessBundle() error { p.processExcessActiveBalanceRestructuring(p.baseMetrics.CurrentState, p.baseMetrics.NextState) p.processConsolidationsForRewardCalculation(p.baseMetrics.CurrentState, p.baseMetrics.NextState) p.processDepositsForRewardCalculation(p.baseMetrics.CurrentState, p.baseMetrics.NextState) - p.processPendingConsolidations(p.baseMetrics.NextState) + p.processPendingConsolidations() if !p.baseMetrics.PrevState.EmptyStateRoot() { // block rewards p.ProcessSlashings() @@ -622,7 +632,19 @@ func (p ElectraMetrics) getParticipationFlags(attestation electra.Attestation, i } // // https://github.com/ethereum/consensus-specs/blob/dev/specs/electra/beacon-chain.md#new-process_pending_consolidations -func (p ElectraMetrics) processPendingConsolidations(s *spec.AgnosticState) { +func (p *ElectraMetrics) processPendingConsolidations() { + p.processPendingConsolidationsFor(p.baseMetrics.NextState) +} + +// processPendingConsolidationsFor walks a state's pending consolidation queue, +// accumulating the counters the epoch row reads. It is safe to call twice on +// the same state: the counters accumulate with += and append, so the flag makes +// the second call a no-op rather than a doubling. +func (p *ElectraMetrics) processPendingConsolidationsFor(s *spec.AgnosticState) { + if s.PendingConsolidationsProcessed { + return + } + nextEpoch := s.Epoch + 1 for index, pendingConsolidation := range s.PendingConsolidations { @@ -651,6 +673,8 @@ func (p ElectraMetrics) processPendingConsolidations(s *spec.AgnosticState) { s.ConsolidationsProcessed = append(s.ConsolidationsProcessed, *consolidationProcessed) s.ConsolidationsProcessedAmount += sourceEffectiveBalance } + + s.PendingConsolidationsProcessed = true } // processConsolidationsForRewardCalculation replays the spec's process_pending_consolidations diff --git a/pkg/spec/state.go b/pkg/spec/state.go index d02d7f09..e49903a9 100644 --- a/pkg/spec/state.go +++ b/pkg/spec/state.go @@ -62,16 +62,21 @@ type AgnosticState struct { PendingConsolidations []*electra.PendingConsolidation PendingPartialWithdrawals []*electra.PendingPartialWithdrawal ConsolidationsProcessed []ConsolidationProcessed - ConsolidationsProcessedAmount phase0.Gwei // total amount of Gwei consolidated - NewExitingValidators []phase0.ValidatorIndex // list of validators that are exiting due to consolidation/withdrawal requests, used for tracking errors of a validator trying to consolidate/withdraw twice on same epoch. - ConsolidatedAmounts map[phase0.ValidatorIndex]phase0.Gwei // map of validator index to consolidated amount (target receives) - ConsolidatedOutAmounts map[phase0.ValidatorIndex]phase0.Gwei // map of validator index to outgoing consolidated amount (source sends) - PendingDeposits []*electra.PendingDeposit - DepositsProcessed []Deposit - DepositedAmounts map[phase0.ValidatorIndex]phase0.Gwei // map of validator index to deposited amount (used for Electra Fork) - DepositBalanceToConsume phase0.Gwei // balance to consume for deposits, used for Electra Fork - Eth1DepositIndex uint64 // index of the next deposit request to be processed, used for Electra Fork - DepositRequestsStartIndex uint64 // index of the next deposit request to be processed, used for Electra Fork + ConsolidationsProcessedAmount phase0.Gwei // total amount of Gwei consolidated + // PendingConsolidationsProcessed records whether ConsolidationsProcessed and + // ConsolidationsProcessedAmount were accumulated for this state object. They + // have the same lifecycle as the deposit counters above: filled while the + // state is NextState, read an epoch later while it is CurrentState. + PendingConsolidationsProcessed bool + NewExitingValidators []phase0.ValidatorIndex // list of validators that are exiting due to consolidation/withdrawal requests, used for tracking errors of a validator trying to consolidate/withdraw twice on same epoch. + ConsolidatedAmounts map[phase0.ValidatorIndex]phase0.Gwei // map of validator index to consolidated amount (target receives) + ConsolidatedOutAmounts map[phase0.ValidatorIndex]phase0.Gwei // map of validator index to outgoing consolidated amount (source sends) + PendingDeposits []*electra.PendingDeposit + DepositsProcessed []Deposit + DepositedAmounts map[phase0.ValidatorIndex]phase0.Gwei // map of validator index to deposited amount (used for Electra Fork) + DepositBalanceToConsume phase0.Gwei // balance to consume for deposits, used for Electra Fork + Eth1DepositIndex uint64 // index of the next deposit request to be processed, used for Electra Fork + DepositRequestsStartIndex uint64 // index of the next deposit request to be processed, used for Electra Fork } func GetCustomState(bstate spec.VersionedBeaconState, duties EpochDuties) (AgnosticState, error) { @@ -156,6 +161,15 @@ func (p *AgnosticState) RefreshBlocks(blockList []*AgnosticBlock) { // per-validator amounts doubled, even though the scalar counters looked // right. p.DepositedAmounts = make(map[phase0.ValidatorIndex]phase0.Gwei) + // The consolidation counters have the same shape as the deposit ones: a + // slice appended to and a total accumulated with +=, both read from + // CurrentState by the epoch row. They were left populated here, so a + // refreshed state kept the old entries and the next pass added to them. + p.ConsolidationsProcessed = make([]ConsolidationProcessed, 0) + p.ConsolidationsProcessedAmount = 0 + p.PendingConsolidationsProcessed = false + p.ConsolidatedAmounts = make(map[phase0.ValidatorIndex]phase0.Gwei) + p.ConsolidatedOutAmounts = make(map[phase0.ValidatorIndex]phase0.Gwei) p.NumAttestations = 0 p.SyncCommitteeParticipation = 0 p.AddBlocks(blockList) From 48abc888b0b8512f47a365dab7daa2ecfbdb05c1 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Mon, 7 Sep 2026 11:03:58 +0200 Subject: [PATCH 28/47] fix(reorg): AdvanceFinalized can retry an epoch it failed to rewrite Three gaps in the reprocess loop, all leaving data wrong or missing with no signal and no way back (#291). A carried epoch's debt was cleared the moment the loop reached it, before anything was known to have been written. DeleteStateMetrics then ran unconditionally while ProcessStateTransitionMetrics writes nothing unless all three of its states are present, so a failed dependency download left the rows deleted, nothing written back, and no debt to retry: a permanent hole made by the mechanism that exists to prevent holes. It was introduced with the carry itself. The obvious fix is the wrong one. Skipping the delete leaves the pre-reorg values in place looking correct, and a hole is detectable by the #285 query where silently wrong values are not. So the delete stays and the debt is what changes: ProcessStateTransitionMetrics now reports whether it wrote, and the debt is settled on that rather than on arrival. An epoch whose state root could not be fetched was logged and skipped, and one whose RefreshStateBlocks failed was reprocessed against pre-reorg block pointers. Both now carry a debt instead. Carrying alone would not have helped: a debt below the finalized boundary was unpayable because CleanUpTo evicts those states and the loop only ever iterated what was still cached. epochsToVisit re-adds an evicted debt and ensureDependencyStates re-downloads what it needs, which is what makes any of the three retryable. The retry is bounded. An unbounded debt would re-download and re-delete the same epoch on every finalized event for the life of the process, so after maxReprocessAttempts it is abandoned with an error, leaving a hole that is at least detectable. Every new test was checked to fail without the part of the fix it covers. --- pkg/analyzer/chain_analyzer.go | 1 + pkg/analyzer/process_state.go | 23 +++--- pkg/analyzer/reorg.go | 111 +++++++++++++++++++++++++---- pkg/analyzer/reorg_order_test.go | 6 +- pkg/analyzer/reorg_pending_test.go | 98 +++++++++++++++++++++---- pkg/analyzer/reorg_visit_test.go | 68 ++++++++++++++++++ 6 files changed, 271 insertions(+), 36 deletions(-) create mode 100644 pkg/analyzer/reorg_visit_test.go diff --git a/pkg/analyzer/chain_analyzer.go b/pkg/analyzer/chain_analyzer.go index 8a7e8a66..4d51d51c 100644 --- a/pkg/analyzer/chain_analyzer.go +++ b/pkg/analyzer/chain_analyzer.go @@ -57,6 +57,7 @@ type ChainAnalyzer struct { validatorsRewardsAggregationsMu sync.Mutex advanceFinalizedMu sync.Mutex // serializes AdvanceFinalized so concurrent invocations cannot race CleanUpTo against in-flight Wait()s pendingReprocess map[uint64]bool // epochs whose derived rows went stale when an earlier epoch was rewritten, and that AdvanceFinalized could not reach in that invocation (#285). Guarded by advanceFinalizedMu. + reprocessAttempts map[uint64]int // how many times each pending epoch has been attempted, so a debt that cannot be paid is abandoned loudly rather than retried forever (#291). Guarded by advanceFinalizedMu. aggregatedEpochsInWindow map[phase0.Epoch]bool // set of unique epochs aggregated in current window; prevents double-counting on reprocessing (#255) epochBoundaryStateRoots sync.Map // slot -> phase0.Root, caches state roots from Head SSE events at epoch boundaries diff --git a/pkg/analyzer/process_state.go b/pkg/analyzer/process_state.go index 42be2e5c..0fc8b921 100644 --- a/pkg/analyzer/process_state.go +++ b/pkg/analyzer/process_state.go @@ -17,10 +17,15 @@ var ( // We always provide the epoch we transition to // To process the transition from epoch 9 to 10, we provide 10 and we retrieve 8, 9, 10 -func (s *ChainAnalyzer) ProcessStateTransitionMetrics(epoch phase0.Epoch) { +// ProcessStateTransitionMetrics reports whether it wrote the metrics for this +// epoch. It writes nothing when any of the three states it needs is missing, +// and a caller that deleted the old rows first has to know that happened: +// otherwise the rows are gone, nothing replaced them, and nothing retries +// (migalabs/goteth#291). +func (s *ChainAnalyzer) ProcessStateTransitionMetrics(epoch phase0.Epoch) bool { if !s.metrics.Epoch { - return + return false } routineKey := fmt.Sprintf("%s%d", epochProcesserTag, epoch) @@ -51,7 +56,7 @@ func (s *ChainAnalyzer) ProcessStateTransitionMetrics(epoch phase0.Epoch) { if err != nil { s.processerBook.FreePage(routineKey) log.Errorf("context cancelled waiting for state at epoch %d: %s", epoch-2, err) - return + return false } } if epoch >= 1 && epoch-1 >= phase0.Epoch(s.initSlot/spec.SlotsPerEpoch) { @@ -59,14 +64,14 @@ func (s *ChainAnalyzer) ProcessStateTransitionMetrics(epoch phase0.Epoch) { if err != nil { s.processerBook.FreePage(routineKey) log.Errorf("context cancelled waiting for state at epoch %d: %s", epoch-1, err) - return + return false } } nextState, err = s.downloadCache.StateHistory.Wait(s.ctx, EpochTo[uint64](epoch)) if err != nil { s.processerBook.FreePage(routineKey) log.Errorf("context cancelled waiting for state at epoch %d: %s", epoch, err) - return + return false } bundle, err := metrics.StateMetricsByForkVersion(nextState, currentState, prevState, s.cli.Api) @@ -75,11 +80,13 @@ func (s *ChainAnalyzer) ProcessStateTransitionMetrics(epoch phase0.Epoch) { log.Errorf("could not parse bundle metrics at epoch: %s", err) s.stop.Store(true) s.cancel() - return + return false } // If prevState, currentState and nextState are filled, we can process proposer duties, epoch metrics and validator rewards + wrote := false if !nextState.EmptyStateRoot() && !currentState.EmptyStateRoot() && !prevState.EmptyStateRoot() { + wrote = true s.processEpochDuties(bundle) s.processValLastStatus(bundle) s.processEpochMetrics(bundle) @@ -97,7 +104,7 @@ func (s *ChainAnalyzer) ProcessStateTransitionMetrics(epoch phase0.Epoch) { } s.processerBook.FreePage(routineKey) - + return wrote } func (s *ChainAnalyzer) processSlashings(bundle metrics.StateMetrics) { @@ -414,7 +421,7 @@ func (s *ChainAnalyzer) getSingleBlockRewards( RewardFees: rewardFees, BurntFees: burntFees, Relays: relayAddresses, - BidCommission: bidCommission, + BidCommission: bidCommission, BuilderPubkeys: builderPubkeys, } } diff --git a/pkg/analyzer/reorg.go b/pkg/analyzer/reorg.go index 044090c1..c4247bc6 100644 --- a/pkg/analyzer/reorg.go +++ b/pkg/analyzer/reorg.go @@ -29,6 +29,8 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { stateKeys := s.downloadCache.StateHistory.GetKeyList() + stateKeys = epochsToVisit(stateKeys, s.pendingReprocess, s.downloadCache.StateHistory.Available) + // Sort keys so we process epochs in ascending order. This guarantees // that when we reach epoch E, we already know whether blocks in // earlier epochs (E-1, E-2) changed and can propagate reprocessing. @@ -47,6 +49,13 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { } advance = true // only set flag if there is something to do + // Re-added from the carried set above and evicted since. Both the + // block loop and the state root check below Wait on entries that are + // no longer in the cache, so they have to be back before either runs. + if !s.downloadCache.StateHistory.Available(epoch) { + s.ensureDependencyStates(epoch) + } + // --- Step 1: verify block roots FIRST --- // Blocks must be checked before state metrics are (re)processed, // because state processing reads block data (e.g. isFlagPossible @@ -83,7 +92,14 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { // Refresh the state's Blocks array so it points to the newly // downloaded block objects instead of the stale pre-reorg ones. if err := s.downloadCache.RefreshStateBlocks(s.ctx, epoch); err != nil { + // The state still points at the pre-reorg blocks, so anything + // derived from it now would be built on the orphaned chain. + // Carry it rather than rewrite it: the old rows stay, which is + // wrong, but the retry can replace them with correct ones, + // whereas rewriting now records the wrong values as final. log.Errorf("failed to refresh state blocks for epoch %d: %s", epoch, err) + s.settleReprocess(epoch, false) + continue } } @@ -96,13 +112,12 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { finalizedStateRoot, err := s.cli.RequestStateRoot(phase0.Slot(cacheState.Slot)) if err != nil { log.Errorf("could not get state root at slot %d: %s", cacheState.Slot, err) - // This epoch is now unresolved: its state may be stale and nothing - // here can tell. Rewriting anyway would derive the rows from a - // state that may still be the pre-reorg copy, and carrying it does - // not help either - see carryStaleDependents for why a debt below - // the finalized boundary can never be paid. Fixing this needs the - // loop to be able to revisit an epoch whose state has been evicted, - // which is a larger change than this one. + // This epoch is unresolved: its state may be stale and nothing here + // can tell. Rewriting anyway would derive the rows from what may + // still be the pre-reorg copy, so carry it instead. The loop can + // revisit an evicted epoch now, which is what makes a debt below + // the finalized boundary payable at all. + s.settleReprocess(epoch, false) continue } @@ -126,7 +141,8 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { if epoch >= 2 && epochsWithChangedBlocks[epoch-2] { needsReprocess = true } - if s.consumeCarriedEpoch(epoch) { + carried := s.carriedEpoch(epoch) + if carried { needsReprocess = true } @@ -154,7 +170,15 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { log.Infof("rewriting metrics for epoch %d (stateRootChanged=%t, blocksChanged=%t, dep=%t)", epoch, stateRootChanged, blocksChanged, (epoch >= 1 && epochsWithChangedBlocks[epoch-1]) || (epoch >= 2 && epochsWithChangedBlocks[epoch-2])) - s.ProcessStateTransitionMetrics(phase0.Epoch(epoch)) + wrote := s.ProcessStateTransitionMetrics(phase0.Epoch(epoch)) + // The rows were deleted just above. If nothing replaced them the + // epoch is a hole, and the debt is what gets it rewritten later. + // Settling only here is the point: clearing it on the way in threw + // the debt away before knowing whether it had been paid (#291). + s.settleReprocess(epoch, wrote) + } else if carried { + // Carried but nothing needed doing, so the debt is discharged. + s.settleReprocess(epoch, true) } } @@ -175,15 +199,78 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { // for the lifetime of the process. // // Callers must hold advanceFinalizedMu. -func (s *ChainAnalyzer) consumeCarriedEpoch(epoch uint64) bool { +// epochsToVisit returns the epochs an invocation should examine: everything +// still in the cache, plus any carried debt whose state has been evicted since +// it was recorded. +// +// Without the second half a debt below the finalized boundary could never be +// paid: CleanUpTo evicts those states at the end of the invocation that +// recorded them, so they are absent from GetKeyList on every later pass and the +// loop never visits them again (migalabs/goteth#291). +func epochsToVisit(cached []uint64, pending map[uint64]bool, available func(uint64) bool) []uint64 { + visit := cached + for epoch := range pending { + if !available(epoch) { + visit = append(visit, epoch) + } + } + return visit +} + +// maxReprocessAttempts bounds the retry. A debt still unpaid after this many +// finalized events is not going to be paid by trying again - most often the +// beacon node no longer serves that state - and an unbounded debt would +// re-download and re-delete the same epoch on every finalized event for the +// life of the process. +const maxReprocessAttempts = 3 + +// carriedEpoch reports whether this epoch is owed a reprocess. It does not +// clear the debt: settleReprocess does that, and only once something is known +// to have been written. Clearing it here meant an epoch whose rewrite wrote +// nothing had its rows deleted and its debt forgotten in the same pass (#291). +func (s *ChainAnalyzer) carriedEpoch(epoch uint64) bool { if !s.pendingReprocess[epoch] { return false } - delete(s.pendingReprocess, epoch) log.Infof("reprocessing epoch %d carried over from an earlier AdvanceFinalized", epoch) return true } +// settleReprocess records the outcome of an attempt to rewrite an epoch. +// +// A successful write clears the debt. A failure keeps it so a later invocation +// retries, up to maxReprocessAttempts, after which the epoch is abandoned with +// an error: its rows are missing, which is at least detectable, and repeating +// the attempt forever would not make them appear. +func (s *ChainAnalyzer) settleReprocess(epoch uint64, wrote bool) { + if s.pendingReprocess == nil { + s.pendingReprocess = make(map[uint64]bool) + } + if s.reprocessAttempts == nil { + s.reprocessAttempts = make(map[uint64]int) + } + + if wrote { + delete(s.pendingReprocess, epoch) + delete(s.reprocessAttempts, epoch) + return + } + + s.reprocessAttempts[epoch]++ + attempts := s.reprocessAttempts[epoch] + if attempts >= maxReprocessAttempts { + delete(s.pendingReprocess, epoch) + delete(s.reprocessAttempts, epoch) + log.Errorf("epoch %d could not be reprocessed after %d attempts; its derived rows are missing and nothing further will retry them", + epoch, maxReprocessAttempts) + return + } + + s.pendingReprocess[epoch] = true + log.Warnf("epoch %d could not be reprocessed (attempt %d of %d); carrying it to the next finalized event", + epoch, attempts, maxReprocessAttempts) +} + // carryStaleDependents records the epochs whose derived rows this invocation // made stale but could not rewrite itself. // @@ -378,7 +465,7 @@ func (s *ChainAnalyzer) rewriteStateMetrics(epochs []phase0.Epoch) { // // The delete and the process are passed in so the order can be asserted without // a database or a beacon node. -func rewriteInOrder(epochs []phase0.Epoch, del func(phase0.Epoch) error, process func(phase0.Epoch)) { +func rewriteInOrder(epochs []phase0.Epoch, del func(phase0.Epoch) error, process func(phase0.Epoch) bool) { for _, epoch := range ascendingEpochs(epochs) { if err := del(epoch); err != nil { log.Errorf("could not delete metrics for epoch %d before rewriting them: %s", epoch, err) diff --git a/pkg/analyzer/reorg_order_test.go b/pkg/analyzer/reorg_order_test.go index 1e7cbabb..9d732067 100644 --- a/pkg/analyzer/reorg_order_test.go +++ b/pkg/analyzer/reorg_order_test.go @@ -103,8 +103,9 @@ func (r *record) del(e phase0.Epoch) error { return nil } -func (r *record) process(e phase0.Epoch) { +func (r *record) process(e phase0.Epoch) bool { r.steps = append(r.steps, "process "+epochStr(e)) + return true } func epochStr(e phase0.Epoch) string { @@ -141,8 +142,9 @@ func TestARewriteStillHappensWhenTheDeleteFails(t *testing.T) { var processed []phase0.Epoch failing := func(phase0.Epoch) error { return errDeleteFailed } - rewriteInOrder([]phase0.Epoch{4}, failing, func(e phase0.Epoch) { + rewriteInOrder([]phase0.Epoch{4}, failing, func(e phase0.Epoch) bool { processed = append(processed, e) + return true }) if len(processed) != 1 || processed[0] != 4 { diff --git a/pkg/analyzer/reorg_pending_test.go b/pkg/analyzer/reorg_pending_test.go index 09a0aa01..7cc24070 100644 --- a/pkg/analyzer/reorg_pending_test.go +++ b/pkg/analyzer/reorg_pending_test.go @@ -10,9 +10,12 @@ import "testing" // them (migalabs/goteth#285). func TestDependentsBelowTheBoundaryAreNotCarried(t *testing.T) { - // The loop reached them itself, so there is no debt - and a debt below the - // boundary could not be paid anyway: CleanUpTo evicts those states at the - // end of the invocation, so no later loop ever visits them again. + // The loop reached them itself, so there is no debt to record. Note the + // second reason this once had - that a debt below the boundary could never + // be paid, because CleanUpTo evicts those states - no longer holds: + // epochsToVisit re-adds an evicted debt and ensureDependencyStates + // re-downloads it. Not carrying these is now purely about not repeating + // work already done in this invocation. s := &ChainAnalyzer{} s.carryStaleDependents(map[uint64]bool{100: true}, 200) @@ -50,24 +53,91 @@ func TestOnlyTheUnreachableDependentIsCarried(t *testing.T) { } } -// A carried epoch must be reprocessed once and then forgotten. Leaving the debt -// in place would retry it on every finalized event for the life of the process. -func TestACarriedEpochIsConsumedExactlyOnce(t *testing.T) { +// A carried epoch that was actually rewritten must be forgotten. Leaving the +// debt in place would retry it on every finalized event for the life of the +// process. +func TestACarriedEpochIsForgottenOnceItIsWritten(t *testing.T) { s := &ChainAnalyzer{} s.carryStaleDependents(map[uint64]bool{199: true}, 200) - if !s.consumeCarriedEpoch(200) { + if !s.carriedEpoch(200) { t.Fatal("the carried epoch was not reported as needing reprocessing") } - if s.consumeCarriedEpoch(200) { - t.Error("the same epoch was reported twice; the debt was never cleared") + s.settleReprocess(200, true) + + if s.carriedEpoch(200) { + t.Error("the same epoch was reported again after being written; the debt was never cleared") + } +} + +// The failure this exists for: the debt used to be cleared when the epoch was +// reached, before anything was known to have been written. A rewrite that wrote +// nothing then left the rows deleted and no debt to retry, so the hole was +// permanent (migalabs/goteth#291). +func TestAnEpochThatWroteNothingKeepsItsDebt(t *testing.T) { + s := &ChainAnalyzer{} + s.carryStaleDependents(map[uint64]bool{199: true}, 200) + + if !s.carriedEpoch(200) { + t.Fatal("the carried epoch was not reported as needing reprocessing") + } + s.settleReprocess(200, false) + + if !s.carriedEpoch(200) { + t.Error("the debt was cleared even though nothing was written; " + + "the rows are deleted and nothing will ever rewrite them") + } +} + +// Reading the debt must not clear it, or the caller cannot report an outcome. +func TestReadingTheDebtDoesNotClearIt(t *testing.T) { + s := &ChainAnalyzer{} + s.carryStaleDependents(map[uint64]bool{199: true}, 200) + + if !s.carriedEpoch(200) || !s.carriedEpoch(200) { + t.Error("reading the debt cleared it, so a failed rewrite could not carry it forward") + } +} + +// An unpayable debt must not be retried forever: the epoch would be +// re-downloaded and re-deleted on every finalized event for the life of the +// process. It is abandoned loudly instead, leaving a hole, which is detectable. +func TestAnUnpayableDebtIsAbandonedAfterABoundedNumberOfAttempts(t *testing.T) { + s := &ChainAnalyzer{} + s.carryStaleDependents(map[uint64]bool{199: true}, 200) + + for attempt := 1; attempt < maxReprocessAttempts; attempt++ { + s.settleReprocess(200, false) + if !s.carriedEpoch(200) { + t.Fatalf("the debt was dropped after %d attempts, before the bound", attempt) + } + } + s.settleReprocess(200, false) + + if s.carriedEpoch(200) { + t.Errorf("the debt survived %d attempts; it would be retried on every "+ + "finalized event forever", maxReprocessAttempts) + } +} + +// A retry that succeeds must reset the count, or an epoch that fails +// intermittently exhausts its budget across unrelated incidents. +func TestASuccessfulRetryResetsTheAttemptCount(t *testing.T) { + s := &ChainAnalyzer{} + s.carryStaleDependents(map[uint64]bool{199: true}, 200) + + s.settleReprocess(200, false) + s.settleReprocess(200, true) + + if s.reprocessAttempts[200] != 0 { + t.Errorf("attempt count survived a successful write: %d", s.reprocessAttempts[200]) } } func TestAnEpochThatWasNeverCarriedIsNotReprocessed(t *testing.T) { s := &ChainAnalyzer{} - if s.consumeCarriedEpoch(200) { + if s.carriedEpoch(200) { t.Error("an epoch nothing made stale was queued for reprocessing") } } @@ -93,18 +163,18 @@ func TestAStateReplacedWithoutBlockChangesIsStillCarried(t *testing.T) { // Consuming a carried epoch must not enqueue its own dependents: it is being // reprocessed because a predecessor changed, not because its own blocks did. // Propagating from it would cascade forward with no end. -func TestConsumingACarriedEpochDoesNotCascade(t *testing.T) { +func TestSettlingACarriedEpochDoesNotCascade(t *testing.T) { s := &ChainAnalyzer{} s.carryStaleDependents(map[uint64]bool{199: true}, 200) - s.consumeCarriedEpoch(200) - s.consumeCarriedEpoch(201) + s.settleReprocess(200, true) + s.settleReprocess(201, true) // The consumed epochs are not added to epochsWithChangedBlocks, so the // next invocation carries nothing on their behalf. s.carryStaleDependents(map[uint64]bool{}, 202) if len(s.pendingReprocess) != 0 { - t.Errorf("consuming carried epochs left %v queued", s.pendingReprocess) + t.Errorf("settling carried epochs left %v queued", s.pendingReprocess) } } diff --git a/pkg/analyzer/reorg_visit_test.go b/pkg/analyzer/reorg_visit_test.go new file mode 100644 index 00000000..ff6d6432 --- /dev/null +++ b/pkg/analyzer/reorg_visit_test.go @@ -0,0 +1,68 @@ +package analyzer + +import ( + "sort" + "testing" +) + +// The loop iterates what GetKeyList returns. A carried epoch whose state was +// evicted by CleanUpTo is not in that list, so before this it was never +// visited again and its debt could not be paid, no matter how many finalized +// events went by (migalabs/goteth#291). + +func inCache(epochs ...uint64) func(uint64) bool { + present := make(map[uint64]bool, len(epochs)) + for _, e := range epochs { + present[e] = true + } + return func(e uint64) bool { return present[e] } +} + +func sorted(in []uint64) []uint64 { + out := append([]uint64(nil), in...) + sort.Slice(out, func(i, j int) bool { return out[i] < out[j] }) + return out +} + +func TestAnEvictedCarriedEpochIsVisitedAgain(t *testing.T) { + got := epochsToVisit([]uint64{300, 301}, map[uint64]bool{200: true}, inCache(300, 301)) + + want := []uint64{200, 300, 301} + if len(got) != len(want) { + t.Fatalf("got %v, want %v; the evicted debt at 200 would never be revisited", sorted(got), want) + } + for i, e := range sorted(got) { + if e != want[i] { + t.Errorf("got %v, want %v", sorted(got), want) + break + } + } +} + +func TestACarriedEpochStillInTheCacheIsNotAddedTwice(t *testing.T) { + // It is already in the key list; adding it again would process it twice in + // one invocation, which is how a processerBook slot leaks (#292). + got := epochsToVisit([]uint64{200, 201}, map[uint64]bool{200: true}, inCache(200, 201)) + + if len(got) != 2 { + t.Errorf("got %v; epoch 200 is in the cache already and was added a second time", sorted(got)) + } +} + +func TestNoDebtMeansNothingIsAdded(t *testing.T) { + got := epochsToVisit([]uint64{200, 201}, nil, inCache(200, 201)) + + if len(got) != 2 { + t.Errorf("got %v, want just the cached epochs", sorted(got)) + } +} + +func TestADebtWithAnEmptyCacheIsStillVisited(t *testing.T) { + // Every state evicted, one debt outstanding: the invocation exists purely + // to pay it, and an empty key list must not mean an empty loop. + got := epochsToVisit(nil, map[uint64]bool{200: true}, inCache()) + + if len(got) != 1 || got[0] != 200 { + t.Errorf("got %v, want [200]", got) + } +} From 04a601a0ca30c38f85aee4c90bdf211f28a41a05 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Mon, 7 Sep 2026 11:10:39 +0200 Subject: [PATCH 29/47] fix(reorg): drop an unreachable branch and never wait on a lost state Audit of the previous commit. The `else if carried` branch could never run: carriedEpoch already sets needsReprocess, so the first arm always takes it. A carried epoch re-added to the loop was passed to ensureDependencyStates and then used regardless of whether that worked. DownloadState adds the state inline, so a state still missing afterwards could not be fetched, and falling through would Wait on it forever while holding advanceFinalizedMu - which stops every later invocation as well, the failure this file already documents in AdvanceFinalized's own comment. It now counts the attempt and moves on. Also removes two comments this change made wrong. consumeCarriedEpoch's doc block outlived the function and had come to sit above epochsToVisit, where it described clearing the debt regardless of the outcome as intended behaviour, which is the bug the previous commit fixed. And carryStaleDependents justified its boundary condition partly on debts below the boundary being unpayable, which is no longer true. --- pkg/analyzer/reorg.go | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/pkg/analyzer/reorg.go b/pkg/analyzer/reorg.go index c4247bc6..4ac1aaaa 100644 --- a/pkg/analyzer/reorg.go +++ b/pkg/analyzer/reorg.go @@ -54,6 +54,17 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { // no longer in the cache, so they have to be back before either runs. if !s.downloadCache.StateHistory.Available(epoch) { s.ensureDependencyStates(epoch) + + // DownloadState adds the state inline, so still missing here means + // it could not be fetched. Falling through would Wait on it + // forever while holding advanceFinalizedMu, which stops every + // later invocation too - the failure mode this loop already + // documents elsewhere. Count the attempt and move on. + if !s.downloadCache.StateHistory.Available(epoch) { + log.Errorf("carried epoch %d could not be restored to the cache; skipping it this pass", epoch) + s.settleReprocess(epoch, false) + continue + } } // --- Step 1: verify block roots FIRST --- @@ -141,8 +152,7 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { if epoch >= 2 && epochsWithChangedBlocks[epoch-2] { needsReprocess = true } - carried := s.carriedEpoch(epoch) - if carried { + if s.carriedEpoch(epoch) { needsReprocess = true } @@ -176,9 +186,6 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { // Settling only here is the point: clearing it on the way in threw // the debt away before knowing whether it had been paid (#291). s.settleReprocess(epoch, wrote) - } else if carried { - // Carried but nothing needed doing, so the debt is discharged. - s.settleReprocess(epoch, true) } } @@ -191,14 +198,6 @@ func (s *ChainAnalyzer) AdvanceFinalized(newFinalizedSlot phase0.Slot) { } } -// consumeCarriedEpoch reports whether this epoch was carried over by an earlier -// invocation that rewrote one of its predecessors, and clears the debt. -// -// The debt is cleared whether or not the rest of the iteration succeeds. An -// epoch that keeps failing would otherwise be retried on every finalized event -// for the lifetime of the process. -// -// Callers must hold advanceFinalizedMu. // epochsToVisit returns the epochs an invocation should examine: everything // still in the cache, plus any carried debt whose state has been evicted since // it was recorded. @@ -289,14 +288,15 @@ func (s *ChainAnalyzer) settleReprocess(epoch uint64, wrote bool) { // reprocessed only because a predecessor changed do not belong here: their own // state is untouched, so nothing downstream of them went stale. // -// Only dependents at or past the boundary are carried, and that condition is -// load-bearing rather than an optimisation. The loop skips epochs at or past -// finalizedEpoch, so it only ever visits epochs below it, and CleanUpTo then -// evicts every state below the finalized slot. An epoch carried from below the -// boundary is therefore gone from StateHistory before any later invocation -// could reach it: the loop never visits it, consumeCarriedEpoch never fires, -// and the debt sits unpayable. Only epochs at or past the boundary survive the -// eviction and become reachable once the boundary advances. +// Only dependents at or past the boundary are carried, because the loop already +// reached the ones below it in this same invocation and carrying them would +// repeat work. +// +// This used to have a second reason: a debt below the boundary was unpayable, +// since CleanUpTo evicts those states and the loop only iterated what was still +// cached. That no longer holds - epochsToVisit re-adds an evicted debt and +// ensureDependencyStates re-downloads it - so the boundary is now about +// avoiding duplicate work rather than about what can be paid at all. // // Callers must hold advanceFinalizedMu. func (s *ChainAnalyzer) carryStaleDependents(changed map[uint64]bool, finalizedEpoch uint64) { From 0e3293a17616e0d1c270f8a4e7a8e6f01810feac Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Mon, 7 Sep 2026 11:29:04 +0200 Subject: [PATCH 30/47] fix(analyzer): stop a duplicate epoch leaking a processerBook slot Three findings that compose into one failure (#292): nothing stopped two goroutines processing the same epoch, the barrier that would have stopped them had never worked, and when it happened the pool lost a slot for good. Acquire is a counting semaphore that also records what holds each slot, not a per-key lock. Two goroutines acquiring one key each took a token, but a page was a bare marker, so the second Set overwrote the first and only one FreePage found a key to delete. One token was never returned. Repeat it and the pool empties, which is the deadlock routines.go already documents. Pages count holders now, and FreePage returns a token for each one, so the accounting is symmetric however many hold a key. It still returns nothing for a key nobody holds: over-filling the channel would make a later send block forever. That accounting change reaches every RoutineBook, including the three in clientapi. NumFreePages reads the channel rather than subtracting the key count, which stopped matching tokens in use once pages held counts, and ActivePages sums holders because the shutdown checks are asking how many routines are running. HandleReorg is spawned per reorg event with no guard, so two events close together walked overlapping ranges and processed the same epochs. It is serialized now. Serialized rather than skipped, unlike AdvanceFinalized: that one is monotonic so a dropped invocation is covered by the next finalized event, while nothing else would ever rewrite a dropped reorg's slots. The end-of-epoch barrier waited on a key built from the slot while the processor registers the epoch, so it asked for "epoch=15096543" against "epoch=471766" and returned immediately every time it ran since it was written. The conversion now lives in epochBarrierKey rather than at the call site, which is where it was missed. Six tests on the book, the first it has had, and five on the barrier keys. Each was checked to fail without the part of the fix it covers, except the reorg guard: it is two lines inside a function that needs a live beacon client and database. --- pkg/analyzer/chain_analyzer.go | 1 + pkg/analyzer/reorg.go | 35 +++++++- pkg/analyzer/reorg_barrier_test.go | 89 +++++++++++++++++++ pkg/utils/book.go | 65 +++++++++----- pkg/utils/book_test.go | 134 +++++++++++++++++++++++++++++ 5 files changed, 300 insertions(+), 24 deletions(-) create mode 100644 pkg/analyzer/reorg_barrier_test.go create mode 100644 pkg/utils/book_test.go diff --git a/pkg/analyzer/chain_analyzer.go b/pkg/analyzer/chain_analyzer.go index 8a7e8a66..81c7949a 100644 --- a/pkg/analyzer/chain_analyzer.go +++ b/pkg/analyzer/chain_analyzer.go @@ -56,6 +56,7 @@ type ChainAnalyzer struct { validatorsRewardsAggregations map[phase0.ValidatorIndex]*spec.ValidatorRewardsAggregation validatorsRewardsAggregationsMu sync.Mutex advanceFinalizedMu sync.Mutex // serializes AdvanceFinalized so concurrent invocations cannot race CleanUpTo against in-flight Wait()s + handleReorgMu sync.Mutex // serializes HandleReorg so two walks cannot process the same epoch at once and leak processerBook slots (#292) pendingReprocess map[uint64]bool // epochs whose derived rows went stale when an earlier epoch was rewritten, and that AdvanceFinalized could not reach in that invocation (#285). Guarded by advanceFinalizedMu. aggregatedEpochsInWindow map[phase0.Epoch]bool // set of unique epochs aggregated in current window; prevents double-counting on reprocessing (#255) epochBoundaryStateRoots sync.Map // slot -> phase0.Root, caches state roots from Head SSE events at epoch boundaries diff --git a/pkg/analyzer/reorg.go b/pkg/analyzer/reorg.go index 044090c1..b5c3a2fb 100644 --- a/pkg/analyzer/reorg.go +++ b/pkg/analyzer/reorg.go @@ -259,7 +259,38 @@ func (s *ChainAnalyzer) ensureDependencyStates(epoch uint64) { } } +// slotBarrierKey and epochBarrierKey build the processerBook keys HandleReorg +// waits on before replacing what a processor may still be reading. +// +// Both take a slot, because HandleReorg walks slots, and the conversion for the +// epoch one belongs here rather than at the call site. Leaving it to the caller +// is exactly how it went wrong: the epoch barrier was built from the raw slot, +// so it asked for "epoch=15096543" while the processor had registered +// "epoch=471766". No such key is ever in the book, so that wait returned +// immediately every time it ran and the state was replaced underneath a live +// reader (migalabs/goteth#292). +func slotBarrierKey(slot phase0.Slot) string { + return fmt.Sprintf("%s%d", slotProcesserTag, uint64(slot)) +} + +func epochBarrierKey(slot phase0.Slot) string { + return fmt.Sprintf("%s%d", epochProcesserTag, uint64(slot)/spec.SlotsPerEpoch) +} + func (s *ChainAnalyzer) HandleReorg(newReorg v1.ChainReorgEvent) { + // Each reorg event fires its own goroutine, and two arriving close together + // walk overlapping slot ranges. Both then call + // ProcessStateTransitionMetrics for the same epoch, which takes two + // processerBook tokens for one key and, before #292, returned only one. + // + // Serialized rather than skipped, which is where this differs from + // AdvanceFinalized: that one is monotonic, so an invocation it drops is + // covered by the next finalized event. A dropped reorg is not covered by + // anything - nothing else rewrites those slots - so the second walk waits + // and then runs against the state the first one left. + s.handleReorgMu.Lock() + defer s.handleReorgMu.Unlock() + depth := newReorg.Depth reorgSlot := newReorg.Slot @@ -282,7 +313,7 @@ func (s *ChainAnalyzer) HandleReorg(newReorg v1.ChainReorgEvent) { if i < reorgSlot && block.Proposed { reorgedSlots += 1 // only count as reorged slot if there was a block porposed and we are not at the reorg slot } - s.processerBook.WaitUntilInactive(fmt.Sprintf("%s%d", slotProcesserTag, i)) // wait until has been processed + s.processerBook.WaitUntilInactive(slotBarrierKey(i)) // wait until has been processed oldBlock := *block s.DownloadBlock(i) // -> inserts into the queue and replaces old block @@ -312,7 +343,7 @@ func (s *ChainAnalyzer) HandleReorg(newReorg v1.ChainReorgEvent) { log.Errorf("context cancelled waiting for state at epoch %d: %s", epoch, err) return } - s.processerBook.WaitUntilInactive(fmt.Sprintf("%s%d", epochProcesserTag, i)) // wait until has been processed + s.processerBook.WaitUntilInactive(epochBarrierKey(i)) oldState := *state s.DownloadState(i) // -> inserts into the queue and replaces old block newState, err := s.downloadCache.StateHistory.Wait(s.ctx, EpochTo[uint64](epoch)) diff --git a/pkg/analyzer/reorg_barrier_test.go b/pkg/analyzer/reorg_barrier_test.go new file mode 100644 index 00000000..58eaa6d4 --- /dev/null +++ b/pkg/analyzer/reorg_barrier_test.go @@ -0,0 +1,89 @@ +package analyzer + +import ( + "fmt" + "testing" + + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/migalabs/goteth/pkg/spec" + "github.com/migalabs/goteth/pkg/utils" +) + +// HandleReorg replaces the state at the end of an epoch while a processor may +// still be reading it, so it waits for that processor first. It walks slots, +// while ProcessStateTransitionMetrics registers epochs, and the conversion was +// left to the call site and not done: the barrier asked for the slot number +// under the epoch tag. No such key is ever in the book, so the wait returned +// immediately every time it ran (migalabs/goteth#292). + +// The key the processor actually registers, built the way process_state.go +// builds it. If that ever changes, this test fails and says so. +func processorEpochKey(epoch uint64) string { + return fmt.Sprintf("%s%d", epochProcesserTag, epoch) +} + +func processorSlotKey(slot uint64) string { + return fmt.Sprintf("%s%d", slotProcesserTag, slot) +} + +func TestTheEpochBarrierAsksForTheKeyTheProcessorRegistered(t *testing.T) { + const epoch uint64 = 471766 + lastSlot := phase0.Slot((epoch+1)*spec.SlotsPerEpoch - 1) + + got := epochBarrierKey(lastSlot) + want := processorEpochKey(epoch) + + if got != want { + t.Errorf("barrier waits on %q while the processor registered %q; "+ + "the wait matches nothing and the state is replaced under a live reader", + got, want) + } +} + +// Every slot of an epoch must resolve to that epoch's key, not just the last +// one: the barrier runs at the end-of-epoch boundary but the conversion has to +// hold wherever it is called from. +func TestEverySlotOfAnEpochResolvesToItsEpochKey(t *testing.T) { + const epoch uint64 = 100 + want := processorEpochKey(epoch) + + for offset := uint64(0); offset < spec.SlotsPerEpoch; offset++ { + slot := phase0.Slot(epoch*spec.SlotsPerEpoch + offset) + if got := epochBarrierKey(slot); got != want { + t.Fatalf("slot %d gave %q, want %q", slot, got, want) + } + } +} + +func TestTheSlotBarrierAsksForTheSlotKey(t *testing.T) { + const slot uint64 = 15096543 + + if got, want := slotBarrierKey(phase0.Slot(slot)), processorSlotKey(slot); got != want { + t.Errorf("slot barrier waits on %q, want %q", got, want) + } +} + +// The two barriers must not collapse onto one another: a slot key and an epoch +// key are different namespaces, and waiting on the wrong one is the bug. +func TestTheTwoBarriersDoNotShareAKey(t *testing.T) { + slot := phase0.Slot(15096543) + + if slotBarrierKey(slot) == epochBarrierKey(slot) { + t.Error("the slot and epoch barriers built the same key") + } +} + +// End to end through the book: a processor registered for the epoch must be +// visible to the barrier that is meant to wait for it. +func TestTheBarrierSeesALiveProcessor(t *testing.T) { + const epoch uint64 = 471766 + lastSlot := phase0.Slot((epoch+1)*spec.SlotsPerEpoch - 1) + + book := utils.NewRoutineBook(4, "test") + book.Acquire(processorEpochKey(epoch)) + + if !book.CheckPageActive(epochBarrierKey(lastSlot)) { + t.Error("the barrier cannot see the processor holding that epoch, so it " + + "would replace the state while the processor is still reading it") + } +} diff --git a/pkg/utils/book.go b/pkg/utils/book.go index ba96c75c..62bd35fe 100644 --- a/pkg/utils/book.go +++ b/pkg/utils/book.go @@ -8,14 +8,21 @@ import ( ) var ( - emptyKey = "" structName = "routinebook" CheckPageInterval = 1 * time.Second ) +// RoutineBook is a counting semaphore that also records what is holding each +// slot, so callers can ask whether a given key is still being worked on. +// +// pages counts holders rather than marking presence. Acquire is not a per-key +// lock: two goroutines can acquire the same key, and each takes its own token. +// While a page was a bare marker the second acquire overwrote the first and +// only one FreePage found a key to delete, so one token was never returned and +// the pool shrank by one every time it happened (migalabs/goteth#292). type RoutineBook struct { sync.Mutex - pages map[string]string + pages map[string]int freeSpaceChan chan struct{} size int64 bookTag string @@ -24,8 +31,8 @@ type RoutineBook struct { func NewRoutineBook(size int, tag string) *RoutineBook { r := &RoutineBook{ - pages: make(map[string]string, size), // contains a list of keys identifying routines - freeSpaceChan: make(chan struct{}, size), // indicates the free position in the array + pages: make(map[string]int, size), // holders per key, see the type comment + freeSpaceChan: make(chan struct{}, size), // indicates the free position in the array size: int64(size), bookTag: tag, } @@ -50,23 +57,36 @@ func (r *RoutineBook) Acquire(key string) { case <-ticker.C: log.WithField("bookTag", r.bookTag).Warnf("Waiting for too long to acquire page %s...", key) case <-r.freeSpaceChan: - r.Set(key, "active") + r.hold(key) return } } } +// FreePage releases one holder of key and returns its token. +// +// The token goes back for every held acquire, not once per key: releasing only +// once would strand the tokens of any other holder of the same key. A key with +// no holders returns nothing, which keeps the channel from being filled past +// its capacity - a send that blocked there would never complete. func (r *RoutineBook) FreePage(key string) { r.Lock() - defer r.Unlock() - _, ok := r.pages[key] - // If the key exists - if ok { + holders, ok := r.pages[key] + if !ok || holders <= 0 { + r.Unlock() + return + } + if holders == 1 { delete(r.pages, key) - r.freeSpaceChan <- struct{}{} + } else { + r.pages[key] = holders - 1 } + r.Unlock() + // Outside the lock: a send here can only block if the channel is full, + // and blocking while holding the mutex would stop every other caller. + r.freeSpaceChan <- struct{}{} } func (r *RoutineBook) CheckPageActive(key string) bool { @@ -93,14 +113,14 @@ func (r *RoutineBook) WaitUntilInactive(key string) bool { } -func (r *RoutineBook) Set(key string, value string) { +// hold records one more holder of key. The caller has already taken a token. +func (r *RoutineBook) hold(key string) { r.Lock() defer r.Unlock() - r.pages[key] = value // book page - + r.pages[key]++ } -func (r *RoutineBook) get(key string) (string, bool) { +func (r *RoutineBook) get(key string) (int, bool) { r.Lock() defer r.Unlock() @@ -110,24 +130,25 @@ func (r *RoutineBook) get(key string) (string, bool) { } +// ActivePages is the number of holders across all keys, which is what the +// shutdown checks mean by "anything still running". Two holders of one key +// count twice, because two routines are running. func (r *RoutineBook) ActivePages() int { r.Lock() defer r.Unlock() result := 0 - for _, item := range r.pages { - if item != emptyKey { - result += 1 - } + for _, holders := range r.pages { + result += holders } return result } +// NumFreePages counts the tokens still available. It reads the channel rather +// than subtracting the number of keys: a key can have several holders, so the +// key count stopped matching the tokens in use once pages began counting. func (r *RoutineBook) NumFreePages() int { - - r.Lock() - defer r.Unlock() - return int(r.size) - len(r.pages) + return len(r.freeSpaceChan) } func (r *RoutineBook) GetKeys() []string { diff --git a/pkg/utils/book_test.go b/pkg/utils/book_test.go new file mode 100644 index 00000000..5fa0f305 --- /dev/null +++ b/pkg/utils/book_test.go @@ -0,0 +1,134 @@ +package utils + +import ( + "sync" + "testing" + "time" +) + +// Acquire looks like a per-key lock but is a counting semaphore that records +// what holds each slot. Two goroutines can acquire the same key, and each takes +// its own token. While a page was a bare marker the second acquire overwrote +// the first, so only one FreePage found a key to delete and one token was never +// returned: the pool shrank by one every time, until nothing could acquire at +// all (migalabs/goteth#292). + +func TestAcquiringOneKeyTwiceReturnsBothTokens(t *testing.T) { + book := NewRoutineBook(4, "test") + + book.Acquire("epoch=471766") + book.Acquire("epoch=471766") + if free := book.NumFreePages(); free != 2 { + t.Fatalf("two acquires left %d of 4 tokens free, want 2", free) + } + + book.FreePage("epoch=471766") + book.FreePage("epoch=471766") + + if free := book.NumFreePages(); free != 4 { + t.Errorf("after releasing both holders %d of 4 tokens are free, want 4; "+ + "the missing one is gone for the life of the process", free) + } +} + +// The consequence, stated as the pool sees it: repeat the leak and the book +// runs out of slots even though nothing is holding any. +func TestRepeatedDuplicateAcquiresDoNotExhaustThePool(t *testing.T) { + book := NewRoutineBook(2, "test") + + for i := 0; i < 10; i++ { + book.Acquire("epoch=100") + book.Acquire("epoch=100") + book.FreePage("epoch=100") + book.FreePage("epoch=100") + } + + if free := book.NumFreePages(); free != 2 { + t.Errorf("after 10 paired acquire/free rounds %d of 2 tokens are free; "+ + "the pool is exhausted and the next Acquire blocks forever", free) + } + if active := book.ActivePages(); active != 0 { + t.Errorf("book reports %d active holders with none outstanding", active) + } +} + +// A key stays active while any holder is still working, or a reorg would +// replace a state out from under a processor that is still reading it. +func TestAKeyStaysActiveUntilTheLastHolderReleasesIt(t *testing.T) { + book := NewRoutineBook(4, "test") + + book.Acquire("epoch=10") + book.Acquire("epoch=10") + + book.FreePage("epoch=10") + if !book.CheckPageActive("epoch=10") { + t.Error("the key went inactive while a second holder was still working") + } + + book.FreePage("epoch=10") + if book.CheckPageActive("epoch=10") { + t.Error("the key stayed active after its last holder released it") + } +} + +// Freeing a key nobody holds must not push a token into the channel: the pool +// would grow past its size, and once the channel is full that send blocks +// forever. +func TestFreeingAKeyNobodyHoldsDoesNotInventTokens(t *testing.T) { + book := NewRoutineBook(2, "test") + + done := make(chan struct{}) + go func() { + defer close(done) + book.FreePage("never-acquired") + book.FreePage("never-acquired") + book.FreePage("never-acquired") + }() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("FreePage blocked on a full channel; the pool was over-filled") + } + + if free := book.NumFreePages(); free != 2 { + t.Errorf("pool holds %d tokens, want 2", free) + } +} + +func TestActivePagesCountsHoldersNotKeys(t *testing.T) { + // The shutdown check asks "is anything still running", so two routines on + // one key must count twice. + book := NewRoutineBook(4, "test") + + book.Acquire("epoch=10") + book.Acquire("epoch=10") + book.Acquire("epoch=11") + + if active := book.ActivePages(); active != 3 { + t.Errorf("ActivePages reported %d, want 3; a shutdown check would think "+ + "fewer routines were running than there are", active) + } +} + +func TestConcurrentAcquireAndFreeKeepsTheAccountingExact(t *testing.T) { + book := NewRoutineBook(8, "test") + + var wg sync.WaitGroup + for i := 0; i < 50; i++ { + wg.Add(1) + go func() { + defer wg.Done() + book.Acquire("epoch=1") + book.FreePage("epoch=1") + }() + } + wg.Wait() + + if free := book.NumFreePages(); free != 8 { + t.Errorf("after 50 paired acquire/free rounds %d of 8 tokens are free", free) + } + if active := book.ActivePages(); active != 0 { + t.Errorf("%d holders still recorded with none outstanding", active) + } +} From f176583a44015864bf2be4d8546ae34d8d7c8664 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Mon, 7 Sep 2026 11:47:01 +0200 Subject: [PATCH 31/47] fix(utils): stop the barrier leaking a timer and waiting for nothing Found auditing the barrier fix, which is what made this matter: that wait had been returning immediately since it was written, and now it actually runs. WaitUntilInactive never stopped its ticker, and an unstopped ticker is never collected. Every call left a runtime timer behind, once per block processed and once per slot of every reorg walk. It also looked only after the first tick, so a key nobody held still cost a whole interval. ProcessBlock waits on its own slot key for every block, so that was an interval per block for no reason. It checks before entering the loop now. The wait stays unbounded, which is the point of it: the caller is about to replace something a holder is still reading. It warns while waiting, the way Acquire does, so a barrier that never clears is visible instead of silent - worth having now that this can genuinely block. --- pkg/utils/book.go | 40 ++++++++++++++++++----- pkg/utils/book_test.go | 72 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/pkg/utils/book.go b/pkg/utils/book.go index 62bd35fe..016ea4a7 100644 --- a/pkg/utils/book.go +++ b/pkg/utils/book.go @@ -97,20 +97,44 @@ func (r *RoutineBook) CheckPageActive(key string) bool { } -func (r *RoutineBook) WaitUntilInactive(key string) bool { +// WaitUntilInactive blocks until nothing holds key. +// +// The ticker is stopped on the way out. It was not, and an unstopped ticker is +// never collected, so every call left a runtime timer behind - once per block +// processed and once per slot of every reorg walk. +// +// The wait is unbounded, which is deliberate: the caller is about to replace +// something the holder is still reading, and giving up would defeat the point. +// It warns while it waits, the way Acquire does, so a barrier that never clears +// is visible rather than silent. +// +// It returned a bool that no caller read and that could only ever be true: the +// old loop ranged over a ticker channel, which is never closed, so the trailing +// `return false` was unreachable. +func (r *RoutineBook) WaitUntilInactive(key string) { ticker := time.NewTicker(CheckPageInterval) + defer ticker.Stop() - for range ticker.C { + warn := time.NewTicker(AcquireWaitIntervalLog) + defer warn.Stop() - _, ok := r.get(key) + // Checked before the first tick. The loop used to wait a whole interval + // before looking, so a key nobody held still cost that second - once per + // block processed, since ProcessBlock waits on its own slot key. + if _, ok := r.get(key); !ok { + return + } - if !ok { - return true + for { + select { + case <-warn.C: + log.WithField("bookTag", r.bookTag).Warnf("Still waiting for page %s to become inactive...", key) + case <-ticker.C: + if _, ok := r.get(key); !ok { + return + } } } - - return false - } // hold records one more holder of key. The caller has already taken a token. diff --git a/pkg/utils/book_test.go b/pkg/utils/book_test.go index 5fa0f305..6a9b3ad8 100644 --- a/pkg/utils/book_test.go +++ b/pkg/utils/book_test.go @@ -132,3 +132,75 @@ func TestConcurrentAcquireAndFreeKeepsTheAccountingExact(t *testing.T) { t.Errorf("%d holders still recorded with none outstanding", active) } } + +// WaitUntilInactive is what stops a reorg replacing a state a processor is +// still reading. It has to return once the last holder releases the key, and +// not before. +func TestWaitUntilInactiveReturnsWhenTheLastHolderReleases(t *testing.T) { + previous := CheckPageInterval + CheckPageInterval = time.Millisecond + defer func() { CheckPageInterval = previous }() + + book := NewRoutineBook(4, "test") + book.Acquire("epoch=10") + book.Acquire("epoch=10") + + returned := make(chan struct{}) + go func() { + defer close(returned) + book.WaitUntilInactive("epoch=10") + }() + + book.FreePage("epoch=10") + select { + case <-returned: + t.Fatal("the wait returned while a second holder was still working; " + + "the caller would replace the state underneath it") + case <-time.After(50 * time.Millisecond): + } + + book.FreePage("epoch=10") + select { + case <-returned: + case <-time.After(5 * time.Second): + t.Fatal("the wait never returned after the last holder released the key") + } +} + +func TestWaitUntilInactiveReturnsImmediatelyForAnUnheldKey(t *testing.T) { + previous := CheckPageInterval + CheckPageInterval = time.Millisecond + defer func() { CheckPageInterval = previous }() + + book := NewRoutineBook(4, "test") + + returned := make(chan struct{}) + go func() { + defer close(returned) + book.WaitUntilInactive("nobody-holds-this") + }() + + select { + case <-returned: + case <-time.After(5 * time.Second): + t.Fatal("waiting on a key nobody holds never returned") + } +} + +// The unheld case has to be free. ProcessBlock waits on its own slot key for +// every block, so a wasted interval there is a wasted interval per block. +func TestWaitUntilInactiveDoesNotPayAnIntervalForAnUnheldKey(t *testing.T) { + previous := CheckPageInterval + CheckPageInterval = 2 * time.Second + defer func() { CheckPageInterval = previous }() + + book := NewRoutineBook(4, "test") + + start := time.Now() + book.WaitUntilInactive("nobody-holds-this") + + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("waiting on an unheld key took %s; it should not wait for a tick "+ + "before looking", elapsed) + } +} From 6f29fb1f8a091cfc6584598ebab1995104cd28b9 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Tue, 8 Sep 2026 15:29:00 +0200 Subject: [PATCH 32/47] ci: build, vet and test this repository There was no CI. No .github/workflows directory, and none in the history. Every change to the indexer serving mainnet merged without being compiled, vetted or tested by anything but whoever wrote it, including the three fixes open right now. The suite could not have run anyway. Thirteen tests in pkg/analyzer build an analyzer against a hardcoded http://localhost:5052 and call t.Errorf when nothing answers, so `go test ./...` was red on any machine without a beacon node. A suite that cannot tell an absent prerequisite from a defect is one nobody can run, and CI would have been red from its first minute. They take their endpoints from GOTETH_TEST_BN_ENDPOINT and GOTETH_TEST_EL_ENDPOINT now, defaulting to the local ones, and skip when nothing answers - matching what goteth-v4 already does. Checked that the skip is what makes them green rather than an empty stub: turn it into a log and all thirteen fail again. Two consequences of removing the builder's err, both preserving existing behaviour rather than changing it. requireNodes returns a pointer, because ChainAnalyzer holds an atomic.Bool and returning a copy trips vet's copylocks check. And several RequestBeaconBlock calls were reusing that err without ever reading it, so they now discard it explicitly. Those tests were already ignoring a request error and asserting on the block regardless; that is untouched here and worth its own look. go build, go vet and go test -race all pass on a clean checkout. --- .github/workflows/ci.yml | 35 ++++++++ pkg/analyzer/metrics_test.go | 150 +++++++++++++++-------------------- 2 files changed, 101 insertions(+), 84 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..65348f7f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: CI + +on: + push: + branches: [main, master, dev] + pull_request: + +# This repository had no CI at all. Every change to the indexer serving +# mainnet merged without being compiled, vetted or tested by anything but +# whoever wrote it. +jobs: + build-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + # go.mod replaces go-relay-client with a local path. + submodules: recursive + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Build + run: go build ./... + + - name: Vet + run: go vet ./... + + - name: Test + # The metrics tests need a live beacon node and skip themselves when + # none answers; set GOTETH_TEST_BN_ENDPOINT (and GOTETH_TEST_EL_ENDPOINT) + # to run them against one. + run: go test -race ./... diff --git a/pkg/analyzer/metrics_test.go b/pkg/analyzer/metrics_test.go index 55467c75..e9184ca1 100644 --- a/pkg/analyzer/metrics_test.go +++ b/pkg/analyzer/metrics_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/big" + "os" "strings" "testing" @@ -16,6 +17,47 @@ import ( "github.com/stretchr/testify/assert" ) +// These exercise the metrics against a live beacon node, and an execution node +// for the ones that need transactions. The endpoints default to the local ones +// a developer runs, and can be pointed elsewhere: +// +// GOTETH_TEST_BN_ENDPOINT beacon node, default http://localhost:5052 +// GOTETH_TEST_EL_ENDPOINT execution node, default http://localhost:8545 +// +// requireNodes skips when nothing answers there. They used to fail instead, +// which meant `go test ./...` was red on any machine without a node running - +// including CI, which is part of why this repository had none. +func testBNEndpoint() string { + if endpoint := os.Getenv("GOTETH_TEST_BN_ENDPOINT"); endpoint != "" { + return endpoint + } + return "http://localhost:5052" +} + +func testELEndpoint() string { + if endpoint := os.Getenv("GOTETH_TEST_EL_ENDPOINT"); endpoint != "" { + return endpoint + } + return "http://localhost:8545" +} + +// requireNodes returns an analyzer, or skips the test when no beacon node is +// reachable. Skipping rather than failing is the point: an absent node is a +// missing prerequisite, not a defect, and a suite that cannot tell the two +// apart is one nobody can run. +// Returns a pointer: ChainAnalyzer holds an atomic.Bool, so handing back a +// copy trips vet's copylocks check. +func requireNodes(t *testing.T, build func() (ChainAnalyzer, error)) *ChainAnalyzer { + t.Helper() + + analyzer, err := build() + if err != nil { + t.Skipf("no beacon node reachable at %s: %s (set GOTETH_TEST_BN_ENDPOINT to point elsewhere)", + testBNEndpoint(), err) + } + return &analyzer +} + func BuildChainAnalyzer() (ChainAnalyzer, error) { ctx := context.Background() @@ -31,9 +73,9 @@ func BuildChainAnalyzer() (ChainAnalyzer, error) { // generate the httpAPI client cli, err := clientapi.NewAPIClient( ctx, - "http://localhost:5052", + testBNEndpoint(), maxRequestRetries, - clientapi.WithELEndpoint("http://localhost:8545"), + clientapi.WithELEndpoint(testELEndpoint()), clientapi.WithDBMetrics(dbMetrics)) if err != nil { return ChainAnalyzer{}, err @@ -51,7 +93,7 @@ func BuildChainAnalyzerWithEL() (ChainAnalyzer, error) { ctx := context.Background() maxRequestRetries := 3 // generate the httpAPI client - cli, err := clientapi.NewAPIClient(ctx, "http://localhost:5052", maxRequestRetries, clientapi.WithELEndpoint("http://localhost:8545")) + cli, err := clientapi.NewAPIClient(ctx, testBNEndpoint(), maxRequestRetries, clientapi.WithELEndpoint(testELEndpoint())) if err != nil { return ChainAnalyzer{}, err } @@ -99,14 +141,10 @@ func BuildEpochTask(analyzer *ChainAnalyzer, slot phase0.Slot) (metrics.StateMet func TestPhase0Epoch(t *testing.T) { - analyzer, err := BuildChainAnalyzer() - if err != nil { - t.Errorf("could not build analyzer: %s", err) - return - } + analyzer := requireNodes(t, BuildChainAnalyzer) // returns the state in a custom struct for Phase0, Altair of Bellatrix - stateMetrics, err := BuildEpochTask(&analyzer, 320031) // epoch 10000 + stateMetrics, err := BuildEpochTask(analyzer, 320031) // epoch 10000 if err != nil { t.Errorf("could not build epoch task: %s", err) return @@ -120,14 +158,10 @@ func TestPhase0Epoch(t *testing.T) { func TestAltairEpoch(t *testing.T) { - analyzer, err := BuildChainAnalyzer() - if err != nil { - t.Errorf("could not build analyzer: %s", err) - return - } + analyzer := requireNodes(t, BuildChainAnalyzer) // returns the state in a custom struct for Phase0, Altair of Bellatrix - stateMetrics, err := BuildEpochTask(&analyzer, 2375711) // epoch 74240 + stateMetrics, err := BuildEpochTask(analyzer, 2375711) // epoch 74240 if err != nil { t.Errorf("could not build epoch task: %s", err) return @@ -147,14 +181,10 @@ func TestAltairEpoch(t *testing.T) { func TestAltairRewards(t *testing.T) { - analyzer, err := BuildChainAnalyzer() - if err != nil { - t.Errorf("could not build analyzer: %s", err) - return - } + analyzer := requireNodes(t, BuildChainAnalyzer) // returns the state in a custom struct for Phase0, Altair of Bellatrix - stateMetrics, err := BuildEpochTask(&analyzer, 6565759) // epoch 205179 + stateMetrics, err := BuildEpochTask(analyzer, 6565759) // epoch 205179 if err != nil { t.Errorf("could not build epoch task: %s", err) return @@ -241,13 +271,9 @@ func TestAltairRewards(t *testing.T) { func TestAltairNegativeRewards(t *testing.T) { - analyzer, err := BuildChainAnalyzer() - if err != nil { - t.Errorf("could not build analyzer: %s", err) - return - } + analyzer := requireNodes(t, BuildChainAnalyzer) // returns the state in a custom struct for Phase0, Altair of Bellatrix - stateMetrics, err := BuildEpochTask(&analyzer, 6565823) // epoch 205181 + stateMetrics, err := BuildEpochTask(analyzer, 6565823) // epoch 205181 if err != nil { t.Errorf("could not build epoch task: %s", err) return @@ -295,12 +321,7 @@ func TestAltairNegativeRewards(t *testing.T) { func TestCapellaBlock(t *testing.T) { - blockAnalyzer, err := BuildChainAnalyzerWithEL() - - if err != nil { - t.Errorf("could not build analyzer: %s", err) - return - } + blockAnalyzer := requireNodes(t, BuildChainAnalyzerWithEL) // Test regular case @@ -363,12 +384,7 @@ func TestCapellaBlock(t *testing.T) { func TestBellatrixBlock(t *testing.T) { - blockAnalyzer, err := BuildChainAnalyzer() - - if err != nil { - t.Errorf("could not build analyzer: %s", err) - return - } + blockAnalyzer := requireNodes(t, BuildChainAnalyzer) // Test regular case @@ -432,16 +448,11 @@ func TestBellatrixBlock(t *testing.T) { func TestAltairBlock(t *testing.T) { - blockAnalyzer, err := BuildChainAnalyzer() - - if err != nil { - t.Errorf("could not build analyzer: %s", err) - return - } + blockAnalyzer := requireNodes(t, BuildChainAnalyzer) // Test regular case - block, err := blockAnalyzer.cli.RequestBeaconBlock(4636687) + block, _ := blockAnalyzer.cli.RequestBeaconBlock(4636687) assert.Equal(t, block.Proposed, true) @@ -462,7 +473,7 @@ func TestAltairBlock(t *testing.T) { // Test missed - block, err = blockAnalyzer.cli.RequestBeaconBlock(4709992) + block, _ = blockAnalyzer.cli.RequestBeaconBlock(4709992) assert.Equal(t, block.Proposed, false) @@ -484,16 +495,11 @@ func TestAltairBlock(t *testing.T) { func TestPhase0Block(t *testing.T) { - blockAnalyzer, err := BuildChainAnalyzer() - - if err != nil { - t.Errorf("could not build analyzer: %s", err) - return - } + blockAnalyzer := requireNodes(t, BuildChainAnalyzer) // Test regular case - block, err := blockAnalyzer.cli.RequestBeaconBlock(2372310) + block, _ := blockAnalyzer.cli.RequestBeaconBlock(2372310) assert.Equal(t, block.Proposed, true) @@ -514,7 +520,7 @@ func TestPhase0Block(t *testing.T) { // Test missed - block, err = blockAnalyzer.cli.RequestBeaconBlock(2372309) + block, _ = blockAnalyzer.cli.RequestBeaconBlock(2372309) assert.Equal(t, block.Proposed, false) @@ -535,12 +541,7 @@ func TestPhase0Block(t *testing.T) { } func TestTransactionGasWhenELIsProvided(t *testing.T) { - blockAnalyzer, err := BuildChainAnalyzerWithEL() - - if err != nil { - t.Errorf("could not build analyzer: %s", err) - return - } + blockAnalyzer := requireNodes(t, BuildChainAnalyzerWithEL) // Test regular case block, err := blockAnalyzer.cli.RequestBeaconBlock(8020631) //block number 18826815 @@ -575,12 +576,7 @@ func TestTransactionGasWhenELIsProvided(t *testing.T) { } func TestTransactionGasWhenELNotProvided(t *testing.T) { - blockAnalyzer, err := BuildChainAnalyzer() - - if err != nil { - t.Errorf("could not build analyzer: %s", err) - return - } + blockAnalyzer := requireNodes(t, BuildChainAnalyzer) // Test regular case block, err := blockAnalyzer.cli.RequestBeaconBlock(8020631) //block number 18826815 @@ -615,24 +611,14 @@ func TestTransactionGasWhenELNotProvided(t *testing.T) { } func TestBlockSizeIsSetWhenELIsProvided(t *testing.T) { - blockAnalyzer, err := BuildChainAnalyzerWithEL() - - if err != nil { - t.Errorf("could not build analyzer: %s", err) - return - } + blockAnalyzer := requireNodes(t, BuildChainAnalyzerWithEL) block, _ := blockAnalyzer.cli.RequestBeaconBlock(5610381) //block number 16442285 assert.Equal(t, block.ExecutionPayload.PayloadSize, uint32(69157)) } func TestBlockSizeNotSetWhenELNotProvided(t *testing.T) { - blockAnalyzer, err := BuildChainAnalyzer() - - if err != nil { - t.Errorf("could not build analyzer: %s", err) - return - } + blockAnalyzer := requireNodes(t, BuildChainAnalyzer) block, _ := blockAnalyzer.cli.RequestBeaconBlock(5610381) //block number 16442285 assert.Equal(t, block.ExecutionPayload.PayloadSize, uint32(0)) @@ -640,11 +626,7 @@ func TestBlockSizeNotSetWhenELNotProvided(t *testing.T) { func TestBlockGasFees(t *testing.T) { - analyzer, err := BuildChainAnalyzer() - if err != nil { - t.Errorf("could not build analyzer: %s", err) - return - } + analyzer := requireNodes(t, BuildChainAnalyzer) block, err := analyzer.cli.RequestBeaconBlock(8790975) if err != nil { From 330b9e5c74aab034ec59218e11834e8fa918911a Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Wed, 16 Sep 2026 09:39:49 +0200 Subject: [PATCH 33/47] fix(clientapi,analyzer): stop leaking a timer per retry Four places created a time.Ticker, waited on one tick, and never stopped it. An unstopped ticker is never collected, so each one went on waking the runtime once a second for the life of the process. All four sat in retry or rate-limit paths, which is what makes it more than untidy: they accumulate exactly when the node is already under stress. A long sync with intermittent beacon node timeouts leaked one per retry and never gave any of them back. pkg/clientapi/block.go one per failed block request retry pkg/clientapi/state.go one per timed-out state request retry pkg/clientapi/transaction.go one per failed receipt request retry pkg/analyzer/routines.go one per rate-limit hit in runHistorical A one-shot wait is a sleep. #295 suggests time.After, but block.go already uses time.Sleep three lines above for its 404 retry, so this follows the file's own idiom rather than introducing a second one. Measured: a ticker's first tick and time.Sleep over the same duration are the same wait. The escalating backoff in block.go is unchanged - the duration still grows with the attempt count. --- pkg/analyzer/routines.go | 8 ++++++-- pkg/clientapi/block.go | 6 ++++-- pkg/clientapi/state.go | 4 ++-- pkg/clientapi/transaction.go | 4 ++-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pkg/analyzer/routines.go b/pkg/analyzer/routines.go index 3dd3e366..8053cee9 100644 --- a/pkg/analyzer/routines.go +++ b/pkg/analyzer/routines.go @@ -18,6 +18,7 @@ func (s *ChainAnalyzer) runDownloadBlocks() { defer s.wgDownload.Done() log.Info("Launching Beacon Block Requester") ticker := time.NewTicker(utils.RoutineFlushTimeout) + defer ticker.Stop() // lives as long as the routine; stopped for a clean shutdown downloadRoutine: for { @@ -80,6 +81,7 @@ func (s *ChainAnalyzer) runHead() { s.eventsObj.SubscribeToReorgsEvents() s.eventsObj.SubscribeToDataColumnSidecarsEvents() ticker := time.NewTicker(utils.RoutineFlushTimeout) + defer ticker.Stop() // lives as long as the routine; stopped for a clean shutdown // loop over the list of slots that we need to analyze for { @@ -319,8 +321,10 @@ func (s *ChainAnalyzer) runHistorical(init phase0.Slot, end phase0.Slot) { } if s.processerBook.NumFreePages() == 0 { log.Debugf("hit limit of concurrent processers") - limitTicker := time.NewTicker(utils.RoutineFlushTimeout) - <-limitTicker.C // if rate limit, wait for ticker + // One-shot wait, so a sleep rather than a ticker nothing stops. + // This is the rate-limit path, so it fires exactly when the node is + // already under load and the leaks accumulate fastest. + time.Sleep(utils.RoutineFlushTimeout) continue } if i%spec.SlotsPerEpoch == 0 { // every time a new epoch is crossed diff --git a/pkg/clientapi/block.go b/pkg/clientapi/block.go index c082abeb..c5178804 100644 --- a/pkg/clientapi/block.go +++ b/pkg/clientapi/block.go @@ -55,9 +55,11 @@ func (s *APIClient) RequestBeaconBlock(slot phase0.Slot) (*local_spec.AgnosticBl } timeoutTime := utils.RoutineFlushTimeout * time.Duration(attempts+1) - ticker := time.NewTicker(timeoutTime) log.Warnf("retrying request: %s. Attempt number: %d", routineKey, attempts) - <-ticker.C + // A plain sleep, like the 404 retry above. A ticker here was never + // stopped, so every retry left one waking the runtime once a second + // for the life of the process. + time.Sleep(timeoutTime) } attempts += 1 diff --git a/pkg/clientapi/state.go b/pkg/clientapi/state.go index b706161b..ff8aca3a 100644 --- a/pkg/clientapi/state.go +++ b/pkg/clientapi/state.go @@ -47,9 +47,9 @@ func (s *APIClient) requestBeaconStateWithID(slot phase0.Slot, stateID string, k }) if errors.Is(err, context.DeadlineExceeded) { - ticker := time.NewTicker(utils.RoutineFlushTimeout) log.Warnf("retrying request: %s", routineKey) - <-ticker.C + // One-shot wait, so a sleep rather than a ticker nothing stops. + time.Sleep(utils.RoutineFlushTimeout) } attempts += 1 diff --git a/pkg/clientapi/transaction.go b/pkg/clientapi/transaction.go index 9ea20ac4..2f91d95a 100644 --- a/pkg/clientapi/transaction.go +++ b/pkg/clientapi/transaction.go @@ -84,9 +84,9 @@ func (s *APIClient) GetTransactionReceipt(iTx bellatrix.Transaction, receipt, err = s.ELApi.TransactionReceipt(s.ctx, parsedTx.Hash()) if err != nil { - ticker := time.NewTicker(utils.RoutineFlushTimeout) log.Warnf("retrying transaction request: %s", parsedTx.Hash().String()) - <-ticker.C + // One-shot wait, so a sleep rather than a ticker nothing stops. + time.Sleep(utils.RoutineFlushTimeout) } attempts += 1 From 641091f39429929938e651bbffd91689745ae98b Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Wed, 16 Sep 2026 09:39:49 +0200 Subject: [PATCH 34/47] fix: stop the long-lived tickers too, so the rule can be checked #295 lists these four as correct but untidy: each is created once per long-running routine and lives as long as the process, so none of them leaks in practice. They are stopped anyway, for a reason the issue does not cover. The invariant worth having is "every ticker is stopped", and that cannot be checked while four known-unstopped tickers exist - the check would have to carry an exception list, which is the kind of thing that grows. Behaviour is unchanged: the deferred Stop runs when the routine returns, which is process shutdown. --- pkg/metrics/service.go | 1 + pkg/validator_window/window.go | 1 + 2 files changed, 2 insertions(+) diff --git a/pkg/metrics/service.go b/pkg/metrics/service.go index ec1bb394..89fd70bd 100644 --- a/pkg/metrics/service.go +++ b/pkg/metrics/service.go @@ -84,6 +84,7 @@ func (p *PrometheusMetrics) launchMetricsUpdater() { defer p.wg.Done() ticker := time.NewTicker(p.RefreshInterval) + defer ticker.Stop() // lives as long as the routine; stopped for a clean shutdown metricsUpdateLoop: for { diff --git a/pkg/validator_window/window.go b/pkg/validator_window/window.go index bd3dfc78..742e45c2 100644 --- a/pkg/validator_window/window.go +++ b/pkg/validator_window/window.go @@ -87,6 +87,7 @@ func (s *ValidatorWindowRunner) Run() { s.eventsObj.SubscribeToFinalizedCheckpointEvents() // every new finalized checkpoint, trigger deletes s.eventsObj.SubscribeToHeadEvents() // for monitorization ticker := time.NewTicker(utils.RoutineFlushTimeout) + defer ticker.Stop() // lives as long as the routine; stopped for a clean shutdown s.routineSyncGroup.Add(1) defer s.routineSyncGroup.Done() From a504bc804925f737b2581a2c881af9f056952142 Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Wed, 16 Sep 2026 09:39:50 +0200 Subject: [PATCH 35/47] test(utils): every ticker is stopped, checked across the repository This is the fifth bug of a shape this repository keeps producing, so the rule is written down where it can fail rather than left as a convention. Walks the tree with go/ast and, for every variable assigned from time.NewTicker, requires a Stop on it in the same function. It reports how many it checked and fails outright if it finds none, so a move or a rename cannot leave it quietly testing nothing. It also flags a NewTicker whose result is not held at all. That was a real blind spot rather than a hypothetical: `<-time.NewTicker(d).C` compiles, leaks exactly like the assigned form, and the first version of this test passed it while reporting "checked 10" - it had not even seen it. Both shapes are now checked, and the count moves to 11 when one appears. Known limit: a ticker assigned to a struct field is not flagged. Its lifecycle is genuinely different - stopped elsewhere, possibly in another method - so flagging it would be a false positive. No such usage exists in the repository today, verified rather than assumed. Mutation-checked in three ways: the original assigned-leak shape, a dropped defer on a long-lived ticker, and the inline form. The guard's count matches what git grep finds, and CI runs it through `go test -race ./...`. --- pkg/utils/ticker_lifecycle_test.go | 164 +++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 pkg/utils/ticker_lifecycle_test.go diff --git a/pkg/utils/ticker_lifecycle_test.go b/pkg/utils/ticker_lifecycle_test.go new file mode 100644 index 00000000..45c81bba --- /dev/null +++ b/pkg/utils/ticker_lifecycle_test.go @@ -0,0 +1,164 @@ +package utils + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + "testing" +) + +// A time.Ticker that is never stopped is never collected: it goes on waking the +// runtime at its interval for the life of the process. That is harmless for a +// ticker created once per long-running routine and a leak for one created +// inside a loop. +// +// goteth had four of the second kind, all in retry or rate-limit paths, so they +// accumulated exactly when the node was already under stress (issue #295). A +// one-shot wait wants time.Sleep; a real ticker wants Stop. +// +// This walks the repository because the rule is about every package, and +// because a convention nobody can see is one that gets broken again. +func TestEveryTickerIsStopped(t *testing.T) { + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + + checked := 0 + err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + // Vendored and generated trees are not ours to police. + switch info.Name() { + case ".git", "vendor", "go-relay-client": + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, path, nil, 0) + if err != nil { + return err + } + rel, _ := filepath.Rel(root, path) + + ast.Inspect(file, func(n ast.Node) bool { + fn, ok := n.(*ast.FuncDecl) + if !ok || fn.Body == nil { + return true + } + named, unnamed := tickersIn(fn) + for name, pos := range named { + checked++ + if !stopsTicker(fn, name) { + t.Errorf("%s:%d: ticker %q is never stopped. If it is a one-shot "+ + "wait use time.Sleep; if it is a real ticker add defer %s.Stop(). "+ + "An unstopped ticker keeps firing for the life of the process.", + rel, fset.Position(pos).Line, name, name) + } + } + // A ticker nobody keeps cannot be stopped by anybody. <-time.NewTicker(d).C + // reads as a one-shot wait and leaks exactly like the assigned version, + // which is why it is named rather than merely uncounted. + for _, pos := range unnamed { + checked++ + t.Errorf("%s:%d: time.NewTicker result is not held, so it can never be "+ + "stopped. For a one-shot wait use time.Sleep.", + rel, fset.Position(pos).Line) + } + return true + }) + return nil + }) + if err != nil { + t.Fatal(err) + } + + // If the walk stops finding tickers, this test has quietly stopped testing + // anything - a rename of the package or a move of the tree would do it. + if checked == 0 { + t.Fatal("found no time.NewTicker calls at all; this test is no longer looking " + + "at the right tree") + } + t.Logf("checked %d tickers", checked) +} + +// tickersIn returns the variables in fn assigned from time.NewTicker, and the +// positions of any NewTicker call whose result is not bound to a variable at +// all. The second group needs no Stop check: nothing holds them, so nothing can +// stop them. +func tickersIn(fn *ast.FuncDecl) (map[string]token.Pos, []token.Pos) { + named := map[string]token.Pos{} + bound := map[token.Pos]bool{} + + ast.Inspect(fn.Body, func(n ast.Node) bool { + assign, ok := n.(*ast.AssignStmt) + if !ok || len(assign.Lhs) != 1 || len(assign.Rhs) != 1 { + return true + } + if !isCallTo(assign.Rhs[0], "time", "NewTicker") { + return true + } + bound[assign.Rhs[0].Pos()] = true + if name, ok := assign.Lhs[0].(*ast.Ident); ok { + named[name.Name] = assign.Pos() + } + return true + }) + + var unnamed []token.Pos + ast.Inspect(fn.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok || !isCallTo(call, "time", "NewTicker") || bound[call.Pos()] { + return true + } + unnamed = append(unnamed, call.Pos()) + return true + }) + return named, unnamed +} + +// stopsTicker reports whether fn calls name.Stop() anywhere, deferred or not. +func stopsTicker(fn *ast.FuncDecl, name string) bool { + stopped := false + ast.Inspect(fn.Body, func(n ast.Node) bool { + if isCallTo(n, name, "Stop") { + stopped = true + } + return !stopped + }) + return stopped +} + +func isCallTo(n ast.Node, receiver, method string) bool { + var fun ast.Expr + switch v := n.(type) { + case *ast.CallExpr: + fun = v.Fun + case *ast.DeferStmt: + fun = v.Call.Fun + case *ast.ExprStmt: + call, ok := v.X.(*ast.CallExpr) + if !ok { + return false + } + fun = call.Fun + default: + return false + } + sel, ok := fun.(*ast.SelectorExpr) + if !ok || sel.Sel.Name != method { + return false + } + ident, ok := sel.X.(*ast.Ident) + return ok && ident.Name == receiver +} From 09d608f503eeaa00067176d13e402e6b387026ed Mon Sep 17 00:00:00 2001 From: ander-deran-arteaga Date: Fri, 18 Sep 2026 12:36:12 +0200 Subject: [PATCH 36/47] fix(clientapi): decide "block not found" from the status code, not the text The check asked whether the rendered error contained the substring "404". The beacon node is addressed by slot, so on a transport failure go-eth2-client joins in the error from net/http, which carries the full request URL: failed to call GET endpoint Get "http://bn:5052/eth/v2/beacon/blocks/404": dial tcp ...: refused 79,447 of the first 15,000,000 slots contain "404" in decimal, about one in two hundred. For those slots an unreachable beacon node satisfied the check. The reverse also held: any status code passed as long as something laundered a "404" into the message, so a 500 naming slot 4040 read as not-found. Five call sites were affected, in two different shapes. RequestBeaconBlock and the three blob.go sites went through response404(err.Error()). RequestBeaconBlock took the missing-block branch and returned CreateMissingBlock(slot) with a nil error: a synthetic empty block was written and the epoch completed clean, indistinguishable downstream from a genuinely empty slot. The blob sites did the same more quietly, returning empty sidecars for a slot never fetched. RequestBlockRoot open-coded the same check inline, which is why it did not turn up alongside the others, and it is the damaging one. It returns a zero root on a match and calls log.Panicf otherwise. A zero root never equals the cached root, so reorg.go deletes that slot's block metrics and redownloads it. A transient beacon node failure during reorg handling, on a 404-shaped slot, therefore deleted correct data, and the redownload hit the first bug and wrote a fabricated empty block in its place. The two compounded. isNotFound asks the error for its status code instead. The slot, the port, the peer address and any hex root echoed back in a body can no longer be mistaken for a status. This changes behaviour, deliberately, in two places. In RequestBeaconBlock a transport failure on a 404-shaped slot now falls through to the retry loop and, if the node stays down, to the existing "unable to retrieve Beacon Block at slot %d" error, so an epoch that used to complete against a flapping node will now stop. In RequestBlockRoot the same failure now reaches the log.Panicf that every other slot already reached. Both make a silent wrong answer into a visible one. The panic in RequestBlockRoot is its own problem and is left alone here rather than folded into this fix. go-eth2-client declares Error() on the value receiver, so api.Error and *api.Error both satisfy the error interface. It returns the pointer form today; both are matched, because a switch to the value form would otherwise stop every genuine 404 being recognised and send every missing slot into a retry loop. The tests cover the statuses accepted and rejected, unwrapping through fmt.Errorf and errors.Join, the value form, and the six shapes that fooled the old check. One test makes a request that really fails against a closed loopback port and asserts the hand-built transport error used by the others still matches it, so those cannot drift into asserting against a fiction. Seven mutations were tried, including restoring the old substring implementation, and all seven were caught; an equivalent mutation was confirmed to survive. isNotFound is at 100%. Fixes #298 --- pkg/clientapi/blob.go | 6 +- pkg/clientapi/block.go | 5 +- pkg/clientapi/utils.go | 37 ++++++-- pkg/clientapi/utils_test.go | 169 ++++++++++++++++++++++++++++++++++++ 4 files changed, 206 insertions(+), 11 deletions(-) create mode 100644 pkg/clientapi/utils_test.go diff --git a/pkg/clientapi/blob.go b/pkg/clientapi/blob.go index ce87d010..6e77e113 100644 --- a/pkg/clientapi/blob.go +++ b/pkg/clientapi/blob.go @@ -19,7 +19,7 @@ func (s *APIClient) RequestBlobSidecars(slot phase0.Slot) ([]*local_spec.Agnosti }) if err != nil { - if response404(err.Error()) { + if isNotFound(err) { return agnosticBlobs, nil } return nil, fmt.Errorf("could not retrieve blob sidecars for slot %d: %s", slot, err) @@ -47,7 +47,7 @@ func (s *APIClient) requestKZGCommitmentFromSignedBlock(slot phase0.Slot) ([]den }) if err != nil { - if response404(err.Error()) { + if isNotFound(err) { return nil, nil } return nil, fmt.Errorf("could not retrieve KZGCommitments for slot %d: %s", slot, err) @@ -65,7 +65,7 @@ func (s *APIClient) RequestFuluBlobs(slot phase0.Slot) ([]*local_spec.AgnosticBl }) if err != nil { - if response404(err.Error()) { + if isNotFound(err) { return blobs, nil } return nil, fmt.Errorf("could not retrieve blobs for slot %d: %s", slot, err) diff --git a/pkg/clientapi/block.go b/pkg/clientapi/block.go index c082abeb..6aadaafb 100644 --- a/pkg/clientapi/block.go +++ b/pkg/clientapi/block.go @@ -3,7 +3,6 @@ package clientapi import ( "errors" "fmt" - "strings" "time" "github.com/attestantio/go-eth2-client/api" @@ -41,7 +40,7 @@ func (s *APIClient) RequestBeaconBlock(slot phase0.Slot) (*local_spec.AgnosticBl Block: fmt.Sprintf("%d", slot), }) if err != nil { - if response404(err.Error()) { + if isNotFound(err) { if attempts < s.maxRetries-1 { // Retry on 404: with Lighthouse v8.1.0+ the Head SSE event can // fire before the block is queryable via the API. @@ -122,7 +121,7 @@ func (s *APIClient) RequestBlockRoot(slot phase0.Slot) phase0.Root { Block: fmt.Sprintf("%d", slot), }) if err != nil { - if strings.Contains(err.Error(), "404") { + if isNotFound(err) { // block was not found => block does not exist return phase0.Root{} } diff --git a/pkg/clientapi/utils.go b/pkg/clientapi/utils.go index 8ba1f9a9..9f1a302f 100644 --- a/pkg/clientapi/utils.go +++ b/pkg/clientapi/utils.go @@ -1,11 +1,38 @@ package clientapi -import "strings" +import ( + "errors" + "net/http" -const ( - missingData = "404" + "github.com/attestantio/go-eth2-client/api" ) -func response404(err string) bool { - return strings.Contains(err, missingData) +// isNotFound reports whether a beacon API call failed because the resource +// does not exist, as opposed to failing to reach the node at all. +// +// It asks the error for its status code rather than searching the rendered +// message for "404". The message is not a safe place to look: the beacon node +// is addressed by slot, so on a transport failure go-eth2-client joins in the +// error from net/http, which carries the full request URL. Every slot whose +// decimal form contains "404" (about one in two hundred) then produced a +// message matching a substring search, and an unreachable node was recorded as +// a missing block instead of being retried. The port, the peer address and any +// hex root echoed back in an error body could do the same. +// +// go-eth2-client declares Error() on the value receiver, so both api.Error and +// *api.Error satisfy the error interface. It returns the pointer form today; +// both are matched here so that changing which one is returned cannot quietly +// turn every genuine 404 into a retry loop. +func isNotFound(err error) bool { + var apiErr *api.Error + if errors.As(err, &apiErr) { + return apiErr.StatusCode == http.StatusNotFound + } + + var apiErrValue api.Error + if errors.As(err, &apiErrValue) { + return apiErrValue.StatusCode == http.StatusNotFound + } + + return false } diff --git a/pkg/clientapi/utils_test.go b/pkg/clientapi/utils_test.go new file mode 100644 index 00000000..3b6080dd --- /dev/null +++ b/pkg/clientapi/utils_test.go @@ -0,0 +1,169 @@ +package clientapi + +import ( + "errors" + "fmt" + "net" + nethttp "net/http" + "net/url" + "strings" + "testing" + + "github.com/attestantio/go-eth2-client/api" +) + +// apiError builds the error go-eth2-client returns for a non-2xx response. +// It constructs the pointer form because that is what http.go returns. +func apiError(status int, body string) error { + return &api.Error{ + Method: nethttp.MethodGet, + StatusCode: status, + Endpoint: "/eth/v2/beacon/blocks/12345", + Data: []byte(body), + } +} + +// transportError builds the error go-eth2-client returns when the request +// never reached the node: it joins its own note to the error from net/http, +// which is a *url.Error carrying the full request URL. The slot is in that URL +// because blocks are addressed by slot. +// +// TestTransportErrorMatchesARealFailedRequest proves this construction still +// matches what a real failed request produces. +func transportError(slot string) error { + return errors.Join( + errors.New("failed to call GET endpoint"), + &url.Error{ + Op: "Get", + URL: "http://beacon:5052/eth/v2/beacon/blocks/" + slot, + Err: errors.New("dial tcp 10.0.0.5:5052: connect: connection refused"), + }, + ) +} + +// A missing block is the one case that may be recorded as missing. Everything +// else has to stay an error so the caller retries. +func TestIsNotFoundAcceptsOnlyAGenuine404(t *testing.T) { + for _, tc := range []struct { + name string + err error + want bool + }{ + {"a 404 with no body", apiError(404, ""), true}, + {"a 404 with the node's body", apiError(404, `{"code":404,"message":"NOT_FOUND: beacon block"}`), true}, + {"a 500", apiError(500, `{"code":500,"message":"internal error"}`), false}, + {"a 503 while syncing", apiError(503, `{"code":503,"message":"syncing"}`), false}, + {"a 400", apiError(400, `{"code":400,"message":"invalid block ID"}`), false}, + {"no error at all", nil, false}, + {"an unrelated error", errors.New("something went wrong"), false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := isNotFound(tc.err); got != tc.want { + t.Fatalf("isNotFound(%v) = %v, want %v", tc.err, got, tc.want) + } + }) + } +} + +// These are the cases the previous implementation got wrong. It searched the +// rendered message for the substring "404", and the slot is part of the +// request URL, so an unreachable beacon node was reported as a missing block +// for roughly one slot in two hundred. Each of these returned true before the +// fix and must return false now. +func TestIsNotFoundIgnoresA404ThatIsNotAStatusCode(t *testing.T) { + for _, tc := range []struct { + name string + err error + }{ + {"the node is unreachable while fetching slot 404", transportError("404")}, + {"the node is unreachable while fetching slot 4040", transportError("4040")}, + {"the node is unreachable while fetching slot 1404000", transportError("1404000")}, + {"a 500 whose body names slot 4040", apiError(500, `{"code":500,"message":"error processing slot 4040"}`)}, + {"a 500 whose body names a root containing 404", apiError(500, `{"message":"bad root 0x404a1b"}`)}, + {"a timeout on a 404-shaped slot", errors.Join( + errors.New("failed to call GET endpoint"), + &url.Error{Op: "Get", URL: "http://beacon:5052/eth/v2/beacon/blocks/404", Err: errors.New("context deadline exceeded")}, + )}, + } { + t.Run(tc.name, func(t *testing.T) { + if !strings.Contains(tc.err.Error(), "404") { + t.Fatalf("this case is meant to contain the substring 404 but does not: %v", tc.err) + } + if isNotFound(tc.err) { + t.Errorf("isNotFound reported a missing block for a failure that is not a 404: %v", tc.err) + } + }) + } +} + +// The error reaches isNotFound through whatever the call stack wrapped it in, +// so the status has to be found through the chain rather than only at the top. +func TestIsNotFoundLooksThroughWrapping(t *testing.T) { + notFound := apiError(404, "") + + for _, tc := range []struct { + name string + err error + }{ + {"fmt.Errorf with %w", fmt.Errorf("downloading block at slot 12345: %w", notFound)}, + {"two levels deep", fmt.Errorf("epoch 380: %w", fmt.Errorf("slot 12345: %w", notFound))}, + {"joined with another error", errors.Join(errors.New("failed to call GET endpoint"), notFound)}, + } { + t.Run(tc.name, func(t *testing.T) { + if !isNotFound(tc.err) { + t.Errorf("isNotFound did not find the 404 through the wrapping: %v", tc.err) + } + }) + } +} + +// go-eth2-client declares Error() on the value receiver, so api.Error and +// *api.Error both satisfy the error interface. It returns the pointer today. +// If that ever changes, a check that matched only the pointer would stop +// recognising any 404 and every missing slot would retry to exhaustion, so +// both forms are accepted. +func TestIsNotFoundAcceptsTheValueFormOfTheError(t *testing.T) { + byValue := api.Error{Method: nethttp.MethodGet, StatusCode: 404} + if !isNotFound(byValue) { + t.Error("isNotFound did not recognise a 404 delivered as a value rather than a pointer") + } + + byValueOther := api.Error{Method: nethttp.MethodGet, StatusCode: 500} + if isNotFound(byValueOther) { + t.Error("isNotFound accepted a 500 delivered as a value") + } +} + +// transportError above is a hand-built stand-in for what net/http produces. +// This makes a request that really fails and checks the stand-in still matches +// it: same error text shape, same answer from isNotFound. Without this the +// other tests could drift into asserting against a fiction. +// +// The request goes to a closed port on the loopback interface, so it needs no +// network and cannot reach anything. +func TestTransportErrorMatchesARealFailedRequest(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Skipf("could not reserve a local port: %s", err) + } + addr := listener.Addr().String() + listener.Close() + + _, realErr := nethttp.Get(fmt.Sprintf("http://%s/eth/v2/beacon/blocks/404", addr)) + if realErr == nil { + t.Skip("the request unexpectedly succeeded; something is listening on the reserved port") + } + joined := errors.Join(errors.New("failed to call GET endpoint"), realErr) + + if !strings.Contains(joined.Error(), "404") { + t.Fatalf("a real transport failure for slot 404 no longer carries the slot, so the bug this guards has changed shape: %v", joined) + } + if isNotFound(joined) { + t.Errorf("isNotFound reported a missing block for a real transport failure: %v", joined) + } + + var urlErr *url.Error + if !errors.As(realErr, &urlErr) { + t.Errorf("net/http no longer returns a *url.Error, so transportError() is no longer faithful: %T", realErr) + } +} From c10847e62bca53ceb3b80f3ec07bdf469d0cbf7a Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Fri, 16 Oct 2026 20:05:50 +0200 Subject: [PATCH 37/47] fix(relay): retry a failed delivered bids request before giving up the epoch Each relay is asked once per processed epoch for its last 32 delivered payloads, and nothing asks again later, so one failed request lost the epoch's bids from that relay: the blocks it delivered were stored with no relay and no bid commission. The failures seen are timeouts and DNS lookups that time out, at times for two relays in the same second. A request is now tried up to three times, waiting 5 s and then 10 s, and counts as one failure for the circuit breaker. A relay whose last request failed, the breaker's half-open probe included, gets a single attempt, so a relay that is down does not hold every epoch up. An empty answer, which the client returns as no bids and no error, is an answer and not a failure. The error message is unchanged. --- pkg/relay/relay_bid_trace.go | 58 ++++++++-- pkg/relay/relay_retry_test.go | 202 ++++++++++++++++++++++++++++++++++ 2 files changed, 252 insertions(+), 8 deletions(-) create mode 100644 pkg/relay/relay_retry_test.go diff --git a/pkg/relay/relay_bid_trace.go b/pkg/relay/relay_bid_trace.go index 9b95f765..ff188227 100644 --- a/pkg/relay/relay_bid_trace.go +++ b/pkg/relay/relay_bid_trace.go @@ -23,8 +23,23 @@ const ( // relay for a cooldown period instead of waiting for timeout every time. cbFailureThreshold = 3 cbCooldown = 2 * time.Minute + + // A failed request is retried before the relay's bids for the epoch are + // given up: they are fetched once per epoch and nothing fetches them again. + // The first retry waits relayRetryBackoff, each next one twice as long. + relayRequestAttempts = 3 + relayRetryBackoff = 5 * time.Second ) +// waitBeforeRetry waits d, or less if ctx is done. A variable so the tests can +// record the waits instead of sleeping. +var waitBeforeRetry = func(ctx context.Context, d time.Duration) { + select { + case <-time.After(d): + case <-ctx.Done(): + } +} + var ( log = logrus.WithField( "module", moduleName) @@ -79,6 +94,13 @@ func (r *RelayClient) isOpen() bool { return true } +// failing tells whether the relay's last query failed. +func (r *RelayClient) failing() bool { + r.mu.Lock() + defer r.mu.Unlock() + return r.consecutiveFail > 0 +} + // recordResult updates the circuit breaker after a query. func (r *RelayClient) recordResult(failed bool) { r.mu.Lock() @@ -103,15 +125,35 @@ func (r *RelayClient) GetDeliveredBidsPerSlotRange(slot phase0.Slot, limit int) return nil, fmt.Errorf("circuit breaker open for %s, skipping", r.address) } - bidsDelivered, err := r.client.(relayclient.DeliveredBidTraceProvider).DeliveredBulkBidTrace(r.ctx, slot, limit) - if err != nil || bidsDelivered == nil { - r.recordResult(true) - return bidsDelivered, fmt.Errorf("error obtaining delivered bid trace from %s: %s", r.address, err) + // A relay whose last query failed gets a single attempt: retrying one that + // is down only delays the epoch, and the circuit breaker deals with it. + attempts := relayRequestAttempts + if r.failing() { + attempts = 1 + } + backoff := relayRetryBackoff + var err error + for attempt := 1; ; attempt++ { + // An empty answer (nil, nil) is an answer: the relay delivered nothing. + var bidsDelivered []*v1.BidTrace + bidsDelivered, err = r.client.(relayclient.DeliveredBidTraceProvider).DeliveredBulkBidTrace(r.ctx, slot, limit) + if err == nil { + r.recordResult(false) + return bidsDelivered, nil + } + if attempt == attempts || r.ctx.Err() != nil { + break + } + log.Warnf("retrying %s at slot %d in %s (attempt %d of %d): %s", r.address, slot, backoff, attempt+1, attempts, err) + waitBeforeRetry(r.ctx, backoff) + if r.ctx.Err() != nil { + break + } + backoff *= 2 } - r.recordResult(false) - return bidsDelivered, nil - + r.recordResult(true) + return nil, fmt.Errorf("error obtaining delivered bid trace from %s: %s", r.address, err) } type RelaysMonitor struct { @@ -150,7 +192,7 @@ func (m RelaysMonitor) GetDeliveredBidsPerSlotRange(slot phase0.Slot, limit int) defer wg.Done() singleRelayBidsDelivered, err := rc.GetDeliveredBidsPerSlotRange(slot, limit) - if err != nil || singleRelayBidsDelivered == nil { + if err != nil { log.Errorf("%s", err) return } diff --git a/pkg/relay/relay_retry_test.go b/pkg/relay/relay_retry_test.go new file mode 100644 index 00000000..e55c88cf --- /dev/null +++ b/pkg/relay/relay_retry_test.go @@ -0,0 +1,202 @@ +package relay + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/attestantio/go-eth2-client/spec/phase0" + v1 "github.com/attestantio/go-relay-client/api/v1" +) + +// A relay's delivered bids are fetched once per epoch and nothing fetches them +// again, so one failed request used to lose the epoch's bids from that relay. + +// scriptedRelay answers each request with the next error of its script, nil +// meaning a successful answer, and repeats the last one when the script runs out. +type scriptedRelay struct { + script []error + calls int + empty bool // answer successfully with no bids, as the client does for [] +} + +func (s *scriptedRelay) Name() string { return "scripted" } +func (s *scriptedRelay) Address() string { return "https://scripted.relay" } +func (s *scriptedRelay) Pubkey() *phase0.BLSPubKey { return nil } + +func (s *scriptedRelay) DeliveredBidTrace(context.Context, phase0.Slot) (*v1.BidTrace, error) { + return nil, errors.New("not used") +} + +func (s *scriptedRelay) DeliveredBulkBidTrace(ctx context.Context, slot phase0.Slot, limit int) ([]*v1.BidTrace, error) { + err := s.script[min(s.calls, len(s.script)-1)] + s.calls++ + if err != nil || s.empty { + return nil, err + } + return []*v1.BidTrace{{Slot: slot}}, nil +} + +// scriptedClient is a relay client over a scripted relay. The waits between +// attempts are recorded instead of slept; onWait, if set, runs in each. +func scriptedClient(t *testing.T, ctx context.Context, onWait func(), script ...error) (*RelayClient, *scriptedRelay, *[]time.Duration) { + t.Helper() + var waits []time.Duration + wait := waitBeforeRetry + waitBeforeRetry = func(_ context.Context, d time.Duration) { + waits = append(waits, d) + if onWait != nil { + onWait() + } + } + t.Cleanup(func() { waitBeforeRetry = wait }) + relay := &scriptedRelay{script: script} + return &RelayClient{ctx: ctx, client: relay, address: relay.Address()}, relay, &waits +} + +var errRelay = errors.New("context deadline exceeded") + +func TestATransientFailureIsRetried(t *testing.T) { + r, relay, waits := scriptedClient(t, context.Background(), nil, errRelay, errRelay, nil) + + bids, err := r.GetDeliveredBidsPerSlotRange(15373855, 32) + + if err != nil || len(bids) != 1 { + t.Fatalf("bids = %v, err = %v, want the third attempt's answer", bids, err) + } + if relay.calls != 3 { + t.Errorf("requests = %d, want 3", relay.calls) + } + if len(*waits) != 2 || (*waits)[0] != relayRetryBackoff || (*waits)[1] != 2*relayRetryBackoff { + t.Errorf("waits = %v, want %v then twice that", *waits, relayRetryBackoff) + } + if r.failing() { + t.Error("the breaker counts a failure for a request that succeeded on a retry") + } +} + +// A relay that fails every attempt is one failure for the breaker, and the +// error keeps the message operators grep for. +func TestARequestFailingEveryAttemptIsOneFailure(t *testing.T) { + r, relay, _ := scriptedClient(t, context.Background(), nil, errRelay) + + _, err := r.GetDeliveredBidsPerSlotRange(15373855, 32) + + if err == nil || !strings.Contains(err.Error(), "error obtaining delivered bid trace from https://scripted.relay: context deadline exceeded") { + t.Fatalf("err = %v, want the delivered bid trace error", err) + } + if relay.calls != relayRequestAttempts { + t.Errorf("requests = %d, want %d", relay.calls, relayRequestAttempts) + } + if r.consecutiveFail != 1 { + t.Errorf("consecutive failures = %d, want 1", r.consecutiveFail) + } +} + +// The client turns an empty list into no bids and no error: the relay +// delivered nothing up to the cursor. That is an answer, not a failure. +func TestAnEmptyAnswerIsAnAnswer(t *testing.T) { + r, relay, _ := scriptedClient(t, context.Background(), nil, nil) + relay.empty = true + + bids, err := r.GetDeliveredBidsPerSlotRange(15373855, 32) + + if err != nil || bids != nil { + t.Errorf("bids = %v, err = %v, want neither", bids, err) + } + if relay.calls != 1 || r.failing() { + t.Errorf("%d requests, failing %v: want one request and no failure", relay.calls, r.failing()) + } +} + +// A relay whose last request failed gets one attempt per request, so a relay +// that is down does not hold every epoch up for the retries. +func TestARelayAlreadyFailingGetsOneAttempt(t *testing.T) { + r, relay, _ := scriptedClient(t, context.Background(), nil, errRelay) + r.consecutiveFail = 1 + + if _, err := r.GetDeliveredBidsPerSlotRange(15373855, 32); err == nil { + t.Fatal("want an error") + } + if relay.calls != 1 { + t.Errorf("requests = %d, want 1", relay.calls) + } +} + +// The probe after the breaker's cooldown is a request of a relay known to be +// failing: one attempt, and the breaker opens again if it fails. +func TestTheHalfOpenProbeGetsOneAttempt(t *testing.T) { + r, relay, _ := scriptedClient(t, context.Background(), nil, errRelay) + r.consecutiveFail = cbFailureThreshold + r.openUntil = time.Now().Add(-time.Second) + + if _, err := r.GetDeliveredBidsPerSlotRange(15373855, 32); err == nil || strings.Contains(err.Error(), "circuit breaker open") { + t.Fatalf("err = %v, want the probe's own failure", err) + } + if relay.calls != 1 { + t.Errorf("requests = %d, want 1", relay.calls) + } + if !r.isOpen() { + t.Error("a failed probe leaves the breaker closed") + } +} + +// Three failed requests still open the breaker, and an open breaker sends none. +func TestThreeFailedRequestsStillOpenTheBreaker(t *testing.T) { + r, relay, _ := scriptedClient(t, context.Background(), nil, errRelay) + + for i := 0; i < cbFailureThreshold; i++ { + r.GetDeliveredBidsPerSlotRange(15373855, 32) + } + sent := relay.calls + _, err := r.GetDeliveredBidsPerSlotRange(15373855, 32) + + if err == nil || !strings.Contains(err.Error(), "circuit breaker open") { + t.Fatalf("err = %v, want the breaker open", err) + } + if relay.calls != sent { + t.Errorf("an open breaker sent %d requests", relay.calls-sent) + } +} + +// Shutting down stops the retries, whether it comes before the request or +// during the wait for the next attempt. +func TestShuttingDownStopsTheRetries(t *testing.T) { + t.Run("before the request", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + r, relay, waits := scriptedClient(t, ctx, nil, errRelay) + if _, err := r.GetDeliveredBidsPerSlotRange(15373855, 32); err == nil { + t.Fatal("want an error") + } + if relay.calls != 1 || len(*waits) != 0 { + t.Errorf("%d requests and %d waits, want one request and no wait", relay.calls, len(*waits)) + } + }) + t.Run("during the wait", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + r, relay, _ := scriptedClient(t, ctx, cancel, errRelay) + if _, err := r.GetDeliveredBidsPerSlotRange(15373855, 32); err == nil { + t.Fatal("want an error") + } + if relay.calls != 1 { + t.Errorf("requests = %d, want 1", relay.calls) + } + }) +} + +// A relay that failed before and answers now is healthy again: the next request +// gets its retries back. +func TestAnAnswerAfterAFailureClearsIt(t *testing.T) { + r, _, _ := scriptedClient(t, context.Background(), nil, nil) + r.consecutiveFail = 1 + + if _, err := r.GetDeliveredBidsPerSlotRange(15373855, 32); err != nil { + t.Fatal(err) + } + if r.failing() { + t.Error("the relay still counts as failing after a successful request") + } +} From 466e5142812b8f2887257ed1af2b8a2a02985f36 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Fri, 16 Oct 2026 20:06:23 +0200 Subject: [PATCH 38/47] fix(relay): remove the bloXroute max-profit mainnet relay bloxroute.max-profit.blxrbdn.com no longer resolves and its last delivered payload is slot 15077175. Every epoch's request to it failed. --- pkg/relay/constants.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/relay/constants.go b/pkg/relay/constants.go index 659ecb7b..41165322 100644 --- a/pkg/relay/constants.go +++ b/pkg/relay/constants.go @@ -3,7 +3,6 @@ package relay const ( mainnetAestusRelay string = "https://0xa15b52576bcbf1072f4a011c0f99f9fb6c66f3e1ff321f11f461d15e31b1cb359caa092c71bbded0bae5b5ea401aab7e@aestus.live" mainnetAgnosticRelay string = "https://0xa7ab7a996c8584251c8f925da3170bdfd6ebc75d50f5ddc4050a6fdc77f2a3b5fce2cc750d0865e05d7228af97d69561@agnostic-relay.net" - mainnetBloxRouteMaxProfitRelay string = "https://0x8b5d2e73e2a3a55c6c87b8b6eb92e0149a125c852751db1422fa951e42a09b82c142c3ea98d0d9930b056a3bc9896b8f@bloxroute.max-profit.blxrbdn.com" mainnetBloxRouteRegulatedRelay string = "https://0xb0b07cd0abef743db4260b0ed50619cf6ad4d82064cb4fbec9d3ec530f7c5e6793d9f286c4e082c0244ffb9f2658fe88@bloxroute.regulated.blxrbdn.com" mainnetTitanGlobalRelay string = "https://0x8c4ed5e24fe5c6ae21018437bde147693f68cda427cd1122cf20819c30eda7ed74f72dece09bb313f2a1855595ab677d@global.titanrelay.xyz" mainnetTitanRegionalRelay string = "https://0x8c4ed5e24fe5c6ae21018437bde147693f68cda427cd1122cf20819c30eda7ed74f72dece09bb313f2a1855595ab677d@regional.titanrelay.xyz" @@ -15,7 +14,6 @@ const ( var mainnetRelayList []string = []string{ mainnetAestusRelay, mainnetAgnosticRelay, - mainnetBloxRouteMaxProfitRelay, mainnetBloxRouteRegulatedRelay, mainnetTitanGlobalRelay, mainnetTitanRegionalRelay, From e61829d1c92e579af14b3eaa1db260bb79fa0f41 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Fri, 16 Oct 2026 20:05:50 +0200 Subject: [PATCH 39/47] fix(relay): retry a failed delivered bids request before giving up the epoch Each relay is asked once per processed epoch for its last 32 delivered payloads, and nothing asks again later, so one failed request lost the epoch's bids from that relay: the blocks it delivered were stored with no relay and no bid commission. The failures seen are timeouts and DNS lookups that time out, at times for two relays in the same second. A request is now tried up to three times, waiting 5 s and then 10 s, and counts as one failure for the circuit breaker. A relay whose last request failed, the breaker's half-open probe included, gets a single attempt, so a relay that is down does not hold every epoch up. An empty answer, which the client returns as no bids and no error, is an answer and not a failure. The error message is unchanged. --- pkg/relay/relay_bid_trace.go | 58 ++++++++-- pkg/relay/relay_retry_test.go | 202 ++++++++++++++++++++++++++++++++++ 2 files changed, 252 insertions(+), 8 deletions(-) create mode 100644 pkg/relay/relay_retry_test.go diff --git a/pkg/relay/relay_bid_trace.go b/pkg/relay/relay_bid_trace.go index 9b95f765..ff188227 100644 --- a/pkg/relay/relay_bid_trace.go +++ b/pkg/relay/relay_bid_trace.go @@ -23,8 +23,23 @@ const ( // relay for a cooldown period instead of waiting for timeout every time. cbFailureThreshold = 3 cbCooldown = 2 * time.Minute + + // A failed request is retried before the relay's bids for the epoch are + // given up: they are fetched once per epoch and nothing fetches them again. + // The first retry waits relayRetryBackoff, each next one twice as long. + relayRequestAttempts = 3 + relayRetryBackoff = 5 * time.Second ) +// waitBeforeRetry waits d, or less if ctx is done. A variable so the tests can +// record the waits instead of sleeping. +var waitBeforeRetry = func(ctx context.Context, d time.Duration) { + select { + case <-time.After(d): + case <-ctx.Done(): + } +} + var ( log = logrus.WithField( "module", moduleName) @@ -79,6 +94,13 @@ func (r *RelayClient) isOpen() bool { return true } +// failing tells whether the relay's last query failed. +func (r *RelayClient) failing() bool { + r.mu.Lock() + defer r.mu.Unlock() + return r.consecutiveFail > 0 +} + // recordResult updates the circuit breaker after a query. func (r *RelayClient) recordResult(failed bool) { r.mu.Lock() @@ -103,15 +125,35 @@ func (r *RelayClient) GetDeliveredBidsPerSlotRange(slot phase0.Slot, limit int) return nil, fmt.Errorf("circuit breaker open for %s, skipping", r.address) } - bidsDelivered, err := r.client.(relayclient.DeliveredBidTraceProvider).DeliveredBulkBidTrace(r.ctx, slot, limit) - if err != nil || bidsDelivered == nil { - r.recordResult(true) - return bidsDelivered, fmt.Errorf("error obtaining delivered bid trace from %s: %s", r.address, err) + // A relay whose last query failed gets a single attempt: retrying one that + // is down only delays the epoch, and the circuit breaker deals with it. + attempts := relayRequestAttempts + if r.failing() { + attempts = 1 + } + backoff := relayRetryBackoff + var err error + for attempt := 1; ; attempt++ { + // An empty answer (nil, nil) is an answer: the relay delivered nothing. + var bidsDelivered []*v1.BidTrace + bidsDelivered, err = r.client.(relayclient.DeliveredBidTraceProvider).DeliveredBulkBidTrace(r.ctx, slot, limit) + if err == nil { + r.recordResult(false) + return bidsDelivered, nil + } + if attempt == attempts || r.ctx.Err() != nil { + break + } + log.Warnf("retrying %s at slot %d in %s (attempt %d of %d): %s", r.address, slot, backoff, attempt+1, attempts, err) + waitBeforeRetry(r.ctx, backoff) + if r.ctx.Err() != nil { + break + } + backoff *= 2 } - r.recordResult(false) - return bidsDelivered, nil - + r.recordResult(true) + return nil, fmt.Errorf("error obtaining delivered bid trace from %s: %s", r.address, err) } type RelaysMonitor struct { @@ -150,7 +192,7 @@ func (m RelaysMonitor) GetDeliveredBidsPerSlotRange(slot phase0.Slot, limit int) defer wg.Done() singleRelayBidsDelivered, err := rc.GetDeliveredBidsPerSlotRange(slot, limit) - if err != nil || singleRelayBidsDelivered == nil { + if err != nil { log.Errorf("%s", err) return } diff --git a/pkg/relay/relay_retry_test.go b/pkg/relay/relay_retry_test.go new file mode 100644 index 00000000..e55c88cf --- /dev/null +++ b/pkg/relay/relay_retry_test.go @@ -0,0 +1,202 @@ +package relay + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/attestantio/go-eth2-client/spec/phase0" + v1 "github.com/attestantio/go-relay-client/api/v1" +) + +// A relay's delivered bids are fetched once per epoch and nothing fetches them +// again, so one failed request used to lose the epoch's bids from that relay. + +// scriptedRelay answers each request with the next error of its script, nil +// meaning a successful answer, and repeats the last one when the script runs out. +type scriptedRelay struct { + script []error + calls int + empty bool // answer successfully with no bids, as the client does for [] +} + +func (s *scriptedRelay) Name() string { return "scripted" } +func (s *scriptedRelay) Address() string { return "https://scripted.relay" } +func (s *scriptedRelay) Pubkey() *phase0.BLSPubKey { return nil } + +func (s *scriptedRelay) DeliveredBidTrace(context.Context, phase0.Slot) (*v1.BidTrace, error) { + return nil, errors.New("not used") +} + +func (s *scriptedRelay) DeliveredBulkBidTrace(ctx context.Context, slot phase0.Slot, limit int) ([]*v1.BidTrace, error) { + err := s.script[min(s.calls, len(s.script)-1)] + s.calls++ + if err != nil || s.empty { + return nil, err + } + return []*v1.BidTrace{{Slot: slot}}, nil +} + +// scriptedClient is a relay client over a scripted relay. The waits between +// attempts are recorded instead of slept; onWait, if set, runs in each. +func scriptedClient(t *testing.T, ctx context.Context, onWait func(), script ...error) (*RelayClient, *scriptedRelay, *[]time.Duration) { + t.Helper() + var waits []time.Duration + wait := waitBeforeRetry + waitBeforeRetry = func(_ context.Context, d time.Duration) { + waits = append(waits, d) + if onWait != nil { + onWait() + } + } + t.Cleanup(func() { waitBeforeRetry = wait }) + relay := &scriptedRelay{script: script} + return &RelayClient{ctx: ctx, client: relay, address: relay.Address()}, relay, &waits +} + +var errRelay = errors.New("context deadline exceeded") + +func TestATransientFailureIsRetried(t *testing.T) { + r, relay, waits := scriptedClient(t, context.Background(), nil, errRelay, errRelay, nil) + + bids, err := r.GetDeliveredBidsPerSlotRange(15373855, 32) + + if err != nil || len(bids) != 1 { + t.Fatalf("bids = %v, err = %v, want the third attempt's answer", bids, err) + } + if relay.calls != 3 { + t.Errorf("requests = %d, want 3", relay.calls) + } + if len(*waits) != 2 || (*waits)[0] != relayRetryBackoff || (*waits)[1] != 2*relayRetryBackoff { + t.Errorf("waits = %v, want %v then twice that", *waits, relayRetryBackoff) + } + if r.failing() { + t.Error("the breaker counts a failure for a request that succeeded on a retry") + } +} + +// A relay that fails every attempt is one failure for the breaker, and the +// error keeps the message operators grep for. +func TestARequestFailingEveryAttemptIsOneFailure(t *testing.T) { + r, relay, _ := scriptedClient(t, context.Background(), nil, errRelay) + + _, err := r.GetDeliveredBidsPerSlotRange(15373855, 32) + + if err == nil || !strings.Contains(err.Error(), "error obtaining delivered bid trace from https://scripted.relay: context deadline exceeded") { + t.Fatalf("err = %v, want the delivered bid trace error", err) + } + if relay.calls != relayRequestAttempts { + t.Errorf("requests = %d, want %d", relay.calls, relayRequestAttempts) + } + if r.consecutiveFail != 1 { + t.Errorf("consecutive failures = %d, want 1", r.consecutiveFail) + } +} + +// The client turns an empty list into no bids and no error: the relay +// delivered nothing up to the cursor. That is an answer, not a failure. +func TestAnEmptyAnswerIsAnAnswer(t *testing.T) { + r, relay, _ := scriptedClient(t, context.Background(), nil, nil) + relay.empty = true + + bids, err := r.GetDeliveredBidsPerSlotRange(15373855, 32) + + if err != nil || bids != nil { + t.Errorf("bids = %v, err = %v, want neither", bids, err) + } + if relay.calls != 1 || r.failing() { + t.Errorf("%d requests, failing %v: want one request and no failure", relay.calls, r.failing()) + } +} + +// A relay whose last request failed gets one attempt per request, so a relay +// that is down does not hold every epoch up for the retries. +func TestARelayAlreadyFailingGetsOneAttempt(t *testing.T) { + r, relay, _ := scriptedClient(t, context.Background(), nil, errRelay) + r.consecutiveFail = 1 + + if _, err := r.GetDeliveredBidsPerSlotRange(15373855, 32); err == nil { + t.Fatal("want an error") + } + if relay.calls != 1 { + t.Errorf("requests = %d, want 1", relay.calls) + } +} + +// The probe after the breaker's cooldown is a request of a relay known to be +// failing: one attempt, and the breaker opens again if it fails. +func TestTheHalfOpenProbeGetsOneAttempt(t *testing.T) { + r, relay, _ := scriptedClient(t, context.Background(), nil, errRelay) + r.consecutiveFail = cbFailureThreshold + r.openUntil = time.Now().Add(-time.Second) + + if _, err := r.GetDeliveredBidsPerSlotRange(15373855, 32); err == nil || strings.Contains(err.Error(), "circuit breaker open") { + t.Fatalf("err = %v, want the probe's own failure", err) + } + if relay.calls != 1 { + t.Errorf("requests = %d, want 1", relay.calls) + } + if !r.isOpen() { + t.Error("a failed probe leaves the breaker closed") + } +} + +// Three failed requests still open the breaker, and an open breaker sends none. +func TestThreeFailedRequestsStillOpenTheBreaker(t *testing.T) { + r, relay, _ := scriptedClient(t, context.Background(), nil, errRelay) + + for i := 0; i < cbFailureThreshold; i++ { + r.GetDeliveredBidsPerSlotRange(15373855, 32) + } + sent := relay.calls + _, err := r.GetDeliveredBidsPerSlotRange(15373855, 32) + + if err == nil || !strings.Contains(err.Error(), "circuit breaker open") { + t.Fatalf("err = %v, want the breaker open", err) + } + if relay.calls != sent { + t.Errorf("an open breaker sent %d requests", relay.calls-sent) + } +} + +// Shutting down stops the retries, whether it comes before the request or +// during the wait for the next attempt. +func TestShuttingDownStopsTheRetries(t *testing.T) { + t.Run("before the request", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + r, relay, waits := scriptedClient(t, ctx, nil, errRelay) + if _, err := r.GetDeliveredBidsPerSlotRange(15373855, 32); err == nil { + t.Fatal("want an error") + } + if relay.calls != 1 || len(*waits) != 0 { + t.Errorf("%d requests and %d waits, want one request and no wait", relay.calls, len(*waits)) + } + }) + t.Run("during the wait", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + r, relay, _ := scriptedClient(t, ctx, cancel, errRelay) + if _, err := r.GetDeliveredBidsPerSlotRange(15373855, 32); err == nil { + t.Fatal("want an error") + } + if relay.calls != 1 { + t.Errorf("requests = %d, want 1", relay.calls) + } + }) +} + +// A relay that failed before and answers now is healthy again: the next request +// gets its retries back. +func TestAnAnswerAfterAFailureClearsIt(t *testing.T) { + r, _, _ := scriptedClient(t, context.Background(), nil, nil) + r.consecutiveFail = 1 + + if _, err := r.GetDeliveredBidsPerSlotRange(15373855, 32); err != nil { + t.Fatal(err) + } + if r.failing() { + t.Error("the relay still counts as failing after a successful request") + } +} From 04281a2d0ded0cd1047e08af55d490555f5826b1 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Fri, 16 Oct 2026 20:06:23 +0200 Subject: [PATCH 40/47] fix(relay): remove the bloXroute max-profit mainnet relay bloxroute.max-profit.blxrbdn.com no longer resolves and its last delivered payload is slot 15077175. Every epoch's request to it failed. --- pkg/relay/constants.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/relay/constants.go b/pkg/relay/constants.go index 88ad4a6f..e5604209 100644 --- a/pkg/relay/constants.go +++ b/pkg/relay/constants.go @@ -3,7 +3,6 @@ package relay const ( mainnetAestusRelay string = "https://0xa15b52576bcbf1072f4a011c0f99f9fb6c66f3e1ff321f11f461d15e31b1cb359caa092c71bbded0bae5b5ea401aab7e@aestus.live" mainnetAgnosticRelay string = "https://0xa7ab7a996c8584251c8f925da3170bdfd6ebc75d50f5ddc4050a6fdc77f2a3b5fce2cc750d0865e05d7228af97d69561@agnostic-relay.net" - mainnetBloxRouteMaxProfitRelay string = "https://0x8b5d2e73e2a3a55c6c87b8b6eb92e0149a125c852751db1422fa951e42a09b82c142c3ea98d0d9930b056a3bc9896b8f@bloxroute.max-profit.blxrbdn.com" mainnetBloxRouteRegulatedRelay string = "https://0xb0b07cd0abef743db4260b0ed50619cf6ad4d82064cb4fbec9d3ec530f7c5e6793d9f286c4e082c0244ffb9f2658fe88@bloxroute.regulated.blxrbdn.com" mainnetTitanGlobalRelay string = "https://0x8c4ed5e24fe5c6ae21018437bde147693f68cda427cd1122cf20819c30eda7ed74f72dece09bb313f2a1855595ab677d@global.titanrelay.xyz" mainnetTitanRegionalRelay string = "https://0x8c4ed5e24fe5c6ae21018437bde147693f68cda427cd1122cf20819c30eda7ed74f72dece09bb313f2a1855595ab677d@regional.titanrelay.xyz" @@ -14,7 +13,6 @@ const ( var mainnetRelayList []string = []string{ mainnetAestusRelay, mainnetAgnosticRelay, - mainnetBloxRouteMaxProfitRelay, mainnetBloxRouteRegulatedRelay, mainnetTitanGlobalRelay, mainnetTitanRegionalRelay, From 0e8bab9258b1ab0056d9bc8852485183d6bfb688 Mon Sep 17 00:00:00 2001 From: cortze <45786396+cortze@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:58:37 +0200 Subject: [PATCH 41/47] add citation.cff file --- CITATION.cff | 29 +++++++++++++++++++++++++++++ README.md | 6 ++++++ 2 files changed, 35 insertions(+) create mode 100644 CITATION.cff diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000..6fbe97b1 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,29 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it using the metadata from this file." +type: software +title: GotEth +abstract: >- + GotEth is a go-written client that indexes all validator-related duties, + parameters and transactions from Ethereum's consensus and execution layers. +authors: + - given-names: Mikel + family-names: Cortes + affiliation: MigaLabs + - given-names: Tarun + family-names: Mohandas + affiliation: MigaLabs + - alias: Zyra-V21 + affiliation: MigaLabs + - given-names: Leo + family-names: Bautista-Gomez + affiliation: MigaLabs +repository-code: "https://github.com/migalabs/goteth" +license: MIT +keywords: + - ethereum + - consensus layer + - beacon chain + - validators + - clickhouse +version: v3.8.2 +date-released: "2026-06-12" diff --git a/README.md b/README.md index 5b54ddae..31fa11e5 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,12 @@ If specific upgrades or downgrades need to be done manually, one could do this w During `v3.0.0` we migrated our database system from PostgreSQL to Clickhouse. If you wish to migrate your existing database, please follow [this](https://migalabs.notion.site/PostgreSQL-to-Clickhouse-migration-611a52a457824cd494d701773365f62f) guide. +# Citation + +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.XXXXXXX.svg)](https://doi.org/10.5281/zenodo.XXXXXXX) + +If you use GotEth in your work, please cite it using the metadata in [CITATION.cff](CITATION.cff), or the "Cite this repository" button on GitHub. + # Maintainers @Zyra-V21 From faf00dd09e4d167e2800a764c3afd02ef5af8c54 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Thu, 22 Oct 2026 15:38:33 +0200 Subject: [PATCH 42/47] Update authors in CITATION.cff --- CITATION.cff | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 6fbe97b1..a60ab0bc 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -6,17 +6,19 @@ abstract: >- GotEth is a go-written client that indexes all validator-related duties, parameters and transactions from Ethereum's consensus and execution layers. authors: - - given-names: Mikel - family-names: Cortes - affiliation: MigaLabs - given-names: Tarun family-names: Mohandas affiliation: MigaLabs - alias: Zyra-V21 + given-names: Ruben + family-names: Planes Llinares affiliation: MigaLabs - given-names: Leo family-names: Bautista-Gomez affiliation: MigaLabs + - given-names: Mikel + family-names: Cortes + affiliation: MigaLabs repository-code: "https://github.com/migalabs/goteth" license: MIT keywords: From b23c3ed39d032b1b9a6839b61f7f72f8e0be44d2 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Thu, 22 Oct 2026 15:51:52 +0200 Subject: [PATCH 43/47] Update cff-version to 3.9.0 --- CITATION.cff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index a60ab0bc..25c48554 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,4 +1,4 @@ -cff-version: 1.2.0 +cff-version: 3.9.0 message: "If you use this software, please cite it using the metadata from this file." type: software title: GotEth From 49b81851a93c782351554f812d030686d45ca611 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Thu, 22 Oct 2026 15:52:05 +0200 Subject: [PATCH 44/47] chore: bump version to v3.9.0 --- pkg/utils/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/utils/constants.go b/pkg/utils/constants.go index d20a9a7f..fa09b67b 100644 --- a/pkg/utils/constants.go +++ b/pkg/utils/constants.go @@ -3,7 +3,7 @@ package utils import "time" const ( - Version = "v3.8.2" + Version = "v3.9.0" CliName = "GotEth" RoutineFlushTimeout = time.Duration(1 * time.Second) AcquireWaitIntervalLog = 1 * time.Minute From f21cf49e7acdf170e2b94f6ac2fe68df564404d7 Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Thu, 22 Oct 2026 15:53:08 +0200 Subject: [PATCH 45/47] Update cff-version from 3.9.0 to 1.2.0 --- CITATION.cff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index 25c48554..a60ab0bc 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,4 +1,4 @@ -cff-version: 3.9.0 +cff-version: 1.2.0 message: "If you use this software, please cite it using the metadata from this file." type: software title: GotEth From c4fba76d4f90aa8f4178b046ae41df73f7fd6efb Mon Sep 17 00:00:00 2001 From: Zyra-V21 Date: Thu, 22 Oct 2026 15:54:17 +0200 Subject: [PATCH 46/47] Update version and release date in CITATION.cff --- CITATION.cff | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index a60ab0bc..fe884468 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -27,5 +27,5 @@ keywords: - beacon chain - validators - clickhouse -version: v3.8.2 -date-released: "2026-06-12" +version: v3.9.0 +date-released: "2026-10-22" From e9d99a6e410c4ed0017b000b1fce087eb2656c61 Mon Sep 17 00:00:00 2001 From: Anadi Pandharkar Date: Wed, 11 Nov 2026 18:07:34 -0700 Subject: [PATCH 47/47] ci: read-only token for the CI workflow (CodeQL) Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65348f7f..07c5e2b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,10 @@ on: branches: [main, master, dev] pull_request: +# The job only reads the code: give its token read access, like build_push.yml. +permissions: + contents: read + # This repository had no CI at all. Every change to the indexer serving # mainnet merged without being compiled, vetted or tested by anything but # whoever wrote it.