From 20ec1dabf8f8da47938fc16e6f29581eae0a4fec Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 4 Sep 2026 21:07:27 -0300 Subject: [PATCH 1/6] fix(fast-inbox): replay published blocks by message count --- yarn-project/archiver/src/errors.ts | 19 ++-- .../archiver/src/store/message_store.test.ts | 62 +++++++++-- .../archiver/src/store/message_store.ts | 46 ++++---- .../archiver/src/test/mock_archiver.ts | 4 + .../src/test/mock_l1_to_l2_message_source.ts | 48 +++++++-- .../src/messaging/l1_to_l2_message_source.ts | 10 +- .../src/proposal_handler.test.ts | 68 ++++++++++++ .../validator-client/src/proposal_handler.ts | 16 +-- .../server_world_state_synchronizer.test.ts | 101 +++++++++++++++++- .../server_world_state_synchronizer.ts | 42 ++++---- .../world-state/src/test/integration.test.ts | 66 ++++++++++++ 11 files changed, 409 insertions(+), 73 deletions(-) diff --git a/yarn-project/archiver/src/errors.ts b/yarn-project/archiver/src/errors.ts index 60661d9ea296..5acfa036b551 100644 --- a/yarn-project/archiver/src/errors.ts +++ b/yarn-project/archiver/src/errors.ts @@ -114,13 +114,20 @@ export class InboxBucketNotSyncedError extends Error { } /** - * Thrown when a cumulative Inbox message count does not resolve to a bucket boundary this archiver has synced, either - * because the count sits inside a bucket or because the bucket is not synced yet. + * Thrown when a cumulative Inbox message-count range is not fully backed by the messages this archiver has synced, + * either because it reaches past the synced tip or because the store is missing an index inside it. */ -export class InboxBucketBoundaryNotSyncedError extends Error { - constructor(public readonly totalMsgCount: bigint) { - super(`No synced Inbox bucket ends at cumulative message count ${totalMsgCount}`); - this.name = 'InboxBucketBoundaryNotSyncedError'; +export class InboxMessageRangeNotSyncedError extends Error { + constructor( + public readonly startLeafCount: bigint, + public readonly endLeafCount: bigint, + public readonly actualCount: bigint, + ) { + super( + `Inbox message range [${startLeafCount}, ${endLeafCount}) is not fully synced ` + + `(got ${actualCount} of ${endLeafCount - startLeafCount} messages)`, + ); + this.name = 'InboxMessageRangeNotSyncedError'; } } diff --git a/yarn-project/archiver/src/store/message_store.test.ts b/yarn-project/archiver/src/store/message_store.test.ts index b9718fcfe818..a22490dcd342 100644 --- a/yarn-project/archiver/src/store/message_store.test.ts +++ b/yarn-project/archiver/src/store/message_store.test.ts @@ -2,12 +2,13 @@ import { CheckpointNumber } from '@aztec/foundation/branded-types'; import { Buffer32 } from '@aztec/foundation/buffer'; import { Fr } from '@aztec/foundation/curves/bn254'; import { toArray } from '@aztec/foundation/iterable'; +import type { AztecAsyncKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { Checkpoint, type PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import { updateInboxRollingHash } from '@aztec/stdlib/messaging'; import '@aztec/stdlib/testing/jest'; -import { InboxBucketBoundaryNotSyncedError, InboxBucketNotSyncedError } from '../errors.js'; +import { InboxBucketNotSyncedError, InboxMessageRangeNotSyncedError } from '../errors.js'; import type { InboxMessage } from '../structs/inbox_message.js'; import { makeInboxMessage, @@ -23,6 +24,7 @@ import { type ArchiverL1SynchPoint, getArchiverSynchPoint } from './data_stores. import { MessageStore, MessageStoreError } from './message_store.js'; describe('MessageStore', () => { + let db: AztecAsyncKVStore; let blockStore: BlockStore; let messageStore: MessageStore; let publishedCheckpoints: PublishedCheckpoint[]; @@ -40,7 +42,7 @@ describe('MessageStore', () => { }); beforeEach(async () => { - const db = await openTmpStore('message_store_test'); + db = await openTmpStore('message_store_test'); blockStore = new BlockStore(db); messageStore = new MessageStore(db); // Create checkpoints sequentially to ensure archive roots are chained properly. @@ -484,23 +486,61 @@ describe('MessageStore', () => { expect(await messageStore.getL1ToL2MessagesBetweenLeafCounts(0n, 0n)).toEqual([]); }); - it('throws when a leaf count does not land on a synced bucket boundary', async () => { + it('returns messages between leaf counts interior to the bucket partition', async () => { const msgs = makeBucketedMessages(threeBucketSpec); await messageStore.addL1ToL2MessageBuckets(msgs); + const leaves = msgs.map(m => m.leaf); - // Counts inside a bucket and past the last synced bucket both fail rather than returning a partial range. - await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(0n, 4n)).rejects.toThrow( - InboxBucketBoundaryNotSyncedError, - ); - await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(4n, 6n)).rejects.toThrow( - InboxBucketBoundaryNotSyncedError, - ); + // Counts 1, 2 and 4 sit inside a bucket. A published block commits to a leaf count, and a reorg can merge away + // the bucket that ended there, so the range is addressed by message index alone. + expect(await messageStore.getL1ToL2MessagesBetweenLeafCounts(1n, 6n)).toEqual(leaves.slice(1)); + expect(await messageStore.getL1ToL2MessagesBetweenLeafCounts(0n, 4n)).toEqual(leaves.slice(0, 4)); + expect(await messageStore.getL1ToL2MessagesBetweenLeafCounts(1n, 4n)).toEqual(leaves.slice(1, 4)); + expect(await messageStore.getL1ToL2MessagesBetweenLeafCounts(4n, 6n)).toEqual(leaves.slice(4)); + // An empty range at an interior count consumes nothing rather than failing. + expect(await messageStore.getL1ToL2MessagesBetweenLeafCounts(4n, 4n)).toEqual([]); + }); + + it('throws on an invalid or unsynced leaf count range', async () => { + const msgs = makeBucketedMessages(threeBucketSpec); + await messageStore.addL1ToL2MessageBuckets(msgs); + + // Ranges reaching past the synced tip fail rather than returning a partial range, empty ones included. await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(3n, 9n)).rejects.toThrow( - InboxBucketBoundaryNotSyncedError, + InboxMessageRangeNotSyncedError, + ); + await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(7n, 7n)).rejects.toThrow( + InboxMessageRangeNotSyncedError, ); + // Reversed and negative bounds are caller errors, reported like every other failure of this API: as a + // rejection, so remote callers behind the archiver RPC see them the same way. await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(5n, 3n)).rejects.toThrow( /Invalid Inbox leaf count range/, ); + await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(-1n, 3n)).rejects.toThrow( + /Invalid Inbox leaf count range/, + ); + await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(0n, -1n)).rejects.toThrow( + /Invalid Inbox leaf count range/, + ); + }); + + it('throws when the leaf count range has a hole', async () => { + const msgs = makeBucketedMessages(threeBucketSpec); + await messageStore.addL1ToL2MessageBuckets(msgs); + + // Defense in depth: insertion is contiguity-checked and removal only ever drops a suffix, so no caller can put + // a hole in the middle of the log. Punch one straight into the map to prove a short read is never returned. + await db.openMap('archiver_l1_to_l2_messages').delete(2); + + await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(0n, 6n)).rejects.toThrow( + InboxMessageRangeNotSyncedError, + ); + await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(2n, 3n)).rejects.toThrow( + InboxMessageRangeNotSyncedError, + ); + // Ranges that do not cover the hole are unaffected. + expect(await messageStore.getL1ToL2MessagesBetweenLeafCounts(3n, 6n)).toEqual(msgs.slice(3).map(m => m.leaf)); }); it('rewinds buckets when messages are removed', async () => { diff --git a/yarn-project/archiver/src/store/message_store.ts b/yarn-project/archiver/src/store/message_store.ts index 19b539332a4e..d2480ece2d82 100644 --- a/yarn-project/archiver/src/store/message_store.ts +++ b/yarn-project/archiver/src/store/message_store.ts @@ -14,7 +14,7 @@ import { } from '@aztec/kv-store'; import { type InboxBucket, updateInboxRollingHash } from '@aztec/stdlib/messaging'; -import { InboxBucketBoundaryNotSyncedError, InboxBucketNotSyncedError } from '../errors.js'; +import { InboxBucketNotSyncedError, InboxMessageRangeNotSyncedError } from '../errors.js'; import { type InboxMessage, deserializeInboxMessage, serializeInboxMessage } from '../structs/inbox_message.js'; /** @@ -474,27 +474,37 @@ export class MessageStore { /** * Returns the message leaves in the cumulative Inbox message-count range `[startLeafCount, endLeafCount)`, in - * insertion order. The bounds are compact L1-to-L2 tree leaf counts, which every block header - * carries, so consumers can ask for the messages a block or checkpoint consumed without resolving buckets - * themselves. Both bounds must land on a bucket boundary this archiver has synced; it throws otherwise, since a - * caller asking for a range always expects the messages in it. + * insertion order. The bounds are compact L1-to-L2 tree leaf counts, which every block header carries, so consumers + * can ask for the messages a block or checkpoint consumed without resolving buckets themselves. + * + * The bounds address canonical compact message indices and need not land on a bucket boundary of the partition this + * archiver currently holds: a published block commits to a leaf count, while an L1 reorg can merge away the bucket + * that once ended there. An invalid range, one reaching past the synced tip, or one the store cannot serve whole + * throws, since a caller asking for a range always expects every message in it. */ public async getL1ToL2MessagesBetweenLeafCounts(startLeafCount: bigint, endLeafCount: bigint): Promise { - if (startLeafCount > endLeafCount) { + if (startLeafCount < 0n || endLeafCount < 0n || startLeafCount > endLeafCount) { throw new Error(`Invalid Inbox leaf count range [${startLeafCount}, ${endLeafCount})`); } - const startBucket = await this.getBucketAtBoundary(startLeafCount); - const endBucket = await this.getBucketAtBoundary(endLeafCount); - return this.getL1ToL2MessagesBetweenBuckets(startBucket.seq, endBucket.seq); - } - - /** Resolves the bucket ending at the given cumulative message count, failing loudly if there is none. */ - private async getBucketAtBoundary(totalMsgCount: bigint): Promise { - const bucket = await this.getInboxBucketByTotalMsgCount(totalMsgCount); - if (bucket === undefined) { - throw new InboxBucketBoundaryNotSyncedError(totalMsgCount); - } - return bucket; + // The synced total and the leaves are read together so a concurrent suffix removal cannot land between them and + // turn a range this store holds whole into a spurious incomplete one. + return await this.db.transactionAsync(async () => { + const syncedTotal = await this.getTotalL1ToL2MessageCount(); + if (endLeafCount > syncedTotal) { + const available = syncedTotal > startLeafCount ? syncedTotal - startLeafCount : 0n; + throw new InboxMessageRangeNotSyncedError(startLeafCount, endLeafCount, available); + } + if (startLeafCount === endLeafCount) { + return []; + } + const leaves = await this.getMessageLeavesInIndexRange(startLeafCount, endLeafCount); + // The map holds at most one entry per index, so a short read is the only way a hole inside the range can show + // up, and the count catches every one of them. + if (BigInt(leaves.length) !== endLeafCount - startLeafCount) { + throw new InboxMessageRangeNotSyncedError(startLeafCount, endLeafCount, BigInt(leaves.length)); + } + return leaves; + }); } /** diff --git a/yarn-project/archiver/src/test/mock_archiver.ts b/yarn-project/archiver/src/test/mock_archiver.ts index 5ef6b4d49500..8285d4e8fcb6 100644 --- a/yarn-project/archiver/src/test/mock_archiver.ts +++ b/yarn-project/archiver/src/test/mock_archiver.ts @@ -18,6 +18,10 @@ export class MockArchiver extends MockL2BlockSource implements L2BlockSource, L1 this.messageSource.setInboxBucket(bucket, msgs); } + public replaceInboxBuckets(buckets: { bucket: InboxBucket; msgs: Fr[] }[]) { + this.messageSource.replaceInboxBuckets(buckets); + } + getL1ToL2MessageIndex(_l1ToL2Message: Fr): Promise { return this.messageSource.getL1ToL2MessageIndex(_l1ToL2Message); } diff --git a/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts b/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts index 53b652f8d279..01913be555d0 100644 --- a/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts +++ b/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts @@ -9,12 +9,34 @@ import type { InboxBucket, L1ToL2MessageSource } from '@aztec/stdlib/messaging'; export class MockL1ToL2MessageSource implements L1ToL2MessageSource { private buckets = new Map(); private messagesPerBucket = new Map(); + /** + * The canonical message log, keyed by compact global index. Kept apart from the bucket partition so a test can + * repartition the buckets (as an L1 reorg does) while the indexed leaves stay exactly as they were. + */ + private leavesByIndex = new Map(); constructor(private blockNumber: number) {} public setInboxBucket(bucket: InboxBucket, msgs: Fr[] = []) { this.buckets.set(bucket.seq, bucket); this.messagesPerBucket.set(bucket.seq, msgs); + // Index from the bucket's own cumulative total rather than call order, so re-registering a bucket or setting them + // out of order keeps every leaf at the index the archiver would give it. + const firstIndex = bucket.totalMsgCount - BigInt(msgs.length); + msgs.forEach((msg, i) => this.leavesByIndex.set(firstIndex + BigInt(i), msg)); + } + + /** + * Replaces the current bucket partition without touching the indexed leaf log, modelling an L1 reorg that re-mines + * the same messages under different bucket boundaries. + */ + public replaceInboxBuckets(buckets: { bucket: InboxBucket; msgs: Fr[] }[]) { + this.buckets = new Map(); + this.messagesPerBucket = new Map(); + for (const { bucket, msgs } of buckets) { + this.buckets.set(bucket.seq, bucket); + this.messagesPerBucket.set(bucket.seq, msgs); + } } public setBlockNumber(blockNumber: number) { @@ -50,13 +72,27 @@ export class MockL1ToL2MessageSource implements L1ToL2MessageSource { return Promise.resolve(seqs.flatMap(seq => this.messagesPerBucket.get(seq) ?? [])); } - async getL1ToL2MessagesBetweenLeafCounts(startLeafCount: bigint, endLeafCount: bigint): Promise { - const startBucket = await this.getInboxBucketByTotalMsgCount(startLeafCount); - const endBucket = await this.getInboxBucketByTotalMsgCount(endLeafCount); - if (startBucket === undefined || endBucket === undefined) { - throw new Error(`No mocked Inbox bucket boundary at ${startLeafCount} or ${endLeafCount}`); + /** + * Slices the indexed leaf log, enforcing the same range contract as the archiver's message store. Every failure is + * a rejection rather than a synchronous throw, so callers see this stand-in behave like the async source it mocks. + */ + getL1ToL2MessagesBetweenLeafCounts(startLeafCount: bigint, endLeafCount: bigint): Promise { + if (startLeafCount < 0n || endLeafCount < 0n || startLeafCount > endLeafCount) { + return Promise.reject(new Error(`Invalid Inbox leaf count range [${startLeafCount}, ${endLeafCount})`)); + } + const leaves: Fr[] = []; + for (let index = startLeafCount; index < endLeafCount; index++) { + const leaf = this.leavesByIndex.get(index); + if (leaf === undefined) { + return Promise.reject( + new Error( + `Inbox message range [${startLeafCount}, ${endLeafCount}) is not fully mocked (missing index ${index})`, + ), + ); + } + leaves.push(leaf); } - return this.getL1ToL2MessagesBetweenBuckets(startBucket.seq, endBucket.seq); + return Promise.resolve(leaves); } getBlockNumber() { diff --git a/yarn-project/stdlib/src/messaging/l1_to_l2_message_source.ts b/yarn-project/stdlib/src/messaging/l1_to_l2_message_source.ts index 4040483f144b..ff6426206fc0 100644 --- a/yarn-project/stdlib/src/messaging/l1_to_l2_message_source.ts +++ b/yarn-project/stdlib/src/messaging/l1_to_l2_message_source.ts @@ -53,9 +53,13 @@ export interface L1ToL2MessageSource { /** * Returns the message leaves in the cumulative Inbox message-count range `[startLeafCount, endLeafCount)`, in - * insertion order. The bounds are compact L1-to-L2 tree leaf counts, which every block header - * carries, so a consumer can ask for the messages a block or checkpoint consumed without resolving Inbox buckets - * itself. Both bounds must land on a bucket boundary the source has synced; it throws otherwise. + * insertion order. The bounds are compact L1-to-L2 tree leaf counts, which every block header carries, so a + * consumer can ask for the messages a block or checkpoint consumed without resolving Inbox buckets itself. + * + * The bounds address canonical compact message indices and need not land on a boundary of the bucket partition the + * source currently holds, so a published block's committed leaf counts stay resolvable after an L1 reorg has merged + * the bucket that ended at one of them. An invalid range, one past the synced tip, or one the source cannot serve + * whole throws, so an empty result always means the range holds no messages. * @param startLeafCount - The cumulative Inbox message count the range starts at, inclusive. * @param endLeafCount - The cumulative Inbox message count the range ends at, exclusive. */ diff --git a/yarn-project/validator-client/src/proposal_handler.test.ts b/yarn-project/validator-client/src/proposal_handler.test.ts index 4bf7316a6f25..8d1f2787b972 100644 --- a/yarn-project/validator-client/src/proposal_handler.test.ts +++ b/yarn-project/validator-client/src/proposal_handler.test.ts @@ -562,6 +562,74 @@ describe('ProposalHandler checkpoint validation', () => { return makeHeader({ epochOutHash, ...overrides }); } + // An L1 reorg can merge the bucket a checkpointed block consumed through into a later one. The archiver never + // prunes a checkpointed block, so that block's committed count stays interior to the current partition forever; + // reading the consumed range by count keeps the parent position resolvable, where resolving it as a bucket would + // derive an empty bundle and make this node decline a valid checkpoint. + it('derives the consumed bundle from a checkpoint-start count interior to the current buckets', async () => { + const header = makeHeader(); + const consumedMessages = [new Fr(1000), new Fr(1001), new Fr(1002), new Fr(1003)]; + setupDeepValidationMocks({ header }); + + const firstBlockNumber = 5; + const block = { + archive: new AppendOnlyTreeSnapshot(archiveRoot, 1), + number: firstBlockNumber, + checkpointNumber: CheckpointNumber(1), + header: { + globalVariables: GlobalVariables.empty({ slotNumber: SlotNumber(1) }), + state: { l1ToL2MessageTree: { nextAvailableLeafIndex: 7 } }, + }, + } as unknown as L2Block; + blockSource.getBlocksForSlot.mockResolvedValue([block]); + blockSource.getBlockData.mockImplementation(query => + Promise.resolve( + 'number' in query && query.number === firstBlockNumber - 1 + ? ({ + header: { state: { l1ToL2MessageTree: { nextAvailableLeafIndex: 3 } } }, + } as unknown as BlockData) + : ({ header: makeBlockHeader() } as BlockData), + ), + ); + + // The checkpoint's final position is still a live bucket boundary; only the parent's is gone. + l1ToL2MessageSource.getInboxBucketByTotalMsgCount.mockImplementation(total => + Promise.resolve( + total === 7n + ? ({ + seq: 1n, + inboxRollingHash: Fr.random(), + totalMsgCount: 7n, + timestamp: 10n, + msgCount: 7, + lastMessageIndex: 6n, + l1BlockNumber: 10n, + l1BlockHash: Buffer32.fromBigInt(10n), + } satisfies InboxBucket) + : undefined, + ), + ); + l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts.mockResolvedValue(consumedMessages); + + await handler.handleCheckpointProposal( + await makeProposal({ archiveRoot, checkpointHeader: header }), + proposalInfo, + ); + + expect(l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts).toHaveBeenCalledWith(3n, 7n); + expect(checkpointsBuilder.openCheckpoint).toHaveBeenCalledWith( + CheckpointNumber(1), + expect.anything(), + expect.anything(), + consumedMessages, + expect.anything(), + expect.anything(), + expect.anything(), + [block], + expect.anything(), + ); + }); + it('returns checkpoint_header_mismatch when headers differ', async () => { const proposalHeader = makeHeader(); const computedHeader = makeHeader({ totalManaUsed: new Fr(999) }); diff --git a/yarn-project/validator-client/src/proposal_handler.ts b/yarn-project/validator-client/src/proposal_handler.ts index 33faaa69ec8d..6d5dfdfd444a 100644 --- a/yarn-project/validator-client/src/proposal_handler.ts +++ b/yarn-project/validator-client/src/proposal_handler.ts @@ -1156,9 +1156,14 @@ export class ProposalHandler { } /** - * Derives the ordered list of L1-to-L2 messages a checkpoint consumed across its blocks, from the Inbox buckets - * between the parent checkpoint's consumed position and the checkpoint's last block. Empty when - * the checkpoint consumed nothing or its consumption cannot be resolved against the local Inbox view. + * Derives the ordered list of L1-to-L2 messages a checkpoint consumed across its blocks: the compact message-count + * range between the parent checkpoint's consumed position and the checkpoint's last block. Empty when the + * checkpoint consumed nothing or its final consumption position does not resolve to a current Inbox bucket. + * + * Only the final position is resolved as a bucket, which is the live rule this proposal must satisfy. The start + * position is a count committed by an already checkpointed block, which the archiver never prunes, so an L1 reorg + * that merges buckets can leave it permanently interior to the current partition; reading the range by count keeps + * that historical bound resolvable instead of silently deriving an empty bundle for a valid proposal. */ private async deriveCheckpointConsumedMessages(blocks: L2Block[]): Promise { const checkpointStartTotal = await this.getPreBlockConsumedTotal(blocks[0].number); @@ -1166,12 +1171,11 @@ export class ProposalHandler { if (checkpointStartTotal === undefined || lastBlockTotal <= checkpointStartTotal) { return []; } - const startBucket = await this.l1ToL2MessageSource.getInboxBucketByTotalMsgCount(checkpointStartTotal); const endBucket = await this.l1ToL2MessageSource.getInboxBucketByTotalMsgCount(lastBlockTotal); - if (startBucket === undefined || endBucket === undefined) { + if (endBucket === undefined) { return []; } - return this.l1ToL2MessageSource.getL1ToL2MessagesBetweenBuckets(startBucket.seq, endBucket.seq); + return this.l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts(checkpointStartTotal, endBucket.totalMsgCount); } async reexecuteTransactions( diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts index 1edcaf2bdb64..7e30c34ed709 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts @@ -51,6 +51,21 @@ describe('ServerWorldStateSynchronizer', () => { beforeEach(() => { blockAndMessagesSource = mock(); blockAndMessagesSource.getBlockNumber.mockResolvedValue(BlockNumber(LATEST_BLOCK_NUMBER)); + // Published-block replay reads the parent block's leaf count and then the messages in the range it opens, so the + // source answers both like an archiver holding the whole mock chain. + blockAndMessagesSource.getBlockData.mockImplementation(async query => { + const block = allBlocks().find(b => 'number' in query && b.number === query.number); + return block === undefined + ? undefined + : { + header: block.header, + archive: block.archive, + blockHash: await block.hash(), + checkpointNumber: block.checkpointNumber, + indexWithinCheckpoint: block.indexWithinCheckpoint, + }; + }); + blockAndMessagesSource.getL1ToL2MessagesBetweenLeafCounts.mockResolvedValue([]); merkleTreeRead = mock(); merkleTreeRead.getInitialHeader.mockReturnValue({ @@ -88,8 +103,10 @@ describe('ServerWorldStateSynchronizer', () => { await server.stop(); }); + const allBlocks = () => checkpoints.flatMap(c => c.checkpoint.blocks); + const pushBlocks = async (from: number, to: number) => { - const blocks = checkpoints.flatMap(c => c.checkpoint.blocks).filter(b => b.number >= from && b.number <= to); + const blocks = allBlocks().filter(b => b.number >= from && b.number <= to); await server.handleBlockStreamEvent({ type: 'blocks-added', blocks, @@ -248,6 +265,88 @@ describe('ServerWorldStateSynchronizer', () => { await expect(pushBlocks(1, 5)).rejects.toThrow(/Test error/i); }); + describe('L1 to L2 message replay', () => { + // Blocks with a strictly growing leaf count, so each one opens a non-empty range over the one before it. + const blockLeafCounts = [3, 3, 7]; + + let messagesByRange: Map; + let originalLeafCounts: number[]; + + afterEach(() => { + const blocks = allBlocks(); + originalLeafCounts.forEach( + (count, i) => (blocks[i].header.state.l1ToL2MessageTree.nextAvailableLeafIndex = count), + ); + }); + + beforeEach(() => { + const blocks = allBlocks(); + originalLeafCounts = blockLeafCounts.map( + (_, i) => blocks[i].header.state.l1ToL2MessageTree.nextAvailableLeafIndex, + ); + blockLeafCounts.forEach((count, i) => (blocks[i].header.state.l1ToL2MessageTree.nextAvailableLeafIndex = count)); + + // Historical replay addresses canonical message indices: resolving a count as a bucket of the current + // partition is exactly what a reorg can make impossible, so it must not happen here. + blockAndMessagesSource.getInboxBucketByTotalMsgCount.mockImplementation(() => { + throw new Error('Published-block replay must not resolve leaf counts as buckets'); + }); + blockAndMessagesSource.getL1ToL2MessagesBetweenBuckets.mockImplementation(() => { + throw new Error('Published-block replay must not read messages by bucket'); + }); + + messagesByRange = new Map([ + ['0-3', [new Fr(10n), new Fr(11n), new Fr(12n)]], + ['3-7', [new Fr(13n), new Fr(14n), new Fr(15n), new Fr(16n)]], + ]); + blockAndMessagesSource.getL1ToL2MessagesBetweenLeafCounts.mockImplementation((start, end) => { + const messages = messagesByRange.get(`${start}-${end}`); + return messages === undefined + ? Promise.reject(new Error(`Unexpected leaf count range [${start}, ${end})`)) + : Promise.resolve(messages); + }); + }); + + it('replays each block from the leaf count range it committed to', async () => { + void server.start(); + await pushBlocks(1, 3); + + const blocks = allBlocks(); + expect(merkleTreeDb.handleL2BlockAndMessages.mock.calls).toEqual([ + [blocks[0], messagesByRange.get('0-3')], + // The second block consumed nothing, so its range is empty and needs no query at all. + [blocks[1], []], + [blocks[2], messagesByRange.get('3-7')], + ]); + expect(blockAndMessagesSource.getL1ToL2MessagesBetweenLeafCounts.mock.calls).toEqual([ + [0n, 3n], + [3n, 7n], + ]); + }); + + it('applies the blocks before an unavailable range and stops there', async () => { + void server.start(); + messagesByRange.delete('3-7'); + + await expect(pushBlocks(1, 3)).rejects.toThrow(/Unexpected leaf count range \[3, 7\)/); + + // The unavailable range fails the sync instead of silently applying the block with no messages, and the blocks + // already applied are kept so the retry resumes from there. + expect(merkleTreeDb.handleL2BlockAndMessages.mock.calls).toEqual([ + [allBlocks()[0], messagesByRange.get('0-3')], + [allBlocks()[1], []], + ]); + }); + + it('fails instead of replaying a whole message history when the parent block is missing', async () => { + void server.start(); + blockAndMessagesSource.getBlockData.mockResolvedValue(undefined); + + await expect(pushBlocks(2, 3)).rejects.toThrow(/block 1 is unavailable/); + expect(merkleTreeDb.handleL2BlockAndMessages).not.toHaveBeenCalled(); + }); + }); + describe('getVerifiedSnapshot', () => { let snapshot: MockProxy; diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index eefe75f2a976..4f9ee689be53 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -369,31 +369,22 @@ export class ServerWorldStateSynchronizer private async handleL2Blocks(l2Blocks: L2Block[]) { this.log.debug(`Handling L2 blocks ${l2Blocks[0].number} to ${l2Blocks.at(-1)!.number}`); - // Derive each block's real L1-to-L2 message bundle from the compact leaf-index range it inserted: - // the messages between the parent block's L1-to-L2 tree leaf count and this block's, resolved via the - // Inbox buckets. Blocks in a batch are consecutive, so we track the running leaf count. - const messagesForBlocks = new Map(); + // Each block's real L1-to-L2 message bundle is the compact leaf-index range it inserted: the messages between the + // parent block's L1-to-L2 tree leaf count and this block's. The range addresses canonical message indices, so it + // is served whole even after an L1 reorg merged away the bucket that once ended at the parent's count. Blocks in + // a batch are consecutive, so we track the running leaf count; each block is fetched and applied before the next + // one is read, so a range the archiver cannot serve yet does not discard the blocks already applied. let prevLeafCount = await this.getL1ToL2LeafCountBefore(l2Blocks[0].number); + let updateStatus: WorldStateStatusFull | undefined = undefined; for (const block of l2Blocks) { const blockLeafCount = BigInt(block.header.state.l1ToL2MessageTree.nextAvailableLeafIndex); - if (blockLeafCount > prevLeafCount) { - const startBucket = await this.l2BlockSource.getInboxBucketByTotalMsgCount(prevLeafCount); - const endBucket = await this.l2BlockSource.getInboxBucketByTotalMsgCount(blockLeafCount); - if (startBucket !== undefined && endBucket !== undefined) { - messagesForBlocks.set( - block.number, - await this.l2BlockSource.getL1ToL2MessagesBetweenBuckets(startBucket.seq, endBucket.seq), - ); - } - } + const messages = + blockLeafCount > prevLeafCount + ? await this.l2BlockSource.getL1ToL2MessagesBetweenLeafCounts(prevLeafCount, blockLeafCount) + : []; prevLeafCount = blockLeafCount; - } - let updateStatus: WorldStateStatusFull | undefined = undefined; - for (const block of l2Blocks) { - const [duration, result] = await elapsed(() => - this.handleL2Block(block, messagesForBlocks.get(block.number) ?? []), - ); + const [duration, result] = await elapsed(() => this.handleL2Block(block, messages)); this.log.info(`World state updated with L2 block ${block.number}`, { eventName: 'l2-block-handled', duration, @@ -410,14 +401,21 @@ export class ServerWorldStateSynchronizer this.instrumentation.updateWorldStateMetrics(updateStatus); } - /** The L1-to-L2 message tree leaf count as of the block before `blockNumber` (0 if that block is genesis). */ + /** + * The L1-to-L2 message tree leaf count as of the block before `blockNumber` (0 if that block is genesis). Throws + * when a non-genesis parent is unavailable: treating it as zero would ask for every message ever received as the + * next block's bundle. + */ private async getL1ToL2LeafCountBefore(blockNumber: BlockNumber): Promise { const parentNumber = blockNumber - 1; if (parentNumber < INITIAL_L2_BLOCK_NUM) { return 0n; } const parentBlock = await this.l2BlockSource.getBlockData({ number: BlockNumber(parentNumber) }); - return parentBlock === undefined ? 0n : BigInt(parentBlock.header.state.l1ToL2MessageTree.nextAvailableLeafIndex); + if (parentBlock === undefined) { + throw new Error(`Cannot derive L1 to L2 messages for block ${blockNumber}: block ${parentNumber} is unavailable`); + } + return BigInt(parentBlock.header.state.l1ToL2MessageTree.nextAvailableLeafIndex); } /** diff --git a/yarn-project/world-state/src/test/integration.test.ts b/yarn-project/world-state/src/test/integration.test.ts index b5c29b7b9f31..e87c33bd722e 100644 --- a/yarn-project/world-state/src/test/integration.test.ts +++ b/yarn-project/world-state/src/test/integration.test.ts @@ -1,6 +1,7 @@ import { MockPrefilledArchiver } from '@aztec/archiver/test'; import { GENESIS_ARCHIVE_ROOT } from '@aztec/constants'; import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; +import { Buffer32 } from '@aztec/foundation/buffer'; import { timesAsync } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; @@ -164,6 +165,71 @@ describe('world-state integration', () => { }); }); + describe('Inbox bucket repartitioning', () => { + // Rebuilds the current bucket partition as a single bucket holding every message the given blocks consumed, as an + // L1 reorg that re-mines the same messages under merged boundaries leaves it. The indexed leaves are unchanged, + // but the boundary each published block consumed through is gone from the current partition. + const mergeInboxBuckets = (blockCount: number) => { + const messages = checkpoints.slice(0, blockCount).flatMap(c => c.messages); + archiver.replaceInboxBuckets([ + { + bucket: { + seq: 0n, + inboxRollingHash: Fr.ZERO, + totalMsgCount: 0n, + timestamp: 0n, + msgCount: 0, + lastMessageIndex: 0n, + l1BlockNumber: 0n, + l1BlockHash: Buffer32.ZERO, + }, + msgs: [], + }, + { + bucket: { + seq: 1n, + inboxRollingHash: Fr.ZERO, + totalMsgCount: BigInt(messages.length), + timestamp: 1n, + msgCount: messages.length, + lastMessageIndex: BigInt(messages.length) - 1n, + l1BlockNumber: 1n, + l1BlockHash: Buffer32.ZERO, + }, + msgs: messages, + }, + ]); + }; + + it('replays published blocks whose consumed boundary the current partition no longer has', async () => { + const blockCount = 3; + await archiver.createBlocks(blockCount); + mergeInboxBuckets(blockCount); + + // Driven directly rather than through the block stream, which swallows and retries the archive-root divergence + // a wrong message bundle causes. + const blocks = checkpoints.slice(0, blockCount).flatMap(c => c.checkpoint.blocks); + await synchronizer.handleBlockStreamEvent({ type: 'blocks-added', blocks }); + + for (let blockNumber = 1; blockNumber <= blockCount; blockNumber++) { + await expectSynchedBlockHashMatches(blockNumber); + } + const lastBlockState = blocks.at(-1)!.header.state; + const messageTree = await db.getCommitted().getTreeInfo(MerkleTreeId.L1_TO_L2_MESSAGE_TREE); + expect(messageTree.root).toEqual(lastBlockState.l1ToL2MessageTree.root.toBuffer()); + expect(messageTree.size).toEqual(BigInt(lastBlockState.l1ToL2MessageTree.nextAvailableLeafIndex)); + }); + + it('syncs the whole chain through the block stream under a merged partition', async () => { + const blockCount = 3; + await archiver.createBlocks(blockCount); + mergeInboxBuckets(blockCount); + + await synchronizer.start(); + await expectSynchedToBlock(blockCount); + }); + }); + describe('reorgs', () => { it('prunes blocks upon a reorg and resyncs', async () => { await archiver.createBlocks(5); From 8f91c9ca8e1f764b683a93cd3db1d8ac7400d42f Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 4 Sep 2026 21:26:53 -0300 Subject: [PATCH 2/6] fix(fast-inbox): derive the validator's checkpoint bundle by message count The checkpoint validator resolved the proposal's final leaf count to a bucket of the current partition before reading the consumed range, and derived an empty bundle when that lookup missed. Both bounds are counts committed by block headers, so the range is read by count alone; whether the final position is a live bucket end stays the publication rule enforced by L1 and the existing censorship guard. A parent block that is unavailable locally is reported as a fetch error instead of an empty bundle, which would have failed the rolling-hash recomputation and been classified as a slashable header mismatch. --- .../src/proposal_handler.test.ts | 115 +++++++++++++----- .../validator-client/src/proposal_handler.ts | 61 ++++++---- .../validator-client/src/validator.test.ts | 11 ++ 3 files changed, 131 insertions(+), 56 deletions(-) diff --git a/yarn-project/validator-client/src/proposal_handler.test.ts b/yarn-project/validator-client/src/proposal_handler.test.ts index 8d1f2787b972..72ff62e9163c 100644 --- a/yarn-project/validator-client/src/proposal_handler.test.ts +++ b/yarn-project/validator-client/src/proposal_handler.test.ts @@ -562,53 +562,49 @@ describe('ProposalHandler checkpoint validation', () => { return makeHeader({ epochOutHash, ...overrides }); } - // An L1 reorg can merge the bucket a checkpointed block consumed through into a later one. The archiver never - // prunes a checkpointed block, so that block's committed count stays interior to the current partition forever; - // reading the consumed range by count keeps the parent position resolvable, where resolving it as a bucket would - // derive an empty bundle and make this node decline a valid checkpoint. - it('derives the consumed bundle from a checkpoint-start count interior to the current buckets', async () => { - const header = makeHeader(); - const consumedMessages = [new Fr(1000), new Fr(1001), new Fr(1002), new Fr(1003)]; - setupDeepValidationMocks({ header }); - - const firstBlockNumber = 5; + /** A checkpoint whose first block is `firstBlockNumber`, with the parent block reporting `parentLeafCount`. */ + function setupCheckpointWithConsumption(opts: { + firstBlockNumber: number; + parentLeafCount: number | undefined; + lastLeafCount: number; + }) { + const { firstBlockNumber, parentLeafCount, lastLeafCount } = opts; const block = { archive: new AppendOnlyTreeSnapshot(archiveRoot, 1), number: firstBlockNumber, checkpointNumber: CheckpointNumber(1), header: { globalVariables: GlobalVariables.empty({ slotNumber: SlotNumber(1) }), - state: { l1ToL2MessageTree: { nextAvailableLeafIndex: 7 } }, + state: { l1ToL2MessageTree: { nextAvailableLeafIndex: lastLeafCount } }, }, } as unknown as L2Block; blockSource.getBlocksForSlot.mockResolvedValue([block]); blockSource.getBlockData.mockImplementation(query => Promise.resolve( 'number' in query && query.number === firstBlockNumber - 1 - ? ({ - header: { state: { l1ToL2MessageTree: { nextAvailableLeafIndex: 3 } } }, - } as unknown as BlockData) + ? parentLeafCount === undefined + ? undefined + : ({ + header: { state: { l1ToL2MessageTree: { nextAvailableLeafIndex: parentLeafCount } } }, + } as unknown as BlockData) : ({ header: makeBlockHeader() } as BlockData), ), ); + return block; + } - // The checkpoint's final position is still a live bucket boundary; only the parent's is gone. - l1ToL2MessageSource.getInboxBucketByTotalMsgCount.mockImplementation(total => - Promise.resolve( - total === 7n - ? ({ - seq: 1n, - inboxRollingHash: Fr.random(), - totalMsgCount: 7n, - timestamp: 10n, - msgCount: 7, - lastMessageIndex: 6n, - l1BlockNumber: 10n, - l1BlockHash: Buffer32.fromBigInt(10n), - } satisfies InboxBucket) - : undefined, - ), - ); + // An L1 reorg can merge the buckets a checkpointed block and the proposal's last block consumed through into + // others. The archiver never prunes a checkpointed block, so the parent position stays interior to the current + // partition forever, and the proposal's own final position may sit interior to it as well while the messages + // themselves are unchanged. The consumed bundle is derived from the committed counts alone, so neither position + // needs to resolve to a bucket; whether the final position is a live bucket end is L1's publication rule. + it('derives the consumed bundle by count when neither checkpoint bound is a current bucket boundary', async () => { + const header = makeHeader(); + const consumedMessages = [new Fr(1000), new Fr(1001), new Fr(1002), new Fr(1003)]; + setupDeepValidationMocks({ header }); + const block = setupCheckpointWithConsumption({ firstBlockNumber: 5, parentLeafCount: 3, lastLeafCount: 7 }); + + l1ToL2MessageSource.getInboxBucketByTotalMsgCount.mockResolvedValue(undefined); l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts.mockResolvedValue(consumedMessages); await handler.handleCheckpointProposal( @@ -617,6 +613,7 @@ describe('ProposalHandler checkpoint validation', () => { ); expect(l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts).toHaveBeenCalledWith(3n, 7n); + expect(l1ToL2MessageSource.getL1ToL2MessagesBetweenBuckets).not.toHaveBeenCalled(); expect(checkpointsBuilder.openCheckpoint).toHaveBeenCalledWith( CheckpointNumber(1), expect.anything(), @@ -630,6 +627,62 @@ describe('ProposalHandler checkpoint validation', () => { ); }); + it('derives an empty bundle without any message query when the checkpoint consumed nothing', async () => { + const header = makeHeader(); + setupDeepValidationMocks({ header }); + const block = setupCheckpointWithConsumption({ firstBlockNumber: 5, parentLeafCount: 3, lastLeafCount: 3 }); + + await handler.handleCheckpointProposal( + await makeProposal({ archiveRoot, checkpointHeader: header }), + proposalInfo, + ); + + expect(l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts).not.toHaveBeenCalled(); + expect(checkpointsBuilder.openCheckpoint).toHaveBeenCalledWith( + CheckpointNumber(1), + expect.anything(), + expect.anything(), + [], + expect.anything(), + expect.anything(), + expect.anything(), + [block], + expect.anything(), + ); + }); + + // A missing parent is a local chain-availability failure. Deriving an empty bundle instead would make the + // rolling-hash recomputation fail and classify the proposer's valid checkpoint as a slashable header mismatch. + it('reports a fetch error instead of deriving an empty bundle when the parent block is unavailable', async () => { + const header = makeHeader(); + setupDeepValidationMocks({ header }); + setupCheckpointWithConsumption({ firstBlockNumber: 5, parentLeafCount: undefined, lastLeafCount: 7 }); + + const result = await handler.handleCheckpointProposal( + await makeProposal({ archiveRoot, checkpointHeader: header }), + proposalInfo, + ); + + expect(result).toEqual({ isValid: false, reason: 'block_fetch_error', checkpointNumber: CheckpointNumber(1) }); + expect(l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts).not.toHaveBeenCalled(); + expect(checkpointsBuilder.openCheckpoint).not.toHaveBeenCalled(); + }); + + it('surfaces an unavailable consumed range instead of deriving an empty bundle', async () => { + const header = makeHeader(); + setupDeepValidationMocks({ header }); + setupCheckpointWithConsumption({ firstBlockNumber: 5, parentLeafCount: 3, lastLeafCount: 7 }); + l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts.mockRejectedValue( + new Error('Inbox message range [3, 7) is not fully synced'), + ); + + await expect( + handler.handleCheckpointProposal(await makeProposal({ archiveRoot, checkpointHeader: header }), proposalInfo), + ).rejects.toThrow(/not fully synced/); + + expect(checkpointsBuilder.openCheckpoint).not.toHaveBeenCalled(); + }); + it('returns checkpoint_header_mismatch when headers differ', async () => { const proposalHeader = makeHeader(); const computedHeader = makeHeader({ totalManaUsed: new Fr(999) }); diff --git a/yarn-project/validator-client/src/proposal_handler.ts b/yarn-project/validator-client/src/proposal_handler.ts index 6d5dfdfd444a..d0fbdec23944 100644 --- a/yarn-project/validator-client/src/proposal_handler.ts +++ b/yarn-project/validator-client/src/proposal_handler.ts @@ -1135,14 +1135,17 @@ export class ProposalHandler { /** * Enforces the streaming-Inbox last-block minimum-consumption (censorship) rule for a checkpoint, mirroring * `ProposeLib.validateInboxConsumption`: the first bucket the checkpoint left unconsumed must be absent, past the - * cutoff, or a cap-escape. Returns true (sufficient) when the checkpoint's consumption cannot be resolved against - * the local Inbox view, deferring to L1 `propose` as the authoritative reject. + * cutoff, or a cap-escape. Returns true (sufficient) when the checkpoint's final consumption position cannot be + * resolved against the local Inbox view, deferring to L1 `propose` as the authoritative reject. */ - private async isLastBlockConsumptionSufficient(slot: SlotNumber, blocks: L2Block[]): Promise { + private async isLastBlockConsumptionSufficient( + slot: SlotNumber, + checkpointStartTotal: bigint, + blocks: L2Block[], + ): Promise { const lastBlockTotal = this.blockLeafCount(blocks[blocks.length - 1]); - const checkpointStartTotal = await this.getPreBlockConsumedTotal(blocks[0].number); const lastConsumedBucket = await this.l1ToL2MessageSource.getInboxBucketByTotalMsgCount(lastBlockTotal); - if (checkpointStartTotal === undefined || lastConsumedBucket === undefined) { + if (lastConsumedBucket === undefined) { return true; } const nextBucket = await this.l1ToL2MessageSource.getInboxBucket(lastConsumedBucket.seq + 1n); @@ -1157,25 +1160,21 @@ export class ProposalHandler { /** * Derives the ordered list of L1-to-L2 messages a checkpoint consumed across its blocks: the compact message-count - * range between the parent checkpoint's consumed position and the checkpoint's last block. Empty when the - * checkpoint consumed nothing or its final consumption position does not resolve to a current Inbox bucket. + * range between the parent checkpoint's consumed position and the checkpoint's last block, read by count from the + * local message log. Empty when the checkpoint consumed nothing. * - * Only the final position is resolved as a bucket, which is the live rule this proposal must satisfy. The start - * position is a count committed by an already checkpointed block, which the archiver never prunes, so an L1 reorg - * that merges buckets can leave it permanently interior to the current partition; reading the range by count keeps - * that historical bound resolvable instead of silently deriving an empty bundle for a valid proposal. + * Neither bound is resolved as an Inbox bucket. Both are counts committed by block headers, and an L1 reorg that + * merges buckets can leave either one interior to the current partition without changing the messages the blocks + * consumed; whether the final position is a live bucket end is a publication rule L1 `propose` enforces, checked + * separately by the censorship guard. Throws when part of the range is not available locally: a local availability + * gap must surface as such rather than as an empty bundle that would make a valid proposal fail its rolling-hash + * recomputation. */ - private async deriveCheckpointConsumedMessages(blocks: L2Block[]): Promise { - const checkpointStartTotal = await this.getPreBlockConsumedTotal(blocks[0].number); + private deriveCheckpointConsumedMessages(checkpointStartTotal: bigint, blocks: L2Block[]): Promise { const lastBlockTotal = this.blockLeafCount(blocks[blocks.length - 1]); - if (checkpointStartTotal === undefined || lastBlockTotal <= checkpointStartTotal) { - return []; - } - const endBucket = await this.l1ToL2MessageSource.getInboxBucketByTotalMsgCount(lastBlockTotal); - if (endBucket === undefined) { - return []; - } - return this.l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts(checkpointStartTotal, endBucket.totalMsgCount); + return lastBlockTotal <= checkpointStartTotal + ? Promise.resolve([]) + : this.l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts(checkpointStartTotal, lastBlockTotal); } async reexecuteTransactions( @@ -1473,9 +1472,21 @@ export class ProposalHandler { const constants = this.extractCheckpointConstants(firstBlock); const checkpointNumber = firstBlock.checkpointNumber; + // The checkpoint's Inbox consumption starts at the leaf count of the block before its first block. Without that + // block the consumed bundle cannot be derived; an empty bundle would make a valid proposal fail its rolling-hash + // recomputation and be classified as a proposer offense, so a missing parent is a local fetch failure instead. + const checkpointStartTotal = await this.getPreBlockConsumedTotal(firstBlock.number); + if (checkpointStartTotal === undefined) { + this.log.warn(`Block before checkpoint proposal's first block ${firstBlock.number} is unavailable locally`, { + ...proposalInfo, + checkpointNumber, + }); + return { isValid: false, reason: 'block_fetch_error', checkpointNumber }; + } + // Streaming Inbox: on the last block of a checkpoint, enforce the minimum-consumption // (censorship) rule before attesting. Reject (no attestation) if a mandatory bucket was left unconsumed. - if (!(await this.isLastBlockConsumptionSufficient(slot, blocks))) { + if (!(await this.isLastBlockConsumptionSufficient(slot, checkpointStartTotal, blocks))) { this.log.warn(`Streaming Inbox last-block censorship check failed, refusing to attest`, { ...proposalInfo, checkpointNumber, @@ -1483,10 +1494,10 @@ export class ProposalHandler { return { isValid: false, reason: 'inbox_consumption_insufficient', checkpointNumber }; } - // Derive the checkpoint's consumed L1-to-L2 message list from the Inbox buckets between the parent checkpoint's - // consumed position and the last block's (compact indexing). The messages are already in the db from per-block + // Derive the checkpoint's consumed L1-to-L2 message list from the message-count range between the parent + // checkpoint's consumed position and the last block's. The messages are already in the db from per-block // validation; this list only drives the checkpoint's rolling-hash recomputation in completeCheckpoint. - const l1ToL2Messages = await this.deriveCheckpointConsumedMessages(blocks); + const l1ToL2Messages = await this.deriveCheckpointConsumedMessages(checkpointStartTotal, blocks); // Collect the out hashes of all the checkpoints before this one in the same epoch. // See note on the analogous block-proposal site: the helper handles pipelining lag. diff --git a/yarn-project/validator-client/src/validator.test.ts b/yarn-project/validator-client/src/validator.test.ts index d148cd87ea52..fa60588a7a19 100644 --- a/yarn-project/validator-client/src/validator.test.ts +++ b/yarn-project/validator-client/src/validator.test.ts @@ -406,6 +406,16 @@ describe('ValidatorClient', () => { } as unknown as L2Block; const disposeFork = jest.fn(); blockSource.getBlocksForSlot.mockResolvedValue([checkpointBlock]); + // The checkpoint's consumed message bundle derives from the leaf count of the block before its first block, so + // that parent must resolve by number; other number queries stay unresolved as in the surrounding setup. + blockSource.getBlockData.mockImplementation(query => + Promise.resolve('number' in query && query.number !== blockNumber - 1 ? undefined : parentBlockData), + ); + // With the parent resolvable, the censorship check runs too: leave no bucket after genesis so nothing is left + // unconsumed and validation reaches the header comparison. + l1ToL2MessageSource.getInboxBucket.mockImplementation(seq => + Promise.resolve(seq === 0n ? genesisInboxBucket : undefined), + ); checkpointsBuilder.getFork.mockResolvedValue({ [Symbol.asyncDispose]: disposeFork, // Match the proposal's expected starting archive so the fork archive check passes and validation @@ -551,6 +561,7 @@ describe('ValidatorClient', () => { l1ToL2MessageSource.getInboxBucket.mockResolvedValue(genesisInboxBucket); l1ToL2MessageSource.getInboxBucketByTotalMsgCount.mockResolvedValue(genesisInboxBucket); l1ToL2MessageSource.getL1ToL2MessagesBetweenBuckets.mockResolvedValue([]); + l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts.mockResolvedValue([]); const clonedBlockHeader = blockHeader.clone(); blockBuildResult = { From ce479a3871f46e62518c664ae392aee071afce38 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 4 Sep 2026 21:26:54 -0300 Subject: [PATCH 3/6] fix(kv-store): keep zero bounds when mapping ranges mapRange dropped a 0n start or end because zero is falsy, turning an exclusive end of zero into an unbounded range. --- yarn-project/kv-store/src/interfaces/common.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/kv-store/src/interfaces/common.ts b/yarn-project/kv-store/src/interfaces/common.ts index 1b6c142d491f..45584336b0ad 100644 --- a/yarn-project/kv-store/src/interfaces/common.ts +++ b/yarn-project/kv-store/src/interfaces/common.ts @@ -18,8 +18,8 @@ export type CustomRange = { /** Maps a custom range into a range of valid key types to iterate over. */ export function mapRange(range: CustomRange, mapFn: (key: CK) => K): Range { return { - start: range.start ? mapFn(range.start) : undefined, - end: range.end ? mapFn(range.end) : undefined, + start: range.start !== undefined ? mapFn(range.start) : undefined, + end: range.end !== undefined ? mapFn(range.end) : undefined, reverse: range.reverse, limit: range.limit, }; From d017f5b16439331df3bac8045ed9ba87c1d95ec3 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 4 Sep 2026 21:26:55 -0300 Subject: [PATCH 4/6] feat(fast-inbox): message position and atomic range reads on the message source Adds InboxMessagePosition and InboxMessageRange next to the L1ToL2MessageSource interface, with getMessagePosition, getSyncedMessagePosition and getL1ToL2MessageRange on the message store, the archiver data source, its RPC schema and the shared mocks. The range read returns the messages and both bounding positions from one store transaction, empty ranges included, and raises the typed availability error when the range or its starting position is unavailable. Positions are read from the existing per-index message records; no schema change. --- yarn-project/archiver/src/errors.ts | 10 +- .../archiver/src/modules/data_source_base.ts | 20 ++- .../archiver/src/store/message_store.test.ts | 116 ++++++++++++++++ .../archiver/src/store/message_store.ts | 126 +++++++++++++++--- .../archiver/src/test/mock_archiver.ts | 23 +++- .../src/test/mock_l1_to_l2_message_source.ts | 61 ++++++++- .../stdlib/src/interfaces/archiver.test.ts | 36 +++++ .../stdlib/src/interfaces/archiver.ts | 12 +- .../src/messaging/l1_to_l2_message_source.ts | 65 ++++++++- 9 files changed, 440 insertions(+), 29 deletions(-) diff --git a/yarn-project/archiver/src/errors.ts b/yarn-project/archiver/src/errors.ts index 5acfa036b551..d5e989b705da 100644 --- a/yarn-project/archiver/src/errors.ts +++ b/yarn-project/archiver/src/errors.ts @@ -115,18 +115,16 @@ export class InboxBucketNotSyncedError extends Error { /** * Thrown when a cumulative Inbox message-count range is not fully backed by the messages this archiver has synced, - * either because it reaches past the synced tip or because the store is missing an index inside it. + * either because it reaches past the synced tip or because the store is missing a message the range needs. + * Distinguishes "not available locally, retry once L1 sync catches up" from a genuinely empty range. */ export class InboxMessageRangeNotSyncedError extends Error { constructor( public readonly startLeafCount: bigint, public readonly endLeafCount: bigint, - public readonly actualCount: bigint, + detail: string, ) { - super( - `Inbox message range [${startLeafCount}, ${endLeafCount}) is not fully synced ` + - `(got ${actualCount} of ${endLeafCount - startLeafCount} messages)`, - ); + super(`Inbox message range [${startLeafCount}, ${endLeafCount}) is not fully synced: ${detail}`); this.name = 'InboxMessageRangeNotSyncedError'; } } diff --git a/yarn-project/archiver/src/modules/data_source_base.ts b/yarn-project/archiver/src/modules/data_source_base.ts index 9e407d9e6765..c37ff585e341 100644 --- a/yarn-project/archiver/src/modules/data_source_base.ts +++ b/yarn-project/archiver/src/modules/data_source_base.ts @@ -40,7 +40,13 @@ import { } from '@aztec/stdlib/epoch-helpers'; import type { L2LogsSource } from '@aztec/stdlib/interfaces/server'; import type { LogResult, PrivateLogsQuery, PublicLogsQuery } from '@aztec/stdlib/logs'; -import type { InboxBucket, L1ToL2MessageSource, L2ToL1MembershipWitness } from '@aztec/stdlib/messaging'; +import type { + InboxBucket, + InboxMessagePosition, + InboxMessageRange, + L1ToL2MessageSource, + L2ToL1MembershipWitness, +} from '@aztec/stdlib/messaging'; import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees'; import type { BlockHeader, IndexedTxEffect, TxHash } from '@aztec/stdlib/tx'; import type { UInt64 } from '@aztec/stdlib/types'; @@ -340,6 +346,18 @@ export abstract class ArchiverDataSourceBase return this.stores.messages.getL1ToL2MessagesBetweenLeafCounts(startLeafCount, endLeafCount); } + public getMessagePosition(totalMessageCount: bigint): Promise { + return this.stores.messages.getMessagePosition(totalMessageCount); + } + + public getSyncedMessagePosition(): Promise { + return this.stores.messages.getSyncedMessagePosition(); + } + + public getL1ToL2MessageRange(startLeafCount: bigint, endLeafCount: bigint): Promise { + return this.stores.messages.getL1ToL2MessageRange(startLeafCount, endLeafCount); + } + private async getPublishedCheckpointFromCheckpointData(checkpoint: CheckpointData): Promise { const blocksForCheckpoint = await this.stores.blocks.getBlocksForCheckpoint(checkpoint.checkpointNumber); if (!blocksForCheckpoint) { diff --git a/yarn-project/archiver/src/store/message_store.test.ts b/yarn-project/archiver/src/store/message_store.test.ts index a22490dcd342..c282d5739c56 100644 --- a/yarn-project/archiver/src/store/message_store.test.ts +++ b/yarn-project/archiver/src/store/message_store.test.ts @@ -226,6 +226,122 @@ describe('MessageStore', () => { }); }); + describe('iterateL1ToL2Messages', () => { + it('honours zero range bounds', async () => { + const msgs = makeInboxMessages(3); + await messageStore.addL1ToL2MessageBuckets(msgs); + + // Zero is a valid compact index, so an exclusive end of zero selects nothing rather than everything. + expect(await toArray(messageStore.iterateL1ToL2Messages({ end: 0n }))).toEqual([]); + expect(await toArray(messageStore.iterateL1ToL2Messages({ start: 0n, end: 2n }))).toEqual(msgs.slice(0, 2)); + expect(await toArray(messageStore.iterateL1ToL2Messages({ start: 0n }))).toEqual(msgs); + }); + }); + + describe('message positions', () => { + const zeroPosition = { totalMessageCount: 0n, rollingHash: Fr.ZERO }; + const positionAfter = (msg: InboxMessage) => ({ + totalMessageCount: msg.index + 1n, + rollingHash: msg.inboxRollingHash, + }); + + it('resolves the position at a synced message count', async () => { + const msgs = makeInboxMessages(6); + await messageStore.addL1ToL2MessageBuckets(msgs); + + // Position zero always resolves; every other count resolves to the hash stored with the message before it. + expect(await messageStore.getMessagePosition(0n)).toEqual(zeroPosition); + expect(await messageStore.getMessagePosition(1n)).toEqual(positionAfter(msgs[0])); + expect(await messageStore.getMessagePosition(4n)).toEqual(positionAfter(msgs[3])); + expect(await messageStore.getMessagePosition(6n)).toEqual(positionAfter(msgs[5])); + // Past the synced tip there is no position yet. + expect(await messageStore.getMessagePosition(7n)).toBeUndefined(); + await expect(messageStore.getMessagePosition(-1n)).rejects.toThrow(/Invalid Inbox message count/); + }); + + it('resolves position zero on an empty store', async () => { + expect(await messageStore.getMessagePosition(0n)).toEqual(zeroPosition); + expect(await messageStore.getMessagePosition(1n)).toBeUndefined(); + expect(await messageStore.getSyncedMessagePosition()).toEqual(zeroPosition); + }); + + it('tracks the synced position through appends and removals', async () => { + const msgs = makeInboxMessages(6); + await messageStore.addL1ToL2MessageBuckets(msgs.slice(0, 4)); + expect(await messageStore.getSyncedMessagePosition()).toEqual(positionAfter(msgs[3])); + + await messageStore.addL1ToL2MessageBuckets(msgs.slice(4)); + expect(await messageStore.getSyncedMessagePosition()).toEqual(positionAfter(msgs[5])); + + await messageStore.removeL1ToL2Messages(2n); + expect(await messageStore.getSyncedMessagePosition()).toEqual(positionAfter(msgs[1])); + // The removed suffix no longer has positions. + expect(await messageStore.getMessagePosition(3n)).toBeUndefined(); + }); + + it('reads a message range together with the positions at both bounds', async () => { + const msgs = makeInboxMessages(6); + await messageStore.addL1ToL2MessageBuckets(msgs); + const leaves = msgs.map(m => m.leaf); + + expect(await messageStore.getL1ToL2MessageRange(0n, 6n)).toEqual({ + messages: leaves, + start: zeroPosition, + end: positionAfter(msgs[5]), + }); + expect(await messageStore.getL1ToL2MessageRange(1n, 4n)).toEqual({ + messages: leaves.slice(1, 4), + start: positionAfter(msgs[0]), + end: positionAfter(msgs[3]), + }); + // An empty range is valid at any synced count and returns equal positions. + expect(await messageStore.getL1ToL2MessageRange(3n, 3n)).toEqual({ + messages: [], + start: positionAfter(msgs[2]), + end: positionAfter(msgs[2]), + }); + expect(await messageStore.getL1ToL2MessageRange(6n, 6n)).toEqual({ + messages: [], + start: positionAfter(msgs[5]), + end: positionAfter(msgs[5]), + }); + }); + + it('reads the empty range at position zero on an empty store', async () => { + expect(await messageStore.getL1ToL2MessageRange(0n, 0n)).toEqual({ + messages: [], + start: zeroPosition, + end: zeroPosition, + }); + }); + + it('throws on an invalid or unsynced message range', async () => { + const msgs = makeInboxMessages(6); + await messageStore.addL1ToL2MessageBuckets(msgs); + + await expect(messageStore.getL1ToL2MessageRange(3n, 9n)).rejects.toThrow(InboxMessageRangeNotSyncedError); + await expect(messageStore.getL1ToL2MessageRange(7n, 7n)).rejects.toThrow(InboxMessageRangeNotSyncedError); + await expect(messageStore.getL1ToL2MessageRange(5n, 3n)).rejects.toThrow(/Invalid Inbox leaf count range/); + await expect(messageStore.getL1ToL2MessageRange(-1n, 3n)).rejects.toThrow(/Invalid Inbox leaf count range/); + }); + + it('throws when the range or its starting position has a hole', async () => { + const msgs = makeInboxMessages(6); + await messageStore.addL1ToL2MessageBuckets(msgs); + await db.openMap('archiver_l1_to_l2_messages').delete(2); + + // Index 2 is missing: ranges over it are short, and a range starting at count 3 has no starting position. + await expect(messageStore.getL1ToL2MessageRange(0n, 6n)).rejects.toThrow(InboxMessageRangeNotSyncedError); + await expect(messageStore.getL1ToL2MessageRange(3n, 6n)).rejects.toThrow(/missing the message at index 2/); + // Ranges that need neither the hole nor a position at it are unaffected. + expect(await messageStore.getL1ToL2MessageRange(4n, 6n)).toEqual({ + messages: msgs.slice(4).map(m => m.leaf), + start: positionAfter(msgs[3]), + end: positionAfter(msgs[5]), + }); + }); + }); + describe('Inbox buckets', () => { // Builds `count` consecutive valid messages, then reassigns their bucket sequence and timestamp per the given // per-message spec so we can exercise multi-message and rollover buckets. diff --git a/yarn-project/archiver/src/store/message_store.ts b/yarn-project/archiver/src/store/message_store.ts index d2480ece2d82..49ef7cba6438 100644 --- a/yarn-project/archiver/src/store/message_store.ts +++ b/yarn-project/archiver/src/store/message_store.ts @@ -12,7 +12,12 @@ import { type CustomRange, mapRange, } from '@aztec/kv-store'; -import { type InboxBucket, updateInboxRollingHash } from '@aztec/stdlib/messaging'; +import { + type InboxBucket, + type InboxMessagePosition, + type InboxMessageRange, + updateInboxRollingHash, +} from '@aztec/stdlib/messaging'; import { InboxBucketNotSyncedError, InboxMessageRangeNotSyncedError } from '../errors.js'; import { type InboxMessage, deserializeInboxMessage, serializeInboxMessage } from '../structs/inbox_message.js'; @@ -110,6 +115,16 @@ const GENESIS_INBOX_BUCKET: InboxBucket = { l1BlockHash: Buffer32.ZERO, }; +/** The position before any message: zero count and zero rolling hash, mirroring the on-chain Inbox base case. */ +const ZERO_MESSAGE_POSITION: InboxMessagePosition = { totalMessageCount: 0n, rollingHash: Fr.ZERO }; + +/** Rejects reversed or negative compact leaf count bounds, which are caller errors rather than sync state. */ +function assertValidLeafCountRange(startLeafCount: bigint, endLeafCount: bigint): void { + if (startLeafCount < 0n || endLeafCount < 0n || startLeafCount > endLeafCount) { + throw new Error(`Invalid Inbox leaf count range [${startLeafCount}, ${endLeafCount})`); + } +} + export class MessageStoreError extends Error { constructor( message: string, @@ -483,30 +498,109 @@ export class MessageStore { * throws, since a caller asking for a range always expects every message in it. */ public async getL1ToL2MessagesBetweenLeafCounts(startLeafCount: bigint, endLeafCount: bigint): Promise { - if (startLeafCount < 0n || endLeafCount < 0n || startLeafCount > endLeafCount) { - throw new Error(`Invalid Inbox leaf count range [${startLeafCount}, ${endLeafCount})`); - } + assertValidLeafCountRange(startLeafCount, endLeafCount); // The synced total and the leaves are read together so a concurrent suffix removal cannot land between them and // turn a range this store holds whole into a spurious incomplete one. return await this.db.transactionAsync(async () => { + await this.assertLeafCountRangeSynced(startLeafCount, endLeafCount); + const messages = await this.getMessagesInLeafCountRange(startLeafCount, endLeafCount); + return messages.map(message => message.leaf); + }); + } + + /** + * Returns the position of the Inbox message sequence after `totalMessageCount` messages: that count and the rolling + * hash over them, which is the rolling hash stored with the message at compact index `totalMessageCount - 1`. + * Position zero always resolves with a zero hash; a count past the synced tip returns undefined. + */ + public async getMessagePosition(totalMessageCount: bigint): Promise { + if (totalMessageCount < 0n) { + throw new Error(`Invalid Inbox message count ${totalMessageCount}`); + } + if (totalMessageCount === 0n) { + return ZERO_MESSAGE_POSITION; + } + const buffer = await this.#l1ToL2Messages.getAsync(this.indexToKey(totalMessageCount - 1n)); + return buffer === undefined + ? undefined + : { totalMessageCount, rollingHash: deserializeInboxMessage(buffer).inboxRollingHash }; + } + + /** Returns the position at the synced tip: the total message count and the rolling hash over every stored message. */ + public getSyncedMessagePosition(): Promise { + return this.db.transactionAsync(async () => { const syncedTotal = await this.getTotalL1ToL2MessageCount(); - if (endLeafCount > syncedTotal) { - const available = syncedTotal > startLeafCount ? syncedTotal - startLeafCount : 0n; - throw new InboxMessageRangeNotSyncedError(startLeafCount, endLeafCount, available); - } - if (startLeafCount === endLeafCount) { - return []; + const position = await this.getMessagePosition(syncedTotal); + if (position === undefined) { + throw new Error(`Inbox message store holds ${syncedTotal} messages but is missing index ${syncedTotal - 1n}`); } - const leaves = await this.getMessageLeavesInIndexRange(startLeafCount, endLeafCount); - // The map holds at most one entry per index, so a short read is the only way a hole inside the range can show - // up, and the count catches every one of them. - if (BigInt(leaves.length) !== endLeafCount - startLeafCount) { - throw new InboxMessageRangeNotSyncedError(startLeafCount, endLeafCount, BigInt(leaves.length)); + return position; + }); + } + + /** + * Returns the messages in the cumulative Inbox message-count range `[startLeafCount, endLeafCount)` together with + * the positions at both bounds. Everything is read in one store transaction, so the ending hash authenticates + * exactly the returned messages appended after the starting position, and a concurrent suffix replacement cannot + * pair the leaves of one version of the sequence with the hash of another. Follows the range contract of + * `getL1ToL2MessagesBetweenLeafCounts`, with the starting position also required to be available; an empty range + * returns equal positions. + */ + public async getL1ToL2MessageRange(startLeafCount: bigint, endLeafCount: bigint): Promise { + assertValidLeafCountRange(startLeafCount, endLeafCount); + return await this.db.transactionAsync(async () => { + await this.assertLeafCountRangeSynced(startLeafCount, endLeafCount); + const start = await this.getMessagePosition(startLeafCount); + if (start === undefined) { + throw new InboxMessageRangeNotSyncedError( + startLeafCount, + endLeafCount, + `the store is missing the message at index ${startLeafCount - 1n}`, + ); } - return leaves; + const messages = await this.getMessagesInLeafCountRange(startLeafCount, endLeafCount); + const lastMessage = messages.at(-1); + const end = + lastMessage === undefined + ? start + : { totalMessageCount: endLeafCount, rollingHash: lastMessage.inboxRollingHash }; + return { messages: messages.map(message => message.leaf), start, end }; }); } + /** Throws unless every message in `[startLeafCount, endLeafCount)` is within the synced total. Empty ranges included. */ + private async assertLeafCountRangeSynced(startLeafCount: bigint, endLeafCount: bigint): Promise { + const syncedTotal = await this.getTotalL1ToL2MessageCount(); + if (endLeafCount > syncedTotal) { + const available = syncedTotal > startLeafCount ? syncedTotal - startLeafCount : 0n; + throw new InboxMessageRangeNotSyncedError( + startLeafCount, + endLeafCount, + `only ${available} of ${endLeafCount - startLeafCount} messages are synced`, + ); + } + } + + /** + * Reads the messages in the compact index range `[startLeafCount, endLeafCount)`, which the caller has established + * lies within the synced total. The map holds at most one entry per index, so a short read is the only way a hole + * inside the range can show up, and the count catches every one of them. + */ + private async getMessagesInLeafCountRange(startLeafCount: bigint, endLeafCount: bigint): Promise { + if (startLeafCount === endLeafCount) { + return []; + } + const messages = await toArray(this.iterateL1ToL2Messages({ start: startLeafCount, end: endLeafCount })); + if (BigInt(messages.length) !== endLeafCount - startLeafCount) { + throw new InboxMessageRangeNotSyncedError( + startLeafCount, + endLeafCount, + `the store holds ${messages.length} of ${endLeafCount - startLeafCount} messages`, + ); + } + return messages; + } + /** * Returns the latest Inbox bucket opened at or before the given L1 timestamp, or undefined if every synced bucket * was opened strictly after it. diff --git a/yarn-project/archiver/src/test/mock_archiver.ts b/yarn-project/archiver/src/test/mock_archiver.ts index 8285d4e8fcb6..70ae2d246b9c 100644 --- a/yarn-project/archiver/src/test/mock_archiver.ts +++ b/yarn-project/archiver/src/test/mock_archiver.ts @@ -2,7 +2,12 @@ import { Buffer32 } from '@aztec/foundation/buffer'; import { Fr } from '@aztec/foundation/curves/bn254'; import type { L2BlockSource } from '@aztec/stdlib/block'; import type { Checkpoint } from '@aztec/stdlib/checkpoint'; -import type { InboxBucket, L1ToL2MessageSource } from '@aztec/stdlib/messaging'; +import type { + InboxBucket, + InboxMessagePosition, + InboxMessageRange, + L1ToL2MessageSource, +} from '@aztec/stdlib/messaging'; import { MockL1ToL2MessageSource } from './mock_l1_to_l2_message_source.js'; import { MockL2BlockSource } from './mock_l2_block_source.js'; @@ -22,6 +27,10 @@ export class MockArchiver extends MockL2BlockSource implements L2BlockSource, L1 this.messageSource.replaceInboxBuckets(buckets); } + public appendL1ToL2Messages(msgs: Fr[]) { + this.messageSource.appendL1ToL2Messages(msgs); + } + getL1ToL2MessageIndex(_l1ToL2Message: Fr): Promise { return this.messageSource.getL1ToL2MessageIndex(_l1ToL2Message); } @@ -45,6 +54,18 @@ export class MockArchiver extends MockL2BlockSource implements L2BlockSource, L1 getL1ToL2MessagesBetweenLeafCounts(startLeafCount: bigint, endLeafCount: bigint): Promise { return this.messageSource.getL1ToL2MessagesBetweenLeafCounts(startLeafCount, endLeafCount); } + + getMessagePosition(totalMessageCount: bigint): Promise { + return this.messageSource.getMessagePosition(totalMessageCount); + } + + getSyncedMessagePosition(): Promise { + return this.messageSource.getSyncedMessagePosition(); + } + + getL1ToL2MessageRange(startLeafCount: bigint, endLeafCount: bigint): Promise { + return this.messageSource.getL1ToL2MessageRange(startLeafCount, endLeafCount); + } } /** diff --git a/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts b/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts index 01913be555d0..32c053b8d50b 100644 --- a/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts +++ b/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts @@ -1,7 +1,13 @@ import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; import { Fr } from '@aztec/foundation/curves/bn254'; import type { CheckpointId, L2BlockId, L2TipId, L2Tips } from '@aztec/stdlib/block'; -import type { InboxBucket, L1ToL2MessageSource } from '@aztec/stdlib/messaging'; +import { + type InboxBucket, + type InboxMessagePosition, + type InboxMessageRange, + type L1ToL2MessageSource, + updateInboxRollingHash, +} from '@aztec/stdlib/messaging'; /** * A mocked implementation of L1ToL2MessageSource to be used in tests. @@ -10,8 +16,9 @@ export class MockL1ToL2MessageSource implements L1ToL2MessageSource { private buckets = new Map(); private messagesPerBucket = new Map(); /** - * The canonical message log, keyed by compact global index. Kept apart from the bucket partition so a test can - * repartition the buckets (as an L1 reorg does) while the indexed leaves stay exactly as they were. + * The canonical message log, keyed by compact global index. This is the primary fixture: message positions and + * count ranges derive from it alone, and it is kept apart from the bucket partition so a test can repartition the + * buckets (as an L1 reorg does) while the indexed leaves stay exactly as they were. */ private leavesByIndex = new Map(); @@ -39,10 +46,34 @@ export class MockL1ToL2MessageSource implements L1ToL2MessageSource { } } + /** Appends leaves to the indexed message log at its synced tip, without registering any bucket for them. */ + public appendL1ToL2Messages(msgs: Fr[]) { + const firstIndex = this.getSyncedMessageCount(); + msgs.forEach((msg, i) => this.leavesByIndex.set(firstIndex + BigInt(i), msg)); + } + public setBlockNumber(blockNumber: number) { this.blockNumber = blockNumber; } + /** The number of leaves indexed contiguously from zero: the mocked synced tip. */ + private getSyncedMessageCount(): bigint { + let count = 0n; + while (this.leavesByIndex.has(count)) { + count++; + } + return count; + } + + /** Recomputes the rolling hash over the first `totalMessageCount` indexed leaves, as the archiver stores per message. */ + private computeRollingHash(totalMessageCount: bigint): Fr { + let hash = Fr.ZERO; + for (let index = 0n; index < totalMessageCount; index++) { + hash = updateInboxRollingHash(hash, this.leavesByIndex.get(index)!); + } + return hash; + } + getL1ToL2MessageIndex(_l1ToL2Message: Fr): Promise { throw new Error('Method not implemented.'); } @@ -95,6 +126,30 @@ export class MockL1ToL2MessageSource implements L1ToL2MessageSource { return Promise.resolve(leaves); } + getMessagePosition(totalMessageCount: bigint): Promise { + if (totalMessageCount < 0n) { + return Promise.reject(new Error(`Invalid Inbox message count ${totalMessageCount}`)); + } + if (totalMessageCount > this.getSyncedMessageCount()) { + return Promise.resolve(undefined); + } + return Promise.resolve({ totalMessageCount, rollingHash: this.computeRollingHash(totalMessageCount) }); + } + + getSyncedMessagePosition(): Promise { + const totalMessageCount = this.getSyncedMessageCount(); + return Promise.resolve({ totalMessageCount, rollingHash: this.computeRollingHash(totalMessageCount) }); + } + + async getL1ToL2MessageRange(startLeafCount: bigint, endLeafCount: bigint): Promise { + const messages = await this.getL1ToL2MessagesBetweenLeafCounts(startLeafCount, endLeafCount); + return { + messages, + start: { totalMessageCount: startLeafCount, rollingHash: this.computeRollingHash(startLeafCount) }, + end: { totalMessageCount: endLeafCount, rollingHash: this.computeRollingHash(endLeafCount) }, + }; + } + getBlockNumber() { return Promise.resolve(BlockNumber(this.blockNumber)); } diff --git a/yarn-project/stdlib/src/interfaces/archiver.test.ts b/yarn-project/stdlib/src/interfaces/archiver.test.ts index d505b005bea0..115ebd151457 100644 --- a/yarn-project/stdlib/src/interfaces/archiver.test.ts +++ b/yarn-project/stdlib/src/interfaces/archiver.test.ts @@ -39,6 +39,7 @@ import type { PrivateLogsQuery, PublicLogsQuery } from '../logs/logs_query.js'; import { SiloedTag } from '../logs/siloed_tag.js'; import { Tag } from '../logs/tag.js'; import type { InboxBucket } from '../messaging/inbox_bucket.js'; +import type { InboxMessagePosition, InboxMessageRange } from '../messaging/l1_to_l2_message_source.js'; import { CheckpointHeader } from '../rollup/checkpoint_header.js'; import { getTokenContractArtifact } from '../tests/fixtures.js'; import { AppendOnlyTreeSnapshot } from '../trees/append_only_tree_snapshot.js'; @@ -242,6 +243,25 @@ describe('ArchiverApiSchema', () => { expect(result).toEqual([expect.any(Fr)]); }); + it('getMessagePosition', async () => { + const result = await context.client.getMessagePosition(3n); + expect(result).toEqual({ totalMessageCount: 3n, rollingHash: expect.any(Fr) }); + }); + + it('getSyncedMessagePosition', async () => { + const result = await context.client.getSyncedMessagePosition(); + expect(result).toEqual({ totalMessageCount: 3n, rollingHash: expect.any(Fr) }); + }); + + it('getL1ToL2MessageRange', async () => { + const result = await context.client.getL1ToL2MessageRange(2n, 3n); + expect(result).toEqual({ + messages: [expect.any(Fr)], + start: { totalMessageCount: 2n, rollingHash: expect.any(Fr) }, + end: { totalMessageCount: 3n, rollingHash: expect.any(Fr) }, + }); + }); + it('registerContractFunctionSignatures', async () => { await context.client.registerContractFunctionSignatures(['test()']); }); @@ -650,6 +670,22 @@ class MockArchiver implements ArchiverApi { expect(typeof endLeafCount).toEqual('bigint'); return Promise.resolve([Fr.random()]); } + getMessagePosition(totalMessageCount: bigint): Promise { + expect(typeof totalMessageCount).toEqual('bigint'); + return Promise.resolve({ totalMessageCount, rollingHash: Fr.random() }); + } + getSyncedMessagePosition(): Promise { + return Promise.resolve({ totalMessageCount: 3n, rollingHash: Fr.random() }); + } + getL1ToL2MessageRange(startLeafCount: bigint, endLeafCount: bigint): Promise { + expect(typeof startLeafCount).toEqual('bigint'); + expect(typeof endLeafCount).toEqual('bigint'); + return Promise.resolve({ + messages: [Fr.random()], + start: { totalMessageCount: startLeafCount, rollingHash: Fr.random() }, + end: { totalMessageCount: endLeafCount, rollingHash: Fr.random() }, + }); + } getL1Constants(): Promise { return Promise.resolve(EmptyL1RollupConstants); } diff --git a/yarn-project/stdlib/src/interfaces/archiver.ts b/yarn-project/stdlib/src/interfaces/archiver.ts index 605f34e03233..1614b2d7ae56 100644 --- a/yarn-project/stdlib/src/interfaces/archiver.ts +++ b/yarn-project/stdlib/src/interfaces/archiver.ts @@ -26,7 +26,11 @@ import { L1RollupConstantsSchema } from '../epoch-helpers/index.js'; import { LogResultSchema } from '../logs/log_result.js'; import { PrivateLogsQuerySchema, PublicLogsQuerySchema } from '../logs/logs_query.js'; import { InboxBucketSchema } from '../messaging/inbox_bucket.js'; -import type { L1ToL2MessageSource } from '../messaging/l1_to_l2_message_source.js'; +import { + InboxMessagePositionSchema, + InboxMessageRangeSchema, + type L1ToL2MessageSource, +} from '../messaging/l1_to_l2_message_source.js'; import { L2ToL1MembershipWitnessSchema } from '../messaging/l2_to_l1_membership.js'; import { optional, schemas } from '../schemas/schemas.js'; import { indexedTxSchema } from '../tx/indexed_tx_effect.js'; @@ -154,6 +158,12 @@ export const ArchiverApiSchema: ApiSchemaFor = { input: z.tuple([schemas.BigInt, schemas.BigInt]), output: z.array(schemas.Fr), }), + getMessagePosition: z.function({ input: z.tuple([schemas.BigInt]), output: InboxMessagePositionSchema.optional() }), + getSyncedMessagePosition: z.function({ input: z.tuple([]), output: InboxMessagePositionSchema }), + getL1ToL2MessageRange: z.function({ + input: z.tuple([schemas.BigInt, schemas.BigInt]), + output: InboxMessageRangeSchema, + }), getDebugFunctionName: z.function({ input: z.tuple([schemas.AztecAddress, schemas.FunctionSelector]), output: optional(z.string()), diff --git a/yarn-project/stdlib/src/messaging/l1_to_l2_message_source.ts b/yarn-project/stdlib/src/messaging/l1_to_l2_message_source.ts index ff6426206fc0..20c1ecdd23f8 100644 --- a/yarn-project/stdlib/src/messaging/l1_to_l2_message_source.ts +++ b/yarn-project/stdlib/src/messaging/l1_to_l2_message_source.ts @@ -1,8 +1,48 @@ -import type { Fr } from '@aztec/foundation/curves/bn254'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { schemas } from '@aztec/foundation/schemas'; + +import { z } from 'zod'; import type { L2Tips } from '../block/l2_block_source.js'; import type { InboxBucket } from './inbox_bucket.js'; +/** + * A position in the ordered Inbox message sequence: the number of messages up to it, which is also the compact index + * of the next message, and the consensus rolling hash over exactly those messages. Position zero has a zero hash. + * Block headers commit to the count (the L1-to-L2 tree leaf count) and checkpoint headers to the hash, so a position + * identifies a message prefix independently of how L1 partitioned the messages into buckets. + */ +export type InboxMessagePosition = { + /** Number of messages in the sequence up to this position. */ + totalMessageCount: bigint; + /** Consensus rolling hash (truncated sha256 chain) over the messages up to this position; zero at position zero. */ + rollingHash: Fr; +}; + +export const InboxMessagePositionSchema = z.object({ + totalMessageCount: schemas.BigInt, + rollingHash: Fr.schema, +}) satisfies z.ZodType; + +/** + * The messages in a compact count range together with the positions the range starts and ends at, all read from one + * snapshot of the source, so `end.rollingHash` authenticates exactly `messages` appended after `start`. + */ +export type InboxMessageRange = { + /** The message leaves in the range, in insertion order. */ + messages: Fr[]; + /** The position the range starts at, inclusive. */ + start: InboxMessagePosition; + /** The position the range ends at, exclusive; equal to `start` for an empty range. */ + end: InboxMessagePosition; +}; + +export const InboxMessageRangeSchema = z.object({ + messages: z.array(schemas.Fr), + start: InboxMessagePositionSchema, + end: InboxMessagePositionSchema, +}) satisfies z.ZodType; + /** * Interface of classes allowing for the retrieval of L1 to L2 messages. */ @@ -65,6 +105,29 @@ export interface L1ToL2MessageSource { */ getL1ToL2MessagesBetweenLeafCounts(startLeafCount: bigint, endLeafCount: bigint): Promise; + /** + * Returns the position of the Inbox message sequence after `totalMessageCount` messages: that count and the rolling + * hash over them. Position zero always resolves, with a zero hash; a count past the synced tip returns undefined, + * and a negative one throws. + * @param totalMessageCount - The cumulative Inbox message count (leaf count) whose position to resolve. + */ + getMessagePosition(totalMessageCount: bigint): Promise; + + /** Returns the position at the source's synced tip: how many messages it holds and the rolling hash over them. */ + getSyncedMessagePosition(): Promise; + + /** + * Returns the messages in the cumulative Inbox message-count range `[startLeafCount, endLeafCount)` together with + * the positions at both bounds, all read from one snapshot of the source, so the ending hash authenticates exactly + * the returned messages and cannot describe a different version of the sequence than they do. An empty range is + * valid and returns equal positions. The range contract is that of `getL1ToL2MessagesBetweenLeafCounts`: an + * invalid range, or one the source cannot serve whole (including its starting position), throws rather than + * returning a partial or empty result. + * @param startLeafCount - The cumulative Inbox message count the range starts at, inclusive. + * @param endLeafCount - The cumulative Inbox message count the range ends at, exclusive. + */ + getL1ToL2MessageRange(startLeafCount: bigint, endLeafCount: bigint): Promise; + /** * Returns the tips of the L2 chain. */ From 2c3d875a4afc3cc5105263db2b7bc3599ebc711a Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 4 Sep 2026 21:37:48 -0300 Subject: [PATCH 5/6] fix(fast-inbox): fresh zero position and a snapshot-consistent typed mock range read getMessagePosition(0) handed out a shared module-level object through a mutable type, so a caller's mutation leaked into later reads. The mock message source yielded between capturing a range's leaves and computing its hashes and never checked the synced tip, so it could pair one version of the log with another's hash and resolve ranges the archiver rejects. Also corrects the validator JSDoc about what the censorship guard establishes, retires the obsolete bucket comment in MockPrefilledArchiver, and covers zero-position mutation, a concurrent removal during a range read, and replay resumption after a failed range. --- .../archiver/src/store/message_store.test.ts | 25 +++++++ .../archiver/src/store/message_store.ts | 11 ++- .../archiver/src/test/mock_archiver.ts | 11 ++- .../test/mock_l1_to_l2_message_source.test.ts | 62 +++++++++++++++++ .../src/test/mock_l1_to_l2_message_source.ts | 67 ++++++++++++------- .../validator-client/src/proposal_handler.ts | 6 +- .../server_world_state_synchronizer.test.ts | 12 +++- 7 files changed, 158 insertions(+), 36 deletions(-) create mode 100644 yarn-project/archiver/src/test/mock_l1_to_l2_message_source.test.ts diff --git a/yarn-project/archiver/src/store/message_store.test.ts b/yarn-project/archiver/src/store/message_store.test.ts index c282d5739c56..677e3ad688a2 100644 --- a/yarn-project/archiver/src/store/message_store.test.ts +++ b/yarn-project/archiver/src/store/message_store.test.ts @@ -265,6 +265,15 @@ describe('MessageStore', () => { expect(await messageStore.getSyncedMessagePosition()).toEqual(zeroPosition); }); + it('hands out positions a caller can mutate without affecting later reads', async () => { + const position = (await messageStore.getMessagePosition(0n))!; + position.rollingHash = new Fr(99); + position.totalMessageCount = 99n; + + expect(await messageStore.getMessagePosition(0n)).toEqual(zeroPosition); + expect((await messageStore.getL1ToL2MessageRange(0n, 0n)).start).toEqual(zeroPosition); + }); + it('tracks the synced position through appends and removals', async () => { const msgs = makeInboxMessages(6); await messageStore.addL1ToL2MessageBuckets(msgs.slice(0, 4)); @@ -307,6 +316,22 @@ describe('MessageStore', () => { }); }); + it('reads the messages and the ending position from one snapshot under a concurrent removal', async () => { + const msgs = makeInboxMessages(6); + await messageStore.addL1ToL2MessageBuckets(msgs); + + // Both operations are queued without awaiting: the read runs as one store transaction, so it sees either the + // full sequence or the truncated one, never the leaves of one with the ending hash of the other. + const rangePromise = messageStore.getL1ToL2MessageRange(0n, 6n); + const removalPromise = messageStore.removeL1ToL2Messages(3n); + const [range] = await Promise.all([rangePromise, removalPromise]); + + expect(range.messages).toEqual(msgs.map(m => m.leaf)); + expect(range.end).toEqual(positionAfter(msgs[5])); + expect(await messageStore.getSyncedMessagePosition()).toEqual(positionAfter(msgs[2])); + await expect(messageStore.getL1ToL2MessageRange(0n, 6n)).rejects.toThrow(InboxMessageRangeNotSyncedError); + }); + it('reads the empty range at position zero on an empty store', async () => { expect(await messageStore.getL1ToL2MessageRange(0n, 0n)).toEqual({ messages: [], diff --git a/yarn-project/archiver/src/store/message_store.ts b/yarn-project/archiver/src/store/message_store.ts index 49ef7cba6438..9503ccc6616e 100644 --- a/yarn-project/archiver/src/store/message_store.ts +++ b/yarn-project/archiver/src/store/message_store.ts @@ -115,8 +115,13 @@ const GENESIS_INBOX_BUCKET: InboxBucket = { l1BlockHash: Buffer32.ZERO, }; -/** The position before any message: zero count and zero rolling hash, mirroring the on-chain Inbox base case. */ -const ZERO_MESSAGE_POSITION: InboxMessagePosition = { totalMessageCount: 0n, rollingHash: Fr.ZERO }; +/** + * The position before any message: zero count and zero rolling hash, mirroring the on-chain Inbox base case. Built + * fresh on every call because positions are plain mutable objects handed out to callers. + */ +function zeroMessagePosition(): InboxMessagePosition { + return { totalMessageCount: 0n, rollingHash: Fr.ZERO }; +} /** Rejects reversed or negative compact leaf count bounds, which are caller errors rather than sync state. */ function assertValidLeafCountRange(startLeafCount: bigint, endLeafCount: bigint): void { @@ -518,7 +523,7 @@ export class MessageStore { throw new Error(`Invalid Inbox message count ${totalMessageCount}`); } if (totalMessageCount === 0n) { - return ZERO_MESSAGE_POSITION; + return zeroMessagePosition(); } const buffer = await this.#l1ToL2Messages.getAsync(this.indexToKey(totalMessageCount - 1n)); return buffer === undefined diff --git a/yarn-project/archiver/src/test/mock_archiver.ts b/yarn-project/archiver/src/test/mock_archiver.ts index 70ae2d246b9c..de301e878be5 100644 --- a/yarn-project/archiver/src/test/mock_archiver.ts +++ b/yarn-project/archiver/src/test/mock_archiver.ts @@ -92,12 +92,11 @@ export class MockPrefilledArchiver extends MockArchiver { this.prefilledMessages[checkpoint.number - 1] = messages; } - // Register the Inbox buckets the streaming world-state synchronizer reconstructs each block's consumed - // message bundle from: a genesis sentinel (totalMsgCount 0) so a leaf count of 0 - // resolves to a bucket, plus one bucket per message-carrying checkpoint whose cumulative totalMsgCount - // matches the block's post-insertion L1-to-L2 leaf count. Rebuilt from the full prefilled chain (not just - // this call's checkpoints) so a reorg re-prefill that replaces a suffix keeps the cumulative aligned. - // Without these the synchronizer derives an empty bundle and the reconstructed block state diverges. + // Index every message-carrying checkpoint's leaves at the compact positions the archiver would give them, which is + // what published-block replay reads by count. The leaves are registered through buckets (a genesis sentinel plus + // one bucket per message-carrying checkpoint) so tests can still model the live partition and repartition it as an + // L1 reorg would. Rebuilt from the full prefilled chain (not just this call's checkpoints) so a reorg re-prefill + // that replaces a suffix keeps the cumulative counts aligned. this.setInboxBucket( { seq: 0n, diff --git a/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.test.ts b/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.test.ts new file mode 100644 index 000000000000..8c9a74fb8d73 --- /dev/null +++ b/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.test.ts @@ -0,0 +1,62 @@ +import { Buffer32 } from '@aztec/foundation/buffer'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { type InboxBucket, updateInboxRollingHash } from '@aztec/stdlib/messaging'; + +import { InboxMessageRangeNotSyncedError } from '../errors.js'; +import { MockL1ToL2MessageSource } from './mock_l1_to_l2_message_source.js'; + +/** A single-message bucket at the start of the Inbox; only its total matters to the leaf index. */ +const singleMessageBucket: InboxBucket = { + seq: 1n, + inboxRollingHash: Fr.ZERO, + totalMsgCount: 1n, + timestamp: 1n, + msgCount: 1, + lastMessageIndex: 0n, + l1BlockNumber: 1n, + l1BlockHash: Buffer32.ZERO, +}; + +describe('MockL1ToL2MessageSource', () => { + let source: MockL1ToL2MessageSource; + + beforeEach(() => { + source = new MockL1ToL2MessageSource(0); + }); + + it('derives positions from the indexed leaves', async () => { + const leaves = [new Fr(11), new Fr(12), new Fr(13)]; + source.appendL1ToL2Messages(leaves); + const hashAfterTwo = updateInboxRollingHash(updateInboxRollingHash(Fr.ZERO, leaves[0]), leaves[1]); + + expect(await source.getMessagePosition(0n)).toEqual({ totalMessageCount: 0n, rollingHash: Fr.ZERO }); + expect(await source.getMessagePosition(2n)).toEqual({ totalMessageCount: 2n, rollingHash: hashAfterTwo }); + expect(await source.getMessagePosition(4n)).toBeUndefined(); + expect((await source.getSyncedMessagePosition()).totalMessageCount).toEqual(3n); + expect(await source.getL1ToL2MessageRange(1n, 2n)).toEqual({ + messages: [leaves[1]], + start: { totalMessageCount: 1n, rollingHash: updateInboxRollingHash(Fr.ZERO, leaves[0]) }, + end: { totalMessageCount: 2n, rollingHash: hashAfterTwo }, + }); + }); + + it('rejects ranges past the mocked tip with the archiver error, empty ones included', async () => { + await expect(source.getL1ToL2MessagesBetweenLeafCounts(7n, 7n)).rejects.toThrow(InboxMessageRangeNotSyncedError); + await expect(source.getL1ToL2MessageRange(7n, 7n)).rejects.toThrow(InboxMessageRangeNotSyncedError); + await expect(source.getL1ToL2MessageRange(0n, 1n)).rejects.toThrow(InboxMessageRangeNotSyncedError); + await expect(source.getL1ToL2MessageRange(2n, 1n)).rejects.toThrow(/Invalid Inbox leaf count range/); + expect(await source.getL1ToL2MessagesBetweenLeafCounts(0n, 0n)).toEqual([]); + }); + + it('reads the leaves and the ending hash of a range from the same version of the log', async () => { + source.setInboxBucket(singleMessageBucket, [new Fr(11)]); + + // Replace the leaf while the read is pending: the result must describe one version, not a mix of both. + const pending = source.getL1ToL2MessageRange(0n, 1n); + source.setInboxBucket(singleMessageBucket, [new Fr(22)]); + + const range = await pending; + expect(range.messages).toEqual([new Fr(11)]); + expect(range.end.rollingHash).toEqual(updateInboxRollingHash(Fr.ZERO, new Fr(11))); + }); +}); diff --git a/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts b/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts index 32c053b8d50b..40caceaf39b8 100644 --- a/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts +++ b/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts @@ -9,6 +9,8 @@ import { updateInboxRollingHash, } from '@aztec/stdlib/messaging'; +import { InboxMessageRangeNotSyncedError } from '../errors.js'; + /** * A mocked implementation of L1ToL2MessageSource to be used in tests. */ @@ -104,26 +106,16 @@ export class MockL1ToL2MessageSource implements L1ToL2MessageSource { } /** - * Slices the indexed leaf log, enforcing the same range contract as the archiver's message store. Every failure is + * Slices the indexed leaf log, enforcing the same range contract as the archiver's message store: invalid bounds and + * ranges past the synced tip are rejected, the latter with the archiver's typed availability error. Every failure is * a rejection rather than a synchronous throw, so callers see this stand-in behave like the async source it mocks. */ getL1ToL2MessagesBetweenLeafCounts(startLeafCount: bigint, endLeafCount: bigint): Promise { - if (startLeafCount < 0n || endLeafCount < 0n || startLeafCount > endLeafCount) { - return Promise.reject(new Error(`Invalid Inbox leaf count range [${startLeafCount}, ${endLeafCount})`)); - } - const leaves: Fr[] = []; - for (let index = startLeafCount; index < endLeafCount; index++) { - const leaf = this.leavesByIndex.get(index); - if (leaf === undefined) { - return Promise.reject( - new Error( - `Inbox message range [${startLeafCount}, ${endLeafCount}) is not fully mocked (missing index ${index})`, - ), - ); - } - leaves.push(leaf); + try { + return Promise.resolve(this.readLeafCountRange(startLeafCount, endLeafCount)); + } catch (err) { + return Promise.reject(err); } - return Promise.resolve(leaves); } getMessagePosition(totalMessageCount: bigint): Promise { @@ -141,13 +133,42 @@ export class MockL1ToL2MessageSource implements L1ToL2MessageSource { return Promise.resolve({ totalMessageCount, rollingHash: this.computeRollingHash(totalMessageCount) }); } - async getL1ToL2MessageRange(startLeafCount: bigint, endLeafCount: bigint): Promise { - const messages = await this.getL1ToL2MessagesBetweenLeafCounts(startLeafCount, endLeafCount); - return { - messages, - start: { totalMessageCount: startLeafCount, rollingHash: this.computeRollingHash(startLeafCount) }, - end: { totalMessageCount: endLeafCount, rollingHash: this.computeRollingHash(endLeafCount) }, - }; + /** + * Reads the range and both positions synchronously from the current leaf log, so a test that mutates the log while + * a range read is pending cannot pair leaves of one version with hashes of another, as the archiver's single-transaction + * read cannot either. + */ + getL1ToL2MessageRange(startLeafCount: bigint, endLeafCount: bigint): Promise { + try { + const messages = this.readLeafCountRange(startLeafCount, endLeafCount); + return Promise.resolve({ + messages, + start: { totalMessageCount: startLeafCount, rollingHash: this.computeRollingHash(startLeafCount) }, + end: { totalMessageCount: endLeafCount, rollingHash: this.computeRollingHash(endLeafCount) }, + }); + } catch (err) { + return Promise.reject(err); + } + } + + private readLeafCountRange(startLeafCount: bigint, endLeafCount: bigint): Fr[] { + if (startLeafCount < 0n || endLeafCount < 0n || startLeafCount > endLeafCount) { + throw new Error(`Invalid Inbox leaf count range [${startLeafCount}, ${endLeafCount})`); + } + const syncedCount = this.getSyncedMessageCount(); + if (endLeafCount > syncedCount) { + const available = syncedCount > startLeafCount ? syncedCount - startLeafCount : 0n; + throw new InboxMessageRangeNotSyncedError( + startLeafCount, + endLeafCount, + `only ${available} of ${endLeafCount - startLeafCount} messages are mocked`, + ); + } + const leaves: Fr[] = []; + for (let index = startLeafCount; index < endLeafCount; index++) { + leaves.push(this.leavesByIndex.get(index)!); + } + return leaves; } getBlockNumber() { diff --git a/yarn-project/validator-client/src/proposal_handler.ts b/yarn-project/validator-client/src/proposal_handler.ts index d0fbdec23944..ded3b5c6e32f 100644 --- a/yarn-project/validator-client/src/proposal_handler.ts +++ b/yarn-project/validator-client/src/proposal_handler.ts @@ -1165,8 +1165,10 @@ export class ProposalHandler { * * Neither bound is resolved as an Inbox bucket. Both are counts committed by block headers, and an L1 reorg that * merges buckets can leave either one interior to the current partition without changing the messages the blocks - * consumed; whether the final position is a live bucket end is a publication rule L1 `propose` enforces, checked - * separately by the censorship guard. Throws when part of the range is not available locally: a local availability + * consumed; whether the final position is a live bucket end is a publication rule L1 `propose` enforces. The + * minimum-consumption guard that runs before this only checks censorship against the buckets it can resolve + * locally and defers an unresolved endpoint to L1. Throws when part of the range is not available locally: a local + * availability * gap must surface as such rather than as an empty bundle that would make a valid proposal fail its rolling-hash * recomputation. */ diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts index 7e30c34ed709..d381c992c026 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts @@ -330,12 +330,20 @@ describe('ServerWorldStateSynchronizer', () => { await expect(pushBlocks(1, 3)).rejects.toThrow(/Unexpected leaf count range \[3, 7\)/); - // The unavailable range fails the sync instead of silently applying the block with no messages, and the blocks - // already applied are kept so the retry resumes from there. + // The unavailable range fails the sync instead of silently applying the block with no messages; the blocks before + // it were fetched and applied one at a time, so the failure does not discard them. expect(merkleTreeDb.handleL2BlockAndMessages.mock.calls).toEqual([ [allBlocks()[0], messagesByRange.get('0-3')], [allBlocks()[1], []], ]); + + // Once the archiver serves the range, replay resumes from the failed block, reading its parent's leaf count. + messagesByRange.set('3-7', [new Fr(13n), new Fr(14n), new Fr(15n), new Fr(16n)]); + await pushBlocks(3, 3); + expect(merkleTreeDb.handleL2BlockAndMessages.mock.calls.at(-1)).toEqual([ + allBlocks()[2], + messagesByRange.get('3-7'), + ]); }); it('fails instead of replaying a whole message history when the parent block is missing', async () => { From baf4eb918e2fd9fb48f8dc47682b05de09f93b5b Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Sat, 5 Sep 2026 03:26:35 -0300 Subject: [PATCH 6/6] test(fast-inbox): read the publisher integration test's message bundles by leaf count The world-state synchronizer now reads each synced block's consumed messages by leaf count, but the publisher integration test's block-source stub still only forwarded the removed bucket lookups, so the first block was never applied to world state and every later block failed to fork from it. --- .../publisher/l1_publisher.integration.test.ts | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/yarn-project/sequencer-client/src/publisher/l1_publisher.integration.test.ts b/yarn-project/sequencer-client/src/publisher/l1_publisher.integration.test.ts index a65d86047d59..ae94abf9ba6c 100644 --- a/yarn-project/sequencer-client/src/publisher/l1_publisher.integration.test.ts +++ b/yarn-project/sequencer-client/src/publisher/l1_publisher.integration.test.ts @@ -149,9 +149,9 @@ describe('L1Publisher integration', () => { let builderDb: NativeWorldStateService; - // Backs the blockSource mock's streaming L1->L2 message queries. The world-state synchronizer reconstructs each - // block's consumed message bundle from Inbox buckets when it syncs a block back, so the test - // registers one bucket per published block here (see buildAndPublishBlock). + // Backs the blockSource mock's streaming L1->L2 message queries. The world-state synchronizer reads each block's + // consumed message bundle by leaf count when it syncs a block back, so the test mirrors every Inbox bucket and its + // leaves here before publishing the block that consumes them (see buildAndPublishBlock). let messageSource: MockL1ToL2MessageSource; // The header of the last block @@ -381,13 +381,10 @@ describe('L1Publisher integration', () => { getBlockNumber(): Promise { return Promise.resolve(BlockNumber(blocks.at(-1)?.number ?? BlockNumber.ZERO)); }, - // Streaming L1->L2 message reconstruction: the world-state synchronizer resolves each - // block's consumed message bundle from the Inbox buckets registered per published block in buildAndPublishBlock. - getInboxBucketByTotalMsgCount(totalMsgCount: bigint) { - return messageSource.getInboxBucketByTotalMsgCount(totalMsgCount); - }, - getL1ToL2MessagesBetweenBuckets(fromExclusive: bigint, toInclusive: bigint) { - return messageSource.getL1ToL2MessagesBetweenBuckets(fromExclusive, toInclusive); + // Streaming L1->L2 message reconstruction: the world-state synchronizer reads each block's consumed message + // bundle by leaf count from the leaves mirrored per published block in buildAndPublishBlock. + getL1ToL2MessagesBetweenLeafCounts(startLeafCount: bigint, endLeafCount: bigint) { + return messageSource.getL1ToL2MessagesBetweenLeafCounts(startLeafCount, endLeafCount); }, });