diff --git a/yarn-project/archiver/src/errors.ts b/yarn-project/archiver/src/errors.ts index 60661d9ea296..d5e989b705da 100644 --- a/yarn-project/archiver/src/errors.ts +++ b/yarn-project/archiver/src/errors.ts @@ -114,13 +114,18 @@ export class InboxBucketNotSyncedError extends Error { } /** - * Thrown when a cumulative Inbox message count does not resolve to a bucket boundary this archiver has synced, either - * because the count sits inside a bucket or because the bucket is not synced yet. + * Thrown when a cumulative Inbox message-count range is not fully backed by the messages this archiver has synced, + * either because it reaches past the synced tip or because the store is missing a message the range needs. + * Distinguishes "not available locally, retry once L1 sync catches up" from a genuinely empty range. */ -export class InboxBucketBoundaryNotSyncedError extends Error { - constructor(public readonly totalMsgCount: bigint) { - super(`No synced Inbox bucket ends at cumulative message count ${totalMsgCount}`); - this.name = 'InboxBucketBoundaryNotSyncedError'; +export class InboxMessageRangeNotSyncedError extends Error { + constructor( + public readonly startLeafCount: bigint, + public readonly endLeafCount: bigint, + detail: string, + ) { + super(`Inbox message range [${startLeafCount}, ${endLeafCount}) is not fully synced: ${detail}`); + this.name = 'InboxMessageRangeNotSyncedError'; } } diff --git a/yarn-project/archiver/src/modules/data_source_base.ts b/yarn-project/archiver/src/modules/data_source_base.ts index 9e407d9e6765..c37ff585e341 100644 --- a/yarn-project/archiver/src/modules/data_source_base.ts +++ b/yarn-project/archiver/src/modules/data_source_base.ts @@ -40,7 +40,13 @@ import { } from '@aztec/stdlib/epoch-helpers'; import type { L2LogsSource } from '@aztec/stdlib/interfaces/server'; import type { LogResult, PrivateLogsQuery, PublicLogsQuery } from '@aztec/stdlib/logs'; -import type { InboxBucket, L1ToL2MessageSource, L2ToL1MembershipWitness } from '@aztec/stdlib/messaging'; +import type { + InboxBucket, + InboxMessagePosition, + InboxMessageRange, + L1ToL2MessageSource, + L2ToL1MembershipWitness, +} from '@aztec/stdlib/messaging'; import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees'; import type { BlockHeader, IndexedTxEffect, TxHash } from '@aztec/stdlib/tx'; import type { UInt64 } from '@aztec/stdlib/types'; @@ -340,6 +346,18 @@ export abstract class ArchiverDataSourceBase return this.stores.messages.getL1ToL2MessagesBetweenLeafCounts(startLeafCount, endLeafCount); } + public getMessagePosition(totalMessageCount: bigint): Promise { + return this.stores.messages.getMessagePosition(totalMessageCount); + } + + public getSyncedMessagePosition(): Promise { + return this.stores.messages.getSyncedMessagePosition(); + } + + public getL1ToL2MessageRange(startLeafCount: bigint, endLeafCount: bigint): Promise { + return this.stores.messages.getL1ToL2MessageRange(startLeafCount, endLeafCount); + } + private async getPublishedCheckpointFromCheckpointData(checkpoint: CheckpointData): Promise { const blocksForCheckpoint = await this.stores.blocks.getBlocksForCheckpoint(checkpoint.checkpointNumber); if (!blocksForCheckpoint) { diff --git a/yarn-project/archiver/src/store/message_store.test.ts b/yarn-project/archiver/src/store/message_store.test.ts index b9718fcfe818..677e3ad688a2 100644 --- a/yarn-project/archiver/src/store/message_store.test.ts +++ b/yarn-project/archiver/src/store/message_store.test.ts @@ -2,12 +2,13 @@ import { CheckpointNumber } from '@aztec/foundation/branded-types'; import { Buffer32 } from '@aztec/foundation/buffer'; import { Fr } from '@aztec/foundation/curves/bn254'; import { toArray } from '@aztec/foundation/iterable'; +import type { AztecAsyncKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { Checkpoint, type PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import { updateInboxRollingHash } from '@aztec/stdlib/messaging'; import '@aztec/stdlib/testing/jest'; -import { InboxBucketBoundaryNotSyncedError, InboxBucketNotSyncedError } from '../errors.js'; +import { InboxBucketNotSyncedError, InboxMessageRangeNotSyncedError } from '../errors.js'; import type { InboxMessage } from '../structs/inbox_message.js'; import { makeInboxMessage, @@ -23,6 +24,7 @@ import { type ArchiverL1SynchPoint, getArchiverSynchPoint } from './data_stores. import { MessageStore, MessageStoreError } from './message_store.js'; describe('MessageStore', () => { + let db: AztecAsyncKVStore; let blockStore: BlockStore; let messageStore: MessageStore; let publishedCheckpoints: PublishedCheckpoint[]; @@ -40,7 +42,7 @@ describe('MessageStore', () => { }); beforeEach(async () => { - const db = await openTmpStore('message_store_test'); + db = await openTmpStore('message_store_test'); blockStore = new BlockStore(db); messageStore = new MessageStore(db); // Create checkpoints sequentially to ensure archive roots are chained properly. @@ -224,6 +226,147 @@ describe('MessageStore', () => { }); }); + describe('iterateL1ToL2Messages', () => { + it('honours zero range bounds', async () => { + const msgs = makeInboxMessages(3); + await messageStore.addL1ToL2MessageBuckets(msgs); + + // Zero is a valid compact index, so an exclusive end of zero selects nothing rather than everything. + expect(await toArray(messageStore.iterateL1ToL2Messages({ end: 0n }))).toEqual([]); + expect(await toArray(messageStore.iterateL1ToL2Messages({ start: 0n, end: 2n }))).toEqual(msgs.slice(0, 2)); + expect(await toArray(messageStore.iterateL1ToL2Messages({ start: 0n }))).toEqual(msgs); + }); + }); + + describe('message positions', () => { + const zeroPosition = { totalMessageCount: 0n, rollingHash: Fr.ZERO }; + const positionAfter = (msg: InboxMessage) => ({ + totalMessageCount: msg.index + 1n, + rollingHash: msg.inboxRollingHash, + }); + + it('resolves the position at a synced message count', async () => { + const msgs = makeInboxMessages(6); + await messageStore.addL1ToL2MessageBuckets(msgs); + + // Position zero always resolves; every other count resolves to the hash stored with the message before it. + expect(await messageStore.getMessagePosition(0n)).toEqual(zeroPosition); + expect(await messageStore.getMessagePosition(1n)).toEqual(positionAfter(msgs[0])); + expect(await messageStore.getMessagePosition(4n)).toEqual(positionAfter(msgs[3])); + expect(await messageStore.getMessagePosition(6n)).toEqual(positionAfter(msgs[5])); + // Past the synced tip there is no position yet. + expect(await messageStore.getMessagePosition(7n)).toBeUndefined(); + await expect(messageStore.getMessagePosition(-1n)).rejects.toThrow(/Invalid Inbox message count/); + }); + + it('resolves position zero on an empty store', async () => { + expect(await messageStore.getMessagePosition(0n)).toEqual(zeroPosition); + expect(await messageStore.getMessagePosition(1n)).toBeUndefined(); + expect(await messageStore.getSyncedMessagePosition()).toEqual(zeroPosition); + }); + + it('hands out positions a caller can mutate without affecting later reads', async () => { + const position = (await messageStore.getMessagePosition(0n))!; + position.rollingHash = new Fr(99); + position.totalMessageCount = 99n; + + expect(await messageStore.getMessagePosition(0n)).toEqual(zeroPosition); + expect((await messageStore.getL1ToL2MessageRange(0n, 0n)).start).toEqual(zeroPosition); + }); + + it('tracks the synced position through appends and removals', async () => { + const msgs = makeInboxMessages(6); + await messageStore.addL1ToL2MessageBuckets(msgs.slice(0, 4)); + expect(await messageStore.getSyncedMessagePosition()).toEqual(positionAfter(msgs[3])); + + await messageStore.addL1ToL2MessageBuckets(msgs.slice(4)); + expect(await messageStore.getSyncedMessagePosition()).toEqual(positionAfter(msgs[5])); + + await messageStore.removeL1ToL2Messages(2n); + expect(await messageStore.getSyncedMessagePosition()).toEqual(positionAfter(msgs[1])); + // The removed suffix no longer has positions. + expect(await messageStore.getMessagePosition(3n)).toBeUndefined(); + }); + + it('reads a message range together with the positions at both bounds', async () => { + const msgs = makeInboxMessages(6); + await messageStore.addL1ToL2MessageBuckets(msgs); + const leaves = msgs.map(m => m.leaf); + + expect(await messageStore.getL1ToL2MessageRange(0n, 6n)).toEqual({ + messages: leaves, + start: zeroPosition, + end: positionAfter(msgs[5]), + }); + expect(await messageStore.getL1ToL2MessageRange(1n, 4n)).toEqual({ + messages: leaves.slice(1, 4), + start: positionAfter(msgs[0]), + end: positionAfter(msgs[3]), + }); + // An empty range is valid at any synced count and returns equal positions. + expect(await messageStore.getL1ToL2MessageRange(3n, 3n)).toEqual({ + messages: [], + start: positionAfter(msgs[2]), + end: positionAfter(msgs[2]), + }); + expect(await messageStore.getL1ToL2MessageRange(6n, 6n)).toEqual({ + messages: [], + start: positionAfter(msgs[5]), + end: positionAfter(msgs[5]), + }); + }); + + it('reads the messages and the ending position from one snapshot under a concurrent removal', async () => { + const msgs = makeInboxMessages(6); + await messageStore.addL1ToL2MessageBuckets(msgs); + + // Both operations are queued without awaiting: the read runs as one store transaction, so it sees either the + // full sequence or the truncated one, never the leaves of one with the ending hash of the other. + const rangePromise = messageStore.getL1ToL2MessageRange(0n, 6n); + const removalPromise = messageStore.removeL1ToL2Messages(3n); + const [range] = await Promise.all([rangePromise, removalPromise]); + + expect(range.messages).toEqual(msgs.map(m => m.leaf)); + expect(range.end).toEqual(positionAfter(msgs[5])); + expect(await messageStore.getSyncedMessagePosition()).toEqual(positionAfter(msgs[2])); + await expect(messageStore.getL1ToL2MessageRange(0n, 6n)).rejects.toThrow(InboxMessageRangeNotSyncedError); + }); + + it('reads the empty range at position zero on an empty store', async () => { + expect(await messageStore.getL1ToL2MessageRange(0n, 0n)).toEqual({ + messages: [], + start: zeroPosition, + end: zeroPosition, + }); + }); + + it('throws on an invalid or unsynced message range', async () => { + const msgs = makeInboxMessages(6); + await messageStore.addL1ToL2MessageBuckets(msgs); + + await expect(messageStore.getL1ToL2MessageRange(3n, 9n)).rejects.toThrow(InboxMessageRangeNotSyncedError); + await expect(messageStore.getL1ToL2MessageRange(7n, 7n)).rejects.toThrow(InboxMessageRangeNotSyncedError); + await expect(messageStore.getL1ToL2MessageRange(5n, 3n)).rejects.toThrow(/Invalid Inbox leaf count range/); + await expect(messageStore.getL1ToL2MessageRange(-1n, 3n)).rejects.toThrow(/Invalid Inbox leaf count range/); + }); + + it('throws when the range or its starting position has a hole', async () => { + const msgs = makeInboxMessages(6); + await messageStore.addL1ToL2MessageBuckets(msgs); + await db.openMap('archiver_l1_to_l2_messages').delete(2); + + // Index 2 is missing: ranges over it are short, and a range starting at count 3 has no starting position. + await expect(messageStore.getL1ToL2MessageRange(0n, 6n)).rejects.toThrow(InboxMessageRangeNotSyncedError); + await expect(messageStore.getL1ToL2MessageRange(3n, 6n)).rejects.toThrow(/missing the message at index 2/); + // Ranges that need neither the hole nor a position at it are unaffected. + expect(await messageStore.getL1ToL2MessageRange(4n, 6n)).toEqual({ + messages: msgs.slice(4).map(m => m.leaf), + start: positionAfter(msgs[3]), + end: positionAfter(msgs[5]), + }); + }); + }); + describe('Inbox buckets', () => { // Builds `count` consecutive valid messages, then reassigns their bucket sequence and timestamp per the given // per-message spec so we can exercise multi-message and rollover buckets. @@ -484,23 +627,61 @@ describe('MessageStore', () => { expect(await messageStore.getL1ToL2MessagesBetweenLeafCounts(0n, 0n)).toEqual([]); }); - it('throws when a leaf count does not land on a synced bucket boundary', async () => { + it('returns messages between leaf counts interior to the bucket partition', async () => { const msgs = makeBucketedMessages(threeBucketSpec); await messageStore.addL1ToL2MessageBuckets(msgs); + const leaves = msgs.map(m => m.leaf); - // Counts inside a bucket and past the last synced bucket both fail rather than returning a partial range. - await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(0n, 4n)).rejects.toThrow( - InboxBucketBoundaryNotSyncedError, - ); - await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(4n, 6n)).rejects.toThrow( - InboxBucketBoundaryNotSyncedError, - ); + // Counts 1, 2 and 4 sit inside a bucket. A published block commits to a leaf count, and a reorg can merge away + // the bucket that ended there, so the range is addressed by message index alone. + expect(await messageStore.getL1ToL2MessagesBetweenLeafCounts(1n, 6n)).toEqual(leaves.slice(1)); + expect(await messageStore.getL1ToL2MessagesBetweenLeafCounts(0n, 4n)).toEqual(leaves.slice(0, 4)); + expect(await messageStore.getL1ToL2MessagesBetweenLeafCounts(1n, 4n)).toEqual(leaves.slice(1, 4)); + expect(await messageStore.getL1ToL2MessagesBetweenLeafCounts(4n, 6n)).toEqual(leaves.slice(4)); + // An empty range at an interior count consumes nothing rather than failing. + expect(await messageStore.getL1ToL2MessagesBetweenLeafCounts(4n, 4n)).toEqual([]); + }); + + it('throws on an invalid or unsynced leaf count range', async () => { + const msgs = makeBucketedMessages(threeBucketSpec); + await messageStore.addL1ToL2MessageBuckets(msgs); + + // Ranges reaching past the synced tip fail rather than returning a partial range, empty ones included. await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(3n, 9n)).rejects.toThrow( - InboxBucketBoundaryNotSyncedError, + InboxMessageRangeNotSyncedError, + ); + await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(7n, 7n)).rejects.toThrow( + InboxMessageRangeNotSyncedError, ); + // Reversed and negative bounds are caller errors, reported like every other failure of this API: as a + // rejection, so remote callers behind the archiver RPC see them the same way. await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(5n, 3n)).rejects.toThrow( /Invalid Inbox leaf count range/, ); + await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(-1n, 3n)).rejects.toThrow( + /Invalid Inbox leaf count range/, + ); + await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(0n, -1n)).rejects.toThrow( + /Invalid Inbox leaf count range/, + ); + }); + + it('throws when the leaf count range has a hole', async () => { + const msgs = makeBucketedMessages(threeBucketSpec); + await messageStore.addL1ToL2MessageBuckets(msgs); + + // Defense in depth: insertion is contiguity-checked and removal only ever drops a suffix, so no caller can put + // a hole in the middle of the log. Punch one straight into the map to prove a short read is never returned. + await db.openMap('archiver_l1_to_l2_messages').delete(2); + + await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(0n, 6n)).rejects.toThrow( + InboxMessageRangeNotSyncedError, + ); + await expect(messageStore.getL1ToL2MessagesBetweenLeafCounts(2n, 3n)).rejects.toThrow( + InboxMessageRangeNotSyncedError, + ); + // Ranges that do not cover the hole are unaffected. + expect(await messageStore.getL1ToL2MessagesBetweenLeafCounts(3n, 6n)).toEqual(msgs.slice(3).map(m => m.leaf)); }); it('rewinds buckets when messages are removed', async () => { diff --git a/yarn-project/archiver/src/store/message_store.ts b/yarn-project/archiver/src/store/message_store.ts index 19b539332a4e..9503ccc6616e 100644 --- a/yarn-project/archiver/src/store/message_store.ts +++ b/yarn-project/archiver/src/store/message_store.ts @@ -12,9 +12,14 @@ import { type CustomRange, mapRange, } from '@aztec/kv-store'; -import { type InboxBucket, updateInboxRollingHash } from '@aztec/stdlib/messaging'; +import { + type InboxBucket, + type InboxMessagePosition, + type InboxMessageRange, + updateInboxRollingHash, +} from '@aztec/stdlib/messaging'; -import { InboxBucketBoundaryNotSyncedError, InboxBucketNotSyncedError } from '../errors.js'; +import { InboxBucketNotSyncedError, InboxMessageRangeNotSyncedError } from '../errors.js'; import { type InboxMessage, deserializeInboxMessage, serializeInboxMessage } from '../structs/inbox_message.js'; /** @@ -110,6 +115,21 @@ const GENESIS_INBOX_BUCKET: InboxBucket = { l1BlockHash: Buffer32.ZERO, }; +/** + * The position before any message: zero count and zero rolling hash, mirroring the on-chain Inbox base case. Built + * fresh on every call because positions are plain mutable objects handed out to callers. + */ +function zeroMessagePosition(): InboxMessagePosition { + return { totalMessageCount: 0n, rollingHash: Fr.ZERO }; +} + +/** Rejects reversed or negative compact leaf count bounds, which are caller errors rather than sync state. */ +function assertValidLeafCountRange(startLeafCount: bigint, endLeafCount: bigint): void { + if (startLeafCount < 0n || endLeafCount < 0n || startLeafCount > endLeafCount) { + throw new Error(`Invalid Inbox leaf count range [${startLeafCount}, ${endLeafCount})`); + } +} + export class MessageStoreError extends Error { constructor( message: string, @@ -474,27 +494,116 @@ export class MessageStore { /** * Returns the message leaves in the cumulative Inbox message-count range `[startLeafCount, endLeafCount)`, in - * insertion order. The bounds are compact L1-to-L2 tree leaf counts, which every block header - * carries, so consumers can ask for the messages a block or checkpoint consumed without resolving buckets - * themselves. Both bounds must land on a bucket boundary this archiver has synced; it throws otherwise, since a - * caller asking for a range always expects the messages in it. + * insertion order. The bounds are compact L1-to-L2 tree leaf counts, which every block header carries, so consumers + * can ask for the messages a block or checkpoint consumed without resolving buckets themselves. + * + * The bounds address canonical compact message indices and need not land on a bucket boundary of the partition this + * archiver currently holds: a published block commits to a leaf count, while an L1 reorg can merge away the bucket + * that once ended there. An invalid range, one reaching past the synced tip, or one the store cannot serve whole + * throws, since a caller asking for a range always expects every message in it. */ public async getL1ToL2MessagesBetweenLeafCounts(startLeafCount: bigint, endLeafCount: bigint): Promise { - if (startLeafCount > endLeafCount) { - throw new Error(`Invalid Inbox leaf count range [${startLeafCount}, ${endLeafCount})`); + assertValidLeafCountRange(startLeafCount, endLeafCount); + // The synced total and the leaves are read together so a concurrent suffix removal cannot land between them and + // turn a range this store holds whole into a spurious incomplete one. + return await this.db.transactionAsync(async () => { + await this.assertLeafCountRangeSynced(startLeafCount, endLeafCount); + const messages = await this.getMessagesInLeafCountRange(startLeafCount, endLeafCount); + return messages.map(message => message.leaf); + }); + } + + /** + * Returns the position of the Inbox message sequence after `totalMessageCount` messages: that count and the rolling + * hash over them, which is the rolling hash stored with the message at compact index `totalMessageCount - 1`. + * Position zero always resolves with a zero hash; a count past the synced tip returns undefined. + */ + public async getMessagePosition(totalMessageCount: bigint): Promise { + if (totalMessageCount < 0n) { + throw new Error(`Invalid Inbox message count ${totalMessageCount}`); + } + if (totalMessageCount === 0n) { + return zeroMessagePosition(); + } + const buffer = await this.#l1ToL2Messages.getAsync(this.indexToKey(totalMessageCount - 1n)); + return buffer === undefined + ? undefined + : { totalMessageCount, rollingHash: deserializeInboxMessage(buffer).inboxRollingHash }; + } + + /** Returns the position at the synced tip: the total message count and the rolling hash over every stored message. */ + public getSyncedMessagePosition(): Promise { + return this.db.transactionAsync(async () => { + const syncedTotal = await this.getTotalL1ToL2MessageCount(); + const position = await this.getMessagePosition(syncedTotal); + if (position === undefined) { + throw new Error(`Inbox message store holds ${syncedTotal} messages but is missing index ${syncedTotal - 1n}`); + } + return position; + }); + } + + /** + * Returns the messages in the cumulative Inbox message-count range `[startLeafCount, endLeafCount)` together with + * the positions at both bounds. Everything is read in one store transaction, so the ending hash authenticates + * exactly the returned messages appended after the starting position, and a concurrent suffix replacement cannot + * pair the leaves of one version of the sequence with the hash of another. Follows the range contract of + * `getL1ToL2MessagesBetweenLeafCounts`, with the starting position also required to be available; an empty range + * returns equal positions. + */ + public async getL1ToL2MessageRange(startLeafCount: bigint, endLeafCount: bigint): Promise { + assertValidLeafCountRange(startLeafCount, endLeafCount); + return await this.db.transactionAsync(async () => { + await this.assertLeafCountRangeSynced(startLeafCount, endLeafCount); + const start = await this.getMessagePosition(startLeafCount); + if (start === undefined) { + throw new InboxMessageRangeNotSyncedError( + startLeafCount, + endLeafCount, + `the store is missing the message at index ${startLeafCount - 1n}`, + ); + } + const messages = await this.getMessagesInLeafCountRange(startLeafCount, endLeafCount); + const lastMessage = messages.at(-1); + const end = + lastMessage === undefined + ? start + : { totalMessageCount: endLeafCount, rollingHash: lastMessage.inboxRollingHash }; + return { messages: messages.map(message => message.leaf), start, end }; + }); + } + + /** Throws unless every message in `[startLeafCount, endLeafCount)` is within the synced total. Empty ranges included. */ + private async assertLeafCountRangeSynced(startLeafCount: bigint, endLeafCount: bigint): Promise { + const syncedTotal = await this.getTotalL1ToL2MessageCount(); + if (endLeafCount > syncedTotal) { + const available = syncedTotal > startLeafCount ? syncedTotal - startLeafCount : 0n; + throw new InboxMessageRangeNotSyncedError( + startLeafCount, + endLeafCount, + `only ${available} of ${endLeafCount - startLeafCount} messages are synced`, + ); } - const startBucket = await this.getBucketAtBoundary(startLeafCount); - const endBucket = await this.getBucketAtBoundary(endLeafCount); - return this.getL1ToL2MessagesBetweenBuckets(startBucket.seq, endBucket.seq); } - /** Resolves the bucket ending at the given cumulative message count, failing loudly if there is none. */ - private async getBucketAtBoundary(totalMsgCount: bigint): Promise { - const bucket = await this.getInboxBucketByTotalMsgCount(totalMsgCount); - if (bucket === undefined) { - throw new InboxBucketBoundaryNotSyncedError(totalMsgCount); + /** + * Reads the messages in the compact index range `[startLeafCount, endLeafCount)`, which the caller has established + * lies within the synced total. The map holds at most one entry per index, so a short read is the only way a hole + * inside the range can show up, and the count catches every one of them. + */ + private async getMessagesInLeafCountRange(startLeafCount: bigint, endLeafCount: bigint): Promise { + if (startLeafCount === endLeafCount) { + return []; + } + const messages = await toArray(this.iterateL1ToL2Messages({ start: startLeafCount, end: endLeafCount })); + if (BigInt(messages.length) !== endLeafCount - startLeafCount) { + throw new InboxMessageRangeNotSyncedError( + startLeafCount, + endLeafCount, + `the store holds ${messages.length} of ${endLeafCount - startLeafCount} messages`, + ); } - return bucket; + return messages; } /** diff --git a/yarn-project/archiver/src/test/mock_archiver.ts b/yarn-project/archiver/src/test/mock_archiver.ts index 5ef6b4d49500..de301e878be5 100644 --- a/yarn-project/archiver/src/test/mock_archiver.ts +++ b/yarn-project/archiver/src/test/mock_archiver.ts @@ -2,7 +2,12 @@ import { Buffer32 } from '@aztec/foundation/buffer'; import { Fr } from '@aztec/foundation/curves/bn254'; import type { L2BlockSource } from '@aztec/stdlib/block'; import type { Checkpoint } from '@aztec/stdlib/checkpoint'; -import type { InboxBucket, L1ToL2MessageSource } from '@aztec/stdlib/messaging'; +import type { + InboxBucket, + InboxMessagePosition, + InboxMessageRange, + L1ToL2MessageSource, +} from '@aztec/stdlib/messaging'; import { MockL1ToL2MessageSource } from './mock_l1_to_l2_message_source.js'; import { MockL2BlockSource } from './mock_l2_block_source.js'; @@ -18,6 +23,14 @@ export class MockArchiver extends MockL2BlockSource implements L2BlockSource, L1 this.messageSource.setInboxBucket(bucket, msgs); } + public replaceInboxBuckets(buckets: { bucket: InboxBucket; msgs: Fr[] }[]) { + this.messageSource.replaceInboxBuckets(buckets); + } + + public appendL1ToL2Messages(msgs: Fr[]) { + this.messageSource.appendL1ToL2Messages(msgs); + } + getL1ToL2MessageIndex(_l1ToL2Message: Fr): Promise { return this.messageSource.getL1ToL2MessageIndex(_l1ToL2Message); } @@ -41,6 +54,18 @@ export class MockArchiver extends MockL2BlockSource implements L2BlockSource, L1 getL1ToL2MessagesBetweenLeafCounts(startLeafCount: bigint, endLeafCount: bigint): Promise { return this.messageSource.getL1ToL2MessagesBetweenLeafCounts(startLeafCount, endLeafCount); } + + getMessagePosition(totalMessageCount: bigint): Promise { + return this.messageSource.getMessagePosition(totalMessageCount); + } + + getSyncedMessagePosition(): Promise { + return this.messageSource.getSyncedMessagePosition(); + } + + getL1ToL2MessageRange(startLeafCount: bigint, endLeafCount: bigint): Promise { + return this.messageSource.getL1ToL2MessageRange(startLeafCount, endLeafCount); + } } /** @@ -67,12 +92,11 @@ export class MockPrefilledArchiver extends MockArchiver { this.prefilledMessages[checkpoint.number - 1] = messages; } - // Register the Inbox buckets the streaming world-state synchronizer reconstructs each block's consumed - // message bundle from: a genesis sentinel (totalMsgCount 0) so a leaf count of 0 - // resolves to a bucket, plus one bucket per message-carrying checkpoint whose cumulative totalMsgCount - // matches the block's post-insertion L1-to-L2 leaf count. Rebuilt from the full prefilled chain (not just - // this call's checkpoints) so a reorg re-prefill that replaces a suffix keeps the cumulative aligned. - // Without these the synchronizer derives an empty bundle and the reconstructed block state diverges. + // Index every message-carrying checkpoint's leaves at the compact positions the archiver would give them, which is + // what published-block replay reads by count. The leaves are registered through buckets (a genesis sentinel plus + // one bucket per message-carrying checkpoint) so tests can still model the live partition and repartition it as an + // L1 reorg would. Rebuilt from the full prefilled chain (not just this call's checkpoints) so a reorg re-prefill + // that replaces a suffix keeps the cumulative counts aligned. this.setInboxBucket( { seq: 0n, diff --git a/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.test.ts b/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.test.ts new file mode 100644 index 000000000000..8c9a74fb8d73 --- /dev/null +++ b/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.test.ts @@ -0,0 +1,62 @@ +import { Buffer32 } from '@aztec/foundation/buffer'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { type InboxBucket, updateInboxRollingHash } from '@aztec/stdlib/messaging'; + +import { InboxMessageRangeNotSyncedError } from '../errors.js'; +import { MockL1ToL2MessageSource } from './mock_l1_to_l2_message_source.js'; + +/** A single-message bucket at the start of the Inbox; only its total matters to the leaf index. */ +const singleMessageBucket: InboxBucket = { + seq: 1n, + inboxRollingHash: Fr.ZERO, + totalMsgCount: 1n, + timestamp: 1n, + msgCount: 1, + lastMessageIndex: 0n, + l1BlockNumber: 1n, + l1BlockHash: Buffer32.ZERO, +}; + +describe('MockL1ToL2MessageSource', () => { + let source: MockL1ToL2MessageSource; + + beforeEach(() => { + source = new MockL1ToL2MessageSource(0); + }); + + it('derives positions from the indexed leaves', async () => { + const leaves = [new Fr(11), new Fr(12), new Fr(13)]; + source.appendL1ToL2Messages(leaves); + const hashAfterTwo = updateInboxRollingHash(updateInboxRollingHash(Fr.ZERO, leaves[0]), leaves[1]); + + expect(await source.getMessagePosition(0n)).toEqual({ totalMessageCount: 0n, rollingHash: Fr.ZERO }); + expect(await source.getMessagePosition(2n)).toEqual({ totalMessageCount: 2n, rollingHash: hashAfterTwo }); + expect(await source.getMessagePosition(4n)).toBeUndefined(); + expect((await source.getSyncedMessagePosition()).totalMessageCount).toEqual(3n); + expect(await source.getL1ToL2MessageRange(1n, 2n)).toEqual({ + messages: [leaves[1]], + start: { totalMessageCount: 1n, rollingHash: updateInboxRollingHash(Fr.ZERO, leaves[0]) }, + end: { totalMessageCount: 2n, rollingHash: hashAfterTwo }, + }); + }); + + it('rejects ranges past the mocked tip with the archiver error, empty ones included', async () => { + await expect(source.getL1ToL2MessagesBetweenLeafCounts(7n, 7n)).rejects.toThrow(InboxMessageRangeNotSyncedError); + await expect(source.getL1ToL2MessageRange(7n, 7n)).rejects.toThrow(InboxMessageRangeNotSyncedError); + await expect(source.getL1ToL2MessageRange(0n, 1n)).rejects.toThrow(InboxMessageRangeNotSyncedError); + await expect(source.getL1ToL2MessageRange(2n, 1n)).rejects.toThrow(/Invalid Inbox leaf count range/); + expect(await source.getL1ToL2MessagesBetweenLeafCounts(0n, 0n)).toEqual([]); + }); + + it('reads the leaves and the ending hash of a range from the same version of the log', async () => { + source.setInboxBucket(singleMessageBucket, [new Fr(11)]); + + // Replace the leaf while the read is pending: the result must describe one version, not a mix of both. + const pending = source.getL1ToL2MessageRange(0n, 1n); + source.setInboxBucket(singleMessageBucket, [new Fr(22)]); + + const range = await pending; + expect(range.messages).toEqual([new Fr(11)]); + expect(range.end.rollingHash).toEqual(updateInboxRollingHash(Fr.ZERO, new Fr(11))); + }); +}); diff --git a/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts b/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts index 53b652f8d279..40caceaf39b8 100644 --- a/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts +++ b/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts @@ -1,7 +1,15 @@ import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; import { Fr } from '@aztec/foundation/curves/bn254'; import type { CheckpointId, L2BlockId, L2TipId, L2Tips } from '@aztec/stdlib/block'; -import type { InboxBucket, L1ToL2MessageSource } from '@aztec/stdlib/messaging'; +import { + type InboxBucket, + type InboxMessagePosition, + type InboxMessageRange, + type L1ToL2MessageSource, + updateInboxRollingHash, +} from '@aztec/stdlib/messaging'; + +import { InboxMessageRangeNotSyncedError } from '../errors.js'; /** * A mocked implementation of L1ToL2MessageSource to be used in tests. @@ -9,18 +17,65 @@ import type { InboxBucket, L1ToL2MessageSource } from '@aztec/stdlib/messaging'; export class MockL1ToL2MessageSource implements L1ToL2MessageSource { private buckets = new Map(); private messagesPerBucket = new Map(); + /** + * The canonical message log, keyed by compact global index. This is the primary fixture: message positions and + * count ranges derive from it alone, and it is kept apart from the bucket partition so a test can repartition the + * buckets (as an L1 reorg does) while the indexed leaves stay exactly as they were. + */ + private leavesByIndex = new Map(); constructor(private blockNumber: number) {} public setInboxBucket(bucket: InboxBucket, msgs: Fr[] = []) { this.buckets.set(bucket.seq, bucket); this.messagesPerBucket.set(bucket.seq, msgs); + // Index from the bucket's own cumulative total rather than call order, so re-registering a bucket or setting them + // out of order keeps every leaf at the index the archiver would give it. + const firstIndex = bucket.totalMsgCount - BigInt(msgs.length); + msgs.forEach((msg, i) => this.leavesByIndex.set(firstIndex + BigInt(i), msg)); + } + + /** + * Replaces the current bucket partition without touching the indexed leaf log, modelling an L1 reorg that re-mines + * the same messages under different bucket boundaries. + */ + public replaceInboxBuckets(buckets: { bucket: InboxBucket; msgs: Fr[] }[]) { + this.buckets = new Map(); + this.messagesPerBucket = new Map(); + for (const { bucket, msgs } of buckets) { + this.buckets.set(bucket.seq, bucket); + this.messagesPerBucket.set(bucket.seq, msgs); + } + } + + /** Appends leaves to the indexed message log at its synced tip, without registering any bucket for them. */ + public appendL1ToL2Messages(msgs: Fr[]) { + const firstIndex = this.getSyncedMessageCount(); + msgs.forEach((msg, i) => this.leavesByIndex.set(firstIndex + BigInt(i), msg)); } public setBlockNumber(blockNumber: number) { this.blockNumber = blockNumber; } + /** The number of leaves indexed contiguously from zero: the mocked synced tip. */ + private getSyncedMessageCount(): bigint { + let count = 0n; + while (this.leavesByIndex.has(count)) { + count++; + } + return count; + } + + /** Recomputes the rolling hash over the first `totalMessageCount` indexed leaves, as the archiver stores per message. */ + private computeRollingHash(totalMessageCount: bigint): Fr { + let hash = Fr.ZERO; + for (let index = 0n; index < totalMessageCount; index++) { + hash = updateInboxRollingHash(hash, this.leavesByIndex.get(index)!); + } + return hash; + } + getL1ToL2MessageIndex(_l1ToL2Message: Fr): Promise { throw new Error('Method not implemented.'); } @@ -50,13 +105,70 @@ export class MockL1ToL2MessageSource implements L1ToL2MessageSource { return Promise.resolve(seqs.flatMap(seq => this.messagesPerBucket.get(seq) ?? [])); } - async getL1ToL2MessagesBetweenLeafCounts(startLeafCount: bigint, endLeafCount: bigint): Promise { - const startBucket = await this.getInboxBucketByTotalMsgCount(startLeafCount); - const endBucket = await this.getInboxBucketByTotalMsgCount(endLeafCount); - if (startBucket === undefined || endBucket === undefined) { - throw new Error(`No mocked Inbox bucket boundary at ${startLeafCount} or ${endLeafCount}`); + /** + * Slices the indexed leaf log, enforcing the same range contract as the archiver's message store: invalid bounds and + * ranges past the synced tip are rejected, the latter with the archiver's typed availability error. Every failure is + * a rejection rather than a synchronous throw, so callers see this stand-in behave like the async source it mocks. + */ + getL1ToL2MessagesBetweenLeafCounts(startLeafCount: bigint, endLeafCount: bigint): Promise { + try { + return Promise.resolve(this.readLeafCountRange(startLeafCount, endLeafCount)); + } catch (err) { + return Promise.reject(err); + } + } + + getMessagePosition(totalMessageCount: bigint): Promise { + if (totalMessageCount < 0n) { + return Promise.reject(new Error(`Invalid Inbox message count ${totalMessageCount}`)); + } + if (totalMessageCount > this.getSyncedMessageCount()) { + return Promise.resolve(undefined); + } + return Promise.resolve({ totalMessageCount, rollingHash: this.computeRollingHash(totalMessageCount) }); + } + + getSyncedMessagePosition(): Promise { + const totalMessageCount = this.getSyncedMessageCount(); + return Promise.resolve({ totalMessageCount, rollingHash: this.computeRollingHash(totalMessageCount) }); + } + + /** + * Reads the range and both positions synchronously from the current leaf log, so a test that mutates the log while + * a range read is pending cannot pair leaves of one version with hashes of another, as the archiver's single-transaction + * read cannot either. + */ + getL1ToL2MessageRange(startLeafCount: bigint, endLeafCount: bigint): Promise { + try { + const messages = this.readLeafCountRange(startLeafCount, endLeafCount); + return Promise.resolve({ + messages, + start: { totalMessageCount: startLeafCount, rollingHash: this.computeRollingHash(startLeafCount) }, + end: { totalMessageCount: endLeafCount, rollingHash: this.computeRollingHash(endLeafCount) }, + }); + } catch (err) { + return Promise.reject(err); + } + } + + private readLeafCountRange(startLeafCount: bigint, endLeafCount: bigint): Fr[] { + if (startLeafCount < 0n || endLeafCount < 0n || startLeafCount > endLeafCount) { + throw new Error(`Invalid Inbox leaf count range [${startLeafCount}, ${endLeafCount})`); + } + const syncedCount = this.getSyncedMessageCount(); + if (endLeafCount > syncedCount) { + const available = syncedCount > startLeafCount ? syncedCount - startLeafCount : 0n; + throw new InboxMessageRangeNotSyncedError( + startLeafCount, + endLeafCount, + `only ${available} of ${endLeafCount - startLeafCount} messages are mocked`, + ); + } + const leaves: Fr[] = []; + for (let index = startLeafCount; index < endLeafCount; index++) { + leaves.push(this.leavesByIndex.get(index)!); } - return this.getL1ToL2MessagesBetweenBuckets(startBucket.seq, endBucket.seq); + return leaves; } getBlockNumber() { diff --git a/yarn-project/kv-store/src/interfaces/common.ts b/yarn-project/kv-store/src/interfaces/common.ts index 1b6c142d491f..45584336b0ad 100644 --- a/yarn-project/kv-store/src/interfaces/common.ts +++ b/yarn-project/kv-store/src/interfaces/common.ts @@ -18,8 +18,8 @@ export type CustomRange = { /** Maps a custom range into a range of valid key types to iterate over. */ export function mapRange(range: CustomRange, mapFn: (key: CK) => K): Range { return { - start: range.start ? mapFn(range.start) : undefined, - end: range.end ? mapFn(range.end) : undefined, + start: range.start !== undefined ? mapFn(range.start) : undefined, + end: range.end !== undefined ? mapFn(range.end) : undefined, reverse: range.reverse, limit: range.limit, }; diff --git a/yarn-project/sequencer-client/src/publisher/l1_publisher.integration.test.ts b/yarn-project/sequencer-client/src/publisher/l1_publisher.integration.test.ts index a65d86047d59..ae94abf9ba6c 100644 --- a/yarn-project/sequencer-client/src/publisher/l1_publisher.integration.test.ts +++ b/yarn-project/sequencer-client/src/publisher/l1_publisher.integration.test.ts @@ -149,9 +149,9 @@ describe('L1Publisher integration', () => { let builderDb: NativeWorldStateService; - // Backs the blockSource mock's streaming L1->L2 message queries. The world-state synchronizer reconstructs each - // block's consumed message bundle from Inbox buckets when it syncs a block back, so the test - // registers one bucket per published block here (see buildAndPublishBlock). + // Backs the blockSource mock's streaming L1->L2 message queries. The world-state synchronizer reads each block's + // consumed message bundle by leaf count when it syncs a block back, so the test mirrors every Inbox bucket and its + // leaves here before publishing the block that consumes them (see buildAndPublishBlock). let messageSource: MockL1ToL2MessageSource; // The header of the last block @@ -381,13 +381,10 @@ describe('L1Publisher integration', () => { getBlockNumber(): Promise { return Promise.resolve(BlockNumber(blocks.at(-1)?.number ?? BlockNumber.ZERO)); }, - // Streaming L1->L2 message reconstruction: the world-state synchronizer resolves each - // block's consumed message bundle from the Inbox buckets registered per published block in buildAndPublishBlock. - getInboxBucketByTotalMsgCount(totalMsgCount: bigint) { - return messageSource.getInboxBucketByTotalMsgCount(totalMsgCount); - }, - getL1ToL2MessagesBetweenBuckets(fromExclusive: bigint, toInclusive: bigint) { - return messageSource.getL1ToL2MessagesBetweenBuckets(fromExclusive, toInclusive); + // Streaming L1->L2 message reconstruction: the world-state synchronizer reads each block's consumed message + // bundle by leaf count from the leaves mirrored per published block in buildAndPublishBlock. + getL1ToL2MessagesBetweenLeafCounts(startLeafCount: bigint, endLeafCount: bigint) { + return messageSource.getL1ToL2MessagesBetweenLeafCounts(startLeafCount, endLeafCount); }, }); diff --git a/yarn-project/stdlib/src/interfaces/archiver.test.ts b/yarn-project/stdlib/src/interfaces/archiver.test.ts index d505b005bea0..115ebd151457 100644 --- a/yarn-project/stdlib/src/interfaces/archiver.test.ts +++ b/yarn-project/stdlib/src/interfaces/archiver.test.ts @@ -39,6 +39,7 @@ import type { PrivateLogsQuery, PublicLogsQuery } from '../logs/logs_query.js'; import { SiloedTag } from '../logs/siloed_tag.js'; import { Tag } from '../logs/tag.js'; import type { InboxBucket } from '../messaging/inbox_bucket.js'; +import type { InboxMessagePosition, InboxMessageRange } from '../messaging/l1_to_l2_message_source.js'; import { CheckpointHeader } from '../rollup/checkpoint_header.js'; import { getTokenContractArtifact } from '../tests/fixtures.js'; import { AppendOnlyTreeSnapshot } from '../trees/append_only_tree_snapshot.js'; @@ -242,6 +243,25 @@ describe('ArchiverApiSchema', () => { expect(result).toEqual([expect.any(Fr)]); }); + it('getMessagePosition', async () => { + const result = await context.client.getMessagePosition(3n); + expect(result).toEqual({ totalMessageCount: 3n, rollingHash: expect.any(Fr) }); + }); + + it('getSyncedMessagePosition', async () => { + const result = await context.client.getSyncedMessagePosition(); + expect(result).toEqual({ totalMessageCount: 3n, rollingHash: expect.any(Fr) }); + }); + + it('getL1ToL2MessageRange', async () => { + const result = await context.client.getL1ToL2MessageRange(2n, 3n); + expect(result).toEqual({ + messages: [expect.any(Fr)], + start: { totalMessageCount: 2n, rollingHash: expect.any(Fr) }, + end: { totalMessageCount: 3n, rollingHash: expect.any(Fr) }, + }); + }); + it('registerContractFunctionSignatures', async () => { await context.client.registerContractFunctionSignatures(['test()']); }); @@ -650,6 +670,22 @@ class MockArchiver implements ArchiverApi { expect(typeof endLeafCount).toEqual('bigint'); return Promise.resolve([Fr.random()]); } + getMessagePosition(totalMessageCount: bigint): Promise { + expect(typeof totalMessageCount).toEqual('bigint'); + return Promise.resolve({ totalMessageCount, rollingHash: Fr.random() }); + } + getSyncedMessagePosition(): Promise { + return Promise.resolve({ totalMessageCount: 3n, rollingHash: Fr.random() }); + } + getL1ToL2MessageRange(startLeafCount: bigint, endLeafCount: bigint): Promise { + expect(typeof startLeafCount).toEqual('bigint'); + expect(typeof endLeafCount).toEqual('bigint'); + return Promise.resolve({ + messages: [Fr.random()], + start: { totalMessageCount: startLeafCount, rollingHash: Fr.random() }, + end: { totalMessageCount: endLeafCount, rollingHash: Fr.random() }, + }); + } getL1Constants(): Promise { return Promise.resolve(EmptyL1RollupConstants); } diff --git a/yarn-project/stdlib/src/interfaces/archiver.ts b/yarn-project/stdlib/src/interfaces/archiver.ts index 605f34e03233..1614b2d7ae56 100644 --- a/yarn-project/stdlib/src/interfaces/archiver.ts +++ b/yarn-project/stdlib/src/interfaces/archiver.ts @@ -26,7 +26,11 @@ import { L1RollupConstantsSchema } from '../epoch-helpers/index.js'; import { LogResultSchema } from '../logs/log_result.js'; import { PrivateLogsQuerySchema, PublicLogsQuerySchema } from '../logs/logs_query.js'; import { InboxBucketSchema } from '../messaging/inbox_bucket.js'; -import type { L1ToL2MessageSource } from '../messaging/l1_to_l2_message_source.js'; +import { + InboxMessagePositionSchema, + InboxMessageRangeSchema, + type L1ToL2MessageSource, +} from '../messaging/l1_to_l2_message_source.js'; import { L2ToL1MembershipWitnessSchema } from '../messaging/l2_to_l1_membership.js'; import { optional, schemas } from '../schemas/schemas.js'; import { indexedTxSchema } from '../tx/indexed_tx_effect.js'; @@ -154,6 +158,12 @@ export const ArchiverApiSchema: ApiSchemaFor = { input: z.tuple([schemas.BigInt, schemas.BigInt]), output: z.array(schemas.Fr), }), + getMessagePosition: z.function({ input: z.tuple([schemas.BigInt]), output: InboxMessagePositionSchema.optional() }), + getSyncedMessagePosition: z.function({ input: z.tuple([]), output: InboxMessagePositionSchema }), + getL1ToL2MessageRange: z.function({ + input: z.tuple([schemas.BigInt, schemas.BigInt]), + output: InboxMessageRangeSchema, + }), getDebugFunctionName: z.function({ input: z.tuple([schemas.AztecAddress, schemas.FunctionSelector]), output: optional(z.string()), diff --git a/yarn-project/stdlib/src/messaging/l1_to_l2_message_source.ts b/yarn-project/stdlib/src/messaging/l1_to_l2_message_source.ts index 4040483f144b..20c1ecdd23f8 100644 --- a/yarn-project/stdlib/src/messaging/l1_to_l2_message_source.ts +++ b/yarn-project/stdlib/src/messaging/l1_to_l2_message_source.ts @@ -1,8 +1,48 @@ -import type { Fr } from '@aztec/foundation/curves/bn254'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { schemas } from '@aztec/foundation/schemas'; + +import { z } from 'zod'; import type { L2Tips } from '../block/l2_block_source.js'; import type { InboxBucket } from './inbox_bucket.js'; +/** + * A position in the ordered Inbox message sequence: the number of messages up to it, which is also the compact index + * of the next message, and the consensus rolling hash over exactly those messages. Position zero has a zero hash. + * Block headers commit to the count (the L1-to-L2 tree leaf count) and checkpoint headers to the hash, so a position + * identifies a message prefix independently of how L1 partitioned the messages into buckets. + */ +export type InboxMessagePosition = { + /** Number of messages in the sequence up to this position. */ + totalMessageCount: bigint; + /** Consensus rolling hash (truncated sha256 chain) over the messages up to this position; zero at position zero. */ + rollingHash: Fr; +}; + +export const InboxMessagePositionSchema = z.object({ + totalMessageCount: schemas.BigInt, + rollingHash: Fr.schema, +}) satisfies z.ZodType; + +/** + * The messages in a compact count range together with the positions the range starts and ends at, all read from one + * snapshot of the source, so `end.rollingHash` authenticates exactly `messages` appended after `start`. + */ +export type InboxMessageRange = { + /** The message leaves in the range, in insertion order. */ + messages: Fr[]; + /** The position the range starts at, inclusive. */ + start: InboxMessagePosition; + /** The position the range ends at, exclusive; equal to `start` for an empty range. */ + end: InboxMessagePosition; +}; + +export const InboxMessageRangeSchema = z.object({ + messages: z.array(schemas.Fr), + start: InboxMessagePositionSchema, + end: InboxMessagePositionSchema, +}) satisfies z.ZodType; + /** * Interface of classes allowing for the retrieval of L1 to L2 messages. */ @@ -53,14 +93,41 @@ export interface L1ToL2MessageSource { /** * Returns the message leaves in the cumulative Inbox message-count range `[startLeafCount, endLeafCount)`, in - * insertion order. The bounds are compact L1-to-L2 tree leaf counts, which every block header - * carries, so a consumer can ask for the messages a block or checkpoint consumed without resolving Inbox buckets - * itself. Both bounds must land on a bucket boundary the source has synced; it throws otherwise. + * insertion order. The bounds are compact L1-to-L2 tree leaf counts, which every block header carries, so a + * consumer can ask for the messages a block or checkpoint consumed without resolving Inbox buckets itself. + * + * The bounds address canonical compact message indices and need not land on a boundary of the bucket partition the + * source currently holds, so a published block's committed leaf counts stay resolvable after an L1 reorg has merged + * the bucket that ended at one of them. An invalid range, one past the synced tip, or one the source cannot serve + * whole throws, so an empty result always means the range holds no messages. * @param startLeafCount - The cumulative Inbox message count the range starts at, inclusive. * @param endLeafCount - The cumulative Inbox message count the range ends at, exclusive. */ getL1ToL2MessagesBetweenLeafCounts(startLeafCount: bigint, endLeafCount: bigint): Promise; + /** + * Returns the position of the Inbox message sequence after `totalMessageCount` messages: that count and the rolling + * hash over them. Position zero always resolves, with a zero hash; a count past the synced tip returns undefined, + * and a negative one throws. + * @param totalMessageCount - The cumulative Inbox message count (leaf count) whose position to resolve. + */ + getMessagePosition(totalMessageCount: bigint): Promise; + + /** Returns the position at the source's synced tip: how many messages it holds and the rolling hash over them. */ + getSyncedMessagePosition(): Promise; + + /** + * Returns the messages in the cumulative Inbox message-count range `[startLeafCount, endLeafCount)` together with + * the positions at both bounds, all read from one snapshot of the source, so the ending hash authenticates exactly + * the returned messages and cannot describe a different version of the sequence than they do. An empty range is + * valid and returns equal positions. The range contract is that of `getL1ToL2MessagesBetweenLeafCounts`: an + * invalid range, or one the source cannot serve whole (including its starting position), throws rather than + * returning a partial or empty result. + * @param startLeafCount - The cumulative Inbox message count the range starts at, inclusive. + * @param endLeafCount - The cumulative Inbox message count the range ends at, exclusive. + */ + getL1ToL2MessageRange(startLeafCount: bigint, endLeafCount: bigint): Promise; + /** * Returns the tips of the L2 chain. */ diff --git a/yarn-project/validator-client/src/proposal_handler.test.ts b/yarn-project/validator-client/src/proposal_handler.test.ts index 4bf7316a6f25..72ff62e9163c 100644 --- a/yarn-project/validator-client/src/proposal_handler.test.ts +++ b/yarn-project/validator-client/src/proposal_handler.test.ts @@ -562,6 +562,127 @@ describe('ProposalHandler checkpoint validation', () => { return makeHeader({ epochOutHash, ...overrides }); } + /** A checkpoint whose first block is `firstBlockNumber`, with the parent block reporting `parentLeafCount`. */ + function setupCheckpointWithConsumption(opts: { + firstBlockNumber: number; + parentLeafCount: number | undefined; + lastLeafCount: number; + }) { + const { firstBlockNumber, parentLeafCount, lastLeafCount } = opts; + const block = { + archive: new AppendOnlyTreeSnapshot(archiveRoot, 1), + number: firstBlockNumber, + checkpointNumber: CheckpointNumber(1), + header: { + globalVariables: GlobalVariables.empty({ slotNumber: SlotNumber(1) }), + state: { l1ToL2MessageTree: { nextAvailableLeafIndex: lastLeafCount } }, + }, + } as unknown as L2Block; + blockSource.getBlocksForSlot.mockResolvedValue([block]); + blockSource.getBlockData.mockImplementation(query => + Promise.resolve( + 'number' in query && query.number === firstBlockNumber - 1 + ? parentLeafCount === undefined + ? undefined + : ({ + header: { state: { l1ToL2MessageTree: { nextAvailableLeafIndex: parentLeafCount } } }, + } as unknown as BlockData) + : ({ header: makeBlockHeader() } as BlockData), + ), + ); + return block; + } + + // An L1 reorg can merge the buckets a checkpointed block and the proposal's last block consumed through into + // others. The archiver never prunes a checkpointed block, so the parent position stays interior to the current + // partition forever, and the proposal's own final position may sit interior to it as well while the messages + // themselves are unchanged. The consumed bundle is derived from the committed counts alone, so neither position + // needs to resolve to a bucket; whether the final position is a live bucket end is L1's publication rule. + it('derives the consumed bundle by count when neither checkpoint bound is a current bucket boundary', async () => { + const header = makeHeader(); + const consumedMessages = [new Fr(1000), new Fr(1001), new Fr(1002), new Fr(1003)]; + setupDeepValidationMocks({ header }); + const block = setupCheckpointWithConsumption({ firstBlockNumber: 5, parentLeafCount: 3, lastLeafCount: 7 }); + + l1ToL2MessageSource.getInboxBucketByTotalMsgCount.mockResolvedValue(undefined); + l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts.mockResolvedValue(consumedMessages); + + await handler.handleCheckpointProposal( + await makeProposal({ archiveRoot, checkpointHeader: header }), + proposalInfo, + ); + + expect(l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts).toHaveBeenCalledWith(3n, 7n); + expect(l1ToL2MessageSource.getL1ToL2MessagesBetweenBuckets).not.toHaveBeenCalled(); + expect(checkpointsBuilder.openCheckpoint).toHaveBeenCalledWith( + CheckpointNumber(1), + expect.anything(), + expect.anything(), + consumedMessages, + expect.anything(), + expect.anything(), + expect.anything(), + [block], + expect.anything(), + ); + }); + + it('derives an empty bundle without any message query when the checkpoint consumed nothing', async () => { + const header = makeHeader(); + setupDeepValidationMocks({ header }); + const block = setupCheckpointWithConsumption({ firstBlockNumber: 5, parentLeafCount: 3, lastLeafCount: 3 }); + + await handler.handleCheckpointProposal( + await makeProposal({ archiveRoot, checkpointHeader: header }), + proposalInfo, + ); + + expect(l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts).not.toHaveBeenCalled(); + expect(checkpointsBuilder.openCheckpoint).toHaveBeenCalledWith( + CheckpointNumber(1), + expect.anything(), + expect.anything(), + [], + expect.anything(), + expect.anything(), + expect.anything(), + [block], + expect.anything(), + ); + }); + + // A missing parent is a local chain-availability failure. Deriving an empty bundle instead would make the + // rolling-hash recomputation fail and classify the proposer's valid checkpoint as a slashable header mismatch. + it('reports a fetch error instead of deriving an empty bundle when the parent block is unavailable', async () => { + const header = makeHeader(); + setupDeepValidationMocks({ header }); + setupCheckpointWithConsumption({ firstBlockNumber: 5, parentLeafCount: undefined, lastLeafCount: 7 }); + + const result = await handler.handleCheckpointProposal( + await makeProposal({ archiveRoot, checkpointHeader: header }), + proposalInfo, + ); + + expect(result).toEqual({ isValid: false, reason: 'block_fetch_error', checkpointNumber: CheckpointNumber(1) }); + expect(l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts).not.toHaveBeenCalled(); + expect(checkpointsBuilder.openCheckpoint).not.toHaveBeenCalled(); + }); + + it('surfaces an unavailable consumed range instead of deriving an empty bundle', async () => { + const header = makeHeader(); + setupDeepValidationMocks({ header }); + setupCheckpointWithConsumption({ firstBlockNumber: 5, parentLeafCount: 3, lastLeafCount: 7 }); + l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts.mockRejectedValue( + new Error('Inbox message range [3, 7) is not fully synced'), + ); + + await expect( + handler.handleCheckpointProposal(await makeProposal({ archiveRoot, checkpointHeader: header }), proposalInfo), + ).rejects.toThrow(/not fully synced/); + + expect(checkpointsBuilder.openCheckpoint).not.toHaveBeenCalled(); + }); + it('returns checkpoint_header_mismatch when headers differ', async () => { const proposalHeader = makeHeader(); const computedHeader = makeHeader({ totalManaUsed: new Fr(999) }); diff --git a/yarn-project/validator-client/src/proposal_handler.ts b/yarn-project/validator-client/src/proposal_handler.ts index 33faaa69ec8d..ded3b5c6e32f 100644 --- a/yarn-project/validator-client/src/proposal_handler.ts +++ b/yarn-project/validator-client/src/proposal_handler.ts @@ -1135,14 +1135,17 @@ export class ProposalHandler { /** * Enforces the streaming-Inbox last-block minimum-consumption (censorship) rule for a checkpoint, mirroring * `ProposeLib.validateInboxConsumption`: the first bucket the checkpoint left unconsumed must be absent, past the - * cutoff, or a cap-escape. Returns true (sufficient) when the checkpoint's consumption cannot be resolved against - * the local Inbox view, deferring to L1 `propose` as the authoritative reject. + * cutoff, or a cap-escape. Returns true (sufficient) when the checkpoint's final consumption position cannot be + * resolved against the local Inbox view, deferring to L1 `propose` as the authoritative reject. */ - private async isLastBlockConsumptionSufficient(slot: SlotNumber, blocks: L2Block[]): Promise { + private async isLastBlockConsumptionSufficient( + slot: SlotNumber, + checkpointStartTotal: bigint, + blocks: L2Block[], + ): Promise { const lastBlockTotal = this.blockLeafCount(blocks[blocks.length - 1]); - const checkpointStartTotal = await this.getPreBlockConsumedTotal(blocks[0].number); const lastConsumedBucket = await this.l1ToL2MessageSource.getInboxBucketByTotalMsgCount(lastBlockTotal); - if (checkpointStartTotal === undefined || lastConsumedBucket === undefined) { + if (lastConsumedBucket === undefined) { return true; } const nextBucket = await this.l1ToL2MessageSource.getInboxBucket(lastConsumedBucket.seq + 1n); @@ -1156,22 +1159,24 @@ export class ProposalHandler { } /** - * Derives the ordered list of L1-to-L2 messages a checkpoint consumed across its blocks, from the Inbox buckets - * between the parent checkpoint's consumed position and the checkpoint's last block. Empty when - * the checkpoint consumed nothing or its consumption cannot be resolved against the local Inbox view. + * Derives the ordered list of L1-to-L2 messages a checkpoint consumed across its blocks: the compact message-count + * range between the parent checkpoint's consumed position and the checkpoint's last block, read by count from the + * local message log. Empty when the checkpoint consumed nothing. + * + * Neither bound is resolved as an Inbox bucket. Both are counts committed by block headers, and an L1 reorg that + * merges buckets can leave either one interior to the current partition without changing the messages the blocks + * consumed; whether the final position is a live bucket end is a publication rule L1 `propose` enforces. The + * minimum-consumption guard that runs before this only checks censorship against the buckets it can resolve + * locally and defers an unresolved endpoint to L1. Throws when part of the range is not available locally: a local + * availability + * gap must surface as such rather than as an empty bundle that would make a valid proposal fail its rolling-hash + * recomputation. */ - private async deriveCheckpointConsumedMessages(blocks: L2Block[]): Promise { - const checkpointStartTotal = await this.getPreBlockConsumedTotal(blocks[0].number); + private deriveCheckpointConsumedMessages(checkpointStartTotal: bigint, blocks: L2Block[]): Promise { const lastBlockTotal = this.blockLeafCount(blocks[blocks.length - 1]); - if (checkpointStartTotal === undefined || lastBlockTotal <= checkpointStartTotal) { - return []; - } - const startBucket = await this.l1ToL2MessageSource.getInboxBucketByTotalMsgCount(checkpointStartTotal); - const endBucket = await this.l1ToL2MessageSource.getInboxBucketByTotalMsgCount(lastBlockTotal); - if (startBucket === undefined || endBucket === undefined) { - return []; - } - return this.l1ToL2MessageSource.getL1ToL2MessagesBetweenBuckets(startBucket.seq, endBucket.seq); + return lastBlockTotal <= checkpointStartTotal + ? Promise.resolve([]) + : this.l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts(checkpointStartTotal, lastBlockTotal); } async reexecuteTransactions( @@ -1469,9 +1474,21 @@ export class ProposalHandler { const constants = this.extractCheckpointConstants(firstBlock); const checkpointNumber = firstBlock.checkpointNumber; + // The checkpoint's Inbox consumption starts at the leaf count of the block before its first block. Without that + // block the consumed bundle cannot be derived; an empty bundle would make a valid proposal fail its rolling-hash + // recomputation and be classified as a proposer offense, so a missing parent is a local fetch failure instead. + const checkpointStartTotal = await this.getPreBlockConsumedTotal(firstBlock.number); + if (checkpointStartTotal === undefined) { + this.log.warn(`Block before checkpoint proposal's first block ${firstBlock.number} is unavailable locally`, { + ...proposalInfo, + checkpointNumber, + }); + return { isValid: false, reason: 'block_fetch_error', checkpointNumber }; + } + // Streaming Inbox: on the last block of a checkpoint, enforce the minimum-consumption // (censorship) rule before attesting. Reject (no attestation) if a mandatory bucket was left unconsumed. - if (!(await this.isLastBlockConsumptionSufficient(slot, blocks))) { + if (!(await this.isLastBlockConsumptionSufficient(slot, checkpointStartTotal, blocks))) { this.log.warn(`Streaming Inbox last-block censorship check failed, refusing to attest`, { ...proposalInfo, checkpointNumber, @@ -1479,10 +1496,10 @@ export class ProposalHandler { return { isValid: false, reason: 'inbox_consumption_insufficient', checkpointNumber }; } - // Derive the checkpoint's consumed L1-to-L2 message list from the Inbox buckets between the parent checkpoint's - // consumed position and the last block's (compact indexing). The messages are already in the db from per-block + // Derive the checkpoint's consumed L1-to-L2 message list from the message-count range between the parent + // checkpoint's consumed position and the last block's. The messages are already in the db from per-block // validation; this list only drives the checkpoint's rolling-hash recomputation in completeCheckpoint. - const l1ToL2Messages = await this.deriveCheckpointConsumedMessages(blocks); + const l1ToL2Messages = await this.deriveCheckpointConsumedMessages(checkpointStartTotal, blocks); // Collect the out hashes of all the checkpoints before this one in the same epoch. // See note on the analogous block-proposal site: the helper handles pipelining lag. diff --git a/yarn-project/validator-client/src/validator.test.ts b/yarn-project/validator-client/src/validator.test.ts index d148cd87ea52..fa60588a7a19 100644 --- a/yarn-project/validator-client/src/validator.test.ts +++ b/yarn-project/validator-client/src/validator.test.ts @@ -406,6 +406,16 @@ describe('ValidatorClient', () => { } as unknown as L2Block; const disposeFork = jest.fn(); blockSource.getBlocksForSlot.mockResolvedValue([checkpointBlock]); + // The checkpoint's consumed message bundle derives from the leaf count of the block before its first block, so + // that parent must resolve by number; other number queries stay unresolved as in the surrounding setup. + blockSource.getBlockData.mockImplementation(query => + Promise.resolve('number' in query && query.number !== blockNumber - 1 ? undefined : parentBlockData), + ); + // With the parent resolvable, the censorship check runs too: leave no bucket after genesis so nothing is left + // unconsumed and validation reaches the header comparison. + l1ToL2MessageSource.getInboxBucket.mockImplementation(seq => + Promise.resolve(seq === 0n ? genesisInboxBucket : undefined), + ); checkpointsBuilder.getFork.mockResolvedValue({ [Symbol.asyncDispose]: disposeFork, // Match the proposal's expected starting archive so the fork archive check passes and validation @@ -551,6 +561,7 @@ describe('ValidatorClient', () => { l1ToL2MessageSource.getInboxBucket.mockResolvedValue(genesisInboxBucket); l1ToL2MessageSource.getInboxBucketByTotalMsgCount.mockResolvedValue(genesisInboxBucket); l1ToL2MessageSource.getL1ToL2MessagesBetweenBuckets.mockResolvedValue([]); + l1ToL2MessageSource.getL1ToL2MessagesBetweenLeafCounts.mockResolvedValue([]); const clonedBlockHeader = blockHeader.clone(); blockBuildResult = { diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts index 1edcaf2bdb64..d381c992c026 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts @@ -51,6 +51,21 @@ describe('ServerWorldStateSynchronizer', () => { beforeEach(() => { blockAndMessagesSource = mock(); blockAndMessagesSource.getBlockNumber.mockResolvedValue(BlockNumber(LATEST_BLOCK_NUMBER)); + // Published-block replay reads the parent block's leaf count and then the messages in the range it opens, so the + // source answers both like an archiver holding the whole mock chain. + blockAndMessagesSource.getBlockData.mockImplementation(async query => { + const block = allBlocks().find(b => 'number' in query && b.number === query.number); + return block === undefined + ? undefined + : { + header: block.header, + archive: block.archive, + blockHash: await block.hash(), + checkpointNumber: block.checkpointNumber, + indexWithinCheckpoint: block.indexWithinCheckpoint, + }; + }); + blockAndMessagesSource.getL1ToL2MessagesBetweenLeafCounts.mockResolvedValue([]); merkleTreeRead = mock(); merkleTreeRead.getInitialHeader.mockReturnValue({ @@ -88,8 +103,10 @@ describe('ServerWorldStateSynchronizer', () => { await server.stop(); }); + const allBlocks = () => checkpoints.flatMap(c => c.checkpoint.blocks); + const pushBlocks = async (from: number, to: number) => { - const blocks = checkpoints.flatMap(c => c.checkpoint.blocks).filter(b => b.number >= from && b.number <= to); + const blocks = allBlocks().filter(b => b.number >= from && b.number <= to); await server.handleBlockStreamEvent({ type: 'blocks-added', blocks, @@ -248,6 +265,96 @@ describe('ServerWorldStateSynchronizer', () => { await expect(pushBlocks(1, 5)).rejects.toThrow(/Test error/i); }); + describe('L1 to L2 message replay', () => { + // Blocks with a strictly growing leaf count, so each one opens a non-empty range over the one before it. + const blockLeafCounts = [3, 3, 7]; + + let messagesByRange: Map; + let originalLeafCounts: number[]; + + afterEach(() => { + const blocks = allBlocks(); + originalLeafCounts.forEach( + (count, i) => (blocks[i].header.state.l1ToL2MessageTree.nextAvailableLeafIndex = count), + ); + }); + + beforeEach(() => { + const blocks = allBlocks(); + originalLeafCounts = blockLeafCounts.map( + (_, i) => blocks[i].header.state.l1ToL2MessageTree.nextAvailableLeafIndex, + ); + blockLeafCounts.forEach((count, i) => (blocks[i].header.state.l1ToL2MessageTree.nextAvailableLeafIndex = count)); + + // Historical replay addresses canonical message indices: resolving a count as a bucket of the current + // partition is exactly what a reorg can make impossible, so it must not happen here. + blockAndMessagesSource.getInboxBucketByTotalMsgCount.mockImplementation(() => { + throw new Error('Published-block replay must not resolve leaf counts as buckets'); + }); + blockAndMessagesSource.getL1ToL2MessagesBetweenBuckets.mockImplementation(() => { + throw new Error('Published-block replay must not read messages by bucket'); + }); + + messagesByRange = new Map([ + ['0-3', [new Fr(10n), new Fr(11n), new Fr(12n)]], + ['3-7', [new Fr(13n), new Fr(14n), new Fr(15n), new Fr(16n)]], + ]); + blockAndMessagesSource.getL1ToL2MessagesBetweenLeafCounts.mockImplementation((start, end) => { + const messages = messagesByRange.get(`${start}-${end}`); + return messages === undefined + ? Promise.reject(new Error(`Unexpected leaf count range [${start}, ${end})`)) + : Promise.resolve(messages); + }); + }); + + it('replays each block from the leaf count range it committed to', async () => { + void server.start(); + await pushBlocks(1, 3); + + const blocks = allBlocks(); + expect(merkleTreeDb.handleL2BlockAndMessages.mock.calls).toEqual([ + [blocks[0], messagesByRange.get('0-3')], + // The second block consumed nothing, so its range is empty and needs no query at all. + [blocks[1], []], + [blocks[2], messagesByRange.get('3-7')], + ]); + expect(blockAndMessagesSource.getL1ToL2MessagesBetweenLeafCounts.mock.calls).toEqual([ + [0n, 3n], + [3n, 7n], + ]); + }); + + it('applies the blocks before an unavailable range and stops there', async () => { + void server.start(); + messagesByRange.delete('3-7'); + + await expect(pushBlocks(1, 3)).rejects.toThrow(/Unexpected leaf count range \[3, 7\)/); + + // The unavailable range fails the sync instead of silently applying the block with no messages; the blocks before + // it were fetched and applied one at a time, so the failure does not discard them. + expect(merkleTreeDb.handleL2BlockAndMessages.mock.calls).toEqual([ + [allBlocks()[0], messagesByRange.get('0-3')], + [allBlocks()[1], []], + ]); + + // Once the archiver serves the range, replay resumes from the failed block, reading its parent's leaf count. + messagesByRange.set('3-7', [new Fr(13n), new Fr(14n), new Fr(15n), new Fr(16n)]); + await pushBlocks(3, 3); + expect(merkleTreeDb.handleL2BlockAndMessages.mock.calls.at(-1)).toEqual([ + allBlocks()[2], + messagesByRange.get('3-7'), + ]); + }); + + it('fails instead of replaying a whole message history when the parent block is missing', async () => { + void server.start(); + blockAndMessagesSource.getBlockData.mockResolvedValue(undefined); + + await expect(pushBlocks(2, 3)).rejects.toThrow(/block 1 is unavailable/); + expect(merkleTreeDb.handleL2BlockAndMessages).not.toHaveBeenCalled(); + }); + }); + describe('getVerifiedSnapshot', () => { let snapshot: MockProxy; diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index eefe75f2a976..4f9ee689be53 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -369,31 +369,22 @@ export class ServerWorldStateSynchronizer private async handleL2Blocks(l2Blocks: L2Block[]) { this.log.debug(`Handling L2 blocks ${l2Blocks[0].number} to ${l2Blocks.at(-1)!.number}`); - // Derive each block's real L1-to-L2 message bundle from the compact leaf-index range it inserted: - // the messages between the parent block's L1-to-L2 tree leaf count and this block's, resolved via the - // Inbox buckets. Blocks in a batch are consecutive, so we track the running leaf count. - const messagesForBlocks = new Map(); + // Each block's real L1-to-L2 message bundle is the compact leaf-index range it inserted: the messages between the + // parent block's L1-to-L2 tree leaf count and this block's. The range addresses canonical message indices, so it + // is served whole even after an L1 reorg merged away the bucket that once ended at the parent's count. Blocks in + // a batch are consecutive, so we track the running leaf count; each block is fetched and applied before the next + // one is read, so a range the archiver cannot serve yet does not discard the blocks already applied. let prevLeafCount = await this.getL1ToL2LeafCountBefore(l2Blocks[0].number); + let updateStatus: WorldStateStatusFull | undefined = undefined; for (const block of l2Blocks) { const blockLeafCount = BigInt(block.header.state.l1ToL2MessageTree.nextAvailableLeafIndex); - if (blockLeafCount > prevLeafCount) { - const startBucket = await this.l2BlockSource.getInboxBucketByTotalMsgCount(prevLeafCount); - const endBucket = await this.l2BlockSource.getInboxBucketByTotalMsgCount(blockLeafCount); - if (startBucket !== undefined && endBucket !== undefined) { - messagesForBlocks.set( - block.number, - await this.l2BlockSource.getL1ToL2MessagesBetweenBuckets(startBucket.seq, endBucket.seq), - ); - } - } + const messages = + blockLeafCount > prevLeafCount + ? await this.l2BlockSource.getL1ToL2MessagesBetweenLeafCounts(prevLeafCount, blockLeafCount) + : []; prevLeafCount = blockLeafCount; - } - let updateStatus: WorldStateStatusFull | undefined = undefined; - for (const block of l2Blocks) { - const [duration, result] = await elapsed(() => - this.handleL2Block(block, messagesForBlocks.get(block.number) ?? []), - ); + const [duration, result] = await elapsed(() => this.handleL2Block(block, messages)); this.log.info(`World state updated with L2 block ${block.number}`, { eventName: 'l2-block-handled', duration, @@ -410,14 +401,21 @@ export class ServerWorldStateSynchronizer this.instrumentation.updateWorldStateMetrics(updateStatus); } - /** The L1-to-L2 message tree leaf count as of the block before `blockNumber` (0 if that block is genesis). */ + /** + * The L1-to-L2 message tree leaf count as of the block before `blockNumber` (0 if that block is genesis). Throws + * when a non-genesis parent is unavailable: treating it as zero would ask for every message ever received as the + * next block's bundle. + */ private async getL1ToL2LeafCountBefore(blockNumber: BlockNumber): Promise { const parentNumber = blockNumber - 1; if (parentNumber < INITIAL_L2_BLOCK_NUM) { return 0n; } const parentBlock = await this.l2BlockSource.getBlockData({ number: BlockNumber(parentNumber) }); - return parentBlock === undefined ? 0n : BigInt(parentBlock.header.state.l1ToL2MessageTree.nextAvailableLeafIndex); + if (parentBlock === undefined) { + throw new Error(`Cannot derive L1 to L2 messages for block ${blockNumber}: block ${parentNumber} is unavailable`); + } + return BigInt(parentBlock.header.state.l1ToL2MessageTree.nextAvailableLeafIndex); } /** diff --git a/yarn-project/world-state/src/test/integration.test.ts b/yarn-project/world-state/src/test/integration.test.ts index b5c29b7b9f31..e87c33bd722e 100644 --- a/yarn-project/world-state/src/test/integration.test.ts +++ b/yarn-project/world-state/src/test/integration.test.ts @@ -1,6 +1,7 @@ import { MockPrefilledArchiver } from '@aztec/archiver/test'; import { GENESIS_ARCHIVE_ROOT } from '@aztec/constants'; import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; +import { Buffer32 } from '@aztec/foundation/buffer'; import { timesAsync } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; @@ -164,6 +165,71 @@ describe('world-state integration', () => { }); }); + describe('Inbox bucket repartitioning', () => { + // Rebuilds the current bucket partition as a single bucket holding every message the given blocks consumed, as an + // L1 reorg that re-mines the same messages under merged boundaries leaves it. The indexed leaves are unchanged, + // but the boundary each published block consumed through is gone from the current partition. + const mergeInboxBuckets = (blockCount: number) => { + const messages = checkpoints.slice(0, blockCount).flatMap(c => c.messages); + archiver.replaceInboxBuckets([ + { + bucket: { + seq: 0n, + inboxRollingHash: Fr.ZERO, + totalMsgCount: 0n, + timestamp: 0n, + msgCount: 0, + lastMessageIndex: 0n, + l1BlockNumber: 0n, + l1BlockHash: Buffer32.ZERO, + }, + msgs: [], + }, + { + bucket: { + seq: 1n, + inboxRollingHash: Fr.ZERO, + totalMsgCount: BigInt(messages.length), + timestamp: 1n, + msgCount: messages.length, + lastMessageIndex: BigInt(messages.length) - 1n, + l1BlockNumber: 1n, + l1BlockHash: Buffer32.ZERO, + }, + msgs: messages, + }, + ]); + }; + + it('replays published blocks whose consumed boundary the current partition no longer has', async () => { + const blockCount = 3; + await archiver.createBlocks(blockCount); + mergeInboxBuckets(blockCount); + + // Driven directly rather than through the block stream, which swallows and retries the archive-root divergence + // a wrong message bundle causes. + const blocks = checkpoints.slice(0, blockCount).flatMap(c => c.checkpoint.blocks); + await synchronizer.handleBlockStreamEvent({ type: 'blocks-added', blocks }); + + for (let blockNumber = 1; blockNumber <= blockCount; blockNumber++) { + await expectSynchedBlockHashMatches(blockNumber); + } + const lastBlockState = blocks.at(-1)!.header.state; + const messageTree = await db.getCommitted().getTreeInfo(MerkleTreeId.L1_TO_L2_MESSAGE_TREE); + expect(messageTree.root).toEqual(lastBlockState.l1ToL2MessageTree.root.toBuffer()); + expect(messageTree.size).toEqual(BigInt(lastBlockState.l1ToL2MessageTree.nextAvailableLeafIndex)); + }); + + it('syncs the whole chain through the block stream under a merged partition', async () => { + const blockCount = 3; + await archiver.createBlocks(blockCount); + mergeInboxBuckets(blockCount); + + await synchronizer.start(); + await expectSynchedToBlock(blockCount); + }); + }); + describe('reorgs', () => { it('prunes blocks upon a reorg and resyncs', async () => { await archiver.createBlocks(5);