diff --git a/contracts/docs/ACTIONS.md b/contracts/docs/ACTIONS.md index 2ad12f90a9..739465eacf 100644 --- a/contracts/docs/ACTIONS.md +++ b/contracts/docs/ACTIONS.md @@ -286,6 +286,8 @@ Every run (every 2 minutes): - `--min-token1 ` (default `500`): USDe held that is worth a pull. - `--target-ousd ` (default `50000`), `--reset-amount ` (default `100`). - `--token-id ` (default `457273`), `--max-gas`, `--module`, `--dryrun`. +- `--lookback-blocks ` (default `500000`), `--log-chunk ` (default + `9999`, capped there): the event search, retried on transient RPC errors. Bought USDe stays in the Safe until it is redeemed at Ethena and minted back into OUSD, which is not automated yet. Redeem only while the quote is @@ -328,7 +330,7 @@ never lets the swap or the new range sell OUSD more than `navToleranceBps` - `--reset-amount ` (default `100`): max OUSD the reset swap sells, capped by the module's `maxAmount0`. - `--lookback-blocks ` (default `500000`), `--log-chunk ` (default - `50000`): the event search. + `9999`, capped there): the event search, retried on transient RPC errors. - `--max-gas ` (default `5000000`): skip a rebalance estimated above this. - `--module
`: override the deployment descriptor (forks). - `--dryrun` simulates and logs without sending. A non-operator signer diff --git a/contracts/tasks/actions/ousdSusdeV4Rebalance.ts b/contracts/tasks/actions/ousdSusdeV4Rebalance.ts index 0822e1c703..04b5d8ba30 100644 --- a/contracts/tasks/actions/ousdSusdeV4Rebalance.ts +++ b/contracts/tasks/actions/ousdSusdeV4Rebalance.ts @@ -2,6 +2,7 @@ import { BigNumber, ethers } from "ethers"; import { formatUnits, parseUnits } from "ethers/lib/utils"; import { action, types, Logger } from "../lib/action"; import { getOrNull } from "../lib/deployments"; +import { findLatestLog, MAX_LOG_CHUNK } from "../lib/logs"; import { getSqrtPriceAtTick } from "../lib/v4TickMath"; import { logTxDetails } from "../../utils/txLogger"; @@ -207,7 +208,7 @@ export function unpackTicks(info: bigint): [number, number] { /** * The position the module minted last: the `mintedTokenId` of its latest * `Rebalanced` event (0 after a burn-only call), searched backwards in - * chunks; `fallback` when the module never rebalanced. + * chunks (see `findLatestLog`); `fallback` when the module never rebalanced. */ async function currentTokenId({ module, @@ -227,17 +228,19 @@ async function currentTokenId({ const latest = await provider.getBlockNumber(); const floor = Math.max(0, latest - lookback); const filter = module.filters.Rebalanced(); - for (let to = latest; to >= floor; to -= chunk) { - const from = Math.max(floor, to - chunk + 1); - const events = await module.queryFilter(filter, from, to); - if (events.length > 0) { - const last = events[events.length - 1]; - const minted = BigInt(last.args!.mintedTokenId.toString()); - log.info( - `Latest Rebalanced at block ${last.blockNumber}: minted ${minted}` - ); - return minted; - } + const last = await findLatestLog({ + query: (from, to) => module.queryFilter(filter, from, to), + latest, + floor, + chunk, + onRetry: (msg) => log.warn(msg), + }); + if (last) { + const minted = BigInt(last.args!.mintedTokenId.toString()); + log.info( + `Latest Rebalanced at block ${last.blockNumber}: minted ${minted}` + ); + return minted; } log.info( `No Rebalanced event in the last ${lookback} blocks; using --token-id ${fallback}` @@ -295,8 +298,8 @@ action({ ); t.addOptionalParam( "logChunk", - "Block range per eth_getLogs call", - 50_000, + "Block range per eth_getLogs call (capped at 9999)", + MAX_LOG_CHUNK, types.int ); t.addOptionalParam( diff --git a/contracts/tasks/actions/ousdUsdeV4Rebalance.ts b/contracts/tasks/actions/ousdUsdeV4Rebalance.ts index 15808879b9..752b7e0de6 100644 --- a/contracts/tasks/actions/ousdUsdeV4Rebalance.ts +++ b/contracts/tasks/actions/ousdUsdeV4Rebalance.ts @@ -2,6 +2,7 @@ import { BigNumber, ethers } from "ethers"; import { formatUnits, parseUnits } from "ethers/lib/utils"; import { action, types, Logger } from "../lib/action"; import { getOrNull } from "../lib/deployments"; +import { findLatestLog, MAX_LOG_CHUNK } from "../lib/logs"; import { bidToTickLower, curveRate, @@ -285,7 +286,7 @@ export function resetAmount({ /** * The position the module minted last on `poolId`: the `mintedTokenId` of * its latest `Rebalanced` event (0 after a burn-only call), searched - * backwards in chunks; `fallback` when the module never rebalanced. + * backwards in chunks (see `findLatestLog`); `fallback` when the module never rebalanced. */ async function currentTokenId({ module, @@ -307,17 +308,19 @@ async function currentTokenId({ const latest = await provider.getBlockNumber(); const floor = Math.max(0, latest - lookback); const filter = module.filters.Rebalanced(poolId); - for (let to = latest; to >= floor; to -= chunk) { - const from = Math.max(floor, to - chunk + 1); - const events = await module.queryFilter(filter, from, to); - if (events.length > 0) { - const last = events[events.length - 1]; - const minted = BigInt(last.args!.mintedTokenId.toString()); - log.info( - `Latest Rebalanced at block ${last.blockNumber}: minted ${minted}` - ); - return minted; - } + const last = await findLatestLog({ + query: (from, to) => module.queryFilter(filter, from, to), + latest, + floor, + chunk, + onRetry: (msg) => log.warn(msg), + }); + if (last) { + const minted = BigInt(last.args!.mintedTokenId.toString()); + log.info( + `Latest Rebalanced at block ${last.blockNumber}: minted ${minted}` + ); + return minted; } log.info( `No Rebalanced event in the last ${lookback} blocks; using --token-id ${fallback}` @@ -481,8 +484,8 @@ action({ ); t.addOptionalParam( "logChunk", - "Block range per eth_getLogs call", - 50_000, + "Block range per eth_getLogs call (capped at 9999)", + MAX_LOG_CHUNK, types.int ); t.addOptionalParam( diff --git a/contracts/tasks/lib/logs.ts b/contracts/tasks/lib/logs.ts new file mode 100644 index 0000000000..0990c64ca4 --- /dev/null +++ b/contracts/tasks/lib/logs.ts @@ -0,0 +1,122 @@ +/** + * Backwards `eth_getLogs` search that survives a load-balanced RPC (dRPC + * routes each call to one of several upstreams). + * + * - Chunks span at most `MAX_LOG_CHUNK` blocks: some upstreams reject a + * wider range ("query block range exceeds server limit ... 10000"). + * - The newest `headLag` blocks are queried on their own, so only that small + * range can hit an upstream that lags the head ("block range extends beyond + * current head block"). It is retried, not skipped: an event there is our + * own previous run, and missing it would read a burned position. + * - Every call is retried with a backoff on transient errors (HTTP 5xx, + * "Temporary internal error", head lag). A range-limit error halves the + * chunk instead of counting as a failure. + */ + +/** Widest range an upstream accepts: `to - from + 1 <= 10000`. */ +export const MAX_LOG_CHUNK = 9_999; + +const RANGE_LIMIT = + /range (exceeds|is too (large|wide))|block range|narrow your filter|too many (blocks|results)/i; +const TRANSIENT = + /temporary|try again|retry|timeout|timed out|rate limit|too many requests|beyond current head|header not found|unknown block|server error|bad gateway|service unavailable|gateway time|ECONNRESET|ETIMEDOUT|socket hang up/i; + +function errorText(err: unknown): string { + const e = err as { + message?: string; + body?: string; + code?: string | number; + status?: number; + error?: unknown; + }; + const parts = [ + e?.message, + e?.body, + String(e?.code ?? ""), + String(e?.status ?? ""), + ]; + if (e?.error && e.error !== err) parts.push(errorText(e.error)); + return parts.filter(Boolean).join(" "); +} + +/** "query block range exceeds server limit, narrow your filter: 10000". */ +export function isRangeLimitError(err: unknown): boolean { + const text = errorText(err); + return RANGE_LIMIT.test(text) && !/beyond current head/i.test(text); +} + +/** HTTP 5xx, JSON-RPC code 19, lagging upstream, network blips. */ +export function isTransientLogError(err: unknown): boolean { + const text = errorText(err); + return ( + TRANSIENT.test(text) || + /SERVER_ERROR|TIMEOUT|NETWORK_ERROR/.test(text) || + /\b5\d\d\b/.test(text) + ); +} + +export interface LatestLogOptions { + /** One `eth_getLogs` over `[from, to]`, oldest first. */ + query: (from: number, to: number) => Promise; + /** Newest block to search. */ + latest: number; + /** Oldest block to search. */ + floor: number; + /** Blocks per call, capped at `MAX_LOG_CHUNK`. */ + chunk?: number; + /** Newest blocks queried on their own (lagging upstreams). */ + headLag?: number; + /** Attempts per range before giving up. */ + attempts?: number; + /** First backoff; doubles on each retry. */ + backoffMs?: number; + sleep?: (ms: number) => Promise; + onRetry?: (msg: string) => void; +} + +/** The newest log in `[floor, latest]`, or undefined when there is none. */ +export async function findLatestLog({ + query, + latest, + floor, + chunk = MAX_LOG_CHUNK, + headLag = 5, + attempts = 3, + backoffMs = 500, + sleep = (ms) => new Promise((r) => setTimeout(r, ms)), + onRetry = () => {}, +}: LatestLogOptions): Promise { + let size = Math.max(1, Math.min(chunk, MAX_LOG_CHUNK)); + let to = latest; + let first = true; + while (to >= floor) { + const width = first ? Math.max(1, Math.min(headLag, size)) : size; + const from = Math.max(floor, to - width + 1); + let logs: T[] | undefined; + for (let attempt = 1; logs === undefined; attempt++) { + try { + logs = await query(from, to); + } catch (err) { + if (isRangeLimitError(err) && to > from) { + size = Math.max(1, Math.floor((to - from + 1) / 2)); + onRetry(`getLogs [${from}, ${to}] too wide; chunk now ${size}`); + break; + } + if (attempt >= attempts || !isTransientLogError(err)) throw err; + const wait = backoffMs * 2 ** (attempt - 1); + onRetry( + `getLogs [${from}, ${to}] failed (attempt ${attempt}/${attempts}), retrying in ${wait}ms: ${ + (err as Error)?.message ?? err + }` + ); + await sleep(wait); + } + } + // Range-limit error: retry the same `to` with the halved chunk. + if (logs === undefined) continue; + if (logs.length > 0) return logs[logs.length - 1]; + first = false; + to = from - 1; + } + return undefined; +} diff --git a/contracts/tasks/test/logs.js b/contracts/tasks/test/logs.js new file mode 100644 index 0000000000..d33c4da493 --- /dev/null +++ b/contracts/tasks/test/logs.js @@ -0,0 +1,194 @@ +const { expect } = require("chai"); + +const { + findLatestLog, + isRangeLimitError, + isTransientLogError, + MAX_LOG_CHUNK, +} = require("../lib/logs"); + +// The three errors dRPC returned to ousdUsdeV4Rebalance in Oct 2026, shaped +// like ethers v5 wraps them. +const rpcError = (message, code) => { + const err = new Error( + `processing response error (body="{\\"jsonrpc\\":\\"2.0\\",\\"id\\":42,\\"error\\":{\\"code\\":${code},\\"message\\":\\"${message}\\"}}", code=SERVER_ERROR)` + ); + err.code = "SERVER_ERROR"; + err.error = { code, message }; + return err; +}; +const temporary = () => rpcError("Temporary internal error. Please retry", 19); +const rangeLimit = () => + rpcError( + "query block range exceeds server limit, narrow your filter: 10000", + -32602 + ); +const beyondHead = () => + rpcError("block range extends beyond current head block", -32000); + +// A fake getLogs: `events` maps block -> log; `failures` is consumed one +// error per call; `maxRange` rejects wider queries like a capped upstream. +function fakeQuery({ events = {}, failures = [], maxRange = Infinity } = {}) { + const calls = []; + const query = async (from, to) => { + calls.push([from, to]); + if (failures.length > 0) throw failures.shift()(); + if (to - from + 1 > maxRange) throw rangeLimit(); + return Object.keys(events) + .map(Number) + .filter((b) => b >= from && b <= to) + .sort((a, b) => a - b) + .map((b) => ({ blockNumber: b, minted: events[b] })); + }; + return { query, calls }; +} + +const noSleep = async () => {}; + +describe("Unit: findLatestLog", () => { + it("classifies the dRPC errors", () => { + expect(isTransientLogError(temporary())).to.equal(true); + expect(isRangeLimitError(temporary())).to.equal(false); + expect(isRangeLimitError(rangeLimit())).to.equal(true); + expect(isRangeLimitError(beyondHead())).to.equal(false); + expect(isTransientLogError(beyondHead())).to.equal(true); + expect(isTransientLogError(new Error("execution reverted"))).to.equal( + false + ); + }); + + it("caps chunks so every query spans at most 10000 blocks", async () => { + const { query, calls } = fakeQuery(); + const found = await findLatestLog({ + query, + latest: 1000000, + floor: 950000, + chunk: 50000, + sleep: noSleep, + }); + expect(found).to.equal(undefined); + expect(MAX_LOG_CHUNK).to.equal(9999); + for (const [from, to] of calls) expect(to - from + 1).to.be.at.most(10000); + // Contiguous, newest first, down to the floor. + expect(calls[0]).to.deep.equal([999996, 1000000]); + for (let i = 1; i < calls.length; i++) { + expect(calls[i][1]).to.equal(calls[i - 1][0] - 1); + } + expect(calls[calls.length - 1][0]).to.equal(950000); + }); + + it("queries the head on its own and still finds an event there", async () => { + const { query, calls } = fakeQuery({ + events: { 900000: BigInt(1), 999999: BigInt(2) }, + }); + const found = await findLatestLog({ + query, + latest: 1000000, + floor: 500000, + sleep: noSleep, + }); + expect(found.minted).to.equal(BigInt(2)); + expect(calls).to.deep.equal([[999996, 1000000]]); + }); + + it("returns the newest event of the first non-empty chunk", async () => { + const { query } = fakeQuery({ + events: { 980000: BigInt(1), 985000: BigInt(2), 900000: BigInt(0) }, + }); + const found = await findLatestLog({ + query, + latest: 1000000, + floor: 500000, + sleep: noSleep, + }); + expect(found).to.deep.equal({ blockNumber: 985000, minted: BigInt(2) }); + }); + + it("retries a transient error, then succeeds", async () => { + const { query, calls } = fakeQuery({ + events: { 990000: BigInt(7) }, + failures: [temporary, beyondHead], + }); + const waits = []; + const found = await findLatestLog({ + query, + latest: 1000000, + floor: 500000, + backoffMs: 100, + sleep: async (ms) => waits.push(ms), + }); + expect(found.minted).to.equal(BigInt(7)); + expect(waits).to.deep.equal([100, 200]); + // Same head range three times, then the next chunk. + expect(calls.slice(0, 3)).to.deep.equal([ + [999996, 1000000], + [999996, 1000000], + [999996, 1000000], + ]); + }); + + it("halves the chunk on a range-limit error", async () => { + const { query, calls } = fakeQuery({ + events: { 970000: BigInt(3) }, + maxRange: 2000, + }); + const found = await findLatestLog({ + query, + latest: 1000000, + floor: 500000, + attempts: 1, + sleep: noSleep, + }); + expect(found.minted).to.equal(BigInt(3)); + // 9999 -> 4999 -> 2499 -> 1249 blocks, all from the same `to`. + expect(calls.slice(1, 5)).to.deep.equal([ + [989997, 999995], + [994997, 999995], + [997497, 999995], + [998747, 999995], + ]); + for (const [from, to] of calls.slice(4)) { + expect(to - from + 1).to.be.at.most(1249); + } + }); + + it("throws once the retries are exhausted", async () => { + const { query, calls } = fakeQuery({ + failures: [temporary, temporary, temporary, temporary], + }); + let err; + try { + await findLatestLog({ + query, + latest: 1000000, + floor: 500000, + attempts: 3, + sleep: noSleep, + }); + } catch (e) { + err = e; + } + expect(err).to.be.an("error"); + expect(err.message).to.contain("Temporary internal error"); + expect(calls).to.have.length(3); + }); + + it("does not retry a non-transient error", async () => { + const { query, calls } = fakeQuery({ + failures: [() => new Error("execution reverted")], + }); + let err; + try { + await findLatestLog({ + query, + latest: 1000000, + floor: 500000, + sleep: noSleep, + }); + } catch (e) { + err = e; + } + expect(err.message).to.equal("execution reverted"); + expect(calls).to.have.length(1); + }); +});