diff --git a/src/common/gateway/gateway.service.ts b/src/common/gateway/gateway.service.ts index 80e5bd2fa..456c4a184 100644 --- a/src/common/gateway/gateway.service.ts +++ b/src/common/gateway/gateway.service.ts @@ -13,7 +13,7 @@ import { NetworkStatus } from "./entities/network.status"; import { NftData } from "./entities/nft.data"; import { TokenData } from "./entities/token.data"; import { Transaction } from "./entities/transaction"; -import { forwardRef, Inject, Injectable } from "@nestjs/common"; +import { forwardRef, HttpStatus, Inject, Injectable } from "@nestjs/common"; import { ApiConfigService } from "../api-config/api.config.service"; import { BinaryUtils, ContextTracker } from "@multiversx/sdk-nestjs-common"; import { ApiService, ApiSettings } from "@multiversx/sdk-nestjs-http"; @@ -42,6 +42,11 @@ export class GatewayService { GatewayComponentRequest.vmQuery, ]); + private readonly maxTriePages = 1000; + private readonly maxTrieWalkRestarts = 1; + private readonly maxTrieBusyRetries = 5; + private readonly trieBusyRetryDelayMs = 200; + constructor( private readonly apiConfigService: ApiConfigService, @Inject(forwardRef(() => ApiService)) @@ -117,12 +122,12 @@ export class GatewayService { } async getEsdtFungibleTokens(): Promise { - const result = await this.get('network/esdt/fungible-tokens', GatewayComponentRequest.allFungibleTokens); + const result = await this.getPaged('network/esdt/fungible-tokens', GatewayComponentRequest.allFungibleTokens, 'tokens'); return result.tokens; } async getAddressEsdtRoles(address: string): Promise { - const result = await this.get(`address/${address}/esdts/roles`, GatewayComponentRequest.addressEsdtAllRoles); + const result = await this.getPaged(`address/${address}/esdts/roles`, GatewayComponentRequest.addressEsdtAllRoles, 'roles'); return result; } @@ -228,6 +233,63 @@ export class GatewayService { return result?.data?.data; } + async getPaged(url: string, component: GatewayComponentRequest, itemsField: string, errorHandler?: (error: any) => Promise): Promise { + for (let restart = 0; ; restart++) { + try { + return await this.getAllPages(url, component, itemsField, errorHandler); + } catch (error: any) { + const isIteratorNodeUnavailable = error?.response?.error?.includes('iterator node unavailable'); + if (!isIteratorNodeUnavailable || restart >= this.maxTrieWalkRestarts) { + throw error; + } + } + } + } + + private async getAllPages(url: string, component: GatewayComponentRequest, itemsField: string, errorHandler?: (error: any) => Promise): Promise { + let result: any = undefined; + let pageUrl = url; + + for (let page = 0; page < this.maxTriePages; page++) { + const data = await this.getTriePage(pageUrl, component, errorHandler); + if (!data) { + return result; + } + + const { iteratorState, ...pageData } = data; + if (result === undefined) { + result = pageData; + } else if (Array.isArray(result[itemsField])) { + result[itemsField] = result[itemsField].concat(pageData[itemsField] ?? []); + } else { + result[itemsField] = { ...result[itemsField], ...pageData[itemsField] }; + } + + if (!iteratorState) { + return result; + } + + const encodedIteratorState = Buffer.from(JSON.stringify(iteratorState)).toString('base64url'); + pageUrl = `${url}${url.includes('?') ? '&' : '?'}iteratorState=${encodedIteratorState}`; + } + + throw new Error(`Trie walk for '${url}' exceeded ${this.maxTriePages} pages`); + } + + private async getTriePage(url: string, component: GatewayComponentRequest, errorHandler?: (error: any) => Promise): Promise { + for (let retry = 0; ; retry++) { + try { + return await this.get(url, component, errorHandler); + } catch (error: any) { + if (error?.status !== HttpStatus.TOO_MANY_REQUESTS || retry >= this.maxTrieBusyRetries) { + throw error; + } + + await new Promise(resolve => setTimeout(resolve, this.trieBusyRetryDelayMs * (retry + 1))); + } + } + } + @LogPerformanceAsync(MetricsEvents.SetGatewayDuration, { argIndex: 1 }) async getRaw(url: string, component: GatewayComponentRequest, errorHandler?: (error: any) => Promise): Promise { const fullUrl = this.getFullUrl(component, url); diff --git a/src/endpoints/esdt/esdt.address.service.ts b/src/endpoints/esdt/esdt.address.service.ts index 39952b24f..e78dc60ae 100644 --- a/src/endpoints/esdt/esdt.address.service.ts +++ b/src/endpoints/esdt/esdt.address.service.ts @@ -338,7 +338,7 @@ export class EsdtAddressService { private async getAllEsdtsForAddressFromGatewayRaw(address: string): Promise<{ [key: string]: any }> { // eslint-disable-next-line require-await - const esdtResult = await this.gatewayService.get(`address/${address}/esdt`, GatewayComponentRequest.addressEsdt, async (error) => { + const esdtResult = await this.gatewayService.getPaged(`address/${address}/esdt`, GatewayComponentRequest.addressEsdt, 'esdts', async (error) => { const errorMessage = error?.response?.data?.error; if (errorMessage) { if (errorMessage.includes('account was not found')) { diff --git a/src/endpoints/proxy/gateway.proxy.controller.ts b/src/endpoints/proxy/gateway.proxy.controller.ts index ed8e53da0..3fd7b5cb3 100644 --- a/src/endpoints/proxy/gateway.proxy.controller.ts +++ b/src/endpoints/proxy/gateway.proxy.controller.ts @@ -47,9 +47,13 @@ export class GatewayProxyController { } @Get('/address/:address/keys') - async getAddressKeys(@Param('address', ParseAddressPipe) address: string) { + async getAddressKeys( + @Param('address', ParseAddressPipe) address: string, + @Query('numKeys', ParseIntPipe) numKeys?: number, + @Query('iteratorState') iteratorState?: string, + ) { try { - return await this.gatewayGet(`address/${address}/keys`, GatewayComponentRequest.addressKeys); + return await this.gatewayGet(`address/${address}/keys`, GatewayComponentRequest.addressKeys, this.getTriePageParams(numKeys, iteratorState)); } catch (error: any) { this.logger.error(`Error fetching address keys for address ${address}: ${error.message}`); throw new BadRequestException(error.response?.data || 'An error occurred while fetching address keys'); @@ -79,9 +83,13 @@ export class GatewayProxyController { } @Get('/address/:address/esdt') - async getAddressEsdt(@Param('address', ParseAddressPipe) address: string) { + async getAddressEsdt( + @Param('address', ParseAddressPipe) address: string, + @Query('numKeys', ParseIntPipe) numKeys?: number, + @Query('iteratorState') iteratorState?: string, + ) { // eslint-disable-next-line require-await - return await this.gatewayGet(`address/${address}/esdt`, GatewayComponentRequest.addressDetails, undefined, async (error) => { + return await this.gatewayGet(`address/${address}/esdt`, GatewayComponentRequest.addressDetails, this.getTriePageParams(numKeys, iteratorState), async (error) => { const message = error.response?.data?.error; if (message && message.includes('account was not found')) { throw error; @@ -377,6 +385,17 @@ export class GatewayProxyController { return await this.gatewayGet('network/gas-configs', GatewayComponentRequest.gasConfigs); } + private getTriePageParams(numKeys?: number, iteratorState?: string): Record | undefined { + if (numKeys === undefined && iteratorState === undefined) { + return undefined; + } + + return { + numKeys, + iteratorState: iteratorState !== undefined ? encodeURIComponent(iteratorState) : undefined, + }; + } + private async gatewayGet(url: string, component: GatewayComponentRequest, params: any = undefined, errorHandler?: (error: any) => Promise) { if (params) { url += '?' + Object.keys(params).filter(key => params[key] !== undefined).map(key => `${key}=${params[key]}`).join('&'); diff --git a/src/test/unit/services/esdt.address.spec.ts b/src/test/unit/services/esdt.address.spec.ts index 7db2e1f05..e5003c3bc 100644 --- a/src/test/unit/services/esdt.address.spec.ts +++ b/src/test/unit/services/esdt.address.spec.ts @@ -48,6 +48,7 @@ describe('EsdtAddressService', () => { provide: GatewayService, useValue: { get: jest.fn(), + getPaged: jest.fn(), getAddressEsdtRoles: jest.fn(), }, }, diff --git a/src/test/unit/services/gateway.paged.spec.ts b/src/test/unit/services/gateway.paged.spec.ts new file mode 100644 index 000000000..fa63cbdf2 --- /dev/null +++ b/src/test/unit/services/gateway.paged.spec.ts @@ -0,0 +1,130 @@ +import { GatewayService } from "src/common/gateway/gateway.service"; +import { GatewayComponentRequest } from "src/common/gateway/entities/gateway.component.request"; + +describe('GatewayService paged trie requests', () => { + let gatewayService: GatewayService; + let apiService: any; + + const firstCheckpoint = { blockNonce: 90, blockHash: 'ab', blockRootHash: 'cd', epoch: 4, iteratorState: ['Dt/+AA=='] }; + const secondCheckpoint = { blockNonce: 90, blockHash: 'ab', blockRootHash: 'cd', epoch: 4, iteratorState: ['Ee/+BB=='] }; + const blockInfo = { nonce: 90, hash: 'ab', rootHash: 'cd' }; + + const firstEncoded = Buffer.from(JSON.stringify(firstCheckpoint)).toString('base64url'); + const secondEncoded = Buffer.from(JSON.stringify(secondCheckpoint)).toString('base64url'); + + beforeEach(() => { + apiService = { get: jest.fn() }; + + gatewayService = new GatewayService( + { getGatewayUrl: () => 'https://gateway', getSnapshotlessGatewayUrl: () => undefined } as any, + apiService, + ); + Object.assign(gatewayService, { eventEmitter: { emit: jest.fn() }, trieBusyRetryDelayMs: 0 }); + }); + + it('should merge object items of every page and drop the checkpoint', async () => { + apiService.get + .mockResolvedValueOnce({ data: { data: { esdts: { 'A-1': { balance: '1' } }, blockInfo, iteratorState: firstCheckpoint } } }) + .mockResolvedValueOnce({ data: { data: { esdts: {}, blockInfo, iteratorState: secondCheckpoint } } }) + .mockResolvedValueOnce({ data: { data: { esdts: { 'B-2': { balance: '2' } }, blockInfo } } }); + + const result = await gatewayService.getPaged('address/erd1/esdt', GatewayComponentRequest.addressEsdt, 'esdts'); + + expect(result).toEqual({ esdts: { 'A-1': { balance: '1' }, 'B-2': { balance: '2' } }, blockInfo }); + expect(apiService.get).toHaveBeenNthCalledWith(1, 'https://gateway/address/erd1/esdt', expect.anything(), undefined); + expect(apiService.get).toHaveBeenNthCalledWith(2, `https://gateway/address/erd1/esdt?iteratorState=${firstEncoded}`, expect.anything(), undefined); + expect(apiService.get).toHaveBeenNthCalledWith(3, `https://gateway/address/erd1/esdt?iteratorState=${secondEncoded}`, expect.anything(), undefined); + }); + + it('should send the checkpoint as unpadded base64url', async () => { + apiService.get + .mockResolvedValueOnce({ data: { data: { tokens: ['A-1'], iteratorState: firstCheckpoint } } }) + .mockResolvedValueOnce({ data: { data: { tokens: ['B-2'] } } }); + + await gatewayService.getPaged('network/esdt/fungible-tokens', GatewayComponentRequest.allFungibleTokens, 'tokens'); + + const continuationUrl: string = apiService.get.mock.calls[1][0]; + const encoded = continuationUrl.split('iteratorState=')[1]; + expect(encoded).toMatch(/^[A-Za-z0-9_-]+$/); + expect(JSON.parse(Buffer.from(encoded, 'base64url').toString())).toEqual(firstCheckpoint); + }); + + it('should concatenate list items of every page', async () => { + apiService.get + .mockResolvedValueOnce({ data: { data: { tokens: ['A-1', 'B-2'], iteratorState: firstCheckpoint } } }) + .mockResolvedValueOnce({ data: { data: { tokens: ['C-3'] } } }); + + expect(await gatewayService.getEsdtFungibleTokens()).toEqual(['A-1', 'B-2', 'C-3']); + }); + + it('should append the checkpoint to an url that already has a query', async () => { + apiService.get + .mockResolvedValueOnce({ data: { data: { pairs: { aa: '01' }, iteratorState: firstCheckpoint } } }) + .mockResolvedValueOnce({ data: { data: { pairs: { bb: '02' } } } }); + + await gatewayService.getPaged('address/erd1/keys?numKeys=1', GatewayComponentRequest.addressKeys, 'pairs'); + + expect(apiService.get).toHaveBeenNthCalledWith(2, `https://gateway/address/erd1/keys?numKeys=1&iteratorState=${firstEncoded}`, expect.anything(), undefined); + }); + + it('should return undefined when the first page is handled by the error handler', async () => { + apiService.get.mockResolvedValueOnce(undefined); + + expect(await gatewayService.getPaged('address/erd1/esdt', GatewayComponentRequest.addressEsdt, 'esdts')).toBeUndefined(); + }); + + it('should retry a page answered with system busy', async () => { + apiService.get + .mockResolvedValueOnce({ data: { data: { tokens: ['A-1'], iteratorState: firstCheckpoint } } }) + .mockRejectedValueOnce({ status: 429, response: { error: 'too many concurrent trie operations', code: 'system_busy' } }) + .mockResolvedValueOnce({ data: { data: { tokens: ['B-2'] } } }); + + expect(await gatewayService.getEsdtFungibleTokens()).toEqual(['A-1', 'B-2']); + expect(apiService.get).toHaveBeenNthCalledWith(3, `https://gateway/network/esdt/fungible-tokens?iteratorState=${firstEncoded}`, expect.anything(), undefined); + }); + + it('should give up after the busy retries are exhausted', async () => { + const busy = { status: 429, response: { error: 'too many concurrent trie operations', code: 'system_busy' } }; + apiService.get.mockRejectedValue(busy); + + await expect(gatewayService.getEsdtFungibleTokens()).rejects.toEqual(busy); + expect(apiService.get).toHaveBeenCalledTimes(6); + }); + + it('should restart the walk once when the checkpoint expired', async () => { + apiService.get + .mockResolvedValueOnce({ data: { data: { tokens: ['A-1'], iteratorState: firstCheckpoint } } }) + .mockRejectedValueOnce({ status: 400, response: { error: 'get esdt tokens error: iterator node unavailable: ab', code: 'bad_request' } }) + .mockResolvedValueOnce({ data: { data: { tokens: ['A-1', 'B-2'] } } }); + + expect(await gatewayService.getEsdtFungibleTokens()).toEqual(['A-1', 'B-2']); + expect(apiService.get).toHaveBeenNthCalledWith(3, 'https://gateway/network/esdt/fungible-tokens', expect.anything(), undefined); + }); + + it('should not restart the walk more than once', async () => { + const expired = { status: 400, response: { error: 'iterator node unavailable: ab', code: 'bad_request' } }; + apiService.get + .mockResolvedValueOnce({ data: { data: { tokens: ['A-1'], iteratorState: firstCheckpoint } } }) + .mockRejectedValueOnce(expired) + .mockResolvedValueOnce({ data: { data: { tokens: ['A-1'], iteratorState: firstCheckpoint } } }) + .mockRejectedValueOnce(expired); + + await expect(gatewayService.getEsdtFungibleTokens()).rejects.toEqual(expired); + }); + + it('should not retry other failures', async () => { + const failure = { status: 500, response: { error: 'the endpoint is only available on metachain nodes', code: 'internal_issue' } }; + apiService.get.mockRejectedValue(failure); + + await expect(gatewayService.getEsdtFungibleTokens()).rejects.toEqual(failure); + expect(apiService.get).toHaveBeenCalledTimes(1); + }); + + it('should stop a walk that never ends', async () => { + Object.assign(gatewayService, { maxTriePages: 3 }); + apiService.get.mockResolvedValue({ data: { data: { tokens: ['A-1'], iteratorState: firstCheckpoint } } }); + + await expect(gatewayService.getEsdtFungibleTokens()).rejects.toThrow("Trie walk for 'network/esdt/fungible-tokens' exceeded 3 pages"); + expect(apiService.get).toHaveBeenCalledTimes(3); + }); +});