Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 65 additions & 3 deletions src/common/gateway/gateway.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import { NetworkStatus } from "./entities/network.status";
import { NftData } from "./entities/nft.data";
import { TokenData } from "./entities/token.data";
import { Transaction } from "./entities/transaction";
import { forwardRef, Inject, Injectable } from "@nestjs/common";
import { forwardRef, HttpStatus, Inject, Injectable } from "@nestjs/common";
import { ApiConfigService } from "../api-config/api.config.service";
import { BinaryUtils, ContextTracker } from "@multiversx/sdk-nestjs-common";
import { ApiService, ApiSettings } from "@multiversx/sdk-nestjs-http";
Expand Down Expand Up @@ -42,6 +42,11 @@ export class GatewayService {
GatewayComponentRequest.vmQuery,
]);

private readonly maxTriePages = 1000;
private readonly maxTrieWalkRestarts = 1;
private readonly maxTrieBusyRetries = 5;
private readonly trieBusyRetryDelayMs = 200;

constructor(
private readonly apiConfigService: ApiConfigService,
@Inject(forwardRef(() => ApiService))
Expand Down Expand Up @@ -117,12 +122,12 @@ export class GatewayService {
}

async getEsdtFungibleTokens(): Promise<string[]> {
const result = await this.get('network/esdt/fungible-tokens', GatewayComponentRequest.allFungibleTokens);
const result = await this.getPaged('network/esdt/fungible-tokens', GatewayComponentRequest.allFungibleTokens, 'tokens');
return result.tokens;
}

async getAddressEsdtRoles(address: string): Promise<EsdtAddressRoles> {
const result = await this.get(`address/${address}/esdts/roles`, GatewayComponentRequest.addressEsdtAllRoles);
const result = await this.getPaged(`address/${address}/esdts/roles`, GatewayComponentRequest.addressEsdtAllRoles, 'roles');
return result;
}

Expand Down Expand Up @@ -228,6 +233,63 @@ export class GatewayService {
return result?.data?.data;
}

async getPaged(url: string, component: GatewayComponentRequest, itemsField: string, errorHandler?: (error: any) => Promise<boolean>): Promise<any> {
for (let restart = 0; ; restart++) {
try {
return await this.getAllPages(url, component, itemsField, errorHandler);
} catch (error: any) {
const isIteratorNodeUnavailable = error?.response?.error?.includes('iterator node unavailable');
if (!isIteratorNodeUnavailable || restart >= this.maxTrieWalkRestarts) {
throw error;
}
}
}
}

private async getAllPages(url: string, component: GatewayComponentRequest, itemsField: string, errorHandler?: (error: any) => Promise<boolean>): Promise<any> {
let result: any = undefined;
let pageUrl = url;

for (let page = 0; page < this.maxTriePages; page++) {
const data = await this.getTriePage(pageUrl, component, errorHandler);
if (!data) {
return result;
}

const { iteratorState, ...pageData } = data;
if (result === undefined) {
result = pageData;
} else if (Array.isArray(result[itemsField])) {
result[itemsField] = result[itemsField].concat(pageData[itemsField] ?? []);
} else {
result[itemsField] = { ...result[itemsField], ...pageData[itemsField] };
}

if (!iteratorState) {
return result;
}

const encodedIteratorState = Buffer.from(JSON.stringify(iteratorState)).toString('base64url');
pageUrl = `${url}${url.includes('?') ? '&' : '?'}iteratorState=${encodedIteratorState}`;
}

throw new Error(`Trie walk for '${url}' exceeded ${this.maxTriePages} pages`);
}

private async getTriePage(url: string, component: GatewayComponentRequest, errorHandler?: (error: any) => Promise<boolean>): Promise<any> {
for (let retry = 0; ; retry++) {
try {
return await this.get(url, component, errorHandler);
} catch (error: any) {
if (error?.status !== HttpStatus.TOO_MANY_REQUESTS || retry >= this.maxTrieBusyRetries) {
throw error;
}

await new Promise(resolve => setTimeout(resolve, this.trieBusyRetryDelayMs * (retry + 1)));
}
}
}

@LogPerformanceAsync(MetricsEvents.SetGatewayDuration, { argIndex: 1 })
async getRaw(url: string, component: GatewayComponentRequest, errorHandler?: (error: any) => Promise<boolean>): Promise<any> {
const fullUrl = this.getFullUrl(component, url);
Expand Down
2 changes: 1 addition & 1 deletion src/endpoints/esdt/esdt.address.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,7 @@ export class EsdtAddressService {

private async getAllEsdtsForAddressFromGatewayRaw(address: string): Promise<{ [key: string]: any }> {
// eslint-disable-next-line require-await
const esdtResult = await this.gatewayService.get(`address/${address}/esdt`, GatewayComponentRequest.addressEsdt, async (error) => {
const esdtResult = await this.gatewayService.getPaged(`address/${address}/esdt`, GatewayComponentRequest.addressEsdt, 'esdts', async (error) => {
const errorMessage = error?.response?.data?.error;
if (errorMessage) {
if (errorMessage.includes('account was not found')) {
Expand Down
27 changes: 23 additions & 4 deletions src/endpoints/proxy/gateway.proxy.controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,13 @@ export class GatewayProxyController {
}

@Get('/address/:address/keys')
async getAddressKeys(@Param('address', ParseAddressPipe) address: string) {
async getAddressKeys(
@Param('address', ParseAddressPipe) address: string,
@Query('numKeys', ParseIntPipe) numKeys?: number,
@Query('iteratorState') iteratorState?: string,
) {
try {
return await this.gatewayGet(`address/${address}/keys`, GatewayComponentRequest.addressKeys);
return await this.gatewayGet(`address/${address}/keys`, GatewayComponentRequest.addressKeys, this.getTriePageParams(numKeys, iteratorState));
} catch (error: any) {
this.logger.error(`Error fetching address keys for address ${address}: ${error.message}`);
throw new BadRequestException(error.response?.data || 'An error occurred while fetching address keys');
Expand Down Expand Up @@ -79,9 +83,13 @@ export class GatewayProxyController {
}

@Get('/address/:address/esdt')
async getAddressEsdt(@Param('address', ParseAddressPipe) address: string) {
async getAddressEsdt(
@Param('address', ParseAddressPipe) address: string,
@Query('numKeys', ParseIntPipe) numKeys?: number,
@Query('iteratorState') iteratorState?: string,
) {
// eslint-disable-next-line require-await
return await this.gatewayGet(`address/${address}/esdt`, GatewayComponentRequest.addressDetails, undefined, async (error) => {
return await this.gatewayGet(`address/${address}/esdt`, GatewayComponentRequest.addressDetails, this.getTriePageParams(numKeys, iteratorState), async (error) => {
const message = error.response?.data?.error;
if (message && message.includes('account was not found')) {
throw error;
Expand Down Expand Up @@ -377,6 +385,17 @@ export class GatewayProxyController {
return await this.gatewayGet('network/gas-configs', GatewayComponentRequest.gasConfigs);
}

private getTriePageParams(numKeys?: number, iteratorState?: string): Record<string, string | number | undefined> | undefined {
if (numKeys === undefined && iteratorState === undefined) {
return undefined;
}

return {
numKeys,
iteratorState: iteratorState !== undefined ? encodeURIComponent(iteratorState) : undefined,
};
}

private async gatewayGet(url: string, component: GatewayComponentRequest, params: any = undefined, errorHandler?: (error: any) => Promise<boolean>) {
if (params) {
url += '?' + Object.keys(params).filter(key => params[key] !== undefined).map(key => `${key}=${params[key]}`).join('&');
Expand Down
1 change: 1 addition & 0 deletions src/test/unit/services/esdt.address.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ describe('EsdtAddressService', () => {
provide: GatewayService,
useValue: {
get: jest.fn(),
getPaged: jest.fn(),
getAddressEsdtRoles: jest.fn(),
},
},
Expand Down
130 changes: 130 additions & 0 deletions src/test/unit/services/gateway.paged.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
import { GatewayService } from "src/common/gateway/gateway.service";
import { GatewayComponentRequest } from "src/common/gateway/entities/gateway.component.request";

describe('GatewayService paged trie requests', () => {
let gatewayService: GatewayService;
let apiService: any;

const firstCheckpoint = { blockNonce: 90, blockHash: 'ab', blockRootHash: 'cd', epoch: 4, iteratorState: ['Dt/+AA=='] };
const secondCheckpoint = { blockNonce: 90, blockHash: 'ab', blockRootHash: 'cd', epoch: 4, iteratorState: ['Ee/+BB=='] };
const blockInfo = { nonce: 90, hash: 'ab', rootHash: 'cd' };

const firstEncoded = Buffer.from(JSON.stringify(firstCheckpoint)).toString('base64url');
const secondEncoded = Buffer.from(JSON.stringify(secondCheckpoint)).toString('base64url');

beforeEach(() => {
apiService = { get: jest.fn() };

gatewayService = new GatewayService(
{ getGatewayUrl: () => 'https://gateway', getSnapshotlessGatewayUrl: () => undefined } as any,
apiService,
);
Object.assign(gatewayService, { eventEmitter: { emit: jest.fn() }, trieBusyRetryDelayMs: 0 });
});

it('should merge object items of every page and drop the checkpoint', async () => {
apiService.get
.mockResolvedValueOnce({ data: { data: { esdts: { 'A-1': { balance: '1' } }, blockInfo, iteratorState: firstCheckpoint } } })
.mockResolvedValueOnce({ data: { data: { esdts: {}, blockInfo, iteratorState: secondCheckpoint } } })
.mockResolvedValueOnce({ data: { data: { esdts: { 'B-2': { balance: '2' } }, blockInfo } } });

const result = await gatewayService.getPaged('address/erd1/esdt', GatewayComponentRequest.addressEsdt, 'esdts');

expect(result).toEqual({ esdts: { 'A-1': { balance: '1' }, 'B-2': { balance: '2' } }, blockInfo });
expect(apiService.get).toHaveBeenNthCalledWith(1, 'https://gateway/address/erd1/esdt', expect.anything(), undefined);
expect(apiService.get).toHaveBeenNthCalledWith(2, `https://gateway/address/erd1/esdt?iteratorState=${firstEncoded}`, expect.anything(), undefined);
expect(apiService.get).toHaveBeenNthCalledWith(3, `https://gateway/address/erd1/esdt?iteratorState=${secondEncoded}`, expect.anything(), undefined);
});

it('should send the checkpoint as unpadded base64url', async () => {
apiService.get
.mockResolvedValueOnce({ data: { data: { tokens: ['A-1'], iteratorState: firstCheckpoint } } })
.mockResolvedValueOnce({ data: { data: { tokens: ['B-2'] } } });

await gatewayService.getPaged('network/esdt/fungible-tokens', GatewayComponentRequest.allFungibleTokens, 'tokens');

const continuationUrl: string = apiService.get.mock.calls[1][0];
const encoded = continuationUrl.split('iteratorState=')[1];
expect(encoded).toMatch(/^[A-Za-z0-9_-]+$/);
expect(JSON.parse(Buffer.from(encoded, 'base64url').toString())).toEqual(firstCheckpoint);
});

it('should concatenate list items of every page', async () => {
apiService.get
.mockResolvedValueOnce({ data: { data: { tokens: ['A-1', 'B-2'], iteratorState: firstCheckpoint } } })
.mockResolvedValueOnce({ data: { data: { tokens: ['C-3'] } } });

expect(await gatewayService.getEsdtFungibleTokens()).toEqual(['A-1', 'B-2', 'C-3']);
});

it('should append the checkpoint to an url that already has a query', async () => {
apiService.get
.mockResolvedValueOnce({ data: { data: { pairs: { aa: '01' }, iteratorState: firstCheckpoint } } })
.mockResolvedValueOnce({ data: { data: { pairs: { bb: '02' } } } });

await gatewayService.getPaged('address/erd1/keys?numKeys=1', GatewayComponentRequest.addressKeys, 'pairs');

expect(apiService.get).toHaveBeenNthCalledWith(2, `https://gateway/address/erd1/keys?numKeys=1&iteratorState=${firstEncoded}`, expect.anything(), undefined);
});

it('should return undefined when the first page is handled by the error handler', async () => {
apiService.get.mockResolvedValueOnce(undefined);

expect(await gatewayService.getPaged('address/erd1/esdt', GatewayComponentRequest.addressEsdt, 'esdts')).toBeUndefined();
});

it('should retry a page answered with system busy', async () => {
apiService.get
.mockResolvedValueOnce({ data: { data: { tokens: ['A-1'], iteratorState: firstCheckpoint } } })
.mockRejectedValueOnce({ status: 429, response: { error: 'too many concurrent trie operations', code: 'system_busy' } })
.mockResolvedValueOnce({ data: { data: { tokens: ['B-2'] } } });

expect(await gatewayService.getEsdtFungibleTokens()).toEqual(['A-1', 'B-2']);
expect(apiService.get).toHaveBeenNthCalledWith(3, `https://gateway/network/esdt/fungible-tokens?iteratorState=${firstEncoded}`, expect.anything(), undefined);
});

it('should give up after the busy retries are exhausted', async () => {
const busy = { status: 429, response: { error: 'too many concurrent trie operations', code: 'system_busy' } };
apiService.get.mockRejectedValue(busy);

await expect(gatewayService.getEsdtFungibleTokens()).rejects.toEqual(busy);
expect(apiService.get).toHaveBeenCalledTimes(6);
});

it('should restart the walk once when the checkpoint expired', async () => {
apiService.get
.mockResolvedValueOnce({ data: { data: { tokens: ['A-1'], iteratorState: firstCheckpoint } } })
.mockRejectedValueOnce({ status: 400, response: { error: 'get esdt tokens error: iterator node unavailable: ab', code: 'bad_request' } })
.mockResolvedValueOnce({ data: { data: { tokens: ['A-1', 'B-2'] } } });

expect(await gatewayService.getEsdtFungibleTokens()).toEqual(['A-1', 'B-2']);
expect(apiService.get).toHaveBeenNthCalledWith(3, 'https://gateway/network/esdt/fungible-tokens', expect.anything(), undefined);
});

it('should not restart the walk more than once', async () => {
const expired = { status: 400, response: { error: 'iterator node unavailable: ab', code: 'bad_request' } };
apiService.get
.mockResolvedValueOnce({ data: { data: { tokens: ['A-1'], iteratorState: firstCheckpoint } } })
.mockRejectedValueOnce(expired)
.mockResolvedValueOnce({ data: { data: { tokens: ['A-1'], iteratorState: firstCheckpoint } } })
.mockRejectedValueOnce(expired);

await expect(gatewayService.getEsdtFungibleTokens()).rejects.toEqual(expired);
});

it('should not retry other failures', async () => {
const failure = { status: 500, response: { error: 'the endpoint is only available on metachain nodes', code: 'internal_issue' } };
apiService.get.mockRejectedValue(failure);

await expect(gatewayService.getEsdtFungibleTokens()).rejects.toEqual(failure);
expect(apiService.get).toHaveBeenCalledTimes(1);
});

it('should stop a walk that never ends', async () => {
Object.assign(gatewayService, { maxTriePages: 3 });
apiService.get.mockResolvedValue({ data: { data: { tokens: ['A-1'], iteratorState: firstCheckpoint } } });

await expect(gatewayService.getEsdtFungibleTokens()).rejects.toThrow("Trie walk for 'network/esdt/fungible-tokens' exceeded 3 pages");
expect(apiService.get).toHaveBeenCalledTimes(3);
});
});
Loading