diff --git a/packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts b/packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts new file mode 100644 index 000000000..58d691771 --- /dev/null +++ b/packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts @@ -0,0 +1,217 @@ +import { Address, Eventual, createLogger, createMetrics } from '@graphprotocol/common-ts' +import { + Allocation, + AllocationsResponse, + NetworkSubgraph, + QueryFeeModels, + QueryResult, + TapCollector, + TapSubgraphResponse, + TapTransaction, + TransactionManager, +} from '@graphprotocol/indexer-common' +import { NetworkContracts as TapContracts } from '@semiotic-labs/tap-contracts-bindings' +import { TAPSubgraph } from '../../tap-subgraph' +import { NetworkSpecification } from 'indexer-common/src/network-specification' +import { createMockAllocation } from '../../indexer-management/__tests__/helpers.test' +import { getContractAddress } from 'ethers/lib/utils' + +const timeout = 30_000 + +// mock allocation subgraph responses +// +// firstPage // 1000 +// secondPage // 1000 +// thirdPage // 999 +const allocations: Allocation[] = [] +const from = '0x8ba1f109551bD432803012645Ac136ddd64DBA72' + +for (let i = 0; i < 2999; i++) { + const mockAllocation = createMockAllocation() + allocations.push({ + ...mockAllocation, + id: getContractAddress({ from, nonce: i }) as Address, + }) +} + +// mock transactions subgraph response +// +// firstPage // 1000 +// secondPage // 1000 +const transactions: TapTransaction[] = [] +for (let i = 0; i < 2000; i++) { + transactions.push({ + id: i.toString(), + sender: { id: 'sender' }, + allocationID: 'allocation id', + timestamp: i, + }) +} + +// Make global Jest variables available +// eslint-disable-next-line @typescript-eslint/no-explicit-any +declare const __LOG_LEVEL__: never +let tapCollector: TapCollector + +function paginateArray( + array: T[], + getId: (item: T) => string, + pageSize: number, + lastId?: string, +): T[] { + // Sort the array by ID to ensure consistent pagination. + array.sort((a, b) => getId(a).localeCompare(getId(b))) + + // Find the index of the item with the given lastId. + let startIndex = 0 + if (lastId) { + startIndex = array.findIndex((item) => getId(item) === lastId) + 1 + } + + // Slice the array to return only the requested page size. + return array.slice(startIndex, startIndex + pageSize) +} + +let mockQueryTapSubgraph = jest + .fn() + .mockImplementation(async (_, variables): Promise> => { + const pageSize: number = variables.pageSize + const lastId: string | undefined = variables.lastId + + const paginatedTransactions = paginateArray( + transactions, + (tx) => tx.id, + pageSize, + lastId, + ) + + return { + data: { + transactions: paginatedTransactions, + _meta: { + block: { + hash: 'blockhash', + timestamp: 100000, + }, + }, + }, + } + }) + +let mockQueryNetworkSubgraph = jest + .fn() + .mockImplementation(async (_, variables): Promise> => { + const pageSize: number = variables.pageSize + const lastId: string | undefined = variables.lastId + + const paginatedAllocations = paginateArray( + allocations, + (allocation) => allocation.id, + pageSize, + lastId, + ) + + return { + data: { + allocations: paginatedAllocations, + meta: { + block: { + hash: 'blockhash', + }, + }, + }, + } + }) + +jest.spyOn(TapCollector.prototype, 'startRAVProcessing').mockImplementation() +const setup = () => { + const logger = createLogger({ + name: 'Indexer API Client', + async: false, + level: __LOG_LEVEL__ ?? 'error', + }) + const metrics = createMetrics() + // Clearing the registry prevents duplicate metric registration in the default registry. + metrics.registry.clear() + const transactionManager = null as unknown as TransactionManager + const models = null as unknown as QueryFeeModels + const tapContracts = null as unknown as TapContracts + const allocations = null as unknown as Eventual + const networkSpecification = { + indexerOptions: { voucherRedemptionThreshold: 0, finalityTime: 0 }, + networkIdentifier: 'test', + } as unknown as NetworkSpecification + + const tapSubgraph = { + query: mockQueryTapSubgraph, + } as unknown as TAPSubgraph + const networkSubgraph = { + query: mockQueryNetworkSubgraph, + } as unknown as NetworkSubgraph + + tapCollector = TapCollector.create({ + logger, + metrics, + transactionManager, + models, + tapContracts, + allocations, + networkSpecification, + + networkSubgraph, + tapSubgraph, + }) +} + +describe('TAP Pagination', () => { + beforeAll(setup, timeout) + test( + 'test `getAllocationsfromAllocationIds` pagination', + async () => { + { + const allocations = await tapCollector['getAllocationsfromAllocationIds']([]) + expect(mockQueryNetworkSubgraph).toBeCalledTimes(3) + expect(allocations.length).toEqual(2999) + } + mockQueryNetworkSubgraph.mockClear() + + const mockAllocation = createMockAllocation() + allocations.push({ + ...mockAllocation, + id: getContractAddress({ from, nonce: 3000 }) as Address, + }) + { + const allocations = await tapCollector['getAllocationsfromAllocationIds']([]) + expect(mockQueryNetworkSubgraph).toBeCalledTimes(4) + expect(allocations.length).toEqual(3000) + } + }, + timeout, + ) + test( + 'test `findTransactionsForRavs` pagination', + async () => { + { + const transactionsResponse = await tapCollector['findTransactionsForRavs']([]) + expect(mockQueryTapSubgraph).toBeCalledTimes(3) + expect(transactionsResponse.transactions.length).toEqual(2000) + } + + mockQueryTapSubgraph.mockClear() + for (let i = 0; i < 500; i++) { + transactions.push({ + id: i.toString(), + sender: { id: 'sender' }, + allocationID: 'allocation id', + timestamp: i, + }) + } + { + const transactionsResponse = await tapCollector['findTransactionsForRavs']([]) + expect(mockQueryTapSubgraph).toBeCalledTimes(3) + expect(transactionsResponse.transactions.length).toEqual(2500) + } + }, + timeout, + ) +}) diff --git a/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts b/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts new file mode 100644 index 000000000..798858167 --- /dev/null +++ b/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts @@ -0,0 +1,87 @@ +import { + defineQueryFeeModels, + GraphNode, + Network, + QueryFeeModels, + TapCollector, +} from '@graphprotocol/indexer-common' +import { + connectDatabase, + createLogger, + createMetrics, + Logger, + Metrics, +} from '@graphprotocol/common-ts' +import { testNetworkSpecification } from '../../indexer-management/__tests__/util' +import { Sequelize } from 'sequelize' + +// Make global Jest variables available +// eslint-disable-next-line @typescript-eslint/no-explicit-any +declare const __DATABASE__: any +declare const __LOG_LEVEL__: never +let logger: Logger +let tapCollector: TapCollector +let metrics: Metrics +let queryFeeModels: QueryFeeModels +let sequelize: Sequelize +const timeout = 30000 + +const setup = async () => { + logger = createLogger({ + name: 'Indexer API Client', + async: false, + level: __LOG_LEVEL__ ?? 'error', + }) + metrics = createMetrics() + // Clearing the registry prevents duplicate metric registration in the default registry. + metrics.registry.clear() + sequelize = await connectDatabase(__DATABASE__) + queryFeeModels = defineQueryFeeModels(sequelize) + sequelize = await sequelize.sync({ force: true }) + + const graphNode = new GraphNode( + logger, + 'https://test-admin-endpoint.xyz', + 'https://test-query-endpoint.xyz', + 'https://test-status-endpoint.xyz', + ) + + const network = await Network.create( + logger, + testNetworkSpecification, + queryFeeModels, + graphNode, + metrics, + ) + tapCollector = network.tapCollector! +} + +jest.spyOn(TapCollector.prototype, 'startRAVProcessing').mockImplementation() +describe('Validate TAP queries', () => { + beforeAll(setup, timeout) + + test( + 'test `getAllocationsfromAllocationIds` query is valid', + async () => { + const mockedFunc = jest.spyOn(tapCollector.networkSubgraph, 'query') + const result = await tapCollector['getAllocationsfromAllocationIds']([]) + expect(result).toEqual([]) + // this subgraph is in an eventual + // we check if it was called more than 0 times + expect(mockedFunc).toBeCalled() + }, + timeout, + ) + + test( + 'test `findTransactionsForRavs` query is valid', + async () => { + const mockedFunc = jest.spyOn(tapCollector.tapSubgraph, 'query') + const result = await tapCollector['findTransactionsForRavs']([]) + expect(result.transactions).toEqual([]) + expect(result._meta.block.hash.length).toEqual(66) + expect(mockedFunc).toBeCalledTimes(1) + }, + timeout, + ) +}) diff --git a/packages/indexer-common/src/allocations/tap-collector.ts b/packages/indexer-common/src/allocations/tap-collector.ts index 99a7873a0..1b0d497d4 100644 --- a/packages/indexer-common/src/allocations/tap-collector.ts +++ b/packages/indexer-common/src/allocations/tap-collector.ts @@ -78,7 +78,7 @@ interface TapMeta { } } -interface TapTransaction { +export interface TapTransaction { id: string allocationID: string timestamp: number @@ -245,8 +245,8 @@ export class TapCollector { allocations( first: $pageSize block: $block - orderBy: id, - orderDirection: asc, + orderBy: id + orderDirection: asc where: { id_gt: $lastId, id_in: $allocationIds } ) { id @@ -275,17 +275,16 @@ export class TapCollector { `, { allocationIds, lastId, pageSize: PAGE_SIZE, block }, ) - console.log("called query!") if (!result.data) { throw `There was an error while querying Network Subgraph. Errors: ${result.error}` } + returnedAllocations.push(...result.data.allocations) + block = { hash: result.data.meta.block.hash } if (result.data.allocations.length < PAGE_SIZE) { break } - block = { hash: result.data.meta.block.hash } lastId = result.data.allocations.slice(-1)[0].id - returnedAllocations.push(...result.data.allocations) } if (returnedAllocations.length == 0) { @@ -414,8 +413,8 @@ export class TapCollector { transactions( first: $pageSize block: $block - orderBy: id, - orderDirection: asc, + orderBy: id + orderDirection: asc where: { id_gt: $lastId type: "redeem" @@ -455,11 +454,11 @@ export class TapCollector { throw `There was an error while querying Tap Subgraph. Errors: ${result.error}` } meta = result.data._meta + transactions.push(...result.data.transactions) if (result.data.transactions.length < PAGE_SIZE) { break } lastId = result.data.transactions.slice(-1)[0].id - transactions.push(...result.data.transactions) } return { diff --git a/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts b/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts index 62c7ca595..a2358926e 100644 --- a/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts +++ b/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts @@ -122,7 +122,7 @@ const setupMonitor = async () => { ) } -const createMockAllocation = (): Allocation => { +export const createMockAllocation = (): Allocation => { const mockDeployment = { id: new SubgraphDeploymentID('QmcpeU4pZxzKB9TJ6fzH6PyZi9h8PJ6pG1c4izb9VAakJq'), deniedAt: 0,