From e5f5f738f3f3cfb80b458c285511346b478e5d81 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Fri, 3 May 2024 07:40:58 -0300 Subject: [PATCH] mina funnel (#343) * mina funnel: wip * wip * add actions fetching * cleanup some lints * split actions and events into 2 cdes * add after/cursor argument to the queries * remove some debugging logs * rename cardano cursors table to generic cursor based pagination this way it can be re-used also for mina or any other system, since there is no fundamental difference * add paginationLimit setting * dehardcode slot duration * replace the node's graphql api with direct db queries * replace the archive's graphql with direct db queries * add limit to the query * add loop to wait for finality * remove debugging logs * minor refactors * remove unused fields from query * handle edge case for the cde generic handler * fix upper range querying extra data * find confirmed timestamp directly instead of getting the slot first * rename the common paginated handling cde function * small refactor in funnel cursors filtering * rework the timestamp to block mapping to not use slots * replace startSlot for startBlockHeight in the extensions * add timestamp checkpointing event * genesisTime field is not used anymore * fix missing cache update of the lower bound * allow overriding archive's confirmation parameters * migrated from postgres to pg * remove debugging logs * cleanup --- package-lock.json | 15 +- packages/engine/paima-funnel/package.json | 3 +- .../paima-funnel/src/cde/minaGeneric.ts | 376 ++++++++++++++++ .../engine/paima-funnel/src/cde/reading.ts | 5 +- .../paima-funnel/src/funnels/FunnelCache.ts | 58 +++ .../paima-funnel/src/funnels/carp/funnel.ts | 28 +- .../paima-funnel/src/funnels/mina/funnel.ts | 418 ++++++++++++++++++ packages/engine/paima-funnel/src/index.ts | 11 + .../paima-runtime/src/cde-config/loading.ts | 31 ++ .../engine/paima-runtime/src/runtime-loops.ts | 5 +- packages/engine/paima-sm/src/cde-generic.ts | 8 +- .../engine/paima-sm/src/cde-processing.ts | 4 + packages/engine/paima-sm/src/index.ts | 60 +-- packages/engine/paima-sm/src/types.ts | 69 ++- packages/node-sdk/paima-db/migrations/up.sql | 8 +- packages/node-sdk/paima-db/src/index.ts | 6 +- .../node-sdk/paima-db/src/paima-tables.ts | 32 +- ...cde-cardano-tracking-pagination.queries.ts | 67 --- .../sql/cde-cardano-tracking-pagination.sql | 15 - .../cde-cursor-tracking-pagination.queries.ts | 67 +++ .../sql/cde-cursor-tracking-pagination.sql | 15 + .../src/sql/mina-checkpoints.queries.ts | 64 +++ .../paima-db/src/sql/mina-checkpoints.sql | 13 + .../paima-utils/src/config/loading.ts | 41 +- .../paima-utils/src/config/singleton.ts | 16 +- .../paima-sdk/paima-utils/src/constants.ts | 5 + packages/paima-sdk/paima-utils/src/index.ts | 4 + 27 files changed, 1307 insertions(+), 137 deletions(-) create mode 100644 packages/engine/paima-funnel/src/cde/minaGeneric.ts create mode 100644 packages/engine/paima-funnel/src/funnels/mina/funnel.ts delete mode 100644 packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.queries.ts delete mode 100644 packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.sql create mode 100644 packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.queries.ts create mode 100644 packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.sql create mode 100644 packages/node-sdk/paima-db/src/sql/mina-checkpoints.queries.ts create mode 100644 packages/node-sdk/paima-db/src/sql/mina-checkpoints.sql diff --git a/package-lock.json b/package-lock.json index fdcfae119..eaa47ab1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20711,6 +20711,18 @@ "node": ">= 0.4" } }, + "node_modules/postgres": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/postgres/-/postgres-3.4.4.tgz", + "integrity": "sha512-IbyN+9KslkqcXa8AO9fxpk97PA4pzewvpi2B3Dwy9u4zpV32QicaEdgmF3eSQUzdRk7ttDHQejNgAEr4XoeH4A==", + "engines": { + "node": ">=12" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/porsager" + } + }, "node_modules/postgres-array": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-3.0.2.tgz", @@ -26991,7 +27003,8 @@ "dependencies": { "@dcspark/cardano-multiplatform-lib-nodejs": "5.2.0", "@dcspark/carp-client": "^3.1.0", - "assert-never": "^1.2.1" + "assert-never": "^1.2.1", + "postgres": "^3.3.5" }, "devDependencies": { "typescript": "^5.3.3" diff --git a/packages/engine/paima-funnel/package.json b/packages/engine/paima-funnel/package.json index d1f33c654..f59aa0d18 100644 --- a/packages/engine/paima-funnel/package.json +++ b/packages/engine/paima-funnel/package.json @@ -19,6 +19,7 @@ "dependencies": { "assert-never": "^1.2.1", "@dcspark/carp-client": "^3.1.0", - "@dcspark/cardano-multiplatform-lib-nodejs": "5.2.0" + "@dcspark/cardano-multiplatform-lib-nodejs": "5.2.0", + "pg": "^8.11.3" } } diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts new file mode 100644 index 000000000..b78647197 --- /dev/null +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -0,0 +1,376 @@ +import type { + CdeMinaActionGenericDatum, + CdeMinaEventGenericDatum, + ChainDataExtensionMinaActionGeneric, + ChainDataExtensionMinaEventGeneric, +} from '@paima/sm'; +import { ChainDataExtensionDatumType } from '@paima/utils'; +import pg from 'pg'; + +export async function getEventCdeData(args: { + pg: pg.Client; + extension: ChainDataExtensionMinaEventGeneric; + fromTimestamp: number; + toTimestamp: number; + getBlockNumber: (minaTimestamp: number) => number; + network: string; + isPresync: boolean; + cursor?: string; + limit?: number; + fromBlockHeight?: number; +}): Promise<(CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]> { + return getCdeData( + getEventsQuery, + ChainDataExtensionDatumType.MinaEventGeneric, + args.pg, + args.extension, + args.fromTimestamp, + args.toTimestamp, + args.getBlockNumber, + args.network, + args.isPresync, + args.cursor, + args.limit, + args.fromBlockHeight + ); +} + +export async function getActionCdeData(args: { + pg: pg.Client; + extension: ChainDataExtensionMinaActionGeneric; + fromTimestamp: number; + toTimestamp: number; + getBlockNumber: (minaTimestamp: number) => number; + network: string; + isPresync: boolean; + cursor?: string; + limit?: number; + fromBlockHeight?: number; +}): Promise<(CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]> { + return getCdeData( + getActionsQuery, + ChainDataExtensionDatumType.MinaActionGeneric, + args.pg, + args.extension, + args.fromTimestamp, + args.toTimestamp, + args.getBlockNumber, + args.network, + args.isPresync, + args.cursor, + args.limit, + args.fromBlockHeight + ); +} + +export async function getCdeData( + query: typeof getEventsQuery | typeof getActionsQuery, + cdeDatumType: + | ChainDataExtensionDatumType.MinaActionGeneric + | ChainDataExtensionDatumType.MinaEventGeneric, + pg: pg.Client, + extension: ChainDataExtensionMinaEventGeneric | ChainDataExtensionMinaActionGeneric, + fromTimestamp: number, + toTimestamp: number, + getBlockNumber: (minaTimestamp: number) => number, + network: string, + isPresync: boolean, + cursor?: string, + limit?: number, + fromBlockHeight?: number +): Promise<(CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]> { + const result = [] as (CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]; + + while (true) { + const unmapped = await query( + pg, + extension.address, + toTimestamp.toString(), + fromTimestamp.toString(), + cursor, + limit?.toString(), + fromBlockHeight?.toString() + ); + + const grouped = groupByTx(unmapped.rows); + + const events = grouped.flatMap(perBlock => + perBlock.eventsData.map(txEvent => ({ + cdeId: extension.cdeId, + cdeDatumType, + blockNumber: getBlockNumber(Number.parseInt(perBlock.blockInfo.timestamp, 10)), + payload: txEvent, + network, + scheduledPrefix: extension.scheduledPrefix, + paginationCursor: { cursor: txEvent.txHash, finished: false }, + })) + ); + + if (events.length > 0) { + const last = events[events.length - 1]; + + cursor = last.paginationCursor.cursor; + } + + events.forEach(element => { + result.push(element); + }); + + if (events.length === 0 || isPresync) { + break; + } + } + + return result; +} + +function groupByTx(events: PerBlock[]) { + const grouped = [] as { + blockInfo: { + height: number; + timestamp: string; + }; + eventsData: { data: string[][]; txHash: string }[]; + }[]; + + for (const block of events) { + const eventData = [] as { data: string[][]; txHash: string }[]; + + for (const blockEvent of block.events) { + if ( + eventData[eventData.length - 1] && + blockEvent.hash == eventData[eventData.length - 1].txHash + ) { + eventData[eventData.length - 1].data.push(blockEvent.data); + } else { + eventData.push({ txHash: blockEvent.hash, data: [blockEvent.data] }); + } + } + + grouped.push({ + blockInfo: { height: block.height, timestamp: block.timestamp }, + eventsData: eventData, + }); + } + + return grouped; +} + +function canonicalChainCTE(toTimestamp?: string, fromTimestamp?: string, fromBlockHeight?: string) { + return ` + canonical_chain AS ( + SELECT + id, state_hash, height, global_slot_since_genesis, timestamp + FROM + blocks b + WHERE + 1=1 + ${fromTimestamp ? `AND b.timestamp::decimal >= ${fromTimestamp}::decimal` : ``} + ${toTimestamp ? `AND b.timestamp::decimal <= ${toTimestamp}::decimal` : ``} + ${fromBlockHeight ? `AND b.height::decimal >= ${fromBlockHeight}::decimal` : ``} + ORDER BY height + ) + `; +} + +function accountIdentifierCTE(address: string) { + return ` + account_identifier AS ( + SELECT + id AS requesting_zkapp_account_identifier_id + FROM + account_identifiers ai + WHERE + ai.public_key_id = (SELECT id FROM public_keys WHERE value = '${address}') + )`; +} + +function blocksAccessedCTE() { + return ` + blocks_accessed AS + ( + SELECT + requesting_zkapp_account_identifier_id, + block_id, + account_identifier_id, + zkapp_id, + id AS account_access_id, + state_hash, + height, + global_slot_since_genesis, + timestamp + FROM + account_identifier ai + INNER JOIN accounts_accessed aa ON ai.requesting_zkapp_account_identifier_id = aa.account_identifier_id + INNER JOIN canonical_chain b ON aa.block_id = b.id + )`; +} + +function emittedZkAppCommandsCTE(after?: string) { + return ` + emitted_zkapp_commands AS ( + SELECT + blocks_accessed.*, + zkcu.id AS zkapp_account_update_id, + zkapp_fee_payer_body_id, + zkapp_account_updates_ids, + authorization_kind, + status, + memo, + hash, + body_id, + events_id, + actions_id + FROM + blocks_accessed + INNER JOIN blocks_zkapp_commands bzkc ON blocks_accessed.block_id = bzkc.block_id + INNER JOIN zkapp_commands zkc ON bzkc.zkapp_command_id = zkc.id + INNER JOIN zkapp_account_update zkcu ON zkcu.id = ANY(zkc.zkapp_account_updates_ids) + INNER JOIN zkapp_account_update_body zkcu_body ON zkcu_body.id = zkcu.body_id + AND zkcu_body.account_identifier_id = requesting_zkapp_account_identifier_id + ${after ? `AND zkc.id > (SELECT id FROM zkapp_commands WHERE zkapp_commands.hash = '${after}')` : ``} + WHERE + bzkc.status <> 'failed' + )`; +} + +function emittedEventsCTE() { + return ` + emitted_events AS ( + SELECT + *, + zke.id AS zkapp_event_id, + zke.element_ids AS zkapp_event_element_ids, + zkfa.id AS zkapp_event_array_id + FROM + emitted_zkapp_commands + INNER JOIN zkapp_events zke ON zke.id = events_id + INNER JOIN zkapp_field_array zkfa ON zkfa.id = ANY(zke.element_ids) + INNER JOIN zkapp_field zkf ON zkf.id = ANY(zkfa.element_ids) + ) + `; +} + +function emittedActionsCTE() { + return ` + emitted_actions AS ( + SELECT + *, + zke.id AS zkapp_event_id, + zke.element_ids AS zkapp_event_element_ids, + zkfa.id AS zkapp_event_array_id + FROM + emitted_zkapp_commands + INNER JOIN zkapp_events zke ON zke.id = actions_id + INNER JOIN zkapp_field_array zkfa ON zkfa.id = ANY(zke.element_ids) + INNER JOIN zkapp_field zkf ON zkf.id = ANY(zkfa.element_ids) + ) + `; +} + +function emittedActionStateCTE() { + return ` + emitted_action_state AS ( + SELECT + emitted_actions.* + FROM + emitted_actions + INNER JOIN zkapp_accounts zkacc ON zkacc.id = emitted_actions.zkapp_id + INNER JOIN zkapp_action_states zks ON zks.id = zkacc.action_state_id + )`; +} + +type PerBlock = { + timestamp: string; + height: number; + + events: { + hash: string; + data: string[]; + }[]; +}; + +export function getEventsQuery( + db_client: pg.Client, + address: string, + toTimestamp?: string, + fromTimestamp?: string, + after?: string, + limit?: string, + fromBlockHeight?: string +) { + let query = ` + WITH + ${canonicalChainCTE(toTimestamp, fromTimestamp, fromBlockHeight)}, + ${accountIdentifierCTE(address)}, + ${blocksAccessedCTE()}, + ${emittedZkAppCommandsCTE(after)}, + ${emittedEventsCTE()}, + grouped_events AS ( + SELECT + MAX(timestamp) timestamp, + MAX(hash) hash, + JSON_AGG(field) events_data, + MAX(height) height + FROM emitted_events + GROUP BY ( + zkapp_account_update_id, + zkapp_event_array_id + ) + ORDER BY height + ) + SELECT + MAX(timestamp) timestamp, + height, + JSON_AGG(JSON_BUILD_OBJECT('hash', hash, 'data', events_data)) events + FROM grouped_events + GROUP BY height + ORDER BY height + ${limit ? `LIMIT ${limit}` : ``} + `; + + return db_client.query(query); +} + +export function getActionsQuery( + db_client: pg.Client, + address: string, + toTimestamp?: string, + fromTimestamp?: string, + after?: string, + limit?: string, + fromBlockHeight?: string +) { + const query = ` + WITH + ${canonicalChainCTE(toTimestamp, fromTimestamp, fromBlockHeight)}, + ${accountIdentifierCTE(address)}, + ${blocksAccessedCTE()}, + ${emittedZkAppCommandsCTE(after)}, + ${emittedActionsCTE()}, + ${emittedActionStateCTE()}, + grouped_events AS ( + SELECT + MAX(timestamp) timestamp, + MAX(hash) hash, + JSON_AGG(field) actions_data, + MAX(height) height + FROM emitted_actions + GROUP BY ( + zkapp_account_update_id, + zkapp_event_array_id + ) + ORDER BY height + ) + SELECT + MAX(timestamp) timestamp, + height, + JSON_AGG(JSON_BUILD_OBJECT('hash', hash, 'data', actions_data)) events + FROM grouped_events + GROUP BY height + ORDER BY height + ${limit ? `LIMIT ${limit}` : ``} + `; + + return db_client.query(query); +} diff --git a/packages/engine/paima-funnel/src/cde/reading.ts b/packages/engine/paima-funnel/src/cde/reading.ts index 0f53e4b1a..7cc43a4ab 100644 --- a/packages/engine/paima-funnel/src/cde/reading.ts +++ b/packages/engine/paima-funnel/src/cde/reading.ts @@ -58,13 +58,14 @@ async function getSpecificCdeData( return await getCdeErc1155Data(extension, fromBlock, toBlock, network); case ChainDataExtensionType.ERC6551Registry: return await getCdeErc6551RegistryData(extension, fromBlock, toBlock, network); + // these are not used by the block funnel case ChainDataExtensionType.CardanoPool: case ChainDataExtensionType.CardanoProjectedNFT: case ChainDataExtensionType.CardanoAssetUtxo: case ChainDataExtensionType.CardanoTransfer: case ChainDataExtensionType.CardanoMintBurn: - // this is used by the block funnel, which can't get information for this - // extension + case ChainDataExtensionType.MinaEventGeneric: + case ChainDataExtensionType.MinaActionGeneric: return []; default: assertNever(extension); diff --git a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts index 161ebdf22..f8e0effcd 100644 --- a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts +++ b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts @@ -1,4 +1,5 @@ import type { ChainData } from '@paima/sm'; +import pg from 'pg'; export interface FunnelCacheEntry { /** @@ -13,6 +14,7 @@ export type CacheMapType = { [RpcCacheEntry.SYMBOL]?: RpcCacheEntry; [CarpFunnelCacheEntry.SYMBOL]?: CarpFunnelCacheEntry; [EvmFunnelCacheEntry.SYMBOL]?: EvmFunnelCacheEntry; + [MinaFunnelCacheEntry.SYMBOL]?: MinaFunnelCacheEntry; }; export class FunnelCacheManager { public cacheEntries: CacheMapType = {}; @@ -178,3 +180,59 @@ export class EvmFunnelCacheEntry implements FunnelCacheEntry { this.cachedData = {}; }; } + +export type MinaFunnelCacheEntryState = { + startingSlotTimestamp: number; + lastPoint: { timestamp: number } | undefined; + pg: pg.Client; + cursors: + | { + [cdeId: number]: { cursor: string; finished: boolean }; + } + | undefined; +}; + +export class MinaFunnelCacheEntry implements FunnelCacheEntry { + private state: MinaFunnelCacheEntryState | null = null; + public static readonly SYMBOL = Symbol('MinaFunnelStartingSlot'); + + public updateStartingTimestamp(startingSlotTimestamp: number, pg: pg.Client): void { + this.state = { + startingSlotTimestamp, + lastPoint: this.state?.lastPoint, + cursors: this.state?.cursors, + pg, + }; + } + + public updateLastPoint(timestamp: number): void { + if (this.state) { + this.state.lastPoint = { timestamp }; + } + } + + public updateCursor(cdeId: number, presyncCursor: { cursor: string; finished: boolean }): void { + if (this.state) { + if (!this.state.cursors) { + this.state.cursors = {}; + } + + this.state.cursors[cdeId] = presyncCursor; + } + } + + public initialized(): boolean { + return !!this.state; + } + + public getState(): Readonly { + if (!this.state) { + throw new Error('[mina-funnel] Uninitialized cache entry'); + } + return this.state; + } + + clear: FunnelCacheEntry['clear'] = () => { + this.state = null; + }; +} diff --git a/packages/engine/paima-funnel/src/funnels/carp/funnel.ts b/packages/engine/paima-funnel/src/funnels/carp/funnel.ts index 1605a7b50..cc0ddd3a5 100644 --- a/packages/engine/paima-funnel/src/funnels/carp/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/carp/funnel.ts @@ -30,7 +30,7 @@ import { query } from '@dcspark/carp-client'; import { Routes } from '@dcspark/carp-client'; import { FUNNEL_PRESYNC_FINISHED, InternalEventType } from '@paima/utils'; import { CarpFunnelCacheEntry } from '../FunnelCache.js'; -import { getCardanoEpoch, getCarpCursors } from '@paima/db'; +import { getCardanoEpoch, getPaginationCursors } from '@paima/db'; import type { BlockTxPair } from '@dcspark/carp-client'; const delayForWaitingForFinalityLoop = 1000; @@ -246,7 +246,7 @@ export class CarpFunnel extends BaseFunnel implements ChainFunnel { Promise.all( this.sharedData.extensions .filter(extension => { - if (!('startSlot' in extension)) { + if (extension.network !== this.chainName) { return false; } @@ -456,14 +456,24 @@ export class CarpFunnel extends BaseFunnel implements ChainFunnel { newEntry.updateEpoch(epoch[0].epoch); } - const cursors = await getCarpCursors.run(undefined, dbTx); - - for (const cursor of cursors) { - newEntry.updateCursor(cursor.cde_id, { - cursor: cursor.cursor, - finished: cursor.finished, + const cursors = await getPaginationCursors.run(undefined, dbTx); + + const extensions = sharedData.extensions + .filter(extensions => extensions.network === chainName) + .map(extension => extension.cdeId) + .reduce((set, cdeId) => { + set.add(cdeId); + return set; + }, new Set()); + + cursors + .filter(cursor => extensions.has(cursor.cde_id)) + .forEach(cursor => { + newEntry.updateCursor(cursor.cde_id, { + cursor: cursor.cursor, + finished: cursor.finished, + }); }); - } return newEntry; })(); diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts new file mode 100644 index 000000000..845acfc4e --- /dev/null +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -0,0 +1,418 @@ +import { + doLog, + logError, + ChainDataExtensionType, + delay, + ENV, + InternalEventType, +} from '@paima/utils'; +import type { ChainFunnel, ReadPresyncDataFrom } from '@paima/runtime'; +import type { + ChainData, + ChainDataExtensionDatum, + MinaPresyncChainData, + PresyncChainData, +} from '@paima/sm'; +import { composeChainData, groupCdeData } from '../../utils.js'; +import { BaseFunnel } from '../BaseFunnel.js'; +import type { FunnelSharedData } from '../BaseFunnel.js'; +import type { PoolClient } from 'pg'; +import { FUNNEL_PRESYNC_FINISHED, ConfigNetworkType } from '@paima/utils'; +import { getMinaCheckpoint, getPaginationCursors } from '@paima/db'; +import { getActionCdeData, getEventCdeData } from '../../cde/minaGeneric.js'; +import type { MinaConfig } from '@paima/utils'; +import { MinaFunnelCacheEntry } from '../FunnelCache.js'; +import pg from 'pg'; +const { Client } = pg; + +const delayForWaitingForFinalityLoop = 1000; + +async function findMinaConfirmedTimestamp( + db: pg.Client, + confirmationDepth?: number +): Promise { + let row; + if (confirmationDepth) { + row = ( + await db.query(` + WITH RECURSIVE chain AS ( + (SELECT parent_id, id, timestamp, height FROM blocks b WHERE height = (select MAX(height) from blocks) + ORDER BY timestamp ASC + LIMIT 1) + UNION ALL + SELECT b.parent_id, b.id, b.timestamp, b.height FROM blocks b + INNER JOIN chain + ON b.id = chain.parent_id AND chain.id <> chain.parent_id + ) SELECT timestamp FROM chain c + LIMIT 1 + OFFSET ${confirmationDepth}; + `) + ).rows; + } else { + const res = await db.query( + `select timestamp from blocks where chain_status = 'canonical' order by height desc limit 1;` + ); + + row = res.rows; + } + + return Number.parseInt(row[0]['timestamp'], 10); +} + +// mina timestamps are in milliseconds, while evm timestamps are in seconds. +function baseChainTimestampToMina(baseChainTimestamp: number, delay: number): number { + return Math.max((baseChainTimestamp - delay) * 1000, 0); +} + +export class MinaFunnel extends BaseFunnel implements ChainFunnel { + config: MinaConfig; + chainName: string; + + protected constructor( + sharedData: FunnelSharedData, + dbTx: PoolClient, + config: MinaConfig, + chainName: string, + private readonly baseFunnel: ChainFunnel, + private cache: MinaFunnelCacheEntry + ) { + super(sharedData, dbTx); + this.readData.bind(this); + this.readPresyncData.bind(this); + this.getDbTx.bind(this); + this.config = config; + this.chainName = chainName; + } + + public override async readData(blockHeight: number): Promise { + const baseData = await this.baseFunnel.readData(blockHeight); + + if (baseData.length === 0) { + return baseData; + } + + let cachedState = this.cache.getState(); + + const maxBaseTimestamp = baseChainTimestampToMina( + baseData[baseData.length - 1].timestamp, + this.config.delay + ); + + while (true) { + const confirmedTimestamp = await findMinaConfirmedTimestamp( + cachedState.pg, + this.config.confirmationDepth + ); + + if (confirmedTimestamp >= maxBaseTimestamp) { + break; + } + + await delay(delayForWaitingForFinalityLoop); + } + + const lastRoundTimestamp = this.cache.getState().lastPoint?.timestamp; + const fromTimestamp = lastRoundTimestamp + ? lastRoundTimestamp + 1 + : cachedState.startingSlotTimestamp; + + const toTimestamp = maxBaseTimestamp; + + const getBlockNumber = (state: { curr: number }) => (ts: number) => { + while ( + state.curr < baseData.length && + baseChainTimestampToMina(baseData[state.curr].timestamp, this.config.delay) <= ts + ) + state.curr++; + + if (state.curr < baseData.length) { + return baseData[state.curr].blockNumber; + } else { + throw new Error('got event out of the expected range'); + } + }; + + const ungroupedCdeData = await Promise.all( + this.sharedData.extensions.reduce( + (promises, extension) => { + if (extension.cdeType === ChainDataExtensionType.MinaEventGeneric) { + const promise = getEventCdeData({ + pg: cachedState.pg, + extension, + fromTimestamp, + toTimestamp, + getBlockNumber: getBlockNumber({ curr: 0 }), + network: this.chainName, + isPresync: false, + cursor: undefined, + limit: this.config.paginationLimit, + }); + + promises.push(promise); + } + + if (extension.cdeType === ChainDataExtensionType.MinaActionGeneric) { + const promise = getActionCdeData({ + pg: cachedState.pg, + extension, + fromTimestamp, + toTimestamp, + getBlockNumber: getBlockNumber({ curr: 0 }), + network: this.chainName, + isPresync: false, + cursor: undefined, + limit: this.config.paginationLimit, + }); + + promises.push(promise); + } + + return promises; + }, + [] as Promise[] + ) + ); + + const grouped = groupCdeData( + this.chainName, + baseData[0].blockNumber, + baseData[baseData.length - 1].blockNumber, + ungroupedCdeData.filter(data => data.length > 0) + ); + + const composed = composeChainData(baseData, grouped); + + this.cache.updateLastPoint(toTimestamp); + + for (const chainData of composed) { + if (!chainData.internalEvents) { + chainData.internalEvents = []; + } + + chainData.internalEvents.push({ + type: InternalEventType.MinaLastTimestamp, + + timestamp: baseChainTimestampToMina(chainData.timestamp, this.config.delay).toString(), + network: this.chainName, + }); + } + + return composed; + } + + public override async readPresyncData(args: ReadPresyncDataFrom): Promise<{ + [network: string]: PresyncChainData[] | typeof FUNNEL_PRESYNC_FINISHED; + }> { + const basePromise = this.baseFunnel.readPresyncData(args); + + const cache = this.cache.getState(); + + const cursors = cache.cursors; + + if (cursors && Object.values(cursors).every(x => x.finished)) { + const data = await basePromise; + data[this.chainName] = FUNNEL_PRESYNC_FINISHED; + + return data; + } + + const mapCursorPaginatedData = (cdeId: number) => (datums: any) => { + const finished = datums.length === 0 || datums.length < this.config.paginationLimit; + + this.cache.updateCursor(cdeId, { + cursor: datums[datums.length - 1] ? datums[datums.length - 1].paginationCursor.cursor : '', + finished, + }); + + if (datums.length > 0) { + datums[datums.length - 1].paginationCursor.finished = finished; + } + + return datums; + }; + + const startingSlotTimestamp = this.cache.getState().startingSlotTimestamp; + + try { + const [baseData, ungroupedCdeData] = await Promise.all([ + basePromise, + Promise.all( + this.sharedData.extensions + .filter(extension => { + if (extension.network !== this.chainName) { + return false; + } + + if (cursors) { + const cursor = cursors[extension.cdeId]; + + if (!cursor) { + return true; + } + + return !cursor.finished; + } else { + return true; + } + }) + .map(async extension => { + if (extension.cdeType === ChainDataExtensionType.MinaEventGeneric) { + const cursor = cursors && cursors[extension.cdeId]; + + const data = await getEventCdeData({ + pg: cache.pg, + extension, + fromTimestamp: 0, + fromBlockHeight: extension.startBlockHeight, + toTimestamp: startingSlotTimestamp - 1, + // the handler for this cde stores the block height unmodified + // (even if the event is scheduled at the correct height), so + // handle this special case here, to have the events properly + // sorted. + getBlockNumber: _x => ENV.SM_START_BLOCKHEIGHT + 1, + network: this.chainName, + isPresync: true, + cursor: cursor?.cursor, + limit: this.config.paginationLimit, + }).then(mapCursorPaginatedData(extension.cdeId)); + + return { + cdeId: extension.cdeId, + cdeType: extension.cdeType, + data, + }; + } else if (extension.cdeType === ChainDataExtensionType.MinaActionGeneric) { + const cursor = cursors && cursors[extension.cdeId]; + + const data = await getActionCdeData({ + pg: cache.pg, + extension, + fromTimestamp: 0, + fromBlockHeight: extension.startBlockHeight, + toTimestamp: startingSlotTimestamp - 1, + getBlockNumber: _x => ENV.SM_START_BLOCKHEIGHT + 1, + network: this.chainName, + isPresync: true, + cursor: cursor?.cursor, + limit: this.config.paginationLimit, + }).then(mapCursorPaginatedData(extension.cdeId)); + + return { + cdeId: extension.cdeId, + cdeType: extension.cdeType, + data, + }; + } else { + throw new Error(`[mina funnel] unhandled extension: ${extension.cdeType}`); + } + }) + ), + ]); + + const list: MinaPresyncChainData[] = []; + + for (const events of ungroupedCdeData) { + for (const event of events.data || []) { + list.push({ + extensionDatums: [event], + network: this.chainName, + networkType: ConfigNetworkType.MINA, + minaCursor: { + cdeId: event.cdeId, + cursor: event.paginationCursor.cursor, + finished: event.paginationCursor.finished, + }, + }); + } + } + + baseData[this.chainName] = list; + + return baseData; + } catch (err) { + doLog(`[paima-funnel::readPresyncData] Exception occurred while reading blocks: ${err}`); + + throw err; + } + } + + public static async recoverState( + sharedData: FunnelSharedData, + dbTx: PoolClient, + baseFunnel: ChainFunnel, + chainName: string, + config: MinaConfig, + startingBlockHeight: number + ): Promise { + const cacheEntry = (async (): Promise => { + const entry = sharedData.cacheManager.cacheEntries[MinaFunnelCacheEntry.SYMBOL]; + if (entry != null && entry.initialized()) return entry; + + const newEntry = new MinaFunnelCacheEntry(); + sharedData.cacheManager.cacheEntries[MinaFunnelCacheEntry.SYMBOL] = newEntry; + + const pg = new Client({ connectionString: config.archiveConnectionString }); + await pg.connect(); + + const startingBlock = await sharedData.web3.eth.getBlock(startingBlockHeight); + const startingBlockTimestamp = startingBlock.timestamp as number; + + const minaTimestamp = baseChainTimestampToMina(startingBlockTimestamp, config.delay); + + newEntry.updateStartingTimestamp(minaTimestamp, pg); + + const cursors = await getPaginationCursors.run(undefined, dbTx); + + const extensions = sharedData.extensions + .filter(extensions => extensions.network === chainName) + .map(extension => extension.cdeId) + .reduce((set, cdeId) => { + set.add(cdeId); + return set; + }, new Set()); + + cursors + .filter(cursor => extensions.has(cursor.cde_id)) + .forEach(cursor => { + newEntry.updateCursor(cursor.cde_id, { + cursor: cursor.cursor, + finished: cursor.finished, + }); + }); + + const checkpoint = await getMinaCheckpoint.run({ network: chainName }, dbTx); + if (checkpoint.length > 0) { + newEntry.updateLastPoint(Number.parseInt(checkpoint[0].timestamp, 10)); + } + + return newEntry; + })(); + + return new MinaFunnel(sharedData, dbTx, config, chainName, baseFunnel, await cacheEntry); + } +} + +export async function wrapToMinaFunnel( + chainFunnel: ChainFunnel, + sharedData: FunnelSharedData, + dbTx: PoolClient, + startingBlockHeight: number, + chainName: string, + config: MinaConfig +): Promise { + try { + const ebp = await MinaFunnel.recoverState( + sharedData, + dbTx, + chainFunnel, + chainName, + config, + startingBlockHeight + ); + return ebp; + } catch (err) { + doLog('[paima-funnel] Unable to initialize mina cde events processor:'); + logError(err); + throw new Error('[paima-funnel] Unable to initialize mina cde events processor'); + } +} diff --git a/packages/engine/paima-funnel/src/index.ts b/packages/engine/paima-funnel/src/index.ts index dda1527d5..347a278ee 100644 --- a/packages/engine/paima-funnel/src/index.ts +++ b/packages/engine/paima-funnel/src/index.ts @@ -18,6 +18,7 @@ import { wrapToCarpFunnel } from './funnels/carp/funnel.js'; import { wrapToParallelEvmFunnel } from './funnels/parallelEvm/funnel.js'; import { ConfigNetworkType } from '@paima/utils'; import type Web3 from 'web3'; +import { wrapToMinaFunnel } from './funnels/mina/funnel.js'; export class FunnelFactory implements IFunnelFactory { private constructor(public sharedData: FunnelSharedData) {} @@ -93,6 +94,16 @@ export class FunnelFactory implements IFunnelFactory { ); } chainFunnel = await wrapToCarpFunnel(chainFunnel, this.sharedData, dbTx, ENV.START_BLOCKHEIGHT); + for (const [chainName, config] of await GlobalConfig.minaConfig()) { + chainFunnel = await wrapToMinaFunnel( + chainFunnel, + this.sharedData, + dbTx, + ENV.START_BLOCKHEIGHT, + chainName, + config + ); + } chainFunnel = await wrapToEmulatedBlocksFunnel( chainFunnel, this.sharedData, diff --git a/packages/engine/paima-runtime/src/cde-config/loading.ts b/packages/engine/paima-runtime/src/cde-config/loading.ts index e92d2b721..9df5c5746 100644 --- a/packages/engine/paima-runtime/src/cde-config/loading.ts +++ b/packages/engine/paima-runtime/src/cde-config/loading.ts @@ -19,6 +19,7 @@ import { defaultEvmMainNetworkName, defaultCardanoNetworkName, getErc1155Contract, + defaultMinaNetworkName, } from '@paima/utils'; import type { @@ -43,6 +44,8 @@ import { ChainDataExtensionErc6551RegistryConfig, ChainDataExtensionErc721Config, ChainDataExtensionGenericConfig, + ChainDataExtensionMinaEventGenericConfig, + ChainDataExtensionMinaActionGenericConfig, } from '@paima/sm'; import { loadAbi } from './utils.js'; import assertNever from 'assert-never'; @@ -162,6 +165,18 @@ export function parseCdeConfigFile(configFileData: string): Static { const cdeId = cdeDatum.cdeId; diff --git a/packages/engine/paima-sm/src/cde-processing.ts b/packages/engine/paima-sm/src/cde-processing.ts index dcce614ae..13b29dd79 100644 --- a/packages/engine/paima-sm/src/cde-processing.ts +++ b/packages/engine/paima-sm/src/cde-processing.ts @@ -48,6 +48,10 @@ export async function cdeTransitionFunction( return await processCardanoTransferDatum(cdeDatum, inPresync); case ChainDataExtensionDatumType.CardanoMintBurn: return await processCardanoMintBurnDatum(cdeDatum, inPresync); + case ChainDataExtensionDatumType.MinaEventGeneric: + return await processGenericDatum(cdeDatum, inPresync); + case ChainDataExtensionDatumType.MinaActionGeneric: + return await processGenericDatum(cdeDatum, inPresync); default: assertNever(cdeDatum); } diff --git a/packages/engine/paima-sm/src/index.ts b/packages/engine/paima-sm/src/index.ts index 107d415d3..9a24626a8 100644 --- a/packages/engine/paima-sm/src/index.ts +++ b/packages/engine/paima-sm/src/index.ts @@ -28,7 +28,8 @@ import { getMainAddress, NO_USER_ID, updateCardanoEpoch, - updateCardanoPaginationCursor, + updatePaginationCursor, + updateMinaCheckpoint, } from '@paima/db'; import Prando from '@paima/prando'; @@ -119,7 +120,7 @@ const SM: GameStateMachineInitializer = { ); } } else if (latestCdeData.networkType === ConfigNetworkType.CARDANO) { - const cdeDataLength = await processCardanoCdeData( + const cdeDataLength = await processPaginatedCdeData( latestCdeData.carpCursor, latestCdeData.extensionDatums, dbTx, @@ -130,6 +131,18 @@ const SM: GameStateMachineInitializer = { `[${latestCdeData.network}] Processed ${cdeDataLength} CDE events in ${latestCdeData.carpCursor.cursor}` ); } + } else if (latestCdeData.networkType === ConfigNetworkType.MINA) { + const cdeDataLength = await processPaginatedCdeData( + latestCdeData.minaCursor, + latestCdeData.extensionDatums, + dbTx, + true + ); + if (cdeDataLength > 0) { + doLog( + `[${latestCdeData.network}] Processed ${cdeDataLength} CDE events in ${latestCdeData.minaCursor.cursor}` + ); + } } }, markPresyncMilestone: async ( @@ -306,34 +319,22 @@ async function processCdeData( }); } -async function processCardanoCdeData( - paginationCursor: - | { cdeId: number; cursor: string; finished: boolean } - | { cdeId: number; slot: number; finished: boolean }, +async function processPaginatedCdeData( + paginationCursor: { cdeId: number; cursor: string; finished: boolean }, cdeData: ChainDataExtensionDatum[] | undefined, dbTx: PoolClient, inPresync: boolean ): Promise { return await processCdeDataBase(cdeData, dbTx, inPresync, async () => { - if ('cursor' in paginationCursor) { - await updateCardanoPaginationCursor.run( - { - cde_id: paginationCursor.cdeId, - cursor: paginationCursor.cursor, - finished: paginationCursor.finished, - }, - dbTx - ); - } else { - await updateCardanoPaginationCursor.run( - { - cde_id: paginationCursor.cdeId, - cursor: paginationCursor.slot.toString(), - finished: paginationCursor.finished, - }, - dbTx - ); - } + await updatePaginationCursor.run( + { + cde_id: paginationCursor.cdeId, + cursor: paginationCursor.cursor, + finished: paginationCursor.finished, + }, + dbTx + ); + return; }); } @@ -488,6 +489,15 @@ async function processInternalEvents( dbTx ); break; + case InternalEventType.MinaLastTimestamp: + await updateMinaCheckpoint.run( + { + timestamp: event.timestamp, + network: event.network, + }, + dbTx + ); + break; default: assertNever(event); } diff --git a/packages/engine/paima-sm/src/types.ts b/packages/engine/paima-sm/src/types.ts index 7ba427681..01a0234cb 100644 --- a/packages/engine/paima-sm/src/types.ts +++ b/packages/engine/paima-sm/src/types.ts @@ -35,13 +35,18 @@ export interface ChainData { internalEvents?: InternalEvent[]; } -export type InternalEvent = CardanoEpochEvent | EvmLastBlockEvent; +export type InternalEvent = CardanoEpochEvent | EvmLastBlockEvent | MinaLastTimestampEvent; export type CardanoEpochEvent = { type: InternalEventType.CardanoBestEpoch; epoch: number }; export type EvmLastBlockEvent = { type: InternalEventType.EvmLastBlock; block: number; network: string; }; +export type MinaLastTimestampEvent = { + type: InternalEventType.MinaLastTimestamp; + timestamp: string; + network: string; +}; export interface EvmPresyncChainData { network: string; @@ -57,7 +62,14 @@ export interface CardanoPresyncChainData { extensionDatums: ChainDataExtensionDatum[]; } -export type PresyncChainData = EvmPresyncChainData | CardanoPresyncChainData; +export interface MinaPresyncChainData { + network: string; + networkType: ConfigNetworkType.MINA; + minaCursor: { cdeId: number; cursor: string; finished: boolean }; + extensionDatums: ChainDataExtensionDatum[]; +} + +export type PresyncChainData = EvmPresyncChainData | CardanoPresyncChainData | MinaPresyncChainData; interface CdeDatumErc20TransferPayload { from: string; @@ -247,6 +259,17 @@ export interface CdeCardanoMintBurnDatum extends CdeDatumBase { paginationCursor: { cursor: string; finished: boolean }; } +export interface CdeMinaEventGenericDatum extends CdeDatumBase { + cdeDatumType: ChainDataExtensionDatumType.MinaEventGeneric; + paginationCursor: { cursor: string; finished: boolean }; + scheduledPrefix: string; +} +export interface CdeMinaActionGenericDatum extends CdeDatumBase { + cdeDatumType: ChainDataExtensionDatumType.MinaActionGeneric; + paginationCursor: { cursor: string; finished: boolean }; + scheduledPrefix: string; +} + export type ChainDataExtensionDatum = | CdeErc20TransferDatum | CdeErc721MintDatum @@ -259,7 +282,9 @@ export type ChainDataExtensionDatum = | CdeCardanoProjectedNFTDatum | CdeCardanoAssetUtxoDatum | CdeCardanoTransferDatum - | CdeCardanoMintBurnDatum; + | CdeCardanoMintBurnDatum + | CdeMinaEventGenericDatum + | CdeMinaActionGenericDatum; export enum CdeEntryTypeName { Generic = 'generic', @@ -273,6 +298,8 @@ export enum CdeEntryTypeName { CardanoDelayedAsset = 'cardano-delayed-asset', CardanoTransfer = 'cardano-transfer', CardanoMintBurn = 'cardano-mint-burn', + MinaEventGeneric = 'mina-event-generic', + MinaActionGeneric = 'mina-action-generic', } const EvmAddress = Type.Transform(Type.RegExp('0x[0-9a-fA-F]{40}')) @@ -462,6 +489,38 @@ export type ChainDataExtensionCardanoMintBurn = ChainDataExtensionBase & cdeType: ChainDataExtensionType.CardanoMintBurn; }; +export const ChainDataExtensionMinaEventGenericConfig = Type.Object({ + type: Type.Literal(CdeEntryTypeName.MinaEventGeneric), + address: Type.String(), + scheduledPrefix: Type.String(), + startBlockHeight: Type.Number(), + name: Type.String(), +}); + +export type TChainDataExtensionMinaEventGenericConfig = Static< + typeof ChainDataExtensionGenericConfig +>; +export type ChainDataExtensionMinaEventGeneric = ChainDataExtensionBase & + Static & { + cdeType: ChainDataExtensionType.MinaEventGeneric; + }; + +export const ChainDataExtensionMinaActionGenericConfig = Type.Object({ + type: Type.Literal(CdeEntryTypeName.MinaActionGeneric), + address: Type.String(), + scheduledPrefix: Type.String(), + startBlockHeight: Type.Number(), + name: Type.String(), +}); + +export type TChainDataExtensionMinaActionGenericConfig = Static< + typeof ChainDataExtensionGenericConfig +>; +export type ChainDataExtensionMinaActionGeneric = ChainDataExtensionBase & + Static & { + cdeType: ChainDataExtensionType.MinaActionGeneric; + }; + export const CdeConfig = Type.Object({ extensions: Type.Array( Type.Intersect([ @@ -477,6 +536,8 @@ export const CdeConfig = Type.Object({ ChainDataExtensionCardanoDelayedAssetConfig, ChainDataExtensionCardanoTransferConfig, ChainDataExtensionCardanoMintBurnConfig, + ChainDataExtensionMinaEventGenericConfig, + ChainDataExtensionMinaActionGenericConfig, ]), Type.Partial(Type.Object({ network: Type.String() })), ]) @@ -509,6 +570,8 @@ export type ChainDataExtension = ( | ChainDataExtensionCardanoDelayedAsset | ChainDataExtensionCardanoTransfer | ChainDataExtensionCardanoMintBurn + | ChainDataExtensionMinaEventGeneric + | ChainDataExtensionMinaActionGeneric ) & { network: string | undefined }; export type GameStateTransitionFunctionRouter = ( diff --git a/packages/node-sdk/paima-db/migrations/up.sql b/packages/node-sdk/paima-db/migrations/up.sql index 3bfc12861..f1eb0dd3d 100644 --- a/packages/node-sdk/paima-db/migrations/up.sql +++ b/packages/node-sdk/paima-db/migrations/up.sql @@ -218,7 +218,7 @@ CREATE TABLE cde_cardano_asset_utxos ( PRIMARY KEY(cde_id,tx_id,output_index,cip14_fingerprint) ); -CREATE TABLE cde_tracking_cardano_pagination ( +CREATE TABLE cde_tracking_cursor_pagination ( cde_id INTEGER PRIMARY KEY, cursor TEXT NOT NULL, finished BOOLEAN NOT NULL @@ -241,3 +241,9 @@ CREATE TABLE cde_cardano_mint_burn( output_addresses JSONB NOT NULL, PRIMARY KEY (cde_id, tx_id) ); + +CREATE TABLE mina_checkpoint ( + timestamp TEXT NOT NULL, + network TEXT NOT NULL, + PRIMARY KEY (network) +); diff --git a/packages/node-sdk/paima-db/src/index.ts b/packages/node-sdk/paima-db/src/index.ts index 13a0639fa..e2f1f3cf5 100644 --- a/packages/node-sdk/paima-db/src/index.ts +++ b/packages/node-sdk/paima-db/src/index.ts @@ -48,11 +48,13 @@ export { cdeSpendCardanoAssetUtxo, ICdeCardanoAssetUtxosByAddressParams, } from './sql/cde-cardano-asset-utxos.queries.js'; -export * from './sql/cde-cardano-tracking-pagination.queries.js'; -export type * from './sql/cde-cardano-tracking-pagination.queries.js'; +export * from './sql/cde-cursor-tracking-pagination.queries.js'; +export type * from './sql/cde-cursor-tracking-pagination.queries.js'; export * from './sql/cde-cardano-transfer.queries.js'; export type * from './sql/cde-cardano-transfer.queries.js'; export { cdeCardanoMintBurnInsert } from './sql/cde-cardano-mint-burn.queries.js'; +export type * from './sql/mina-checkpoints.queries.js'; +export * from './sql/mina-checkpoints.queries.js'; export { tx, diff --git a/packages/node-sdk/paima-db/src/paima-tables.ts b/packages/node-sdk/paima-db/src/paima-tables.ts index 6566161b9..9b197d1c4 100644 --- a/packages/node-sdk/paima-db/src/paima-tables.ts +++ b/packages/node-sdk/paima-db/src/paima-tables.ts @@ -118,16 +118,16 @@ const TABLE_DATA_CDE_TRACKING_CARDANO: TableData = { creationQuery: QUERY_CREATE_TABLE_CDE_TRACKING_CARDANO, }; -const QUERY_CREATE_TABLE_CDE_TRACKING_CARDANO_PAGINATION = ` -CREATE TABLE cde_tracking_cardano_pagination ( +const QUERY_CREATE_TABLE_CDE_TRACKING_CURSOR_PAGINATION = ` +CREATE TABLE cde_tracking_cursor_pagination ( cde_id INTEGER PRIMARY KEY, cursor TEXT NOT NULL, finished BOOLEAN NOT NULL ); `; -const TABLE_DATA_CDE_TRACKING_CARDANO_PAGINATION: TableData = { - tableName: 'cde_tracking_cardano_pagination', +const TABLE_DATA_CDE_TRACKING_CURSOR_PAGINATION: TableData = { + tableName: 'cde_tracking_cursor_pagination', primaryKeyColumns: ['cde_id'], columnData: packTuples([ ['cde_id', 'integer', 'NO', ''], @@ -135,7 +135,7 @@ const TABLE_DATA_CDE_TRACKING_CARDANO_PAGINATION: TableData = { ['finished', 'boolean', 'NO', ''], ]), serialColumns: [], - creationQuery: QUERY_CREATE_TABLE_CDE_TRACKING_CARDANO_PAGINATION, + creationQuery: QUERY_CREATE_TABLE_CDE_TRACKING_CURSOR_PAGINATION, }; const QUERY_CREATE_TABLE_CDE = ` @@ -576,6 +576,25 @@ const TABLE_DATA_DELEGATIONS: TableData = { creationQuery: QUERY_CREATE_TABLE_DELEGATIONS, }; +const QUERY_CREATE_TABLE_MINA_CHECKPOINT = ` +CREATE TABLE mina_checkpoint ( + timestamp TEXT NOT NULL, + network TEXT NOT NULL, + PRIMARY KEY (network) +); +`; + +const TABLE_DATA_MINA_CHECKPOINT: TableData = { + tableName: 'mina_checkpoint', + primaryKeyColumns: ['network'], + columnData: packTuples([ + ['timestamp', 'text', 'NO', ''], + ['network', 'text', 'NO', ''], + ]), + serialColumns: [], + creationQuery: QUERY_CREATE_TABLE_MINA_CHECKPOINT, +}; + const FUNCTION_NOTIFY_WALLET_CONNECT: string = ` create or replace function public.notify_wallet_connect() returns trigger @@ -660,7 +679,8 @@ export const TABLES: TableData[] = [ TABLE_DATA_ADDRESSES, TABLE_DATA_DELEGATIONS, TABLE_DATA_CARDANO_LAST_EPOCH, - TABLE_DATA_CDE_TRACKING_CARDANO_PAGINATION, + TABLE_DATA_CDE_TRACKING_CURSOR_PAGINATION, TABLE_DATA_CDE_CARDANO_TRANSFER, TABLE_DATA_CDE_CARDANO_MINT_BURN, + TABLE_DATA_MINA_CHECKPOINT, ]; diff --git a/packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.queries.ts b/packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.queries.ts deleted file mode 100644 index 5e5d5a524..000000000 --- a/packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.queries.ts +++ /dev/null @@ -1,67 +0,0 @@ -/** Types generated for queries found in "src/sql/cde-cardano-tracking-pagination.sql" */ -import { PreparedQuery } from '@pgtyped/runtime'; - -/** 'GetCarpCursors' parameters type */ -export type IGetCarpCursorsParams = void; - -/** 'GetCarpCursors' return type */ -export interface IGetCarpCursorsResult { - cde_id: number; - cursor: string; - finished: boolean; -} - -/** 'GetCarpCursors' query type */ -export interface IGetCarpCursorsQuery { - params: IGetCarpCursorsParams; - result: IGetCarpCursorsResult; -} - -const getCarpCursorsIR: any = {"usedParamSet":{},"params":[],"statement":"select * from cde_tracking_cardano_pagination"}; - -/** - * Query generated from SQL: - * ``` - * select * from cde_tracking_cardano_pagination - * ``` - */ -export const getCarpCursors = new PreparedQuery(getCarpCursorsIR); - - -/** 'UpdateCardanoPaginationCursor' parameters type */ -export interface IUpdateCardanoPaginationCursorParams { - cde_id: number; - cursor: string; - finished: boolean; -} - -/** 'UpdateCardanoPaginationCursor' return type */ -export type IUpdateCardanoPaginationCursorResult = void; - -/** 'UpdateCardanoPaginationCursor' query type */ -export interface IUpdateCardanoPaginationCursorQuery { - params: IUpdateCardanoPaginationCursorParams; - result: IUpdateCardanoPaginationCursorResult; -} - -const updateCardanoPaginationCursorIR: any = {"usedParamSet":{"cde_id":true,"cursor":true,"finished":true},"params":[{"name":"cde_id","required":true,"transform":{"type":"scalar"},"locs":[{"a":89,"b":96}]},{"name":"cursor","required":true,"transform":{"type":"scalar"},"locs":[{"a":101,"b":108},{"a":170,"b":177}]},{"name":"finished","required":true,"transform":{"type":"scalar"},"locs":[{"a":113,"b":122},{"a":191,"b":200}]}],"statement":"INSERT INTO cde_tracking_cardano_pagination(\n cde_id,\n cursor,\n finished\n) VALUES (\n :cde_id!,\n :cursor!,\n :finished!\n)\nON CONFLICT (cde_id)\nDO UPDATE SET cursor = :cursor!, finished = :finished!"}; - -/** - * Query generated from SQL: - * ``` - * INSERT INTO cde_tracking_cardano_pagination( - * cde_id, - * cursor, - * finished - * ) VALUES ( - * :cde_id!, - * :cursor!, - * :finished! - * ) - * ON CONFLICT (cde_id) - * DO UPDATE SET cursor = :cursor!, finished = :finished! - * ``` - */ -export const updateCardanoPaginationCursor = new PreparedQuery(updateCardanoPaginationCursorIR); - - diff --git a/packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.sql b/packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.sql deleted file mode 100644 index 31307a50b..000000000 --- a/packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.sql +++ /dev/null @@ -1,15 +0,0 @@ -/* @name getCarpCursors */ -select * from cde_tracking_cardano_pagination; - -/* @name updateCardanoPaginationCursor */ -INSERT INTO cde_tracking_cardano_pagination( - cde_id, - cursor, - finished -) VALUES ( - :cde_id!, - :cursor!, - :finished! -) -ON CONFLICT (cde_id) -DO UPDATE SET cursor = :cursor!, finished = :finished!; \ No newline at end of file diff --git a/packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.queries.ts b/packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.queries.ts new file mode 100644 index 000000000..de0533d8e --- /dev/null +++ b/packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.queries.ts @@ -0,0 +1,67 @@ +/** Types generated for queries found in "src/sql/cde-cursor-tracking-pagination.sql" */ +import { PreparedQuery } from '@pgtyped/runtime'; + +/** 'GetPaginationCursors' parameters type */ +export type IGetPaginationCursorsParams = void; + +/** 'GetPaginationCursors' return type */ +export interface IGetPaginationCursorsResult { + cde_id: number; + cursor: string; + finished: boolean; +} + +/** 'GetPaginationCursors' query type */ +export interface IGetPaginationCursorsQuery { + params: IGetPaginationCursorsParams; + result: IGetPaginationCursorsResult; +} + +const getPaginationCursorsIR: any = {"usedParamSet":{},"params":[],"statement":"select * from cde_tracking_cursor_pagination"}; + +/** + * Query generated from SQL: + * ``` + * select * from cde_tracking_cursor_pagination + * ``` + */ +export const getPaginationCursors = new PreparedQuery(getPaginationCursorsIR); + + +/** 'UpdatePaginationCursor' parameters type */ +export interface IUpdatePaginationCursorParams { + cde_id: number; + cursor: string; + finished: boolean; +} + +/** 'UpdatePaginationCursor' return type */ +export type IUpdatePaginationCursorResult = void; + +/** 'UpdatePaginationCursor' query type */ +export interface IUpdatePaginationCursorQuery { + params: IUpdatePaginationCursorParams; + result: IUpdatePaginationCursorResult; +} + +const updatePaginationCursorIR: any = {"usedParamSet":{"cde_id":true,"cursor":true,"finished":true},"params":[{"name":"cde_id","required":true,"transform":{"type":"scalar"},"locs":[{"a":88,"b":95}]},{"name":"cursor","required":true,"transform":{"type":"scalar"},"locs":[{"a":100,"b":107},{"a":169,"b":176}]},{"name":"finished","required":true,"transform":{"type":"scalar"},"locs":[{"a":112,"b":121},{"a":190,"b":199}]}],"statement":"INSERT INTO cde_tracking_cursor_pagination(\n cde_id,\n cursor,\n finished\n) VALUES (\n :cde_id!,\n :cursor!,\n :finished!\n)\nON CONFLICT (cde_id)\nDO UPDATE SET cursor = :cursor!, finished = :finished!"}; + +/** + * Query generated from SQL: + * ``` + * INSERT INTO cde_tracking_cursor_pagination( + * cde_id, + * cursor, + * finished + * ) VALUES ( + * :cde_id!, + * :cursor!, + * :finished! + * ) + * ON CONFLICT (cde_id) + * DO UPDATE SET cursor = :cursor!, finished = :finished! + * ``` + */ +export const updatePaginationCursor = new PreparedQuery(updatePaginationCursorIR); + + diff --git a/packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.sql b/packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.sql new file mode 100644 index 000000000..9e25d295d --- /dev/null +++ b/packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.sql @@ -0,0 +1,15 @@ +/* @name getPaginationCursors */ +select * from cde_tracking_cursor_pagination; + +/* @name updatePaginationCursor */ +INSERT INTO cde_tracking_cursor_pagination( + cde_id, + cursor, + finished +) VALUES ( + :cde_id!, + :cursor!, + :finished! +) +ON CONFLICT (cde_id) +DO UPDATE SET cursor = :cursor!, finished = :finished!; \ No newline at end of file diff --git a/packages/node-sdk/paima-db/src/sql/mina-checkpoints.queries.ts b/packages/node-sdk/paima-db/src/sql/mina-checkpoints.queries.ts new file mode 100644 index 000000000..0c61c3d68 --- /dev/null +++ b/packages/node-sdk/paima-db/src/sql/mina-checkpoints.queries.ts @@ -0,0 +1,64 @@ +/** Types generated for queries found in "src/sql/mina-checkpoints.sql" */ +import { PreparedQuery } from '@pgtyped/runtime'; + +/** 'UpdateMinaCheckpoint' parameters type */ +export interface IUpdateMinaCheckpointParams { + network: string; + timestamp: string; +} + +/** 'UpdateMinaCheckpoint' return type */ +export type IUpdateMinaCheckpointResult = void; + +/** 'UpdateMinaCheckpoint' query type */ +export interface IUpdateMinaCheckpointQuery { + params: IUpdateMinaCheckpointParams; + result: IUpdateMinaCheckpointResult; +} + +const updateMinaCheckpointIR: any = {"usedParamSet":{"timestamp":true,"network":true},"params":[{"name":"timestamp","required":true,"transform":{"type":"scalar"},"locs":[{"a":71,"b":81},{"a":149,"b":159}]},{"name":"network","required":true,"transform":{"type":"scalar"},"locs":[{"a":88,"b":96}]}],"statement":"INSERT INTO mina_checkpoint(\n timestamp,\n network\n) VALUES (\n :timestamp!,\n :network!\n) \nON CONFLICT (network) DO\nUPDATE SET timestamp = :timestamp!"}; + +/** + * Query generated from SQL: + * ``` + * INSERT INTO mina_checkpoint( + * timestamp, + * network + * ) VALUES ( + * :timestamp!, + * :network! + * ) + * ON CONFLICT (network) DO + * UPDATE SET timestamp = :timestamp! + * ``` + */ +export const updateMinaCheckpoint = new PreparedQuery(updateMinaCheckpointIR); + + +/** 'GetMinaCheckpoint' parameters type */ +export interface IGetMinaCheckpointParams { + network: string; +} + +/** 'GetMinaCheckpoint' return type */ +export interface IGetMinaCheckpointResult { + timestamp: string; +} + +/** 'GetMinaCheckpoint' query type */ +export interface IGetMinaCheckpointQuery { + params: IGetMinaCheckpointParams; + result: IGetMinaCheckpointResult; +} + +const getMinaCheckpointIR: any = {"usedParamSet":{"network":true},"params":[{"name":"network","required":true,"transform":{"type":"scalar"},"locs":[{"a":54,"b":62}]}],"statement":"SELECT timestamp FROM mina_checkpoint WHERE network = :network! LIMIT 1"}; + +/** + * Query generated from SQL: + * ``` + * SELECT timestamp FROM mina_checkpoint WHERE network = :network! LIMIT 1 + * ``` + */ +export const getMinaCheckpoint = new PreparedQuery(getMinaCheckpointIR); + + diff --git a/packages/node-sdk/paima-db/src/sql/mina-checkpoints.sql b/packages/node-sdk/paima-db/src/sql/mina-checkpoints.sql new file mode 100644 index 000000000..da8f6e9e1 --- /dev/null +++ b/packages/node-sdk/paima-db/src/sql/mina-checkpoints.sql @@ -0,0 +1,13 @@ +/* @name updateMinaCheckpoint */ +INSERT INTO mina_checkpoint( + timestamp, + network +) VALUES ( + :timestamp!, + :network! +) +ON CONFLICT (network) DO +UPDATE SET timestamp = :timestamp!; + +/* @name getMinaCheckpoint */ +SELECT timestamp FROM mina_checkpoint WHERE network = :network! LIMIT 1; diff --git a/packages/paima-sdk/paima-utils/src/config/loading.ts b/packages/paima-sdk/paima-utils/src/config/loading.ts index 34bd8cb0e..18e68611a 100644 --- a/packages/paima-sdk/paima-utils/src/config/loading.ts +++ b/packages/paima-sdk/paima-utils/src/config/loading.ts @@ -9,6 +9,7 @@ export enum ConfigNetworkType { EVM = 'evm-main', EVM_OTHER = 'evm-other', CARDANO = 'cardano', + MINA = 'mina', } export type EvmConfig = Static; @@ -60,6 +61,15 @@ export const CardanoConfigSchema = Type.Object({ export type CardanoConfig = Static; +export const MinaConfigSchema = Type.Object({ + archiveConnectionString: Type.String(), + delay: Type.Number(), + paginationLimit: Type.Number({ default: 50 }), + confirmationDepth: Type.Optional(Type.Number()), +}); + +export type MinaConfig = Static; + // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export const TaggedEvmConfig = (T: T, MAIN_CONFIG: U) => Type.Union([ @@ -88,9 +98,21 @@ export const TaggedCardanoConfig = (T: T) => Type.Object({ type: Type.Literal(ConfigNetworkType.CARDANO) }), ]); +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export const TaggedMinaConfig = (T: T) => + Type.Intersect([ + T ? MinaConfigSchema : Type.Partial(MinaConfigSchema), + Type.Object({ type: Type.Literal(ConfigNetworkType.MINA) }), + ]); + // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export const TaggedConfig = (T: T) => - Type.Union([TaggedEvmMainConfig(T), TaggedEvmOtherConfig(T), TaggedCardanoConfig(T)]); + Type.Union([ + TaggedEvmMainConfig(T), + TaggedEvmOtherConfig(T), + TaggedCardanoConfig(T), + TaggedMinaConfig(T), + ]); // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export const BaseConfig = (T: T) => Type.Record(Type.String(), TaggedConfig(T)); @@ -109,10 +131,17 @@ const cardanoConfigDefaults = { paginationLimits: 50, }; +const minaConfigDefaults = { + // lightnet defaults + confirmationDepth: 30, + delay: 30 * 40, +}; + // used as a placeholder name for the ENV fallback mechanism // will need to be removed afterwards export const defaultEvmMainNetworkName = 'evm'; export const defaultCardanoNetworkName = 'cardano'; +export const defaultMinaNetworkName = 'mina'; export async function loadConfig(): Promise | undefined> { let configFileData: string; @@ -170,6 +199,9 @@ export async function loadConfig(): Promise; @@ -58,4 +64,12 @@ export class GlobalConfig { .filter(key => instance[key].type === ConfigNetworkType.EVM_OTHER) .map(key => [key, instance[key] as EvmConfig]); } + + public static async minaConfig(): Promise<[string, MinaConfig][]> { + const instance = await GlobalConfig.getInstance(); + + return Object.keys(instance) + .filter(key => instance[key].type === ConfigNetworkType.MINA) + .map(key => [key, instance[key] as MinaConfig]); + } } diff --git a/packages/paima-sdk/paima-utils/src/constants.ts b/packages/paima-sdk/paima-utils/src/constants.ts index 67c9cbc1a..831c575ce 100644 --- a/packages/paima-sdk/paima-utils/src/constants.ts +++ b/packages/paima-sdk/paima-utils/src/constants.ts @@ -25,6 +25,8 @@ export const enum ChainDataExtensionType { CardanoTransfer = 10, CardanoMintBurn = 11, ERC1155 = 12, + MinaEventGeneric = 13, + MinaActionGeneric = 14, } export const enum ChainDataExtensionDatumType { @@ -40,6 +42,8 @@ export const enum ChainDataExtensionDatumType { CardanoTransfer, CardanoMintBurn, Erc1155Transfer, + MinaEventGeneric, + MinaActionGeneric, } export const FUNNEL_PRESYNC_FINISHED = 'finished'; @@ -47,4 +51,5 @@ export const FUNNEL_PRESYNC_FINISHED = 'finished'; export const enum InternalEventType { CardanoBestEpoch, EvmLastBlock, + MinaLastTimestamp, } diff --git a/packages/paima-sdk/paima-utils/src/index.ts b/packages/paima-sdk/paima-utils/src/index.ts index 8c3d1671c..4b0c7b820 100644 --- a/packages/paima-sdk/paima-utils/src/index.ts +++ b/packages/paima-sdk/paima-utils/src/index.ts @@ -10,9 +10,11 @@ import { GlobalConfig } from './config/singleton.js'; import { EvmConfig, CardanoConfig, + MinaConfig, ConfigNetworkType, defaultEvmMainNetworkName, defaultCardanoNetworkName, + defaultMinaNetworkName, } from './config/loading.js'; export * from './config.js'; @@ -34,9 +36,11 @@ export { GlobalConfig, EvmConfig, CardanoConfig, + MinaConfig, ConfigNetworkType, defaultEvmMainNetworkName, defaultCardanoNetworkName, + defaultMinaNetworkName, }; export const DEFAULT_GAS_PRICE = '61000000000' as const;