From 1e046bed549e47d029c7187f245d3d2e2095d2ec Mon Sep 17 00:00:00 2001 From: Ford Date: Wed, 13 Nov 2024 13:34:32 -0800 Subject: [PATCH 1/5] agent, common: Add 'syncingNetwork' column to Actions table - Syncing network is the network/chain that the subgraph deployment extracts data from. --- .../17-actions-add-syncing-network.ts | 53 +++++++++++++ packages/indexer-cli/src/actions.ts | 4 + packages/indexer-common/src/actions.ts | 74 +++++++++++-------- packages/indexer-common/src/errors.ts | 2 + .../__tests__/helpers.test.ts | 1 + .../src/indexer-management/client.ts | 3 + .../src/indexer-management/models/action.ts | 9 +++ .../src/indexer-management/monitor.ts | 42 +++++++++++ .../indexer-management/resolvers/actions.ts | 10 --- packages/indexer-common/src/operator.ts | 1 + 10 files changed, 159 insertions(+), 40 deletions(-) create mode 100644 packages/indexer-agent/src/db/migrations/17-actions-add-syncing-network.ts diff --git a/packages/indexer-agent/src/db/migrations/17-actions-add-syncing-network.ts b/packages/indexer-agent/src/db/migrations/17-actions-add-syncing-network.ts new file mode 100644 index 000000000..0c34f61e3 --- /dev/null +++ b/packages/indexer-agent/src/db/migrations/17-actions-add-syncing-network.ts @@ -0,0 +1,53 @@ +import { Logger } from '@graphprotocol/common-ts' +import { DataTypes, QueryInterface } from 'sequelize' + +interface MigrationContext { + queryInterface: QueryInterface + logger: Logger +} + +interface Context { + context: MigrationContext +} + +export async function up({ context }: Context): Promise { + const { queryInterface, logger } = context + + logger.debug(`Checking if actions table exists`) + const tables = await queryInterface.showAllTables() + if (!tables.includes('Actions')) { + logger.info(`Actions table does not exist, migration not necessary`) + return + } + + logger.debug(`Checking if 'Actions' table needs to be migrated`) + const table = await queryInterface.describeTable('Actions') + const syncingNetworkColumn = table.syncingNetwork + if (syncingNetworkColumn) { + logger.info( + `'syncingNetwork' columns already exist, migration not necessary`, + ) + return + } + + logger.info(`Add 'syncingNetwork' column to 'Actions' table`) + await queryInterface.addColumn('Actions', 'syncingNetwork', { + type: DataTypes.BOOLEAN, + defaultValue: false, + }) +} + +export async function down({ context }: Context): Promise { + const { queryInterface, logger } = context + + return await queryInterface.sequelize.transaction({}, async transaction => { + const tables = await queryInterface.showAllTables() + + if (tables.includes('Actions')) { + logger.info(`Remove 'syncingNetwork' column`) + await context.queryInterface.removeColumn('Actions', 'syncingNetwork', { + transaction, + }) + } + }) +} diff --git a/packages/indexer-cli/src/actions.ts b/packages/indexer-cli/src/actions.ts index f7ccab61c..7b5790271 100644 --- a/packages/indexer-cli/src/actions.ts +++ b/packages/indexer-cli/src/actions.ts @@ -47,6 +47,7 @@ export async function buildActionInput( status, priority, protocolNetwork, + syncingNetwork: 'unknown', } case ActionType.UNALLOCATE: { let poi = actionParams.param2 @@ -64,6 +65,7 @@ export async function buildActionInput( status, priority, protocolNetwork, + syncingNetwork: 'unknown', } } case ActionType.REALLOCATE: { @@ -83,6 +85,7 @@ export async function buildActionInput( status, priority, protocolNetwork, + syncingNetwork: 'unknown', } } } @@ -399,6 +402,7 @@ export async function fetchActions( ) { id protocolNetwork + syncingNetwork type allocationID deploymentID diff --git a/packages/indexer-common/src/actions.ts b/packages/indexer-common/src/actions.ts index 9aec66814..88ce6a230 100644 --- a/packages/indexer-common/src/actions.ts +++ b/packages/indexer-common/src/actions.ts @@ -46,36 +46,36 @@ export interface ActionInput { status: ActionStatus priority: number | undefined protocolNetwork: string + syncingNetwork: string } export const isValidActionInput = ( /* eslint-disable @typescript-eslint/no-explicit-any */ - variableToCheck: any, -): variableToCheck is ActionInput => { - if (!('type' in variableToCheck)) { + actionToCheck: any, +): actionToCheck is ActionInput => { + if (!('type' in actionToCheck)) { return false } let hasActionParams = false - switch (variableToCheck.type) { + switch (actionToCheck.type) { case ActionType.ALLOCATE: - hasActionParams = 'deploymentID' in variableToCheck && 'amount' in variableToCheck + hasActionParams = 'deploymentID' in actionToCheck && 'amount' in actionToCheck break case ActionType.UNALLOCATE: - hasActionParams = - 'deploymentID' in variableToCheck && 'allocationID' in variableToCheck + hasActionParams = 'deploymentID' in actionToCheck && 'allocationID' in actionToCheck break case ActionType.REALLOCATE: hasActionParams = - 'deploymentID' in variableToCheck && - 'allocationID' in variableToCheck && - 'amount' in variableToCheck + 'deploymentID' in actionToCheck && + 'allocationID' in actionToCheck && + 'amount' in actionToCheck } return ( hasActionParams && - 'source' in variableToCheck && - 'reason' in variableToCheck && - 'status' in variableToCheck && - 'priority' in variableToCheck + 'source' in actionToCheck && + 'reason' in actionToCheck && + 'status' in actionToCheck && + 'priority' in actionToCheck ) } @@ -92,22 +92,6 @@ export const validateActionInputs = async ( throw Error("Cannot set an action without the field 'protocolNetwork'") } - try { - // Set the parsed network identifier back in the action input object - action.protocolNetwork = validateNetworkIdentifier(action.protocolNetwork) - } catch (e) { - throw Error(`Invalid value for the field 'protocolNetwork'. ${e}`) - } - - // Must have the required params for the action type - if (!isValidActionInput(action)) { - throw new Error( - `Failed to queue action: Invalid action input, actionInput: ${JSON.stringify( - action, - )}`, - ) - } - // Must have status QUEUED or APPROVED if ( [ @@ -122,6 +106,15 @@ export const validateActionInputs = async ( ) } + // Must have the required params for the action type + if (!isValidActionInput(action)) { + throw new Error( + `Failed to queue action: Invalid action input, actionInput: ${JSON.stringify( + action, + )}`, + ) + } + // Action must target an existing subgraph deployment const subgraphDeployment = await networkMonitor.subgraphDeployment( action.deploymentID, @@ -132,6 +125,25 @@ export const validateActionInputs = async ( ) } + try { + // Set the parsed protocol network identifier back in the action input object + action.protocolNetwork = validateNetworkIdentifier(action.protocolNetwork) + } catch (e) { + throw Error(`Invalid value for the field 'protocolNetwork'. ${e}`) + } + + try { + // Fetch syncing network, parse alias, and set the parsed value back in the action input object + const syncingNetwork = await networkMonitor.deploymentSyncingNetwork( + action.deploymentID, + ) + action.syncingNetwork = validateNetworkIdentifier(syncingNetwork) + } catch (e) { + throw Error( + `Could not resolve 'syncingNetwork' for deployment '${action.deploymentID}'. ${e}`, + ) + } + // Unallocate & reallocate actions must target an active allocationID if ([ActionType.UNALLOCATE, ActionType.REALLOCATE].includes(action.type)) { // allocationID must belong to active allocation @@ -161,6 +173,7 @@ export interface ActionFilter { reason?: string updatedAt?: WhereOperators protocolNetwork?: string + syncingNetwork?: string } export const actionFilterToWhereOptions = (filter: ActionFilter): WhereOptions => { @@ -192,6 +205,7 @@ export interface ActionResult { failureReason: string | null transaction: string | null protocolNetwork: string + syncingNetwork: string } export enum ActionType { diff --git a/packages/indexer-common/src/errors.ts b/packages/indexer-common/src/errors.ts index 34dc610d9..5f93a62fe 100644 --- a/packages/indexer-common/src/errors.ts +++ b/packages/indexer-common/src/errors.ts @@ -88,6 +88,7 @@ export enum IndexerErrorCode { IE075 = 'IE075', IE076 = 'IE076', IE077 = 'IE077', + IE078 = 'IE078', } export const INDEXER_ERROR_MESSAGES: Record = { @@ -169,6 +170,7 @@ export const INDEXER_ERROR_MESSAGES: Record = { IE075: 'Failed to connect to network contracts', IE076: 'Failed to resume subgraph deployment', IE077: 'Failed to allocate: subgraph not healthily syncing', + IE078: 'Failed to query subgraph features from network subgraph', } export type IndexerErrorCause = unknown diff --git a/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts b/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts index a2358926e..1d6a8093a 100644 --- a/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts +++ b/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts @@ -236,6 +236,7 @@ describe('Actions', () => { priority: 0, // When writing directly to the database, `protocolNetwork` must be in the CAIP2-ID format. protocolNetwork: 'eip155:421614', + syncingNetwork: 'eip155:1', } await models.Action.upsert(action) diff --git a/packages/indexer-common/src/indexer-management/client.ts b/packages/indexer-common/src/indexer-management/client.ts index 164aa404f..64b449028 100644 --- a/packages/indexer-common/src/indexer-management/client.ts +++ b/packages/indexer-common/src/indexer-management/client.ts @@ -135,6 +135,7 @@ const SCHEMA_SDL = gql` createdAt: BigInt! updatedAt: BigInt protocolNetwork: String! + syncingNetwork: String! } input ActionInput { @@ -149,6 +150,7 @@ const SCHEMA_SDL = gql` reason: String! priority: Int! protocolNetwork: String! + syncingNetwork: String! } input ActionUpdateInput { @@ -201,6 +203,7 @@ const SCHEMA_SDL = gql` input ActionFilter { id: Int protocolNetwork: String + syncingNetwork: String type: ActionType status: String source: String diff --git a/packages/indexer-common/src/indexer-management/models/action.ts b/packages/indexer-common/src/indexer-management/models/action.ts index 642a9fbca..2b2a27414 100644 --- a/packages/indexer-common/src/indexer-management/models/action.ts +++ b/packages/indexer-common/src/indexer-management/models/action.ts @@ -36,6 +36,7 @@ export class Action extends Model< declare updatedAt: CreationOptional declare protocolNetwork: string + declare syncingNetwork: string // eslint-disable-next-line @typescript-eslint/ban-types public toGraphQL(): object { @@ -151,6 +152,14 @@ export const defineActionModels = (sequelize: Sequelize): ActionModels => { is: caip2IdRegex, }, }, + syncingNetwork: { + type: DataTypes.STRING(50), + primaryKey: false, + allowNull: false, + validate: { + is: caip2IdRegex, + }, + }, }, { modelName: 'Action', diff --git a/packages/indexer-common/src/indexer-management/monitor.ts b/packages/indexer-common/src/indexer-management/monitor.ts index 4d347ae3e..6efaec773 100644 --- a/packages/indexer-common/src/indexer-management/monitor.ts +++ b/packages/indexer-common/src/indexer-management/monitor.ts @@ -520,6 +520,48 @@ export class NetworkMonitor { } } + async deploymentSyncingNetwork(ipfsHash: string): Promise { + try { + const result = await this.networkSubgraph.checkedQuery( + gql` + query subgraphDeploymentManifest($ipfsHash: String!) { + subgraphDeploymentManifest(id: $ipfsHash) { + network + } + } + `, + { + ipfsHash: ipfsHash, + }, + ) + + if (result.error) { + throw result.error + } + + if (!result.data || !result.data.subgraphDeploymentManifest) { + throw new Error( + `SubgraphDeployment with ipfsHash = ${ipfsHash} not found on chain`, + ) + } + + if (result.data.subgraphDeploymentManifest.network == undefined) { + return 'unknown' + } + + return result.data.subgraphDeploymentManifest.network + } catch (error) { + const err = indexerError(IndexerErrorCode.IE078, error) + this.logger.error( + `Failed to query subgraphDeploymentManifest with ipfsHash = ${ipfsHash}`, + { + err, + }, + ) + throw err + } + } + async transferredDeployments(): Promise { this.logger.debug('Querying the Network for transferred subgraph deployments') try { diff --git a/packages/indexer-common/src/indexer-management/resolvers/actions.ts b/packages/indexer-common/src/indexer-management/resolvers/actions.ts index 1e246754c..fb6eb2447 100644 --- a/packages/indexer-common/src/indexer-management/resolvers/actions.ts +++ b/packages/indexer-common/src/indexer-management/resolvers/actions.ts @@ -15,7 +15,6 @@ import { NetworkMapped, OrderDirection, validateActionInputs, - validateNetworkIdentifier, } from '@graphprotocol/indexer-common' import { literal, Op, Transaction } from 'sequelize' import { ActionManager } from '../actions' @@ -161,15 +160,6 @@ export default { throw Error('IndexerManagementClient must be in `network` mode to modify actions') } - // Sanitize protocol network identifier - actions.forEach((action) => { - try { - action.protocolNetwork = validateNetworkIdentifier(action.protocolNetwork) - } catch (e) { - throw Error(`Invalid value for the field 'protocolNetwork'. ${e}`) - } - }) - // Let Network Monitors validate actions based on their protocol networks await multiNetworks.mapNetworkMapped( groupBy(actions, (action) => action.protocolNetwork), diff --git a/packages/indexer-common/src/operator.ts b/packages/indexer-common/src/operator.ts index 1d97904c4..d62522ee4 100644 --- a/packages/indexer-common/src/operator.ts +++ b/packages/indexer-common/src/operator.ts @@ -278,6 +278,7 @@ export class Operator { reason: action.reason, priority: 0, protocolNetwork: action.protocolNetwork, + syncingNetork: 'unknown', } this.logger.trace(`Queueing action input`, { actionInput, From fce97f8ab7a483511111f24f286645c937d90335 Mon Sep 17 00:00:00 2001 From: Ford Date: Fri, 15 Nov 2024 12:49:44 -0800 Subject: [PATCH 2/5] cli: Add support for filtering 'actions get' command by syncing-network --- packages/indexer-cli/src/actions.ts | 4 +- packages/indexer-cli/src/command-helpers.ts | 41 ++++++++++++++++++- .../src/commands/indexer/actions/get.ts | 23 +++++++---- .../src/commands/indexer/allocations/close.ts | 4 +- .../commands/indexer/allocations/collect.ts | 4 +- .../commands/indexer/allocations/create.ts | 4 +- .../indexer-common/src/parsers/basic-types.ts | 8 +++- .../indexer-common/src/parsers/validators.ts | 7 +++- 8 files changed, 74 insertions(+), 21 deletions(-) diff --git a/packages/indexer-cli/src/actions.ts b/packages/indexer-cli/src/actions.ts index 7b5790271..c70e42d90 100644 --- a/packages/indexer-cli/src/actions.ts +++ b/packages/indexer-cli/src/actions.ts @@ -10,7 +10,7 @@ import { nullPassThrough, OrderDirection, parseBoolean, - validateNetworkIdentifier, + validateSupportedNetworkIdentifier, } from '@graphprotocol/indexer-common' import { validatePOI, validateRequiredParams } from './command-helpers' import gql from 'graphql-tag' @@ -215,7 +215,7 @@ const ACTION_PARAMS_PARSERS: Record any> type: x => validateActionType(x), status: x => validateActionStatus(x), reason: nullPassThrough, - protocolNetwork: x => validateNetworkIdentifier(x), + protocolNetwork: x => validateSupportedNetworkIdentifier(x), } /** diff --git a/packages/indexer-cli/src/command-helpers.ts b/packages/indexer-cli/src/command-helpers.ts index 2d8b1aef8..0b6d0a48b 100644 --- a/packages/indexer-cli/src/command-helpers.ts +++ b/packages/indexer-cli/src/command-helpers.ts @@ -36,7 +36,10 @@ export enum OutputFormat { import yaml from 'yaml' import { GluegunParameters, GluegunPrint } from 'gluegun' import { utils } from 'ethers' -import { validateNetworkIdentifier } from '@graphprotocol/indexer-common' +import { + validateNetworkIdentifier, + validateSupportedNetworkIdentifier, +} from '@graphprotocol/indexer-common' export const fixParameters = ( parameters: GluegunParameters, @@ -237,7 +240,7 @@ export function extractProtocolNetworkOption( const input = (network ?? n) as string try { - return validateNetworkIdentifier(input) + return validateSupportedNetworkIdentifier(input) } catch (parseError) { throw new Error(`Invalid value for the option '--network'. ${parseError}`) } @@ -251,3 +254,37 @@ export function requireProtocolNetworkOption(options: { [key: string]: any }): s } return protocolNetwork } + +export function extractSyncingNetworkOption( + options: { + [key: string]: any + }, + required = false, +): string | undefined { + const { s, syncing } = options + + // Tries to extract the --network option from Gluegun options. + // Throws if required is set to true and the option is not found. + if (!s && !syncing) { + if (required) { + throw new Error("The option '--syncing' is required") + } else { + return undefined + } + } + + // Check for invalid usage + const allowedUsages = + (s === undefined && typeof syncing === 'string') || + (syncing === undefined && typeof s === 'string') + if (!allowedUsages) { + throw new Error("Invalid usage of the option '--network'") + } + const input = (syncing ?? s) as string + + try { + return validateNetworkIdentifier(input) + } catch (parseError) { + throw new Error(`Invalid value for the option '--syncing'. ${parseError}`) + } +} diff --git a/packages/indexer-cli/src/commands/indexer/actions/get.ts b/packages/indexer-cli/src/commands/indexer/actions/get.ts index ce73d2740..1b8e0fb24 100644 --- a/packages/indexer-cli/src/commands/indexer/actions/get.ts +++ b/packages/indexer-cli/src/commands/indexer/actions/get.ts @@ -14,6 +14,7 @@ import { fixParameters, printObjectOrArray, extractProtocolNetworkOption, + extractSyncingNetworkOption, } from '../../../command-helpers' import { fetchAction, fetchActions } from '../../../actions' @@ -23,6 +24,7 @@ ${chalk.dim('Options:')} -h, --help Show usage information -n, --network Filter by protocol network (mainnet, arbitrum-one, sepolia, arbitrum-sepolia) + -s, --syncing Filter by the syncing network (see https://thegraph.com/networks/ for supported networks) --type allocate|unallocate|reallocate|collect Filter by type --status queued|approved|pending|success|failed|canceled Filter by status --source Fetch only actions queued by a specific source @@ -37,6 +39,7 @@ ${chalk.dim('Options:')} const actionFields: (keyof Action)[] = [ 'id', 'protocolNetwork', + 'syncingNetwork', 'type', 'deploymentID', 'allocationID', @@ -51,7 +54,7 @@ const actionFields: (keyof Action)[] = [ 'reason', ] -/// Validates input for the `--fieds` option. +/// Validates input for the `--fields` option. function validateFields(fields: string | undefined): (keyof Action)[] { if (fields === undefined) { return [] @@ -95,10 +98,8 @@ module.exports = { let orderByParam = ActionParams.ID let orderDirectionValue = OrderDirection.DESC const outputFormat = o || output || 'table' - - const protocolNetwork: string | undefined = extractProtocolNetworkOption( - parameters.options, - ) + let protocolNetwork: string | undefined = undefined + let syncingNetwork: string | undefined = undefined if (help || h) { inputSpinner.stopAndPersist({ symbol: '💁', text: HELP }) @@ -106,6 +107,10 @@ module.exports = { } let selectedFields: (keyof Action)[] try { + protocolNetwork = extractProtocolNetworkOption(parameters.options) + + syncingNetwork = extractSyncingNetworkOption(parameters.options) + if (!['json', 'yaml', 'table'].includes(outputFormat)) { throw Error( `Invalid output format "${outputFormat}" must be one of ['json', 'yaml' or 'table']`, @@ -195,6 +200,7 @@ module.exports = { source, reason, protocolNetwork, + syncingNetwork, }, first, orderByParam, @@ -213,9 +219,10 @@ module.exports = { ) // Format Actions 'protocolNetwork' field to display human-friendly chain aliases instead of CAIP2-IDs - actions.forEach( - action => (action.protocolNetwork = resolveChainAlias(action.protocolNetwork)), - ) + actions.forEach(action => { + action.protocolNetwork = resolveChainAlias(action.protocolNetwork) + action.syncingNetwork = resolveChainAlias(action.syncingNetwork) + }) printObjectOrArray(print, outputFormat, actions, displayProperties) } catch (error) { diff --git a/packages/indexer-cli/src/commands/indexer/allocations/close.ts b/packages/indexer-cli/src/commands/indexer/allocations/close.ts index 796493939..06b5a93d4 100644 --- a/packages/indexer-cli/src/commands/indexer/allocations/close.ts +++ b/packages/indexer-cli/src/commands/indexer/allocations/close.ts @@ -5,7 +5,7 @@ import { loadValidatedConfig } from '../../../config' import { createIndexerManagementClient } from '../../../client' import { closeAllocation } from '../../../allocations' import { validatePOI, printObjectOrArray } from '../../../command-helpers' -import { validateNetworkIdentifier } from '@graphprotocol/indexer-common' +import { validateSupportedNetworkIdentifier } from '@graphprotocol/indexer-common' const HELP = ` ${chalk.bold('graph indexer allocations close')} [options] @@ -63,7 +63,7 @@ module.exports = { return } else { try { - protocolNetwork = validateNetworkIdentifier(network) + protocolNetwork = validateSupportedNetworkIdentifier(network) } catch (error) { spinner.fail(`Invalid value for argument 'network': '${network}' `) process.exitCode = 1 diff --git a/packages/indexer-cli/src/commands/indexer/allocations/collect.ts b/packages/indexer-cli/src/commands/indexer/allocations/collect.ts index 4730988d2..6a95015dc 100644 --- a/packages/indexer-cli/src/commands/indexer/allocations/collect.ts +++ b/packages/indexer-cli/src/commands/indexer/allocations/collect.ts @@ -4,7 +4,7 @@ import chalk from 'chalk' import { loadValidatedConfig } from '../../../config' import { createIndexerManagementClient } from '../../../client' import { submitCollectReceiptsJob } from '../../../allocations' -import { validateNetworkIdentifier } from '@graphprotocol/indexer-common' +import { validateSupportedNetworkIdentifier } from '@graphprotocol/indexer-common' const HELP = ` ${chalk.bold('graph indexer allocations collect')} [options] @@ -60,7 +60,7 @@ module.exports = { return } else { try { - protocolNetwork = validateNetworkIdentifier(network) + protocolNetwork = validateSupportedNetworkIdentifier(network) } catch (error) { spinner.fail(`Invalid value for argument 'network': '${network}' `) process.exitCode = 1 diff --git a/packages/indexer-cli/src/commands/indexer/allocations/create.ts b/packages/indexer-cli/src/commands/indexer/allocations/create.ts index 6986bfbe2..659ce2393 100644 --- a/packages/indexer-cli/src/commands/indexer/allocations/create.ts +++ b/packages/indexer-cli/src/commands/indexer/allocations/create.ts @@ -8,7 +8,7 @@ import { createAllocation } from '../../../allocations' import { processIdentifier, SubgraphIdentifierType, - validateNetworkIdentifier, + validateSupportedNetworkIdentifier, } from '@graphprotocol/indexer-common' import { printObjectOrArray } from '../../../command-helpers' @@ -64,7 +64,7 @@ module.exports = { // This nested try block is necessary to complement the parsing error with the 'network' field. try { - validateNetworkIdentifier(protocolNetwork) + validateSupportedNetworkIdentifier(protocolNetwork) } catch (parsingError) { throw new Error(`Invalid 'network' provided. ${parsingError}`) } diff --git a/packages/indexer-common/src/parsers/basic-types.ts b/packages/indexer-common/src/parsers/basic-types.ts index 68b945b5e..15b43e0c2 100644 --- a/packages/indexer-common/src/parsers/basic-types.ts +++ b/packages/indexer-common/src/parsers/basic-types.ts @@ -20,14 +20,18 @@ export const url = P.regex(/^https?:.*/) // Source: https://github.com/ChainAgnostic/CAIPs/blob/master/CAIPs/caip-2.md export const caip2IdRegex = /^[-a-z0-9]{3,8}:[-_a-zA-Z0-9]{1,32}$/ -const caip2Id = P.regex(caip2IdRegex).chain(validateNetworkIdentifier) +const caip2Id = P.regex(caip2IdRegex) +const supportedCaip2Id = P.regex(caip2IdRegex).chain(validateNetworkIdentifier) // A valid human friendly network name / alias. -const networkAlias = P.regex(/[a-z-]+/).chain(validateNetworkIdentifier) +const networkAlias = P.regex(/[a-z-]+/) +const supportedNetworkAlias = P.regex(/[a-z-]+/).chain(validateNetworkIdentifier) // Either a CAIP-2 or an alias. export const networkIdentifier = P.alt(caip2Id, networkAlias) +export const supportedNetworkIdentifier = P.alt(supportedCaip2Id, supportedNetworkAlias) + // A basic `base58btc` parser for CIDv0 (IPFS Hashes) export const base58 = P.regex(/^Qm[1-9A-HJ-NP-Za-km-z]{44,}$/).desc( 'An IPFS Content Identifer (Qm...)', diff --git a/packages/indexer-common/src/parsers/validators.ts b/packages/indexer-common/src/parsers/validators.ts index cf81e70a2..718fd0b58 100644 --- a/packages/indexer-common/src/parsers/validators.ts +++ b/packages/indexer-common/src/parsers/validators.ts @@ -3,7 +3,7 @@ import P from 'parsimmon' -import { networkIdentifier, base58 } from './basic-types' +import { base58, networkIdentifier, supportedNetworkIdentifier } from './basic-types' export { caip2IdRegex } from './basic-types' // Generic function that takes a parser of type T and attempts to parse it from a string. If it @@ -21,10 +21,15 @@ function parse(parser: P.Parser, input: string): T { `Failed to parse "${input}". Expected: ${expected}. Parsed up to: "${parsed}". Remaining: "${remaining}"`, ) } + export function validateNetworkIdentifier(input: string): string { return parse(networkIdentifier, input) } +export function validateSupportedNetworkIdentifier(input: string): string { + return parse(supportedNetworkIdentifier, input) +} + export function validateIpfsHash(input: string): string { return parse(base58, input) } From 40091ffec0d7554ecf98a5277dc13458b30b48bd Mon Sep 17 00:00:00 2001 From: Ford Date: Mon, 18 Nov 2024 14:29:26 -0800 Subject: [PATCH 3/5] cli: Add support for approving actions by their syncing-network - Add -s,--syncing option to `indexer actions approve queued ...` so user can approve actions by syncing-network. --- .../src/commands/indexer/actions/approve.ts | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/indexer-cli/src/commands/indexer/actions/approve.ts b/packages/indexer-cli/src/commands/indexer/actions/approve.ts index 52e32f55c..b8b6871b7 100644 --- a/packages/indexer-cli/src/commands/indexer/actions/approve.ts +++ b/packages/indexer-cli/src/commands/indexer/actions/approve.ts @@ -8,18 +8,22 @@ import { printObjectOrArray, parseOutputFormat, extractProtocolNetworkOption, + extractSyncingNetworkOption, } from '../../../command-helpers' import { approveActions, fetchActions } from '../../../actions' import { ActionStatus, resolveChainAlias } from '@graphprotocol/indexer-common' const HELP = ` ${chalk.bold('graph indexer actions approve')} [options] [ ...] -${chalk.bold('graph indexer actions approve')} [options] queued --network +${chalk.bold( + 'graph indexer actions approve', +)} [options] queued --network --syncing ${chalk.dim('Options:')} -h, --help Show usage information - -n, --network Filter action selection by their protocol network (mainnet, arbitrum-one, sepolia, arbitrum-sepolia) + -n, --network Filter actions by their protocol network (mainnet, arbitrum-one, sepolia, arbitrum-sepolia) + -s, --syncing Filter (optional) by the syncing network (see https://thegraph.com/networks/ for supported networks) -o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML ` @@ -47,12 +51,17 @@ module.exports = { return } - const protocolNetwork = extractProtocolNetworkOption(parameters.options) let numericActionIDs: number[] + let protocolNetwork: string | undefined = undefined + let syncingNetwork: string | undefined = undefined const config = loadValidatedConfig() const client = await createIndexerManagementClient({ url: config.api }) try { + protocolNetwork = extractProtocolNetworkOption(parameters.options) + + syncingNetwork = extractSyncingNetworkOption(parameters.options) + if (!actionIDs || actionIDs.length === 0) { throw Error(`Missing required argument: 'actionID'`) } @@ -67,6 +76,7 @@ module.exports = { const queuedActions = await fetchActions(client, { status: ActionStatus.QUEUED, protocolNetwork, + syncingNetwork, }) numericActionIDs = queuedActions.map(action => action.id) @@ -101,15 +111,17 @@ module.exports = { const queuedAction = await approveActions(client, numericActionIDs) // Format Actions 'protocolNetwork' field to display human-friendly chain aliases instead of CAIP2-IDs - queuedAction.forEach( - action => (action.protocolNetwork = resolveChainAlias(action.protocolNetwork)), - ) + queuedAction.forEach(action => { + action.protocolNetwork = resolveChainAlias(action.protocolNetwork) + action.syncingNetwork = resolveChainAlias(action.syncingNetwork) + }) actionSpinner.succeed(`Actions approved`) printObjectOrArray(print, outputFormat, queuedAction, [ 'id', 'type', 'protocolNetwork', + 'syncingNetwork', 'deploymentID', 'allocationID', 'amount', From 6f5083ff7c46a4f44b4f9599d3a5a6c7d4e7eb30 Mon Sep 17 00:00:00 2001 From: Ford Date: Mon, 18 Nov 2024 14:41:09 -0800 Subject: [PATCH 4/5] cli: Add filtering of action updates by syncing and protocol networks - Add optional --network and --syncing options to `indexer actions update ...` so user can filter the actions to be updated by protocol network or syncing network. --- packages/indexer-cli/src/actions.ts | 8 +++++ .../src/commands/indexer/actions/update.ts | 34 +++++++++++++++---- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/packages/indexer-cli/src/actions.ts b/packages/indexer-cli/src/actions.ts index c70e42d90..97fb2ec23 100644 --- a/packages/indexer-cli/src/actions.ts +++ b/packages/indexer-cli/src/actions.ts @@ -145,6 +145,8 @@ export function buildActionFilter( status: string | undefined, source: string | undefined, reason: string | undefined, + protocolNetwork: string | undefined, + syncingNetwork: string | undefined, ): ActionFilter { const filter: ActionFilter = {} if (id) { @@ -162,6 +164,12 @@ export function buildActionFilter( if (reason) { filter.reason = reason } + if (protocolNetwork) { + filter.protocolNetwork = protocolNetwork + } + if (syncingNetwork) { + filter.syncingNetwork = syncingNetwork + } if (Object.keys(filter).length === 0) { throw Error( `No action filter provided, please specify at least one filter using ['--id', '--type', '--status', '--source', '--reason']`, diff --git a/packages/indexer-cli/src/commands/indexer/actions/update.ts b/packages/indexer-cli/src/commands/indexer/actions/update.ts index 6ea470cac..9c1307ddd 100644 --- a/packages/indexer-cli/src/commands/indexer/actions/update.ts +++ b/packages/indexer-cli/src/commands/indexer/actions/update.ts @@ -9,7 +9,12 @@ import { } from '@graphprotocol/indexer-common' import { loadValidatedConfig } from '../../../config' import { createIndexerManagementClient } from '../../../client' -import { fixParameters, printObjectOrArray } from '../../../command-helpers' +import { + extractProtocolNetworkOption, + extractSyncingNetworkOption, + fixParameters, + printObjectOrArray, +} from '../../../command-helpers' import { buildActionFilter, parseActionUpdateInput, @@ -22,7 +27,9 @@ ${chalk.bold('graph indexer actions update')} [options] [ ...] ${chalk.dim('Options:')} - -h, --help Show usage information + -h, --help Show usage information + -n, --network Filter by protocol network (mainnet, arbitrum-one, sepolia, arbitrum-sepolia) + -s, --syncing Filter by the syncing network (see https://thegraph.com/networks/ for supported networks) --id Filter by actionID --type allocate|unallocate|reallocate Filter by type --status queued|approved|pending|success|failed|canceled Filter by status @@ -47,6 +54,8 @@ module.exports = { let actionFilter: ActionFilter = {} const outputFormat = o || output || 'table' + let protocolNetwork: string | undefined = undefined + let syncingNetwork: string | undefined = undefined // eslint-disable-next-line @typescript-eslint/no-explicit-any if (help || h) { @@ -54,6 +63,9 @@ module.exports = { return } try { + protocolNetwork = extractProtocolNetworkOption(parameters.options) + + syncingNetwork = extractSyncingNetworkOption(parameters.options) if (!['json', 'yaml', 'table'].includes(outputFormat)) { throw Error( `Invalid output format "${outputFormat}" must be one of ['json', 'yaml' or 'table']`, @@ -74,7 +86,15 @@ module.exports = { ...Object.fromEntries([...partition(2, 2, kvs)]), }) - actionFilter = buildActionFilter(id, type, status, source, reason) + actionFilter = buildActionFilter( + id, + type, + status, + source, + reason, + protocolNetwork, + syncingNetwork, + ) inputSpinner.succeed('Processed input parameters') } catch (error) { @@ -104,6 +124,7 @@ module.exports = { 'id', 'type', 'protocolNetwork', + 'syncingNetwork', 'deploymentID', 'allocationID', 'amount', @@ -118,9 +139,10 @@ module.exports = { ] // Format Actions 'protocolNetwork' field to display human-friendly chain aliases instead of CAIP2-IDs - actionsUpdated.forEach( - action => (action.protocolNetwork = resolveChainAlias(action.protocolNetwork)), - ) + actionsUpdated.forEach(action => { + action.protocolNetwork = resolveChainAlias(action.protocolNetwork) + action.syncingNetwork = resolveChainAlias(action.syncingNetwork) + }) printObjectOrArray(print, outputFormat, actionsUpdated, displayProperties) } catch (error) { From d59ed259020848d827c9d2fc1635355d7c0fff26 Mon Sep 17 00:00:00 2001 From: Ford Date: Mon, 18 Nov 2024 15:36:08 -0800 Subject: [PATCH 5/5] cli: Add support for deleting subgraphs by protocol or syncing network - Can now supply multiple filters to `indexer actions delete`: --status, --syncing, and --network. --- .../src/commands/indexer/actions/delete.ts | 67 ++++++++++++++----- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/packages/indexer-cli/src/commands/indexer/actions/delete.ts b/packages/indexer-cli/src/commands/indexer/actions/delete.ts index 84940bf0e..f93d1e316 100644 --- a/packages/indexer-cli/src/commands/indexer/actions/delete.ts +++ b/packages/indexer-cli/src/commands/indexer/actions/delete.ts @@ -3,19 +3,29 @@ import chalk from 'chalk' import { loadValidatedConfig } from '../../../config' import { createIndexerManagementClient } from '../../../client' -import { fixParameters } from '../../../command-helpers' +import { + extractProtocolNetworkOption, + extractSyncingNetworkOption, + fixParameters, +} from '../../../command-helpers' import { deleteActions, fetchActions } from '../../../actions' const HELP = ` ${chalk.bold('graph indexer actions delete')} [options] all ${chalk.bold('graph indexer actions delete')} [options] [ ...] +${chalk.bold('graph indexer actions delete')} [options] ${chalk.dim('Options:')} -h, --help Show usage information + -n, --network Filter by protocol network (mainnet, arbitrum-one, sepolia, arbitrum-sepolia) + -s, --syncing Filter by the syncing network (see https://thegraph.com/networks/ for supported networks) --status queued|approved|pending|success|failed|canceled Filter by status -o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML ` +function isNumber(value?: string | number): boolean { + return value != null && value !== '' && !isNaN(Number(value.toString())) +} module.exports = { name: 'delete', @@ -32,18 +42,37 @@ module.exports = { const outputFormat = o || output || 'table' const toHelp = help || h || undefined + let protocolNetwork: string | undefined = undefined + let syncingNetwork: string | undefined = undefined + let deleteType: 'ids' | 'all' | 'filter' = 'filter' + if (toHelp) { inputSpinner.stopAndPersist({ symbol: '💁', text: HELP }) return } try { + protocolNetwork = extractProtocolNetworkOption(parameters.options) + + syncingNetwork = extractSyncingNetworkOption(parameters.options) + if (!['json', 'yaml', 'table'].includes(outputFormat)) { throw Error( `Invalid output format "${outputFormat}", must be one of ['json', 'yaml', 'table']`, ) } + if ( + !status && + !syncingNetwork && + !protocolNetwork && + (!actionIDs || actionIDs.length === 0) + ) { + throw Error( + `Required at least one argument: actionID(s), 'all', '--status' filter, '--network' filter, or '--syncing' filter`, + ) + } + if ( status && !['queued', 'approved', 'pending', 'success', 'failed', 'canceled'].includes( @@ -55,18 +84,22 @@ module.exports = { ) } - if (actionIDs[0] == 'all') { - if (status || actionIDs.length > 1) { + if (actionIDs && actionIDs[0] == 'all') { + deleteType = 'all' + if (status || protocolNetwork || syncingNetwork || actionIDs.length > 1) { throw Error( - `Invalid query, cannot specify '--status' filter or multiple ids in addition to 'action = all'`, + `Invalid query, cannot specify '--status'|'--network'|'--syncing' filters or action ids in addition to 'action = all'`, ) } } - if (!status && (!actionIDs || actionIDs.length === 0)) { - throw Error( - `Required at least one argument: actionID(s), 'all', or '--status' filter`, - ) + if (actionIDs && isNumber(actionIDs[0])) { + deleteType = 'ids' + if (status || protocolNetwork || syncingNetwork || actionIDs.length > 1) { + throw Error( + `Invalid query, cannot specify '--status'|'--network'|'--syncing' filters or action ids in addition to 'action = all'`, + ) + } } inputSpinner.succeed('Processed input parameters') @@ -81,13 +114,17 @@ module.exports = { try { const config = loadValidatedConfig() const client = await createIndexerManagementClient({ url: config.api }) - - const numericActionIDs: number[] = - actionIDs[0] == 'all' - ? (await fetchActions(client, {})).map(action => action.id) - : status - ? (await fetchActions(client, { status })).map(action => action.id) - : actionIDs.map(action => +action) + let numericActionIDs: number[] = [] + + if (deleteType === 'all') { + numericActionIDs = (await fetchActions(client, {})).map(action => action.id) + } else if (deleteType === 'filter') { + numericActionIDs = ( + await fetchActions(client, { status, protocolNetwork, syncingNetwork }) + ).map(action => action.id) + } else if (deleteType === 'ids') { + numericActionIDs = actionIDs.map(action => +action) + } const numDeleted = await deleteActions(client, numericActionIDs)