diff --git a/common/changes/@subsquid/evm-processor/field-selection-schema_2024-02-21-13-33.json b/common/changes/@subsquid/evm-processor/field-selection-schema_2024-02-21-13-33.json new file mode 100644 index 000000000..7098e8e0d --- /dev/null +++ b/common/changes/@subsquid/evm-processor/field-selection-schema_2024-02-21-13-33.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/evm-processor", + "comment": "add field selection schema validation", + "type": "minor" + } + ], + "packageName": "@subsquid/evm-processor" +} \ No newline at end of file diff --git a/common/changes/@subsquid/substrate-processor/field-selection-schema_2024-02-21-13-33.json b/common/changes/@subsquid/substrate-processor/field-selection-schema_2024-02-21-13-33.json new file mode 100644 index 000000000..f0e2131ac --- /dev/null +++ b/common/changes/@subsquid/substrate-processor/field-selection-schema_2024-02-21-13-33.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/substrate-processor", + "comment": "add field selection schema validation", + "type": "minor" + } + ], + "packageName": "@subsquid/substrate-processor" +} \ No newline at end of file diff --git a/common/changes/@subsquid/util-internal-validation/field-selection-schema_2024-02-21-13-33.json b/common/changes/@subsquid/util-internal-validation/field-selection-schema_2024-02-21-13-33.json new file mode 100644 index 000000000..8425cfe00 --- /dev/null +++ b/common/changes/@subsquid/util-internal-validation/field-selection-schema_2024-02-21-13-33.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/util-internal-validation", + "comment": "add `BOOLEAN` primitive", + "type": "minor" + } + ], + "packageName": "@subsquid/util-internal-validation" +} \ No newline at end of file diff --git a/evm/evm-processor/src/mapping/selection.ts b/evm/evm-processor/src/mapping/selection.ts new file mode 100644 index 000000000..548cc9f81 --- /dev/null +++ b/evm/evm-processor/src/mapping/selection.ts @@ -0,0 +1,125 @@ +import {FieldSelection} from '../interfaces/data' +import {object, option, BOOLEAN} from '@subsquid/util-internal-validation' + + +type GetFieldSelectionSchema = {[K in keyof T]-?: typeof FIELD} + + +const FIELD = option(BOOLEAN) + + +export function getBlockHeaderSelectionValidator() { + let fields: GetFieldSelectionSchema = { + nonce: FIELD, + sha3Uncles: FIELD, + logsBloom: FIELD, + transactionsRoot: FIELD, + stateRoot: FIELD, + receiptsRoot: FIELD, + mixHash: FIELD, + miner: FIELD, + difficulty: FIELD, + totalDifficulty: FIELD, + extraData: FIELD, + size: FIELD, + gasLimit: FIELD, + gasUsed: FIELD, + baseFeePerGas: FIELD, + timestamp: FIELD, + l1BlockNumber: FIELD, + } + return object(fields) +} + + +export function getTxSelectionValidator() { + let fields: GetFieldSelectionSchema = { + hash: FIELD, + from: FIELD, + to: FIELD, + gas: FIELD, + gasPrice: FIELD, + maxFeePerGas: FIELD, + maxPriorityFeePerGas: FIELD, + sighash: FIELD, + input: FIELD, + nonce: FIELD, + value: FIELD, + v: FIELD, + r: FIELD, + s: FIELD, + yParity: FIELD, + chainId: FIELD, + gasUsed: FIELD, + cumulativeGasUsed: FIELD, + effectiveGasPrice: FIELD, + contractAddress: FIELD, + type: FIELD, + status: FIELD, + } + return object(fields) +} + + +export function getLogSelectionValidator() { + let fields: GetFieldSelectionSchema = { + transactionHash: FIELD, + address: FIELD, + data: FIELD, + topics: FIELD, + } + return object(fields) +} + + +export function getTraceSelectionValidator() { + let fields: GetFieldSelectionSchema = { + callCallType: FIELD, + callFrom: FIELD, + callGas: FIELD, + callInput: FIELD, + callResultGasUsed: FIELD, + callResultOutput: FIELD, + callSighash: FIELD, + callTo: FIELD, + callValue: FIELD, + createFrom: FIELD, + createGas: FIELD, + createInit: FIELD, + createResultAddress: FIELD, + createResultCode: FIELD, + createResultGasUsed: FIELD, + createValue: FIELD, + error: FIELD, + revertReason: FIELD, + rewardAuthor: FIELD, + rewardType: FIELD, + rewardValue: FIELD, + subtraces: FIELD, + suicideAddress: FIELD, + suicideBalance: FIELD, + suicideRefundAddress: FIELD, + } + return object(fields) +} + + +export function getStateDiffSelectionValidator() { + let fields: GetFieldSelectionSchema = { + kind: FIELD, + next: FIELD, + prev: FIELD, + } + return object(fields) +} + + +export function getFieldSelectionValidator() { + return object({ + block: option(getBlockHeaderSelectionValidator()), + log: option(getLogSelectionValidator()), + transaction: option(getTxSelectionValidator()), + trace: option(getTraceSelectionValidator()), + stateDiff: option(getStateDiffSelectionValidator()), + }) +} \ No newline at end of file diff --git a/evm/evm-processor/src/processor.ts b/evm/evm-processor/src/processor.ts index a7748fbc1..8a2b8be01 100644 --- a/evm/evm-processor/src/processor.ts +++ b/evm/evm-processor/src/processor.ts @@ -5,12 +5,14 @@ import {assertNotNull, def, runProgram} from '@subsquid/util-internal' import {ArchiveClient} from '@subsquid/util-internal-archive-client' import {Database, getOrGenerateSquidId, PrometheusServer, Runner} from '@subsquid/util-internal-processor-tools' import {applyRangeBound, mergeRangeRequests, Range, RangeRequest} from '@subsquid/util-internal-range' +import {cast} from '@subsquid/util-internal-validation' import assert from 'assert' import {EvmArchive} from './ds-archive/client' import {EvmRpcDataSource} from './ds-rpc/client' import {Chain} from './interfaces/chain' import {BlockData, DEFAULT_FIELDS, FieldSelection} from './interfaces/data' import {DataRequest, LogRequest, StateDiffRequest, TraceRequest, TransactionRequest} from './interfaces/data-request' +import {getFieldSelectionValidator} from './mapping/selection' export interface RpcEndpointSettings { @@ -314,7 +316,8 @@ export class EvmBatchProcessor { */ setFields(fields: T): EvmBatchProcessor { this.assertNotRunning() - this.fields = fields + let validator = getFieldSelectionValidator() + this.fields = cast(validator, fields) return this as any } diff --git a/substrate/substrate-processor/package.json b/substrate/substrate-processor/package.json index 0ca263684..66528482b 100644 --- a/substrate/substrate-processor/package.json +++ b/substrate/substrate-processor/package.json @@ -27,7 +27,8 @@ "@subsquid/util-internal-ingest-tools": "^1.1.0", "@subsquid/util-internal-json": "^1.2.2", "@subsquid/util-internal-processor-tools": "^4.0.1", - "@subsquid/util-internal-range": "^0.1.0" + "@subsquid/util-internal-range": "^0.1.0", + "@subsquid/util-internal-validation": "~0.2.0" }, "peerDependencies": { "@subsquid/substrate-runtime": "^1.0.3" diff --git a/substrate/substrate-processor/src/processor.ts b/substrate/substrate-processor/src/processor.ts index 672be01ee..84e0a6889 100644 --- a/substrate/substrate-processor/src/processor.ts +++ b/substrate/substrate-processor/src/processor.ts @@ -15,6 +15,7 @@ import {assertNotNull, def, runProgram} from '@subsquid/util-internal' import {ArchiveClient} from '@subsquid/util-internal-archive-client' import {Batch, Database, getOrGenerateSquidId, PrometheusServer, Runner} from '@subsquid/util-internal-processor-tools' import {applyRangeBound, mergeRangeRequests, Range, RangeRequest} from '@subsquid/util-internal-range' +import {cast} from '@subsquid/util-internal-validation' import assert from 'assert' import {Chain} from './chain' import {SubstrateArchive} from './ds-archive' @@ -30,6 +31,7 @@ import { GearMessageQueuedRequest, GearUserMessageSentRequest } from './interfaces/data-request' +import {getFieldSelectionValidator} from './selection' export interface RpcEndpointSettings { @@ -284,7 +286,8 @@ export class SubstrateBatchProcessor { */ setFields(fields: T): SubstrateBatchProcessor { this.assertNotRunning() - this.fields = fields + let validator = getFieldSelectionValidator() + this.fields = cast(validator, fields) return this as any } diff --git a/substrate/substrate-processor/src/selection.ts b/substrate/substrate-processor/src/selection.ts new file mode 100644 index 000000000..9e881ad68 --- /dev/null +++ b/substrate/substrate-processor/src/selection.ts @@ -0,0 +1,66 @@ +import {FieldSelection} from './interfaces/data' +import {object, option, BOOLEAN} from '@subsquid/util-internal-validation' + + +type GetFieldSelectionSchema = {[K in keyof T]-?: typeof FIELD} + + +const FIELD = option(BOOLEAN) + + +export function getBlockHeaderSelectionValidator() { + let fields: GetFieldSelectionSchema = { + digest: FIELD, + extrinsicsRoot: FIELD, + stateRoot: FIELD, + timestamp: FIELD, + validator: FIELD, + } + return object(fields) +} + + +export function getExtrinsicSelectionValidator() { + let fields: GetFieldSelectionSchema = { + hash: FIELD, + error: FIELD, + fee: FIELD, + signature: FIELD, + success: FIELD, + tip: FIELD, + version: FIELD, + } + return object(fields) +} + + +export function getEventSelectionValidator() { + let fields: GetFieldSelectionSchema = { + args: FIELD, + name: FIELD, + phase: FIELD, + } + return object(fields) +} + + +export function getCallSelectionValidator() { + let fields: GetFieldSelectionSchema = { + args: FIELD, + error: FIELD, + name: FIELD, + origin: FIELD, + success: FIELD + } + return object(fields) +} + + +export function getFieldSelectionValidator() { + return object({ + block: option(getBlockHeaderSelectionValidator()), + extrinsic: option(getExtrinsicSelectionValidator()), + call: option(getCallSelectionValidator()), + event: option(getEventSelectionValidator()), + }) +} \ No newline at end of file diff --git a/test/erc20-transfers/src/processor.ts b/test/erc20-transfers/src/processor.ts index 981712d82..a446b5148 100644 --- a/test/erc20-transfers/src/processor.ts +++ b/test/erc20-transfers/src/processor.ts @@ -14,7 +14,7 @@ const processor = new EvmBatchProcessor() .setBlockRange({from: 153000000}) .setFields({ block: {size: true}, - log: {transactionHash: true} + log: {transactionHash: true, foo: true} }) .addLog({ address: [CONTRACT], diff --git a/util/util-internal-validation/src/primitives.ts b/util/util-internal-validation/src/primitives.ts index cf5c06b59..1bc2e8a46 100644 --- a/util/util-internal-validation/src/primitives.ts +++ b/util/util-internal-validation/src/primitives.ts @@ -140,3 +140,16 @@ export const BYTES: Validator = { return '0x' } } + +export const BOOLEAN: Validator = { + cast(value: unknown): boolean | ValidationFailure { + return this.validate(value) || value as boolean + }, + validate(value: unknown): ValidationFailure | undefined { + if (typeof value === 'boolean') return + return new ValidationFailure(value, `{value} is not a boolean`) + }, + phantom(): boolean { + return false + } +} \ No newline at end of file