From 16e7880a1a8428d173c6b50a3b4c9a653e47e93c Mon Sep 17 00:00:00 2001 From: Eldar Gabdullin Date: Thu, 30 Nov 2023 15:56:52 +0400 Subject: [PATCH] RPC data ingestion reform (#217) --- .../evm-processor/subs_2023-11-30-06-20.json | 20 + .../http-client/subs_2023-11-30-06-20.json | 10 + .../rpc-client/subs_2023-11-30-06-20.json | 15 + .../subs_2023-11-30-06-20.json | 10 + .../substrate-data/subs_2023-11-30-06-20.json | 10 + .../substrate-dump/subs_2023-11-30-06-20.json | 10 + .../subs_2023-11-30-06-20.json | 10 + .../subs_2023-11-30-06-20.json | 20 + .../subs_2023-11-30-06-20.json | 10 + .../subs_2023-11-30-06-20.json | 10 + .../subs_2023-11-30-06-20.json | 20 + .../subs_2023-11-30-06-20.json | 10 + .../subs_2023-11-30-06-20.json | 10 + .../util-internal/subs_2023-11-30-06-20.json | 15 + common/config/rush/pnpm-lock.yaml | 114 +-- evm/evm-processor/package.json | 7 +- evm/evm-processor/src/ds-archive/client.ts | 157 +++- evm/evm-processor/src/ds-archive/gateway.ts | 36 - evm/evm-processor/src/ds-archive/mapping.ts | 313 ------- evm/evm-processor/src/ds-archive/schema.ts | 50 ++ evm/evm-processor/src/ds-rpc/client.ts | 719 +++++----------- evm/evm-processor/src/ds-rpc/filter.ts | 226 +++++ evm/evm-processor/src/ds-rpc/mapping.ts | 780 +++++++----------- evm/evm-processor/src/ds-rpc/request.ts | 118 +++ evm/evm-processor/src/ds-rpc/rpc-data.ts | 217 +++++ evm/evm-processor/src/ds-rpc/rpc.ts | 704 +++++++++++----- evm/evm-processor/src/ds-rpc/schema.ts | 201 +++++ evm/evm-processor/src/ds-rpc/util.ts | 23 + evm/evm-processor/src/interfaces/base.ts | 5 + .../src/interfaces/data-request.ts | 3 +- evm/evm-processor/src/interfaces/data.ts | 51 +- evm/evm-processor/src/interfaces/evm.ts | 52 +- evm/evm-processor/src/mapping/entities.ts | 326 ++++++++ evm/evm-processor/src/mapping/relations.ts | 79 ++ evm/evm-processor/src/mapping/schema.ts | 220 +++++ evm/evm-processor/src/processor.ts | 431 +++++++--- evm/evm-processor/src/util.ts | 21 - rush.json | 10 +- substrate/substrate-data-raw/package.json | 6 +- .../substrate-data-raw/src/datasource.ts | 342 +++++--- substrate/substrate-data-raw/src/fetch1.ts | 258 ++++++ substrate/substrate-data-raw/src/fetcher.ts | 124 --- substrate/substrate-data-raw/src/index.ts | 1 + .../substrate-data-raw/src/interfaces.ts | 26 +- substrate/substrate-data-raw/src/rpc.ts | 93 ++- .../src/runtimeVersionTracker.ts | 63 ++ substrate/substrate-data-raw/src/util.ts | 4 +- substrate/substrate-data/src/datasource.ts | 45 +- .../substrate-data/src/interfaces/data-raw.ts | 2 + substrate/substrate-data/src/parser.ts | 129 +-- .../substrate-data/src/runtime-tracker.ts | 38 +- substrate/substrate-dump/src/dumper.ts | 13 +- substrate/substrate-dump/src/prometheus.ts | 6 +- substrate/substrate-ingest/src/ingest.ts | 9 +- substrate/substrate-processor/package.json | 4 +- .../substrate-processor/src/ds-archive.ts | 61 +- .../src/{filter.ts => ds-rpc-filter.ts} | 186 +---- substrate/substrate-processor/src/ds-rpc.ts | 74 +- substrate/substrate-processor/src/mapping.ts | 69 +- .../substrate-processor/src/processor.ts | 334 +++++--- test/balances/src/processor.ts | 12 +- .../.env | 0 .../Makefile | 0 .../db/migrations/1682961487386-Data.js | 0 .../docker-compose.yml | 0 .../erc20.json | 0 .../package.json | 2 +- .../schema.graphql | 0 .../src/abi/abi.support.ts | 0 .../src/abi/erc20.abi.ts | 0 .../src/abi/erc20.ts | 0 .../src/model/generated/index.ts | 0 .../src/model/generated/marshal.ts | 0 .../src/model/generated/transfer.model.ts | 0 .../src/model/index.ts | 0 .../src/processor.ts | 17 +- .../tsconfig.json | 0 util/rpc-client/src/client.ts | 119 ++- util/rpc-client/src/index.ts | 1 + util/rpc-client/src/interfaces.ts | 17 +- util/rpc-client/src/subscriptions.ts | 171 ++++ util/rpc-client/src/transport/ws.ts | 42 +- .../util-internal-archive-client/package.json | 4 +- .../src/client.ts | 18 +- util/util-internal-ingest-tools/package.json | 9 + .../util-internal-ingest-tools/src/archive.ts | 64 ++ util/util-internal-ingest-tools/src/cold.ts | 76 ++ .../src/consistency-error.ts | 20 + util/util-internal-ingest-tools/src/hot.ts | 160 ++-- util/util-internal-ingest-tools/src/index.ts | 6 +- .../util-internal-ingest-tools/src/invalid.ts | 27 + util/util-internal-ingest-tools/src/ref.ts | 30 + .../src/requests-tracker.ts | 42 - .../src/datasource.ts | 6 +- .../src/filter.ts | 154 ++++ .../src/index.ts | 5 +- .../src/ingest.ts | 322 -------- .../src/runner.ts | 50 +- .../util-internal-processor-tools/src/util.ts | 22 +- util/util-internal-range/src/index.ts | 1 + util/util-internal-range/src/requests.ts | 57 ++ util/util-internal-range/src/util.ts | 10 + util/util-internal-validation/package.json | 32 + .../src/composite/array.ts | 37 + .../src/composite/constant.ts | 34 + .../src/composite/key-tagged-union.ts | 79 ++ .../src/composite/nullable.ts | 24 + .../src/composite/object.ts | 50 ++ .../src/composite/one-of.ts | 89 ++ .../src/composite/option.ts | 23 + .../src/composite/record.ts | 44 + .../src/composite/ref.ts | 28 + .../src/composite/sentinel.ts | 65 ++ .../src/composite/tagged-union.ts | 56 ++ util/util-internal-validation/src/dsl.ts | 116 +++ util/util-internal-validation/src/error.ts | 37 + util/util-internal-validation/src/index.ts | 4 + .../util-internal-validation/src/interface.ts | 18 + .../src/primitives.ts | 142 ++++ util/util-internal-validation/src/util.ts | 16 + util/util-internal-validation/tsconfig.json | 18 + util/util-internal/src/async.ts | 98 ++- util/util-internal/src/misc.ts | 34 + 123 files changed, 6287 insertions(+), 3151 deletions(-) create mode 100644 common/changes/@subsquid/evm-processor/subs_2023-11-30-06-20.json create mode 100644 common/changes/@subsquid/http-client/subs_2023-11-30-06-20.json create mode 100644 common/changes/@subsquid/rpc-client/subs_2023-11-30-06-20.json create mode 100644 common/changes/@subsquid/substrate-data-raw/subs_2023-11-30-06-20.json create mode 100644 common/changes/@subsquid/substrate-data/subs_2023-11-30-06-20.json create mode 100644 common/changes/@subsquid/substrate-dump/subs_2023-11-30-06-20.json create mode 100644 common/changes/@subsquid/substrate-ingest/subs_2023-11-30-06-20.json create mode 100644 common/changes/@subsquid/substrate-processor/subs_2023-11-30-06-20.json create mode 100644 common/changes/@subsquid/util-internal-archive-client/subs_2023-11-30-06-20.json create mode 100644 common/changes/@subsquid/util-internal-ingest-tools/subs_2023-11-30-06-20.json create mode 100644 common/changes/@subsquid/util-internal-processor-tools/subs_2023-11-30-06-20.json create mode 100644 common/changes/@subsquid/util-internal-range/subs_2023-11-30-06-20.json create mode 100644 common/changes/@subsquid/util-internal-validation/subs_2023-11-30-06-20.json create mode 100644 common/changes/@subsquid/util-internal/subs_2023-11-30-06-20.json delete mode 100644 evm/evm-processor/src/ds-archive/gateway.ts delete mode 100644 evm/evm-processor/src/ds-archive/mapping.ts create mode 100644 evm/evm-processor/src/ds-archive/schema.ts create mode 100644 evm/evm-processor/src/ds-rpc/filter.ts create mode 100644 evm/evm-processor/src/ds-rpc/request.ts create mode 100644 evm/evm-processor/src/ds-rpc/rpc-data.ts create mode 100644 evm/evm-processor/src/ds-rpc/schema.ts create mode 100644 evm/evm-processor/src/ds-rpc/util.ts create mode 100644 evm/evm-processor/src/interfaces/base.ts create mode 100644 evm/evm-processor/src/mapping/entities.ts create mode 100644 evm/evm-processor/src/mapping/relations.ts create mode 100644 evm/evm-processor/src/mapping/schema.ts delete mode 100644 evm/evm-processor/src/util.ts create mode 100644 substrate/substrate-data-raw/src/fetch1.ts delete mode 100644 substrate/substrate-data-raw/src/fetcher.ts create mode 100644 substrate/substrate-data-raw/src/runtimeVersionTracker.ts rename substrate/substrate-processor/src/{filter.ts => ds-rpc-filter.ts} (55%) rename test/{eth-usdc-transfers => erc20-transfers}/.env (100%) rename test/{eth-usdc-transfers => erc20-transfers}/Makefile (100%) rename test/{eth-usdc-transfers => erc20-transfers}/db/migrations/1682961487386-Data.js (100%) rename test/{eth-usdc-transfers => erc20-transfers}/docker-compose.yml (100%) rename test/{eth-usdc-transfers => erc20-transfers}/erc20.json (100%) rename test/{eth-usdc-transfers => erc20-transfers}/package.json (94%) rename test/{eth-usdc-transfers => erc20-transfers}/schema.graphql (100%) rename test/{eth-usdc-transfers => erc20-transfers}/src/abi/abi.support.ts (100%) rename test/{eth-usdc-transfers => erc20-transfers}/src/abi/erc20.abi.ts (100%) rename test/{eth-usdc-transfers => erc20-transfers}/src/abi/erc20.ts (100%) rename test/{eth-usdc-transfers => erc20-transfers}/src/model/generated/index.ts (100%) rename test/{eth-usdc-transfers => erc20-transfers}/src/model/generated/marshal.ts (100%) rename test/{eth-usdc-transfers => erc20-transfers}/src/model/generated/transfer.model.ts (100%) rename test/{eth-usdc-transfers => erc20-transfers}/src/model/index.ts (100%) rename test/{eth-usdc-transfers => erc20-transfers}/src/processor.ts (76%) rename test/{eth-usdc-transfers => erc20-transfers}/tsconfig.json (100%) create mode 100644 util/rpc-client/src/subscriptions.ts create mode 100644 util/util-internal-ingest-tools/src/archive.ts create mode 100644 util/util-internal-ingest-tools/src/cold.ts create mode 100644 util/util-internal-ingest-tools/src/consistency-error.ts create mode 100644 util/util-internal-ingest-tools/src/invalid.ts create mode 100644 util/util-internal-ingest-tools/src/ref.ts delete mode 100644 util/util-internal-ingest-tools/src/requests-tracker.ts create mode 100644 util/util-internal-processor-tools/src/filter.ts delete mode 100644 util/util-internal-processor-tools/src/ingest.ts create mode 100644 util/util-internal-range/src/requests.ts create mode 100644 util/util-internal-validation/package.json create mode 100644 util/util-internal-validation/src/composite/array.ts create mode 100644 util/util-internal-validation/src/composite/constant.ts create mode 100644 util/util-internal-validation/src/composite/key-tagged-union.ts create mode 100644 util/util-internal-validation/src/composite/nullable.ts create mode 100644 util/util-internal-validation/src/composite/object.ts create mode 100644 util/util-internal-validation/src/composite/one-of.ts create mode 100644 util/util-internal-validation/src/composite/option.ts create mode 100644 util/util-internal-validation/src/composite/record.ts create mode 100644 util/util-internal-validation/src/composite/ref.ts create mode 100644 util/util-internal-validation/src/composite/sentinel.ts create mode 100644 util/util-internal-validation/src/composite/tagged-union.ts create mode 100644 util/util-internal-validation/src/dsl.ts create mode 100644 util/util-internal-validation/src/error.ts create mode 100644 util/util-internal-validation/src/index.ts create mode 100644 util/util-internal-validation/src/interface.ts create mode 100644 util/util-internal-validation/src/primitives.ts create mode 100644 util/util-internal-validation/src/util.ts create mode 100644 util/util-internal-validation/tsconfig.json diff --git a/common/changes/@subsquid/evm-processor/subs_2023-11-30-06-20.json b/common/changes/@subsquid/evm-processor/subs_2023-11-30-06-20.json new file mode 100644 index 000000000..84e45c21c --- /dev/null +++ b/common/changes/@subsquid/evm-processor/subs_2023-11-30-06-20.json @@ -0,0 +1,20 @@ +{ + "changes": [ + { + "packageName": "@subsquid/evm-processor", + "comment": "introduce RPC data filtering", + "type": "minor" + }, + { + "packageName": "@subsquid/evm-processor", + "comment": "support chain head tracking via RPC subscription", + "type": "minor" + }, + { + "packageName": "@subsquid/evm-processor", + "comment": "improve hot block data ingestion speed via batch processing", + "type": "minor" + } + ], + "packageName": "@subsquid/evm-processor" +} diff --git a/common/changes/@subsquid/http-client/subs_2023-11-30-06-20.json b/common/changes/@subsquid/http-client/subs_2023-11-30-06-20.json new file mode 100644 index 000000000..b009bb7d6 --- /dev/null +++ b/common/changes/@subsquid/http-client/subs_2023-11-30-06-20.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/http-client", + "comment": "update dependencies", + "type": "patch" + } + ], + "packageName": "@subsquid/http-client" +} diff --git a/common/changes/@subsquid/rpc-client/subs_2023-11-30-06-20.json b/common/changes/@subsquid/rpc-client/subs_2023-11-30-06-20.json new file mode 100644 index 000000000..55084bd7b --- /dev/null +++ b/common/changes/@subsquid/rpc-client/subs_2023-11-30-06-20.json @@ -0,0 +1,15 @@ +{ + "changes": [ + { + "packageName": "@subsquid/rpc-client", + "comment": "introduce subscriptions and notifications", + "type": "minor" + }, + { + "packageName": "@subsquid/rpc-client", + "comment": "introduce `.validateError` call option", + "type": "minor" + } + ], + "packageName": "@subsquid/rpc-client" +} diff --git a/common/changes/@subsquid/substrate-data-raw/subs_2023-11-30-06-20.json b/common/changes/@subsquid/substrate-data-raw/subs_2023-11-30-06-20.json new file mode 100644 index 000000000..e73249d9d --- /dev/null +++ b/common/changes/@subsquid/substrate-data-raw/subs_2023-11-30-06-20.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/substrate-data-raw", + "comment": "overhaul hot block ingestion", + "type": "major" + } + ], + "packageName": "@subsquid/substrate-data-raw" +} diff --git a/common/changes/@subsquid/substrate-data/subs_2023-11-30-06-20.json b/common/changes/@subsquid/substrate-data/subs_2023-11-30-06-20.json new file mode 100644 index 000000000..6eebf2daa --- /dev/null +++ b/common/changes/@subsquid/substrate-data/subs_2023-11-30-06-20.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/substrate-data", + "comment": "overhaul hot block ingestion", + "type": "major" + } + ], + "packageName": "@subsquid/substrate-data" +} diff --git a/common/changes/@subsquid/substrate-dump/subs_2023-11-30-06-20.json b/common/changes/@subsquid/substrate-dump/subs_2023-11-30-06-20.json new file mode 100644 index 000000000..c5fbd24e6 --- /dev/null +++ b/common/changes/@subsquid/substrate-dump/subs_2023-11-30-06-20.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/substrate-dump", + "comment": "upgrade `@subsquid/*` dependencies", + "type": "minor" + } + ], + "packageName": "@subsquid/substrate-dump" +} \ No newline at end of file diff --git a/common/changes/@subsquid/substrate-ingest/subs_2023-11-30-06-20.json b/common/changes/@subsquid/substrate-ingest/subs_2023-11-30-06-20.json new file mode 100644 index 000000000..cef206706 --- /dev/null +++ b/common/changes/@subsquid/substrate-ingest/subs_2023-11-30-06-20.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/substrate-ingest", + "comment": "upgrade `@subsquid/*` dependencies", + "type": "minor" + } + ], + "packageName": "@subsquid/substrate-ingest" +} \ No newline at end of file diff --git a/common/changes/@subsquid/substrate-processor/subs_2023-11-30-06-20.json b/common/changes/@subsquid/substrate-processor/subs_2023-11-30-06-20.json new file mode 100644 index 000000000..a07440aa5 --- /dev/null +++ b/common/changes/@subsquid/substrate-processor/subs_2023-11-30-06-20.json @@ -0,0 +1,20 @@ +{ + "changes": [ + { + "packageName": "@subsquid/substrate-processor", + "comment": "support chain head tracking via RPC subscription", + "type": "minor" + }, + { + "packageName": "@subsquid/substrate-processor", + "comment": "deprecate `.setDataSource()` in favour of separate `.setArchive()` and `.setRpcEndpoint()` methods", + "type": "minor" + }, + { + "packageName": "@subsquid/substrate-processor", + "comment": "introduce `.setRpcDataIngestionSettings()` and deprecate `.setChainPollInterval()`, `.useArchiveOnly()`", + "type": "minor" + } + ], + "packageName": "@subsquid/substrate-processor" +} diff --git a/common/changes/@subsquid/util-internal-archive-client/subs_2023-11-30-06-20.json b/common/changes/@subsquid/util-internal-archive-client/subs_2023-11-30-06-20.json new file mode 100644 index 000000000..2642c40e1 --- /dev/null +++ b/common/changes/@subsquid/util-internal-archive-client/subs_2023-11-30-06-20.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/util-internal-archive-client", + "comment": "better type archive query result", + "type": "minor" + } + ], + "packageName": "@subsquid/util-internal-archive-client" +} \ No newline at end of file diff --git a/common/changes/@subsquid/util-internal-ingest-tools/subs_2023-11-30-06-20.json b/common/changes/@subsquid/util-internal-ingest-tools/subs_2023-11-30-06-20.json new file mode 100644 index 000000000..f106f5712 --- /dev/null +++ b/common/changes/@subsquid/util-internal-ingest-tools/subs_2023-11-30-06-20.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/util-internal-ingest-tools", + "comment": "overhaul all tools", + "type": "major" + } + ], + "packageName": "@subsquid/util-internal-ingest-tools" +} \ No newline at end of file diff --git a/common/changes/@subsquid/util-internal-processor-tools/subs_2023-11-30-06-20.json b/common/changes/@subsquid/util-internal-processor-tools/subs_2023-11-30-06-20.json new file mode 100644 index 000000000..2fb9486f3 --- /dev/null +++ b/common/changes/@subsquid/util-internal-processor-tools/subs_2023-11-30-06-20.json @@ -0,0 +1,20 @@ +{ + "changes": [ + { + "packageName": "@subsquid/util-internal-processor-tools", + "comment": "migrate to callback based hot data ingestion", + "type": "major" + }, + { + "packageName": "@subsquid/util-internal-processor-tools", + "comment": "remove ingest tools", + "type": "major" + }, + { + "packageName": "@subsquid/util-internal-processor-tools", + "comment": "add item filtering tools", + "type": "minor" + } + ], + "packageName": "@subsquid/util-internal-processor-tools" +} diff --git a/common/changes/@subsquid/util-internal-range/subs_2023-11-30-06-20.json b/common/changes/@subsquid/util-internal-range/subs_2023-11-30-06-20.json new file mode 100644 index 000000000..0d4af8bd2 --- /dev/null +++ b/common/changes/@subsquid/util-internal-range/subs_2023-11-30-06-20.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/util-internal-range", + "comment": "add tools to work with `RangeRequest` lists", + "type": "minor" + } + ], + "packageName": "@subsquid/util-internal-range" +} \ No newline at end of file diff --git a/common/changes/@subsquid/util-internal-validation/subs_2023-11-30-06-20.json b/common/changes/@subsquid/util-internal-validation/subs_2023-11-30-06-20.json new file mode 100644 index 000000000..77ddc7de5 --- /dev/null +++ b/common/changes/@subsquid/util-internal-validation/subs_2023-11-30-06-20.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/util-internal-validation", + "comment": "", + "type": "none" + } + ], + "packageName": "@subsquid/util-internal-validation" +} \ No newline at end of file diff --git a/common/changes/@subsquid/util-internal/subs_2023-11-30-06-20.json b/common/changes/@subsquid/util-internal/subs_2023-11-30-06-20.json new file mode 100644 index 000000000..c82e56714 --- /dev/null +++ b/common/changes/@subsquid/util-internal/subs_2023-11-30-06-20.json @@ -0,0 +1,15 @@ +{ + "changes": [ + { + "packageName": "@subsquid/util-internal", + "comment": "rework `AsyncQueue`", + "type": "major" + }, + { + "packageName": "@subsquid/util-internal", + "comment": "add `weakMemo()`, `partitionBy()`, `safeCall()` functions", + "type": "minor" + } + ], + "packageName": "@subsquid/util-internal" +} diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 3b5df0749..89f60d080 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -47,9 +47,9 @@ dependencies: '@rush-temp/data-test': specifier: file:./projects/data-test.tgz version: file:projects/data-test.tgz - '@rush-temp/eth-usdc-transfers': - specifier: file:./projects/eth-usdc-transfers.tgz - version: file:projects/eth-usdc-transfers.tgz(supports-color@8.1.1) + '@rush-temp/erc20-transfers': + specifier: file:./projects/erc20-transfers.tgz + version: file:projects/erc20-transfers.tgz(supports-color@8.1.1) '@rush-temp/evm-processor': specifier: file:./projects/evm-processor.tgz version: file:projects/evm-processor.tgz @@ -194,6 +194,9 @@ dependencies: '@rush-temp/util-internal-read-lines': specifier: file:./projects/util-internal-read-lines.tgz version: file:projects/util-internal-read-lines.tgz + '@rush-temp/util-internal-validation': + specifier: file:./projects/util-internal-validation.tgz + version: file:projects/util-internal-validation.tgz '@rush-temp/util-naming': specifier: file:./projects/util-naming.tgz version: file:projects/util-naming.tgz @@ -4955,7 +4958,7 @@ packages: dev: false file:projects/astar-erc20.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-Ns5lxhWXIj5i++nHYEOd/KA1ng6rZBtaU7AjGs91697yNwkEP7IQ9mW0iGp+An4awKzMAMQxlFJaFIEAkbXV/A==, tarball: file:projects/astar-erc20.tgz} + resolution: {integrity: sha512-otrucSMsxPH6kFSJBRIxdKoXIUcr8VTL54bxnvSEdMuozizWshZQCTJSvJgNJ9xfH7dSkQEcpAEVhWRD2/3tkg==, tarball: file:projects/astar-erc20.tgz} id: file:projects/astar-erc20.tgz name: '@rush-temp/astar-erc20' version: 0.0.0 @@ -4990,7 +4993,7 @@ packages: dev: false file:projects/balances.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-8GFKJfHOYe1aWqiRakntC0eigezs2nj1T6H1OIESzP8wzxy0V3E+XFl6MddRg8C/BdRs6ocUWvsM2kV63AAu1w==, tarball: file:projects/balances.tgz} + resolution: {integrity: sha512-m8Ezz8/7rGVG5W2sEUYkz3AM0RQojyjLZRMXi5JJNQpAl5n2kcAbeZ95/rVAZHHqdAwVw6N3GzBoJrffxvL6Kg==, tarball: file:projects/balances.tgz} id: file:projects/balances.tgz name: '@rush-temp/balances' version: 0.0.0 @@ -5029,7 +5032,7 @@ packages: dev: false file:projects/chain-status-service.tgz: - resolution: {integrity: sha512-HYKNEhV4UN8TVLTxUvV3mnAPgQozYsMaecIdvhvH129gaIy9/iZ9V0wA9FWEzYpB8G0Ct9oFfcfu5hhKGWHDSg==, tarball: file:projects/chain-status-service.tgz} + resolution: {integrity: sha512-EwiA2L/Jh4AhkB2C8uJMorLPrgqu6Ygel5+fVg8e+DayG72I1HoJsKLIMGywJawHritYt7SHWNb8sBvHmGPtFQ==, tarball: file:projects/chain-status-service.tgz} name: '@rush-temp/chain-status-service' version: 0.0.0 dependencies: @@ -5040,7 +5043,7 @@ packages: dev: false file:projects/commands.tgz: - resolution: {integrity: sha512-Vkce44kxTF1i+/72Ku9vF8WHuHeTOTxKkJcWCbqxqOxOfTLF0utRqKkuj3xtj3msJMqq2Ase2YMftr2j/fLBbQ==, tarball: file:projects/commands.tgz} + resolution: {integrity: sha512-7zLtByR25xVn/hidMl6yoEPpV1QBhJQZvoViNyz+hGm+oPVCOS+h9Ri5mHB0OjZHEdhmFfckDO/kuwsTQXe2lQ==, tarball: file:projects/commands.tgz} name: '@rush-temp/commands' version: 0.0.0 dependencies: @@ -5053,7 +5056,7 @@ packages: dev: false file:projects/data-test.tgz: - resolution: {integrity: sha512-RhFMQ3P5WW5rgMLNofLwbCD+e2ctEiZJQXlJbbwyuw6js8lPmfhCfgP3B9eKTee+kr1XJBLMLccU77tWfCgpUg==, tarball: file:projects/data-test.tgz} + resolution: {integrity: sha512-NLzLYJjEju9RroL3p0wQyT9zOg8bl8J5kvVj3eFFHLPYH+VmhDSEayfPgW4WZknar9h/PGMfAkac5++Mkq19cA==, tarball: file:projects/data-test.tgz} name: '@rush-temp/data-test' version: 0.0.0 dependencies: @@ -5066,10 +5069,10 @@ packages: - pg-native dev: false - file:projects/eth-usdc-transfers.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-P1X9LqVmididc2TM0gxS9s+kGk2A7TNZLw787s9+Lm6ELWQMXSttESfxyopVz9PRHSF6s8ll4S1+uZEKD9ORNg==, tarball: file:projects/eth-usdc-transfers.tgz} - id: file:projects/eth-usdc-transfers.tgz - name: '@rush-temp/eth-usdc-transfers' + file:projects/erc20-transfers.tgz(supports-color@8.1.1): + resolution: {integrity: sha512-lZxvY9oUJYDJHr2yyglLol8WISFvL/jkrgrRW+pyktzb6mz2LDBgL/KRCzxWzhKJFZ5H3gv1Uqd2RZKaPQ1dnw==, tarball: file:projects/erc20-transfers.tgz} + id: file:projects/erc20-transfers.tgz + name: '@rush-temp/erc20-transfers' version: 0.0.0 dependencies: '@types/node': 18.18.0 @@ -5101,16 +5104,18 @@ packages: dev: false file:projects/evm-processor.tgz: - resolution: {integrity: sha512-/Av0xfqN/nswjCuKkIVx8W1Lmj/jz6FvjZd5/89/7E+ZX1uPKueU9bXKMAWjlYOxV8B5T9gbWn5PVFxlKl4Upw==, tarball: file:projects/evm-processor.tgz} + resolution: {integrity: sha512-atJiIrADd3qTV+aJ4v8+SqyYE9KZEUqgRHd5cQ+sptjt9t7HmtPYUu+VFGppWRWljt9XnbbkqpMyQS4vwwT4Tg==, tarball: file:projects/evm-processor.tgz} name: '@rush-temp/evm-processor' version: 0.0.0 dependencies: + '@exodus/schemasafe': 1.3.0 '@types/node': 18.18.0 + ajv: 8.12.0 typescript: 5.2.2 dev: false file:projects/evm-typegen.tgz: - resolution: {integrity: sha512-JhyNsZ//U+20gpR7vJJJsMxsYJ0QZN9F+B6eKbn4wHrYVkxumLqNZeTURfH0oXMy3hf59KW8roMvfcSCj9Mc9g==, tarball: file:projects/evm-typegen.tgz} + resolution: {integrity: sha512-EBhjQzP8JQnU5jI9mGIVs8W+x6BP3UhkDRk/DYVZVeF+RbyY+hX2jQ+/guerK2ccwIj1wc5cez5R5UWTuFHpEA==, tarball: file:projects/evm-typegen.tgz} name: '@rush-temp/evm-typegen' version: 0.0.0 dependencies: @@ -5124,7 +5129,7 @@ packages: dev: false file:projects/frontier.tgz: - resolution: {integrity: sha512-3eOGSNdgUE/z3FDvLVcLN0j4pk7RBp7MTud1JDvW76v3ZVmx8piVwHRSTAercr9QUMURi29u/nNT1Ewn54GT4w==, tarball: file:projects/frontier.tgz} + resolution: {integrity: sha512-20rTovtKz+zq5XUKEgK8HwiFX+Vc2unjv+ebXS5UhVlDwdkMwyertepsDNoV0gcI+t0Jx19vsL6O5PtVYEkhDg==, tarball: file:projects/frontier.tgz} name: '@rush-temp/frontier' version: 0.0.0 dependencies: @@ -5137,7 +5142,7 @@ packages: dev: false file:projects/gql-test-client.tgz(graphql@15.8.0): - resolution: {integrity: sha512-qbTI0BhWi3kSRhbhhDK7ffT7tWOXsJiG1Z0l5kaW24V7QkLR28QsRuw7uB8qwDIVu/MP8n6rnXcUJyyCCXa7FQ==, tarball: file:projects/gql-test-client.tgz} + resolution: {integrity: sha512-BFGNG5F98MPfXjGiDdHhjTxdjBdrddtpsb21lMFP7hrAdwB44oOGkw91X9gGTvSwobKVWvOQ4LWIB3lqYX8Xsw==, tarball: file:projects/gql-test-client.tgz} id: file:projects/gql-test-client.tgz name: '@rush-temp/gql-test-client' version: 0.0.0 @@ -5155,7 +5160,7 @@ packages: dev: false file:projects/graphql-server.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-ZhNzX0NEOGecxmgz3tW97uFnj/oiVYrNk9D2zGEBz/BGd8IR23glx4tJb7oDDVOf7KHkBFW46cFj5gAtuxriLA==, tarball: file:projects/graphql-server.tgz} + resolution: {integrity: sha512-+P6jwvpPOH+o5mWOwDjCj19RICCjZ4g1H0PoWkvSSfyRooIAFfKIW4LtbfKRJpYWbI3noRYPrbEzlNjD1oyRzQ==, tarball: file:projects/graphql-server.tgz} id: file:projects/graphql-server.tgz name: '@rush-temp/graphql-server' version: 0.0.0 @@ -5212,7 +5217,7 @@ packages: dev: false file:projects/http-client.tgz: - resolution: {integrity: sha512-Ps9+whTB3ZUsmtYPUvDvPVYEKMjco3KuS//D0wvWUCNpRL9+CHf/m+eSIfQZUaWBmY6WtycvMxbtzVx0DAGgzw==, tarball: file:projects/http-client.tgz} + resolution: {integrity: sha512-umL6yLO4laCJHD8Z0gELgvTBM0hkRHpFgemqVDM5IoL9BWgEcVCTbDjILy2DHq+axew/tfo2uh/REa5yWoQ6Cw==, tarball: file:projects/http-client.tgz} name: '@rush-temp/http-client' version: 0.0.0 dependencies: @@ -5222,7 +5227,7 @@ packages: dev: false file:projects/ink-abi.tgz: - resolution: {integrity: sha512-kgXAIHY6mSgjyNpJX482nGcwOmL9+Jq7AI9R8ezHFT5UXfeTrIHwNPKgeFQ5h08TRYxT3TSoMYuMMbKBKNeS2A==, tarball: file:projects/ink-abi.tgz} + resolution: {integrity: sha512-JI8yyAZTf4T1azFdfvI+M8isqF5RgcpqmwPLxWjHcFQkeqk6PyR3fAJ7vwKeF+oRuyTjimlpGd/ORd/cB981iA==, tarball: file:projects/ink-abi.tgz} name: '@rush-temp/ink-abi' version: 0.0.0 dependencies: @@ -5235,7 +5240,7 @@ packages: dev: false file:projects/ink-typegen.tgz: - resolution: {integrity: sha512-74hkCJnULZIGWwHcV2KwAxRa+FsYdKMuLBveftUeOjcZPJwRcddHmv4VqnzPmgqt7u2IfiIxF9kMOuAy2WmjkQ==, tarball: file:projects/ink-typegen.tgz} + resolution: {integrity: sha512-Bo04ztYhd3pEAq82/NOssdPQTV1O4fV0qBFaN64FE70hax8GcE2ncZvJtCDzE8N1joCr8t7JpxWiE5ORhA8Tcg==, tarball: file:projects/ink-typegen.tgz} name: '@rush-temp/ink-typegen' version: 0.0.0 dependencies: @@ -5245,7 +5250,7 @@ packages: dev: false file:projects/logger.tgz: - resolution: {integrity: sha512-EuMMn5R/UOaS57g7knEgGrYNiKj70geScmomcbMOPlag22YpxGG0g4c/4H+oZUEq4uedHqzRYzeG/9BrJSZ92A==, tarball: file:projects/logger.tgz} + resolution: {integrity: sha512-dftE3ieibo4OUnMqiuveHb5lArAW+D+HDiipOeNSwi/8bsiq1yhUdfc8qjFzDv0HG1yKTQSInfye68RgRAfTbA==, tarball: file:projects/logger.tgz} name: '@rush-temp/logger' version: 0.0.0 dependencies: @@ -5259,7 +5264,7 @@ packages: dev: false file:projects/openreader.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-KplqJjNQYMwA8hRZA8HzE74GVkGOxgzwQzaFBPV4XYhJRJsLx2Tcuq5V+8hzrJDcUrBaxiQ4YJC7+dKNp6swlQ==, tarball: file:projects/openreader.tgz} + resolution: {integrity: sha512-9ZANNChMNmUTdlntUpLCmRdm6V0s3DdIiUJqVv4PV7gd+j7wSOaXWdzhVvl1W6zajhpmbVezS/nQ+brwnRIqmQ==, tarball: file:projects/openreader.tgz} id: file:projects/openreader.tgz name: '@rush-temp/openreader' version: 0.0.0 @@ -5295,7 +5300,7 @@ packages: dev: false file:projects/ops-xcm-typegen.tgz: - resolution: {integrity: sha512-/8nb8upxKn1q4JVqd+8rbrjwB5Hy9jwxZ9kcrOZtnMKxXuB5797bn1Aa3MKlY77VZBCuM4dEn3at8olNFVQTzA==, tarball: file:projects/ops-xcm-typegen.tgz} + resolution: {integrity: sha512-CYLXUZE+2dbaCGpjK3hQJO+w/znoGcLEg6E57+Hx/w1XiYnAZQ9dPg73GBYPo0Tb/jwkPSkN/+1CDXvGxcVl9w==, tarball: file:projects/ops-xcm-typegen.tgz} name: '@rush-temp/ops-xcm-typegen' version: 0.0.0 dependencies: @@ -5304,7 +5309,7 @@ packages: dev: false file:projects/rpc-client.tgz: - resolution: {integrity: sha512-yXnApkjUHgYlJGrX7AmTg/IzMSsgElHyquaoBcx+FhxpHxDqChoGVnjJwiQ5vFoHxsdKP9toMXCmpdnmk7VdQA==, tarball: file:projects/rpc-client.tgz} + resolution: {integrity: sha512-HisqgeJJZIlJV0DX9BnQX11deCdD0x4vY7JZamyObdelioHH8ScVUFiwa09bonQ0X09Du1yZRtKFsdJfEodaQQ==, tarball: file:projects/rpc-client.tgz} name: '@rush-temp/rpc-client' version: 0.0.0 dependencies: @@ -5315,7 +5320,7 @@ packages: dev: false file:projects/scale-codec.tgz: - resolution: {integrity: sha512-lMyTVrcyccjYa3Fjflpa7Qak9RRa9jDf+GEtypV0uU1aHHXNiUzjzMiRLdeMcGb9odVGZ0JdblfQ6PdIMQQOEA==, tarball: file:projects/scale-codec.tgz} + resolution: {integrity: sha512-i5u4T6dpC5YfC2jOdVHBhFc06/4Z5fF/7PyiXaVVZMg7XhQxAYkzrQszDpZFRX7p9CQr8Izt+Rw40XwEfJBmIQ==, tarball: file:projects/scale-codec.tgz} name: '@rush-temp/scale-codec' version: 0.0.0 dependencies: @@ -5324,7 +5329,7 @@ packages: dev: false file:projects/scale-type-system.tgz: - resolution: {integrity: sha512-NcYUHTyUXmbyIwrDzKz39p6qgOdHyVzwLO+K41J3uPFhvCvzYTSrJztGlYg7OBM9X/w0WcwPfJqwk2sN5O4FWQ==, tarball: file:projects/scale-type-system.tgz} + resolution: {integrity: sha512-WwJ4dmoyPnNxIggMMJFeRQ9T1sU45Mc3+Ge4lEEUq0MoAyGo34rJiaxMyRLyniJB1Sy11CKZRqcTtFPOxE+uPg==, tarball: file:projects/scale-type-system.tgz} name: '@rush-temp/scale-type-system' version: 0.0.0 dependencies: @@ -5333,7 +5338,7 @@ packages: dev: false file:projects/shibuya-psp22.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-yq0z1xgXvEXO/U4KbknrhPLrkxN9p557crCWRg2YAwuSlDpRG7KFb8xpgOodUGjOfXy9HSV/HT/bZOoBhm1fCw==, tarball: file:projects/shibuya-psp22.tgz} + resolution: {integrity: sha512-Py3B7m6TSBG4RO614pM0dUZWYStRmJRa48f5iZ1WUvEC2m298R4cWvk/wCjYwtg7qLApefkBTFDBiHZr5MVj4g==, tarball: file:projects/shibuya-psp22.tgz} id: file:projects/shibuya-psp22.tgz name: '@rush-temp/shibuya-psp22' version: 0.0.0 @@ -5375,7 +5380,7 @@ packages: dev: false file:projects/ss58.tgz: - resolution: {integrity: sha512-Cp5nw67GO94PbG2u2vamzEIMrFQa4RH+P5KNlB+Tg2+tybPVXKG3VKbOW4c0eau+SQzEqyaq3SItxrGcNWNKlw==, tarball: file:projects/ss58.tgz} + resolution: {integrity: sha512-b25j8/ttmoj84WDF86biAXZBPYVPby2FKIWF/inMzlpGEKVvWCm7bF43C0BKTwucrm6Qs5UWIvBVHSA6+byD1w==, tarball: file:projects/ss58.tgz} name: '@rush-temp/ss58' version: 0.0.0 dependencies: @@ -5384,7 +5389,7 @@ packages: dev: false file:projects/substrate-data-raw.tgz: - resolution: {integrity: sha512-K5SiKfyX8hulKF1BmKPsqsDDuZpu6UEKWs/FD6SyO1OmYmFWmILvGFzYLXMLGx/d0eNdgq23gYwPo3eR0LnChQ==, tarball: file:projects/substrate-data-raw.tgz} + resolution: {integrity: sha512-fAo9TuHtvJ+3TaeZJpFucB5AHTahMmD9ITQDaCGAf/8IdsYPRGvXBC5tmENbRxahG9Skflvu474gCl6x9Jk0Ug==, tarball: file:projects/substrate-data-raw.tgz} name: '@rush-temp/substrate-data-raw' version: 0.0.0 dependencies: @@ -5393,7 +5398,7 @@ packages: dev: false file:projects/substrate-data.tgz: - resolution: {integrity: sha512-BDHVv1fVe8fa1lWO9PDX0T9kwhjZeT1+NUcxNW3nVPyoAkbjTzxmcTpU6PksVEjNnEvHtf3jmDJXO3H435qbwA==, tarball: file:projects/substrate-data.tgz} + resolution: {integrity: sha512-aH+SthQX9HHtpPAN3im9zCuKGAhclob1PHO23z1J9lpgBv9DYoZhMP+cV5zj/SL78I0T8cSDQ5iC1SdLMioCqQ==, tarball: file:projects/substrate-data.tgz} name: '@rush-temp/substrate-data' version: 0.0.0 dependencies: @@ -5404,7 +5409,7 @@ packages: dev: false file:projects/substrate-dump.tgz: - resolution: {integrity: sha512-pc1xdIGLAqcnruzO+xNE3p+cCnoF3992oo7cSnGf6dHlyZxqagfyfHcRX1PqwsJ9OvZH/c7GE8p2Erbtua/1bA==, tarball: file:projects/substrate-dump.tgz} + resolution: {integrity: sha512-Wi9n4n27sGQNQvreDGh3hvursqIcBjz93h3Luaz/H3Ph4khl7ci4DWe5xnMs7cKfQaJHmaADh2GHg+VGeee7Wg==, tarball: file:projects/substrate-dump.tgz} name: '@rush-temp/substrate-dump' version: 0.0.0 dependencies: @@ -5415,7 +5420,7 @@ packages: dev: false file:projects/substrate-ingest.tgz: - resolution: {integrity: sha512-3o7kFFQNfknsmK9ZJj/q9cjQfH6j0aP/OZoh+14FEMbcKjZNJ8pyfmaFWOkvswC1E2dWDEyrasxoRwvxw6aQxA==, tarball: file:projects/substrate-ingest.tgz} + resolution: {integrity: sha512-fI2EWYm16iNGIK8DZb0gAsIf5xZP7Tt7Gjz8PhmQLyDPS788U/Z7wPT010XFrRPOxeyBoDi3PxmVNgNuy5/sXQ==, tarball: file:projects/substrate-ingest.tgz} name: '@rush-temp/substrate-ingest' version: 0.0.0 dependencies: @@ -5424,7 +5429,7 @@ packages: dev: false file:projects/substrate-metadata-explorer.tgz: - resolution: {integrity: sha512-+ngQcYl4izVTBbVspaStn+nTu3VAyMgy21UC744AAuqFeMVyWH91MxO9LqcF3t49jYH8zmyfvSoHLA4TO9Yh9Q==, tarball: file:projects/substrate-metadata-explorer.tgz} + resolution: {integrity: sha512-cE3o/veUNl3ptRwrO5g8JLujuItQTq3JjaQsWSZ97WvLW/DBpdFqzCDgtw+66CZNmc7Ydu5YQMFAM6D1nFWeNg==, tarball: file:projects/substrate-metadata-explorer.tgz} name: '@rush-temp/substrate-metadata-explorer' version: 0.0.0 dependencies: @@ -5434,7 +5439,7 @@ packages: dev: false file:projects/substrate-metadata-service.tgz: - resolution: {integrity: sha512-4lYJOhpHsVkBRg/ENg9a6z/CeBSN7Dh3WH+6WpFiqYtyD6cFRmO0RGU/vkK4hDhy39C0qCoUHfiVpVg6N10HfQ==, tarball: file:projects/substrate-metadata-service.tgz} + resolution: {integrity: sha512-OE/P/6+zHx2QQM0RU15iu5Y7mSrg57yX25ZMMsrutQxScFQSbIAtKFpC69o1YnZ1yQiRiCWLDU/oW/TmLv4S2g==, tarball: file:projects/substrate-metadata-service.tgz} name: '@rush-temp/substrate-metadata-service' version: 0.0.0 dependencies: @@ -5446,7 +5451,7 @@ packages: dev: false file:projects/substrate-processor.tgz: - resolution: {integrity: sha512-3H2n456zv/bT+E1Gs0wo0p+wmvMUMLeyt6Coxh2NUx9F46bwEsOZUDQKM6oxmUK/8FnU0rKsY1+O557FNwz5Ig==, tarball: file:projects/substrate-processor.tgz} + resolution: {integrity: sha512-b0sSm3c4ew+H9N0C1ns9r0V6tpe0dI0cGrGfnZath0/9SdJzZx2/K/D0SfCrZQvuR63HGX2+IiVMYTamhb6zDw==, tarball: file:projects/substrate-processor.tgz} name: '@rush-temp/substrate-processor' version: 0.0.0 dependencies: @@ -5455,7 +5460,7 @@ packages: dev: false file:projects/substrate-runtime.tgz: - resolution: {integrity: sha512-z12woAo78lq4cHf/UpQ+OJOzaQ+wzwvKR6xQ6fHffUZCkjtrCqRKs8yJhuUpMpTizGS/Es6Ip+A7j7GywskMQg==, tarball: file:projects/substrate-runtime.tgz} + resolution: {integrity: sha512-OPFYGUWOT3w0Cuit4imFiVH5ffeKqEQjEnVkMdrrRO94q4GhQnCYxoWHJFhjrNOA9emO6vVlhIKTABv10RjcsA==, tarball: file:projects/substrate-runtime.tgz} name: '@rush-temp/substrate-runtime' version: 0.0.0 dependencies: @@ -5468,7 +5473,7 @@ packages: dev: false file:projects/substrate-typegen.tgz: - resolution: {integrity: sha512-230723OQnwKLCh3wJtAKJGq+B9AWK7gkUhden9Qva6HPogJMFwTR5xHPfUdKnd9W0g9E79K6isOdooA0ZX6XXg==, tarball: file:projects/substrate-typegen.tgz} + resolution: {integrity: sha512-b2IvjBsj3ofHOXHTcZuxW5YFTcuX5FfN9KPHZWc785knaGjOrGNJkJxs4qTZM5/S+TIKf2flm9OFuowGhPk+Tw==, tarball: file:projects/substrate-typegen.tgz} name: '@rush-temp/substrate-typegen' version: 0.0.0 dependencies: @@ -5478,7 +5483,7 @@ packages: dev: false file:projects/typeorm-codegen.tgz: - resolution: {integrity: sha512-c0hGW5rduhNCkj1SJXka6KExjz33aS7NTJcqUP6UwO2YLin4JWkx0OPtEFlq4pc6Szb+IB9pK03l0dZTeOoGqA==, tarball: file:projects/typeorm-codegen.tgz} + resolution: {integrity: sha512-oRMh8RjOCGdoRPrXrTQRW9kHl0bMKKWboSd1GmTG/fzGkUWIAnjc3hkMLNebOuf5Evp+3tYtt0B4w56Sm9jz0Q==, tarball: file:projects/typeorm-codegen.tgz} name: '@rush-temp/typeorm-codegen' version: 0.0.0 dependencies: @@ -5488,7 +5493,7 @@ packages: dev: false file:projects/typeorm-config.tgz(pg@8.11.3)(supports-color@8.1.1): - resolution: {integrity: sha512-We4z+Y3z5qjXwpEeY3/FVTzchjFSx9iwhX0lum3qFENjFRuGjham0oxtRxXvGJsBWLI4yW14p5ZDLgHgANmnQg==, tarball: file:projects/typeorm-config.tgz} + resolution: {integrity: sha512-saVCXXLXLNCVSBxzGAg8FQXoNmtD1vDg8U8fP0mfuVlvSVsZcZaxqwomBNexQJ5LZgN9SdSgo5Zl8zozrlJ2ww==, tarball: file:projects/typeorm-config.tgz} id: file:projects/typeorm-config.tgz name: '@rush-temp/typeorm-config' version: 0.0.0 @@ -5518,7 +5523,7 @@ packages: dev: false file:projects/typeorm-migration.tgz(pg@8.11.3)(supports-color@8.1.1): - resolution: {integrity: sha512-iYd0yS2E9Jpz4cUsiOPvyHnjt/OYs9nYW2BvX9nScNDG4XzvYdi7GaPMaDq56EDicuAAD2pdF1pZ/o8h2YLFUQ==, tarball: file:projects/typeorm-migration.tgz} + resolution: {integrity: sha512-qt/pv0SQuzVjSqccy5xlEwpSn8qv53GeXs/609npsq1v8pjCYaSgmghSA1Ei4aIq4piWfLyH6gOFKtWX5g+Dng==, tarball: file:projects/typeorm-migration.tgz} id: file:projects/typeorm-migration.tgz name: '@rush-temp/typeorm-migration' version: 0.0.0 @@ -5550,7 +5555,7 @@ packages: dev: false file:projects/typeorm-store.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-DDc0SAj6pbkwQLEYjJ4vC0kWiM60DlWxQd5FT9PUy7qmnCgFq3jrCQm4P5TnLhnMwS4J3tfiuU/iLqahNp8bTQ==, tarball: file:projects/typeorm-store.tgz} + resolution: {integrity: sha512-i/0Yf5v5duI1VYYkG1Icjwy5CcNp8XcvximTSbaL9s+XmWA2n5jS5JO5RKZ9pY16uox8zrh36rBQP0Y5cFf7CA==, tarball: file:projects/typeorm-store.tgz} id: file:projects/typeorm-store.tgz name: '@rush-temp/typeorm-store' version: 0.0.0 @@ -5584,7 +5589,7 @@ packages: dev: false file:projects/types-test.tgz: - resolution: {integrity: sha512-MOWCjQyJ4zQQ4VVLPa6LkS2JcLNWLAO13QbshwQdcbj2bTF8snZs+EaPbF9JAKTg+2VgeyGLW/AQKh2aOpZEjg==, tarball: file:projects/types-test.tgz} + resolution: {integrity: sha512-g7X8l+rutSThwokRI4UFVpbtYeBehhlFUjFm4HD3ZvEGc6mPnC0LMkSEm1H9sUIFZKYTeqY9LPNL1+9Ia0ty3Q==, tarball: file:projects/types-test.tgz} name: '@rush-temp/types-test' version: 0.0.0 dependencies: @@ -5593,7 +5598,7 @@ packages: dev: false file:projects/util-internal-archive-client.tgz: - resolution: {integrity: sha512-ww+7VOebiF+8GKRcVvK3cu2NjGme/q8Vwlhbq2rGZFLtl6DlJVJKVEhDJt+yLsX5Tm0CAmM9EjQl5vzCJCM/Zw==, tarball: file:projects/util-internal-archive-client.tgz} + resolution: {integrity: sha512-RElU5MXMxk6CW25MRDbUePbZ0Y4CC3Ymj1wugI/cmTyg94HmgkD+BTjh3NpfLEe2VJeRqm64GfdUwAsnVw0DOw==, tarball: file:projects/util-internal-archive-client.tgz} name: '@rush-temp/util-internal-archive-client' version: 0.0.0 dependencies: @@ -5602,7 +5607,7 @@ packages: dev: false file:projects/util-internal-archive-layout.tgz: - resolution: {integrity: sha512-m1/jENN5Tp2rIikXga/l8vQvkrOR1odQMaMy6Wlxm+9Xr1x6kZOEbdh/03O/6JjS/xFFUZp/7MKTrU1RPSjk+w==, tarball: file:projects/util-internal-archive-layout.tgz} + resolution: {integrity: sha512-y4B0X1+zwHmCDnIXj6okIsTxPKjkAU3ZULf9uX2gUCnguHf9WtryoGBh6n1/o2iixzFSDSO6PEa2OO6Dag2kKg==, tarball: file:projects/util-internal-archive-layout.tgz} name: '@rush-temp/util-internal-archive-layout' version: 0.0.0 dependencies: @@ -5689,7 +5694,7 @@ packages: dev: false file:projects/util-internal-ingest-tools.tgz: - resolution: {integrity: sha512-BfKDnrXGmrC5OMfXIK5hho/3HsU4GjlFKdjp82bofGJMKrAXJrWZ7riNCliIxbETzwvIqz0GvlM5uYnbtbD4TA==, tarball: file:projects/util-internal-ingest-tools.tgz} + resolution: {integrity: sha512-g6OE+Dchup+btQiQT+keDt++UcpcA1p9bB34VmVaNFXINEYGZelcACjerZPLfiQSbZFfxEPbOjZJRHZrmf6HNg==, tarball: file:projects/util-internal-ingest-tools.tgz} name: '@rush-temp/util-internal-ingest-tools' version: 0.0.0 dependencies: @@ -5698,7 +5703,7 @@ packages: dev: false file:projects/util-internal-json.tgz: - resolution: {integrity: sha512-OU46IF4g3SEensjwj3uvDWiAehf3AZx7tp0wwhR/FNTQl/sfaHd45QIbt7lcAj3+bp2PsqgWAhPHFbFu2nATZw==, tarball: file:projects/util-internal-json.tgz} + resolution: {integrity: sha512-Iw4z29WIY3nE4/vqgBWtqVGMzxnyCkED6tVG6KCHQGsz+/YZalC+yJKy9ifyAX4lQrMiU7NxVCjld9xdM6/6Sw==, tarball: file:projects/util-internal-json.tgz} name: '@rush-temp/util-internal-json' version: 0.0.0 dependencies: @@ -5707,7 +5712,7 @@ packages: dev: false file:projects/util-internal-processor-tools.tgz: - resolution: {integrity: sha512-1PgK9fDVEBqrqCdc32gbV44devRZ4obOHw6TXkWknFVIgXNGECtob7Em8jqfyymkiDAyldZWXjBi5vnrB937zw==, tarball: file:projects/util-internal-processor-tools.tgz} + resolution: {integrity: sha512-6CZWI5lZKO8u5MNUmdDM1GPgBx+GuTmvWE4S2tH2tc3uMBS/9wpGTVl5+jSuk6wrkoW4+pyF10M2GutT8cyQ/g==, tarball: file:projects/util-internal-processor-tools.tgz} name: '@rush-temp/util-internal-processor-tools' version: 0.0.0 dependencies: @@ -5717,7 +5722,7 @@ packages: dev: false file:projects/util-internal-prometheus-server.tgz: - resolution: {integrity: sha512-GoRSFxU+Y3WuNMil2XrXtW1iRxk2t5vPdZC9dKYOcOMB4iTcj2KXs8tzH77WvMnq0hYLDHDeaL/2otavwBSAsw==, tarball: file:projects/util-internal-prometheus-server.tgz} + resolution: {integrity: sha512-a1a2+8qS801x1gLy17+IZcdEO8SE6A9NCZfWXWRwaWsxet2T+oqp/7zp5Pg3KAkPvSifNf7TLiJNlfRveQEmaQ==, tarball: file:projects/util-internal-prometheus-server.tgz} name: '@rush-temp/util-internal-prometheus-server' version: 0.0.0 dependencies: @@ -5727,7 +5732,7 @@ packages: dev: false file:projects/util-internal-range.tgz: - resolution: {integrity: sha512-CKiwIN8s3IKGnqwFQJFNux5sbh9OHJyMEGdfbjbyze31RG8fDA9EQMQDYKPynMCnPkWwkS/Hq7aqXQw1SnegNw==, tarball: file:projects/util-internal-range.tgz} + resolution: {integrity: sha512-7MJEcnCSMVy9cGNTQz8ofuPg475hl4ozyWQtGBOzbxLwczezEeHOWz1K9hhhTsLuPV7r3yxKruX7q469A/kaFQ==, tarball: file:projects/util-internal-range.tgz} name: '@rush-temp/util-internal-range' version: 0.0.0 dependencies: @@ -5745,6 +5750,15 @@ packages: typescript: 5.2.2 dev: false + file:projects/util-internal-validation.tgz: + resolution: {integrity: sha512-NsNA1wlqYaVBIvkOp4qf1SKCZp1150/k1UVCTMzXrhW9+kA9TmoRIal+cEhJeeZHsUVNptOqFAzeaAlnQWKh2Q==, tarball: file:projects/util-internal-validation.tgz} + name: '@rush-temp/util-internal-validation' + version: 0.0.0 + dependencies: + '@types/node': 18.18.0 + typescript: 5.2.2 + dev: false + file:projects/util-internal.tgz: resolution: {integrity: sha512-6ig7wMujNUZv/cCIft/pd4ySbvQqy0JPFjGNUh+F5gfdylvvAW4ocCK5PIqnYSBTj/lJb2NSATO8puGUfKLpxQ==, tarball: file:projects/util-internal.tgz} name: '@rush-temp/util-internal' @@ -5788,7 +5802,7 @@ packages: dev: false file:projects/workspace.tgz: - resolution: {integrity: sha512-2DYbjhcUqgDrcRDGx/KQMKYm/bEQGWEGf/1askhxiBmircqAHrdfwyF3OHWHtka79fGJ9V/4S2jhEdxPDXleLA==, tarball: file:projects/workspace.tgz} + resolution: {integrity: sha512-S+YqeXk+TII3XF00BJqp8OGNunyhoQgTDEaWb/2x4RmAB/MJgcJTgwn5tZ7D82MwWLNlD1QPryNfjQatKnNJaQ==, tarball: file:projects/workspace.tgz} name: '@rush-temp/workspace' version: 0.0.0 dependencies: diff --git a/evm/evm-processor/package.json b/evm/evm-processor/package.json index 652bc5f6d..ef874b3fc 100644 --- a/evm/evm-processor/package.json +++ b/evm/evm-processor/package.json @@ -23,10 +23,13 @@ "@subsquid/util-internal": "^2.5.2", "@subsquid/util-internal-archive-client": "^0.0.1", "@subsquid/util-internal-hex": "^1.2.1", - "@subsquid/util-internal-processor-tools": "^3.1.0" + "@subsquid/util-internal-ingest-tools": "^0.0.2", + "@subsquid/util-internal-processor-tools": "^3.1.0", + "@subsquid/util-internal-range": "^0.0.1", + "@subsquid/util-internal-validation": "^0.0.0", + "@subsquid/util-timeout": "^2.3.1" }, "devDependencies": { - "@subsquid/typeorm-store": "^1.2.4", "@types/node": "^18.18.0", "typescript": "~5.2.2" } diff --git a/evm/evm-processor/src/ds-archive/client.ts b/evm/evm-processor/src/ds-archive/client.ts index 9f22fc66a..98516897e 100644 --- a/evm/evm-processor/src/ds-archive/client.ts +++ b/evm/evm-processor/src/ds-archive/client.ts @@ -1,21 +1,34 @@ +import {addErrorContext, assertNotNull, unexpectedCase} from '@subsquid/util-internal' import {ArchiveClient} from '@subsquid/util-internal-archive-client' -import { - archiveIngest, - Batch, - DataSource, - PollingHeightTracker, - RangeRequest, - SplitRequest -} from '@subsquid/util-internal-processor-tools' +import {archiveIngest} from '@subsquid/util-internal-ingest-tools' +import {Batch, DataSource} from '@subsquid/util-internal-processor-tools' +import {getRequestAt, RangeRequest} from '@subsquid/util-internal-range' +import {cast} from '@subsquid/util-internal-validation' import assert from 'assert' -import {AllFields, BlockData} from '../interfaces/data' +import {Bytes32} from '../interfaces/base' +import {FieldSelection} from '../interfaces/data' import {DataRequest} from '../interfaces/data-request' -import {Bytes32} from '../interfaces/evm' -import * as gw from './gateway' -import {mapGatewayBlock, withDefaultFields} from './mapping' +import { + Block, + BlockHeader, + Log, + StateDiff, + StateDiffAdd, + StateDiffChange, + StateDiffDelete, + StateDiffNoChange, + Trace, + TraceCall, + TraceCreate, + TraceReward, + TraceSuicide, + Transaction +} from '../mapping/entities' +import {setUpRelations} from '../mapping/relations' +import {getBlockValidator} from './schema' -type Block = BlockData +const NO_FIELDS = {} export class EvmArchive implements DataSource { @@ -26,7 +39,7 @@ export class EvmArchive implements DataSource { } async getBlockHash(height: number): Promise { - let blocks = await this.query({ + let blocks = await this.client.query({ fromBlock: height, toBlock: height, includeAllBlocks: true @@ -35,31 +48,107 @@ export class EvmArchive implements DataSource { return blocks[0].header.hash } - getFinalizedBlocks(requests: RangeRequest[], stopOnHead?: boolean | undefined): AsyncIterable> { - return archiveIngest({ + async *getFinalizedBlocks(requests: RangeRequest[], stopOnHead?: boolean | undefined): AsyncIterable> { + for await (let batch of archiveIngest({ requests, - heightTracker: new PollingHeightTracker(() => this.getFinalizedHeight(), 10_000), - query: s => this.fetchSplit(s), + client: this.client, stopOnHead - }) - } + })) { + let fields = getRequestAt(requests, batch.blocks[0].header.number)?.fields || NO_FIELDS - private async fetchSplit(s: SplitRequest): Promise { - let blocks = await this.query({ - fromBlock: s.range.from, - toBlock: s.range.to, - fields: withDefaultFields(s.request.fields), - includeAllBlocks: !!s.request.includeAllBlocks, - transactions: s.request.transactions, - logs: s.request.logs, - traces: s.request.traces, - stateDiffs: s.request.stateDiffs - }) + let blocks = batch.blocks.map(b => { + try { + return this.mapBlock(b, fields) + } catch(err: any) { + throw addErrorContext(err, { + blockHeight: b.header.number, + blockHash: b.header.hash + }) + } + }) - return blocks.map(mapGatewayBlock) + yield {blocks, isHead: batch.isHead} + } } - private query(q: gw.BatchRequest): Promise { - return this.client.query(q) + private mapBlock(rawBlock: unknown, fields: FieldSelection): Block { + let validator = getBlockValidator(fields) + + let src = cast(validator, rawBlock) + + let {number, hash, parentHash, ...hdr} = src.header + let header = new BlockHeader(number, hash, parentHash) + Object.assign(header, hdr) + + let block = new Block(header) + + if (src.transactions) { + for (let {transactionIndex, ...props} of src.transactions) { + let tx = new Transaction(header, transactionIndex) + Object.assign(tx, props) + block.transactions.push(tx) + } + } + + if (src.logs) { + for (let {logIndex, transactionIndex, ...props} of src.logs) { + let log = new Log(header, logIndex, transactionIndex) + Object.assign(log, props) + block.logs.push(log) + } + } + + if (src.traces) { + for (let {transactionIndex, traceAddress, type, ...props} of src.traces) { + transactionIndex = assertNotNull(transactionIndex) + let trace: Trace + switch(type) { + case 'create': + trace = new TraceCreate(header, transactionIndex, traceAddress) + break + case 'call': + trace = new TraceCall(header, transactionIndex, traceAddress) + break + case 'suicide': + trace = new TraceSuicide(header, transactionIndex, traceAddress) + break + case 'reward': + trace = new TraceReward(header, transactionIndex, traceAddress) + break + default: + throw unexpectedCase() + } + Object.assign(trace, props) + block.traces.push(trace) + } + } + + if (src.stateDiffs) { + for (let {transactionIndex, address, key, kind, ...props} of src.stateDiffs) { + let diff: StateDiff + switch(kind) { + case '=': + diff = new StateDiffNoChange(header, transactionIndex, address, key) + break + case '+': + diff = new StateDiffAdd(header, transactionIndex, address, key) + break + case '*': + diff = new StateDiffChange(header, transactionIndex, address, key) + break + case '-': + diff = new StateDiffDelete(header, transactionIndex, address, key) + break + default: + throw unexpectedCase() + } + Object.assign(diff, props) + block.stateDiffs.push(diff) + } + } + + setUpRelations(block) + + return block } } diff --git a/evm/evm-processor/src/ds-archive/gateway.ts b/evm/evm-processor/src/ds-archive/gateway.ts deleted file mode 100644 index 96622cf79..000000000 --- a/evm/evm-processor/src/ds-archive/gateway.ts +++ /dev/null @@ -1,36 +0,0 @@ -import {DataRequest} from '../interfaces/data-request' -import {EvmBlock, EvmLog, EvmStateDiff, EvmTrace, EvmTransaction, Qty} from '../interfaces/evm' - - -export interface BatchRequest extends DataRequest { - fromBlock: number - toBlock?: number -} - - -type ReplaceBigintToQty = { - [K in keyof T]: bigint extends T[K] ? Qty : T[K] -} - - -export type Block = Omit, 'height'> & {number: number} -export type Transaction = ReplaceBigintToQty -export type Log = EvmLog -export type StateDiff = EvmStateDiff - - -type PatchTrace = { - [K in keyof T]: K extends 'action' | 'result' ? ReplaceBigintToQty : T[K] -} - - -export type Trace = PatchTrace - - -export interface BlockData { - header: Block - transactions?: Transaction[] - logs?: Log[] - traces?: Trace[] - stateDiffs?: EvmStateDiff[] -} diff --git a/evm/evm-processor/src/ds-archive/mapping.ts b/evm/evm-processor/src/ds-archive/mapping.ts deleted file mode 100644 index 496226f2a..000000000 --- a/evm/evm-processor/src/ds-archive/mapping.ts +++ /dev/null @@ -1,313 +0,0 @@ -import {addErrorContext} from '@subsquid/util-internal' -import { - AllFields, - BlockData, - BlockHeader, - DEFAULT_FIELDS, - FieldSelection, - Log, - StateDiff, - Trace, - TraceCall, - TraceCallAction, - TraceCallResult, - TraceCreate, - TraceCreateAction, - TraceCreateResult, - TraceReward, - TraceRewardAction, - TraceSuicide, - TraceSuicideAction, - Transaction -} from '../interfaces/data' -import {formatId} from '../util' -import * as gw from './gateway' - - -export const NO_LOGS_BLOOM = '0x'+Buffer.alloc(256).toString('hex') - - -export function mapGatewayBlock(src: gw.BlockData): BlockData { - try { - return tryMapGatewayBlock(src) - } catch(e: any) { - throw addErrorContext(e, { - blockHeight: src.header.number, - blockHash: src.header.hash - }) - } -} - - -function tryMapGatewayBlock(src: gw.BlockData): BlockData { - let header = mapBlockHeader(src.header) - - let block: BlockData = { - header, - transactions: [], - logs: [], - traces: [], - stateDiffs: [] - } - - let txIndex = new Map>() - - for (let go of src.transactions || []) { - let transaction = mapTransaction(header, go) - txIndex.set(transaction.transactionIndex, transaction) - block.transactions.push(transaction) - } - - for (let go of src.logs || []) { - let log: Log = { - id: formatId(header.height, header.hash, go.logIndex), - ...go, - block: header - } - let transaction = txIndex.get(log.transactionIndex) - if (transaction) { - log.transaction = transaction - } - block.logs.push(log) - } - - for (let go of src.traces || []) { - let trace = mapTrace(go) - trace.block = header - let transaction = txIndex.get(go.transactionIndex) - if (transaction) { - trace.transaction = transaction - } - block.traces.push(trace as Trace) - } - - for (let go of src.stateDiffs || []) { - let diff: StateDiff = { - ...go, - block: header - } - let transaction = txIndex.get(go.transactionIndex) - if (transaction) { - diff.transaction = transaction - } - block.stateDiffs.push(diff) - } - - return block -} - - -function mapBlockHeader(src: gw.Block): BlockHeader { - let header: Partial> = { - id: formatId(src.number, src.hash) - } - - let key: keyof gw.Block - for (key in src) { - if (src[key] == null) continue - switch(key) { - case 'number': - header.height = src.number - break - case 'timestamp': - header.timestamp = src.timestamp * 1000 - break - case 'difficulty': - case 'totalDifficulty': - case 'size': - case 'gasUsed': - case 'gasLimit': - case 'baseFeePerGas': - header[key] = BigInt(src[key]!) - break - default: - header[key] = src[key] - } - } - - return header as BlockHeader -} - - -function mapTransaction(block: BlockHeader, src: gw.Transaction): Transaction { - let tx: Partial> = { - id: formatId(block.height, block.hash, src.transactionIndex) - } - - let key: keyof gw.Transaction - for (key in src) { - if (src[key] == null) continue - switch(key) { - case 'gas': - case 'gasPrice': - case 'gasUsed': - case 'cumulativeGasUsed': - case 'effectiveGasPrice': - case 'value': - case 'v': - case 'maxFeePerGas': - case 'maxPriorityFeePerGas': - tx[key] = BigInt(src[key]!) - break - case 'transactionIndex': - case 'chainId': - case 'yParity': - case 'nonce': - case 'type': - case 'status': - tx[key] = src[key] - break - default: - tx[key] = src[key] - } - } - - tx.block = block - - return tx as Transaction -} - - -function mapTrace(src: gw.Trace): Partial> { - switch(src.type) { - case 'create': { - let {action, result, ...common} = src - let tr: Partial> = common - if (action) { - tr.action = {} as TraceCreateAction - let key: keyof TraceCreateAction - for (key in action) { - switch(key) { - case 'value': - case 'gas': - tr.action[key] = BigInt(action[key]) - break - default: - tr.action[key] = action[key] - } - } - } - if (result) { - tr.result = {} as TraceCreateResult - let key: keyof TraceCreateResult - for (key in result) { - switch(key) { - case 'gasUsed': - tr.result.gasUsed = BigInt(result.gasUsed) - break - default: - tr.result[key] = result[key] - } - } - } - return tr - } - case 'call': { - let {action, result, ...common} = src - let tr: Partial> = common - if (action) { - tr.action = {} as TraceCallAction - let key: keyof TraceCallAction - for (key in action) { - switch(key) { - case 'gas': - tr.action[key] = BigInt(action[key]) - break - case 'value': - let val = action[key] - if (val != null) { - tr.action[key] = BigInt(val) - } - break - default: - tr.action[key] = action[key] - } - } - } - if (result) { - tr.result = {} as TraceCallResult - let key: keyof TraceCallResult - for (key in result) { - switch(key) { - case 'gasUsed': - tr.result.gasUsed = BigInt(result.gasUsed) - break - default: - tr.result[key] = result[key] - } - } - } - return tr - } - case 'reward': { - let {action, ...common} = src - let tr: Partial> = common - if (action) { - tr.action = {} as TraceRewardAction - let key: keyof TraceRewardAction - for (key in action) { - switch(key) { - case 'value': - tr.action.value = BigInt(action.value) - break - default: - tr.action[key] = action[key] - } - } - } - return tr - } - case 'suicide': { - let {action, ...common} = src - let tr: Partial> = common - if (action) { - tr.action = {} as TraceSuicideAction - let key: keyof TraceSuicideAction - for (key in action) { - switch(key) { - case 'balance': - tr.action.balance = BigInt(action.balance) - break - default: - tr.action[key] = action[key] - } - } - } - return tr - } - } -} - - -export function withDefaultFields(fields?: FieldSelection): FieldSelection { - return { - block: mergeDefaultFields(DEFAULT_FIELDS.block, fields?.block), - transaction: mergeDefaultFields(DEFAULT_FIELDS.transaction, fields?.transaction), - log: mergeDefaultFields(DEFAULT_FIELDS.log, fields?.log), - trace: mergeDefaultFields(DEFAULT_FIELDS.trace, fields?.trace), - stateDiff: mergeDefaultFields(DEFAULT_FIELDS.stateDiff, fields?.stateDiff) - } -} - - -type Selector = { - [P in Props]?: boolean -} - - -function mergeDefaultFields( - defaults: Selector, - selection?: Selector -): Selector { - let result: Selector = {...defaults} - for (let key in selection) { - if (selection[key] != null) { - if (selection[key]) { - result[key] = true - } else { - delete result[key] - } - } - } - return result -} diff --git a/evm/evm-processor/src/ds-archive/schema.ts b/evm/evm-processor/src/ds-archive/schema.ts new file mode 100644 index 000000000..a014c5c89 --- /dev/null +++ b/evm/evm-processor/src/ds-archive/schema.ts @@ -0,0 +1,50 @@ +import {weakMemo} from '@subsquid/util-internal' +import {array, BYTES, NAT, object, option, STRING, taggedUnion} from '@subsquid/util-internal-validation' +import {FieldSelection} from '../interfaces/data' +import { + getBlockHeaderProps, + getLogProps, + getTraceFrameValidator, + getTxProps, + getTxReceiptProps, + project +} from '../mapping/schema' + + +export const getBlockValidator = weakMemo((fields: FieldSelection) => { + let BlockHeader = object(getBlockHeaderProps(fields.block, true)) + + let Transaction = object({ + hash: fields.transaction?.hash ? BYTES : undefined, + ...getTxProps(fields.transaction, true), + sighash: fields.transaction?.sighash ? BYTES : undefined, + ...getTxReceiptProps(fields.transaction, true) + }) + + let Log = object( + getLogProps(fields.log, true) + ) + + let Trace = getTraceFrameValidator(fields.trace, true) + + let stateDiffBase = { + transactionIndex: NAT, + address: BYTES, + key: STRING + } + + let StateDiff = taggedUnion('kind', { + ['=']: object({...stateDiffBase}), + ['+']: object({...stateDiffBase, ...project(fields.stateDiff, {next: BYTES})}), + ['*']: object({...stateDiffBase, ...project(fields.stateDiff, {prev: BYTES, next: BYTES})}), + ['-']: object({...stateDiffBase, ...project(fields.stateDiff, {prev: BYTES})}) + }) + + return object({ + header: BlockHeader, + transactions: option(array(Transaction)), + logs: option(array(Log)), + traces: option(array(Trace)), + stateDiffs: option(array(StateDiff)) + }) +}) diff --git a/evm/evm-processor/src/ds-rpc/client.ts b/evm/evm-processor/src/ds-rpc/client.ts index 26b97d410..e97979830 100644 --- a/evm/evm-processor/src/ds-rpc/client.ts +++ b/evm/evm-processor/src/ds-rpc/client.ts @@ -1,37 +1,45 @@ -import {Logger} from '@subsquid/logger' -import {RetryError, RpcClient, RpcError} from '@subsquid/rpc-client' -import {RpcRequest} from '@subsquid/rpc-client/lib/interfaces' -import {assertNotNull, concurrentMap, def, groupBy, last, wait} from '@subsquid/util-internal' +import {Logger, LogLevel} from '@subsquid/logger' +import {RpcClient} from '@subsquid/rpc-client' +import {AsyncQueue, ensureError, last, maybeLast, Throttler, wait} from '@subsquid/util-internal' import { - Batch, - ForkNavigator, - generateFetchStrides, - getHeightUpdates, - HotDatabaseState, - HotDataSource, - HotUpdate, - PollingHeightTracker, + BlockConsistencyError, + BlockHeader as Head, + BlockRef, + coldIngest, + HashAndHeight, + HotProcessor, + isDataConsistencyError +} from '@subsquid/util-internal-ingest-tools' +import {Batch, HotDatabaseState, HotDataSource, HotUpdate} from '@subsquid/util-internal-processor-tools' +import { + getRequestAt, + mapRangeRequestList, + rangeEnd, RangeRequest, - RequestsTracker, + RangeRequestList, + splitRange, + splitRangeByRequest, SplitRequest -} from '@subsquid/util-internal-processor-tools' +} from '@subsquid/util-internal-range' +import {BYTES, cast, object, SMALL_QTY} from '@subsquid/util-internal-validation' +import {addTimeout, TimeoutError} from '@subsquid/util-timeout' import assert from 'assert' -import {NO_LOGS_BLOOM} from '../ds-archive/mapping' -import {AllFields, BlockData} from '../interfaces/data' +import {Bytes32} from '../interfaces/base' import {DataRequest} from '../interfaces/data-request' -import {Bytes32, Qty} from '../interfaces/evm' -import {getBlockHeight, getBlockName, getTxHash, mapBlock, qty2Int, toQty, toRpcDataRequest} from './mapping' -import * as rpc from './rpc' +import {Block} from '../mapping/entities' +import {mapBlock} from './mapping' +import {MappingRequest, toMappingRequest} from './request' +import {Rpc} from './rpc' -type Block = BlockData +const NO_REQUEST = toMappingRequest() export interface EvmRpcDataSourceOptions { rpc: RpcClient finalityConfirmation: number - pollInterval?: number - strideSize?: number + newHeadTimeout?: number + headPollInterval?: number preferTraceApi?: boolean useDebugApiForStateDiffs?: boolean log?: Logger @@ -39,568 +47,225 @@ export interface EvmRpcDataSourceOptions { export class EvmRpcDataSource implements HotDataSource { - private rpc: RpcClient - private strideSize: number + private rpc: Rpc private finalityConfirmation: number - private pollInterval: number - private useDebugApiForStateDiffs: boolean - private preferTraceApi: boolean + private headPollInterval: number + private newHeadTimeout: number + private preferTraceApi?: boolean + private useDebugApiForStateDiffs?: boolean private log?: Logger constructor(options: EvmRpcDataSourceOptions) { - this.rpc = options.rpc + this.rpc = new Rpc(options.rpc) this.finalityConfirmation = options.finalityConfirmation - this.strideSize = options.strideSize ?? 10 - this.pollInterval = options.pollInterval ?? 1000 - this.useDebugApiForStateDiffs = options.useDebugApiForStateDiffs ?? false - this.preferTraceApi = options.preferTraceApi ?? false + this.headPollInterval = options.headPollInterval || 5_000 + this.newHeadTimeout = options.newHeadTimeout || 0 + this.preferTraceApi = options.preferTraceApi + this.useDebugApiForStateDiffs = options.useDebugApiForStateDiffs this.log = options.log } async getFinalizedHeight(): Promise { - let height = await this.getHeight() + let height = await this.rpc.getHeight() return Math.max(0, height - this.finalityConfirmation) } - private async getHeight(): Promise { - let height: Qty = await this.rpc.call('eth_blockNumber') - return qty2Int(height) - } - - async getBlockHash(height: number): Promise { - let block: rpc.Block | null = await this.rpc.call( - 'eth_getBlockByNumber', - [toQty(height), false] - ) - return block?.hash - } - - @def - async getGenesisHash(): Promise { - let hash = await this.getBlockHash(0) - return assertNotNull(hash, `block 0 is not known by ${this.rpc.url}`) - } - - async *getHotBlocks(requests: RangeRequest[], state: HotDatabaseState): AsyncIterable> { - let requestsTracker = new RequestsTracker( - requests.map(toRpcBatchRequest) - ) - - let heightTracker = new PollingHeightTracker( - () => this.getHeight(), - this.pollInterval - ) - - let nav = new ForkNavigator( - state, - ref => { - let height = assertNotNull(ref.height) - let req = requestsTracker.getRequestAt(height) - return this.fetchHotBlock({height, hash: ref.hash}, req) - }, - block => block.header - ) - - for await (let top of getHeightUpdates(heightTracker, nav.getHeight() + 1)) { - let finalized = Math.max(top - this.finalityConfirmation, 0) - for (let number = nav.getHeight() + 1; number <= top; number++) { - let update = await this.getHotUpdate(nav, number, finalized) - yield update - if (!requestsTracker.hasRequestsAfter(update.finalizedHead.height)) return - } - } - } - - private async getHotUpdate(nav: ForkNavigator, blockHeight: number, finalizedHeight: number): Promise> { - let retries = 0 - while (true) { - try { - return await nav.move({ - best: blockHeight, - finalized: Math.min(blockHeight, finalizedHeight) - }) - } catch(err: any) { - if (isConsistencyError(err) && retries < 10) { - retries += 1 - if (retries > 2) { - this.log?.warn(err.message) - } - await wait(200 * retries) - } else { - throw err - } - } - } - } - - private async fetchHotBlock(ref: {height: number, hash?: string}, req: rpc.DataRequest | undefined): Promise { - let withTransactions = !!req?.transactions - let block0: rpc.Block | null - if (ref.hash) { - block0 = await this.rpc.call('eth_getBlockByHash', [ref.hash, withTransactions]) - } else { - block0 = await this.rpc.call('eth_getBlockByNumber', [toQty(ref.height), withTransactions]) - } - if (block0 == null) { - throw new ConsistencyError(ref) - } - let processed = await this.processBlocks([block0], req, 0) - return processed[0] + getBlockHash(height: number): Promise { + return this.rpc.getBlockHash(height) } getFinalizedBlocks( requests: RangeRequest[], stopOnHead?: boolean ): AsyncIterable> { - let heightTracker = new PollingHeightTracker(() => this.getFinalizedHeight(), this.pollInterval) - return concurrentMap( - 5, - generateFetchStrides({ - requests: requests.map(toRpcBatchRequest), - heightTracker, - strideSize: this.strideSize, - stopOnHead - }), - async s => { - let blocks0 = await this.getStride0(s) - let blocks = await this.processBlocks(blocks0, s.request) - return { - blocks, - isHead: s.range.to >= heightTracker.getLastHeight() - } - } - ) - } - - private async getStride0(s: SplitRequest): Promise { - let call = [] - for (let i = s.range.from; i <= s.range.to; i++) { - call.push({ - method: 'eth_getBlockByNumber', - params: [toQty(i), s.request.transactions] - }) - } - let blocks: rpc.Block[] = await this.rpc.batchCall(call, { - priority: s.range.from, - validateResult: nonNull + return coldIngest({ + getFinalizedHeight: () => this.getFinalizedHeight(), + getSplit: req => this._getSplit(req), + requests: mapRangeRequestList(requests, req => this.toMappingRequest(req)), + splitSize: 10, + concurrency: Math.min(5, this.rpc.client.getConcurrency()), + stopOnHead, + headPollInterval: this.headPollInterval }) - for (let i = 1; i < blocks.length; i++) { - assert.strictEqual( - blocks[i - 1].hash, - blocks[i].parentHash, - 'perhaps finality confirmation was not large enough' - ) - } - return blocks } - private async processBlocks(blocks: rpc.Block[], request?: rpc.DataRequest, finalizedHeight?: number): Promise { - if (blocks.length == 0) return [] - let req = request ?? toRpcDataRequest() - await this.fetchRequestedData(blocks, req, finalizedHeight) - return blocks.map(b => mapBlock(b, !!req.transactionList)) + private async _getSplit(req: SplitRequest): Promise { + let rpc = this.rpc.withPriority(req.range.from) + let blocks = await rpc.getColdSplit(req) + return blocks.map(b => mapBlock(b, req.request)) } - private async fetchRequestedData(blocks: rpc.Block[], req: rpc.DataRequest, finalizedHeight?: number): Promise { - let subtasks = [] - - if (req.logs && !req.receipts) { - subtasks.push(catching( - this.fetchLogs(blocks) - )) - } - - if (req.receipts) { - let byBlockMethod = await this.getBlockReceiptsMethod() - if (byBlockMethod) { - subtasks.push(catching( - this.fetchReceiptsByBlock(blocks, byBlockMethod) - )) - } else { - subtasks.push(catching( - this.fetchReceiptsByTx(blocks) - )) - } - } - - if (req.traces || req.stateDiffs) { - let isArbitrumOne = await this.getGenesisHash() === '0x7ee576b35482195fc49205cec9af72ce14f003b9ae69f6ba0faef4514be8b442' - if (isArbitrumOne) { - subtasks.push(this.fetchArbitrumOneTraces(blocks, req)) - } else { - subtasks.push(this.fetchTraces(blocks, req, finalizedHeight ?? Number.MAX_SAFE_INTEGER)) - } - } - - await Promise.all(subtasks) + private toMappingRequest(req?: DataRequest): MappingRequest { + let r = toMappingRequest(req) + r.preferTraceApi = this.preferTraceApi + r.useDebugApiForStateDiffs = this.useDebugApiForStateDiffs + return r } - private async fetchLogs(blocks: rpc.Block[]): Promise { - let logs: rpc.Log[] = await this.requestLogs( - getBlockHeight(blocks[0]), - getBlockHeight(last(blocks)) - ) - - let logsByBlock = groupBy(logs, log => log.blockHash) + async processHotBlocks( + requests: RangeRequestList, + state: HotDatabaseState, + cb: (upd: HotUpdate) => Promise + ): Promise { + if (requests.length == 0) return - for (let block of blocks) { - let logs = logsByBlock.get(block.hash) || [] - if (logs.length == 0 && block.logsBloom !== NO_LOGS_BLOOM) { - throw new ConsistencyError(block) - } else { - block._logs = logs - } - } - } + let mappingRequests = mapRangeRequestList(requests, req => this.toMappingRequest(req)) - private async requestLogs(from: number, to: number): Promise { - return this.rpc.call('eth_getLogs', [{ - fromBlock: toQty(from), - toBlock: toQty(to) - }], { - priority: from - }).catch(async err => { - let range = toTryAnotherRangeError(err) - if (range && range.from == from && from <= range.to && range.to < to) { - let result = await Promise.all([ - this.requestLogs(range.from, range.to), - this.requestLogs(range.to + 1, to) - ]) - return result[0].concat(result[1]) - } else { - throw err - } - }) - } + let self = this - private async fetchReceiptsByBlock( - blocks: rpc.Block[], - method: 'eth_getBlockReceipts' | 'alchemy_getTransactionReceipts' - ): Promise { - let call = blocks.map(block => { - if (method == 'eth_getBlockReceipts') { - return { - method, - params: [block.number] + let proc = new HotProcessor(state, { + process: cb, + getBlock: async ref => { + let req = getRequestAt(mappingRequests, ref.height) || NO_REQUEST + let block = await this.rpc.getColdBlock(ref.hash, req, proc.getFinalizedHeight()) + return mapBlock(block, req) + }, + async *getBlockRange(from: number, to: BlockRef): AsyncIterable { + assert(to.height != null) + if (from > to.height) { + from = to.height } - } else { - return { - method, - params: [{blockHash: block.hash}] + for (let split of splitRangeByRequest(mappingRequests, {from, to: to.height})) { + let request = split.request || NO_REQUEST + for (let range of splitRange(10, split.range)) { + let rpcBlocks = await self.rpc.getHotSplit({ + range, + request, + finalizedHeight: proc.getFinalizedHeight() + }) + let blocks = rpcBlocks.map(b => mapBlock(b, request)) + let lastBlock = maybeLast(blocks)?.header.height ?? range.from - 1 + yield blocks + if (lastBlock < range.to) { + throw new BlockConsistencyError({height: lastBlock + 1}) + } + } } + }, + getHeader(block) { + return block.header } }) - let results: rpc.TransactionReceipt[][] = await this.rpc.batchCall(call, { - priority: getBlockHeight(blocks[0]), - validateResult: nonNull - }) - - for (let i = 0; i < blocks.length; i++) { - let block = blocks[i] - let receipts = results[i] - if (block.transactions.length !== receipts.length) throw new ConsistencyError(block) - for (let receipt of receipts) { - if (receipt.blockHash !== block.hash) throw new ConsistencyError(block) - } - block._receipts = receipts - } - } - - @def - private async getBlockReceiptsMethod(): Promise<'eth_getBlockReceipts' | 'alchemy_getTransactionReceipts' | undefined> { - let alchemy = await this.rpc.call('alchemy_getTransactionReceipts', [{blockNumber: '0x0'}]).then( - res => Array.isArray(res), - () => false - ) - if (alchemy) return 'alchemy_getTransactionReceipts' - - let eth = await this.rpc.call('eth_getBlockReceipts', ['latest']).then( - res => Array.isArray(res), - () => false - ) - if (eth) return 'eth_getBlockReceipts' - - return undefined - } - - private async fetchReceiptsByTx(blocks: rpc.Block[]): Promise { - let call = [] - for (let block of blocks) { - for (let tx of block.transactions) { - call.push({ - method: 'eth_getTransactionReceipt', - params: [getTxHash(tx)] - }) - } - } - - let receipts: (rpc.TransactionReceipt | null)[] = await this.rpc.batchCall(call, { - priority: getBlockHeight(blocks[0]) - }) - - let receiptsByBlock = groupBy( - receipts.filter(r => r != null) as rpc.TransactionReceipt[], - r => r.blockHash - ) - - for (let block of blocks) { - let rs = receiptsByBlock.get(block.hash) || [] - if (rs.length !== block.transactions.length) { - throw new ConsistencyError(block) - } - block._receipts = rs - } - } - - private fetchTraces(blocks: rpc.Block[], req: rpc.DataRequest, finalizedHeight: number): Promise { - let tasks: Promise[] = [] - let replayTracers: rpc.TraceTracers[] = [] + let isEnd = () => proc.getFinalizedHeight() >= rangeEnd(last(requests).range) - if (req.stateDiffs) { - if (finalizedHeight < getBlockHeight(last(blocks)) || this.useDebugApiForStateDiffs) { - tasks.push(catching( - this.fetchDebugStateDiffs(blocks) - )) - } else { - replayTracers.push('stateDiff') - } - } - - if (req.traces) { - if (this.preferTraceApi) { - if (finalizedHeight < getBlockHeight(last(blocks)) || replayTracers.length == 0) { - tasks.push(catching( - this.fetchTraceBlock(blocks) - )) - } else { - replayTracers.push('trace') + let navigate = (head: {height: number, hash?: Bytes32}): Promise => { + return proc.goto({ + best: head, + finalized: { + height: Math.max(head.height - this.finalityConfirmation, 0) } - } else { - tasks.push(catching( - this.fetchDebugFrames(blocks) - )) - } + }) } - if (replayTracers.length) { - tasks.push(catching( - this.fetchReplays(blocks, replayTracers) - )) + if (this.rpc.client.supportsNotifications()) { + return this.subscription(navigate, isEnd) + } else { + return this.polling(navigate, isEnd) } - - return Promise.all(tasks).then() } - private async fetchReplays( - blocks: rpc.Block[], - tracers: rpc.TraceTracers[], - method: string = 'trace_replayBlockTransactions' - ): Promise { - if (tracers.length == 0) return - - let call = blocks.map(block => ({ - method, - params: [block.number, tracers] - })) - - let replaysByBlock: rpc.TraceTransactionReplay[][] = await this.rpc.batchCall(call, { - priority: getBlockHeight(blocks[0]) - }) - - for (let i = 0; i < blocks.length; i++) { - let block = blocks[i] - let replays = replaysByBlock[i] - let txs = new Set(block.transactions.map(getTxHash)) - - for (let rep of replays) { - if (!rep.transactionHash) { // FIXME: Who behaves like that? Arbitrum? - let txHash: Bytes32 | undefined = undefined - for (let frame of rep.trace || []) { - assert(txHash == null || txHash === frame.transactionHash) - txHash = txHash || frame.transactionHash + private async polling(cb: (head: {height: number}) => Promise, isEnd: () => boolean): Promise { + let prev = -1 + let height = new Throttler(() => this.rpc.getHeight(), this.headPollInterval) + while (!isEnd()) { + let next = await height.call() + if (next <= prev) continue + prev = next + for (let i = 0; i < 100; i++) { + try { + await cb({height: next}) + break + } catch(err: any) { + if (isDataConsistencyError(err)) { + this.log?.write( + i > 0 ? LogLevel.WARN : LogLevel.DEBUG, + err.message + ) + await wait(100) + } else { + throw err } - assert(txHash, "Can't match transaction replay with its transaction") - rep.transactionHash = txHash - } - // Sometimes replays might be missing. FIXME: when? - if (!txs.has(rep.transactionHash)) { - throw new ConsistencyError(block) } } - - block._traceReplays = replays } } - private async fetchTraceBlock(blocks: rpc.Block[]): Promise { - let call = blocks.map(block => ({ - method: 'trace_block', - params: [block.number] - })) - - let results: rpc.TraceFrame[][] = await this.rpc.batchCall(call, { - priority: getBlockHeight(blocks[0]) - }) - - for (let i = 0; i < blocks.length; i++) { - let block = blocks[i] - let frames = results[i] - if (frames.length == 0) { - if (block.transactions.length > 0) throw new ConsistencyError(block) - } else { - for (let frame of frames) { - if (frame.blockHash !== block.hash) throw new ConsistencyError(block) - } - block._traceReplays = [] - let byTx = groupBy(frames, f => f.transactionHash) - for (let [transactionHash, txFrames] of byTx.entries()) { - if (transactionHash) { - block._traceReplays.push({ - transactionHash, - trace: txFrames - }) + private async subscription(cb: (head: HashAndHeight) => Promise, isEnd: () => boolean): Promise { + let lastHead: HashAndHeight = {height: -1, hash: '0x'} + let heads = this.subscribeNewHeads() + try { + while (!isEnd()) { + let next = await addTimeout(heads.take(), this.newHeadTimeout).catch(ensureError) + assert(next) + if (next instanceof TimeoutError) { + this.log?.warn(`resetting RPC connection, because we haven't seen a new head for ${this.newHeadTimeout} ms`) + this.rpc.client.reset() + } else if (next instanceof Error) { + throw next + } else if (next.height >= lastHead.height) { + lastHead = next + for (let i = 0; i < 3; i++) { + try { + await cb(next) + break + } catch(err: any) { + if (isDataConsistencyError(err)) { + this.log?.write( + i > 0 ? LogLevel.WARN : LogLevel.DEBUG, + err.message + ) + await wait(100) + if (heads.peek()) break + } else { + throw err + } + } } } } + } finally { + heads.close() } } - private async fetchDebugFrames(blocks: rpc.Block[]): Promise { - let traceConfig = { - tracer: 'callTracer', - tracerConfig: { - onlyTopCall: false, - withLog: false // will study log <-> frame matching problem later - } - } - - let call = blocks.map(block => ({ - method: 'debug_traceBlockByHash', - params: [block.hash, traceConfig] - })) - - let results: any[][] = await this.rpc.batchCall(call, { - priority: getBlockHeight(blocks[0]) - }) - - for (let i = 0; i < blocks.length; i++) { - let block = blocks[i] - let frames = results[i] - - assert(block.transactions.length === frames.length) - - // Moonbeam quirk - for (let j = 0; j < frames.length; j++) { - if (!frames[j].result) { - frames[j] = {result: frames[j]} + private subscribeNewHeads(): AsyncQueue { + let queue = new AsyncQueue(1) + + let handle = this.rpc.client.subscribe({ + method: 'eth_subscribe', + params: ['newHeads'], + notification: 'eth_subscription', + unsubscribe: 'eth_unsubscribe', + onMessage: msg => { + try { + let {number, hash, parentHash} = cast(NewHeadMessage, msg) + queue.forcePut({ + height: number, + hash, + parentHash + }) + } catch(err: any) { + queue.forcePut(ensureError(err)) + queue.close() } - } - - block._debugFrames = frames - } - } - - private async fetchDebugStateDiffs(blocks: rpc.Block[]): Promise { - let traceConfig = { - tracer: 'prestateTracer', - tracerConfig: { - onlyTopCall: false, // passing this option is incorrect, but required by Alchemy endpoints - diffMode: true - } - } - - let call = blocks.map(block => ({ - method: 'debug_traceBlockByHash', - params: [block.hash, traceConfig] - })) - - let results: rpc.DebugStateDiffResult[][] = await this.rpc.batchCall(call, { - priority: getBlockHeight(blocks[0]) + }, + onError: err => { + queue.forcePut(ensureError(err)) + queue.close() + }, + resubscribeOnConnectionLoss: true }) - for (let i = 0; i < blocks.length; i++) { - let block = blocks[i] - let diffs = results[i] - assert(block.transactions.length === diffs.length) - block._debugStateDiffs = diffs - } - } - - private async fetchArbitrumOneTraces(blocks: rpc.Block[], req: rpc.DataRequest): Promise { - if (req.stateDiffs) { - throw new Error('State diffs are not supported on Arbitrum One') - } - if (!req.traces) return - - let arbBlocks = blocks.filter(b => getBlockHeight(b) <= 22207815) - let debugBlocks = blocks.filter(b => getBlockHeight(b) >= 22207818) - - if (arbBlocks.length) { - await this.fetchReplays(arbBlocks, ['trace'], 'arbtrace_replayBlockTransactions') - } - - if (debugBlocks.length) { - await this.fetchDebugFrames(debugBlocks) - } - } -} - - -class ConsistencyError extends Error { - constructor(block: rpc.Block | {height: number, hash?: string, number?: undefined} | number | string) { - let name = typeof block == 'object' ? getBlockName(block) : block - super(`Seems like the chain node navigated to another branch while we were fetching block ${name} or lost it`) - } -} - - -function toRpcBatchRequest(request: RangeRequest): RangeRequest { - return { - range: request.range, - request: toRpcDataRequest(request.request) - } -} - - -function isConsistencyError(err: unknown): err is Error { - if (err instanceof ConsistencyError) return true - if (err instanceof RpcError) { - // eth_gelBlockByNumber on Moonbeam reacts like that when block is not present - if (/Expect block number from id/i.test(err.message)) return true - } - return false -} - - -function catching(promise: Promise): Promise { - // prevent unhandled promise rejection crashes - promise.catch(() => {}) - return promise -} + queue.addCloseListener(() => handle.close()) - -class UnexpectedResponse extends RetryError { - get name(): string { - return 'UnexpectedResponse' + return queue } } -function nonNull(result: any, req: RpcRequest): any { - if (result == null) throw new UnexpectedResponse( - `Result of call ${JSON.stringify(req)} was null. Perhaps, you should find a better endpoint.` - ) - return result -} - - -function toTryAnotherRangeError(err: unknown): {from: number, to: number} | undefined { - if (!(err instanceof RpcError)) return - let m = /Try with this block range \[(0x[0-9a-f]+), (0x[0-9a-f]+)]/i.exec(err.message) - if (m == null) return - return { - from: qty2Int(m[1]), - to: qty2Int(m[2]) - } -} +const NewHeadMessage = object({ + number: SMALL_QTY, + hash: BYTES, + parentHash: BYTES +}) diff --git a/evm/evm-processor/src/ds-rpc/filter.ts b/evm/evm-processor/src/ds-rpc/filter.ts new file mode 100644 index 000000000..3023b0da8 --- /dev/null +++ b/evm/evm-processor/src/ds-rpc/filter.ts @@ -0,0 +1,226 @@ +import {assertNotNull, weakMemo} from '@subsquid/util-internal' +import {EntityFilter, FilterBuilder} from '@subsquid/util-internal-processor-tools' +import {Block, Log, StateDiff, Trace, Transaction} from '../mapping/entities' +import {DataRequest} from '../interfaces/data-request' + + +function buildLogFilter(dataRequest: DataRequest): EntityFilter { + let items = new EntityFilter() + for (let req of dataRequest.logs || []) { + let {address, topic0, topic1, topic2, topic3, ...relations} = req + let filter = new FilterBuilder() + filter.propIn('address', address) + filter.getIn(log => assertNotNull(log.topics)[0], topic0) + filter.getIn(log => assertNotNull(log.topics)[1], topic1) + filter.getIn(log => assertNotNull(log.topics)[2], topic2) + filter.getIn(log => assertNotNull(log.topics)[3], topic3) + items.add(filter, relations) + } + return items +} + + +function buildTransactionFilter(dataRequest: DataRequest): EntityFilter { + let items = new EntityFilter() + for (let req of dataRequest.transactions || []) { + let {to, from, sighash, ...relations} = req + let filter = new FilterBuilder() + filter.propIn('to', to) + filter.propIn('from', from) + filter.propIn('sighash', sighash) + items.add(filter, relations) + } + return items +} + + +function buildTraceFilter(dataRequest: DataRequest): EntityFilter { + let items = new EntityFilter() + for (let req of dataRequest.traces || []) { + let { + type, + createFrom, + callTo, + callSighash, + suicideRefundAddress, + rewardAuthor, + ...relations + } = req + let filter = new FilterBuilder() + filter.propIn('type', type as Trace['type'][]) + filter.getIn(trace => trace.type === 'create' && assertNotNull(trace.action?.from), createFrom) + filter.getIn(trace => trace.type === 'call' && assertNotNull(trace.action?.to), callTo) + filter.getIn(trace => trace.type === 'call' && assertNotNull(trace.action?.sighash), callSighash) + filter.getIn(trace => trace.type === 'suicide' && assertNotNull(trace.action?.refundAddress), suicideRefundAddress) + filter.getIn(trace => trace.type === 'reward' && assertNotNull(trace.action?.author), rewardAuthor) + items.add(filter, relations) + } + return items +} + + +function buildStateDiffFilter(dataRequest: DataRequest): EntityFilter { + let items = new EntityFilter() + for (let req of dataRequest.stateDiffs || []) { + let { + address, + key, + kind, + ...relations + } = req + let filter = new FilterBuilder() + filter.propIn('address', address) + filter.propIn('key', key) + filter.propIn('kind', kind) + items.add(filter, relations) + } + return items +} + + +const getItemFilter = weakMemo((dataRequest: DataRequest) => { + return { + logs: buildLogFilter(dataRequest), + transactions: buildTransactionFilter(dataRequest), + traces: buildTraceFilter(dataRequest), + stateDiffs: buildStateDiffFilter(dataRequest) + } +}) + + +class IncludeSet { + logs = new Set() + transactions = new Set() + traces = new Set() + stateDiffs = new Set() + + addLog(log?: Log): void { + if (log) this.logs.add(log) + } + + addTransaction(tx?: Transaction): void { + if (tx) this.transactions.add(tx) + } + + addTrace(trace?: Trace): void { + if (trace) this.traces.add(trace) + } + + addTraceStack(trace?: Trace): void { + while (trace) { + this.traces.add(trace) + trace = trace.parent + } + } + + addStateDiff(diff?: StateDiff): void { + if (diff) this.stateDiffs.add(diff) + } +} + + +export function filterBlock(block: Block, dataRequest: DataRequest): void { + let items = getItemFilter(dataRequest) + + let include = new IncludeSet() + + if (items.logs.present()) { + for (let log of block.logs) { + let rel = items.logs.match(log) + if (rel == null) continue + include.addLog(log) + if (rel.transaction) { + include.addTransaction(log.transaction) + } + } + } + + if (items.transactions.present()) { + for (let tx of block.transactions) { + let rel = items.transactions.match(tx) + if (rel == null) continue + include.addTransaction(tx) + if (rel.logs) { + for (let log of tx.logs) { + include.addLog(log) + } + } + if (rel.traces) { + for (let trace of tx.traces) { + include.addTrace(trace) + } + } + if (rel.stateDiffs) { + for (let diff of tx.stateDiffs) { + include.addStateDiff(diff) + } + } + } + } + + if (items.traces.present()) { + for (let trace of block.traces) { + let rel = items.traces.match(trace) + if (rel == null) continue + include.addTrace(trace) + if (rel.parents) { + include.addTraceStack(trace.parent) + } + if (rel.subtraces) { + for (let sub of trace.children) { + include.addTrace(sub) + } + } + if (rel.transaction) { + include.addTransaction(trace.transaction) + } + } + } + + if (items.stateDiffs.present()) { + for (let diff of block.stateDiffs) { + let rel = items.stateDiffs.match(diff) + if (rel == null) continue + include.addStateDiff(diff) + if (rel.transaction) { + include.addTransaction(diff.transaction) + } + } + } + + block.logs = block.logs.filter(log => { + if (!include.logs.has(log)) return false + if (log.transaction && !include.transactions.has(log.transaction)) { + log.transaction = undefined + } + return true + }) + + block.transactions = block.transactions.filter(tx => { + if (!include.transactions.has(tx)) return false + tx.logs = tx.logs.filter(it => include.logs.has(it)) + tx.traces = tx.traces.filter(it => include.traces.has(it)) + tx.stateDiffs = tx.stateDiffs.filter(it => include.stateDiffs.has(it)) + return true + }) + + block.traces = block.traces.filter(trace => { + if (!include.traces.has(trace)) return false + if (trace.transaction && !include.transactions.has(trace.transaction)) { + trace.transaction = undefined + } + if (trace.parent && !include.traces.has(trace.parent)) { + trace.parent = undefined + } + trace.children = trace.children.filter(it => include.traces.has(it)) + return true + }) +} diff --git a/evm/evm-processor/src/ds-rpc/mapping.ts b/evm/evm-processor/src/ds-rpc/mapping.ts index bc4e6c984..f1e79d187 100644 --- a/evm/evm-processor/src/ds-rpc/mapping.ts +++ b/evm/evm-processor/src/ds-rpc/mapping.ts @@ -1,417 +1,361 @@ import {addErrorContext, assertNotNull, unexpectedCase} from '@subsquid/util-internal' +import {cast, GetCastType} from '@subsquid/util-internal-validation' +import {GetPropsCast} from '@subsquid/util-internal-validation/lib/composite/object' import assert from 'assert' -import {AllFields, BlockData, BlockHeader, FieldSelection, Transaction} from '../interfaces/data' -import {DataRequest} from '../interfaces/data-request' +import {Bytes, Bytes20, Bytes32} from '../interfaces/base' +import {FieldSelection} from '../interfaces/data' import { - Bytes, - Bytes20, - Bytes32, - EvmStateDiff, - EvmTrace, - EvmTraceBase, - EvmTraceCall, - EvmTraceCreate, - Qty + EvmTraceCallAction, + EvmTraceCallResult, + EvmTraceCreateAction, + EvmTraceCreateResult, + EvmTraceSuicideAction } from '../interfaces/evm' -import {formatId} from '../util' -import * as rpc from './rpc' - - -export function mapBlock(block: rpc.Block, transactionsRequested: boolean): BlockData { +import { + Block, + BlockHeader, + Log, + StateDiff, + StateDiffAdd, + StateDiffChange, + StateDiffDelete, + StateDiffNoChange, + Trace, + TraceCall, + TraceCreate, + TraceReward, + TraceSuicide, + Transaction +} from '../mapping/entities' +import {setUpRelations} from '../mapping/relations' +import {getLogProps, getTraceFrameValidator, isEmpty} from '../mapping/schema' +import {filterBlock} from './filter' +import {MappingRequest} from './request' +import {Block as RpcBlock, DebugStateDiffResult, DebugStateMap, TraceDiff, TraceStateDiff} from './rpc-data' +import {DebugFrame, getBlockValidator} from './schema' +import {getTxHash} from './util' + + +export function mapBlock(rpcBlock: RpcBlock, req: MappingRequest): Block { try { - return tryMapBlock(block, transactionsRequested) + return tryMapBlock(rpcBlock, req) } catch(err: any) { throw addErrorContext(err, { - blockHeight: getBlockHeight(block), - blockHash: block.hash + blockHash: rpcBlock.hash, + blockHeight: rpcBlock.height }) } } -function tryMapBlock(src: rpc.Block, transactionsRequested: boolean): BlockData { - let block: BlockData = { - header: mapBlockHeader(src), - transactions: [], - logs: [], - traces: [], - stateDiffs: [] - } +function tryMapBlock(rpcBlock: RpcBlock, req: MappingRequest): Block { + let src = cast(getBlockValidator(req), rpcBlock) + + let {number, hash, parentHash, transactions, ...headerProps} = src.block + let header = new BlockHeader(number, hash, parentHash) + Object.assign(header, headerProps) - if (transactionsRequested) { - for (let i = 0; i < src.transactions.length; i++) { - let stx = src.transactions[i] - let tx: Transaction - let id = formatId(block.header.height, block.header.hash, i) + let block = new Block(header) + + if (req.transactionList) { + for (let i = 0; i < transactions.length; i++) { + let stx = transactions[i] + let tx = new Transaction(header, i) if (typeof stx == 'string') { - tx = { - id, - transactionIndex: i, - hash: stx, - block: block.header - } as Transaction - } else { - tx = { - id, - transactionIndex: i, - hash: stx.hash, - from: stx.from, - to: stx.to || undefined, - input: stx.input, - nonce: qty2Int(stx.nonce), - v: stx.v == null ? undefined : BigInt(stx.v), - r: stx.r, - s: stx.s, - value: BigInt(stx.value), - gas: BigInt(stx.gas), - gasPrice: BigInt(stx.gasPrice), - chainId: stx.chainId == null ? undefined : qty2Int(stx.chainId), - sighash: stx.input.slice(0, 10), - block: block.header + if (req.fields.transaction?.hash) { + tx.hash = stx } + } else { + let {transactionIndex, ...props} = stx + Object.assign(tx, props) + assert(transactionIndex === i) } + block.transactions.push(tx) + } + } - if (src._receipts) { - let receipt = src._receipts[i] - assert(receipt.transactionHash === tx.hash) - tx.gasUsed = BigInt(receipt.gasUsed) - tx.cumulativeGasUsed = BigInt(receipt.cumulativeGasUsed) - tx.effectiveGasPrice = BigInt(receipt.effectiveGasPrice) - tx.contractAddress = receipt.contractAddress || undefined - tx.type = qty2Int(receipt.type) - tx.status = qty2Int(receipt.status) + if (req.receipts) { + let receipts = assertNotNull(src.receipts) + for (let i = 0; i < receipts.length; i++) { + let {transactionIndex, logs, ...props} = receipts[i] + assert(transactionIndex === i) + Object.assign(block.transactions[i], props) + if (req.logList) { + for (let log of assertNotNull(logs)) { + block.logs.push(makeLog(header, log)) + } } - - block.transactions.push(tx) } } - for (let log of iterateLogs(src)) { - let logIndex = qty2Int(log.logIndex) - let transactionIndex = qty2Int(log.transactionIndex) - block.logs.push({ - id: formatId(block.header.height, block.header.hash, logIndex), - logIndex, - transactionIndex, - transactionHash: log.transactionHash, - address: log.address, - topics: log.topics, - data: log.data, - block: block.header, - transaction: block.transactions[transactionIndex] - }) + if (src.logs) { + assert(block.logs.length == 0) + for (let log of src.logs) { + block.logs.push(makeLog(header, log)) + } } - if (src._traceReplays) { - let hash2idx = new Map(src.transactions.map((tx, idx) => [getTxHash(tx), idx])) - for (let rep of src._traceReplays) { - let transactionIndex = assertNotNull(hash2idx.get(rep.transactionHash)) + if (src.traceReplays) { + let txIndex = new Map(src.block.transactions.map((tx, idx) => { + return [getTxHash(tx), idx] + })) + for (let rep of src.traceReplays) { + let transactionIndex = assertNotNull(txIndex.get(rep.transactionHash)) if (rep.trace) { for (let frame of rep.trace) { - block.traces.push({ - ...mapTraceFrame(transactionIndex, frame), - block: block.header, - transaction: block.transactions[transactionIndex] - }) + block.traces.push( + makeTraceRecordFromReplayFrame(header, transactionIndex, frame) + ) } } if (rep.stateDiff) { - for (let diff of iterateTraceStateDiffs(transactionIndex, rep.stateDiff)) { - block.stateDiffs.push({ - ...diff, - block: block.header, - transaction: block.transactions[transactionIndex] - }) + for (let diff of mapReplayStateDiff(header, transactionIndex, rep.stateDiff)) { + block.stateDiffs.push(diff) } } } } - if (src._debugFrames) { + if (src.debugFrames) { assert(block.traces.length == 0) - assert(src._debugFrames.length === src.transactions.length) - for (let i = 0; i < src._debugFrames.length; i++) { - for (let frame of mapDebugFrame(i, src._debugFrames[i])) { - block.traces.push({ - ...frame, - block: block.header, - transaction: block.transactions[i] - }) + for (let i = 0; i < src.debugFrames.length; i++) { + for (let trace of mapDebugFrame(header, i, src.debugFrames[i], req.fields)) { + block.traces.push(trace) } } } - if (src._debugStateDiffs) { + if (src.debugStateDiffs) { assert(block.stateDiffs.length == 0) - assert(src._debugStateDiffs.length === src.transactions.length) - for (let i = 0; i < src._debugStateDiffs.length; i++) { - for (let diff of mapDebugStateDiff(i, src._debugStateDiffs[i])) { - block.stateDiffs.push({ - ...diff, - block: block.header, - transaction: block.transactions[i] - }) + for (let i = 0; i < src.debugStateDiffs.length; i++) { + for (let diff of mapDebugStateDiff(header, i, src.debugStateDiffs[i])) { + block.stateDiffs.push(diff) } } } - return block -} + setUpRelations(block) + filterBlock(block, req.dataRequest) - -function mapBlockHeader(block: rpc.Block): BlockHeader { - let height = qty2Int(block.number) - return { - id: formatId(height, block.hash), - height, - hash: block.hash, - parentHash: block.parentHash, - timestamp: qty2Int(block.timestamp) * 1000, - stateRoot: block.stateRoot, - transactionsRoot: block.transactionsRoot, - receiptsRoot: block.receiptsRoot, - logsBloom: block.logsBloom, - extraData: block.extraData, - sha3Uncles: block.sha3Uncles, - miner: block.miner, - nonce: block.nonce, - size: BigInt(block.size), - gasLimit: BigInt(block.gasLimit), - gasUsed: BigInt(block.gasUsed), - difficulty: block.difficulty == null ? undefined : BigInt(block.difficulty) - } + return block } -function* iterateLogs(block: rpc.Block): Iterable { - if (block._receipts) { - for (let receipt of block._receipts) { - yield* receipt.logs - } - } else if (block._logs) { - yield* block._logs - } +function makeLog(blockHeader: BlockHeader, src: GetPropsCast>): Log { + let {logIndex, transactionIndex, ...props} = src + let log = new Log(blockHeader, logIndex, transactionIndex) + Object.assign(log, props) + return log } -function mapTraceFrame(transactionIndex: number, src: rpc.TraceFrame): EvmTrace { - switch(src.type) { - case 'create': { - let rec: EvmTraceCreate = { - transactionIndex, - traceAddress: src.traceAddress, - subtraces: src.subtraces, - error: src.error, - type: src.type, - action: { - from: src.action.from, - value: BigInt(src.action.value), - gas: BigInt(src.action.gas), - init: src.action.init - } - } - if (src.result) { - rec.result = { - address: src.result.address, - code: src.result.code, - gasUsed: BigInt(src.result.gasUsed) - } - } - return rec - } - case 'call': { - let rec: EvmTraceCall = { - transactionIndex, - traceAddress: src.traceAddress, - subtraces: src.subtraces, - error: src.error, - type: src.type, - action: { - callType: src.action.callType, - from: src.action.from, - to: src.action.to, - value: BigInt(src.action.value), - gas: BigInt(src.action.gas), - input: src.action.input, - sighash: src.action.input.slice(0, 10) - } - } - if (src.result) { - rec.result = { - gasUsed: BigInt(src.result.gasUsed), - output: src.result.output - } - } - return rec - } - case 'suicide': { - return { - transactionIndex, - traceAddress: src.traceAddress, - subtraces: src.subtraces, - error: src.error, - type: src.type, - action: { - address: src.action.address, - refundAddress: src.action.refundAddress, - balance: BigInt(src.action.balance) - } - } - } - case 'reward': { - return { - transactionIndex, - traceAddress: src.traceAddress, - subtraces: src.subtraces, - error: src.error, - type: src.type, - action: { - author: src.action.author, - value: BigInt(src.action.value), - type: src.action.type - } - } - } +function makeTraceRecordFromReplayFrame( + header: BlockHeader, + transactionIndex: number, + frame: GetCastType> +): Trace { + let {traceAddress, type, ...props} = frame + let trace: Trace + switch(type) { + case 'create': + trace = new TraceCreate(header, transactionIndex, traceAddress) + break + case 'call': + trace = new TraceCall(header, transactionIndex, traceAddress) + break + case 'suicide': + trace = new TraceSuicide(header, transactionIndex, traceAddress) + break + case 'reward': + trace = new TraceReward(header, transactionIndex, traceAddress) + break default: - throw unexpectedCase((src as any).type) + throw unexpectedCase(type) } + Object.assign(trace, props) + return trace } -function* iterateTraceStateDiffs( +function* mapReplayStateDiff( + header: BlockHeader, transactionIndex: number, - stateDiff?: Record -): Iterable { - if (stateDiff == null) return + stateDiff: Record +): Iterable { for (let address in stateDiff) { let diffs = stateDiff[address] - yield mapTraceStateDiff(transactionIndex, address, 'code', diffs.code) - yield mapTraceStateDiff(transactionIndex, address, 'balance', diffs.balance) - yield mapTraceStateDiff(transactionIndex, address, 'nonce', diffs.nonce) + yield makeStateDiffFromReplay(header, transactionIndex, address, 'code', diffs.code) + yield makeStateDiffFromReplay(header, transactionIndex, address, 'balance', diffs.balance) + yield makeStateDiffFromReplay(header, transactionIndex, address, 'nonce', diffs.nonce) for (let key in diffs.storage) { - yield mapTraceStateDiff(transactionIndex, address, key, diffs.storage[key]) + yield makeStateDiffFromReplay(header, transactionIndex, address, key, diffs.storage[key]) } } } -function mapTraceStateDiff(transactionIndex: number, address: Bytes20, key: string, diff: rpc.TraceDiff): EvmStateDiff { +function makeStateDiffFromReplay( + header: BlockHeader, + transactionIndex: number, + address: Bytes20, + key: string, + diff: TraceDiff +): StateDiff { if (diff === '=') { - return { - transactionIndex, - address, - key, - kind: '=' - } + return new StateDiffNoChange(header, transactionIndex, address, key) } if (diff['+']) { - return { - transactionIndex, - address, - key, - kind: '+', - next: diff['+'] - } + let rec = new StateDiffAdd(header, transactionIndex, address, key) + rec.next = diff['+'] + return rec } if (diff['*']) { - return { - transactionIndex, - address, - key, - kind: '*', - prev: diff['*'].from, - next: diff['*'].to - } + let rec = new StateDiffChange(header, transactionIndex, address, key) + rec.prev = diff['*'].from + rec.next = diff['*'].to + return rec } if (diff['-']) { - return { - transactionIndex, - address, - key, - kind: '-', - prev: diff['-'] - } + let rec = new StateDiffDelete(header, transactionIndex, address, key) + rec.prev = diff['-'] + return rec } throw unexpectedCase() } -function* mapDebugFrame(transactionIndex: number, debugFrameResult: rpc.DebugFrameResult): Iterable { +function* mapDebugFrame( + header: BlockHeader, + transactionIndex: number, + debugFrameResult: {result: DebugFrame}, + fields: FieldSelection | undefined +): Iterable { if (debugFrameResult.result.type == 'STOP' || debugFrameResult.result.type == 'INVALID') { assert(!debugFrameResult.result.calls?.length) return } - for (let rec of traverseDebugFrame(debugFrameResult.result, [])) { - let base: EvmTraceBase = { - transactionIndex, - traceAddress: rec.traceAddress, - subtraces: rec.subtraces, - error: rec.frame.error ?? null, - revertReason: rec.frame.revertReason - } - switch(rec.frame.type) { + let projection = fields?.trace || {} + for (let {traceAddress, subtraces, frame} of traverseDebugFrame(debugFrameResult.result, [])) { + let trace: Trace + switch(frame.type) { case 'CREATE': - case 'CREATE2': - yield { - ...base, - type: 'create', - action: { - from: rec.frame.from, - value: BigInt(assertNotNull(rec.frame.value)), - gas: BigInt(rec.frame.gas), - init: rec.frame.input - }, - result: { - gasUsed: BigInt(rec.frame.gasUsed), - code: rec.frame.output, - address: rec.frame.to - } + case 'CREATE2': { + trace = new TraceCreate(header, transactionIndex, traceAddress) + let action: Partial = {} + + action.from = frame.from + + if (projection.createValue) { + action.value = frame.value + } + if (projection.createGas) { + action.gas = frame.gas + } + if (projection.createInit) { + action.init = frame.input + } + if (!isEmpty(action)) { + trace.action = action + } + let result: Partial = {} + if (projection.createResultGasUsed) { + result.gasUsed = frame.gasUsed + } + if (projection.createResultCode) { + result.code = frame.output + } + if (projection.createResultAddress) { + result.address = frame.to + } + if (!isEmpty(result)) { + trace.result = result } break + } case 'CALL': case 'CALLCODE': - case 'STATICCALL': case 'DELEGATECALL': - yield { - ...base, - type: 'call', - action: { - callType: rec.frame.type.toLowerCase(), - from: rec.frame.from, - to: rec.frame.to, - value: rec.frame.value == null ? undefined : BigInt(rec.frame.value), - gas: BigInt(rec.frame.gas), - input: rec.frame.input, - sighash: rec.frame.input.slice(0, 10) - }, - result: { - gasUsed: BigInt(rec.frame.gasUsed), - output: rec.frame.output + case 'STATICCALL': { + trace = new TraceCall(header, transactionIndex, traceAddress) + let action: Partial = {} + let hasAction = false + if (projection.callCallType) { + action.callType = frame.type.toLowerCase() + } + if (projection.callFrom) { + action.from = frame.from + } + + action.to = frame.to + + if (projection.callValue) { + hasAction = true + if (frame.value != null) { + action.value = frame.value } } + if (projection.callGas) { + action.gas = frame.gas + } + if (projection.callInput) { + action.input = frame.input + } + + action.sighash = frame.input.slice(0, 10) + + if (hasAction || !isEmpty(action)) { + trace.action = action + } + let result: Partial = {} + if (projection.callResultGasUsed) { + result.gasUsed = frame.gasUsed + } + if (projection.callResultOutput) { + result.output = frame.output + } + if (!isEmpty(result)) { + trace.result = result + } break - case 'SELFDESTRUCT': - yield { - ...base, - type: 'suicide', - action: { - address: rec.frame.from, - refundAddress: rec.frame.to, - balance: BigInt(assertNotNull(rec.frame.value)) - } + } + case 'SELFDESTRUCT': { + trace = new TraceSuicide(header, transactionIndex, traceAddress) + let action: Partial = {} + if (projection.suicideAddress) { + action.address = frame.from + } + if (projection.suicideBalance) { + action.balance = frame.value + } + + action.refundAddress = frame.to + + if (!isEmpty(action)) { + trace.action = action } break + } default: - throw unexpectedCase(rec.frame.type) + throw unexpectedCase() + } + if (projection.subtraces) { + trace.subtraces = subtraces } + if (frame.error != null) { + trace.error = frame.error + } + if (frame.revertReason != null) { + trace.revertReason = frame.revertReason + } + yield trace } } -function* traverseDebugFrame(frame: rpc.DebugFrame, traceAddress: number[]): Iterable<{ +function* traverseDebugFrame(frame: DebugFrame, traceAddress: number[]): Iterable<{ traceAddress: number[] subtraces: number - frame: rpc.DebugFrame + frame: DebugFrame }> { let subcalls = frame.calls || [] yield {traceAddress, subtraces: subcalls.length, frame} @@ -421,202 +365,88 @@ function* traverseDebugFrame(frame: rpc.DebugFrame, traceAddress: number[]): Ite } -function* mapDebugStateDiff(transactionIndex: number, debugDiffResult: rpc.DebugStateDiffResult): Iterable { +function* mapDebugStateDiff( + header: BlockHeader, + transactionIndex: number, + debugDiffResult: GetCastType +): Iterable { let {pre, post} = debugDiffResult.result for (let address in pre) { let prev = pre[address] let next = post[address] || {} - yield* mapDebugDiff(transactionIndex, address, prev, next) + yield* mapDebugStateMap(header, transactionIndex, address, prev, next) } for (let address in post) { if (pre[address] == null) { - yield* mapDebugDiff(transactionIndex, address, {}, post[address]) + yield* mapDebugStateMap(header, transactionIndex, address, {}, post[address]) } } } -function* mapDebugDiff( +function* mapDebugStateMap( + header: BlockHeader, transactionIndex: number, address: Bytes20, - prev: rpc.DebugStateMap, - next: rpc.DebugStateMap -): Iterable { + prev: GetCastType, + next: GetCastType +): Iterable { if (next.code) { - yield makeDiffRecord(transactionIndex, address, 'code', prev.code, next.code) + yield makeDebugStateDiffRecord(header, transactionIndex, address, 'code', prev.code, next.code) } if (next.balance) { - yield makeDiffRecord(transactionIndex, address, 'balance', prev.balance, next.balance) + yield makeDebugStateDiffRecord( + header, + transactionIndex, + address, + 'balance', + '0x'+(prev.balance || 0).toString(16), + '0x'+next.balance.toString(16) + ) } if (next.nonce) { - yield makeDiffRecord( + yield makeDebugStateDiffRecord( + header, transactionIndex, address, 'nonce', - '0x'+prev.nonce?.toString(16), - '0x'+next.nonce?.toString(16) + '0x'+(prev.nonce ?? 0).toString(16), + '0x'+next.nonce.toString(16) ) } for (let key in prev.storage) { - yield makeDiffRecord(transactionIndex, address, key, prev.storage[key], next.storage?.[key]) + yield makeDebugStateDiffRecord(header, transactionIndex, address, key, prev.storage[key], next.storage?.[key]) } for (let key in next.storage) { if (prev.storage?.[key] == null) { - yield makeDiffRecord(transactionIndex, address, key, undefined, next.storage[key]) + yield makeDebugStateDiffRecord(header, transactionIndex, address, key, undefined, next.storage[key]) } } } -function makeDiffRecord(transactionIndex: number, key: Bytes32, address: Bytes20, prev?: Bytes, next?: Bytes20): EvmStateDiff { +function makeDebugStateDiffRecord( + header: BlockHeader, + transactionIndex: number, + address: Bytes20, + key: Bytes32, + prev?: Bytes, + next?: Bytes +): StateDiff { if (prev == null) { - return { - transactionIndex, - address, - key, - kind: '+', - next: assertNotNull(next) - } + let diff = new StateDiffAdd(header, transactionIndex, address, key) + diff.next = assertNotNull(next) + return diff } if (next == null) { - return { - transactionIndex, - address, - key, - kind: '-', - prev - } - } - return { - transactionIndex, - address, - key, - kind: '*', - prev, - next - } -} - -export function qty2Int(qty: Qty): number { - let i = parseInt(qty, 16) - assert(Number.isSafeInteger(i)) - return i -} - - -export function toQty(i: number): Qty { - return '0x'+i.toString(16) -} - - -export function toRpcDataRequest(req?: DataRequest): rpc.DataRequest { - return { - transactionList: transactionsRequested(req), - transactions: transactionsRequested(req) && transactionRequired(req), - logs: !!req?.logs?.length, - receipts: receiptsRequested(req), - traces: tracesRequested(req), - stateDiffs: stateDiffsRequested(req) + let diff = new StateDiffDelete(header, transactionIndex, address, key) + diff.prev = assertNotNull(prev) + return diff } -} - - -function transactionsRequested(req?: DataRequest): boolean { - if (req == null) return false - if (req.transactions?.length) return true - for (let items of [req.logs, req.traces, req.stateDiffs]) { - if (items) { - for (let it of items) { - if (it.transaction) return true - } - } + { + let diff = new StateDiffChange(header, transactionIndex, address, key) + diff.prev = prev + diff.next = next + return diff } - return false -} - - -function transactionRequired(req?: DataRequest): boolean { - let f: keyof Exclude - for (f in req?.fields?.transaction) { - switch(f) { - case 'hash': - case 'status': - case 'type': - case 'gasUsed': - case 'cumulativeGasUsed': - case 'effectiveGasPrice': - case 'contractAddress': - break - default: - return true - } - } - return false -} - - -function receiptsRequested(req?: DataRequest): boolean { - if (!transactionsRequested(req)) return false - let fields = req?.fields?.transaction - if (fields == null) return false - return !!( - fields.status || - fields.type || - fields.gasUsed || - fields.cumulativeGasUsed || - fields.effectiveGasPrice || - fields.contractAddress - ) -} - - -function tracesRequested(req?: DataRequest): boolean { - if (req == null) return false - if (req.traces?.length) return true - for (let tx of req.transactions || []) { - if (tx.traces) return true - } - return false -} - - -function stateDiffsRequested(req?: DataRequest): boolean { - if (req == null) return false - if (req.stateDiffs?.length) return true - for (let tx of req.transactions || []) { - if (tx.stateDiffs) return true - } - return false -} - - -export function getTxHash(tx: Bytes32 | rpc.Transaction): Bytes32 { - if (typeof tx == 'string') { - return tx - } else { - return tx.hash - } -} - - -export function getBlockName(block: rpc.Block | {height: number, hash?: string, number?: undefined}): string { - let height: number - let hash: string | undefined - if (block.number == null) { - height = block.height - hash = block.hash - } else { - height = qty2Int(block.number) - hash = block.hash - } - if (hash) { - return `${height}#${hash.slice(2, 8)}` - } else { - return ''+height - } -} - - -export function getBlockHeight(block: rpc.Block): number { - return qty2Int(block.number) } diff --git a/evm/evm-processor/src/ds-rpc/request.ts b/evm/evm-processor/src/ds-rpc/request.ts new file mode 100644 index 000000000..f7968eeaf --- /dev/null +++ b/evm/evm-processor/src/ds-rpc/request.ts @@ -0,0 +1,118 @@ +import {FieldSelection} from '../interfaces/data' +import {DataRequest} from '../interfaces/data-request' +import {_EvmTx, _EvmTxReceipt} from '../interfaces/evm' +import {DataRequest as RpcDataRequest} from './rpc-data' + + +export interface MappingRequest extends RpcDataRequest { + fields: FieldSelection + transactionList: boolean + logList: boolean + dataRequest: DataRequest +} + + +export function toMappingRequest(req?: DataRequest): MappingRequest { + let txs = transactionsRequested(req) + let logs = logsRequested(req) + let receipts = txs && isRequested(TX_RECEIPT_FIELDS, req?.fields?.transaction) + return { + fields: req?.fields || {}, + transactionList: txs, + logList: logs, + // include transactions if we potentially have item filters or when tx fields are requested + transactions: !!req?.transactions?.length || txs && isRequested(TX_FIELDS, req?.fields?.transaction), + logs: logs && !receipts, + receipts, + traces: tracesRequested(req), + stateDiffs: stateDiffsRequested(req), + dataRequest: req || {} + } +} + + +function transactionsRequested(req?: DataRequest): boolean { + if (req == null) return false + if (req.transactions?.length) return true + for (let items of [req.logs, req.traces, req.stateDiffs]) { + if (items) { + for (let it of items) { + if (it.transaction) return true + } + } + } + return false +} + + +function logsRequested(req?: DataRequest): boolean { + if (req == null) return false + if (req.logs?.length) return true + if (req.transactions) { + for (let tx of req.transactions) { + if (tx.logs) return true + } + } + return false +} + + +function tracesRequested(req?: DataRequest): boolean { + if (req == null) return false + if (req.traces?.length) return true + if (req.transactions) { + for (let tx of req.transactions) { + if (tx.traces) return true + } + } + return false +} + + +function stateDiffsRequested(req?: DataRequest): boolean { + if (req == null) return false + if (req.stateDiffs?.length) return true + if (req.transactions) { + for (let tx of req.transactions) { + if (tx.stateDiffs) return true + } + } + return false +} + + +const TX_FIELDS: {[K in Exclude]: true} = { + from: true, + to: true, + gas: true, + gasPrice: true, + maxFeePerGas: true, + maxPriorityFeePerGas: true, + input: true, + nonce: true, + value: true, + v: true, + r: true, + s: true, + yParity: true, + chainId: true +} + + +const TX_RECEIPT_FIELDS: {[K in keyof _EvmTxReceipt]: true} = { + gasUsed: true, + cumulativeGasUsed: true, + effectiveGasPrice: true, + contractAddress: true, + type: true, + status: true +} + + +function isRequested(set: Record, selection?: Record): boolean { + if (selection == null) return false + for (let key in selection) { + if (set[key] && selection[key]) return true + } + return false +} diff --git a/evm/evm-processor/src/ds-rpc/rpc-data.ts b/evm/evm-processor/src/ds-rpc/rpc-data.ts new file mode 100644 index 000000000..9eeb40725 --- /dev/null +++ b/evm/evm-processor/src/ds-rpc/rpc-data.ts @@ -0,0 +1,217 @@ +import { + array, + BYTES, + constant, + GetSrcType, + keyTaggedUnion, + NAT, + object, + oneOf, + option, + QTY, + record, + ref, + SMALL_QTY, + STRING, + Validator +} from '@subsquid/util-internal-validation' +import {Bytes, Bytes20, Bytes32, Qty} from '../interfaces/base' +import {project} from '../mapping/schema' + + +export interface DataRequest { + logs?: boolean + transactions?: boolean + receipts?: boolean + traces?: boolean + stateDiffs?: boolean + preferTraceApi?: boolean + useDebugApiForStateDiffs?: boolean +} + + +export interface Block { + height: number + hash: Bytes32 + block: GetBlock + receipts?: TransactionReceipt[] + logs?: Log[] + traceReplays?: TraceTransactionReplay[] + debugFrames?: DebugFrameResult[] + debugStateDiffs?: DebugStateDiffResult[] + _isInvalid?: boolean +} + + +const Transaction = object({ + blockNumber: SMALL_QTY, + blockHash: BYTES, + transactionIndex: SMALL_QTY, + hash: BYTES, + input: BYTES +}) + + +export type Transaction = GetSrcType + + +export const GetBlockWithTransactions = object({ + number: SMALL_QTY, + hash: BYTES, + parentHash: BYTES, + logsBloom: BYTES, + transactions: array(Transaction) +}) + + +export const GetBlockNoTransactions = object({ + number: SMALL_QTY, + hash: BYTES, + parentHash: BYTES, + logsBloom: BYTES, + transactions: array(BYTES) +}) + + +export interface GetBlock { + number: Qty + hash: Bytes32 + parentHash: Bytes32 + logsBloom: Bytes + transactions: Bytes32[] | Transaction[] +} + + +export const Log = object({ + blockNumber: SMALL_QTY, + blockHash: BYTES, + logIndex: SMALL_QTY, + transactionIndex: SMALL_QTY +}) + + +export type Log = GetSrcType + + +export const TransactionReceipt = object({ + blockNumber: SMALL_QTY, + blockHash: BYTES, + transactionIndex: SMALL_QTY, + logs: array(Log) +}) + + +export type TransactionReceipt = GetSrcType + + +export const DebugFrame: Validator = object({ + type: STRING, + input: BYTES, + calls: option(array(ref(() => DebugFrame))) +}) + + +export interface DebugFrame { + type: string + input: Bytes + calls?: DebugFrame[] | null +} + + +export const DebugFrameResult = object({ + result: DebugFrame +}) + + +export type DebugFrameResult = GetSrcType + + +export const DebugStateMap = object({ + balance: option(QTY), + code: option(BYTES), + nonce: option(SMALL_QTY), + storage: option(record(BYTES, BYTES)) +}) + + +export type DebugStateMap = GetSrcType + + +export const DebugStateDiff = object({ + pre: record(BYTES, DebugStateMap), + post: record(BYTES, DebugStateMap) +}) + + +export type DebugStateDiff = GetSrcType + + +export const DebugStateDiffResult = object({ + result: DebugStateDiff +}) + + +export type DebugStateDiffResult = GetSrcType + + +export const TraceFrame = object({ + blockHash: option(BYTES), + transactionHash: option(BYTES), + traceAddress: array(NAT), + type: STRING, + action: object({}) +}) + + +export type TraceFrame = GetSrcType + + +const TraceDiffObj = keyTaggedUnion({ + '+': BYTES, + '*': object({from: BYTES, to: BYTES}), + '-': BYTES +}) + + +const TraceDiff = oneOf({ + '= sign': constant('='), + 'diff object': TraceDiffObj +}) + + +export type TraceDiff = GetSrcType + + +export const TraceStateDiff = object({ + balance: TraceDiff, + code: TraceDiff, + nonce: TraceDiff, + storage: record(BYTES, TraceDiff) +}) + + +export type TraceStateDiff = GetSrcType + + +export interface TraceTransactionReplay { + transactionHash?: Bytes32 | null + trace?: TraceFrame[] + stateDiff?: Record +} + + +export interface TraceReplayTraces { + trace?: boolean + stateDiff?: boolean +} + + +export function getTraceTransactionReplayValidator(tracers: TraceReplayTraces): Validator { + return object({ + transactionHash: option(BYTES), + ...project(tracers, { + trace: array(TraceFrame), + stateDiff: record(BYTES, TraceStateDiff) + }) + }) +} diff --git a/evm/evm-processor/src/ds-rpc/rpc.ts b/evm/evm-processor/src/ds-rpc/rpc.ts index dfa4a855e..9c19a956a 100644 --- a/evm/evm-processor/src/ds-rpc/rpc.ts +++ b/evm/evm-processor/src/ds-rpc/rpc.ts @@ -1,256 +1,572 @@ -import {Bytes, Bytes20, Bytes32, Bytes8, Qty} from '../interfaces/evm' - - -export interface Block { - number: Qty - hash: Bytes32 - parentHash: Bytes32 - nonce?: Bytes8 - sha3Uncles: Bytes32 - logsBloom: Bytes - transactionsRoot: Bytes32 - stateRoot: Bytes32 - receiptsRoot: Bytes32 - mixHash?: Bytes - miner: Bytes20 - difficulty?: Qty - totalDifficulty?: Qty - extraData: Bytes - size: Qty - gasLimit: Qty - gasUsed: Qty - timestamp: Qty - baseFeePerGas?: Qty - transactions: Bytes32[] | Transaction[] - _receipts?: TransactionReceipt[] - _logs?: Log[] - _traceReplays?: TraceTransactionReplay[] - _debugFrames?: DebugFrameResult[] - _debugStateDiffs?: DebugStateDiffResult[] +import {CallOptions, RpcClient, RpcError} from '@subsquid/rpc-client' +import {groupBy, last} from '@subsquid/util-internal' +import {assertIsValid, BlockConsistencyError, trimInvalid} from '@subsquid/util-internal-ingest-tools' +import {FiniteRange, SplitRequest} from '@subsquid/util-internal-range' +import {array, DataValidationError, GetSrcType, nullable, Validator} from '@subsquid/util-internal-validation' +import assert from 'assert' +import {Bytes, Bytes32, Qty} from '../interfaces/base' +import {isEmpty} from '../mapping/schema' +import { + Block, + DataRequest, + DebugFrameResult, + DebugStateDiffResult, + GetBlock, + GetBlockNoTransactions, + GetBlockWithTransactions, + getTraceTransactionReplayValidator, + Log, + TraceFrame, + TraceReplayTraces, + TransactionReceipt +} from './rpc-data' +import {getTxHash, qty2Int, toQty} from './util' + + +const NO_LOGS_BLOOM = '0x'+Buffer.alloc(256).toString('hex') + + +function getResultValidator(validator: V): (result: unknown) => GetSrcType { + return function(result: unknown) { + let err = validator.validate(result) + if (err) { + throw new DataValidationError(`server returned unexpected result: ${err.toString()}`) + } else { + return result as any + } + } } -export interface Transaction { - blockNumber: Qty - blockHash: Bytes32 - from: Bytes20 - gas: Qty - gasPrice: Qty - maxFeePerGas?: Qty - maxPriorityFeePerGas?: Qty - hash: Bytes32 - input: Bytes - nonce: Qty - to?: Bytes20 - transactionIndex: Qty - value: Qty - v?: Qty - r?: Bytes32 - s?: Bytes32 - yParity?: Qty - chainId?: Qty -} +export class Rpc { + private props: RpcProps + constructor( + public readonly client: RpcClient, + private genesisHeight: number = 0, + private priority: number = 0, + props?: RpcProps + ) { + this.props = props || new RpcProps(this.client, this.genesisHeight) + } -export interface TransactionReceipt { - transactionHash: Bytes32 - transactionIndex: Qty - blockHash: Bytes32 - blockNumber: Qty - cumulativeGasUsed: Qty - effectiveGasPrice: Qty - gasUsed: Qty - contractAddress: Bytes20 | null - logs: Log[] - type: Qty - status: Qty -} + withPriority(priority: number): Rpc { + return new Rpc(this.client, this.genesisHeight, priority, this.props) + } + call(method: string, params?: any[], options?: CallOptions): Promise { + return this.client.call(method, params, {priority: this.priority, ...options}) + } -export interface Log { - blockNumber: Qty - blockHash: Bytes32 - logIndex: Qty - transactionIndex: Qty - transactionHash: Bytes32 - address: Bytes20 - data: Bytes - topics: Bytes32[] -} + batchCall(batch: {method: string, params?: any[]}[], options?: CallOptions): Promise { + return this.client.batchCall(batch, {priority: this.priority, ...options}) + } + getBlockByNumber(height: number, withTransactions: boolean): Promise { + return this.call('eth_getBlockByNumber', [ + toQty(height), + withTransactions + ], { + validateResult: getResultValidator( + withTransactions ? nullable(GetBlockWithTransactions) : nullable(GetBlockNoTransactions) + ) + }) + } -export interface TraceFrameBase { - traceAddress: number[] - subtraces: number - error: string | null - transactionHash?: Bytes32 - blockHash?: Bytes32 -} + getBlockByHash(hash: Bytes, withTransactions: boolean): Promise { + return this.call('eth_getBlockByHash', [hash, withTransactions], { + validateResult: getResultValidator( + withTransactions ? nullable(GetBlockWithTransactions) : nullable(GetBlockNoTransactions) + ) + }) + } + async getBlockHash(height: number): Promise { + let block = await this.getBlockByNumber(height, false) + return block?.hash + } -export interface TraceCreateFrame extends TraceFrameBase { - type: 'create' - action: TraceCreateAction - result?: TraceCreateResult -} + async getHeight(): Promise { + let height: Qty = await this.call('eth_blockNumber') + return qty2Int(height) + } + async getColdBlock(blockHash: Bytes32, req?: DataRequest, finalizedHeight?: number): Promise { + let block = await this.getBlockByHash(blockHash, req?.transactions || false).then(toBlock) + if (block == null) throw new BlockConsistencyError({hash: blockHash}) + if (req) { + await this.addRequestedData([block], req, finalizedHeight) + } + if (block._isInvalid) throw new BlockConsistencyError(block) + return block + } -export interface TraceCreateAction { - from: Bytes20 - value: Qty - gas: Qty - init: Bytes -} + async getColdSplit(req: SplitRequest): Promise { + let result = await this.getBlockSplit(req) + for (let i = 0; i < result.length; i++) { + if (result[i] == null) throw new BlockConsistencyError({height: req.range.from + i}) + if (i > 0 && result[i - 1]!.hash !== result[i]!.block.parentHash) + throw new BlockConsistencyError(result[i]!) + } -export interface TraceCreateResult { - gasUsed: Qty - code: Bytes - address: Bytes20 -} + let blocks = result as Block[] + await this.addRequestedData(blocks, req.request) -export interface TraceCallFrame extends TraceFrameBase { - type: 'call' - action: TraceCallAction - result?: TraceCallResult -} + assertIsValid(blocks) + return blocks + } -export interface TraceCallAction { - callType: string - from: Bytes20 - to: Bytes20 - value: Qty - gas: Qty - input: Bytes -} + async getHotSplit(req: SplitRequest & {finalizedHeight: number}): Promise { + let blocks = await this.getBlockSplit(req) + let chain: Block[] = [] -export interface TraceCallResult { - gasUsed: bigint - output: Bytes -} + for (let i = 0; i < blocks.length; i++) { + let block = blocks[i] + if (block == null) break + if (i > 0 && chain[i - 1].hash !== block.block.parentHash) break + chain.push(block) + } + await this.addRequestedData(chain, req.request, req.finalizedHeight) -export interface TraceSuicideFrame extends TraceFrameBase { - type: 'suicide' - action: TraceSuicideAction -} + return trimInvalid(chain) + } + private async getBlockSplit(req: SplitRequest): Promise<(Block | undefined)[]> { + let call = [] + for (let i = req.range.from; i <= req.range.to; i++) { + call.push({ + method: 'eth_getBlockByNumber', + params: [toQty(i), req.request.transactions || false] + }) + } + let blocks = await this.batchCall(call, { + validateResult: getResultValidator( + req.request.transactions ? nullable(GetBlockWithTransactions) : nullable(GetBlockNoTransactions) + ) + }) + return blocks.map(toBlock) + } -export interface TraceSuicideAction { - address: Bytes20 - refundAddress: Bytes20 - balance: Qty -} + private async addRequestedData(blocks: Block[], req: DataRequest, finalizedHeight?: number): Promise { + if (blocks.length == 0) return + let subtasks = [] -export interface TraceRewardFrame extends TraceFrameBase { - type: 'reward' - action: TraceRewardAction -} + if (req.logs) { + subtasks.push(this.addLogs(blocks)) + } + if (req.receipts) { + subtasks.push(this.addReceipts(blocks)) + } -export interface TraceRewardAction { - author: Bytes20 - value: Qty - type: string -} + if (req.traces || req.stateDiffs) { + subtasks.push(this.addTraces(blocks, req, finalizedHeight)) + } + await Promise.all(subtasks) + } -export type TraceFrame = TraceCreateFrame | TraceCallFrame | TraceSuicideFrame | TraceRewardFrame + private async addLogs(blocks: Block[]): Promise { + if (blocks.length == 0) return + let logs = await this.getLogs( + blocks[0].height, + last(blocks).height + ) -interface TraceAddDiff { - '+': Bytes - '*'?: undefined - '-'?: undefined -} + let logsByBlock = groupBy(logs, log => log.blockHash) + for (let block of blocks) { + let logs = logsByBlock.get(block.hash) || [] + if (logs.length == 0 && block.block.logsBloom !== NO_LOGS_BLOOM) { + block._isInvalid = true + } else { + block.logs = logs + } + } + } -interface TraceChangeDiff { - '+'?: undefined - '*': { - from: Bytes - to: Bytes + getLogs(from: number, to: number): Promise { + return this.call('eth_getLogs', [{ + fromBlock: toQty(from), + toBlock: toQty(to) + }], { + validateResult: getResultValidator(array(Log)) + }).catch(async err => { + let range = asTryAnotherRangeError(err) + if (range && range.from == from && from <= range.to && range.to < to) { + let result = await Promise.all([ + this.getLogs(range.from, range.to), + this.getLogs(range.to + 1, to) + ]) + return result[0].concat(result[1]) + } else { + throw err + } + }) } - '-'?: undefined -} + private async addReceipts(blocks: Block[]): Promise { + let method = await this.props.getReceiptsMethod() + switch(method) { + case 'alchemy_getTransactionReceipts': + case 'eth_getBlockReceipts': + return this.addReceiptsByBlock(blocks, method) + default: + return this.addReceiptsByTx(blocks) + } + } -interface TraceDeleteDiff { - '+'?: undefined - '*'?: undefined - '-': Bytes -} + private async addReceiptsByBlock( + blocks: Block[], + method: 'eth_getBlockReceipts' | 'alchemy_getTransactionReceipts' + ): Promise { + let call = blocks.map(block => { + if (method == 'eth_getBlockReceipts') { + return { + method, + params: [block.block.number] + } + } else { + return { + method, + params: [{blockHash: block.hash}] + } + } + }) + + let results: (TransactionReceipt[] | null)[] = await this.batchCall(call, { + validateResult: getResultValidator(nullable(array(TransactionReceipt))) + }) + + for (let i = 0; i < blocks.length; i++) { + let block = blocks[i] + let receipts = results[i] + if (receipts != null && block.block.transactions.length === receipts.length) { + for (let receipt of receipts) { + if (receipt.blockHash !== block.hash) { + block._isInvalid = true + } + } + block.receipts = receipts + } else { + block._isInvalid = true + } + } + } + private async addReceiptsByTx(blocks: Block[]): Promise { + let call = [] + for (let block of blocks) { + for (let tx of block.block.transactions) { + call.push({ + method: 'eth_getTransactionReceipt', + params: [getTxHash(tx)] + }) + } + } + + let receipts = await this.batchCall(call, { + validateResult: getResultValidator(nullable(TransactionReceipt)) + }) + + let receiptsByBlock = groupBy( + receipts.filter(r => r != null) as TransactionReceipt[], + r => r.blockHash + ) + + for (let block of blocks) { + let rs = receiptsByBlock.get(block.hash) || [] + if (rs.length === block.block.transactions.length) { + block.receipts = rs + } else { + block._isInvalid = true + } + } + } -export type TraceDiff = '=' | TraceAddDiff | TraceChangeDiff | TraceDeleteDiff + private async addTraceTxReplays( + blocks: Block[], + traces: TraceReplayTraces, + method: string = 'trace_replayBlockTransactions' + ): Promise { + let tracers: string[] = [] + + if (traces.trace) { + tracers.push('trace') + } + + if (traces.stateDiff) { + tracers.push('stateDiff') + } + + if (tracers.length == 0) return + + let call = blocks.map(block => ({ + method, + params: [block.block.number, tracers] + })) + + let replaysByBlock = await this.batchCall(call, { + validateResult: getResultValidator( + array(getTraceTransactionReplayValidator(traces)) + ) + }) + + for (let i = 0; i < blocks.length; i++) { + let block = blocks[i] + let replays = replaysByBlock[i] + let txs = new Set(block.block.transactions.map(getTxHash)) + + for (let rep of replays) { + if (!rep.transactionHash) { // FIXME: Who behaves like that? Arbitrum? + let txHash: Bytes32 | null | undefined = undefined + for (let frame of rep.trace || []) { + assert(txHash == null || txHash === frame.transactionHash) + txHash = txHash || frame.transactionHash + } + assert(txHash, "Can't match transaction replay with its transaction") + rep.transactionHash = txHash + } + // Sometimes replays might be missing. FIXME: when? + if (!txs.has(rep.transactionHash)) { + block._isInvalid = true + } + } + + block.traceReplays = replays + } + } + private async addTraceBlockTraces(blocks: Block[]): Promise { + let call = blocks.map(block => ({ + method: 'trace_block', + params: [block.block.number] + })) + + let results = await this.batchCall(call, { + validateResult: getResultValidator(array(TraceFrame)) + }) + + for (let i = 0; i < blocks.length; i++) { + let block = blocks[i] + let frames = results[i] + if (frames.length == 0) { + if (block.block.transactions.length > 0) { + block._isInvalid = true + } + } else { + for (let frame of frames) { + if (frame.blockHash !== block.hash) { + block._isInvalid = true + break + } + } + if (!block._isInvalid) { + block.traceReplays = [] + let byTx = groupBy(frames, f => f.transactionHash) + for (let [transactionHash, txFrames] of byTx.entries()) { + if (transactionHash) { + block.traceReplays.push({ + transactionHash, + trace: txFrames + }) + } + } + } + } + } + } -export interface TraceStateDiff { - balance: TraceDiff - code: TraceDiff - nonce: TraceDiff - storage: Record -} + private async addDebugFrames(blocks: Block[]): Promise { + let traceConfig = { + tracer: 'callTracer', + tracerConfig: { + onlyTopCall: false, + withLog: false // will study log <-> frame matching problem later + } + } + + let call = blocks.map(block => ({ + method: 'debug_traceBlockByHash', + params: [block.hash, traceConfig] + })) + + let validateFrameResult = getResultValidator(array(DebugFrameResult)) + + let results = await this.batchCall(call, { + validateResult: result => { + if (Array.isArray(result)) { + // Moonbeam quirk + for (let i = 0; i < result.length; i++) { + if (!('result' in result[i])) { + result[i] = {result: result[i]} + } + } + } + return validateFrameResult(result) + } + }) + + for (let i = 0; i < blocks.length; i++) { + let block = blocks[i] + let frames = results[i] + assert(block.block.transactions.length === frames.length) + block.debugFrames = frames + } + } + private async addDebugStateDiffs(blocks: Block[]): Promise { + let traceConfig = { + tracer: 'prestateTracer', + tracerConfig: { + onlyTopCall: false, // passing this option is incorrect, but required by Alchemy endpoints + diffMode: true + } + } + + let call = blocks.map(block => ({ + method: 'debug_traceBlockByHash', + params: [block.hash, traceConfig] + })) + + let results = await this.batchCall(call, { + validateResult: getResultValidator(array(DebugStateDiffResult)) + }) + + for (let i = 0; i < blocks.length; i++) { + let block = blocks[i] + let diffs = results[i] + assert(block.block.transactions.length === diffs.length) + block.debugStateDiffs = diffs + } + } -export interface TraceTransactionReplay { - transactionHash: Bytes32 - trace?: TraceFrame[] - stateDiff?: Record -} + private async addArbitrumOneTraces(blocks: Block[], req: DataRequest): Promise { + if (req.stateDiffs) { + throw new Error('State diffs are not supported on Arbitrum One') + } + if (!req.traces) return + let arbBlocks = blocks.filter(b => b.height <= 22207815) + let debugBlocks = blocks.filter(b => b.height >= 22207818) -export type TraceTracers = 'trace' | 'stateDiff' + if (arbBlocks.length) { + await this.addTraceTxReplays(arbBlocks, {trace: true}, 'arbtrace_replayBlockTransactions') + } + if (debugBlocks.length) { + await this.addDebugFrames(debugBlocks) + } + } -export interface DebugFrame { - type: 'CALL' | 'CALLCODE' | 'STATICCALL' | 'DELEGATECALL' | 'CREATE' | 'CREATE2' | 'SELFDESTRUCT' | 'INVALID' | 'STOP' - from: Bytes20 - to: Bytes20 - value?: Qty - gas: Qty - gasUsed: Qty - input: Bytes - output: Bytes - error?: string - revertReason?: string - calls?: DebugFrame[] + private async addTraces( + blocks: Block[], + req: DataRequest, + finalizedHeight: number = Number.MAX_SAFE_INTEGER + ): Promise { + let isArbitrumOne = await this.props.getGenesisHash() === '0x7ee576b35482195fc49205cec9af72ce14f003b9ae69f6ba0faef4514be8b442' + if (isArbitrumOne) return this.addArbitrumOneTraces(blocks, req) + + let tasks: Promise[] = [] + let replayTraces: TraceReplayTraces = {} + + if (req.stateDiffs) { + if (finalizedHeight < last(blocks).height || req.useDebugApiForStateDiffs) { + tasks.push(this.addDebugStateDiffs(blocks)) + } else { + replayTraces.stateDiff = true + } + } + + if (req.traces) { + if (req.preferTraceApi) { + if (finalizedHeight < last(blocks).height || isEmpty(replayTraces)) { + tasks.push(this.addTraceBlockTraces(blocks)) + } else { + replayTraces.trace = true + } + } else { + tasks.push(this.addDebugFrames(blocks)) + } + } + + if (!isEmpty(replayTraces)) { + tasks.push(this.addTraceTxReplays(blocks, replayTraces)) + } + + await Promise.all(tasks) + } } -export interface DebugFrameResult { - result: DebugFrame -} +type GetReceiptsMethod = + 'eth_getTransactionReceipt' | + 'eth_getBlockReceipts' | + 'alchemy_getTransactionReceipts' -export interface DebugStateMap { - balance?: Qty - code?: Bytes - nonce?: number - storage?: Record -} +class RpcProps { + private genesisHash?: Bytes + private receiptsMethod?: GetReceiptsMethod + constructor( + private client: RpcClient, + private genesisHeight: number = 0 + ) {} -export interface DebugStateDiff { - pre: Record - post: Record + async getGenesisHash(): Promise { + if (this.genesisHash) return this.genesisHash + let rpc = new Rpc(this.client) + let hash = await rpc.getBlockHash(this.genesisHeight) + if (hash == null) throw new Error(`block ${this.genesisHeight} is not known to ${this.client.url}`) + return this.genesisHash = hash + } + + async getReceiptsMethod(): Promise { + if (this.receiptsMethod) return this.receiptsMethod + + let alchemy = await this.client.call('alchemy_getTransactionReceipts', [{blockNumber: '0x1'}]).then( + res => Array.isArray(res), + () => false + ) + if (alchemy) return this.receiptsMethod = 'alchemy_getTransactionReceipts' + + let eth = await this.client.call('eth_getBlockReceipts', ['latest']).then( + res => Array.isArray(res), + () => false + ) + if (eth) return this.receiptsMethod = 'eth_getBlockReceipts' + + return this.receiptsMethod = 'eth_getTransactionReceipt' + } } -export interface DebugStateDiffResult { - result: DebugStateDiff +function asTryAnotherRangeError(err: unknown): FiniteRange | undefined { + if (!(err instanceof RpcError)) return + let m = /Try with this block range \[(0x[0-9a-f]+), (0x[0-9a-f]+)]/i.exec(err.message) + if (m == null) return + let from = qty2Int(m[1]) + let to = qty2Int(m[2]) + if (from <= to) return {from, to} } -export interface DataRequest { - logs: boolean - transactions: boolean - receipts: boolean - traces: boolean - stateDiffs: boolean - transactionList?: boolean +function toBlock(getBlock: GetBlock): Block +function toBlock(getBlock?: null): undefined +function toBlock(getBlock?: GetBlock | null): Block | undefined +function toBlock(getBlock?: GetBlock | null): Block | undefined { + if (getBlock == null) return + return { + height: qty2Int(getBlock.number), + hash: getBlock.hash, + block: getBlock + } } diff --git a/evm/evm-processor/src/ds-rpc/schema.ts b/evm/evm-processor/src/ds-rpc/schema.ts new file mode 100644 index 000000000..196aeb518 --- /dev/null +++ b/evm/evm-processor/src/ds-rpc/schema.ts @@ -0,0 +1,201 @@ +import {weakMemo} from '@subsquid/util-internal' +import { + array, + BYTES, + NAT, + object, + option, + QTY, + record, + ref, + SMALL_QTY, + STRING, + taggedUnion, + Validator +} from '@subsquid/util-internal-validation' +import {Bytes, Bytes20} from '../interfaces/base' +import {FieldSelection} from '../interfaces/data' +import { + getBlockHeaderProps, + getLogProps, + getTraceFrameValidator, + getTxProps, + getTxReceiptProps, + project +} from '../mapping/schema' +import {MappingRequest} from './request' +import {DebugStateDiffResult, TraceStateDiff} from './rpc-data' + +// Here we must be careful to include all fields, +// that can potentially be used in item filters +// (no matter what field selection is telling us to omit) +export const getBlockValidator = weakMemo((req: MappingRequest) => { + let Transaction = req.transactions + ? object({ + ...getTxProps(req.fields.transaction, false), + hash: BYTES, + input: BYTES, + from: BYTES, + to: option(BYTES), + }) + : BYTES + + let GetBlock = object({ + ...getBlockHeaderProps(req.fields.block, false), + transactions: array(Transaction) + }) + + let Log = object({ + ...getLogProps( + {...req.fields.log, address: true, topics: true}, + false + ), + address: BYTES, + topics: array(BYTES) + }) + + let Receipt = object({ + transactionIndex: SMALL_QTY, + ...getTxReceiptProps(req.fields.transaction, false), + logs: req.logList ? array(Log) : undefined + }) + + let TraceFrame = getTraceFrameValidator(req.fields.trace, false) + + let DebugFrame = getDebugFrameValidator(req.fields.trace) + + return object({ + height: NAT, + hash: BYTES, + block: GetBlock, + ...project({ + logs: req.logs, + receipts: req.receipts + }, { + logs: array(Log), + receipts: array(Receipt) + }), + traceReplays: option(array(object({ + transactionHash: BYTES, + trace: option(array(TraceFrame)), + stateDiff: option(record(BYTES, TraceStateDiff)) + }))), + debugFrames: option(array(object({ + result: DebugFrame + }))), + debugStateDiffs: option(array(DebugStateDiffResult)) + }) +}) + + +function getDebugFrameValidator(fields: FieldSelection['trace']) { + let Frame: Validator + + let base = project(fields, { + error: option(STRING), + revertReason: option(STRING), + calls: option(array(ref(() => Frame))) + }) + + let Create = object({ + ...base, + from: BYTES, + ...project({ + value: fields?.createValue, + gas: fields?.createGas, + input: fields?.createInit, + gasUsed: fields?.createResultGasUsed, + output: fields?.createResultCode, + to: fields?.createResultAddress + }, { + value: QTY, + gas: QTY, + input: BYTES, + gasUsed: QTY, + output: BYTES, + to: BYTES + }) + }) + + let Call = object({ + ...base, + to: BYTES, + input: BYTES, + ...project({ + from: fields?.callFrom, + value: fields?.callValue, + gas: fields?.callGas, + }, { + from: BYTES, + value: option(QTY), + gas: QTY, + }) + }) + + let Suicide = object({ + ...base, + to: BYTES, + ...project({ + from: fields?.suicideAddress, + value: fields?.suicideBalance + }, { + from: BYTES, + value: QTY + }) + }) + + Frame = taggedUnion('type', { + CALL: Call, + CALLCODE: Call, + STATICCALL: Call, + DELEGATECALL: Call, + CREATE: Create, + CREATE2: Create, + SELFDESTRUCT: Suicide, + INVALID: object({}), + STOP: object({}) + }) + + return Frame +} + + +export type DebugFrame = DebugCreateFrame | DebugCallFrame | DebugSuicideFrame | DebugInvalidFrame + + +interface DebugCreateFrame extends DebugFrameBase { + type: 'CREATE' | 'CREATE2' + from: Bytes20 +} + + +interface DebugCallFrame extends DebugFrameBase { + type: 'CALL' | 'CALLCODE' | 'STATICCALL' | 'DELEGATECALL' + to: Bytes20 + input: Bytes +} + + +interface DebugSuicideFrame extends DebugFrameBase { + type: 'SELFDESTRUCT' + to: Bytes20 +} + + +interface DebugInvalidFrame extends DebugFrameBase { + type: 'INVALID' | 'STOP' +} + + +interface DebugFrameBase { + error?: string + revertReason?: string + from?: Bytes20 + to?: Bytes20 + value?: bigint + gas?: bigint + gasUsed?: bigint + input?: Bytes + output?: Bytes + calls?: DebugFrame[] +} diff --git a/evm/evm-processor/src/ds-rpc/util.ts b/evm/evm-processor/src/ds-rpc/util.ts new file mode 100644 index 000000000..e041ea38a --- /dev/null +++ b/evm/evm-processor/src/ds-rpc/util.ts @@ -0,0 +1,23 @@ +import assert from 'assert' +import {Bytes32, Qty} from '../interfaces/base' + + +export function qty2Int(qty: Qty): number { + let i = parseInt(qty, 16) + assert(Number.isSafeInteger(i)) + return i +} + + +export function toQty(i: number): Qty { + return '0x' + i.toString(16) +} + + +export function getTxHash(tx: Bytes32 | {hash: Bytes32}): Bytes32 { + if (typeof tx == 'string') { + return tx + } else { + return tx.hash + } +} diff --git a/evm/evm-processor/src/interfaces/base.ts b/evm/evm-processor/src/interfaces/base.ts new file mode 100644 index 000000000..a50143fb8 --- /dev/null +++ b/evm/evm-processor/src/interfaces/base.ts @@ -0,0 +1,5 @@ +export type Bytes = string +export type Bytes8 = string +export type Bytes20 = string +export type Bytes32 = string +export type Qty = string diff --git a/evm/evm-processor/src/interfaces/data-request.ts b/evm/evm-processor/src/interfaces/data-request.ts index 534f56e54..206a0b9e3 100644 --- a/evm/evm-processor/src/interfaces/data-request.ts +++ b/evm/evm-processor/src/interfaces/data-request.ts @@ -1,5 +1,6 @@ +import {Bytes, Bytes20, Bytes32} from './base' import {FieldSelection} from './data' -import {Bytes, Bytes20, Bytes32, EvmStateDiff} from './evm' +import {EvmStateDiff} from './evm' export interface DataRequest { diff --git a/evm/evm-processor/src/interfaces/data.ts b/evm/evm-processor/src/interfaces/data.ts index 29c54b38b..4b5a6a6a1 100644 --- a/evm/evm-processor/src/interfaces/data.ts +++ b/evm/evm-processor/src/interfaces/data.ts @@ -1,7 +1,8 @@ import { - EvmBlock, + EvmBlockHeader, EvmLog, - EvmStateDiff, EvmStateDiffBase, + EvmStateDiff, + EvmStateDiffBase, EvmTraceBase, EvmTraceCallAction, EvmTraceCallResult, @@ -18,7 +19,7 @@ type Simplify = { } & {} -type Selector = { +type Selector = { [P in Exclude]?: boolean } @@ -26,15 +27,15 @@ type Selector = { type AddPrefix = `${Prefix}${Capitalize}` -type BlockRequiredFields = 'height' | 'hash' | 'parentHash' -type TransactionRequiredFields = 'transactionIndex' -type LogRequiredFields = 'logIndex' | 'transactionIndex' -type TraceRequiredFields = 'transactionIndex' | 'traceAddress' | 'type' -type StateDiffRequiredFields = 'transactionIndex' | 'address' | 'key' +export type BlockRequiredFields = 'height' | 'hash' | 'parentHash' +export type TransactionRequiredFields = 'transactionIndex' +export type LogRequiredFields = 'logIndex' | 'transactionIndex' +export type TraceRequiredFields = 'transactionIndex' | 'traceAddress' | 'type' +export type StateDiffRequiredFields = 'transactionIndex' | 'address' | 'key' export interface FieldSelection { - block?: Selector + block?: Selector transaction?: Selector log?: Selector trace?: Selector< @@ -103,8 +104,8 @@ type Select = T extends any ? Simplify>> : nev export type BlockHeader = Simplify< {id: string} & - Pick & - Select> + Pick & + Select> > @@ -112,7 +113,12 @@ export type Transaction = Simplify< {id: string} & Pick & Select> & - {block: BlockHeader} + { + block: BlockHeader + logs: Log[] + traces: Trace[] + stateDiffs: StateDiff[] + } > @@ -120,7 +126,11 @@ export type Log = Simplify< {id: string} & Pick & Select> & - {block: BlockHeader, transaction?: Transaction} + { + block: BlockHeader, + transaction?: Transaction + getTransaction(): Transaction + } > @@ -169,7 +179,14 @@ export type TraceRewardAction = Select< type TraceBase = Pick> & Select> & - {block: BlockHeader, transaction?: Transaction} + { + block: BlockHeader, + transaction?: Transaction + getTransaction(): Transaction + parent?: Trace + getParent(): Trace + children: Trace[] + } type RemoveEmptyObjects = { @@ -215,7 +232,11 @@ export type Trace = export type StateDiff = Simplify< EvmStateDiffBase & Select> & - {block: BlockHeader, transaction?: Transaction} + { + block: BlockHeader + transaction?: Transaction + getTransaction(): Transaction + } > diff --git a/evm/evm-processor/src/interfaces/evm.ts b/evm/evm-processor/src/interfaces/evm.ts index b421eb219..22b0292f4 100644 --- a/evm/evm-processor/src/interfaces/evm.ts +++ b/evm/evm-processor/src/interfaces/evm.ts @@ -1,57 +1,61 @@ -export type Bytes = string -export type Bytes8 = string -export type Bytes20 = string -export type Bytes32 = string -export type Qty = string +import {Bytes, Bytes20, Bytes32, Bytes8} from './base' -export interface EvmBlock { +export interface EvmBlockHeader { height: number hash: Bytes32 parentHash: Bytes32 - nonce?: Bytes8 + nonce: Bytes8 sha3Uncles: Bytes32 logsBloom: Bytes transactionsRoot: Bytes32 stateRoot: Bytes32 receiptsRoot: Bytes32 - mixHash?: Bytes + mixHash: Bytes miner: Bytes20 - difficulty?: bigint - totalDifficulty?: bigint + difficulty: bigint + totalDifficulty: bigint extraData: Bytes size: bigint gasLimit: bigint gasUsed: bigint timestamp: number - baseFeePerGas?: bigint + baseFeePerGas: bigint } -export interface EvmTransaction { +export interface EvmTransaction extends _EvmTx, _EvmTxReceipt { + transactionIndex: number + sighash: Bytes +} + + +export interface _EvmTx { + hash: Bytes32 from: Bytes20 + to?: Bytes20 gas: bigint gasPrice: bigint maxFeePerGas?: bigint maxPriorityFeePerGas?: bigint - hash: Bytes32 input: Bytes nonce: number - to?: Bytes20 - transactionIndex: number value: bigint - v?: bigint - r?: Bytes32 - s?: Bytes32 + v: bigint + r: Bytes32 + s: Bytes32 yParity?: number chainId?: number - gasUsed?: bigint - cumulativeGasUsed?: bigint - effectiveGasPrice?: bigint +} + + +export interface _EvmTxReceipt { + gasUsed: bigint + cumulativeGasUsed: bigint + effectiveGasPrice: bigint contractAddress?: Bytes32 - type?: number - status?: number - sighash: Bytes + type: number + status: number } diff --git a/evm/evm-processor/src/mapping/entities.ts b/evm/evm-processor/src/mapping/entities.ts new file mode 100644 index 000000000..e4523ba52 --- /dev/null +++ b/evm/evm-processor/src/mapping/entities.ts @@ -0,0 +1,326 @@ +import {formatId} from '@subsquid/util-internal-processor-tools' +import assert from 'assert' +import {Bytes, Bytes20, Bytes32, Bytes8} from '../interfaces/base' +import { + EvmTraceCallAction, + EvmTraceCallResult, + EvmTraceCreateAction, + EvmTraceCreateResult, + EvmTraceRewardAction, + EvmTraceSuicideAction +} from '../interfaces/evm' + + +export class Block { + header: BlockHeader + transactions: Transaction[] = [] + logs: Log[] = [] + traces: Trace[] = [] + stateDiffs: StateDiff[] = [] + + constructor(header: BlockHeader) { + this.header = header + } +} + + +export class BlockHeader { + id: string + height: number + hash: Bytes32 + parentHash: Bytes32 + nonce?: Bytes8 + sha3Uncles?: Bytes32 + logsBloom?: Bytes + transactionsRoot?: Bytes32 + stateRoot?: Bytes32 + receiptsRoot?: Bytes32 + mixHash?: Bytes + miner?: Bytes20 + difficulty?: bigint + totalDifficulty?: bigint + extraData?: Bytes + size?: bigint + gasLimit?: bigint + gasUsed?: bigint + timestamp?: number + baseFeePerGas?: bigint + + constructor( + height: number, + hash: Bytes20, + parentHash: Bytes20 + ) { + this.id = formatId({height, hash}) + this.height = height + this.hash = hash + this.parentHash = parentHash + } +} + + +export class Transaction { + id: string + transactionIndex: number + from?: Bytes20 + gas?: bigint + gasPrice?: bigint + maxFeePerGas?: bigint + maxPriorityFeePerGas?: bigint + hash?: Bytes32 + input?: Bytes + nonce?: number + to?: Bytes20 + value?: bigint + v?: bigint + r?: Bytes32 + s?: Bytes32 + yParity?: number + chainId?: number + gasUsed?: bigint + cumulativeGasUsed?: bigint + effectiveGasPrice?: bigint + contractAddress?: Bytes32 + type?: number + status?: number + sighash?: Bytes + #block: BlockHeader + #logs?: Log[] + #traces?: Trace[] + #stateDiffs?: StateDiff[] + + constructor( + block: BlockHeader, + transactionIndex: number + ) { + this.id = formatId(block, transactionIndex) + this.transactionIndex = transactionIndex + this.#block = block + } + + get block(): BlockHeader { + return this.#block + } + + get logs(): Log[] { + return this.#logs || (this.#logs = []) + } + + set logs(value: Log[]) { + this.#logs = value + } + + get traces(): Trace[] { + return this.#traces || (this.#traces = []) + } + + set traces(value: Trace[]) { + this.#traces = value + } + + get stateDiffs(): StateDiff[] { + return this.#stateDiffs || (this.#stateDiffs = []) + } + + set stateDiffs(value: StateDiff[]) { + this.#stateDiffs = value + } +} + + +export class Log { + id: string + logIndex: number + transactionIndex: number + transactionHash?: Bytes32 + address?: Bytes20 + data?: Bytes + topics?: Bytes32[] + #block: BlockHeader + #transaction?: Transaction + + constructor( + block: BlockHeader, + logIndex: number, + transactionIndex: number + ) { + this.id = formatId(block, logIndex) + this.logIndex = logIndex + this.transactionIndex = transactionIndex + this.#block = block + } + + get block(): BlockHeader { + return this.#block + } + + get transaction(): Transaction | undefined { + return this.#transaction + } + + set transaction(value: Transaction | undefined) { + this.#transaction = value + } + + getTransaction(): Transaction { + assert(this.transaction != null) + return this.transaction + } +} + + +class TraceBase { + id: string + transactionIndex: number + traceAddress: number[] + subtraces?: number + error?: string | null + revertReason?: string + #block: BlockHeader + #transaction?: Transaction + #parent?: Trace + #children?: Trace[] + + constructor(block: BlockHeader, transactionIndex: number, traceAddress: number[]) { + this.id = formatId(block, transactionIndex, ...traceAddress) + this.transactionIndex = transactionIndex + this.traceAddress = traceAddress + this.#block = block + } + + get block(): BlockHeader { + return this.#block + } + + get transaction(): Transaction | undefined { + return this.#transaction + } + + set transaction(value: Transaction | undefined) { + this.#transaction = value + } + + getTransaction(): Transaction { + assert(this.transaction != null) + return this.transaction + } + + get parent(): Trace | undefined { + return this.#parent + } + + set parent(value: Trace | undefined) { + this.#parent = value + } + + getParent(): Trace { + assert(this.parent != null) + return this.parent + } + + get children(): Trace[] { + return this.#children || (this.#children = []) + } + + set children(value: Trace[]) { + this.#children = value + } +} + + +export class TraceCreate extends TraceBase { + type: 'create' = 'create' + action?: Partial + result?: Partial +} + + +export class TraceCall extends TraceBase { + type: 'call' = 'call' + action?: Partial + result?: Partial +} + + +export class TraceSuicide extends TraceBase { + type: 'suicide' = 'suicide' + action?: Partial +} + + +export class TraceReward extends TraceBase { + type: 'reward' = 'reward' + action?: Partial +} + + +export type Trace = TraceCreate | TraceCall | TraceSuicide | TraceReward + + +class StateDiffBase { + transactionIndex: number + address: Bytes20 + key: 'balance' | 'code' | 'nonce' | Bytes32 + #block: BlockHeader + #transaction?: Transaction + + constructor( + block: BlockHeader, + transactionIndex: number, + address: Bytes20, + key: 'balance' | 'code' | 'nonce' | Bytes32 + ) { + this.transactionIndex = transactionIndex + this.address = address + this.key = key + this.#block = block + } + + get block(): BlockHeader { + return this.#block + } + + get transaction(): Transaction | undefined { + return this.#transaction + } + + set transaction(value: Transaction | undefined) { + this.#transaction = value + } + + getTransaction(): Transaction { + assert(this.transaction != null) + return this.transaction + } +} + + +export class StateDiffNoChange extends StateDiffBase { + kind: '=' = '=' + prev?: null + next?: null +} + + +export class StateDiffAdd extends StateDiffBase { + kind: '+' = '+' + prev?: null + next?: Bytes +} + + +export class StateDiffChange extends StateDiffBase { + kind: '*' = '*' + prev?: Bytes + next?: Bytes +} + + +export class StateDiffDelete extends StateDiffBase { + kind: '-' = '-' + prev?: Bytes + next?: null +} + + +export type StateDiff = StateDiffNoChange | StateDiffAdd | StateDiffChange | StateDiffDelete diff --git a/evm/evm-processor/src/mapping/relations.ts b/evm/evm-processor/src/mapping/relations.ts new file mode 100644 index 000000000..ae1ae8de4 --- /dev/null +++ b/evm/evm-processor/src/mapping/relations.ts @@ -0,0 +1,79 @@ +import {maybeLast} from '@subsquid/util-internal' +import {Block, Trace, Transaction} from './entities' + + +export function setUpRelations(block: Block): void { + block.transactions.sort((a, b) => a.transactionIndex - b.transactionIndex) + block.logs.sort((a, b) => a.logIndex - b.logIndex) + block.traces.sort(traceCompare) + + let txs: (Transaction | undefined)[] = new Array((maybeLast(block.transactions)?.transactionIndex ?? -1) + 1) + for (let tx of block.transactions) { + txs[tx.transactionIndex] = tx + } + + for (let rec of block.logs) { + let tx = txs[rec.transactionIndex] + if (tx) { + rec.transaction = tx + tx.logs.push(rec) + } + } + + for (let i = 0; i < block.traces.length; i++) { + let rec = block.traces[i] + let tx = txs[rec.transactionIndex] + if (tx) { + rec.transaction = tx + tx.traces.push(rec) + } + + if (i > 0) { + let prev = block.traces[i - 1] + if (isChild(prev, rec)) { + rec.parent = prev + populateSubtraces(prev, rec) + } + } + } + + for (let rec of block.stateDiffs) { + let tx = txs[rec.transactionIndex] + if (tx) { + rec.transaction = tx + tx.stateDiffs.push(rec) + } + } +} + + +function traceCompare(a: Trace, b: Trace) { + return a.transactionIndex - b.transactionIndex || addressCompare(a.traceAddress, b.traceAddress) +} + + +function addressCompare(a: number[], b: number[]): number { + for (let i = 0; i < Math.min(a.length, b.length); i++) { + let order = a[i] - b[i] + if (order) return order + } + return a.length - b.length // this differs from substrate call ordering +} + + +function isChild(parent: Trace, child: Trace): boolean { + if (parent.transactionIndex != child.transactionIndex) return false + if (parent.traceAddress.length > child.traceAddress.length) return false + for (let i = 0; i < parent.traceAddress.length; i++) { + if (parent.traceAddress[i] != child.traceAddress[i]) return false + } + return true +} + + +function populateSubtraces(parent: Trace | undefined, child: Trace): void { + while (parent) { + parent.children.push(child) + parent = parent.parent + } +} diff --git a/evm/evm-processor/src/mapping/schema.ts b/evm/evm-processor/src/mapping/schema.ts new file mode 100644 index 000000000..dee613d91 --- /dev/null +++ b/evm/evm-processor/src/mapping/schema.ts @@ -0,0 +1,220 @@ +import {FieldSelection} from '../interfaces/data' +import {array, BYTES, NAT, object, option, QTY, SMALL_QTY, STRING, taggedUnion, withSentinel} from '@subsquid/util-internal-validation' + + +export function getBlockHeaderProps(fields: FieldSelection['block'], forArchive: boolean) { + let natural = forArchive ? NAT : SMALL_QTY + return { + number: natural, + hash: BYTES, + parentHash: BYTES, + ...project(fields, { + nonce: withSentinel('BlockHeader.nonce', '0x', BYTES), + sha3Uncles: withSentinel('BlockHeader.sha3Uncles', '0x', BYTES), + logsBloom: withSentinel('BlockHeader.logsBloom', '0x', BYTES), + transactionsRoot: withSentinel('BlockHeader.transactionsRoot', '0x', BYTES), + stateRoot: withSentinel('BlockHeader.stateRoot', '0x', BYTES), + receiptsRoot: withSentinel('BlockHeader.receiptsRoot', '0x', BYTES), + mixHash: withSentinel('BlockHeader.mixHash', '0x', BYTES), + miner: withSentinel('BlockHeader.miner', '0x', BYTES), + difficulty: withSentinel('BlockHeader.difficulty', -1n, QTY), + totalDifficulty: withSentinel('BlockHeader.totalDifficulty', -1n, QTY), + extraData: withSentinel('BlockHeader.extraData', '0x', BYTES), + size: withSentinel('BlockHeader.size', -1, SMALL_QTY), + gasLimit: withSentinel('BlockHeader.gasLimit', -1n, QTY), + gasUsed: withSentinel('BlockHeader.gasUsed', -1n, QTY), + baseFeePerGas: withSentinel('BlockHeader.baseFeePerGas', -1n, QTY), + timestamp: withSentinel('BlockHeader.timestamp', 0, natural) + }) + } +} + + +export function getTxProps(fields: FieldSelection['transaction'], forArchive: boolean) { + let natural = forArchive ? NAT : SMALL_QTY + return { + transactionIndex: natural, + ...project(fields, { + hash: BYTES, + from: BYTES, + to: option(BYTES), + gas: withSentinel('Transaction.gas', -1n, QTY), + gasPrice: withSentinel('Transaction.gasPrice', -1n, QTY), + maxFeePerGas: option(QTY), + maxPriorityFeePerGas: option(QTY), + input: BYTES, + nonce: withSentinel('Transaction.nonce', -1, NAT), + value: withSentinel('Transaction.value', -1n, QTY), + v: withSentinel('Transaction.v', -1n, QTY), + r: withSentinel('Transaction.r', '0x', BYTES), + s: withSentinel('Transaction.s', '0x', BYTES), + yParity: option(natural), + chainId: option(natural), + }) + } +} + + +export function getTxReceiptProps(fields: FieldSelection['transaction'], forArchive: boolean) { + let natural = forArchive ? NAT : SMALL_QTY + return project(fields, { + gasUsed: withSentinel('Receipt.gasUsed', -1n, QTY), + cumulativeGasUsed: withSentinel('Receipt.cumulativeGasUsed', -1n, QTY), + effectiveGasPrice: withSentinel('Receipt.effectiveGasPrice', -1n, QTY), + contractAddress: option(BYTES), + type: withSentinel('Receipt.type', -1, natural), + status: withSentinel('Receipt.status', -1, natural), + }) +} + + +export function getLogProps(fields: FieldSelection['log'], forArchive: boolean) { + let natural = forArchive ? NAT : SMALL_QTY + return { + logIndex: natural, + transactionIndex: natural, + ...project(fields, { + transactionHash: BYTES, + address: BYTES, + data: BYTES, + topics: array(BYTES) + }) + } +} + + +export function getTraceFrameValidator(fields: FieldSelection['trace'], forArchive: boolean) { + let traceBase = { + transactionIndex: forArchive ? NAT : undefined, + traceAddress: array(NAT), + ...project(fields, { + subtraces: NAT, + error: option(STRING), + revertReason: option(STRING) + }) + } + + let traceCreateAction = project({ + from: fields?.createFrom || !forArchive, + value: fields?.createValue, + gas: fields?.createGas, + init: fields?.createInit + }, { + from: BYTES, + value: QTY, + gas: QTY, + init: BYTES + }) + + let traceCreateResult = project({ + gasUsed: fields?.createResultGasUsed, + code: fields?.createResultCode, + address: fields?.createResultAddress + }, { + gasUsed: QTY, + code: BYTES, + address: BYTES + }) + + let TraceCreate = object({ + ...traceBase, + action: isEmpty(traceCreateAction) ? undefined : object(traceCreateAction), + result: isEmpty(traceCreateResult) ? undefined : option(object(traceCreateResult)) + }) + + let traceCallAction = project({ + callType: fields?.callCallType, + from: fields?.callFrom, + to: forArchive ? fields?.callTo : true, + value: fields?.callValue, + gas: fields?.callGas, + input: forArchive ? fields?.callInput : true, + sighash: forArchive ? fields?.callSighash : false + }, { + callType: STRING, + from: BYTES, + to: BYTES, + value: option(QTY), + gas: QTY, + input: BYTES, + sighash: BYTES + }) + + let traceCallResult = project({ + gasUsed: fields?.callResultGasUsed, + output: fields?.callResultOutput + }, { + gasUsed: QTY, + output: BYTES + }) + + let TraceCall = object({ + ...traceBase, + action: isEmpty(traceCallAction) ? undefined : object(traceCallAction), + result: isEmpty(traceCallResult) ? undefined : option(object(traceCallResult)) + }) + + let traceSuicideAction = project({ + address: fields?.suicideAddress, + refundAddress: forArchive ? fields?.suicideRefundAddress : true, + balance: fields?.suicideBalance + }, { + address: BYTES, + refundAddress: BYTES, + balance: QTY + }) + + let TraceSuicide = object({ + ...traceBase, + action: isEmpty(traceSuicideAction) ? undefined : object(traceSuicideAction) + }) + + let traceRewardAction = project({ + author: forArchive ? fields?.rewardAuthor : true, + value: fields?.rewardValue, + type: fields?.rewardType + }, { + author: BYTES, + value: QTY, + type: STRING + }) + + let TraceReward = object({ + ...traceBase, + action: isEmpty(traceRewardAction) ? undefined : object(traceRewardAction) + }) + + return taggedUnion('type', { + create: TraceCreate, + call: TraceCall, + suicide: TraceSuicide, + reward: TraceReward + }) +} + + +export function project( + fields: F | undefined, + obj: T +): Partial { + if (fields == null) return {} + let result: Partial = {} + let key: keyof T + for (key in obj) { + if (fields[key]) { + result[key] = obj[key] + } + } + return result +} + + +export function isEmpty(obj: object): boolean { + for (let _ in obj) { + return false + } + return true +} + + +export function assertAssignable(): void {} diff --git a/evm/evm-processor/src/processor.ts b/evm/evm-processor/src/processor.ts index 859478eb7..e244b4af3 100644 --- a/evm/evm-processor/src/processor.ts +++ b/evm/evm-processor/src/processor.ts @@ -3,51 +3,102 @@ import {createLogger, Logger} from '@subsquid/logger' import {RpcClient} from '@subsquid/rpc-client' import {assertNotNull, def, runProgram} from '@subsquid/util-internal' import {ArchiveClient} from '@subsquid/util-internal-archive-client' -import { - applyRangeBound, - Database, - getOrGenerateSquidId, - mergeRangeRequests, - PrometheusServer, - Range, - RangeRequest, - Runner -} from '@subsquid/util-internal-processor-tools' +import {Database, getOrGenerateSquidId, PrometheusServer, Runner} from '@subsquid/util-internal-processor-tools' +import {applyRangeBound, mergeRangeRequests, Range, RangeRequest} from '@subsquid/util-internal-range' import assert from 'assert' import {EvmArchive} from './ds-archive/client' import {EvmRpcDataSource} from './ds-rpc/client' import {Chain} from './interfaces/chain' -import {BlockData, FieldSelection} from './interfaces/data' +import {BlockData, DEFAULT_FIELDS, FieldSelection} from './interfaces/data' import {DataRequest, LogRequest, StateDiffRequest, TraceRequest, TransactionRequest} from './interfaces/data-request' -export type DataSource = ArchiveDataSource | ChainDataSource - - -type ChainRpc = string | { +export interface RpcEndpointSettings { + /** + * RPC endpoint URL (either http(s) or ws(s)) + */ url: string + /** + * Maximum number of ongoing concurrent requests + */ capacity?: number + /** + * Maximum number of requests per second + */ rateLimit?: number + /** + * Request timeout in `ms` + */ requestTimeout?: number + /** + * Maximum number of requests in a single batch call + */ maxBatchCallSize?: number } -type ArchiveConnection = string | { +export interface RpcDataIngestionSettings { + /** + * By default, `debug_traceBlockByHash` is used to obtain call traces, + * this flag instructs the processor to utilize `trace_` methods instead. + * + * This setting is only effective for finalized blocks. + */ + preferTraceApi?: boolean + /** + * By default, `trace_replayBlockTransactions` is used to obtain state diffs for finalized blocks, + * this flag instructs the processor to utilize `debug_traceBlockByHash` instead. + * + * This setting is only effective for finalized blocks. + */ + useDebugApiForStateDiffs?: boolean + /** + * Poll interval for new blocks in `ms` + * + * Poll mechanism is used to get new blocks via HTTP connection. + */ + headPollInterval?: number + /** + * When websocket subscription is used to get new blocks, + * this setting specifies timeout in `ms` after which connection + * will be reset and subscription re-initiated if no new block where received. + */ + newHeadTimeout?: number + /** + * Disable RPC data ingestion entirely + */ + disabled?: boolean +} + + +export interface ArchiveSettings { + /** + * Subsquid archive URL + */ url: string + /** + * Request timeout in ms + */ requestTimeout?: number } +/** + * @deprecated + */ +export type DataSource = ArchiveDataSource | ChainDataSource + + + interface ArchiveDataSource { /** * Subsquid evm archive endpoint URL */ - archive: ArchiveConnection + archive: string | ArchiveSettings /** * Chain node RPC endpoint URL */ - chain?: ChainRpc + chain?: string | RpcEndpointSettings } @@ -56,7 +107,7 @@ interface ChainDataSource { /** * Chain node RPC endpoint URL */ - chain: ChainRpc + chain: string | RpcEndpointSettings } @@ -68,11 +119,31 @@ interface BlockRange { } +/** + * API and data that is passed to the data handler + */ export interface DataHandlerContext { + /** + * @internal + */ _chain: Chain + /** + * An instance of a structured logger. + */ log: Logger + /** + * Storage interface provided by the database + */ store: Store + /** + * List of blocks to map and process + */ blocks: BlockData[] + /** + * Signals, that the processor reached the head of a chain. + * + * The head block is always included in `.blocks`. + */ isHead: boolean } @@ -85,146 +156,224 @@ export type EvmBatchProcessorFields = T extends EvmBatchProcessor ? */ export class EvmBatchProcessor { private requests: RangeRequest[] = [] - private src?: DataSource private blockRange?: Range private fields?: FieldSelection private finalityConfirmation?: number - private _preferTraceApi?: boolean - private _useDebugApiForStateDiffs?: boolean - private _useArchiveOnly?: boolean - private chainPollInterval?: number + private archive?: ArchiveSettings + private rpcIngestSettings?: RpcDataIngestionSettings + private rpcEndpoint?: RpcEndpointSettings private running = false - private add(request: DataRequest, range?: Range): void { - this.requests.push({ - range: range || {from: 0}, - request - }) + /** + * Set Subsquid Archive endpoint. + * + * Subsquid Archive allows to get data from finalized blocks up to + * infinite times faster and more efficient than via regular RPC. + * + * @example + * processor.setArchive('https://v2.archive.subsquid.io/network/ethereum-mainnet') + */ + setArchive(url: string | ArchiveSettings): this { + this.assertNotRunning() + if (typeof url == 'string') { + this.archive = {url} + } else { + this.archive = url + } + return this } /** - * Configure a set of fetched fields + * Set chain RPC endpoint + * + * @example + * // just pass a URL + * processor.setRpcEndpoint('https://eth-mainnet.public.blastapi.io') + * + * // adjust some connection options + * processor.setRpcEndpoint({ + * url: 'https://eth-mainnet.public.blastapi.io', + * rateLimit: 10 + * }) */ - setFields(fields: T): EvmBatchProcessor { + setRpcEndpoint(url: string | RpcEndpointSettings | undefined): this { this.assertNotRunning() - this.fields = fields - return this as any + if (typeof url == 'string') { + this.rpcEndpoint = {url} + } else { + this.rpcEndpoint = url + } + return this } - addLog(options: LogRequest & BlockRange): this { + /** + * Sets blockchain data source. + * + * @example + * processor.setDataSource({ + * archive: 'https://v2.archive.subsquid.io/network/ethereum-mainnet', + * chain: 'https://eth-mainnet.public.blastapi.io' + * }) + * + * @deprecated Use separate {@link .setArchive()} and {@link .setRpcEndpoint()} methods + * to specify data sources. + */ + setDataSource(src: DataSource): this { this.assertNotRunning() - this.add({ - logs: [mapRequest(options)] - }, options.range) + if (src.archive) { + this.setArchive(src.archive) + } else { + this.archive = undefined + } + if (src.chain) { + this.setRpcEndpoint(src.chain) + } else { + this.rpcEndpoint = undefined + } return this } - addTransaction(options: TransactionRequest & BlockRange): this { + /** + * Set up RPC data ingestion settings + */ + setRpcDataIngestionSettings(settings: RpcDataIngestionSettings): this { this.assertNotRunning() - this.add({ - transactions: [mapRequest(options)] - }, options.range) + this.rpcIngestSettings = settings return this } - addTrace(options: TraceRequest & BlockRange): this { + /** + * @deprecated Use {@link .setRpcDataIngestionSettings()} instead + */ + setChainPollInterval(ms: number): this { + assert(ms >= 0) this.assertNotRunning() - this.add({ - traces: [mapRequest(options)] - }, options.range) + this.rpcIngestSettings = {...this.rpcIngestSettings, headPollInterval: ms} return this } - addStateDiff(options: StateDiffRequest & BlockRange): this { + /** + * @deprecated Use {@link .setRpcDataIngestionSettings()} instead + */ + preferTraceApi(yes?: boolean): this { this.assertNotRunning() - this.add({ - stateDiffs: [mapRequest(options)] - }, options.range) + this.rpcIngestSettings = {...this.rpcIngestSettings, preferTraceApi: yes !== false} return this } /** - * Sets the port for a built-in prometheus metrics server. - * - * By default, the value of `PROMETHEUS_PORT` environment - * variable is used. When it is not set, - * the processor will pick up an ephemeral port. + * @deprecated Use {@link .setRpcDataIngestionSettings()} instead */ - setPrometheusPort(port: number | string): this { + useDebugApiForStateDiffs(yes?: boolean): this { this.assertNotRunning() - this.getPrometheusServer().setPort(port) + this.rpcIngestSettings = {...this.rpcIngestSettings, useDebugApiForStateDiffs: yes !== false} return this } /** - * By default, the processor will fetch only blocks - * which contain requested items. This method - * modifies such behaviour to fetch all chain blocks. + * Never use RPC endpoint for data ingestion. * - * Optionally a range of blocks can be specified - * for which the setting should be effective. + * @deprecated This is the same as `.setRpcDataIngestionSettings({disabled: true})` */ - includeAllBlocks(range?: Range): this { + useArchiveOnly(yes?: boolean): this { this.assertNotRunning() - this.add({includeAllBlocks: true}, range) + this.rpcIngestSettings = {...this.rpcIngestSettings, disabled: yes !== false} return this } /** - * Limits the range of blocks to be processed. - * - * When the upper bound is specified, - * the processor will terminate with exit code 0 once it reaches it. + * Distance from the head block behind which all blocks are considered to be finalized. */ - setBlockRange(range?: Range): this { + setFinalityConfirmation(nBlocks: number): this { this.assertNotRunning() - this.blockRange = range + this.finalityConfirmation = nBlocks return this } /** - * Sets blockchain data source. + * Configure a set of fetched fields + */ + setFields(fields: T): EvmBatchProcessor { + this.assertNotRunning() + this.fields = fields + return this as any + } + + private add(request: DataRequest, range?: Range): void { + this.requests.push({ + range: range || {from: 0}, + request + }) + } + + /** + * By default, the processor will fetch only blocks + * which contain requested items. This method + * modifies such behaviour to fetch all chain blocks. * - * @example - * processor.setDataSource({ - * archive: 'https://v2.archive.subsquid.io/network/ethereum-mainnet', - * chain: 'https://eth-mainnet.public.blastapi.io' - * }) + * Optionally a range of blocks can be specified + * for which the setting should be effective. */ - setDataSource(src: DataSource): this { + includeAllBlocks(range?: Range): this { this.assertNotRunning() - this.src = src + this.add({includeAllBlocks: true}, range) return this } - setFinalityConfirmation(nBlocks: number): this { + addLog(options: LogRequest & BlockRange): this { this.assertNotRunning() - this.finalityConfirmation = nBlocks + this.add({ + logs: [mapRequest(options)] + }, options.range) return this } - setChainPollInterval(ms: number): this { - assert(ms >= 0) + addTransaction(options: TransactionRequest & BlockRange): this { this.assertNotRunning() - this.chainPollInterval = ms + this.add({ + transactions: [mapRequest(options)] + }, options.range) return this } - preferTraceApi(yes?: boolean): this { + addTrace(options: TraceRequest & BlockRange): this { this.assertNotRunning() - this._preferTraceApi = yes !== false + this.add({ + traces: [mapRequest(options)] + }, options.range) return this } - useDebugApiForStateDiffs(yes?: boolean): this { + addStateDiff(options: StateDiffRequest & BlockRange): this { this.assertNotRunning() - this._useDebugApiForStateDiffs = yes !== false + this.add({ + stateDiffs: [mapRequest(options)] + }, options.range) return this } - useArchiveOnly(yes?: boolean): this { + /** + * Limits the range of blocks to be processed. + * + * When the upper bound is specified, + * the processor will terminate with exit code 0 once it reaches it. + */ + setBlockRange(range?: Range): this { + this.assertNotRunning() + this.blockRange = range + return this + } + + /** + * Sets the port for a built-in prometheus metrics server. + * + * By default, the value of `PROMETHEUS_PORT` environment + * variable is used. When it is not set, + * the processor will pick up an ephemeral port. + */ + setPrometheusPort(port: number | string): this { this.assertNotRunning() - this._useArchiveOnly = yes !== false + this.getPrometheusServer().setPort(port) return this } @@ -249,30 +398,19 @@ export class EvmBatchProcessor { return new PrometheusServer() } - private getDataSource(): DataSource { - if (this.src == null) { - throw new Error('use .setDataSource() to specify archive and/or chain RPC endpoint') - } - return this.src - } - @def private getChainRpcClient(): RpcClient { - let options = this.src?.chain - if (options == null) { - throw new Error(`use .setDataSource() to specify chain RPC endpoint`) - } - if (typeof options == 'string') { - options = {url: options} + if (this.rpcEndpoint == null) { + throw new Error(`use .setRpcEndpoint() to specify chain RPC endpoint`) } let client = new RpcClient({ - url: options.url, - maxBatchCallSize: options.maxBatchCallSize ?? 100, - requestTimeout: options.requestTimeout ?? 30_000, - capacity: options.capacity ?? 10, - rateLimit: options.rateLimit, + url: this.rpcEndpoint.url, + maxBatchCallSize: this.rpcEndpoint.maxBatchCallSize ?? 100, + requestTimeout: this.rpcEndpoint.requestTimeout ?? 30_000, + capacity: this.rpcEndpoint.capacity ?? 10, + rateLimit: this.rpcEndpoint.rateLimit, retryAttempts: Number.MAX_SAFE_INTEGER, - log: this.getLogger().child('rpc', {rpcUrl: options.url}) + log: this.getLogger().child('rpc', {rpcUrl: this.rpcEndpoint.url}) }) this.getPrometheusServer().addChainRpcMetrics(() => client.getMetrics()) return client @@ -296,19 +434,17 @@ export class EvmBatchProcessor { return new EvmRpcDataSource({ rpc: this.getChainRpcClient(), finalityConfirmation: this.finalityConfirmation, - preferTraceApi: this._preferTraceApi, - useDebugApiForStateDiffs: this._useDebugApiForStateDiffs, - pollInterval: this.chainPollInterval, + preferTraceApi: this.rpcIngestSettings?.preferTraceApi, + useDebugApiForStateDiffs: this.rpcIngestSettings?.useDebugApiForStateDiffs, + headPollInterval: this.rpcIngestSettings?.headPollInterval, + newHeadTimeout: this.rpcIngestSettings?.newHeadTimeout, log: this.getLogger().child('rpc', {rpcUrl: this.getChainRpcClient().url}) }) } @def private getArchiveDataSource(): EvmArchive { - let archive = assertNotNull(this.getDataSource().archive) - if (typeof archive == 'string') { - archive = {url: archive} - } + let archive = assertNotNull(this.archive) let log = this.getLogger().child('archive') @@ -319,15 +455,15 @@ export class EvmBatchProcessor { agent: new HttpAgent({ keepAlive: true }), - log: log.child('http') + log }) return new EvmArchive( new ArchiveClient({ http, - log, url: archive.url, - queryTimeout: archive.requestTimeout + queryTimeout: archive.requestTimeout, + log }) ) } @@ -346,10 +482,9 @@ export class EvmBatchProcessor { return res }) - if (this.fields) { - requests.forEach(req => { - req.request.fields = this.fields - }) + let fields = addDefaultFields(this.fields) + for (let req of requests) { + req.request.fields = fields } return applyRangeBound(requests, this.blockRange) @@ -365,7 +500,7 @@ export class EvmBatchProcessor { * @param database - database is responsible for providing storage to data handlers * and persisting mapping progress and status. * - * @param handler - The data handler, see {@link BatchContext} for an API available to the handler. + * @param handler - The data handler, see {@link DataHandlerContext} for an API available to the handler. */ run(database: Database, handler: (ctx: DataHandlerContext) => Promise): void { this.assertNotRunning() @@ -373,19 +508,27 @@ export class EvmBatchProcessor { let log = this.getLogger() runProgram(async () => { - let src = this.getDataSource() let chain = this.getChain() let mappingLog = log.child('mapping') - if (src.archive == null && this._useArchiveOnly) { - throw new Error('Archive URL is required when .useArchiveOnly() flag is set') + if (this.archive == null && this.rpcEndpoint == null) { + throw new Error( + 'No data source where specified. ' + + 'Use .setArchive() to specify Subsquid Archive and/or .setRpcEndpoint() to specify RPC endpoint.' + ) + } + + if (this.archive == null && this.rpcIngestSettings?.disabled) { + throw new Error('Subsquid Archive is required when RPC data ingestion is disabled') } return new Runner({ database, requests: this.getBatchRequests(), - archive: src.archive ? this.getArchiveDataSource() : undefined, - hotDataSource: src.chain && !this._useArchiveOnly ? this.getHotDataSource() : undefined, + archive: this.archive ? this.getArchiveDataSource() : undefined, + hotDataSource: this.rpcEndpoint && !this.rpcIngestSettings?.disabled + ? this.getHotDataSource() + : undefined, allBlocksAreFinal: this.finalityConfirmation === 0, prometheus: this.getPrometheusServer(), log, @@ -426,3 +569,37 @@ function concatRequestLists(a?: T[], b?: T[]): T[] | undefined } return result.length == 0 ? undefined : result } + + +function addDefaultFields(fields?: FieldSelection): FieldSelection { + return { + block: mergeDefaultFields(DEFAULT_FIELDS.block, fields?.block), + transaction: mergeDefaultFields(DEFAULT_FIELDS.transaction, fields?.transaction), + log: mergeDefaultFields(DEFAULT_FIELDS.log, fields?.log), + trace: mergeDefaultFields(DEFAULT_FIELDS.trace, fields?.trace), + stateDiff: {...mergeDefaultFields(DEFAULT_FIELDS.stateDiff, fields?.stateDiff), kind: true} + } +} + + +type Selector = { + [P in Props]?: boolean +} + + +function mergeDefaultFields( + defaults: Selector, + selection?: Selector +): Selector { + let result: Selector = {...defaults} + for (let key in selection) { + if (selection[key] != null) { + if (selection[key]) { + result[key] = true + } else { + delete result[key] + } + } + } + return result +} diff --git a/evm/evm-processor/src/util.ts b/evm/evm-processor/src/util.ts deleted file mode 100644 index 6b3ae8d53..000000000 --- a/evm/evm-processor/src/util.ts +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Formats the event id into a fixed-length string. When formatted the natural string ordering - * is the same as the ordering - * in the blockchain (first ordered by block height, then by block ID) - * - * @return id in the format 000000..00-000- - * - */ -export function formatId(height: number, hash: string, index?: number): string { - let blockPart = String(height).padStart(10, '0') - let indexPart = - index !== undefined - ? '-' + String(index).padStart(6, '0') - : '' - let _hash = hash.startsWith('0x') ? hash.slice(2) : hash - let shortHash = - _hash.length < 5 - ? _hash.padEnd(5, '0') - : _hash.slice(0, 5) - return `${blockPart}${indexPart}-${shortHash}` -} diff --git a/rush.json b/rush.json index fa4385cfc..8f780c7dc 100644 --- a/rush.json +++ b/rush.json @@ -742,6 +742,12 @@ "shouldPublish": true, "versionPolicyName": "npm" }, + { + "packageName": "@subsquid/util-internal-validation", + "projectFolder": "util/util-internal-validation", + "shouldPublish": true, + "versionPolicyName": "npm" + }, { "packageName": "@subsquid/util-naming", "projectFolder": "util/util-naming", @@ -771,8 +777,8 @@ "shouldPublish": false }, { - "packageName": "eth-usdc-transfers", - "projectFolder": "test/eth-usdc-transfers", + "packageName": "erc20-transfers", + "projectFolder": "test/erc20-transfers", "shouldPublish": false }, { diff --git a/substrate/substrate-data-raw/package.json b/substrate/substrate-data-raw/package.json index c033e9b26..eca2515d1 100644 --- a/substrate/substrate-data-raw/package.json +++ b/substrate/substrate-data-raw/package.json @@ -1,7 +1,7 @@ { "name": "@subsquid/substrate-data-raw", "version": "0.1.0", - "description": "Raw data fetcher for substrate based chains", + "description": "Raw RPC data fetcher for substrate based chains", "license": "GPL-3.0-or-later", "repository": "git@github.com:subsquid/squid.git", "publishConfig": { @@ -21,7 +21,9 @@ "@subsquid/util-internal-range": "^0.0.1" }, "peerDependencies": { - "@subsquid/rpc-client": "^4.4.2" + "@subsquid/logger": "^1.3.1", + "@subsquid/rpc-client": "^4.4.2", + "@subsquid/util-timeout": "^2.3.1" }, "devDependencies": { "@subsquid/rpc-client": "^4.4.2", diff --git a/substrate/substrate-data-raw/src/datasource.ts b/substrate/substrate-data-raw/src/datasource.ts index 4b7d10b96..521d57ddb 100644 --- a/substrate/substrate-data-raw/src/datasource.ts +++ b/substrate/substrate-data-raw/src/datasource.ts @@ -1,140 +1,143 @@ +import {Logger, LogLevel} from '@subsquid/logger' import {RpcClient} from '@subsquid/rpc-client' -import {assertNotNull, concurrentMap, Throttler} from '@subsquid/util-internal' -import {HotProcessor, HotState, HotUpdate, RequestsTracker} from '@subsquid/util-internal-ingest-tools' -import {assertRangeList, RangeRequest, RangeRequestList, splitRange} from '@subsquid/util-internal-range' -import {Fetcher, matchesRequest0, Stride} from './fetcher' -import {BlockBatch, BlockData, DataRequest, Hash} from './interfaces' +import {AsyncQueue, ensureError, maybeLast, partitionBy, Throttler, wait} from '@subsquid/util-internal' +import { + assertIsValid, + Batch, + BlockConsistencyError, + BlockRef, + ChainHeads, + DataConsistencyError, + HashAndHeight, + HotProcessor, + HotState, + HotUpdate, isDataConsistencyError, + coldIngest +} from '@subsquid/util-internal-ingest-tools' +import { + assertRangeList, + getRequestAt, + RangeRequestList, + splitRange, + splitRangeByRequest +} from '@subsquid/util-internal-range' +import {addTimeout, TimeoutError} from '@subsquid/util-timeout' +import assert from 'assert' +import {Fetch1} from './fetch1' +import {BlockData, BlockHeader, DataRequest} from './interfaces' import {Rpc} from './rpc' +import {RuntimeVersionTracker} from './runtimeVersionTracker' +import {qty2Int} from './util' export interface RpcDataSourceOptions { rpc: RpcClient - pollInterval?: number - strides?: number + headPollInterval?: number + newHeadTimeout?: number + log?: Logger } export class RpcDataSource { public readonly rpc: Rpc - private pollInterval: number - private strides: number + private headPollInterval: number + private newHeadTimeout: number + private log?: Logger constructor(options: RpcDataSourceOptions) { this.rpc = new Rpc(options.rpc) - this.pollInterval = options.pollInterval ?? 1000 - this.strides = options.strides || 5 + this.headPollInterval = options.headPollInterval ?? 5000 + this.newHeadTimeout = options.newHeadTimeout ?? 0 + this.log = options.log } async getFinalizedHeight(): Promise { let head = await this.rpc.getFinalizedHead() - let block = await this.rpc.getBlock0(head) - return block.height + let header = await this.rpc.getBlockHeader(head) + assert(header, 'finalized blocks must be always available') + return qty2Int(header.number) } - async *getFinalizedBlocks(requests: RangeRequestList, stopOnHead?: boolean): AsyncIterable { + async *getFinalizedBlocks(requests: RangeRequestList, stopOnHead?: boolean): AsyncIterable> { assertRangeList(requests.map(req => req.range)) - let batches1 = concurrentMap( - this.strides, - this.generateStrides(requests, stopOnHead), - async s => { - let blocks = await new Fetcher(this.rpc.withPriority(s.range.from)).getStride1(s) - return {blocks, stride: s} - } - ) - - let fetcher = new Fetcher(this.rpc) + let runtimeVersionTracker = new RuntimeVersionTracker() - for await (let {blocks, stride} of batches1) { - if (stride.request.runtimeVersion) { - await fetcher.fetchRuntimeVersion(blocks) - } - yield { - blocks, - isHead: !!stride.lastBlock - } - } - } + let stream = coldIngest({ + getFinalizedHeight: () => this.getFinalizedHeight(), + getSplit: req => { + let fetch = new Fetch1(this.rpc.withPriority(req.range.from)) + return fetch.getColdSplit(req.range.from, req.range.to, req.request) + }, + requests, + concurrency: Math.min(5, this.rpc.client.getConcurrency()), + splitSize: 10, + stopOnHead, + headPollInterval: this.headPollInterval + }) - private async *generateStrides( - requests: RangeRequestList, - stopOnHead?: boolean - ): AsyncIterable { - let head = new Throttler(() => this.rpc.getFinalizedHead(), this.pollInterval) - let top = await this.rpc.getBlock0(await head.get()) - for (let req of requests) { - let beg = req.range.from - let end = req.range.to ?? Infinity - while (beg <= end) { - if (top.height < beg) { - top = await this.getHeadBlock( - await head.get(), - top, - beg, - req.request - ) - } - if (top.height < beg) { - if (stopOnHead) return - await head.call() - } else { - let to = Math.min(top.height, end) - for (let range of splitRange(10, { - from: beg, - to - })) { - let stride: Stride = { - range, - request: req.request - } - if (range.to == top.height) { - stride.lastBlock = top - } - yield stride - } - beg = to + 1 - } + for await (let batch of stream) { + let request = getRequestAt(requests, batch.blocks[0].height) + if (request?.runtimeVersion) { + await runtimeVersionTracker.addRuntimeVersion(this.rpc, batch.blocks) + assertIsValid(batch.blocks) } + yield batch } } - private async getHeadBlock( - head: Hash, - current: BlockData, - desiredHeight: number, - req: DataRequest - ): Promise { - if (head === current.hash) return current - return this.rpc.getBlock0( - head, - desiredHeight == current.height + 1 ? req : undefined - ) - } - async processHotBlocks( - requests: RangeRequest[], + requests: RangeRequestList, state: HotState, cb: (upd: HotUpdate) => Promise ): Promise { - let requestsTracker = new RequestsTracker(requests) - let fetcher = new Fetcher(this.rpc) - - let processor = new HotProcessor({ - state, - getBlock: async ref => { - let hash = ref.hash || await this.rpc.getBlockHash(assertNotNull(ref.height)) - if (ref.height == null) { - let request = requestsTracker.getRequestAt(processor.getHeight() + 1) - let block = await this.rpc.getBlock0(hash, request) - request = requestsTracker.getRequestAt(block.height) - if (matchesRequest0(block, request)) { - return block - } else { - return this.rpc.getBlock0(hash, request) + let runtimeVersionTracker = new RuntimeVersionTracker() + let rpc = this.rpc + let fetch = new Fetch1(rpc) + + let proc = new HotProcessor(state, { + process: async upd => { + for (let pack of partitionBy(upd.blocks, b => !!getRequestAt(requests, b.height)?.runtimeVersion)) { + if (pack.value) { + await runtimeVersionTracker.addRuntimeVersion(rpc, pack.items) } + } + assertIsValid(upd.blocks) + await cb(upd) + }, + async getBlock(ref: HashAndHeight): Promise { + let blocks = await fetch.getColdSplit(ref.height, ref, { + ...getRequestAt(requests, ref.height), + runtimeVersion: false + }) + return blocks[0] + }, + async *getBlockRange(from: number, to: BlockRef): AsyncIterable { + let top: number + let headBlock: BlockData | undefined + if (to.height == null) { + headBlock = await fetch.getBlock0(to.hash, getRequestAt(requests, from) || {}) + if (headBlock == null) throw new BlockConsistencyError(to) + top = headBlock.height } else { - let request = requestsTracker.getRequestAt(ref.height) - return this.rpc.getBlock0(hash, request) + top = to.height + } + if (from > top) { + from = top + } + for (let split of splitRangeByRequest(requests, {from, to: top})) { + for (let range of splitRange(10, split.range)) { + let blocks = await fetch.getHotSplit( + from, + range.to === headBlock?.height ? headBlock : range.to, + split.request || {} + ) + let lastBlock = maybeLast(blocks)?.height ?? range.from - 1 + yield blocks + if (lastBlock < range.to) { + throw new BlockConsistencyError({height: lastBlock + 1}) + } + } } }, getHeader(block: BlockData) { @@ -143,38 +146,125 @@ export class RpcDataSource { hash: block.hash, parentHash: block.block.block.header.parentHash } + } + }) + + function isEnd(): boolean { + return proc.getFinalizedHeight() >= (maybeLast(requests)?.range.to ?? Infinity) + } + + if (this.rpc.client.supportsNotifications()) { + await this.subscription(heads => proc.goto(heads), isEnd) + } else { + await this.polling(heads => proc.goto(heads), isEnd) + } + } + + private async polling(cb: (heads: ChainHeads) => Promise, isEnd: () => boolean): Promise { + let headSrc = new Throttler(() => this.rpc.getHead(), this.headPollInterval) + let prev = '' + while (!isEnd()) { + let head = await headSrc.call() + if (head === prev) continue + let finalizedHead = await this.rpc.getFinalizedHead() + await this.handleNewHeads({ + best: {hash: head}, + finalized: {hash: finalizedHead} + }, cb) + } + } + + private async subscription(cb: (heads: ChainHeads) => Promise, isEnd: () => boolean): Promise { + let queue = new AsyncQueue(1) + let finalizedHeight = 0 + let prevHeight = 0 + + let finalizedHeadsHandle = this.rpc.client.subscribe({ + method: 'chain_subscribeFinalizedHeads', + unsubscribe: 'chain_unsubscribeFinalizedHeads', + notification: 'chain_finalizedHead', + onMessage(head: BlockHeader) { + try { + let height = qty2Int(head.number) + finalizedHeight = Math.max(finalizedHeight, height) + } catch(err: any) { + close(err) + } }, - getBlockHeight: async hash => { - let b = await this.rpc.getBlock0(hash) - return b.height + onError(err: Error) { + close(ensureError(err)) }, - process: async upd => { - for (let {blocks, request} of requestsTracker.splitBlocksByRequest(upd.blocks, b => b.height)) { - if (request) { - await fetcher.fetch1(blocks, request) - if (request.runtimeVersion) { - await fetcher.fetchRuntimeVersion(blocks) - } + resubscribeOnConnectionLoss: true + }) + + let newHeadsHandle = this.rpc.client.subscribe({ + method: 'chain_subscribeNewHeads', + unsubscribe: 'chain_unsubscribeNewHeads', + notification: 'chain_newHead', + onMessage(head: BlockHeader) { + try { + let height = qty2Int(head.number) + if (height >= prevHeight) { + prevHeight = height + queue.forcePut(height) } + } catch(err: any) { + close(err) } - await cb(upd) - } + }, + onError(err: Error) { + close(ensureError(err)) + }, + resubscribeOnConnectionLoss: true }) - for await (let head of this.getHeads()) { - await processor.goto(head) + function close(err?: Error) { + newHeadsHandle.close() + finalizedHeadsHandle.close() + if (err) { + queue.forcePut(err) + } + queue.close() + } + + try { + while (!isEnd()) { + let height = await addTimeout(queue.take(), this.newHeadTimeout).catch(ensureError) + if (height instanceof TimeoutError) { + this.log?.warn(`resetting RPC connection, because we haven't seen a new head for ${this.newHeadTimeout} ms`) + this.rpc.client.reset() + } else if (height instanceof Error) { + throw height + } else { + assert(height != null) + let hash = await this.rpc.getBlockHash(height) + if (hash) { + await this.handleNewHeads({ + best: {height, hash}, + finalized: {height: finalizedHeight} + }, cb) + } + } + } + } finally { + close() } } - private async *getHeads(): AsyncIterable<{best: Hash, finalized: Hash}> { - let heads = new Throttler(() => this.rpc.getHead(), this.pollInterval) - let prevBest: Hash | undefined - while (true) { - let best = await heads.call() - if (best !== prevBest) { - let finalized = await this.rpc.getFinalizedHead() - yield {best, finalized} - prevBest = best + private async handleNewHeads(heads: ChainHeads, cb: (heads: ChainHeads) => Promise): Promise { + for (let i = 0; i < 3; i++) { + try { + return await cb(heads) + } catch(err: any) { + if (isDataConsistencyError(err)) { + this.log?.write( + i > 0 ? LogLevel.WARN : LogLevel.DEBUG, + err.message + ) + await wait(100) + } else { + throw err + } } } } diff --git a/substrate/substrate-data-raw/src/fetch1.ts b/substrate/substrate-data-raw/src/fetch1.ts new file mode 100644 index 000000000..39db08713 --- /dev/null +++ b/substrate/substrate-data-raw/src/fetch1.ts @@ -0,0 +1,258 @@ +import {RpcCall} from '@subsquid/rpc-client/lib/interfaces' +import {last} from '@subsquid/util-internal' +import { + assertIsValid, + BlockConsistencyError, + HashAndHeight, + setInvalid, + trimInvalid +} from '@subsquid/util-internal-ingest-tools' +import assert from 'assert' +import {BlockData, DataRequest, DataRequest0, Hash, PartialBlockData} from './interfaces' +import {captureMissingBlock, Rpc} from './rpc' +import {qty2Int, toQty} from './util' + + +export class Fetch1 { + constructor(private rpc: Rpc) {} + + async getColdSplit0(from: number, to: number | HashAndHeight, req: DataRequest0): Promise { + let top = typeof to == 'number' ? to : to.height + assert(from <= top) + + let hash: Hash + if (typeof to == 'number') { + hash = await this.rpc.getBlockHash(top).then(hash => { + if (hash == null) throw new BlockConsistencyError({height: top}) + return hash + }) + } else { + hash = to.hash + } + + let blocks: BlockData[] = new Array(top - from + 1) + for (let i = blocks.length - 1; i >= 0; i--) { + let block = await this.getBlock0(hash, req) + if (block == null) throw new BlockConsistencyError({hash, height: from + i}) + blocks[i] = block + hash = block.block.block.header.parentHash + } + + return blocks + } + + async getBlock0(blockHash: Hash, req: DataRequest0): Promise { + if (req.extrinsics) { + let block = await this.rpc.getBlock(blockHash) + if (block == null) return + return { + height: qty2Int(block.block.header.number), + hash: blockHash, + block + } + } else { + let header = await this.rpc.getBlockHeader(blockHash) + if (header == null) return + return { + height: qty2Int(header.number), + hash: blockHash, + block: { + block: {header} + } + } + } + } + + async getColdSplit(from: number, to: number | HashAndHeight, req: DataRequest): Promise { + let blocks = await this.getColdSplit0(from, to, req) + await this.addRequestedData(blocks, req) + assertIsValid(blocks) + return blocks + } + + async getHotSplit(from: number, to: number | PartialBlockData, req: DataRequest): Promise { + let heads = await this.getHotHeads(from, to) + + await Promise.all([ + this.addBlock(heads, req), + this.addRequestedData(heads, req) + ]) + + let blocks = trimInvalid(heads) as BlockData[] + + for (let i = 1; i < blocks.length; i++) { + if (blocks[i].block.block.header.parentHash !== blocks[i-1].hash) return blocks.slice(0, i) + } + + return blocks + } + + private async addBlock(blocks: PartialBlockData[], req: DataRequest): Promise { + if (blocks.length == 0) return + let last = blocks.length - 1 + let lastBlock = blocks[last] + if (lastBlock.block && (lastBlock.block.block.extrinsics || !req.extrinsics)) { + last -= 1 + } + + let call: RpcCall[] = [] + for (let i = 0; i <= last; i++) { + let block = blocks[i] + if (req.extrinsics) { + call.push({ + method: 'chain_getBlock', + params: [block.hash] + }) + } else { + call.push({ + method: 'chain_getHeader', + params: [block.hash] + }) + } + } + + let batch = await this.rpc.batchCall(call) + + for (let i = 0; i < batch.length; i++) { + let block = batch[i] + if (block == null) return setInvalid(blocks, i) + if (req.extrinsics) { + blocks[i].block = block + } else { + blocks[i].block = { + block: {header: block} + } + } + } + } + + private async getHotHeads(from: number, to: number | PartialBlockData): Promise { + let top = typeof to == 'number' ? to: to.height + if (from > top) { + from = top + } + + let blocks: PartialBlockData[] + if (typeof to == 'object') { + if (to.block) { + let missingHashHeight = to.height - 2 + if (missingHashHeight - from >= 0) { + blocks = await this.getHashes(from, missingHashHeight) + } else { + blocks = [] + } + if (from < to.height) blocks.push({ + height: to.height - 1, + hash: to.block.block.header.parentHash + }) + blocks.push(to) + } else { + let missingHashHeight = to.height - 1 + if (missingHashHeight - from >= 0) { + blocks = await this.getHashes(from, missingHashHeight) + } else { + blocks = [] + } + blocks.push(to) + } + } else { + blocks = await this.getHashes(from, to) + } + + for (let i = 1; i < blocks.length; i++) { + if (blocks[i].height - blocks[i-1].height != 1) return blocks.slice(0, i) + } + + return blocks + } + + private async getHashes(from: number, to: number): Promise { + let call: RpcCall[] = [] + for (let height = from; height <= to; height++) { + call.push({ + method: 'chain_getBlockHash', + params: [toQty(height)] + }) + } + let hashes: (Hash | null)[] = await this.rpc.batchCall(call) + let heads: HashAndHeight[] = [] + for (let i = 0; i < hashes.length; i++) { + let height = from + i + let hash = hashes[i] + if (hash == null) return heads + heads.push({hash, height}) + } + return heads + } + + async addRequestedData(blocks: PartialBlockData[], req: DataRequest): Promise { + if (blocks.length == 0) return + + let tasks: Promise[] = [] + + if (req.events) { + tasks.push(this.addEvents(blocks)) + } + + if (req.trace != null) { + tasks.push(this.addTrace(blocks, req.trace)) + } + + if (req.runtimeVersion) { + let block = last(blocks) + tasks.push( + this.rpc.getRuntimeVersion(block.hash).then(v => { + if (v == null) { + block._isInvalid = true + } else { + block.runtimeVersion = v + } + }) + ) + } + + await Promise.all(tasks) + } + + private async addEvents(blocks: PartialBlockData[]): Promise { + let events = await this.rpc.getStorageMany(blocks.map(b => { + return [ + '0x26aa394eea5630e07c48ae0c9558cef780d41e5e16056765bc8461851072c9d7', + b.hash + ] + })) + + for (let i = 0; i < blocks.length; i++) { + let bytes = events[i] + if (bytes == null) { + blocks[i]._isInvalid = true + } else { + blocks[i].events = bytes + } + } + } + + private async addTrace(blocks: PartialBlockData[], targets: string): Promise { + let tasks = [] + for (let i = 0; i < blocks.length; i++) { + let block = blocks[i] + if (block.height != 0) { + tasks.push(this.rpc.call('state_traceBlock', [ + block.hash, + targets, + '', + '' + ], { + validateError: captureMissingBlock + }).then(trace => { + if (trace === undefined) { + block._isInvalid = true + } else { + block.trace = trace + } + })) + } + } + await Promise.all(tasks) + } +} diff --git a/substrate/substrate-data-raw/src/fetcher.ts b/substrate/substrate-data-raw/src/fetcher.ts deleted file mode 100644 index 2fc065091..000000000 --- a/substrate/substrate-data-raw/src/fetcher.ts +++ /dev/null @@ -1,124 +0,0 @@ -import {last} from '@subsquid/util-internal' -import {SplitRequest} from '@subsquid/util-internal-range' -import assert from 'assert' -import {BlockData, Bytes, DataRequest, RuntimeVersion} from './interfaces' -import {Rpc} from './rpc' -import {Prev, runtimeVersionEquals} from './util' - - -export interface Stride extends SplitRequest { - lastBlock?: BlockData -} - - -export class Fetcher { - private prevRuntimeVersion = new Prev() - - constructor(private rpc: Rpc) {} - - async getStride1(s: Stride): Promise { - let blocks = await this.getStride0(s) - await this.fetch1(blocks, s.request) - return blocks - } - - async getStride0(s: Stride): Promise { - let blocks: BlockData[] = new Array(s.range.to - s.range.from + 1) - if (s.lastBlock) { - assert(s.range.to === s.lastBlock.height) - if (matchesRequest0(s.lastBlock, s.request)) { - blocks[blocks.length - 1] = s.lastBlock - } else { - blocks[blocks.length - 1] = await this.rpc.getBlock0(s.lastBlock.hash, s.request) - } - } else { - let hash = await this.rpc.getBlockHash(s.range.to) - blocks[blocks.length - 1] = await this.rpc.getBlock0(hash, s.request) - } - for (let i = blocks.length - 2; i >= 0; i--) { - blocks[i] = await this.rpc.getBlock0(blocks[i + 1].block.block.header.parentHash, s.request) - } - return blocks - } - - async fetch1(blocks: BlockData[], req: DataRequest): Promise { - let tasks: Promise[] = [] - - if (req.events) { - tasks.push(this.fetchEvents(blocks)) - } - - if (req.trace != null) { - tasks.push(this.fetchTrace(blocks, req.trace)) - } - - await Promise.all(tasks) - } - - private async fetchEvents(blocks: BlockData[]): Promise { - let call = blocks.map(b => ({ - method: 'state_getStorage', - params: [ - '0x26aa394eea5630e07c48ae0c9558cef780d41e5e16056765bc8461851072c9d7', - b.hash - ] - })) - - let events: Bytes[] = await this.rpc.batchCall(call) - - for (let i = 0; i < blocks.length; i++) { - blocks[i].events = events[i] - } - } - - private fetchTrace(blocks: BlockData[], targets: string): Promise { - let tasks = [] - for (let i = 0; i < blocks.length; i++) { - let block = blocks[i] - if (block.height != 0) { - tasks.push(async () => { - block.trace = await this.rpc.call('state_traceBlock', [ - block.hash, - targets, - '', - '' - ]) - }); - } - } - return Promise.all(tasks).then() - } - - async fetchRuntimeVersion(blocks: BlockData[]): Promise { - if (blocks.length == 0) return - - let prev = this.prevRuntimeVersion.get(blocks[0].height) - if (prev == null) { - prev = blocks[0].runtimeVersion = await this.rpc.getRuntimeVersion(blocks[0].hash) - this.prevRuntimeVersion.set(blocks[0].height, prev) - } - - let lastBlock = last(blocks) - if (lastBlock.runtimeVersion == null) { - lastBlock.runtimeVersion = await this.rpc.getRuntimeVersion(lastBlock.hash) - } - - for (let block of blocks) { - if (block.runtimeVersion == null) { - block.runtimeVersion = runtimeVersionEquals(prev, lastBlock.runtimeVersion) - ? prev - : await this.rpc.getRuntimeVersion(block.hash) - } - if (runtimeVersionEquals(prev, block.runtimeVersion)) { - block.runtimeVersion = prev - } else { - prev = block.runtimeVersion - this.prevRuntimeVersion.set(block.height, prev) - } - } - } -} - -export function matchesRequest0(block: BlockData, request?: DataRequest): boolean { - return !!block.block.block.extrinsics || !request?.extrinsics -} diff --git a/substrate/substrate-data-raw/src/index.ts b/substrate/substrate-data-raw/src/index.ts index 5a2ab3560..687c286df 100644 --- a/substrate/substrate-data-raw/src/index.ts +++ b/substrate/substrate-data-raw/src/index.ts @@ -1,4 +1,5 @@ export * from './datasource' +export * from './fetch1' export * from './interfaces' export * from './rpc' export * from './util' diff --git a/substrate/substrate-data-raw/src/interfaces.ts b/substrate/substrate-data-raw/src/interfaces.ts index d2ce24912..47f3a1d7c 100644 --- a/substrate/substrate-data-raw/src/interfaces.ts +++ b/substrate/substrate-data-raw/src/interfaces.ts @@ -65,10 +65,15 @@ export interface RuntimeVersion extends RuntimeVersionId { } -export interface BlockData { - hash: Hash - height: number +export interface BlockData extends PartialBlockData { block: PartialGetBlockResult +} + + +export interface PartialBlockData { + height: number + hash: Hash + block?: PartialGetBlockResult runtimeVersion?: RuntimeVersion metadata?: Bytes /** @@ -76,19 +81,17 @@ export interface BlockData { */ events?: Bytes trace?: any + _isInvalid?: boolean } -export interface BlockBatch { - blocks: BlockData[] - isHead: boolean +export interface DataRequest0 { + extrinsics?: boolean } -export interface DataRequest { +export interface DataRequest1 extends DataRequest0 { events?: boolean - extrinsics?: boolean - runtimeVersion?: boolean /** * List of trace targets or an empty string to fetch all */ @@ -96,7 +99,6 @@ export interface DataRequest { } -export interface HashAndHeight { - hash: Hash - height: number +export interface DataRequest extends DataRequest1 { + runtimeVersion?: boolean } diff --git a/substrate/substrate-data-raw/src/rpc.ts b/substrate/substrate-data-raw/src/rpc.ts index b72e78498..cc6bf4426 100644 --- a/substrate/substrate-data-raw/src/rpc.ts +++ b/substrate/substrate-data-raw/src/rpc.ts @@ -1,32 +1,28 @@ -import {CallOptions, RpcClient} from '@subsquid/rpc-client' -import { - BlockData, - BlockHeader, - Bytes, - DataRequest, - GetBlockResult, - Hash, - PartialGetBlockResult, - RuntimeVersion -} from './interfaces' -import {qty2Int, toQty} from './util' +import {CallOptions, RpcClient, RpcError} from '@subsquid/rpc-client' +import {RpcErrorInfo} from '@subsquid/rpc-client/lib/interfaces' +import {BlockHeader, Bytes, GetBlockResult, Hash, RuntimeVersion} from './interfaces' +import {toQty} from './util' export class Rpc { constructor( public readonly client: RpcClient, - private options: CallOptions = {} + public readonly options: CallOptions = {} ) {} withPriority(priority: number): Rpc { - return this.withOptions({ + return new Rpc(this.client, { ...this.options, priority }) } - withOptions(options: CallOptions): Rpc { - return new Rpc(this.client, options) + call(method: string, params?: any[], options?: CallOptions): Promise { + return this.client.call(method, params, {...this.options, ...options}) + } + + batchCall(batch: {method: string, params?: any[]}[], options?: CallOptions): Promise { + return this.client.batchCall(batch, {...this.options, ...options}) } getFinalizedHead(): Promise { @@ -37,50 +33,57 @@ export class Rpc { return this.call('chain_getHead') } - getBlockHash(height: number): Promise { + getBlockHash(height: number): Promise { return this.call('chain_getBlockHash', [toQty(height)]) } - getBlockHeader(hash: Hash): Promise { - return this.call('chain_getHeader', [hash]) + getBlockHeader(blockHash: Hash): Promise { + return this.call('chain_getHeader', [blockHash]) } - getBlock(hash: Hash): Promise { - return this.call('chain_getBlock', [hash]) + getBlock(blockHash: Hash): Promise { + return this.call('chain_getBlock', [blockHash]) } - getRuntimeVersion(blockHash: Hash): Promise { - return this.call('state_getRuntimeVersion', [blockHash]) + getRuntimeVersion(blockHash: Hash): Promise { + return this.call('state_getRuntimeVersion', [blockHash], { + validateError: captureMissingBlock + }) } - getMetadata(blockHash: Hash): Promise { - return this.call('state_getMetadata', [blockHash]) + getMetadata(blockHash: Hash): Promise { + return this.call('state_getMetadata', [blockHash], { + validateError: captureMissingBlock + }) } - getStorage(blockHash: Hash, key: Bytes): Promise { - return this.call('state_getStorageAt', [key, blockHash]) + getStorage(key: Bytes, blockHash: Hash): Promise { + return this.call('state_getStorageAt', [key, blockHash], { + validateError: captureMissingBlock + }) } - async getBlock0(hash: Hash, req?: DataRequest): Promise { - let block: PartialGetBlockResult - if (req?.extrinsics) { - block = await this.getBlock(hash) - } else { - let header = await this.getBlockHeader(hash) - block = {block: {header}} - } - return { - hash, - height: qty2Int(block.block.header.number), - block - } + getStorageMany(query: [key: Bytes, blockHash: Hash][]): Promise<(Bytes | null | undefined)[]> { + let call = query.map(q => ({ + method: 'state_getStorageAt', + params: q + })) + return this.batchCall(call, { + validateError: captureMissingBlock + }) } +} - call(method: string, params?: any[]): Promise { - return this.client.call(method, params, this.options) - } - batchCall(batch: {method: string, params?: any[]}[]): Promise { - return this.client.batchCall(batch, this.options) +export function captureMissingBlock(info: RpcErrorInfo): undefined { + if (isMissingBlockError(info)) { + return undefined + } else { + throw new RpcError(info) } } + + +export function isMissingBlockError(info: RpcErrorInfo): boolean { + return info.message.includes(' not found') +} diff --git a/substrate/substrate-data-raw/src/runtimeVersionTracker.ts b/substrate/substrate-data-raw/src/runtimeVersionTracker.ts new file mode 100644 index 000000000..11f46cfce --- /dev/null +++ b/substrate/substrate-data-raw/src/runtimeVersionTracker.ts @@ -0,0 +1,63 @@ +import {setInvalid} from '@subsquid/util-internal-ingest-tools' +import {BlockData, RuntimeVersion} from './interfaces' +import {Rpc} from './rpc' +import {Prev, runtimeVersionEquals} from './util' + + +export class RuntimeVersionTracker { + private prevRuntimeVersion = new Prev() + + async addRuntimeVersion(rpc: Rpc, blocks: BlockData[]): Promise { + if (blocks.length == 0) return + + let prev: RuntimeVersion + let maybePrev = this.prevRuntimeVersion.get(blocks[0].height) + if (maybePrev == null) { + let v = blocks[0].runtimeVersion || await rpc.getRuntimeVersion(blocks[0].hash) + if (v == null) return setInvalid(blocks) + prev = blocks[0].runtimeVersion = v + this.prevRuntimeVersion.set(blocks[0].height, prev) + } else { + prev = maybePrev + } + + let last = blocks.length - 1 + let lastRuntimeVersion: RuntimeVersion | undefined + while (last >= 0) { + let block = blocks[last] + if (block.runtimeVersion == null) { + lastRuntimeVersion = await rpc.getRuntimeVersion(block.hash) + if (lastRuntimeVersion) { + block.runtimeVersion = lastRuntimeVersion + break + } else { + last -= 1 + } + } else { + lastRuntimeVersion = block.runtimeVersion + break + } + } + + if (lastRuntimeVersion == null) return setInvalid(blocks) + + for (let i = 0; i < last; i++) { + let block = blocks[i] + if (block.runtimeVersion == null) { + if (runtimeVersionEquals(prev, lastRuntimeVersion)) { + block.runtimeVersion = prev + } else { + block.runtimeVersion = await rpc.getRuntimeVersion(block.hash) + if (block.runtimeVersion == null) return setInvalid(blocks, i) + } + } + if (runtimeVersionEquals(prev, block.runtimeVersion)) { + // maintain same object reference + block.runtimeVersion = prev + } else { + prev = block.runtimeVersion + this.prevRuntimeVersion.set(block.height, prev) + } + } + } +} diff --git a/substrate/substrate-data-raw/src/util.ts b/substrate/substrate-data-raw/src/util.ts index cdc95be55..b81b91723 100644 --- a/substrate/substrate-data-raw/src/util.ts +++ b/substrate/substrate-data-raw/src/util.ts @@ -23,7 +23,7 @@ export function runtimeVersionEquals(a: RuntimeVersionId, b: RuntimeVersionId): } -interface PrevItem { +export interface PrevItem { height: number value: T } @@ -41,7 +41,7 @@ export class Prev { } getItem(height: number): PrevItem | undefined { - this.rollbackTo(height) + this.rollbackTo(height - 1) return maybeLast(this.items) } diff --git a/substrate/substrate-data/src/datasource.ts b/substrate/substrate-data/src/datasource.ts index 70b06fa89..984711af5 100644 --- a/substrate/substrate-data/src/datasource.ts +++ b/substrate/substrate-data/src/datasource.ts @@ -2,39 +2,41 @@ import type {RpcClient} from '@subsquid/rpc-client' import * as raw from '@subsquid/substrate-data-raw' import {OldSpecsBundle, OldTypesBundle} from '@subsquid/substrate-runtime/lib/metadata' import {Batch, HotState, HotUpdate} from '@subsquid/util-internal-ingest-tools' -import {RangeRequest, RangeRequestList} from '@subsquid/util-internal-range' +import {mapRangeRequestList, RangeRequestList} from '@subsquid/util-internal-range' import {Block, DataRequest} from './interfaces/data' import {Parser} from './parser' export interface RpcDataSourceOptions { rpc: RpcClient - pollInterval?: number + headPollInterval?: number + newHeadTimeout?: number typesBundle?: OldTypesBundle | OldSpecsBundle } export class RpcDataSource { - private rds: raw.RpcDataSource + private rawDataSource: raw.RpcDataSource private typesBundle?: OldTypesBundle | OldSpecsBundle constructor(options: RpcDataSourceOptions) { - this.rds = new raw.RpcDataSource({ + this.rawDataSource = new raw.RpcDataSource({ rpc: options.rpc, - pollInterval: options.pollInterval + headPollInterval: options.headPollInterval, + newHeadTimeout: options.newHeadTimeout }) this.typesBundle = options.typesBundle } get rpc(): raw.Rpc { - return this.rds.rpc + return this.rawDataSource.rpc } getFinalizedHeight(): Promise { - return this.rds.getFinalizedHeight() + return this.rawDataSource.getFinalizedHeight() } - getBlockHash(height: number): Promise { + getBlockHash(height: number): Promise { return this.rpc.getBlockHash(height) } @@ -44,11 +46,11 @@ export class RpcDataSource { ): AsyncIterable> { let parser = new Parser(this.rpc, requests, this.typesBundle) - for await (let batch of this.rds.getFinalizedBlocks( - requests.map(toRawRangeRequest), + for await (let batch of this.rawDataSource.getFinalizedBlocks( + mapRangeRequestList(requests, toRawRequest), stopOnHead )) { - let blocks = await parser.parse(batch.blocks) + let blocks = await parser.parseCold(batch.blocks) yield { ...batch, blocks @@ -56,18 +58,19 @@ export class RpcDataSource { } } - processHotBlocks( - requests: RangeRequest[], + async processHotBlocks( + requests: RangeRequestList, state: HotState, cb: (upd: HotUpdate) => Promise ): Promise { let parser = new Parser(this.rpc, requests, this.typesBundle) - return this.rds.processHotBlocks( - requests.map(toRawRangeRequest), + + return this.rawDataSource.processHotBlocks( + mapRangeRequestList(requests, toRawRequest), state, async upd => { - let blocks = await parser.parse(upd.blocks) - await cb({ + let blocks = await parser.parseCold(upd.blocks) + return cb({ ...upd, blocks }) @@ -77,14 +80,6 @@ export class RpcDataSource { } -function toRawRangeRequest(req: RangeRequest): RangeRequest { - return { - range: req.range, - request: toRawRequest(req.request) - } -} - - function toRawRequest(req: DataRequest): raw.DataRequest { return { runtimeVersion: true, diff --git a/substrate/substrate-data/src/interfaces/data-raw.ts b/substrate/substrate-data/src/interfaces/data-raw.ts index 0287ae448..848580273 100644 --- a/substrate/substrate-data/src/interfaces/data-raw.ts +++ b/substrate/substrate-data/src/interfaces/data-raw.ts @@ -1,6 +1,7 @@ import type {BlockData, Bytes} from '@subsquid/substrate-data-raw' import type {Runtime} from '@subsquid/substrate-runtime' import type {AccountId} from '../parsing/validator' +import type {Block as ParsedBlock} from './data' export interface RawBlock extends BlockData { @@ -15,4 +16,5 @@ export interface RawBlock extends BlockData { storage?: { [key: Bytes]: Bytes | null } + parsed?: ParsedBlock } diff --git a/substrate/substrate-data/src/parser.ts b/substrate/substrate-data/src/parser.ts index 503d2591f..3a0a009e6 100644 --- a/substrate/substrate-data/src/parser.ts +++ b/substrate/substrate-data/src/parser.ts @@ -1,8 +1,8 @@ -import {HashAndHeight, Prev, Rpc} from '@subsquid/substrate-data-raw' +import {Prev, Rpc} from '@subsquid/substrate-data-raw' import {OldSpecsBundle, OldTypesBundle} from '@subsquid/substrate-runtime/lib/metadata' -import {addErrorContext, annotateAsyncError, assertNotNull, groupBy, last} from '@subsquid/util-internal' -import {RequestsTracker} from '@subsquid/util-internal-ingest-tools' -import {RangeRequestList} from '@subsquid/util-internal-range' +import {addErrorContext, annotateAsyncError, assertNotNull, groupBy} from '@subsquid/util-internal' +import {assertIsValid, HashAndHeight, setInvalid, trimInvalid} from '@subsquid/util-internal-ingest-tools' +import {RangeRequestList, splitBlocksByRequest} from '@subsquid/util-internal-range' import assert from 'assert' import {Block, Bytes, DataRequest} from './interfaces/data' import {RawBlock} from './interfaces/data-raw' @@ -20,65 +20,74 @@ const STORAGE = { export class Parser { - private requests: RequestsTracker private prevValidators = new Prev<{session: Bytes, validators: AccountId[]}>() private runtimeTracker: RuntimeTracker constructor( private rpc: Rpc, - requests: RangeRequestList, + private requests: RangeRequestList, typesBundle?: OldTypesBundle | OldSpecsBundle ) { - this.requests = new RequestsTracker(requests) - this.runtimeTracker = new RuntimeTracker( + this.rpc, block => ({ height: block.height, hash: block.hash, parentHash: block.block.block.header.parentHash }), block => assertNotNull(block.runtimeVersion), - this.rpc, typesBundle ) } - async parse(blocks: RawBlock[]): Promise { - if (blocks.length == 0) return [] + async parseCold(blocks: RawBlock[]): Promise { + await this.parse(blocks) + assertIsValid(blocks) + return blocks.map(b => assertNotNull(b.parsed)) + } + + async parse(blocks: RawBlock[]): Promise { + if (blocks.length == 0) return await this.runtimeTracker.setRuntime(blocks) + blocks = trimInvalid(blocks) - let result = [] + for (let batch of splitBlocksByRequest(this.requests, blocks, b => b.height)) { + let batchBlocks = batch.blocks - for (let batch of this.requests.splitBlocksByRequest(blocks, b => b.height)) { if (batch.request?.blockValidator) { - await this.setValidators(batch.blocks) + await this.setValidators(batchBlocks) + batchBlocks = trimInvalid(batchBlocks) } if (batch.request?.extrinsics?.fee) { - for (let [runtime, blocks] of groupBy(batch.blocks, b => b.runtime!).entries()) { + for (let [runtime, blocks] of groupBy(batchBlocks, b => b.runtime!).entries()) { if (!runtime.hasEvent('TransactionPayment.TransactionFeePaid') && supportsFeeCalc(runtime)) { await this.setFeeMultiplier(blocks) } } + batchBlocks = trimInvalid(batchBlocks) } - for (let block of batch.blocks) { - let parsed = await this.parseBlock(block, batch.request) - result.push(parsed) + for (let block of batchBlocks) { + await this.parseBlock(block, batch.request) + if (block._isInvalid) return } } - - return result } - private async parseBlock(rawBlock: RawBlock, options?: DataRequest): Promise { + private async parseBlock(rawBlock: RawBlock, options?: DataRequest): Promise { while (true) { try { - return parseBlock(rawBlock, options ?? {}) + rawBlock.parsed = parseBlock(rawBlock, options ?? {}) + return } catch(err: any) { if (err instanceof MissingStorageValue) { - let val = await this.rpc.getStorage(rawBlock.block.block.header.parentHash, err.key) + let val = await this.rpc.getStorage(err.key, rawBlock.block.block.header.parentHash) + if (val === undefined) { + rawBlock._isInvalid = true + return + } let storage = rawBlock.storage || (rawBlock.storage = {}) storage[err.key] = val } else { @@ -91,38 +100,64 @@ export class Parser { private async setValidators(blocks: RawBlock[]): Promise { blocks = blocks.filter(b => b.runtime!.hasStorageItem('Session.Validators')) - if (blocks.length == 0) return + if (blocks.length == 0 || blocks[0]._isInvalid) return - let prev = this.prevValidators.get(blocks[0].height) - if (prev == null) { - prev = await this.fetchValidators(blocks[0]) + let prev: {session: Bytes, validators: AccountId[]} + let maybePrev = this.prevValidators.get(blocks[0].height) + if (maybePrev == null) { + maybePrev = await this.fetchValidators(blocks[0]) + if (maybePrev == null) return setInvalid(blocks) + prev = maybePrev + } else { + prev = maybePrev } - let lastBlock = last(blocks) - if (lastBlock.session == null) { - lastBlock.session = await this.rpc.getStorage(lastBlock.hash, STORAGE.session) + let last = blocks.length - 1 + let lastBlock: RawBlock | undefined + while (last >= 0) { + lastBlock = blocks[last] + if (lastBlock.session == null) { + let session = await this.rpc.getStorage(STORAGE.session, lastBlock.hash) + if (session == null) { + last -= 1 + } else { + lastBlock.session = session + break + } + } else { + break + } } - for (let block of blocks) { - block.session = prev.session == lastBlock.session - ? prev.session - : await this.rpc.getStorage(block.hash, STORAGE.session) + if (lastBlock == null) return setInvalid(blocks) + for (let i = 0; i < last; i++) { + let block = blocks[i] + if (prev.session == lastBlock.session) { + block.session = prev.session + } else { + let session = await this.rpc.getStorage(STORAGE.session, block.hash) + if (session == null) return setInvalid(blocks, i) + block.session = session + } if (prev.session == block.session) { block.session = prev.session block.validators = prev.validators } else { - prev = await this.fetchValidators(block) + let maybePrev = await this.fetchValidators(block) + if (maybePrev == null) return setInvalid(blocks, i) + prev = maybePrev } } } @annotateAsyncError(getRefCtx) - private async fetchValidators(block: RawBlock): Promise<{session: Bytes, validators: AccountId[]}> { + private async fetchValidators(block: RawBlock): Promise<{session: Bytes, validators: AccountId[]} | undefined> { let [session, data] = await Promise.all([ - block.session ? Promise.resolve(block.session) : this.rpc.getStorage(block.hash, STORAGE.session), - this.rpc.getStorage(block.hash, STORAGE.validators) + block.session ? Promise.resolve(block.session) : this.rpc.getStorage(STORAGE.session, block.hash), + this.rpc.getStorage(STORAGE.validators, block.hash) ]) + if (session == null || data === undefined) return let runtime = assertNotNull(block.runtime) let validators = runtime.decodeStorageValue('Session.Validators', data) assert(Array.isArray(validators)) @@ -134,16 +169,18 @@ export class Parser { } private async setFeeMultiplier(blocks: RawBlock[]): Promise { - let call = blocks.map(b => { + let values = await this.rpc.getStorageMany(blocks.map(b => { let parentHash = b.height == 0 ? b.hash : b.block.block.header.parentHash - return { - method: 'state_getStorageAt', - params: [STORAGE.nextFeeMultiplier, parentHash] - } - }) - let values: Bytes[] = await this.rpc.batchCall(call) + return [STORAGE.nextFeeMultiplier, parentHash] + })) for (let i = 0; i < blocks.length; i++) { - blocks[i].feeMultiplier = values[i] + let value = values[i] + let block = blocks[i] + if (value === undefined) { + block._isInvalid = true + } else if (value) { + block.feeMultiplier = value + } } } } diff --git a/substrate/substrate-data/src/runtime-tracker.ts b/substrate/substrate-data/src/runtime-tracker.ts index bfc914d5d..8e1a72f68 100644 --- a/substrate/substrate-data/src/runtime-tracker.ts +++ b/substrate/substrate-data/src/runtime-tracker.ts @@ -1,7 +1,8 @@ -import {Hash, HashAndHeight, Prev, Rpc, runtimeVersionEquals, RuntimeVersionId} from '@subsquid/substrate-data-raw' +import {Hash, Prev, PrevItem, Rpc, runtimeVersionEquals, RuntimeVersionId} from '@subsquid/substrate-data-raw' import {Runtime} from '@subsquid/substrate-runtime' import {OldSpecsBundle, OldTypesBundle} from '@subsquid/substrate-runtime/lib/metadata' import {annotateAsyncError} from '@subsquid/util-internal' +import {HashAndHeight, setInvalid} from '@subsquid/util-internal-ingest-tools' interface Header extends HashAndHeight { @@ -12,6 +13,7 @@ interface Header extends HashAndHeight { export interface WithRuntime { runtime?: Runtime runtimeOfPrevBlock?: Runtime + _isInvalid?: boolean } @@ -19,24 +21,32 @@ export class RuntimeTracker { private prev = new Prev() constructor( + private rpc: Rpc, private getBlockHeader: (block: B) => Header, private getBlockRuntimeVersion: (block: B) => RuntimeVersionId, - private rpc: Rpc, private typesBundle?: OldTypesBundle | OldSpecsBundle ) { } async setRuntime(blocks: B[]): Promise { - if (blocks.length == 0) return + if (blocks.length == 0 || blocks[0]._isInvalid) return + let prev: PrevItem let parentParentHeight = Math.max(0, this.getBlockHeader(blocks[0]).height - 2) - let prev = this.prev.getItem(parentParentHeight) - if (prev == null) { - prev = await this.fetchRuntime(await this.getParent(getParent(this.getBlockHeader(blocks[0])))) + let maybePrev = this.prev.getItem(parentParentHeight) + if (maybePrev == null) { + let parentParentRef = await this.getParent(getParent(this.getBlockHeader(blocks[0]))) + if (parentParentRef == null) return setInvalid(blocks) + maybePrev = await this.fetchRuntime(parentParentRef) + if (maybePrev == null) return setInvalid(blocks) + prev = maybePrev + } else { + prev = maybePrev } for (let i = 0; i < blocks.length; i++) { let block = blocks[i] + if (block._isInvalid) return let header = this.getBlockHeader(block) let parentParentHeight = Math.max(0, header.height - 2) let parentHeight = Math.max(0, header.height - 1) @@ -45,7 +55,11 @@ export class RuntimeTracker { if (runtimeVersionEquals(prev.value, rtv) || prev.height == parentParentHeight) { block.runtimeOfPrevBlock = prev.value } else { - prev = await this.fetchRuntime(await this.getParent(getParent(header))) + let parentParentRef = await this.getParent(getParent(header)) + if (parentParentRef == null) return setInvalid(blocks, i) + let maybePrev = await this.fetchRuntime(parentParentRef) + if (maybePrev == null) return setInvalid(blocks, i) + prev = maybePrev block.runtimeOfPrevBlock = prev.value } @@ -56,7 +70,9 @@ export class RuntimeTracker { value: prev.value } } else { - prev = await this.fetchRuntime(getParent(header)) + let maybePrev = await this.fetchRuntime(getParent(header)) + if (maybePrev == null) return setInvalid(blocks, i) + prev = maybePrev block.runtime = prev.value } } @@ -65,19 +81,21 @@ export class RuntimeTracker { @annotateAsyncError(getRefCtx) private async fetchRuntime( ref: HashAndHeight - ): Promise<{height: number, value: Runtime}> { + ): Promise<{height: number, value: Runtime} | undefined> { let [runtimeVersion, metadata] = await Promise.all([ this.rpc.getRuntimeVersion(ref.hash), this.rpc.getMetadata(ref.hash) ]) + if (runtimeVersion == null || metadata == null) return undefined let runtime = new Runtime(runtimeVersion, metadata, this.typesBundle, this.rpc.client) this.prev.set(ref.height, runtime) return {height: ref.height, value: runtime} } - private async getParent(ref: HashAndHeight): Promise { + private async getParent(ref: HashAndHeight): Promise { if (ref.height == 0) return ref let header = await this.rpc.getBlockHeader(ref.hash) + if (header == null) return null return { height: ref.height - 1, hash: header.parentHash diff --git a/substrate/substrate-dump/src/dumper.ts b/substrate/substrate-dump/src/dumper.ts index 828662195..deeac9c23 100644 --- a/substrate/substrate-dump/src/dumper.ts +++ b/substrate/substrate-dump/src/dumper.ts @@ -1,7 +1,6 @@ import {createLogger, Logger} from '@subsquid/logger' import {RpcClient} from '@subsquid/rpc-client' import { - BlockBatch, BlockData, DataRequest, RpcDataSource, @@ -13,6 +12,7 @@ import {ArchiveLayout, getShortHash} from '@subsquid/util-internal-archive-layou import {printTimeInterval, Progress} from '@subsquid/util-internal-counters' import {createFs, Fs} from '@subsquid/util-internal-fs' import {assertRange, printRange, Range, rangeEnd} from '@subsquid/util-internal-range' +import assert from 'assert' import {MetadataWriter} from './metadata' import {PrometheusServer} from './prometheus' @@ -79,8 +79,7 @@ export class Dumper { src(): RpcDataSource { return new RpcDataSource({ rpc: this.rpc(), - pollInterval: 10_000, - strides: Math.max(2, this.getEndpointCapacity() - 2) + headPollInterval: 10_000 }) } @@ -89,7 +88,7 @@ export class Dumper { return new PrometheusServer(this.options.metrics ?? 0, this.rpc()) } - ingest(range: Range): AsyncIterable { + ingest(range: Range): AsyncIterable<{blocks: BlockData[], isHead: boolean}> { let request: DataRequest = { runtimeVersion: true, extrinsics: true, @@ -185,8 +184,10 @@ export class Dumper { implVersion: v.implVersion, blockHeight: block.height, blockHash: getShortHash(block.hash) - }, () => { - return this.src().rpc.getMetadata(block.hash) + }, async () => { + let metadata = await this.src().rpc.getMetadata(block.hash) + assert(metadata, 'finalized blocks are supposed to be always available') + return metadata }) prevRuntimeVersion = v } diff --git a/substrate/substrate-dump/src/prometheus.ts b/substrate/substrate-dump/src/prometheus.ts index e5408c0f0..4e8f1d3ef 100644 --- a/substrate/substrate-dump/src/prometheus.ts +++ b/substrate/substrate-dump/src/prometheus.ts @@ -2,6 +2,7 @@ import {createLogger} from '@subsquid/logger' import {RpcClient} from '@subsquid/rpc-client' import {Rpc} from '@subsquid/substrate-data-raw' import {createPrometheusServer, ListeningServer} from '@subsquid/util-internal-prometheus-server' +import assert from 'assert' import {collectDefaultMetrics, Gauge, Registry} from 'prom-client' @@ -29,8 +30,9 @@ export class PrometheusServer { try { let head = await rpc.getFinalizedHead() - let header = await rpc.getBlock0(head) - chainHeight = header.height + let header = await rpc.getBlockHeader(head) + assert(header?.number, 'finalized blocks supposed to be always available') + chainHeight = parseInt(header.number) } catch(err: any) { LOG.error(err, 'failed to acquire chain height') } diff --git a/substrate/substrate-ingest/src/ingest.ts b/substrate/substrate-ingest/src/ingest.ts index f1be05870..588cd7d5f 100644 --- a/substrate/substrate-ingest/src/ingest.ts +++ b/substrate/substrate-ingest/src/ingest.ts @@ -2,7 +2,12 @@ import {createLogger} from '@subsquid/logger' import {RpcClient} from '@subsquid/rpc-client' import {Block, DataRequest, Parser, RpcDataSource} from '@subsquid/substrate-data' import * as raw from '@subsquid/substrate-data-raw' -import {getOldTypesBundle, OldSpecsBundle, OldTypesBundle, readOldTypesBundle} from '@subsquid/substrate-runtime/lib/metadata' +import { + getOldTypesBundle, + OldSpecsBundle, + OldTypesBundle, + readOldTypesBundle +} from '@subsquid/substrate-runtime/lib/metadata' import {assertNotNull, def, ensureError, wait} from '@subsquid/util-internal' import {ArchiveLayout, DataChunk, getChunkPath} from '@subsquid/util-internal-archive-layout' import {createFs} from '@subsquid/util-internal-fs' @@ -92,7 +97,7 @@ export class Ingest { const process = async (rawBlocks: raw.BlockData[]) => { if (rawBlocks.length == 0) return - let blocks = await parser.parse(rawBlocks) + let blocks = await parser.parseCold(rawBlocks) await cb(blocks) } diff --git a/substrate/substrate-processor/package.json b/substrate/substrate-processor/package.json index 6523af9d6..718f749b5 100644 --- a/substrate/substrate-processor/package.json +++ b/substrate/substrate-processor/package.json @@ -24,8 +24,10 @@ "@subsquid/util-internal": "^2.5.2", "@subsquid/util-internal-archive-client": "^0.0.1", "@subsquid/util-internal-hex": "^1.2.1", + "@subsquid/util-internal-ingest-tools": "^0.0.2", "@subsquid/util-internal-json": "^1.2.1", - "@subsquid/util-internal-processor-tools": "^3.1.0" + "@subsquid/util-internal-processor-tools": "^3.1.0", + "@subsquid/util-internal-range": "^0.0.1" }, "peerDependencies": { "@subsquid/substrate-runtime": "^1.0.1" diff --git a/substrate/substrate-processor/src/ds-archive.ts b/substrate/substrate-processor/src/ds-archive.ts index 9b41b883a..251aee2d1 100644 --- a/substrate/substrate-processor/src/ds-archive.ts +++ b/substrate/substrate-processor/src/ds-archive.ts @@ -3,14 +3,9 @@ import {Rpc, RuntimeTracker, WithRuntime} from '@subsquid/substrate-data' import {OldSpecsBundle, OldTypesBundle} from '@subsquid/substrate-runtime/lib/metadata' import {annotateSyncError, assertNotNull} from '@subsquid/util-internal' import {ArchiveClient} from '@subsquid/util-internal-archive-client' -import { - archiveIngest, - Batch, - DataSource, - PollingHeightTracker, - RangeRequest, - RangeRequestList -} from '@subsquid/util-internal-processor-tools' +import {archiveIngest, assertIsValid, IsInvalid} from '@subsquid/util-internal-ingest-tools' +import {Batch, DataSource} from '@subsquid/util-internal-processor-tools' +import {mapRangeRequestList, RangeRequestList} from '@subsquid/util-internal-range' import {DEFAULT_FIELDS, FieldSelection} from './interfaces/data' import {ArchiveBlock, ArchiveBlockHeader} from './interfaces/data-partial' import {DataRequest} from './interfaces/data-request' @@ -19,8 +14,6 @@ import {Block, BlockHeader, Call, Event, Extrinsic, setUpItems} from './mapping' interface ArchiveQuery extends DataRequest { type: 'substrate' - fromBlock: number - toBlock?: number } @@ -46,43 +39,43 @@ export class SubstrateArchive implements DataSource { return this.client.getHeight() } - getBlockHash(height: number): Promise { + getBlockHash(height: number): Promise { return this.rpc.getBlockHash(height) } - getFinalizedBlocks(requests: RangeRequestList, stopOnHead?: boolean): AsyncIterable> { + async *getFinalizedBlocks(requests: RangeRequestList, stopOnHead?: boolean): AsyncIterable> { let runtimeTracker = new RuntimeTracker( + this.rpc, hdr => ({height: hdr.number, hash: hdr.hash, parentHash: hdr.parentHash}), hdr => hdr, - this.rpc, this.typesBundle ) - return archiveIngest({ - requests, - heightTracker: new PollingHeightTracker(() => this.getFinalizedHeight(), 30_000), - query: async s => { - let blocks = await this.query(s) - await runtimeTracker.setRuntime(blocks.map(b => b.header)) - return blocks.map(b => this.mapBlock(b)) - }, - stopOnHead + let archiveRequests = mapRangeRequestList(requests, req => { + let {fields, ...items} = req + let q: ArchiveQuery = { + type: 'substrate', + fields: getFields(fields), + ...items + } + return q }) - } - private query(req: RangeRequest): Promise { - let {fields, ...items} = req.request - - let q: ArchiveQuery = { - type: 'substrate', - fromBlock: req.range.from, - toBlock: req.range.to, - fields: getFields(fields), - ...items + for await (let {blocks, isHead} of archiveIngest({ + client: this.client, + requests: archiveRequests, + stopOnHead + })) { + let headers: (ArchiveBlockHeader & IsInvalid)[] = blocks.map(b => b.header) + await runtimeTracker.setRuntime(headers) + assertIsValid(headers) + + yield { + blocks: blocks.map(b => this.mapBlock(b)), + isHead + } } - - return this.client.query(q) } @annotateSyncError((src: ArchiveBlock) => ({blockHeight: src.header.number, blockHash: src.header.hash})) diff --git a/substrate/substrate-processor/src/filter.ts b/substrate/substrate-processor/src/ds-rpc-filter.ts similarity index 55% rename from substrate/substrate-processor/src/filter.ts rename to substrate/substrate-processor/src/ds-rpc-filter.ts index 059484c81..da50e0c4d 100644 --- a/substrate/substrate-processor/src/filter.ts +++ b/substrate/substrate-processor/src/ds-rpc-filter.ts @@ -1,121 +1,18 @@ -import {RangeRequest, RequestsTracker} from '@subsquid/util-internal-processor-tools' +import {weakMemo} from '@subsquid/util-internal' +import {EntityFilter, FilterBuilder} from '@subsquid/util-internal-processor-tools' +import {getRequestAt, RangeRequest} from '@subsquid/util-internal-range' import {CallRelations, DataRequest, EventRelations} from './interfaces/data-request' import {Block, Call, Event, Extrinsic} from './mapping' -interface Filter { - match(obj: T): boolean -} - - -class Requests { - private requests: { - filter: Filter - relations: R - }[] = [] - - match(obj: T): R | undefined { - let relations: R | undefined - for (let req of this.requests) { - if (req.filter.match(obj)) { - relations = {...relations, ...req.relations} - } - } - return relations - } - - present(): boolean { - return this.requests.length > 0 - } - - add(filter: FilterBuilder, relations: R): void { - if (filter.isNever()) return - this.requests.push({ - filter: filter.build(), - relations - }) - } -} - - -class AndFilter implements Filter { - constructor(private filters: Filter[]) {} - - match(obj: T): boolean { - for (let f of this.filters) { - if (!f.match(obj)) return false - } - return true - } -} - - -const OK: Filter = { - match(obj: unknown): boolean { - return true - } -} - - -class PropInFilter implements Filter { - private values: Set - - constructor(private prop: P, values: T[P][]) { - this.values = new Set(values) - } - - match(obj: T): boolean { - return this.values.has(obj[this.prop]) - } -} - - -class PropEqFilter implements Filter { - constructor(private prop: P, private value: T[P]) {} - - match(obj: T): boolean { - return obj[this.prop] === this.value - } -} - - -class FilterBuilder { - private filters: Filter[] = [] - private never = false - - propIn

(prop: P, values?: T[P][]): this { - if (values == null) return this - if (values.length == 0) { - this.never = true - } - let filter = values.length == 1 - ? new PropEqFilter(prop, values[0]) - : new PropInFilter(prop, values) - this.filters.push(filter) - return this - } - - isNever(): boolean { - return this.never - } - - build(): Filter { - switch(this.filters.length) { - case 0: return OK - case 1: return this.filters[0] - default: return new AndFilter(this.filters) - } - } -} - -function buildCallRequests(dataRequest: DataRequest): Requests { - let requests = new Requests() +function buildCallFilter(dataRequest: DataRequest): EntityFilter { + let calls = new EntityFilter() dataRequest.calls?.forEach(req => { let {name, ...relations} = req let filter = new FilterBuilder() filter.propIn('name', name) - requests.add(filter, relations) + calls.add(filter, relations) }) dataRequest.ethereumTransactions?.forEach(req => { @@ -123,21 +20,21 @@ function buildCallRequests(dataRequest: DataRequest): Requests() filter.propIn('_ethereumTransactTo', to) filter.propIn('_ethereumTransactSighash', sighash) - requests.add(filter, relations) + calls.add(filter, relations) }) - return requests + return calls } -function buildEventRequests(dataRequest: DataRequest): Requests { - let requests = new Requests +function buildEventFilter(dataRequest: DataRequest): EntityFilter { + let events = new EntityFilter dataRequest.events?.forEach(req => { let {name, ...relations} = req let filter = new FilterBuilder() filter.propIn('name', name) - requests.add(filter, relations) + events.add(filter, relations) }) dataRequest.evmLogs?.forEach(req => { @@ -148,14 +45,14 @@ function buildEventRequests(dataRequest: DataRequest): Requests { let {contractAddress, ...relations} = req let filter = new FilterBuilder() filter.propIn('_contractAddress', contractAddress) - requests.add(filter, relations) + events.add(filter, relations) }) dataRequest.gearMessagesQueued?.forEach(req => { @@ -163,7 +60,7 @@ function buildEventRequests(dataRequest: DataRequest): Requests() filter.propIn('name', ['Gear.MessageQueued']) filter.propIn('_gearProgramId', programId) - requests.add(filter, relations) + events.add(filter, relations) }) dataRequest.gearUserMessagesSent?.forEach(req => { @@ -171,33 +68,19 @@ function buildEventRequests(dataRequest: DataRequest): Requests() filter.propIn('name', ['Gear.UserMessageSent']) filter.propIn('_gearProgramId', programId) - requests.add(filter, relations) + events.add(filter, relations) }) - return requests -} - - -interface ItemRequests { - events: Requests - calls: Requests + return events } -const ITEM_REQUESTS = new WeakMap - - -function getItemRequests(dataRequest: DataRequest): ItemRequests { - let items = ITEM_REQUESTS.get(dataRequest) - if (items == null) { - items = { - calls: buildCallRequests(dataRequest), - events: buildEventRequests(dataRequest) - } - ITEM_REQUESTS.set(dataRequest, items) +const getItemFilter = weakMemo((dataRequest: DataRequest) => { + return { + calls: buildCallFilter(dataRequest), + events: buildEventFilter(dataRequest) } - return items -} +}) class IncludeSet { @@ -233,12 +116,13 @@ class IncludeSet { function filterBlock(block: Block, dataRequest: DataRequest): void { - let req = getItemRequests(dataRequest) + let items = getItemFilter(dataRequest) + let include = new IncludeSet() - if (req.events.present()) { + if (items.events.present()) { for (let event of block.events) { - let rel = req.events.match(event) + let rel = items.events.match(event) if (rel == null) continue include.addEvent(event) if (rel.stack) { @@ -252,9 +136,9 @@ function filterBlock(block: Block, dataRequest: DataRequest): void { } } - if (req.calls.present()) { + if (items.calls.present()) { for (let call of block.calls) { - let rel = req.calls.match(call) + let rel = items.calls.match(call) if (rel == null) continue include.addCall(call) if (rel.events) { @@ -294,13 +178,25 @@ function filterBlock(block: Block, dataRequest: DataRequest): void { call.events = call.events.filter(event => include.events.has(event)) return true }) + + block.extrinsics = block.extrinsics.filter(ex => { + if (!include.extrinsics.has(ex)) return false + if (ex.call && !include.calls.has(ex.call)) { + ex.call = undefined + } + ex.subcalls = ex.subcalls.filter(sub => include.calls.has(sub)) + ex.events = ex.events.filter(event => include.events.has(event)) + return true + }) } export function filterBlockBatch(requests: RangeRequest[], blocks: Block[]): void { - let requestsTracker = new RequestsTracker(requests) for (let block of blocks) { - let dataRequest = requestsTracker.getRequestAt(block.header.height) || {} + let dataRequest = getRequestAt(requests, block.header.height) || NO_DATA_REQUEST filterBlock(block, dataRequest) } } + + +const NO_DATA_REQUEST: DataRequest = {} diff --git a/substrate/substrate-processor/src/ds-rpc.ts b/substrate/substrate-processor/src/ds-rpc.ts index 1d8d779b5..6b313fe21 100644 --- a/substrate/substrate-processor/src/ds-rpc.ts +++ b/substrate/substrate-processor/src/ds-rpc.ts @@ -1,53 +1,44 @@ import type {RpcClient} from '@subsquid/rpc-client' import * as base from '@subsquid/substrate-data' import {OldSpecsBundle, OldTypesBundle} from '@subsquid/substrate-runtime' -import {annotateSyncError, AsyncQueue, ensureError} from '@subsquid/util-internal' +import {annotateSyncError} from '@subsquid/util-internal' import {toJSON} from '@subsquid/util-internal-json' -import { - Batch, - HotDatabaseState, - HotDataSource, - HotUpdate, - RangeRequest, - RangeRequestList -} from '@subsquid/util-internal-processor-tools' -import {filterBlockBatch} from './filter' +import {Batch, HotDatabaseState, HotDataSource, HotUpdate} from '@subsquid/util-internal-processor-tools' +import {mapRangeRequestList, RangeRequestList} from '@subsquid/util-internal-range' +import {filterBlockBatch} from './ds-rpc-filter' import {DataRequest} from './interfaces/data-request' import {Block, BlockHeader, Call, Event, Extrinsic, setUpItems} from './mapping' export interface RpcDataSourceOptions { rpc: RpcClient - pollInterval?: number + headPollInterval?: number + newHeadTimeout?: number typesBundle?: OldTypesBundle | OldSpecsBundle } export class RpcDataSource implements HotDataSource { - private ds: base.RpcDataSource + private baseDataSource: base.RpcDataSource constructor(options: RpcDataSourceOptions) { - this.ds = new base.RpcDataSource({ - rpc: options.rpc, - pollInterval: options.pollInterval, - typesBundle: options.typesBundle - }) + this.baseDataSource = new base.RpcDataSource(options) } - getBlockHash(height: number): Promise { - return this.ds.getBlockHash(height) + getBlockHash(height: number): Promise { + return this.baseDataSource.getBlockHash(height) } getFinalizedHeight(): Promise { - return this.ds.getFinalizedHeight() + return this.baseDataSource.getFinalizedHeight() } async *getFinalizedBlocks( requests: RangeRequestList, stopOnHead?: boolean ): AsyncIterable> { - for await (let batch of this.ds.getFinalizedBlocks( - requests.map(toBaseRangeRequest), + for await (let batch of this.baseDataSource.getFinalizedBlocks( + mapRangeRequestList(requests, toBaseDataRequest), stopOnHead )) { let blocks = batch.blocks.map(b => this.mapBlock(b)) @@ -59,33 +50,20 @@ export class RpcDataSource implements HotDataSource { } } - async *getHotBlocks( + async processHotBlocks( requests: RangeRequestList, - state: HotDatabaseState - ): AsyncIterable> { - let queue = new AsyncQueue | Error>(1) - - this.ds.processHotBlocks( - requests.map(toBaseRangeRequest), + state: HotDatabaseState, + cb: (upd: HotUpdate) => Promise + ): Promise { + return this.baseDataSource.processHotBlocks( + mapRangeRequestList(requests, toBaseDataRequest), state, - upd => queue.put(upd) - ).then( - () => queue.close(), - err => queue.put(ensureError(err)).catch(err => {}) - ) - - for await (let upd of queue.iterate()) { - if (upd instanceof Error) { - throw upd - } else { + upd => { let blocks = upd.blocks.map(b => this.mapBlock(b)) filterBlockBatch(requests, blocks) - yield { - ...upd, - blocks - } + return cb({...upd, blocks}) } - } + ) } @annotateSyncError((src: base.Block) => ({blockHeight: src.header.height, blockHash: src.header.hash})) @@ -180,14 +158,6 @@ export class RpcDataSource implements HotDataSource { } -function toBaseRangeRequest(req: RangeRequest): RangeRequest { - return { - range: req.range, - request: toBaseDataRequest(req.request) - } -} - - function toBaseDataRequest(req: DataRequest): base.DataRequest { let events = !!req.events?.length || !!req.evmLogs?.length diff --git a/substrate/substrate-processor/src/mapping.ts b/substrate/substrate-processor/src/mapping.ts index 45d19411e..f6d760a8f 100644 --- a/substrate/substrate-processor/src/mapping.ts +++ b/substrate/substrate-processor/src/mapping.ts @@ -1,7 +1,7 @@ import {Bytes, ExtrinsicSignature, Hash, QualifiedName} from '@subsquid/substrate-data' import {Runtime} from '@subsquid/substrate-runtime' -import {assertNotNull} from '@subsquid/util-internal' -import {HashAndHeight} from '@subsquid/util-internal-processor-tools' +import {assertNotNull, maybeLast} from '@subsquid/util-internal' +import {formatId} from '@subsquid/util-internal-processor-tools' import {ParentBlockHeader} from './interfaces/data' import {PartialBlockHeader} from './interfaces/data-partial' @@ -382,28 +382,40 @@ export function setUpItems(block: Block): void { block.extrinsics.sort((a, b) => a.index - b.index) block.calls.sort(callCompare) - let extrinsicsByIndex = new Map(block.extrinsics.map(ex => [ex.index, ex])) + let extrinsics: (Extrinsic | undefined)[] = new Array((maybeLast(block.extrinsics)?.index ?? -1) + 1) + for (let rec of block.extrinsics) { + extrinsics[rec.index] = rec + } - for (let i = 0; i < block.calls.length; i++) { - let call = block.calls[i] - let extrinsic = extrinsicsByIndex.get(call.extrinsicIndex) + for (let i = block.calls.length - 1; i >= 0; i--) { + let rec = block.calls[i] + let extrinsic = extrinsics[rec.extrinsicIndex] if (extrinsic) { - if (call.address.length == 0) { - extrinsic.call = call + if (rec.address.length == 0) { + extrinsic.call = rec + } + rec.extrinsic = extrinsic + extrinsic.subcalls.push(rec) + } + + if (i < block.calls.length - 1) { + let prev = block.calls[i + 1] + if (isSubcall(prev, rec)) { + rec.parentCall = prev + populateSubcalls(prev, rec) } - call.extrinsic = extrinsic - extrinsic.subcalls.push(call) } - setUpCallTree(block.calls, i) } for (let event of block.events) { if (event.extrinsicIndex == null) continue - let extrinsic = extrinsicsByIndex.get(event.extrinsicIndex) + + let extrinsic = extrinsics[event.extrinsicIndex] if (extrinsic) { extrinsic.events.push(event) event.extrinsic = extrinsic } + if (event.callAddress && block.calls.length) { let pos = bisectCalls(block.calls, event.extrinsicIndex, event.callAddress) for (let i = pos; i < block.calls.length; i++) { @@ -441,21 +453,11 @@ function bisectCalls(calls: Call[], extrinsicIndex: number, callAddress: number[ } -function setUpCallTree(calls: Call[], pos: number): void { - let offset = -1 - let parent = calls[pos] - for (let i = pos - 1; i >= 0; i--) { - if (isSubcall(parent, calls[i])) { - if (calls[i].address.length == parent.address.length + 1) { - calls[i].parentCall = parent - } - offset = i - } else { - break - } +function populateSubcalls(parent: Call | undefined, child: Call): void { + while (parent) { + parent.subcalls.push(child) + parent = parent.parentCall } - if (offset < 0) return - parent.subcalls = calls.slice(offset, pos) } @@ -469,7 +471,7 @@ function addressCompare(a: number[], b: number[]): number { let order = a[i] - b[i] if (order) return order } - return b.length - a.length + return b.length - a.length // this differs from EVM call ordering } @@ -484,16 +486,3 @@ function isSubcall(parent: CallKey, call: CallKey): boolean { } return true } - - -function formatId(block: HashAndHeight, ...address: number[]): string { - let no = block.height.toString().padStart(10, '0') - let hash = block.hash.startsWith('0x') - ? block.hash.slice(2, 7) - : block.hash.slice(0, 5) - let id = `${no}-${hash}` - for (let index of address) { - id += '-' + index.toString().padStart(6, '0') - } - return id -} diff --git a/substrate/substrate-processor/src/processor.ts b/substrate/substrate-processor/src/processor.ts index 0dc169d29..ce27e62ea 100644 --- a/substrate/substrate-processor/src/processor.ts +++ b/substrate/substrate-processor/src/processor.ts @@ -13,17 +13,8 @@ import { } from '@subsquid/substrate-runtime/lib/metadata/old/typesBundle-polkadotjs' import {assertNotNull, def, runProgram} from '@subsquid/util-internal' import {ArchiveClient} from '@subsquid/util-internal-archive-client' -import { - applyRangeBound, - Batch, - Database, - getOrGenerateSquidId, - mergeRangeRequests, - PrometheusServer, - Range, - RangeRequest, - Runner -} from '@subsquid/util-internal-processor-tools' +import {Batch, Database, getOrGenerateSquidId, PrometheusServer, Runner} from '@subsquid/util-internal-processor-tools' +import {applyRangeBound, mergeRangeRequests, Range, RangeRequest} from '@subsquid/util-internal-range' import assert from 'assert' import {Chain} from './chain' import {SubstrateArchive} from './ds-archive' @@ -41,24 +32,74 @@ import { } from './interfaces/data-request' -export interface DataSource { +export interface RpcEndpointSettings { /** - * Subsquid archive endpoint URL + * RPC endpoint URL (either http(s) or ws(s)) */ - archive?: string + url: string /** - * Chain node RPC endpoint URL + * Maximum number of ongoing concurrent requests + */ + capacity?: number + /** + * Maximum number of requests per second + */ + rateLimit?: number + /** + * Request timeout in `ms` + */ + requestTimeout?: number + /** + * Maximum number of requests in a single batch call + */ + maxBatchCallSize?: number +} + + +export interface RpcDataIngestionSettings { + /** + * Poll interval for new blocks in `ms` + * + * Poll mechanism is used to get new blocks via HTTP connection. + */ + headPollInterval?: number + /** + * When websocket subscription is used to get new blocks, + * this setting specifies timeout in `ms` after which connection + * will be reset and subscription re-initiated if no new block where received. + */ + newHeadTimeout?: number + /** + * Disable RPC data ingestion entirely */ - chain: ChainRpc + disabled?: boolean } -type ChainRpc = string | { +export interface ArchiveSettings { + /** + * Subsquid archive URL + */ url: string - capacity?: number - rateLimit?: number + /** + * Request timeout in ms + */ requestTimeout?: number - maxBatchCallSize?: number +} + + +/** + * @deprecated + */ +export interface DataSource { + /** + * Subsquid archive endpoint URL + */ + archive?: string + /** + * Chain node RPC endpoint URL + */ + chain: string | RpcEndpointSettings } @@ -67,13 +108,25 @@ interface BlockRange { } +/** + * API and data that is passed to the data handler + */ export interface DataHandlerContext { /** * @internal */ _chain: Chain + /** + * An instance of a structured logger. + */ log: Logger + /** + * Storage interface provided by the database + */ store: Store + /** + * List of blocks to map and process + */ blocks: Block[] /** * Signals, that the processor reached the head of a chain. @@ -94,75 +147,139 @@ export class SubstrateBatchProcessor { private requests: RangeRequest[] = [] private fields?: FieldSelection private blockRange?: Range - private src?: DataSource - private chainPollInterval?: number + private archive?: ArchiveSettings + private rpcEndpoint?: RpcEndpointSettings + private rpcIngestSettings?: RpcDataIngestionSettings private typesBundle?: OldTypesBundle | OldSpecsBundle private prometheus = new PrometheusServer() private running = false - private _useArchiveOnly = false - - private add(request: DataRequest, range?: Range): void { - this.requests.push({ - range: range || {from: 0}, - request - }) - } /** - * Configure a set of fetched fields + * Set Subsquid Archive endpoint. + * + * Subsquid Archive allows to get data from finalized blocks up to + * infinite times faster and more efficient than via regular RPC. + * + * @example + * processor.setArchive('https://v2.archive.subsquid.io/network/kusama') */ - setFields(fields: T): SubstrateBatchProcessor { + setArchive(url: string | ArchiveSettings): this { this.assertNotRunning() - this.fields = fields - return this as any + if (typeof url == 'string') { + this.archive = {url} + } else { + this.archive = url + } + return this } - addEvent(options: EventRequest & BlockRange): this { + /** + * Set chain RPC endpoint + * + * @example + * // just pass a URL + * processor.setRpcEndpoint('https://kusama-rpc.polkadot.io') + * + * // adjust some connection options + * processor.setRpcEndpoint({ + * url: 'https://kusama-rpc.polkadot.io', + * rateLimit: 10 + * }) + */ + setRpcEndpoint(url: string | RpcEndpointSettings): this { this.assertNotRunning() - this.add({events: [options]}, options.range) + if (typeof url == 'string') { + this.rpcEndpoint = {url} + } else { + this.rpcEndpoint = url + } return this } - addCall(options: CallRequest & BlockRange): this { + /** + * Sets blockchain data source. + * + * @example + * processor.setDataSource({ + * archive: 'https://v2.archive.subsquid.io/network/kusama', + * chain: 'https://kusama-rpc.polkadot.io' + * }) + * + * @deprecated Use separate {@link .setArchive()} and {@link .setRpcEndpoint()} methods + * to specify data sources. + */ + setDataSource(src: DataSource): this { this.assertNotRunning() - this.add({calls: [options]}, options.range) + if (src.archive) { + this.setArchive(src.archive) + } else { + this.archive = undefined + } + if (src.chain) { + this.setRpcEndpoint(src.chain) + } else { + this.rpcEndpoint = undefined + } return this } - addEvmLog(options: EvmLogRequest & BlockRange): this { + /** + * Set up RPC data ingestion settings + */ + setRpcDataIngestionSettings(settings: RpcDataIngestionSettings): this { this.assertNotRunning() - this.add({evmLogs: [{ - ...options, - address: options.address?.map(s => s.toLowerCase()) - }]}, options.range) + this.rpcIngestSettings = settings return this } - addEthereumTransaction(options: EthereumTransactRequest & BlockRange): this { + /** + * @deprecated Use {@link .setRpcDataIngestionSettings()} instead + */ + setChainPollInterval(ms: number): this { + assert(ms >= 0) this.assertNotRunning() - this.add({ethereumTransactions: [{ - ...options, - to: options.to?.map(s => s.toLowerCase()) - }]}, options.range) + this.rpcIngestSettings = {...this.rpcIngestSettings, headPollInterval: ms} return this } - addContractsContractEmitted(options: ContractsContractEmittedRequest & BlockRange): this { + /** + * Never use RPC endpoint for data ingestion. + * + * @deprecated This is the same as `.setRpcDataIngestionSettings({disabled: true})` + */ + useArchiveOnly(yes?: boolean): this { this.assertNotRunning() - this.add({contractsEvents: [options]}, options.range) + this.rpcIngestSettings = {...this.rpcIngestSettings, disabled: true} return this } - addGearMessageQueued(options: GearMessageQueuedRequest & BlockRange): this { + + /** + * Limits the range of blocks to be processed. + * + * When the upper bound is specified, + * the processor will terminate with exit code 0 once it reaches it. + */ + setBlockRange(range?: Range): this { this.assertNotRunning() - this.add({gearMessagesQueued: [options]}, options.range) + this.blockRange = range return this } - addGearUserMessageSent(options: GearUserMessageSentRequest & BlockRange): this { + /** + * Configure a set of fetched fields + */ + setFields(fields: T): SubstrateBatchProcessor { this.assertNotRunning() - this.add({gearUserMessagesSent: [options]}, options.range) - return this + this.fields = fields + return this as any + } + + private add(request: DataRequest, range?: Range): void { + this.requests.push({ + range: range || {from: 0}, + request + }) } /** @@ -179,46 +296,51 @@ export class SubstrateBatchProcessor { return this } - /** - * Limits the range of blocks to be processed. - * - * When the upper bound is specified, - * the processor will terminate with exit code 0 once it reaches it. - */ - setBlockRange(range?: Range): this { + addEvent(options: EventRequest & BlockRange): this { this.assertNotRunning() - this.blockRange = range + this.add({events: [options]}, options.range) return this } - /** - * Sets blockchain data source. - * - * @example - * processor.setDataSource({ - * chain: 'wss://rpc.polkadot.io', - * archive: 'https://substrate.archive.subsquid.io/polkadot' - * }) - */ - setDataSource(src: DataSource): this { + addCall(options: CallRequest & BlockRange): this { this.assertNotRunning() - this.src = src + this.add({calls: [options]}, options.range) return this } - setChainPollInterval(ms: number): this { - assert(ms >= 0) + addEvmLog(options: EvmLogRequest & BlockRange): this { this.assertNotRunning() - this.chainPollInterval = ms + this.add({evmLogs: [{ + ...options, + address: options.address?.map(s => s.toLowerCase()) + }]}, options.range) return this } - /** - * Never use RPC endpoint as a data source - */ - useArchiveOnly(yes?: boolean): this { + addEthereumTransaction(options: EthereumTransactRequest & BlockRange): this { + this.assertNotRunning() + this.add({ethereumTransactions: [{ + ...options, + to: options.to?.map(s => s.toLowerCase()) + }]}, options.range) + return this + } + + addContractsContractEmitted(options: ContractsContractEmittedRequest & BlockRange): this { this.assertNotRunning() - this._useArchiveOnly = yes !== false + this.add({contractsEvents: [options]}, options.range) + return this + } + + addGearMessageQueued(options: GearMessageQueuedRequest & BlockRange): this { + this.assertNotRunning() + this.add({gearMessagesQueued: [options]}, options.range) + return this + } + + addGearUserMessageSent(options: GearUserMessageSentRequest & BlockRange): this { + this.assertNotRunning() + this.add({gearUserMessagesSent: [options]}, options.range) return this } @@ -284,21 +406,17 @@ export class SubstrateBatchProcessor { @def private getChainRpcClient(): RpcClient { - let options = this.src?.chain - if (options == null) { - throw new Error(`use .setDataSource() to specify chain RPC endpoint`) - } - if (typeof options == 'string') { - options = {url: options} + if (this.rpcEndpoint == null) { + throw new Error(`use .setRpcEndpoint() to specify chain RPC endpoint`) } let client = new RpcClient({ - url: options.url, - maxBatchCallSize: options.maxBatchCallSize ?? 100, - requestTimeout: options.requestTimeout ?? 30_000, - capacity: options.capacity ?? 10, - rateLimit: options.rateLimit, + url: this.rpcEndpoint.url, + maxBatchCallSize: this.rpcEndpoint.maxBatchCallSize ?? 100, + requestTimeout: this.rpcEndpoint.requestTimeout ?? 30_000, + capacity: this.rpcEndpoint.capacity ?? 10, + rateLimit: this.rpcEndpoint.rateLimit, retryAttempts: Number.MAX_SAFE_INTEGER, - log: this.getLogger().child('rpc', {rpcUrl: options.url}) + log: this.getLogger().child('rpc', {rpcUrl: this.rpcEndpoint.url}) }) this.prometheus.addChainRpcMetrics(() => client.getMetrics()) return client @@ -308,14 +426,15 @@ export class SubstrateBatchProcessor { private getRpcDataSource(): RpcDataSource { return new RpcDataSource({ rpc: this.getChainRpcClient(), - pollInterval: this.chainPollInterval, + headPollInterval: this.rpcIngestSettings?.headPollInterval, + newHeadTimeout: this.rpcIngestSettings?.newHeadTimeout, typesBundle: this.typesBundle }) } @def private getArchiveDataSource(): SubstrateArchive { - let url = assertNotNull(this.src?.archive) + let options = assertNotNull(this.archive) let log = this.getLogger().child('archive') @@ -326,11 +445,16 @@ export class SubstrateBatchProcessor { agent: new HttpAgent({ keepAlive: true }), - log: log.child('http') + log }) return new SubstrateArchive({ - client: new ArchiveClient({http, url, log}), + client: new ArchiveClient({ + http, + url: options.url, + queryTimeout: options.requestTimeout, + log + }), rpc: this.getChainRpcClient(), typesBundle: this.typesBundle }) @@ -420,14 +544,20 @@ export class SubstrateBatchProcessor { this.running = true let log = this.getLogger() runProgram(async () => { - if (this._useArchiveOnly && this.src?.archive == null) { - throw new Error('Archive URL is required when .useArchiveOnly() flag is set') + if (this.rpcEndpoint == null) { + throw new Error('Chain RPC endpoint is always required. Use .setRpcEndpoint() to specify it.') + } + if (this.rpcIngestSettings?.disabled && this.archive == null) { + throw new Error( + 'Archive is required when RPC data ingestion is disabled. ' + + 'Use .setArchive() to specify it.' + ) } return new Runner({ database, requests: this.getBatchRequests(), - archive: this.src?.archive == null ? undefined : this.getArchiveDataSource(), - hotDataSource: this._useArchiveOnly ? undefined : this.getRpcDataSource(), + archive: this.archive == null ? undefined : this.getArchiveDataSource(), + hotDataSource: this.rpcIngestSettings?.disabled ? undefined : this.getRpcDataSource(), process: (s, b) => this.processBatch(s, b as any, handler), prometheus: this.prometheus, log diff --git a/test/balances/src/processor.ts b/test/balances/src/processor.ts index 8796e7592..3608572dc 100644 --- a/test/balances/src/processor.ts +++ b/test/balances/src/processor.ts @@ -8,19 +8,17 @@ import {events} from './types' const processor = new SubstrateBatchProcessor() - .setDataSource({ - chain: 'https://kusama-rpc.polkadot.io', - archive: 'https://v2.archive.subsquid.io/network/kusama' - }) - .addEvent({ - name: [events.balances.transfer.name] - }) + .setArchive('https://v2.archive.subsquid.io/network/kusama') + .setRpcEndpoint(process.env.KUSAMA_NODE_WS || 'wss://kusama-rpc.polkadot.io') .setFields({ block: { timestamp: true } }) .setBlockRange({from: 19_666_100}) + .addEvent({ + name: [events.balances.transfer.name] + }) processor.run(new TypeormDatabase(), async ctx => { diff --git a/test/eth-usdc-transfers/.env b/test/erc20-transfers/.env similarity index 100% rename from test/eth-usdc-transfers/.env rename to test/erc20-transfers/.env diff --git a/test/eth-usdc-transfers/Makefile b/test/erc20-transfers/Makefile similarity index 100% rename from test/eth-usdc-transfers/Makefile rename to test/erc20-transfers/Makefile diff --git a/test/eth-usdc-transfers/db/migrations/1682961487386-Data.js b/test/erc20-transfers/db/migrations/1682961487386-Data.js similarity index 100% rename from test/eth-usdc-transfers/db/migrations/1682961487386-Data.js rename to test/erc20-transfers/db/migrations/1682961487386-Data.js diff --git a/test/eth-usdc-transfers/docker-compose.yml b/test/erc20-transfers/docker-compose.yml similarity index 100% rename from test/eth-usdc-transfers/docker-compose.yml rename to test/erc20-transfers/docker-compose.yml diff --git a/test/eth-usdc-transfers/erc20.json b/test/erc20-transfers/erc20.json similarity index 100% rename from test/eth-usdc-transfers/erc20.json rename to test/erc20-transfers/erc20.json diff --git a/test/eth-usdc-transfers/package.json b/test/erc20-transfers/package.json similarity index 94% rename from test/eth-usdc-transfers/package.json rename to test/erc20-transfers/package.json index 6f1fb12dc..bb5da9166 100644 --- a/test/eth-usdc-transfers/package.json +++ b/test/erc20-transfers/package.json @@ -1,5 +1,5 @@ { - "name": "eth-usdc-transfers", + "name": "erc20-transfers", "version": "0.0.0", "private": true, "scripts": { diff --git a/test/eth-usdc-transfers/schema.graphql b/test/erc20-transfers/schema.graphql similarity index 100% rename from test/eth-usdc-transfers/schema.graphql rename to test/erc20-transfers/schema.graphql diff --git a/test/eth-usdc-transfers/src/abi/abi.support.ts b/test/erc20-transfers/src/abi/abi.support.ts similarity index 100% rename from test/eth-usdc-transfers/src/abi/abi.support.ts rename to test/erc20-transfers/src/abi/abi.support.ts diff --git a/test/eth-usdc-transfers/src/abi/erc20.abi.ts b/test/erc20-transfers/src/abi/erc20.abi.ts similarity index 100% rename from test/eth-usdc-transfers/src/abi/erc20.abi.ts rename to test/erc20-transfers/src/abi/erc20.abi.ts diff --git a/test/eth-usdc-transfers/src/abi/erc20.ts b/test/erc20-transfers/src/abi/erc20.ts similarity index 100% rename from test/eth-usdc-transfers/src/abi/erc20.ts rename to test/erc20-transfers/src/abi/erc20.ts diff --git a/test/eth-usdc-transfers/src/model/generated/index.ts b/test/erc20-transfers/src/model/generated/index.ts similarity index 100% rename from test/eth-usdc-transfers/src/model/generated/index.ts rename to test/erc20-transfers/src/model/generated/index.ts diff --git a/test/eth-usdc-transfers/src/model/generated/marshal.ts b/test/erc20-transfers/src/model/generated/marshal.ts similarity index 100% rename from test/eth-usdc-transfers/src/model/generated/marshal.ts rename to test/erc20-transfers/src/model/generated/marshal.ts diff --git a/test/eth-usdc-transfers/src/model/generated/transfer.model.ts b/test/erc20-transfers/src/model/generated/transfer.model.ts similarity index 100% rename from test/eth-usdc-transfers/src/model/generated/transfer.model.ts rename to test/erc20-transfers/src/model/generated/transfer.model.ts diff --git a/test/eth-usdc-transfers/src/model/index.ts b/test/erc20-transfers/src/model/index.ts similarity index 100% rename from test/eth-usdc-transfers/src/model/index.ts rename to test/erc20-transfers/src/model/index.ts diff --git a/test/eth-usdc-transfers/src/processor.ts b/test/erc20-transfers/src/processor.ts similarity index 76% rename from test/eth-usdc-transfers/src/processor.ts rename to test/erc20-transfers/src/processor.ts index d1ae65008..c6b14ffc5 100644 --- a/test/eth-usdc-transfers/src/processor.ts +++ b/test/erc20-transfers/src/processor.ts @@ -4,22 +4,21 @@ import * as erc20 from './abi/erc20' import {Transfer} from './model' -const CONTRACT = '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48' +const CONTRACT = '0xFd086bC7CD5C481DCC9C85ebE478A1C0b69FCbb9'.toLowerCase() const processor = new EvmBatchProcessor() - .setDataSource({ - archive: 'https://v2.archive.subsquid.io/network/ethereum-mainnet', - chain: 'https://rpc.ankr.com/eth' + .setArchive('https://v2.archive.subsquid.io/network/arbitrum-one') + .setRpcEndpoint(process.env.ARB_NODE_WS) + .setFinalityConfirmation(500) + .setBlockRange({from: 150_000_000}) + .setFields({ + log: {transactionHash: true} }) .addLog({ address: [CONTRACT], topic0: [erc20.events.Transfer.topic] }) - .setFields({ - log: {transactionHash: true} - }) - .setFinalityConfirmation(50) processor.run(new TypeormDatabase({supportHotBlocks: true}), async ctx => { @@ -32,7 +31,7 @@ processor.run(new TypeormDatabase({supportHotBlocks: true}), async ctx => { transfers.push(new Transfer({ id: log.id, blockNumber: block.header.height, - timestamp: new Date(block.header.timestamp), + timestamp: new Date(block.header.timestamp * 1000), tx: log.transactionHash, from, to, diff --git a/test/eth-usdc-transfers/tsconfig.json b/test/erc20-transfers/tsconfig.json similarity index 100% rename from test/eth-usdc-transfers/tsconfig.json rename to test/erc20-transfers/tsconfig.json diff --git a/util/rpc-client/src/client.ts b/util/rpc-client/src/client.ts index 0edb3550e..ccef0c067 100644 --- a/util/rpc-client/src/client.ts +++ b/util/rpc-client/src/client.ts @@ -1,11 +1,12 @@ import {HttpError, HttpTimeoutError, isHttpConnectionError} from '@subsquid/http-client' import {createLogger, Logger} from '@subsquid/logger' -import {addErrorContext, last, splitParallelWork, wait} from '@subsquid/util-internal' +import {addErrorContext, def, last, splitParallelWork, wait} from '@subsquid/util-internal' import {Heap} from '@subsquid/util-internal-binary-heap' import assert from 'assert' import {RetryError, RpcConnectionError, RpcError} from './errors' -import {Connection, RpcRequest, RpcResponse} from './interfaces' +import {Connection, RpcCall, RpcErrorInfo, RpcNotification, RpcRequest, RpcResponse} from './interfaces' import {RateMeter} from './rate' +import {Subscription, SubscriptionHandle, Subscriptions} from './subscriptions' import {HttpConnection} from './transport/http' import {WsConnection} from './transport/ws' @@ -33,10 +34,12 @@ export interface CallOptions { * Otherwise, `client.call(...).then(validateResult)` is a better option. */ validateResult?: ResultValidator + validateError?: ErrorValidator } type ResultValidator = (result: any, req: RpcRequest) => R +type ErrorValidator = (info: RpcErrorInfo, req: RpcRequest) => R interface Req { @@ -47,6 +50,7 @@ interface Req { resolve(result: any): void reject(error: Error): void validateResult?: ResultValidator | undefined + validateError?: ErrorValidator | undefined } @@ -60,6 +64,7 @@ export class RpcClient { private retrySchedule: number[] private retryAttempts: number private capacity: number + private maxCapacity: number private log?: Logger private rate?: RateMeter private rateLimit: number = Number.MAX_SAFE_INTEGER @@ -67,14 +72,17 @@ export class RpcClient { private connectionErrorsInRow = 0 private connectionErrors = 0 private requestsServed = 0 + private notificationsReceived = 0 private backoffEpoch = 0 + private notificationListeners: ((msg: RpcNotification) => void)[] = [] + private resetListeners: ((reason: Error) => void)[] = [] private closed = false constructor(options: RpcClientOptions) { this.url = trimCredentials(options.url) this.con = this.createConnection(options.url) this.maxBatchCallSize = options.maxBatchCallSize ?? Number.MAX_SAFE_INTEGER - this.capacity = options.capacity ?? 10 + this.capacity = this.maxCapacity = options.capacity || 10 this.requestTimeout = options.requestTimeout ?? 0 this.retryAttempts = options.retryAttempts ?? 0 this.retrySchedule = options.retrySchedule ?? [10, 100, 500, 2000, 10000, 20000] @@ -97,7 +105,16 @@ export class RpcClient { switch(protocol) { case 'ws:': case 'wss:': - return new WsConnection(url) + return new WsConnection({ + url, + onNotificationMessage: msg => this.onNotification(msg), + onReset: reason => { + if (this.closed) return + for (let cb of this.resetListeners) { + this.safeCallback(cb, reason) + } + } + }) case 'http:': case 'https:': return new HttpConnection(url, this.log) @@ -106,14 +123,65 @@ export class RpcClient { } } + getConcurrency(): number { + return this.maxCapacity + } + getMetrics() { return { url: this.url, requestsServed: this.requestsServed, - connectionErrors: this.connectionErrors + connectionErrors: this.connectionErrors, + notificationsReceived: this.notificationsReceived + } + } + + private onNotification(msg: RpcNotification): void { + this.notificationsReceived += 1 + this.log?.debug({rpcMsg: msg}, 'rpc notification') + for (let cb of this.notificationListeners) { + this.safeCallback(cb, msg) } } + private safeCallback(cb: (arg: T) => void, arg: T): void { + try { + cb(arg) + } catch(err: any) { + this.log?.error(err, 'callback error') + } + } + + addNotificationListener(cb: (msg: RpcNotification) => void): void { + this.notificationListeners.push(cb) + } + + removeNotificationListener(cb: (msg: RpcNotification) => void): void { + removeItem(this.notificationListeners, cb) + } + + addResetListener(cb: (reason: Error) => void): void { + this.resetListeners.push(cb) + } + + removeResetListener(cb: (reason: Error) => void): void { + removeItem(this.resetListeners, cb) + } + + subscribe(sub: Subscription): SubscriptionHandle { + return this.subscriptions().add(sub) + } + + @def + private subscriptions(): Subscriptions { + assert(this.supportsNotifications(), 'subscriptions are only supported by websocket connections') + return new Subscriptions(this) + } + + supportsNotifications(): boolean { + return this.con instanceof WsConnection + } + call(method: string, params?: any[], options?: CallOptions): Promise { return new Promise((resolve, reject) => { let call: RpcRequest = { @@ -138,12 +206,13 @@ export class RpcClient { retryAttempts: options?.retryAttempts ?? this.retryAttempts, resolve, reject, - validateResult: options?.validateResult + validateResult: options?.validateResult, + validateError: options?.validateError }) }) } - batchCall(batch: {method: string, params?: any[]}[], options?: CallOptions): Promise { + batchCall(batch: RpcCall[], options?: CallOptions): Promise { return splitParallelWork( this.maxBatchCallSize, batch, @@ -151,7 +220,7 @@ export class RpcClient { ) } - private batchCallInternal(batch: {method: string, params?: any[]}[], options?: CallOptions): Promise { + private batchCallInternal(batch: RpcCall[], options?: CallOptions): Promise { if (batch.length == 0) return Promise.resolve([]) if (batch.length == 1) return this.call(batch[0].method, batch[0].params, options).then(res => [res]) return new Promise((resolve, reject) => { @@ -182,7 +251,8 @@ export class RpcClient { retryAttempts: options?.retryAttempts ?? this.retryAttempts, resolve, reject, - validateResult: options?.validateResult + validateResult: options?.validateResult, + validateError: options?.validateError }) }) } @@ -242,7 +312,7 @@ export class RpcClient { promise = this.con.batchCall(call, req.timeout).then(res => { let result = new Array(res.length) for (let i = 0; i < res.length; i++) { - result[i] = this.receiveResult(call[i], res[i], req.validateResult) + result[i] = this.receiveResult(call[i], res[i], req.validateResult, req.validateError) } return result }) @@ -250,7 +320,7 @@ export class RpcClient { let call = req.call this.log?.debug({rpcId: call.id}, 'rpc send') promise = this.con.call(call, req.timeout).then(res => { - return this.receiveResult(call, res, req.validateResult) + return this.receiveResult(call, res, req.validateResult, req.validateError) }) } promise.then(result => { @@ -313,7 +383,12 @@ export class RpcClient { return this.retrySchedule[idx] } - private receiveResult(call: RpcRequest, res: RpcResponse, validateResult: ResultValidator | undefined): any { + private receiveResult( + call: RpcRequest, + res: RpcResponse, + validateResult: ResultValidator | undefined, + validateError: ErrorValidator | undefined + ): any { if (this.log?.isDebug()) { this.log.debug({ rpcId: call.id, @@ -324,7 +399,11 @@ export class RpcClient { } try { if (res.error) { - throw new RpcError(res.error) + if (validateError) { + return validateError(res.error, call) + } else { + throw new RpcError(res.error) + } } else if (validateResult) { return validateResult(res.result, call) } else { @@ -359,6 +438,13 @@ export class RpcClient { return false } + reset(reason?: RpcConnectionError): void { + if (this.closed) return + if (this.con instanceof WsConnection) { + this.con.close(reason || new RpcConnectionError('client was reset')) + } + } + close(err?: Error) { if (this.closed) return this.closed = true @@ -404,3 +490,10 @@ function trimCredentials(url: string): string { function isRateLimitError(err: unknown): boolean { return err instanceof RpcError && /rate limit/i.test(err.message) } + + +function removeItem(arr: T[], item: T): void { + let index = arr.indexOf(item) + if (index < 0) return + arr.splice(index, 1) +} diff --git a/util/rpc-client/src/index.ts b/util/rpc-client/src/index.ts index 545eb17d9..cb39de5b5 100644 --- a/util/rpc-client/src/index.ts +++ b/util/rpc-client/src/index.ts @@ -1,2 +1,3 @@ export * from './errors' export * from './client' +export {Subscription, SubscriptionHandle} from './subscriptions' diff --git a/util/rpc-client/src/interfaces.ts b/util/rpc-client/src/interfaces.ts index deaacbf46..929fbded5 100644 --- a/util/rpc-client/src/interfaces.ts +++ b/util/rpc-client/src/interfaces.ts @@ -1,9 +1,19 @@ +export interface RpcCall { + method: string + params?: unknown[] +} -export interface RpcRequest { + +export interface RpcRequest extends RpcCall { id: number jsonrpc: '2.0' +} + + +export interface RpcNotification { + jsonrpc: '2.0' method: string - params?: unknown[] + params?: any } @@ -15,6 +25,9 @@ export interface RpcResponse { } +export type RpcIncomingMessage = RpcNotification | RpcResponse + + export interface RpcErrorInfo { code: number message: string diff --git a/util/rpc-client/src/subscriptions.ts b/util/rpc-client/src/subscriptions.ts new file mode 100644 index 000000000..393138531 --- /dev/null +++ b/util/rpc-client/src/subscriptions.ts @@ -0,0 +1,171 @@ +import {addErrorContext} from '@subsquid/util-internal' +import type {RpcClient} from './client' +import {RpcConnectionError, RpcError} from './errors' +import {RpcErrorInfo, RpcNotification, RpcRequest} from './interfaces' + + +export interface Subscription { + method: string + params?: unknown[] + notification: string + unsubscribe: string + onMessage: (msg: T) => void + onError: (err: Error) => void + resubscribeOnConnectionLoss?: boolean + retryAttempts?: number +} + + +export interface SubscriptionHandle { + readonly isActive: boolean + readonly isClosed: boolean + close(): void +} + + +type SubId = string + + +export class Subscriptions { + private active = new Map() + + constructor(private client: RpcClient) { + this.client.addNotificationListener(msg => this.onNotification(msg)) + this.client.addResetListener(err => this.onReset(err)) + } + + add(sub: Subscription): SubscriptionHandle { + return new Handle(sub, this.client, this.active) + } + + private onNotification(msg: RpcNotification): void { + let subscription: unknown = msg.params?.subscription + switch(typeof subscription) { + case 'number': + case 'string': + break + default: + return + } + let id = `${msg.method}::${subscription}` + let handle = this.active.get(id) + if (handle == null) return + + let params = msg.params as { + result: any + error?: undefined + } | { + result?: undefined + error: RpcErrorInfo + } + + if (params.error) { + let err = new RpcError(params.error) + handle.sub.onError(err) + } else { + handle.sub.onMessage(params.result) + } + } + + private onReset(err: Error): void { + for (let handle of this.active.values()) { + handle.onConnectionReset(err) + } + } +} + + +class Handle implements SubscriptionHandle { + private closed = false + private id?: SubId + + constructor( + public readonly sub: Subscription, + private client: RpcClient, + private active: Map + ) { + this.subscribe() + } + + get isActive(): boolean { + return !this.closed + } + + get isClosed(): boolean { + return this.closed + } + + close(): void { + if (this.closed) return + this.closed = true + this.unsubscribe() + } + + onConnectionReset(reason: Error): void { + if (this.closed) return + this.active.delete(this.id!) + this.id = undefined + if (reason instanceof RpcConnectionError && this.sub.resubscribeOnConnectionLoss) { + this.subscribe() + } else { + this.closed = true + this.sub.onError(reason) + } + } + + private subscribe(): void { + this.client.call(this.sub.method, this.sub.params, { + retryAttempts: this.sub.retryAttempts, + validateResult: (result, req) => this.validateSubscriptionResult(result, req) + }).then(id => { + this.id = id + if (this.isActive) { + this.active.set(this.id, this) + } else { + this.unsubscribe() + } + }, err => { + if (this.closed) return + this.closed = true + this.sub.onError(err) + }) + } + + private unsubscribe(): void { + if (this.id == null) return + this.active.delete(this.id) + this.client.call(this.sub.unsubscribe, [this.id], {retryAttempts: 0}).catch(err => { + if (err instanceof RpcConnectionError) return + this.client.reset( + addErrorContext( + new RpcConnectionError('connection was reset due to subscription cancellation error'), + {rpcSubscriptionCancellationError: err} + ) + ) + }) + } + + private validateSubscriptionResult(result: unknown, req: RpcRequest): SubId { + switch(typeof result) { + case 'string': + case 'number': + break + default: + this.client.reset() + throw addErrorContext( + new Error( + 'unexpected subscription result: ' + + 'only numbers and strings are accepted as subscription ids' + ), { + rpcResult: result + } + ) + } + let id = `${this.sub.notification}::${result}` + if (this.active.has(id)) { + this.client.reset() + throw new Error(`got duplicate subscription: ${result}`) + } + return id + } +} diff --git a/util/rpc-client/src/transport/ws.ts b/util/rpc-client/src/transport/ws.ts index 4114a594a..e8e184dd6 100644 --- a/util/rpc-client/src/transport/ws.ts +++ b/util/rpc-client/src/transport/ws.ts @@ -1,7 +1,7 @@ import assert from 'assert' import {w3cwebsocket as WebSocket} from 'websocket' import {RpcConnectionError, RpcProtocolError} from '../errors' -import {Connection, RpcRequest, RpcResponse} from '../interfaces' +import {Connection, RpcIncomingMessage, RpcNotification, RpcRequest, RpcResponse} from '../interfaces' const MB = 1024 * 1024 @@ -13,12 +13,26 @@ interface RequestHandle { } +export interface WsConnectionOptions { + url: string + onNotificationMessage?: (msg: RpcNotification) => void + onReset?: (err: Error) => void +} + + export class WsConnection implements Connection { + private url: string + private onNotificationMessage?: (msg: RpcNotification) => void + private onReset?: (err: Error) => void private _ws?: WebSocket private connected = false private requests = new Map() - constructor(private url: string) {} + constructor(options: WsConnectionOptions) { + this.url = options.url + this.onNotificationMessage = options.onNotificationMessage + this.onReset = options.onReset + } connect(): Promise { return new Promise((resolve, reject) => { @@ -74,6 +88,7 @@ export class WsConnection implements Connection { this.requests.clear() this._ws = undefined this.connected = false + this.onReset?.(err) } close(err?: Error): void { @@ -93,7 +108,7 @@ export class WsConnection implements Connection { if (typeof data != 'string') { throw new RpcProtocolError(1003, 'Received non-text frame') } - let msg: RpcResponse | RpcResponse[] + let msg: RpcIncomingMessage | RpcIncomingMessage[] try { msg = JSON.parse(data) } catch(e: any) { @@ -108,14 +123,18 @@ export class WsConnection implements Connection { } } - private handleResponse(res: RpcResponse): void { + private handleResponse(res: RpcIncomingMessage): void { // TODO: more strictness, more validation - let h = this.requests.get(res.id) - if (h == null) { - throw new RpcProtocolError(1008, `Got response for unknown request ${res.id}`) + if (isNotification(res)) { + this.onNotificationMessage?.(res) + } else { + let h = this.requests.get(res.id) + if (h == null) { + throw new RpcProtocolError(1008, `Got response for unknown request ${res.id}`) + } + this.requests.delete(res.id) + h.resolve(res) } - this.requests.delete(res.id) - h.resolve(res) } private ws(): WebSocket { @@ -193,3 +212,8 @@ class BatchItemHandle { this.handle.reject(err) } } + + +function isNotification(res: RpcResponse | RpcNotification): res is RpcNotification { + return typeof (res as any).method == 'string' +} diff --git a/util/util-internal-archive-client/package.json b/util/util-internal-archive-client/package.json index f5872e0b5..3417e1fd9 100644 --- a/util/util-internal-archive-client/package.json +++ b/util/util-internal-archive-client/package.json @@ -16,11 +16,11 @@ "build": "rm -rf lib && tsc" }, "dependencies": { - "@subsquid/http-client": "^1.3.1", "@subsquid/util-internal": "^2.5.2", "@subsquid/util-internal-range": "^0.0.1" }, "peerDependencies": { + "@subsquid/http-client": "^1.3.1", "@subsquid/logger": "^1.3.1" }, "peerDependenciesMeta": { @@ -29,6 +29,8 @@ } }, "devDependencies": { + "@subsquid/http-client": "^1.3.1", + "@subsquid/logger": "^1.3.1", "@types/node": "^18.18.0", "typescript": "~5.2.2" } diff --git a/util/util-internal-archive-client/src/client.ts b/util/util-internal-archive-client/src/client.ts index f27c0cbce..73fbc8b19 100644 --- a/util/util-internal-archive-client/src/client.ts +++ b/util/util-internal-archive-client/src/client.ts @@ -1,5 +1,5 @@ import {HttpClient, HttpTimeoutError} from '@subsquid/http-client' -import {Logger} from '@subsquid/logger' +import type {Logger} from '@subsquid/logger' import {wait, withErrorContext} from '@subsquid/util-internal' import assert from 'assert' @@ -10,11 +10,19 @@ export interface ArchiveQuery { } +export interface Block { + header: { + number: number + hash: string + } +} + + export interface ArchiveClientOptions { http: HttpClient url: string - log?: Logger queryTimeout?: number + log?: Logger } @@ -22,14 +30,14 @@ export class ArchiveClient { private url: URL private http: HttpClient private queryTimeout: number - private log?: Logger private retrySchedule = [5000, 10000, 20000, 30000, 60000] + private log?: Logger constructor(options: ArchiveClientOptions) { this.url = new URL(options.url) this.http = options.http - this.log = options.log this.queryTimeout = options.queryTimeout ?? 180_000 + this.log = options.log } private getRouterUrl(path: string): string { @@ -54,7 +62,7 @@ export class ArchiveClient { }) } - query(query: ArchiveQuery): Promise { + query(query: Q): Promise { return this.retry(async () => { let worker: string = await this.http.get(this.getRouterUrl(`${query.fromBlock}/worker`), { retryAttempts: 0, diff --git a/util/util-internal-ingest-tools/package.json b/util/util-internal-ingest-tools/package.json index 3eb6f4926..8a87b82d3 100644 --- a/util/util-internal-ingest-tools/package.json +++ b/util/util-internal-ingest-tools/package.json @@ -20,7 +20,16 @@ "@subsquid/util-internal": "^2.5.2", "@subsquid/util-internal-range": "^0.0.1" }, + "peerDependencies": { + "@subsquid/util-internal-archive-client": "^0.0.1" + }, + "peerDependenciesMeta": { + "@subsquid/util-internal-archive-client": { + "optional": true + } + }, "devDependencies": { + "@subsquid/util-internal-archive-client": "^0.0.1", "@types/node": "^18.18.0", "typescript": "~5.2.2" } diff --git a/util/util-internal-ingest-tools/src/archive.ts b/util/util-internal-ingest-tools/src/archive.ts new file mode 100644 index 000000000..b470c7d96 --- /dev/null +++ b/util/util-internal-ingest-tools/src/archive.ts @@ -0,0 +1,64 @@ +import {concurrentMap, last, Throttler} from '@subsquid/util-internal' +import type {ArchiveClient, Block} from '@subsquid/util-internal-archive-client' +import {RangeRequestList} from '@subsquid/util-internal-range' +import assert from 'assert' +import {Batch} from './interfaces' + + +export interface ArchiveIngestOptions { + client: ArchiveClient + requests: RangeRequestList + stopOnHead?: boolean + pollInterval?: number +} + + +export function archiveIngest(args: ArchiveIngestOptions): AsyncIterable> { + let { + client, + requests, + stopOnHead = false, + pollInterval = 20_000 + } = args + + let height = new Throttler(() => client.getHeight(), pollInterval) + + async function *ingest(): AsyncIterable> { + let top = await height.get() + for (let req of requests) { + let beg = req.range.from + let end = req.range.to ?? Infinity + while (beg <= end) { + if (top < beg) { + top = await height.get() + } + while (top < beg) { + if (stopOnHead) return + top = await height.call() + } + let blocks = await client.query({ + fromBlock: beg, + toBlock: req.range.to, + ...req.request + }) + assert(blocks.length > 0, 'boundary blocks are expected to be included') + let lastBlock = last(blocks).header.number + assert(lastBlock >= beg) + beg = lastBlock + 1 + if (beg > top) { + top = await height.get() + } + yield { + blocks, + isHead: beg > top + } + } + } + } + + return concurrentMap( + 2, + ingest(), + batch => Promise.resolve(batch) + ) +} diff --git a/util/util-internal-ingest-tools/src/cold.ts b/util/util-internal-ingest-tools/src/cold.ts new file mode 100644 index 000000000..6dcd5c68a --- /dev/null +++ b/util/util-internal-ingest-tools/src/cold.ts @@ -0,0 +1,76 @@ +import {concurrentMap, Throttler} from '@subsquid/util-internal' +import {RangeRequestList, splitRange, SplitRequest} from '@subsquid/util-internal-range' +import assert from 'assert' +import {Batch} from './interfaces' + + +export interface ColdIngestOptions { + getFinalizedHeight: () => Promise + getSplit: (req: SplitRequest) => Promise + requests: RangeRequestList + splitSize: number + concurrency: number + stopOnHead?: boolean + headPollInterval?: number +} + + +export function coldIngest(args: ColdIngestOptions): AsyncIterable> { + let { + getFinalizedHeight, + getSplit, + requests, + splitSize, + concurrency, + stopOnHead, + headPollInterval = 10_000 + } = args + + assert(splitSize >= 1) + assert(concurrency >= 1) + + let height = new Throttler(getFinalizedHeight, headPollInterval) + + async function *strides(): AsyncIterable<{split: SplitRequest, isHead: boolean}> { + let top = await height.get() + for (let req of requests) { + let beg = req.range.from + let end = req.range.to ?? Infinity + while (beg <= end) { + if (top < beg) { + top = await height.get() + } + while (top < beg) { + if (stopOnHead) return + top = await height.call() + } + for (let range of splitRange(splitSize, { + from: beg, + to: Math.min(top, end) + })) { + let split = { + range, + request: req.request + } + beg = range.to + 1 + if (beg > top) { + top = await height.get() + } + yield { + split, + isHead: beg > top + } + } + } + } + } + + return concurrentMap( + concurrency, + strides(), + async ({split, isHead}) => { + let blocks = await getSplit(split) + return {blocks, isHead} + } + ) +} diff --git a/util/util-internal-ingest-tools/src/consistency-error.ts b/util/util-internal-ingest-tools/src/consistency-error.ts new file mode 100644 index 000000000..aecf304d7 --- /dev/null +++ b/util/util-internal-ingest-tools/src/consistency-error.ts @@ -0,0 +1,20 @@ +import {BlockRef, getBlockName} from './ref' + + +export class DataConsistencyError extends Error { + get __sqd_data_consistency_error(): boolean { + return true + } +} + + +export class BlockConsistencyError extends DataConsistencyError { + constructor(ref: BlockRef) { + super(`Failed to fetch block ${getBlockName(ref)}, perhaps chain node navigated to another branch.`) + } +} + + +export function isDataConsistencyError(err: unknown): err is Error { + return err instanceof Error && !!(err as any).__sqd_data_consistency_error +} diff --git a/util/util-internal-ingest-tools/src/hot.ts b/util/util-internal-ingest-tools/src/hot.ts index 57913a132..4530316b1 100644 --- a/util/util-internal-ingest-tools/src/hot.ts +++ b/util/util-internal-ingest-tools/src/hot.ts @@ -1,46 +1,36 @@ -import {last, unexpectedCase} from '@subsquid/util-internal' +import {last} from '@subsquid/util-internal' import assert from 'assert' import {BlockHeader, Hash, HashAndHeight, HotState, HotUpdate} from './interfaces' - - -type AnyHead = HashAndHeight | number | Hash +import {BlockRef} from './ref' export interface ChainHeads { - best: AnyHead - finalized: AnyHead + best: BlockRef + finalized: BlockRef } export interface HotProcessorOptions { - state: HotState - process: (update: HotUpdate) => Promise - getBlock: (ref: Partial) => Promise - getHeader: (block: B) => BlockHeader - getBlockHeight?: (hash: Hash) => Promise - maxUpdateSize?: number + process(update: HotUpdate): Promise + getBlock(ref: HashAndHeight): Promise + /** + * This method must handle situations where `from > to`, + * in such cases `from` must be coerced to `to`. + */ + getBlockRange(from: number, to: BlockRef): AsyncIterable + getHeader(block: B): BlockHeader + getFinalizedBlockHeight?(hash: Hash): Promise } export class HotProcessor { + private o: HotProcessorOptions private chain: HashAndHeight[] - private process: (update: HotUpdate) => Promise - private getBlock: (ref: Partial) => Promise - private getHeader: (block: B) => BlockHeader - private maxUpdateSize: number - private getBlockHeight?: (hash: Hash) => Promise - private finalizedHead?: AnyHead - - constructor(options: HotProcessorOptions) { - this.chain = [options.state, ...options.state.top] - this.getBlock = options.getBlock - this.process = options.process - this.getHeader = options.getHeader - this.maxUpdateSize = options.maxUpdateSize ?? 10 - this.assertInvariants() - } + private finalizedHead?: BlockRef - private assertInvariants(): void { + constructor(state: HotState, options: HotProcessorOptions) { + this.o = options + this.chain = [state, ...state.top] for (let i = 1; i < this.chain.length; i++) { assert(this.chain[i].height == this.chain[i-1].height + 1) } @@ -51,87 +41,75 @@ export class HotProcessor { } getFinalizedHeight(): number { - return this.chain[0].height + return Math.max(this.chain[0].height, this.getPassedFinalizedHeight()) } - goto(heads: ChainHeads): Promise { - this.finalizedHead = heads.finalized - switch(typeof heads.best) { - case 'number': - return this.moveToHeight(heads.best) - case 'string': - return this.moveToHash(heads.best) - case 'object': - return this.moveToHead(heads.best) - default: - throw unexpectedCase() + private getPassedFinalizedHeight(): number { + if (this.finalizedHead == null) return 0 + if (this.finalizedHead.height == null) { + return this.chain.find(h => h.hash === this.finalizedHead?.hash)?.height ?? 0 + } else { + return this.finalizedHead.height } } - private async moveToHeight(height: number): Promise { - while (this.getHeight() < height) { - let nextHeight = Math.min(height, this.getHeight() + this.maxUpdateSize) - let block = await this.getBlock({height: nextHeight}) - await this.moveToBlock(block) + async goto(heads: ChainHeads): Promise { + if (this.isKnownBlock(heads.best)) return + this.finalizedHead = heads.finalized + for await (let blocks of this.o.getBlockRange(this.getHeight() + 1, heads.best)) { + await this.moveToBlocks(blocks) } } - private async moveToHash(hash: Hash): Promise { - if (this.chain.some(b => b.hash == hash)) return - let block = await this.getBlock({hash}) - let head = this.getHeader(block) - if (head.height > this.getHeight() + this.maxUpdateSize) { - await this.moveToHeight(head.height - this.maxUpdateSize) + private isKnownBlock(ref: BlockRef): boolean { + if (ref.height == null) { + return !!this.chain.find(b => b.hash === ref.hash) + } else { + if (ref.hash == null) return ref.height < this.getHeight() + let pos = ref.height - this.chain[0].height + return this.chain[pos]?.hash === ref.hash } - return this.moveToBlock(block) } - private async moveToHead(head: HashAndHeight): Promise { - if (head.height <= this.getHeight()) { - let pos = head.height - this.chain[0].height - assert(pos >= 0) - if (this.chain[pos].hash == head.hash) return - } - if (head.height > this.getHeight() + this.maxUpdateSize) { - await this.moveToHeight(head.height - this.maxUpdateSize) + private async moveToBlocks(blocks: B[]): Promise { + if (blocks.length == 0) return + + for (let i = 1; i < blocks.length; i++) { + assert(this.o.getHeader(blocks[i-1]).hash === this.o.getHeader(blocks[i]).parentHash) } - let block = await this.getBlock(head) - return this.moveToBlock(block) - } - private async moveToBlock(block: B): Promise { - let newBlocks = [block] - let head = getParent(this.getHeader(block)) + let newBlocks = blocks.slice().reverse() + let head = getParent(this.o.getHeader(blocks[0])) assert(head.height >= this.chain[0].height) let chain = this.chain.slice(0, head.height - this.chain[0].height + 1) while (last(chain).height < head.height) { - let block = await this.getBlock(head) + let block = await this.o.getBlock(head) newBlocks.push(block) - head = getParent(this.getHeader(block)) + head = getParent(this.o.getHeader(block)) } assert(last(chain).height === head.height) while (last(chain).hash !== head.hash) { - let block = await this.getBlock(head) + let block = await this.o.getBlock(head) newBlocks.push(block) - head = getParent(this.getHeader(block)) + head = getParent(this.o.getHeader(block)) chain.pop() } newBlocks = newBlocks.reverse() for (let block of newBlocks) { - chain.push(this.getHeader(block)) + chain.push(this.o.getHeader(block)) } chain = await this.finalize(chain) let baseHead = newBlocks.length - ? getParent(this.getHeader(newBlocks[0])) + ? getParent(this.o.getHeader(newBlocks[0])) : last(chain) - await this.process({ + await this.o.process({ baseHead, finalizedHead: chain[0], blocks: newBlocks @@ -143,20 +121,23 @@ export class HotProcessor { private async finalize(chain: HashAndHeight[]): Promise { if (this.finalizedHead == null) return chain - if (typeof this.finalizedHead == 'string') { - this.finalizedHead = chain.find(b => b.hash == this.finalizedHead) - || await this.getBlockRef(this.finalizedHead) - } + let finalizedHeight: number - let pos: number + if (this.finalizedHead.height == null) { + finalizedHeight = chain.find(b => b.hash == this.finalizedHead?.hash)?.height + || await this.getFinalizedBlockHeight(this.finalizedHead.hash) - if (typeof this.finalizedHead == 'object') { - pos = this.finalizedHead.height - chain[0].height - if (0 <= pos && pos < chain.length) { - assert(chain[pos].hash === this.finalizedHead.hash) + this.finalizedHead = { + height: finalizedHeight, + hash: this.finalizedHead.hash } } else { - pos = this.finalizedHead - chain[0].height + finalizedHeight = this.finalizedHead.height + } + + let pos = finalizedHeight - chain[0].height + if (this.finalizedHead.hash && 0 <= pos && pos < chain.length) { + assert(chain[pos].hash === this.finalizedHead.hash) } pos = Math.min(pos, chain.length - 1) @@ -168,14 +149,11 @@ export class HotProcessor { } } - private async getBlockRef(hash: Hash): Promise { - if (this.getBlockHeight) { - let height = await this.getBlockHeight(hash) - return {hash, height} - } else { - let block = await this.getBlock({hash}) - return this.getHeader(block) - } + private getFinalizedBlockHeight(blockHash: Hash): Promise { + if (this.o.getFinalizedBlockHeight == null) throw new Error( + `.getFinalizedBlockHeight() method is not available` + ) + return this.o.getFinalizedBlockHeight(blockHash) } } diff --git a/util/util-internal-ingest-tools/src/index.ts b/util/util-internal-ingest-tools/src/index.ts index 9bab2db47..b457ad686 100644 --- a/util/util-internal-ingest-tools/src/index.ts +++ b/util/util-internal-ingest-tools/src/index.ts @@ -1,3 +1,7 @@ +export * from './archive' +export * from './cold' +export * from './consistency-error' export * from './hot' export * from './interfaces' -export * from './requests-tracker' +export * from './invalid' +export * from './ref' diff --git a/util/util-internal-ingest-tools/src/invalid.ts b/util/util-internal-ingest-tools/src/invalid.ts new file mode 100644 index 000000000..1ccfe3e48 --- /dev/null +++ b/util/util-internal-ingest-tools/src/invalid.ts @@ -0,0 +1,27 @@ +import {BlockConsistencyError} from './consistency-error' +import {BlockRef} from './ref' + + +export interface IsInvalid { + _isInvalid?: boolean +} + + +export function setInvalid(blocks: IsInvalid[], index?: number): void { + blocks[index || 0]._isInvalid = true +} + + +export function assertIsValid(blocks: (IsInvalid & BlockRef)[]): void { + for (let block of blocks) { + if (block._isInvalid) throw new BlockConsistencyError(block) + } +} + + +export function trimInvalid(blocks: B[]): B[] { + for (let i = 0; i < blocks.length; i++) { + if (blocks[i]._isInvalid) return blocks.slice(0, i) + } + return blocks +} diff --git a/util/util-internal-ingest-tools/src/ref.ts b/util/util-internal-ingest-tools/src/ref.ts new file mode 100644 index 000000000..6da60be1b --- /dev/null +++ b/util/util-internal-ingest-tools/src/ref.ts @@ -0,0 +1,30 @@ +import {HashAndHeight} from './interfaces' + + +export type BlockRef = HashAndHeight | { + height: number + hash?: undefined +} | { + height?: undefined + hash: string +} + + +export function getBlockName(ref: BlockRef): string { + if (ref.hash == null) { + return ''+ref.height + } else if (ref.height == null) { + return ref.hash + } else { + return `${ref.height}#${shortHash(ref.hash)}` + } +} + + +function shortHash(hash: string): string { + if (hash.startsWith('0x')) { + return hash.slice(2, 7) + } else { + return hash.slice(0, 5) + } +} diff --git a/util/util-internal-ingest-tools/src/requests-tracker.ts b/util/util-internal-ingest-tools/src/requests-tracker.ts deleted file mode 100644 index eb6fd0e04..000000000 --- a/util/util-internal-ingest-tools/src/requests-tracker.ts +++ /dev/null @@ -1,42 +0,0 @@ -import {RangeRequest} from '@subsquid/util-internal-range' - - -export class RequestsTracker { - constructor(private requests: RangeRequest[]) {} - - getRequestAt(height: number): R | undefined { - for (let req of this.requests) { - let from = req.range.from - let to = req.range.to ?? Infinity - if (from <= height && height <= to) return req.request - } - } - - hasRequestsAfter(height: number): boolean { - for (let req of this.requests) { - let to = req.range.to ?? Infinity - if (height < to) return true - } - return false - } - - *splitBlocksByRequest(blocks: B[], getBlockHeight: (b: B) => number): Iterable<{blocks: B[], request?: R}> { - let pack: B[] = [] - let packRequest: R | undefined = undefined - for (let b of blocks) { - let req = this.getRequestAt(getBlockHeight(b)) - if (req === packRequest) { - pack.push(b) - } else { - if (pack.length) { - yield {blocks: pack, request: packRequest} - } - pack = [b] - packRequest = req - } - } - if (pack.length) { - yield {blocks: pack, request: packRequest} - } - } -} diff --git a/util/util-internal-processor-tools/src/datasource.ts b/util/util-internal-processor-tools/src/datasource.ts index 8d91a36d4..c7bba419c 100644 --- a/util/util-internal-processor-tools/src/datasource.ts +++ b/util/util-internal-processor-tools/src/datasource.ts @@ -35,5 +35,9 @@ export interface DataSource { export interface HotDataSource extends DataSource { - getHotBlocks(requests: RangeRequestList, state: HotDatabaseState): AsyncIterable> + processHotBlocks( + requests: RangeRequestList, + state: HotDatabaseState, + cb: (upd: HotUpdate) => Promise + ): Promise } diff --git a/util/util-internal-processor-tools/src/filter.ts b/util/util-internal-processor-tools/src/filter.ts new file mode 100644 index 000000000..c22b24e6f --- /dev/null +++ b/util/util-internal-processor-tools/src/filter.ts @@ -0,0 +1,154 @@ + +export interface Filter { + match(obj: T): boolean +} + + +export class EntityFilter { + private requests: { + filter: Filter + relations: R + }[] = [] + + present(): boolean { + return this.requests.length > 0 + } + + match(obj: T): R | undefined { + let relations: R | undefined + for (let req of this.requests) { + if (req.filter.match(obj)) { + relations = this.mergeRelations(relations, req.relations) + } + } + return relations + } + + mergeRelations(a: R | undefined, b: R): R { + if (a == null) return b + let result = {...a} + let key: keyof R + for (key in b) { + if (b[key]) { + result[key] = b[key] + } + } + return result + } + + add(filter: Filter | FilterBuilder, relations: R): void { + if (filter instanceof FilterBuilder) { + if (filter.isNever()) return + filter = filter.build() + } + this.requests.push({ + filter, + relations + }) + } +} + + +export class FilterBuilder { + private filters: Filter[] = [] + private never = false + + propIn

(prop: P, values?: T[P][]): this { + if (values == null) return this + if (values.length == 0) { + this.never = true + } + let filter = values.length == 1 + ? new PropEqFilter(prop, values[0]) + : new PropInFilter(prop, values) + this.filters.push(filter) + return this + } + + getIn

(get: (obj: T) => P, values?: P[]): this { + if (values == null) return this + if (values.length == 0) { + this.never = true + } + let filter = values.length == 1 + ? new GetEqFilter(get, values[0]) + : new GetInFilter(get, values) + this.filters.push(filter) + return this + } + + isNever(): boolean { + return this.never + } + + build(): Filter { + switch(this.filters.length) { + case 0: return OK + case 1: return this.filters[0] + default: return new AndFilter(this.filters) + } + } +} + + +const OK: Filter = { + match(obj: unknown): boolean { + return true + } +} + + +class PropInFilter implements Filter { + public readonly values: Set + + constructor(public readonly prop: P, values: T[P][]) { + this.values = new Set(values) + } + + match(obj: T): boolean { + return this.values.has(obj[this.prop]) + } +} + + +class PropEqFilter implements Filter { + constructor(public readonly prop: P, public readonly value: T[P]) {} + + match(obj: T): boolean { + return obj[this.prop] === this.value + } +} + + +class GetInFilter implements Filter { + public readonly values: Set

+ + constructor(public readonly get: (obj: T) => P, values: P[]) { + this.values = new Set(values) + } + + match(obj: T): boolean { + return this.values.has(this.get(obj)) + } +} + + +class GetEqFilter implements Filter { + constructor(public readonly get: (obj: T) => P, public readonly value: P) {} + + match(obj: T): boolean { + return this.get(obj) === this.value + } +} + + +class AndFilter implements Filter { + constructor(public readonly filters: Filter[]) {} + + match(obj: T): boolean { + for (let f of this.filters) { + if (!f.match(obj)) return false + } + return true + } +} diff --git a/util/util-internal-processor-tools/src/index.ts b/util/util-internal-processor-tools/src/index.ts index 2f9d17224..b7de63903 100644 --- a/util/util-internal-processor-tools/src/index.ts +++ b/util/util-internal-processor-tools/src/index.ts @@ -1,7 +1,6 @@ -export * from '@subsquid/util-internal-range' export * from './database' export * from './datasource' -export * from './ingest' +export * from './filter' export * from './prometheus' export * from './runner' -export {getOrGenerateSquidId} from './util' +export {getOrGenerateSquidId, shortHash, formatId} from './util' diff --git a/util/util-internal-processor-tools/src/ingest.ts b/util/util-internal-processor-tools/src/ingest.ts deleted file mode 100644 index 9c7d99ea8..000000000 --- a/util/util-internal-processor-tools/src/ingest.ts +++ /dev/null @@ -1,322 +0,0 @@ -import {concurrentMap, last, wait} from '@subsquid/util-internal' -import {RangeRequest, splitRange, SplitRequest} from '@subsquid/util-internal-range' -import assert from 'assert' -import {HashAndHeight, HotDatabaseState} from './database' -import {Batch, Block, BlockHeader, HotUpdate} from './datasource' - - -export interface HeightTracker { - getHeight(): Promise - getLastHeight(): number - wait(height: number): Promise -} - - -export class PollingHeightTracker implements HeightTracker { - private lastAccess = 0 - private lastHeight = 0 - - constructor( - private height: () => Promise, - public readonly pollInterval: number - ) { - } - - async getHeight(): Promise { - let height = await this.height() - this.lastAccess = Date.now() - this.lastHeight = height - return height - } - - getLastHeight(): number { - return this.lastHeight - } - - async wait(height: number): Promise { - let current = this.lastHeight - while (current < height) { - let pause = this.pollInterval - Math.min(Date.now() - this.lastAccess, this.pollInterval) - if (pause) { - await wait(pause) - } - current = await this.getHeight() - } - return current - } -} - - -export async function* getHeightUpdates(heightTracker: HeightTracker, from: number): AsyncIterable { - while (true) { - yield from = await heightTracker.wait(from) - from += 1 - } -} - - -export async function* generateSplitRequests( - args: { - requests: RangeRequest[] - heightTracker: HeightTracker - stopOnHead?: boolean - } -): AsyncIterable> { - let top = args.heightTracker.getLastHeight() - for (let req of args.requests) { - let from = req.range.from - let end = req.range.to ?? Infinity - while (from <= end) { - if (top < from) { - top = await args.heightTracker.getHeight() - if (top < from) { - if (args.stopOnHead) return - top = await args.heightTracker.wait(from) - } - } - let to = Math.min(end, top) - yield { - range: {from, to}, - request: req.request - } - from = to + 1 - } - } -} - - -export async function* generateFetchStrides( - args: { - requests: RangeRequest[] - heightTracker: HeightTracker - strideSize?: number - stopOnHead?: boolean - } -): AsyncIterable> { - let {strideSize = 10, ...params} = args - for await (let s of generateSplitRequests(params)) { - for (let range of splitRange(strideSize, s.range)) { - yield { - range, - request: s.request - } - } - } -} - - -export function archiveIngest( - args: { - requests: RangeRequest[] - heightTracker: HeightTracker - query: (s: SplitRequest) => Promise - stopOnHead?: boolean - } -): AsyncIterable> { - let {query, ...params} = args - - async function* ingest(): AsyncIterable> { - for await (let s of generateSplitRequests(params)) { - let from = s.range.from - let to = s.range.to - while (from <= to) { - let blocks = await query({ - range: {from, to}, - request: s.request - }) - assert(blocks.length > 0, 'boundary blocks are expected to be included') - let top = last(blocks).header.height - yield {blocks, isHead: top >= args.heightTracker.getLastHeight()} - from = top + 1 - } - } - } - - return concurrentMap( - 2, - ingest(), - b => Promise.resolve(b) - ) -} - - -export interface ChainHeads { - best?: HashAndHeight | number | string - finalized?: HashAndHeight | number | string -} - - -export class ForkNavigator { - private chain: HashAndHeight[] - - constructor( - state: HotDatabaseState, - private getBlock: (ref: Partial) => Promise, - private getHeader: (block: B) => BlockHeader - ) { - this.chain = [state, ...state.top] - this.assertInvariants() - } - - private assertInvariants(): void { - for (let i = 1; i < this.chain.length; i++) { - assert(this.chain[i].height == this.chain[i-1].height + 1) - } - } - - getHeight(): number { - return last(this.chain).height - } - - async move(heads: ChainHeads): Promise> { - let chain = this.chain.slice() - let newBlocks: B[] = [] - let bestHead: HashAndHeight | undefined - - if (typeof heads.best == 'number') { - if (heads.best > last(chain).height) { - let newBlock = await this.getBlock({height: heads.best}) - newBlocks.push(newBlock) - bestHead = getParent(this.getHeader(newBlock)) - } - } else if (typeof heads.best == 'string') { - bestHead = chain.find(ref => ref.hash === heads.best) - bestHead = bestHead || await this.getBlock({hash: heads.best}).then(b => this.getHeader(b)) - } else { - bestHead = heads.best - } - - if (bestHead) { - assert(bestHead.height >= chain[0].height) - - if (last(chain).height > bestHead.height) { - let bestPos = bestHead.height - chain[0].height - if (chain[bestPos].hash === bestHead.hash) { - // no fork - } else { - // we have a proper fork - chain = chain.slice(0, bestPos) - } - } - - while (last(chain).height < bestHead.height) { - let block = await this.getBlock(bestHead) - newBlocks.push(block) - bestHead = getParent(this.getHeader(block)) - } - - while (last(chain).hash !== bestHead.hash) { - let block = await this.getBlock(bestHead) - newBlocks.push(block) - bestHead = getParent(this.getHeader(block)) - chain.pop() - } - } - - newBlocks = newBlocks.reverse() - for (let block of newBlocks) { - chain.push(this.getHeader(block)) - } - - let finalizedHead: HashAndHeight | undefined - if (typeof heads.finalized == 'number') { - assert(heads.finalized <= last(chain).height) - if (heads.finalized > chain[0].height) { - finalizedHead = chain[heads.finalized - chain[0].height] - assert(finalizedHead.height == heads.finalized) - } - } else if (typeof heads.finalized == 'string') { - finalizedHead = chain.find(ref => ref.hash === heads.finalized) - finalizedHead = finalizedHead || await this.getBlock({hash: heads.finalized}).then(b => this.getHeader(b)) - } else { - finalizedHead = heads.finalized - } - - if (finalizedHead && finalizedHead.height >= chain[0].height) { - assert(finalizedHead.height <= last(chain).height) - let finalizedHeadPos = finalizedHead.height - chain[0].height - assert(chain[finalizedHeadPos].hash === finalizedHead.hash) - chain = chain.slice(finalizedHeadPos) - } - - // don't change the state until no error is guaranteed to occur - this.chain = chain - - return { - blocks: newBlocks, - baseHead: newBlocks.length ? getParent(this.getHeader(newBlocks[0])) : last(this.chain), - finalizedHead: this.chain[0] - } - } - - async transact(cb: () => Promise): Promise { - let chain = this.chain - try { - return await cb() - } catch(err: any) { - this.chain = chain - throw err - } - } -} - - -function getParent(hdr: BlockHeader): HashAndHeight { - return { - hash: hdr.parentHash, - height: hdr.height - 1 - } -} - - -export class RequestsTracker { - constructor(private requests: RangeRequest[]) {} - - getRequestAt(height: number): R | undefined { - for (let req of this.requests) { - let from = req.range.from - let to = req.range.to ?? Infinity - if (from <= height && height <= to) return req.request - } - } - - hasRequestsAfter(height: number): boolean { - for (let req of this.requests) { - let to = req.range.to ?? Infinity - if (height < to) return true - } - return false - } - - *splitBlocksByRequest(blocks: B[], getBlockHeight: (b: B) => number): Iterable<{blocks: B[], request?: R}> { - let pack: B[] = [] - let packRequest: R | undefined = undefined - for (let b of blocks) { - let req = this.getRequestAt(getBlockHeight(b)) - if (req === packRequest) { - pack.push(b) - } else { - if (pack.length) { - yield {blocks: pack, request: packRequest} - } - pack = [b] - packRequest = req - } - } - if (pack.length) { - yield {blocks: pack, request: packRequest} - } - } - - async processBlocks( - blocks: I[], - getBlockHeight: (b: I) => number, - process: (blocks: I[], req?: R) => Promise - ): Promise { - let result: O[] = [] - for (let pack of this.splitBlocksByRequest(blocks, getBlockHeight)) { - result.push(...await process(pack.blocks, pack.request)) - } - return result - } -} diff --git a/util/util-internal-processor-tools/src/runner.ts b/util/util-internal-processor-tools/src/runner.ts index 363bc341a..f758a8627 100644 --- a/util/util-internal-processor-tools/src/runner.ts +++ b/util/util-internal-processor-tools/src/runner.ts @@ -190,35 +190,39 @@ export class Runner { let db = this.config.database let ds = assertNotNull(this.config.hotDataSource) let lastHead = maybeLast(state.top) || state + return ds.processHotBlocks( + this.getLeftRequests(state), + state, + async upd => { + let newHead = maybeLast(upd.blocks)?.header || upd.baseHead + if (upd.baseHead.hash !== lastHead.hash) { + this.log.info(`navigating a fork between ${formatHead(lastHead)} to ${formatHead(newHead)} with a common base ${formatHead(upd.baseHead)}`) + } - for await (let upd of ds.getHotBlocks(this.getLeftRequests(state), state)) { - let newHead = maybeLast(upd.blocks)?.header || upd.baseHead - if (upd.baseHead.hash !== lastHead.hash) { - this.log.info(`navigating a fork between ${formatHead(lastHead)} to ${formatHead(newHead)} with a common base ${formatHead(upd.baseHead)}`) - } - - this.log.debug({hotUpdate: upd}) + this.log.debug({hotUpdate: upd}) - await this.withProgressMetrics(upd.blocks, () => { - return db.transactHot({ - finalizedHead: upd.finalizedHead, - baseHead: upd.baseHead, - newBlocks: upd.blocks.map(b => b.header) - }, (store, ref) => { - let idx = ref.height - upd.baseHead.height - 1 - let block = upd.blocks[idx] + await this.withProgressMetrics(upd.blocks, () => { + return db.transactHot({ + finalizedHead: upd.finalizedHead, + baseHead: upd.baseHead, + newBlocks: upd.blocks.map(b => b.header) + }, (store, ref) => { + let idx = ref.height - upd.baseHead.height - 1 + let block = upd.blocks[idx] - assert.strictEqual(block.header.hash, ref.hash) - assert.strictEqual(block.header.height, ref.height) + assert.strictEqual(block.header.hash, ref.hash) + assert.strictEqual(block.header.height, ref.height) - return this.config.process(store, { - blocks: [block], - isHead: newHead.height === ref.height + return this.config.process(store, { + blocks: [block], + isHead: newHead.height === ref.height + }) }) }) - }) - lastHead = newHead - } + + lastHead = newHead + } + ) } private async withProgressMetrics(blocks: Block[], handler: () => R): Promise { diff --git a/util/util-internal-processor-tools/src/util.ts b/util/util-internal-processor-tools/src/util.ts index af361e15e..eefb97dd6 100644 --- a/util/util-internal-processor-tools/src/util.ts +++ b/util/util-internal-processor-tools/src/util.ts @@ -45,5 +45,25 @@ export function getItemsCount(blocks: any[]): number { export function formatHead(head: HashAndHeight): string { - return `${head.height}#${head.hash.slice(2, 10)}` + return `${head.height}#${shortHash(head.hash)}` +} + + +export function shortHash(hash: string): string { + if (hash.startsWith('0x')) { + return hash.slice(2, 7) + } else { + return hash.slice(0, 5) + } +} + + +export function formatId(block: HashAndHeight, ...address: number[]): string { + let no = block.height.toString().padStart(10, '0') + let hash = shortHash(block.hash) + let id = `${no}-${hash}` + for (let index of address) { + id += '-' + index.toString().padStart(6, '0') + } + return id } diff --git a/util/util-internal-range/src/index.ts b/util/util-internal-range/src/index.ts index d44af69b4..883d0ab7b 100644 --- a/util/util-internal-range/src/index.ts +++ b/util/util-internal-range/src/index.ts @@ -1,2 +1,3 @@ export * from './interfaces' export * from './util' +export * from './requests' diff --git a/util/util-internal-range/src/requests.ts b/util/util-internal-range/src/requests.ts new file mode 100644 index 000000000..ea1c7ad30 --- /dev/null +++ b/util/util-internal-range/src/requests.ts @@ -0,0 +1,57 @@ +import {assertNotNull, partitionBy} from '@subsquid/util-internal' +import {FiniteRange, RangeRequest} from './interfaces' +import {applyRangeBound} from './util' + + +export function getRequestAt(requests: RangeRequest[], height: number): R | undefined { + for (let req of requests) { + let from = req.range.from + let to = req.range.to ?? Infinity + if (from <= height && height <= to) return req.request + } +} + + +export function hasRequestsAfter(requests: RangeRequest[], height: number): boolean { + for (let req of requests) { + let to = req.range.to ?? Infinity + if (height < to) return true + } + return false +} + + +export function *splitBlocksByRequest( + requests: RangeRequest[], + blocks: B[], + getBlockHeight: (b: B) => number +): Iterable<{ + blocks: B[] + request?: R +}> { + for (let pack of partitionBy(blocks, b => getRequestAt(requests, getBlockHeight(b)))) { + yield { + blocks: pack.items, + request: pack.value + } + } +} + + +export function* splitRangeByRequest(requests: RangeRequest[], range: FiniteRange): Iterable<{ + range: FiniteRange, + request?: R +}> { + requests = applyRangeBound(requests, range) + for (let i = 0; i < requests.length; i++) { + let req = requests[i] as {range: FiniteRange, request: R} + if (i > 0) { + let from = assertNotNull(requests[i-1].range.to) + 1 + let to = req.range.from - 1 + if (from <= to) { + yield {range: {from, to}} + } + } + yield req + } +} diff --git a/util/util-internal-range/src/util.ts b/util/util-internal-range/src/util.ts index 15f125357..5ed9f4ea5 100644 --- a/util/util-internal-range/src/util.ts +++ b/util/util-internal-range/src/util.ts @@ -141,3 +141,13 @@ export function printRange(range?: Range): string { return `[${range?.from ?? 0})` } } + + +export function mapRangeRequestList(requests: RangeRequestList, f: (req: T) => R): RangeRequestList { + return requests.map(req => { + return { + range: req.range, + request: f(req.request) + } + }) +} diff --git a/util/util-internal-validation/package.json b/util/util-internal-validation/package.json new file mode 100644 index 000000000..cbe88f9a8 --- /dev/null +++ b/util/util-internal-validation/package.json @@ -0,0 +1,32 @@ +{ + "name": "@subsquid/util-internal-validation", + "version": "0.0.0", + "description": "JSON data validation framework", + "license": "GPL-3.0-or-later", + "repository": "git@github.com:subsquid/squid.git", + "publishConfig": { + "access": "public" + }, + "main": "lib/index.js", + "files": [ + "lib", + "src" + ], + "scripts": { + "build": "rm -rf lib && tsc" + }, + "dependencies": {}, + "peerDependencies": { + "@subsquid/logger": "^1.3.1" + }, + "peerDependenciesMeta": { + "@subsquid/logger": { + "optional": true + } + }, + "devDependencies": { + "@subsquid/logger": "^1.3.1", + "@types/node": "^18.18.0", + "typescript": "~5.2.2" + } +} diff --git a/util/util-internal-validation/src/composite/array.ts b/util/util-internal-validation/src/composite/array.ts new file mode 100644 index 000000000..4ce135087 --- /dev/null +++ b/util/util-internal-validation/src/composite/array.ts @@ -0,0 +1,37 @@ +import {ValidationFailure} from '../error' +import {Validator} from '../interface' + + +export class ArrayValidator implements Validator { + constructor(public readonly item: Validator) {} + + cast(array: unknown): ValidationFailure | T[] { + if (!Array.isArray(array)) return new ValidationFailure(array, `{value} is not an array`) + let result: any[] = new Array(array.length) + for (let i = 0; i < array.length; i++) { + let val = this.item.cast(array[i]) + if (val instanceof ValidationFailure) { + val.path.push(i) + return val + } else { + result[i] = val + } + } + return result + } + + validate(array: unknown): ValidationFailure | undefined { + if (!Array.isArray(array)) return new ValidationFailure(array, `{value} is not an array`) + for (let i = 0; i < array.length; i++) { + let err = this.item.validate(array[i]) + if (err) { + err.path.push(i) + return err + } + } + } + + phantom(): S[] { + return [] + } +} diff --git a/util/util-internal-validation/src/composite/constant.ts b/util/util-internal-validation/src/composite/constant.ts new file mode 100644 index 000000000..aff3f4b09 --- /dev/null +++ b/util/util-internal-validation/src/composite/constant.ts @@ -0,0 +1,34 @@ +import {ValidationFailure} from '../error' +import {Validator} from '../interface' +import {print} from '../util' + + +export class ConstantValidator implements Validator { + private message: string + + constructor( + public readonly value: T, + public readonly equals: (a: unknown, b: T) => boolean = strictEquals + ) { + this.message = `value {value} is not equal to ${print(this.value)}` + } + + cast(value: unknown): ValidationFailure | T { + if (this.equals(value, this.value)) return this.value + return new ValidationFailure(value, this.message) + } + + validate(value: unknown): ValidationFailure | undefined { + if (this.equals(value, this.value)) return + return new ValidationFailure(value, this.message) + } + + phantom(): T { + return this.value + } +} + + +function strictEquals(a: unknown, b: T): boolean { + return a === b +} diff --git a/util/util-internal-validation/src/composite/key-tagged-union.ts b/util/util-internal-validation/src/composite/key-tagged-union.ts new file mode 100644 index 000000000..4aa48e971 --- /dev/null +++ b/util/util-internal-validation/src/composite/key-tagged-union.ts @@ -0,0 +1,79 @@ +import {ValidationFailure} from '../error' +import {GetCastType, GetSrcType, Validator} from '../interface' +import {AddOptionToUndefined, Simplify, print} from '../util' + + +export type GetKeyTaggedUnionCast = Simplify<{ + [C in keyof U]: Simplify : undefined + }>> +}[keyof U]> + + +export type GetKeyTaggedUnionSrc = Simplify<{ + [C in keyof U]: AddOptionToUndefined<{ + [K in keyof U]: K extends C ? GetSrcType : undefined + }> +}[keyof U]> + + +export class KeyTaggedUnionValidator>> + implements Validator, GetKeyTaggedUnionSrc> +{ + private onlyOneOfMessage?: string + private noPropsMessage?: string + + constructor(public readonly union: U) { + } + + cast(value: any): ValidationFailure | GetKeyTaggedUnionCast { + let tag = this.getTag(value) + if (tag instanceof ValidationFailure) return tag + let val = this.union[tag].cast(value[tag]) + if (val instanceof ValidationFailure) { + val.path.push(tag) + return val + } + return {[tag]: val} as any + } + + validate(value: any): ValidationFailure | undefined { + let tag = this.getTag(value) + if (tag instanceof ValidationFailure) return tag + let err = this.union[tag].validate(value[tag]) + if (err) { + err.path.push(tag) + return err + } + } + + phantom(): GetKeyTaggedUnionSrc { + throw new Error() + } + + private getTag(value: any): string | ValidationFailure { + if (typeof value != 'object' || !value) return new ValidationFailure(value, '{value} is not an object') + let tag: string | undefined + for (let key in value) { + let validator = this.union[key] + if (validator == null) continue + if (tag == null) { + tag = key + } else { + return new ValidationFailure(value, this.getOnlyOneOfMessage()) + } + } + if (tag == null) return new ValidationFailure(value, this.getNoPropsMessage()) + return tag + } + + private getOnlyOneOfMessage(): string { + if (this.onlyOneOfMessage) return this.onlyOneOfMessage + return this.onlyOneOfMessage = `only one of ${print(Object.keys(this.union))} properties expected to be present in the object, but got {value}` + } + + private getNoPropsMessage(): string { + if (this.noPropsMessage) return this.noPropsMessage + return this.noPropsMessage = `expected an object with one of ${print(Object.keys(this.union))} properties, but got {value}` + } +} diff --git a/util/util-internal-validation/src/composite/nullable.ts b/util/util-internal-validation/src/composite/nullable.ts new file mode 100644 index 000000000..e92c8dc37 --- /dev/null +++ b/util/util-internal-validation/src/composite/nullable.ts @@ -0,0 +1,24 @@ +import {ValidationFailure} from '../error' +import {Validator} from '../interface' + + +export class NullableValidator implements Validator { + constructor(public readonly value: Validator) {} + + cast(value: unknown): ValidationFailure | T | null { + if (value === null) { + return null + } else { + return this.value.cast(value) + } + } + + validate(value: unknown): ValidationFailure | undefined { + if (value === null) return + return this.value.validate(value) + } + + phantom(): S | null { + return null + } +} diff --git a/util/util-internal-validation/src/composite/object.ts b/util/util-internal-validation/src/composite/object.ts new file mode 100644 index 000000000..f8b7f3897 --- /dev/null +++ b/util/util-internal-validation/src/composite/object.ts @@ -0,0 +1,50 @@ +import {ValidationFailure} from '../error' +import {GetCastType, GetSrcType, Validator} from '../interface' +import {AddOptionToUndefined, Simplify} from '../util' + + +export type GetPropsCast = Simplify +}>> + + +export type GetPropsSrc = Simplify +}>> + + +export class ObjectValidator> + implements Validator, GetPropsSrc> +{ + constructor(public readonly props: Props) {} + + cast(object: any): ValidationFailure | GetPropsCast { + if (typeof object != 'object' || !object) return new ValidationFailure(object, `{value} is not an object`) + let result: any = {} + for (let key in this.props) { + let val = this.props[key].cast(object[key]) + if (val === undefined) continue + if (val instanceof ValidationFailure) { + val.path.push(key) + return val + } + result[key] = val + } + return result + } + + validate(object: any): ValidationFailure | undefined { + if (typeof object != 'object' || !object) return new ValidationFailure(object, `{value} is not an object`) + for (let key in this.props) { + let err = this.props[key].validate(object[key]) + if (err) { + err.path.push(key) + return err + } + } + } + + phantom(): GetPropsSrc { + throw new Error() + } +} diff --git a/util/util-internal-validation/src/composite/one-of.ts b/util/util-internal-validation/src/composite/one-of.ts new file mode 100644 index 000000000..1dba22de8 --- /dev/null +++ b/util/util-internal-validation/src/composite/one-of.ts @@ -0,0 +1,89 @@ +import assert from 'assert' +import {ValidationFailure} from '../error' +import {GetCastType, GetSrcType, Validator} from '../interface' +import {Simplify} from '../util' + + +export type GetOneOfCast

= Simplify<{ + [K in keyof P]: GetCastType +}[keyof P]> + + +export type GetOneOfSrc

= Simplify<{ + [K in keyof P]: GetSrcType +}[keyof P]> + + +export class OneOfValidator

>> implements Validator, GetOneOfSrc

> { + private patternNames: string[] + private errors: (ValidationFailure | undefined)[] + + + constructor(public readonly patterns: P) { + this.patternNames = Object.keys(this.patterns) + this.errors = new Array(this.patternNames.length) + assert(this.patternNames.length > 1) + } + + private clearErrors(): void { + for (let i = 0; i < this.errors.length; i++) { + this.errors[i] = undefined + } + } + + cast(value: unknown): ValidationFailure | GetOneOfCast

{ + for (let i = 0; i < this.patternNames.length; i++) { + let key = this.patternNames[i] + let validator = this.patterns[key] + let result = validator.cast(value) + if (result instanceof ValidationFailure) { + this.errors[i] = result + } else { + this.clearErrors() + return result + } + } + let failure = new OneOfValidationFailure(value, this.errors.slice() as ValidationFailure[], this.patternNames) + this.clearErrors() + return failure + } + + validate(value: unknown): ValidationFailure | undefined { + for (let i = 0; i < this.patternNames.length; i++) { + let key = this.patternNames[i] + let validator = this.patterns[key] + let err = validator.validate(value) + if (err) { + this.errors[i] = err + } else { + this.clearErrors() + return + } + } + let failure = new OneOfValidationFailure(value, this.errors.slice() as ValidationFailure[], this.patternNames) + this.clearErrors() + return failure + } + + phantom(): GetOneOfSrc

{ + throw new Error() + } +} + + +export class OneOfValidationFailure extends ValidationFailure { + constructor(value: unknown, public errors: ValidationFailure[], public patternNames: string[]) { + super(value, 'given value does not match any of the expected patterns') + } + + toString(): string { + let msg = 'given value does not match any of the expected patterns:' + for (let i = 0; i < this.patternNames.length; i++) { + msg += `\n ${this.patternNames[i]}: ${this.errors[i].toString()}` + } + if (this.path.length) { + msg = `invalid value at ${this.getPathString()}: ${msg}` + } + return msg + } +} diff --git a/util/util-internal-validation/src/composite/option.ts b/util/util-internal-validation/src/composite/option.ts new file mode 100644 index 000000000..3c9eaf974 --- /dev/null +++ b/util/util-internal-validation/src/composite/option.ts @@ -0,0 +1,23 @@ +import {ValidationFailure} from '../error' +import {Validator} from '../interface' + + +export class OptionValidator implements Validator { + constructor(public readonly value: Validator) {} + + cast(value: unknown): ValidationFailure | T | undefined { + if (value == null) { + return undefined + } else { + return this.value.cast(value) + } + } + + validate(value: unknown): ValidationFailure | undefined { + if (value != null) return this.value.validate(value) + } + + phantom(): S | undefined | null { + throw new Error() + } +} diff --git a/util/util-internal-validation/src/composite/record.ts b/util/util-internal-validation/src/composite/record.ts new file mode 100644 index 000000000..19d3113d4 --- /dev/null +++ b/util/util-internal-validation/src/composite/record.ts @@ -0,0 +1,44 @@ +import {ValidationFailure} from '../error' +import {Validator} from '../interface' + + +export class RecordValidator implements Validator, Record> { + constructor( + public readonly key: Validator, + public readonly value: Validator + ) {} + + cast(record: unknown): ValidationFailure | Record { + if (typeof record != 'object' || !record) return new ValidationFailure(record, `{value} is not an object`) + let result: any = {} + for (let key in record) { + let k = this.key.cast(key) + if (k instanceof ValidationFailure) { + k.path.push(key) + return k + } + let v = this.value.cast((record as any)[key]) + if (v instanceof ValidationFailure) { + v.path.push(key) + return v + } + result[k] = v + } + return result + } + + validate(record: unknown): ValidationFailure | undefined { + if (typeof record != 'object' || !record) return new ValidationFailure(record, `{value} is not an object`) + for (let key in record) { + let err = this.key.validate(key) || this.value.validate((record as any)[key]) + if (err) { + err.path.push(key) + return err + } + } + } + + phantom(): Record { + throw new Error() + } +} diff --git a/util/util-internal-validation/src/composite/ref.ts b/util/util-internal-validation/src/composite/ref.ts new file mode 100644 index 000000000..73364204a --- /dev/null +++ b/util/util-internal-validation/src/composite/ref.ts @@ -0,0 +1,28 @@ +import {ValidationFailure} from '../error' +import {Validator} from '../interface' + + +export class RefValidator implements Validator { + private validator?: Validator + + constructor(private getter: () => Validator) {} + + getValidator(): Validator { + if (this.validator == null) { + this.validator = this.getter() + } + return this.validator + } + + cast(value: unknown): ValidationFailure | T { + return this.getValidator().cast(value) + } + + validate(value: unknown): ValidationFailure | undefined { + return this.getValidator().validate(value) + } + + phantom(): S { + throw new Error() + } +} diff --git a/util/util-internal-validation/src/composite/sentinel.ts b/util/util-internal-validation/src/composite/sentinel.ts new file mode 100644 index 000000000..41e727de7 --- /dev/null +++ b/util/util-internal-validation/src/composite/sentinel.ts @@ -0,0 +1,65 @@ +import type {Logger} from '@subsquid/logger' +import {ValidationFailure} from '../error' +import {Validator} from '../interface' + + +const SUPPRESSED_LABELS: { + [label: string]: boolean +} = initSuppressedLabels() + + +let LOG: Logger | undefined +try { + LOG = require('@subsquid/logger').createLogger('sqd:validation') +} catch(err: any) {} + + +function warn(label: string): void { + if (LOG == null || SUPPRESSED_LABELS[label]) return + SUPPRESSED_LABELS[label] = true + LOG.error( + `Sentinel value was used in place of ${label}. ` + + `This message will be printed only once. ` + + `To suppress it entirely set SQD_ALLOW_SENTINEL=${label} env variable. ` + + `Use commas (,) to separate multiple labels.` + ) +} + + +export class Sentinel implements Validator{ + constructor( + public readonly label: string, + public readonly value: T, + public readonly item: Validator + ) {} + + cast(value: unknown): ValidationFailure | T { + if (value == null) { + warn(this.label) + return this.value + } else { + return this.item.cast(value) + } + } + + validate(value: unknown): ValidationFailure | undefined { + if (value == null) return + return this.item.validate(value) + } + + phantom(): S | null | undefined { + return undefined + } +} + + +function initSuppressedLabels(): Record { + let rec: Record = {} + if (typeof process.env.SQD_ALLOW_SENTINEL == 'string') { + let labels = process.env.SQD_ALLOW_SENTINEL.split(',').map(l => l.trim()) + for (let l of labels) { + rec[l] = true + } + } + return rec +} diff --git a/util/util-internal-validation/src/composite/tagged-union.ts b/util/util-internal-validation/src/composite/tagged-union.ts new file mode 100644 index 000000000..fda6f26dc --- /dev/null +++ b/util/util-internal-validation/src/composite/tagged-union.ts @@ -0,0 +1,56 @@ +import {ValidationFailure} from '../error' +import {GetCastType, GetSrcType, Validator} from '../interface' +import {Simplify, print} from '../util' + + +export type GetTaggedUnionCast = Simplify<{ + [C in keyof U]: GetCastType & {[T in F]: C} +}[keyof U]> + + +export type GetTaggedUnionSrc = Simplify<{ + [C in keyof U]: GetSrcType & {[T in F]: C} +}[keyof U]> + + +export class TaggedUnion>> + implements Validator, GetTaggedUnionSrc> +{ + private wrongTagMessage: string + + constructor( + public readonly tagField: F, + public readonly variants: U + ) { + this.wrongTagMessage = `got {value}, but expected one of ${print(Object.keys(this.variants))}` + } + + cast(value: any): ValidationFailure | GetTaggedUnionCast { + let variant = this.getVariant(value) + if (variant instanceof ValidationFailure) return variant + let result = variant.cast(value) + if (result instanceof ValidationFailure) return result + result[this.tagField] = value[this.tagField] + return result + } + + validate(value: unknown): ValidationFailure | undefined { + let variant = this.getVariant(value) + if (variant instanceof ValidationFailure) return variant + return variant.validate(value) + } + + private getVariant(object: any): Validator| ValidationFailure { + if (typeof object != 'object' || !object) return new ValidationFailure(object, `{value} is not an object`) + let tag = object[this.tagField] + let variant = this.variants[tag] + if (variant) return variant + let failure = new ValidationFailure(tag, this.wrongTagMessage) + failure.path.push(this.tagField) + return failure + } + + phantom(): GetTaggedUnionSrc { + throw new Error() + } +} diff --git a/util/util-internal-validation/src/dsl.ts b/util/util-internal-validation/src/dsl.ts new file mode 100644 index 000000000..a707cc915 --- /dev/null +++ b/util/util-internal-validation/src/dsl.ts @@ -0,0 +1,116 @@ +import {ArrayValidator} from './composite/array' +import {ConstantValidator} from './composite/constant' +import {GetKeyTaggedUnionCast, GetKeyTaggedUnionSrc, KeyTaggedUnionValidator} from './composite/key-tagged-union' +import {NullableValidator} from './composite/nullable' +import {GetPropsCast, GetPropsSrc, ObjectValidator} from './composite/object' +import {GetOneOfCast, GetOneOfSrc, OneOfValidator} from './composite/one-of' +import {OptionValidator} from './composite/option' +import {RecordValidator} from './composite/record' +import {RefValidator} from './composite/ref' +import {Sentinel} from './composite/sentinel' +import {GetTaggedUnionCast, GetTaggedUnionSrc, TaggedUnion} from './composite/tagged-union' +import {DataValidationError, ValidationFailure} from './error' +import {GetCastType, GetSrcType, Validator} from './interface' + + +export function object | undefined>>( + props: Props +): Validator, GetPropsSrc> { + let presentProps: Record> = {} + for (let key in props) { + let v = props[key] + if (v) { + presentProps[key] = v + } + } + return new ObjectValidator(presentProps) as any +} + + +export function record, V extends Validator>( + key: K, + value: V +): Validator< + Record, GetCastType>, + Record, GetSrcType> +> { + return new RecordValidator(key, value) +} + + +export function taggedUnion>>( + field: F, + variants: U +): Validator, GetTaggedUnionSrc> { + return new TaggedUnion(field, variants) +} + + +export function keyTaggedUnion>>( + variants: U +): Validator, GetKeyTaggedUnionSrc> { + return new KeyTaggedUnionValidator(variants) +} + + +export function array>(item: V): Validator[], GetSrcType[]> { + return new ArrayValidator(item) +} + + +export function option>(item: V): Validator< + GetCastType | undefined, + GetSrcType | undefined | null +> { + return new OptionValidator(item) +} + + +export function nullable>(item: V): Validator< + GetCastType | null, + GetSrcType | null +> { + return new NullableValidator(item) +} + + +export function withSentinel>( + label: string, + value: GetCastType, + validator: V +): Validator, GetSrcType | undefined | null> { + return new Sentinel(label, value, validator) +} + + +export function ref>(get: () => V): Validator, GetSrcType> { + return new RefValidator(get) +} + + +export function oneOf

>>( + patterns: P +): Validator, GetOneOfSrc

> { + return new OneOfValidator(patterns) +} + + +export function constant(value: T, equals?: (a: unknown, b: T) => boolean) { + return new ConstantValidator(value, equals) +} + + +export function cast>(validator: V, value: unknown): GetCastType { + let result = validator.cast(value) + if (result instanceof ValidationFailure) throw new DataValidationError(result.toString()) + return result +} + + +export function assertValidity>( + validator: V, + value: unknown +): asserts value is GetSrcType { + let err = validator.validate(value) + if (err) throw new DataValidationError(err.toString()) +} diff --git a/util/util-internal-validation/src/error.ts b/util/util-internal-validation/src/error.ts new file mode 100644 index 000000000..51341ed6f --- /dev/null +++ b/util/util-internal-validation/src/error.ts @@ -0,0 +1,37 @@ +import {print} from './util' + + +export class ValidationFailure { + path: (string | number)[] = [] + + constructor( + public value: unknown, + public message: string + ) {} + + toString(): string { + let msg = this.message + if (msg.includes('{value}')) { + msg = msg.replace('{value}', print(this.value)) + } + if (this.path.length) { + msg = `invalid value at ${this.getPathString()}: ${msg}` + } + return msg + } + + getPathString(): string { + let s = '' + for (let i = this.path.length - 1; i >= 0; i--) { + s += '/' + this.path[i] + } + return s + } +} + + +export class DataValidationError extends Error { + get name(): string { + return 'DataValidationError' + } +} diff --git a/util/util-internal-validation/src/index.ts b/util/util-internal-validation/src/index.ts new file mode 100644 index 000000000..96b54b756 --- /dev/null +++ b/util/util-internal-validation/src/index.ts @@ -0,0 +1,4 @@ +export * from './dsl' +export * from './error' +export * from './interface' +export * from './primitives' diff --git a/util/util-internal-validation/src/interface.ts b/util/util-internal-validation/src/interface.ts new file mode 100644 index 000000000..71c618c72 --- /dev/null +++ b/util/util-internal-validation/src/interface.ts @@ -0,0 +1,18 @@ +import {ValidationFailure} from './error' + + +export interface Validator { + cast(value: unknown): T | ValidationFailure + validate(value: unknown): ValidationFailure | undefined + phantom(): S +} + + +export type GetCastType = V extends Validator + ? T + : V extends undefined ? undefined : never + + +export type GetSrcType = V extends Validator + ? S + : V extends undefined ? undefined : never diff --git a/util/util-internal-validation/src/primitives.ts b/util/util-internal-validation/src/primitives.ts new file mode 100644 index 000000000..06aaaf17e --- /dev/null +++ b/util/util-internal-validation/src/primitives.ts @@ -0,0 +1,142 @@ +import {ValidationFailure} from './error' +import {Validator} from './interface' + + +export const STRING: Validator = { + cast(value: unknown): ValidationFailure | string { + if (typeof value == 'string') { + return value + } else { + return new ValidationFailure(value, '{value} is not a string') + } + }, + validate(value: unknown): ValidationFailure | undefined { + if (typeof value == 'string') return + return new ValidationFailure(value, '{value} is not a string') + }, + phantom(): string { + return '' + } +} + + +/** + * Safe integer + */ +export const INT: Validator = { + cast(value: unknown): number | ValidationFailure { + if (isInteger(value)) { + return value + } else { + return new ValidationFailure(value, '{value} is not an integer') + } + }, + validate(value: unknown): ValidationFailure | undefined { + if (isInteger(value)) return + return new ValidationFailure(value, '{value} is not an integer') + }, + phantom(): number { + return 0 + } +} + + +function isInteger(value: unknown): value is number { + return typeof value == 'number' && Number.isSafeInteger(value) +} + + +/** + * Safe integer greater or equal to 0 + */ +export const NAT: Validator = { + cast(value: unknown): number | ValidationFailure { + if (isInteger(value) && value >= 0) { + return value + } else { + return new ValidationFailure(value, '{value} is not a safe natural number') + } + }, + validate(value: unknown): ValidationFailure | undefined { + if (isInteger(value) && value >= 0) return + return new ValidationFailure(value, '{value} is not a safe natural number') + }, + phantom(): number { + return 0 + } +} + + +/** + * Hex encoded binary string or natural number + */ +type Bytes = string + + +function isBytes(value: unknown): value is Bytes { + return typeof value == 'string' && /^0x[0-9a-f]*$/.test(value) +} + + +/** + * Hex encoded natural number of an arbitrary size + */ +export const QTY: Validator = { + cast(value: unknown): ValidationFailure | bigint { + if (isBytes(value)) { + return BigInt(value) + } else { + return new ValidationFailure(value, `{value} is not a hex encoded natural number`) + } + }, + validate(value: unknown): ValidationFailure | undefined { + if (isBytes(value)) return + return new ValidationFailure(value, `{value} is not a hex encoded natural number`) + }, + phantom(): string { + return '0x0' + } +} + + +/** + * Hex encoded safe natural number + */ +export const SMALL_QTY: Validator = { + cast(value: unknown): number | ValidationFailure { + if (isBytes(value)) { + let val = parseInt(value) + if (Number.isSafeInteger(val)) { + return val + } else { + return new ValidationFailure(value, `{value} is not a safe integer`) + } + } else { + return new ValidationFailure(value, `{value} is not a hex encoded natural number`) + } + }, + validate(value: unknown): ValidationFailure | undefined { + let i = this.cast(value) + if (i instanceof ValidationFailure) return i + }, + phantom(): string { + return '0x0' + } +} + + +/** + * Hex encoded binary string + */ +export const BYTES: Validator = { + cast(value: unknown): string | ValidationFailure { + return this.validate(value) || value as Bytes + }, + validate(value: unknown): ValidationFailure | undefined { + if (isBytes(value)) return + return new ValidationFailure(value, `{value} is not a hex encoded binary string`) + }, + phantom(): Bytes { + return '0x' + } +} diff --git a/util/util-internal-validation/src/util.ts b/util/util-internal-validation/src/util.ts new file mode 100644 index 000000000..5c84f28f3 --- /dev/null +++ b/util/util-internal-validation/src/util.ts @@ -0,0 +1,16 @@ +export type AddOptionToUndefined = { + [K in keyof T as undefined extends T[K] ? never : K]: T[K] +} & { + [K in keyof T as undefined extends T[K] ? K : never]+?: T[K] +} + + +export type Simplify = { + [K in keyof T]: T[K] +} & {} + + +export function print(val: unknown): string { + if (val === undefined) return 'undefined' + return JSON.stringify(val) +} diff --git a/util/util-internal-validation/tsconfig.json b/util/util-internal-validation/tsconfig.json new file mode 100644 index 000000000..3cd33f5d4 --- /dev/null +++ b/util/util-internal-validation/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es2020", + "outDir": "lib", + "rootDir": "src", + "strict": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "esModuleInterop": true, + "skipLibCheck": true + }, + "include": ["src"], + "exclude": [ + "node_modules" + ] +} diff --git a/util/util-internal/src/async.ts b/util/util-internal/src/async.ts index 5f68ce28a..702c9abb0 100644 --- a/util/util-internal/src/async.ts +++ b/util/util-internal/src/async.ts @@ -1,4 +1,5 @@ import assert from 'assert' +import * as process from 'process' import {assertNotNull} from './misc' @@ -40,52 +41,76 @@ export class AsyncQueue { private closed = false private putFuture?: Future private takeFuture?: Future + private closeListeners?: (() => void)[] - constructor(size: number) { - this.buf = new Array(size) + constructor(maxsize: number) { + assert(maxsize >= 1) + this.buf = new Array(maxsize) + } + + isClosed(): boolean { + return this.closed } async put(val: T): Promise { if (this.closed) throw new ClosedQueueError() + + assert(this.size < this.buf.length && this.putFuture == null, 'concurrent puts are not allowed') + + if (this.takeFuture) { + this.takeFuture.resolve(val) + this.takeFuture = undefined + } else { + this.buf[(this.pos + this.size) % this.buf.length] = val + this.size += 1 + if (this.size == this.buf.length) { + this.putFuture = createFuture() + await this.putFuture.promise() + } + } + } + + forcePut(val: T): void { + if (this.closed) throw new ClosedQueueError() if (this.takeFuture) { this.takeFuture.resolve(val) this.takeFuture = undefined } else if (this.size < this.buf.length) { - this.putToBuffer(val) + this.buf[(this.pos + this.size) % this.buf.length] = val + this.size += 1 } else { - assert(this.putFuture == null, 'Concurrent puts are not allowed') - this.putFuture = createFuture() - await this.putFuture.promise() - this.putToBuffer(val) + this.buf[(this.pos + this.size - 1) % this.buf.length] = val } } - private putToBuffer(val: T): void { - assert(this.size < this.buf.length) - this.buf[(this.pos + this.size) % this.buf.length] = val - this.size += 1 + tryPut(val: T): void { + this.put(val).catch(err => {}) } async take(): Promise { - if (this.putFuture) { - this.putFuture.resolve() - this.putFuture = undefined - } if (this.size > 0) { let val = this.buf[this.pos]! this.buf[this.pos] = undefined this.pos = (this.pos + 1) % this.buf.length this.size -= 1 + if (this.putFuture) { + this.putFuture.resolve() + this.putFuture = undefined + } return val } else if (this.closed) { return undefined } else { - assert(this.takeFuture == null, 'Concurrent takes are not allowed') + assert(this.takeFuture == null, 'concurrent takes are not allowed') this.takeFuture = createFuture() return this.takeFuture.promise() } } + peek(): T | undefined { + return this.buf[this.pos] + } + close(): void { this.closed = true if (this.putFuture) { @@ -96,13 +121,32 @@ export class AsyncQueue { this.takeFuture.resolve(undefined) this.takeFuture = undefined } + if (this.closeListeners) { + for (let cb of this.closeListeners) { + safeCall(cb) + } + this.closeListeners = undefined + } + } + + addCloseListener(cb: () => void): void { + if (this.closed) return process.nextTick(() => safeCall(cb)) + if (this.closeListeners == null) { + this.closeListeners = [cb] + } else { + this.closeListeners.push(cb) + } } async *iterate(): AsyncIterable { - while (true) { - let val = await this.take() - if (val === undefined) return - yield val + try { + while (true) { + let val = await this.take() + if (val === undefined) return + yield val + } + } finally { + this.close() } } } @@ -113,8 +157,7 @@ export async function* concurrentMap( stream: AsyncIterable, f: (val: T) => Promise ): AsyncIterable { - assert(concurrency > 1) - let queue = new AsyncQueue<{promise: Promise}>(concurrency - 1) + let queue = new AsyncQueue<{promise: Promise}>(concurrency) async function map() { for await (let val of stream) { @@ -126,10 +169,19 @@ export async function* concurrentMap( map().then( () => queue.close(), - err => queue.put({promise: Promise.reject(err)}) + err => queue.tryPut({promise: Promise.reject(err)}) ) for await (let item of queue.iterate()) { yield await item.promise } } + + +export function safeCall(cb: () => void): void { + try { + cb() + } catch(err: any) { + Promise.reject(err) + } +} diff --git a/util/util-internal/src/misc.ts b/util/util-internal/src/misc.ts index 72fe7128d..469136ef1 100644 --- a/util/util-internal/src/misc.ts +++ b/util/util-internal/src/misc.ts @@ -205,3 +205,37 @@ export async function splitParallelWork(maxSize: number, tasks: T[], run: } return result } + + +export function* partitionBy(items: T[], value: (a: T) => V): Iterable<{items: T[], value: V}> { + if (items.length == 0) return + let pack: T[] = [items[0]] + let packValue = value(items[0]) + for (let i = 1; i < items.length; i++) { + let item = items[i] + let itemValue = value(item) + if (itemValue === packValue) { + pack.push(item) + } else { + yield {items: pack, value: packValue} + pack = [item] + packValue = itemValue + } + } + if (pack.length > 0) { + yield {items: pack, value: packValue} + } +} + + +export function weakMemo(f: (obj: T) => R): (obj: T) => R { + let cache = new WeakMap() + return function(obj: T): R { + let val = cache.get(obj) + if (val === undefined) { + val = f(obj) + cache.set(obj, val) + } + return val + } +}