Merge branch 'master' into new-typegen

subsquid · Mar 4, 2024 · b6c7cd4 · b6c7cd4
2 parents 031c5c8 + 6d81291
commit b6c7cd4
Show file tree

Hide file tree

Showing 59 changed files with 482 additions and 108 deletions.
diff --git a/evm/evm-processor/CHANGELOG.json b/evm/evm-processor/CHANGELOG.json
@@ -1,6 +1,40 @@
 {
   "name": "@subsquid/evm-processor",
   "entries": [
+    {
+      "version": "1.17.0",
+      "tag": "@subsquid/evm-processor_v1.17.0",
+      "date": "Thu, 29 Feb 2024 15:27:11 GMT",
+      "comments": {
+        "minor": [
+          {
+            "comment": "fix debug trace validation for failed `CREATE` case and make `EvmTraceCreateResult.address` non-optional again"
+          }
+        ],
+        "patch": [
+          {
+            "comment": "Handle Avalanche RPC inconsistency issues"
+          }
+        ],
+        "dependency": [
+          {
+            "comment": "Updating dependency \"@subsquid/logger\" from `^1.3.2` to `^1.3.3`"
+          },
+          {
+            "comment": "Updating dependency \"@subsquid/util-internal-archive-client\" from `^0.1.0` to `^0.1.1`"
+          },
+          {
+            "comment": "Updating dependency \"@subsquid/util-internal-ingest-tools\" from `^1.1.0` to `^1.1.1`"
+          },
+          {
+            "comment": "Updating dependency \"@subsquid/util-internal-processor-tools\" from `^4.0.1` to `^4.0.2`"
+          },
+          {
+            "comment": "Updating dependency \"@subsquid/util-internal-range\" from `^0.1.0` to `^0.2.0`"
+          }
+        ]
+      }
+    },
     {
       "version": "1.16.0",
       "tag": "@subsquid/evm-processor_v1.16.0",

diff --git a/evm/evm-processor/CHANGELOG.md b/evm/evm-processor/CHANGELOG.md
@@ -1,6 +1,17 @@
 # Change Log - @subsquid/evm-processor
 
-This log was last generated on Wed, 21 Feb 2024 19:22:05 GMT and should not be manually modified.
+This log was last generated on Thu, 29 Feb 2024 15:27:11 GMT and should not be manually modified.
+
+## 1.17.0
+Thu, 29 Feb 2024 15:27:11 GMT
+
+### Minor changes
+
+- fix debug trace validation for failed `CREATE` case and make `EvmTraceCreateResult.address` non-optional again
+
+### Patches
+
+- Handle Avalanche RPC inconsistency issues
 
 ## 1.16.0
 Wed, 21 Feb 2024 19:22:05 GMT

diff --git a/evm/evm-processor/package.json b/evm/evm-processor/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@subsquid/evm-processor",
-  "version": "1.16.0",
+  "version": "1.17.0",
   "description": "Data fetcher and mappings executor for EVM-based chains",
   "license": "GPL-3.0-or-later",
   "repository": "[email protected]:subsquid/squid.git",
@@ -18,14 +18,14 @@
   },
   "dependencies": {
     "@subsquid/http-client": "^1.3.2",
-    "@subsquid/logger": "^1.3.2",
+    "@subsquid/logger": "^1.3.3",
     "@subsquid/rpc-client": "^4.6.0",
     "@subsquid/util-internal": "^3.0.0",
-    "@subsquid/util-internal-archive-client": "^0.1.0",
+    "@subsquid/util-internal-archive-client": "^0.1.1",
     "@subsquid/util-internal-hex": "^1.2.2",
-    "@subsquid/util-internal-ingest-tools": "^1.1.0",
-    "@subsquid/util-internal-processor-tools": "^4.0.1",
-    "@subsquid/util-internal-range": "^0.1.0",
+    "@subsquid/util-internal-ingest-tools": "^1.1.1",
+    "@subsquid/util-internal-processor-tools": "^4.0.2",
+    "@subsquid/util-internal-range": "^0.2.0",
     "@subsquid/util-internal-validation": "^0.3.0",
     "@subsquid/util-timeout": "^2.3.2"
   },

diff --git a/evm/evm-processor/src/ds-rpc/rpc.ts b/evm/evm-processor/src/ds-rpc/rpc.ts
@@ -2,8 +2,8 @@ import {Logger} from '@subsquid/logger'
 import {CallOptions, RpcClient, RpcError} from '@subsquid/rpc-client'
 import {RpcErrorInfo} from '@subsquid/rpc-client/lib/interfaces'
 import {groupBy, last} from '@subsquid/util-internal'
-import {assertIsValid, BlockConsistencyError, getBlockName, trimInvalid} from '@subsquid/util-internal-ingest-tools'
-import {FiniteRange, SplitRequest} from '@subsquid/util-internal-range'
+import {assertIsValid, BlockConsistencyError, trimInvalid} from '@subsquid/util-internal-ingest-tools'
+import {FiniteRange, rangeToArray, SplitRequest} from '@subsquid/util-internal-range'
 import {array, DataValidationError, GetSrcType, nullable, Validator} from '@subsquid/util-internal-validation'
 import assert from 'assert'
 import {Bytes, Bytes32, Qty} from '../interfaces/base'
@@ -105,28 +105,76 @@ export class Rpc {
     }
 
     async getColdSplit(req: SplitRequest<DataRequest>): Promise<Block[]> {
-        let result = await this.getBlockSplit(req)
+        let blocks = await this.getColdBlockBatch(
+            rangeToArray(req.range),
+            req.request.transactions ?? false,
+            1
+        )
+        return this.addColdRequestedData(blocks, req.request, 1)
+    }
+
+    private async addColdRequestedData(blocks: Block[], req: DataRequest, depth: number): Promise<Block[]> {
+        let result = blocks.map(b => ({...b}))
+
+        await this.addRequestedData(result, req)
 
+        if (depth > 9) {
+            assertIsValid(result)
+            return result
+        }
+
+        let missing: number[] = []
         for (let i = 0; i < result.length; i++) {
-            if (result[i] == null) throw new BlockConsistencyError({height: req.range.from + i})
-            if (i > 0 && result[i - 1]!.hash !== result[i]!.block.parentHash)
-                throw new BlockConsistencyError(
-                    result[i]!,
-                    `given block is not a parent of ${getBlockName(result[i-1]!)}`
-                )
+            if (result[i]._isInvalid) {
+                missing.push(i)
+            }
         }
 
-        let blocks = result as Block[]
+        if (missing.length == 0) return result
 
-        await this.addRequestedData(blocks, req.request)
+        let missed = await this.addColdRequestedData(
+            missing.map(i => blocks[i]),
+            req,
+            depth + 1
+        )
 
-        assertIsValid(blocks)
+        for (let i = 0; i < missing.length; i++) {
+            result[missing[i]] = missed[i]
+        }
 
-        return blocks
+        return result
+    }
+
+    private async getColdBlockBatch(numbers: number[], withTransactions: boolean, depth: number): Promise<Block[]> {
+        let result = await this.getBlockBatch(numbers, withTransactions)
+        let missing: number[] = []
+        for (let i = 0; i < result.length; i++) {
+            if (result[i] == null) {
+                missing.push(i)
+            }
+        }
+
+        if (missing.length == 0) return result as Block[]
+
+        if (depth > 9) throw new BlockConsistencyError({
+            height: numbers[missing[0]]
+        }, `failed to get finalized block after ${depth} attempts`)
+
+        let missed = await this.getColdBlockBatch(
+            missing.map(i => numbers[i]),
+            withTransactions,
+            depth + 1
+        )
+
+        for (let i = 0; i < missing.length; i++) {
+            result[missing[i]] = missed[i]
+        }
+
+        return result as Block[]
     }
 
     async getHotSplit(req: SplitRequest<DataRequest> & {finalizedHeight: number}): Promise<Block[]> {
-        let blocks = await this.getBlockSplit(req)
+        let blocks = await this.getBlockBatch(rangeToArray(req.range), req.request.transactions ?? false)
 
         let chain: Block[] = []
 
@@ -142,18 +190,22 @@ export class Rpc {
         return trimInvalid(chain)
     }
 
-    private async getBlockSplit(req: SplitRequest<DataRequest>): Promise<(Block | undefined)[]> {
-        let call = []
-        for (let i = req.range.from; i <= req.range.to; i++) {
-            call.push({
+    private async getBlockBatch(numbers: number[], withTransactions: boolean): Promise<(Block | undefined)[]> {
+        let call = numbers.map(height => {
+            return {
                 method: 'eth_getBlockByNumber',
-                params: [toQty(i), req.request.transactions || false]
-            })
-        }
+                params: [toQty(height), withTransactions]
+            }
+        })
         let blocks = await this.batchCall(call, {
             validateResult: getResultValidator(
-                req.request.transactions ? nullable(GetBlockWithTransactions) : nullable(GetBlockNoTransactions)
-            )
+                withTransactions ? nullable(GetBlockWithTransactions) : nullable(GetBlockNoTransactions)
+            ),
+            validateError: info => {
+                // Avalanche
+                if (info.message.includes('cannot query unfinalized data')) return null
+                throw new RpcError(info)
+            }
         })
         return blocks.map(toBlock)
     }
@@ -204,7 +256,15 @@ export class Rpc {
             fromBlock: toQty(from),
             toBlock: toQty(to)
         }], {
-            validateResult: getResultValidator(array(Log))
+            validateResult: getResultValidator(array(Log)),
+            validateError: info => {
+                if (info.message.includes('after last accepted block')) {
+                    // Regular RVM networks simply return an empty array in case
+                    // of out of range request, but Avalanche returns an error.
+                    return []
+                }
+                throw new RpcError(info)
+            }
         }).catch(async err => {
             let range = asTryAnotherRangeError(err)
             if (range && range.from == from && from <= range.to && range.to < to) {

diff --git a/evm/evm-processor/src/ds-rpc/schema.ts b/evm/evm-processor/src/ds-rpc/schema.ts
@@ -116,7 +116,7 @@ function getDebugFrameValidator(fields: FieldSelection['trace']) {
             input: BYTES,
             gasUsed: QTY,
             output: withDefault('0x', BYTES),
-            to: BYTES
+            to: withDefault('0x0000000000000000000000000000000000000000', BYTES)
         })
     })
 

diff --git a/evm/evm-processor/src/interfaces/evm.ts b/evm/evm-processor/src/interfaces/evm.ts
@@ -101,7 +101,7 @@ export interface EvmTraceCreateAction {
 export interface EvmTraceCreateResult {
     gasUsed: bigint
     code: Bytes
-    address?: Bytes20
+    address: Bytes20
 }
 
 

diff --git a/evm/evm-processor/src/mapping/schema.ts b/evm/evm-processor/src/mapping/schema.ts
@@ -126,7 +126,7 @@ export function getTraceFrameValidator(fields: FieldSelection['trace'], forArchi
     }, {
         gasUsed: QTY,
         code: withDefault('0x', BYTES),
-        address: option(BYTES)
+        address: withDefault('0x0000000000000000000000000000000000000000', BYTES)
     })
 
     let TraceCreate = object({

diff --git a/evm/evm-typegen/package.json b/evm/evm-typegen/package.json
@@ -20,7 +20,7 @@
   },
   "dependencies": {
     "@subsquid/http-client": "^1.3.2",
-    "@subsquid/logger": "^1.3.2",
+    "@subsquid/logger": "^1.3.3",
     "@subsquid/util-internal": "^3.0.0",
     "@subsquid/util-internal-code-printer": "^1.2.2",
     "@subsquid/util-internal-commander": "^1.3.2",

diff --git a/graphql/graphql-server/package.json b/graphql/graphql-server/package.json
@@ -26,7 +26,7 @@
     "@graphql-tools/schema": "^10.0.2",
     "@subsquid/openreader": "^4.5.0",
     "@subsquid/typeorm-config": "^4.1.0",
-    "@subsquid/logger": "^1.3.2",
+    "@subsquid/logger": "^1.3.3",
     "@subsquid/util-internal": "^3.0.0",
     "@subsquid/util-internal-commander": "^1.3.2",
     "@subsquid/util-internal-http-server": "^1.2.2",

diff --git a/graphql/openreader/package.json b/graphql/openreader/package.json
@@ -27,7 +27,7 @@
   "dependencies": {
     "@graphql-tools/merge": "^9.0.1",
     "@subsquid/graphiql-console": "^0.3.0",
-    "@subsquid/logger": "^1.3.2",
+    "@subsquid/logger": "^1.3.3",
     "@subsquid/util-internal": "^3.0.0",
     "@subsquid/util-internal-commander": "^1.3.2",
     "@subsquid/util-internal-hex": "^1.2.2",

diff --git a/substrate/substrate-data-raw/CHANGELOG.json b/substrate/substrate-data-raw/CHANGELOG.json
@@ -1,6 +1,24 @@
 {
   "name": "@subsquid/substrate-data-raw",
   "entries": [
+    {
+      "version": "1.1.1",
+      "tag": "@subsquid/substrate-data-raw_v1.1.1",
+      "date": "Thu, 29 Feb 2024 15:27:11 GMT",
+      "comments": {
+        "dependency": [
+          {
+            "comment": "Updating dependency \"@subsquid/util-internal-ingest-tools\" from `^1.1.0` to `^1.1.1`"
+          },
+          {
+            "comment": "Updating dependency \"@subsquid/util-internal-range\" from `^0.1.0` to `^0.2.0`"
+          },
+          {
+            "comment": "Updating dependency \"@subsquid/logger\" from `^1.3.2` to `^1.3.3`"
+          }
+        ]
+      }
+    },
     {
       "version": "1.1.0",
       "tag": "@subsquid/substrate-data-raw_v1.1.0",

diff --git a/substrate/substrate-data-raw/CHANGELOG.md b/substrate/substrate-data-raw/CHANGELOG.md
@@ -1,6 +1,11 @@
 # Change Log - @subsquid/substrate-data-raw
 
-This log was last generated on Thu, 04 Jan 2024 17:31:35 GMT and should not be manually modified.
+This log was last generated on Thu, 29 Feb 2024 15:27:11 GMT and should not be manually modified.
+
+## 1.1.1
+Thu, 29 Feb 2024 15:27:11 GMT
+
+_Version update only_
 
 ## 1.1.0
 Thu, 04 Jan 2024 17:31:35 GMT

diff --git a/substrate/substrate-data-raw/package.json b/substrate/substrate-data-raw/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@subsquid/substrate-data-raw",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "description": "Raw RPC data fetcher for substrate based chains",
   "license": "GPL-3.0-or-later",
   "repository": "[email protected]:subsquid/squid.git",
@@ -17,12 +17,12 @@
   },
   "dependencies": {
     "@subsquid/util-internal": "^3.0.0",
-    "@subsquid/util-internal-ingest-tools": "^1.1.0",
-    "@subsquid/util-internal-range": "^0.1.0",
+    "@subsquid/util-internal-ingest-tools": "^1.1.1",
+    "@subsquid/util-internal-range": "^0.2.0",
     "@subsquid/util-timeout": "^2.3.2"
   },
   "peerDependencies": {
-    "@subsquid/logger": "^1.3.2",
+    "@subsquid/logger": "^1.3.3",
     "@subsquid/rpc-client": "^4.6.0"
   },
   "devDependencies": {

diff --git a/substrate/substrate-data/CHANGELOG.json b/substrate/substrate-data/CHANGELOG.json
@@ -1,6 +1,24 @@
 {
   "name": "@subsquid/substrate-data",
   "entries": [
+    {
+      "version": "4.0.2",
+      "tag": "@subsquid/substrate-data_v4.0.2",
+      "date": "Thu, 29 Feb 2024 15:27:11 GMT",
+      "comments": {
+        "dependency": [
+          {
+            "comment": "Updating dependency \"@subsquid/substrate-data-raw\" from `^1.1.0` to `^1.1.1`"
+          },
+          {
+            "comment": "Updating dependency \"@subsquid/util-internal-ingest-tools\" from `^1.1.0` to `^1.1.1`"
+          },
+          {
+            "comment": "Updating dependency \"@subsquid/util-internal-range\" from `^0.1.0` to `^0.2.0`"
+          }
+        ]
+      }
+    },
     {
       "version": "4.0.1",
       "tag": "@subsquid/substrate-data_v4.0.1",

diff --git a/substrate/substrate-data/CHANGELOG.md b/substrate/substrate-data/CHANGELOG.md
@@ -1,6 +1,11 @@
 # Change Log - @subsquid/substrate-data
 
-This log was last generated on Thu, 04 Jan 2024 17:31:35 GMT and should not be manually modified.
+This log was last generated on Thu, 29 Feb 2024 15:27:11 GMT and should not be manually modified.
+
+## 4.0.2
+Thu, 29 Feb 2024 15:27:11 GMT
+
+_Version update only_
 
 ## 4.0.1
 Thu, 04 Jan 2024 17:31:35 GMT