substrate data ingestion fixes

common/changes/@subsquid/substrate-data-raw/master_2024-01-04-17-13.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/substrate-data-raw",
+      "comment": "allow `block.events` to be null",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@subsquid/substrate-data-raw"
+}

common/changes/@subsquid/substrate-data/master_2024-01-04-17-13.json

@@ -0,0 +1,15 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/substrate-data",
+      "comment": "fix genesis block fetch error by allowing `null` event storage values",
+      "type": "patch"
+    },
+    {
+      "packageName": "@subsquid/substrate-data",
+      "comment": "fix: sometimes block validator is not set",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@subsquid/substrate-data"
+}

common/changes/@subsquid/substrate-dump/master_2024-01-04-17-13.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/substrate-dump",
+      "comment": "fix genesis block fetch error by allowing `null` event storage values",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@subsquid/substrate-dump"
+}

substrate/substrate-data-raw/src/fetch1.ts

@@ -224,7 +224,7 @@ export class Fetch1 {
 
         for (let i = 0; i < blocks.length; i++) {
             let bytes = events[i]
-            if (bytes == null) {
+            if (bytes === undefined) {
                 blocks[i]._isInvalid = true
             } else {
                 blocks[i].events = bytes

substrate/substrate-data-raw/src/interfaces.ts

@@ -79,7 +79,7 @@ export interface PartialBlockData {
     /**
      * Contents of `System.Events` storage
      */
-    events?: Bytes
+    events?: Bytes | null
     trace?: any
     _isInvalid?: boolean
 }

substrate/substrate-data/src/index.ts

@@ -3,3 +3,4 @@ export * from './datasource'
 export * from './interfaces/data'
 export * from './parser'
 export * from './runtime-tracker'
+export {STORAGE} from './storage'

substrate/substrate-data/src/interfaces/data-raw.ts

@@ -4,12 +4,21 @@ import type {AccountId} from '../parsing/validator'
 import type {Block as ParsedBlock} from './data'
 
 
+/**
+ * Decoded value of Session.CurrentIndex storage
+ */
+export type SessionIndex = number | bigint
+
+
 export interface RawBlock extends BlockData {
     validators?: AccountId[]
-    session?: Bytes
+    session?: SessionIndex
     runtime?: Runtime
     runtimeOfPrevBlock?: Runtime
-    feeMultiplier?: Bytes
+    /**
+     * Decoded value of TransactionPayment.NextFeeMultiplier storage
+     */
+    feeMultiplier?: number | bigint
     /**
      * Storage values of previous block
      */

substrate/substrate-data/src/parser.ts

@@ -3,24 +3,17 @@ import {OldSpecsBundle, OldTypesBundle} from '@subsquid/substrate-runtime/lib/me
 import {addErrorContext, annotateAsyncError, assertNotNull, groupBy} from '@subsquid/util-internal'
 import {assertIsValid, HashAndHeight, setInvalid, trimInvalid} from '@subsquid/util-internal-ingest-tools'
 import {RangeRequestList, splitBlocksByRequest} from '@subsquid/util-internal-range'
-import assert from 'assert'
 import {Block, Bytes, DataRequest} from './interfaces/data'
-import {RawBlock} from './interfaces/data-raw'
+import {RawBlock, SessionIndex} from './interfaces/data-raw'
 import {parseBlock} from './parsing/block'
 import {supportsFeeCalc} from './parsing/fee/calc'
 import {AccountId} from './parsing/validator'
 import {RuntimeTracker} from './runtime-tracker'
-
-
-const STORAGE = {
-    nextFeeMultiplier: '0x3f1467a096bcd71a5b6a0c8155e208103f2edf3bdf381debe331ab7446addfdc',
-    validators: '0xcec5070d609dd3497f72bde07fc96ba088dcde934c658227ee1dfafcd6e16903',
-    session: '0xcec5070d609dd3497f72bde07fc96ba072763800a36a99fdfc7c10f6415f6ee6'
-}
+import {STORAGE} from './storage'
 
 
 export class Parser {
-    private prevValidators = new Prev<{session: Bytes, validators: AccountId[]}>()
+    private prevValidators = new Prev<{session: SessionIndex, validators: AccountId[]}>()
     private runtimeTracker: RuntimeTracker<RawBlock>
 
     constructor(
@@ -98,11 +91,11 @@ export class Parser {
     }
 
     private async setValidators(blocks: RawBlock[]): Promise<void> {
-        blocks = blocks.filter(b => b.runtime!.hasStorageItem('Session.Validators'))
+        blocks = blocks.filter(b => STORAGE.validators.isDefined(b))
 
         if (blocks.length == 0 || blocks[0]._isInvalid) return
 
-        let prev: {session: Bytes, validators: AccountId[]}
+        let prev: {session: SessionIndex, validators: AccountId[]}
         let maybePrev = this.prevValidators.get(blocks[0].height)
         if (maybePrev == null) {
             maybePrev = await this.fetchValidators(blocks[0])
@@ -117,8 +110,8 @@ export class Parser {
         while (last >= 0) {
             lastBlock = blocks[last]
             if (lastBlock.session == null) {
-                let session = await this.rpc.getStorage(STORAGE.session, lastBlock.hash)
-                if (session == null) {
+                let session = await STORAGE.sessionIndex.get(this.rpc, lastBlock)
+                if (session === undefined) {
                     last -= 1
                 } else {
                     lastBlock.session = session
@@ -131,13 +124,13 @@ export class Parser {
 
         if (lastBlock == null) return setInvalid(blocks)
 
-        for (let i = 0; i < last; i++) {
+        for (let i = 0; i <= last; i++) {
             let block = blocks[i]
             if (prev.session == lastBlock.session) {
                 block.session = prev.session
             } else {
-                let session = await this.rpc.getStorage(STORAGE.session, block.hash)
-                if (session == null) return setInvalid(blocks, i)
+                let session = await STORAGE.sessionIndex.get(this.rpc, block)
+                if (session === undefined) return setInvalid(blocks, i)
                 block.session = session
             }
             if (prev.session == block.session) {
@@ -149,18 +142,19 @@ export class Parser {
                 prev = maybePrev
             }
         }
+
+        if (last + 1 < blocks.length) {
+            setInvalid(blocks, last + 1)
+        }
     }
 
     @annotateAsyncError(getRefCtx)
-    private async fetchValidators(block: RawBlock): Promise<{session: Bytes, validators: AccountId[]} | undefined> {
-        let [session, data] = await Promise.all([
-            block.session ? Promise.resolve(block.session) : this.rpc.getStorage(STORAGE.session, block.hash),
-            this.rpc.getStorage(STORAGE.validators, block.hash)
+    private async fetchValidators(block: RawBlock): Promise<{session: SessionIndex, validators: AccountId[]} | undefined> {
+        let [session, validators] = await Promise.all([
+            block.session ? Promise.resolve(block.session) : STORAGE.sessionIndex.get(this.rpc, block),
+            STORAGE.validators.get(this.rpc, block)
         ])
-        if (session == null || data === undefined) return
-        let runtime = assertNotNull(block.runtime)
-        let validators = runtime.decodeStorageValue('Session.Validators', data)
-        assert(Array.isArray(validators))
+        if (session === undefined || validators === undefined) return
         block.session = session
         block.validators = validators
         let item = {session, validators}
@@ -171,15 +165,15 @@ export class Parser {
     private async setFeeMultiplier(blocks: RawBlock[]): Promise<void> {
         let values = await this.rpc.getStorageMany(blocks.map(b => {
             let parentHash = b.height == 0 ? b.hash : b.block.block.header.parentHash
-            return [STORAGE.nextFeeMultiplier, parentHash]
+            return [STORAGE.nextFeeMultiplier.key(), parentHash]
         }))
         for (let i = 0; i < blocks.length; i++) {
             let value = values[i]
             let block = blocks[i]
             if (value === undefined) {
                 block._isInvalid = true
-            } else if (value) {
-                block.feeMultiplier = value
+            } else {
+                block.feeMultiplier = STORAGE.nextFeeMultiplier.decode(block, value)
             }
         }
     }

substrate/substrate-data/src/parsing/block.ts

@@ -108,6 +108,7 @@ class BlockParser {
     @def
     events(): Event[] {
         assert('events' in this.src, 'event data is not provided')
+        if (this.src.events == null) return []
         return decodeEvents(this.runtime, this.src.events)
     }
 
@@ -152,7 +153,7 @@ class BlockParser {
         if (this.runtime.hasEvent('TransactionPayment.TransactionFeePaid')) {
             setExtrinsicFeesFromPaidEvent(this.runtime, this.extrinsics(), this.events())
         } else if (supportsFeeCalc(this.runtime)) {
-            assert('feeMultiplier' in this.src, 'fee multiplier value is not provided')
+            assert(this.src.feeMultiplier != null, 'fee multiplier value is not provided')
             let extrinsics = this.extrinsics()
             let rawExtrinsics = assertNotNull(this.src.block.block.extrinsics)
             setExtrinsicFeesFromCalc(

substrate/substrate-data/src/parsing/event.ts

@@ -17,7 +17,7 @@ const EventItem = struct({
 const EventItemList = array(EventItem)
 
 
-export function decodeEvents(runtime: Runtime, eventsStorageValue: Bytes | undefined): Event[] {
+export function decodeEvents(runtime: Runtime, eventsStorageValue: Bytes | undefined | null): Event[] {
     assertStorage(runtime, 'System.Events', ['Required', 'Default'], [], EventItemList)
 
     let items: GetType<typeof EventItem>[] = runtime.decodeStorageValue(

substrate/substrate-data/src/parsing/fee/calc.ts

@@ -2,6 +2,7 @@ import {Bytes, QualifiedName, Runtime} from '@subsquid/substrate-runtime'
 import * as sts from '@subsquid/substrate-runtime/lib/sts'
 import {def} from '@subsquid/util-internal'
 import {CalcFee as SubstrateFeeCalc} from '@substrate/calc'
+import {STORAGE} from '../../storage'
 import {
     BlockWeightsConst,
     ExtrinsicBaseWeightConst,
@@ -25,7 +26,7 @@ export function supportsFeeCalc(runtime: Runtime): boolean {
 
 export function getFeeCalc(
     runtime: Runtime,
-    feeMultiplier: Bytes | undefined,
+    feeMultiplier: number | bigint,
     specName: string,
     specVersion: number
 ): Calc | undefined {
@@ -73,16 +74,11 @@ class CalcFactory {
 
     @def
     private hasNextFeeMultiplier(): boolean {
-        return this.runtime.checkStorageType(
-            'TransactionPayment.NextFeeMultiplier',
-            'Default',
-            [],
-            NextFeeMultiplier
-        )
+        return STORAGE.nextFeeMultiplier.check(this.runtime)
     }
 
     get(
-        feeMultiplier: Bytes | undefined,
+        feeMultiplier: number | bigint,
         specName: string,
         specVersion: number
     ): Calc | undefined {
@@ -91,12 +87,7 @@ class CalcFactory {
         const baseWeights = this.baseWeights()
         if (baseWeights == null) return
 
-        const multiplier: bigint | number = this.runtime.decodeStorageValue(
-            'TransactionPayment.NextFeeMultiplier',
-            feeMultiplier
-        )
-
-        const calc = this.createCalc(multiplier, specName, specVersion)
+        const calc = this.createCalc(feeMultiplier, specName, specVersion)
         if (calc == null) return
 
         return (dispatchInfo, len) => {

substrate/substrate-data/src/parsing/fee/index.ts

@@ -46,7 +46,7 @@ export function setExtrinsicFeesFromCalc(
     events: Event[],
     prevBlockSpecName: string,
     prevBlockSpecVersion: number,
-    feeMultiplier: Bytes | undefined
+    feeMultiplier: number | bigint
 ): void {
     let calc = getFeeCalc(
         runtime,

substrate/substrate-data/src/storage.ts

@@ -0,0 +1,56 @@
+import {Rpc} from '@subsquid/substrate-data-raw'
+import {Bytes, Runtime} from '@subsquid/substrate-runtime'
+import {getNameHash} from '@subsquid/substrate-runtime/lib/runtime/storage'
+import {parseQualifiedName} from '@subsquid/substrate-runtime/lib/runtime/util'
+import {array, bytes, GetType, numeric, Type} from '@subsquid/substrate-runtime/lib/sts'
+import {assertNotNull, def} from '@subsquid/util-internal'
+import {RawBlock} from './interfaces/data-raw'
+import {assertStorage} from './types/util'
+
+
+class ScalarStorage<T extends Type> {
+    constructor(private name: string, private type: T) {}
+
+    assert(runtime: Runtime | RawBlock): void {
+        assertStorage(getRuntime(runtime), this.name, ['Default', 'Required'], [], this.type)
+    }
+
+    check(runtime: Runtime | RawBlock): boolean {
+        return getRuntime(runtime).checkStorageType(this.name, ['Default', 'Required'], [], this.type)
+    }
+
+    isDefined(runtime: Runtime | RawBlock): boolean {
+        return getRuntime(runtime).hasStorageItem(this.name)
+    }
+
+    @def
+    key(): Bytes {
+        let [pallet, name] = parseQualifiedName(this.name)
+        return getNameHash(pallet) + getNameHash(name).slice(2)
+    }
+
+    async get(rpc: Rpc, block: RawBlock): Promise<GetType<T> | undefined> {
+        this.assert(block)
+        let value = await rpc.getStorage(this.key(), block.hash)
+        if (value === undefined) return
+        return this.decode(block, value)
+    }
+
+    decode(runtime: RawBlock | Runtime, value: Bytes | null): GetType<T> {
+        this.assert(runtime)
+        return getRuntime(runtime).decodeStorageValue(this.name, value)
+    }
+}
+
+
+function getRuntime(blockOrRuntime: RawBlock | Runtime): Runtime {
+    if (blockOrRuntime instanceof Runtime) return blockOrRuntime
+    return assertNotNull(blockOrRuntime.runtime)
+}
+
+
+export const STORAGE = {
+    nextFeeMultiplier: new ScalarStorage('TransactionPayment.NextFeeMultiplier', numeric()),
+    sessionIndex: new ScalarStorage('Session.CurrentIndex', numeric()),
+    validators: new ScalarStorage('Session.Validators', array(bytes()))
+}