fix handling of JSON arrays during hot block rollbacks

common/changes/@subsquid/typeorm-store/master_2023-10-19-08-40.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/typeorm-store",
+      "comment": "fix handling of JSON arrays during hot block rollbacks",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@subsquid/typeorm-store"
+}

typeorm/typeorm-store/src/hot.ts

@@ -1,4 +1,6 @@
+import {assertNotNull} from '@subsquid/util-internal'
 import type {EntityManager, EntityMetadata} from 'typeorm'
+import {ColumnMetadata} from 'typeorm/metadata/ColumnMetadata'
 import {Entity, EntityClass} from './store'
 
 
@@ -61,7 +63,7 @@ export class ChangeTracker {
         let meta = this.getEntityMetadata(type)
 
         let touchedRows = await this.fetchEntities(
-            meta.tableName,
+            meta,
             entities.map(e => e.id)
         ).then(
             entities => new Map(
@@ -90,7 +92,7 @@ export class ChangeTracker {
 
     async trackDelete(type: EntityClass<Entity>, ids: string[]): Promise<void> {
         let meta = this.getEntityMetadata(type)
-        let deletedEntities = await this.fetchEntities(meta.tableName, ids)
+        let deletedEntities = await this.fetchEntities(meta, ids)
         return this.writeChangeRows(deletedEntities.map(e => {
             let {id, ...fields} = e
             return {
@@ -102,16 +104,17 @@ export class ChangeTracker {
         }))
     }
 
-    private async fetchEntities(table: string, ids: string[]): Promise<Entity[]> {
+    private async fetchEntities(meta: EntityMetadata, ids: string[]): Promise<Entity[]> {
         let entities = await this.em.query(
-            `SELECT * FROM ${this.escape(table)} WHERE id = ANY($1::text[])`,
+            `SELECT * FROM ${this.escape(meta.tableName)} WHERE id = ANY($1::text[])`,
             [ids]
         )
 
-        // Use different representation for raw bytes.
-        // That's because we can't serialize Buffer values in change records
-        // via `JSON.stringify()` (even with replacement function).
-        // It would be better to handle this issue during change record serialization,
+        // Here we transform the row object returned by the driver to its
+        // JSON variant in such a way, that `driver.query('UPDATE entity SET field = $1', [json.field])`
+        // would be always correctly handled.
+        //
+        // It would be better to handle it during change record serialization,
         // but it is just easier to do it here...
         for (let e of entities) {
             for (let key in e) {
@@ -121,6 +124,8 @@ export class ChangeTracker {
                         ? value
                         : Buffer.from(value.buffer, value.byteOffset, value.byteLength)
                     e[key] = '\\x' + value.toString('hex').toUpperCase()
+                } else if (Array.isArray(value) && isJsonProp(meta, key)) {
+                    e[key] = JSON.stringify(value)
                 }
             }
         }
@@ -207,3 +212,33 @@ export async function rollbackBlock(
 function escape(em: EntityManager, name: string): string {
     return em.connection.driver.escape(name)
 }
+
+
+const ENTITY_COLUMNS = new WeakMap<EntityMetadata, Map<string, ColumnMetadata>>()
+
+
+function getColumn(meta: EntityMetadata, fieldName: string): ColumnMetadata {
+    let columns = ENTITY_COLUMNS.get(meta)
+    if (columns == null) {
+        columns = new Map()
+        ENTITY_COLUMNS.set(meta, columns)
+    }
+    let col = columns.get(fieldName)
+    if (col == null) {
+        col = assertNotNull(meta.findColumnWithDatabaseName(fieldName))
+        columns.set(fieldName, col)
+    }
+    return col
+}
+
+
+function isJsonProp(meta: EntityMetadata, fieldName: string): boolean {
+    let col = getColumn(meta, fieldName)
+    switch(col.type) {
+        case 'jsonb':
+        case 'json':
+            return true
+        default:
+            return false
+    }
+}

typeorm/typeorm-store/src/test/database.test.ts

@@ -16,6 +16,7 @@ describe('TypeormDatabase', function() {
             big_integer numeric,
             date_time timestamp with time zone,
             "bytes" bytea,
+            "json" jsonb,
             item_id text references item
         )`
     ])
@@ -107,7 +108,8 @@ describe('TypeormDatabase', function() {
             integerArray: [1, 10],
             bigInteger: 1000000000000000000000000000000000000000000000000000000000n,
             dateTime: new Date(1000000000000),
-            bytes: Buffer.from([100, 100, 100])
+            bytes: Buffer.from([100, 100, 100]),
+            json: [1, {foo: 'bar'}]
         })
 
         let a2 = new Data({
@@ -118,7 +120,8 @@ describe('TypeormDatabase', function() {
             integerArray: [2, 20],
             bigInteger: 2000000000000000000000000000000000000000000000000000000000n,
             dateTime: new Date(2000000000000),
-            bytes: Buffer.from([200, 200, 200])
+            bytes: Buffer.from([200, 200, 200]),
+            json: [2, {foo: 'baz'}]
         })
 
         let a3 = new Data({
@@ -129,7 +132,8 @@ describe('TypeormDatabase', function() {
             integerArray: [30, 300],
             bigInteger: 3000000000000000000000000000000000000000000000000000000000n,
             dateTime: new Date(3000000000000),
-            bytes: Buffer.from([3, 3, 3])
+            bytes: Buffer.from([3, 3, 3]),
+            json: [3, {foo: 'qux'}]
         })
 
         await db.transactHot({
@@ -162,7 +166,8 @@ describe('TypeormDatabase', function() {
             integerArray: [10, 100],
             bigInteger: 8000000000000000000000000000000000000000000000000000000000_000_000n,
             dateTime: new Date(100000),
-            bytes: Buffer.from([1, 1, 1])
+            bytes: Buffer.from([1, 1, 1]),
+            json: ["b1", {foo: 'bar'}]
         })
 
         let b2 = new Data({
@@ -173,7 +178,8 @@ describe('TypeormDatabase', function() {
             integerArray: [20, 200],
             bigInteger: 9000000000000000000000000000000000000000000000000000000000_000n,
             dateTime: new Date(2000),
-            bytes: Buffer.from([2, 2, 2])
+            bytes: Buffer.from([2, 2, 2]),
+            json: {b2: true}
         })
 
         await db.transactHot({

typeorm/typeorm-store/src/test/lib/model.ts

@@ -1,4 +1,4 @@
-import {Column, Entity, ManyToOne, PrimaryColumn} from 'typeorm'
+import {Column as Column_, Column, Entity, ManyToOne, PrimaryColumn} from 'typeorm'
 
 
 @Entity()
@@ -61,6 +61,9 @@ export class Data {
     @Column('bytea')
     bytes?: Uint8Array | null
 
+    @Column_("jsonb", {nullable: true})
+    json?: unknown | null
+
     @ManyToOne(() => Item)
     item?: Item | null
 }