diff --git a/common/changes/@subsquid/typeorm-store/master_2023-10-19-08-40.json b/common/changes/@subsquid/typeorm-store/master_2023-10-19-08-40.json new file mode 100644 index 000000000..46efc0584 --- /dev/null +++ b/common/changes/@subsquid/typeorm-store/master_2023-10-19-08-40.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/typeorm-store", + "comment": "fix handling of JSON arrays during hot block rollbacks", + "type": "patch" + } + ], + "packageName": "@subsquid/typeorm-store" +} \ No newline at end of file diff --git a/typeorm/typeorm-store/src/hot.ts b/typeorm/typeorm-store/src/hot.ts index 4a8b1fb82..33030331b 100644 --- a/typeorm/typeorm-store/src/hot.ts +++ b/typeorm/typeorm-store/src/hot.ts @@ -1,4 +1,6 @@ +import {assertNotNull} from '@subsquid/util-internal' import type {EntityManager, EntityMetadata} from 'typeorm' +import {ColumnMetadata} from 'typeorm/metadata/ColumnMetadata' import {Entity, EntityClass} from './store' @@ -61,7 +63,7 @@ export class ChangeTracker { let meta = this.getEntityMetadata(type) let touchedRows = await this.fetchEntities( - meta.tableName, + meta, entities.map(e => e.id) ).then( entities => new Map( @@ -90,7 +92,7 @@ export class ChangeTracker { async trackDelete(type: EntityClass, ids: string[]): Promise { let meta = this.getEntityMetadata(type) - let deletedEntities = await this.fetchEntities(meta.tableName, ids) + let deletedEntities = await this.fetchEntities(meta, ids) return this.writeChangeRows(deletedEntities.map(e => { let {id, ...fields} = e return { @@ -102,16 +104,17 @@ export class ChangeTracker { })) } - private async fetchEntities(table: string, ids: string[]): Promise { + private async fetchEntities(meta: EntityMetadata, ids: string[]): Promise { let entities = await this.em.query( - `SELECT * FROM ${this.escape(table)} WHERE id = ANY($1::text[])`, + `SELECT * FROM ${this.escape(meta.tableName)} WHERE id = ANY($1::text[])`, [ids] ) - // Use different representation for raw bytes. - // That's because we can't serialize Buffer values in change records - // via `JSON.stringify()` (even with replacement function). - // It would be better to handle this issue during change record serialization, + // Here we transform the row object returned by the driver to its + // JSON variant in such a way, that `driver.query('UPDATE entity SET field = $1', [json.field])` + // would be always correctly handled. + // + // It would be better to handle it during change record serialization, // but it is just easier to do it here... for (let e of entities) { for (let key in e) { @@ -121,6 +124,8 @@ export class ChangeTracker { ? value : Buffer.from(value.buffer, value.byteOffset, value.byteLength) e[key] = '\\x' + value.toString('hex').toUpperCase() + } else if (Array.isArray(value) && isJsonProp(meta, key)) { + e[key] = JSON.stringify(value) } } } @@ -207,3 +212,33 @@ export async function rollbackBlock( function escape(em: EntityManager, name: string): string { return em.connection.driver.escape(name) } + + +const ENTITY_COLUMNS = new WeakMap>() + + +function getColumn(meta: EntityMetadata, fieldName: string): ColumnMetadata { + let columns = ENTITY_COLUMNS.get(meta) + if (columns == null) { + columns = new Map() + ENTITY_COLUMNS.set(meta, columns) + } + let col = columns.get(fieldName) + if (col == null) { + col = assertNotNull(meta.findColumnWithDatabaseName(fieldName)) + columns.set(fieldName, col) + } + return col +} + + +function isJsonProp(meta: EntityMetadata, fieldName: string): boolean { + let col = getColumn(meta, fieldName) + switch(col.type) { + case 'jsonb': + case 'json': + return true + default: + return false + } +} diff --git a/typeorm/typeorm-store/src/test/database.test.ts b/typeorm/typeorm-store/src/test/database.test.ts index a66042549..48ffbf1c4 100644 --- a/typeorm/typeorm-store/src/test/database.test.ts +++ b/typeorm/typeorm-store/src/test/database.test.ts @@ -16,6 +16,7 @@ describe('TypeormDatabase', function() { big_integer numeric, date_time timestamp with time zone, "bytes" bytea, + "json" jsonb, item_id text references item )` ]) @@ -107,7 +108,8 @@ describe('TypeormDatabase', function() { integerArray: [1, 10], bigInteger: 1000000000000000000000000000000000000000000000000000000000n, dateTime: new Date(1000000000000), - bytes: Buffer.from([100, 100, 100]) + bytes: Buffer.from([100, 100, 100]), + json: [1, {foo: 'bar'}] }) let a2 = new Data({ @@ -118,7 +120,8 @@ describe('TypeormDatabase', function() { integerArray: [2, 20], bigInteger: 2000000000000000000000000000000000000000000000000000000000n, dateTime: new Date(2000000000000), - bytes: Buffer.from([200, 200, 200]) + bytes: Buffer.from([200, 200, 200]), + json: [2, {foo: 'baz'}] }) let a3 = new Data({ @@ -129,7 +132,8 @@ describe('TypeormDatabase', function() { integerArray: [30, 300], bigInteger: 3000000000000000000000000000000000000000000000000000000000n, dateTime: new Date(3000000000000), - bytes: Buffer.from([3, 3, 3]) + bytes: Buffer.from([3, 3, 3]), + json: [3, {foo: 'qux'}] }) await db.transactHot({ @@ -162,7 +166,8 @@ describe('TypeormDatabase', function() { integerArray: [10, 100], bigInteger: 8000000000000000000000000000000000000000000000000000000000_000_000n, dateTime: new Date(100000), - bytes: Buffer.from([1, 1, 1]) + bytes: Buffer.from([1, 1, 1]), + json: ["b1", {foo: 'bar'}] }) let b2 = new Data({ @@ -173,7 +178,8 @@ describe('TypeormDatabase', function() { integerArray: [20, 200], bigInteger: 9000000000000000000000000000000000000000000000000000000000_000n, dateTime: new Date(2000), - bytes: Buffer.from([2, 2, 2]) + bytes: Buffer.from([2, 2, 2]), + json: {b2: true} }) await db.transactHot({ diff --git a/typeorm/typeorm-store/src/test/lib/model.ts b/typeorm/typeorm-store/src/test/lib/model.ts index 5efcb23bc..8cd68bfc3 100644 --- a/typeorm/typeorm-store/src/test/lib/model.ts +++ b/typeorm/typeorm-store/src/test/lib/model.ts @@ -1,4 +1,4 @@ -import {Column, Entity, ManyToOne, PrimaryColumn} from 'typeorm' +import {Column as Column_, Column, Entity, ManyToOne, PrimaryColumn} from 'typeorm' @Entity() @@ -61,6 +61,9 @@ export class Data { @Column('bytea') bytes?: Uint8Array | null + @Column_("jsonb", {nullable: true}) + json?: unknown | null + @ManyToOne(() => Item) item?: Item | null }