From 269ac182b135ad91d0751536d8e3f48e0e934e92 Mon Sep 17 00:00:00 2001 From: belopash Date: Mon, 27 May 2024 00:44:05 +0500 Subject: [PATCH 01/11] save --- typeorm/typeorm-store/package.json | 3 +- typeorm/typeorm-store/src/database.ts | 22 +- .../decorators/columns/BigDecimalColumn.ts | 2 +- .../src/decorators/columns/BigIntColumn.ts | 2 +- .../src/decorators/columns/FloatColumn.ts | 2 +- typeorm/typeorm-store/src/index.ts | 4 +- typeorm/typeorm-store/src/store.ts | 621 ++++++++++++++---- .../typeorm-store/src/test/database.test.ts | 4 +- typeorm/typeorm-store/src/test/store.test.ts | 55 +- typeorm/typeorm-store/src/utils/cacheMap.ts | 101 +++ .../typeorm-store/src/utils/changeTracker.ts | 105 +++ .../src/{hot.ts => utils/changeWriter.ts} | 38 +- .../typeorm-store/src/utils/commitOrder.ts | 62 ++ typeorm/typeorm-store/src/utils/misc.ts | 86 +++ .../src/{ => utils}/transformers.ts | 15 +- typeorm/typeorm-store/src/{ => utils}/tx.ts | 2 +- 16 files changed, 931 insertions(+), 193 deletions(-) create mode 100644 typeorm/typeorm-store/src/utils/cacheMap.ts create mode 100644 typeorm/typeorm-store/src/utils/changeTracker.ts rename typeorm/typeorm-store/src/{hot.ts => utils/changeWriter.ts} (85%) create mode 100644 typeorm/typeorm-store/src/utils/commitOrder.ts create mode 100644 typeorm/typeorm-store/src/utils/misc.ts rename typeorm/typeorm-store/src/{ => utils}/transformers.ts (65%) rename typeorm/typeorm-store/src/{ => utils}/tx.ts (95%) diff --git a/typeorm/typeorm-store/package.json b/typeorm/typeorm-store/package.json index 8596fb28d..4424dce6b 100644 --- a/typeorm/typeorm-store/package.json +++ b/typeorm/typeorm-store/package.json @@ -19,7 +19,8 @@ }, "dependencies": { "@subsquid/typeorm-config": "^4.1.1", - "@subsquid/util-internal": "^3.2.0" + "@subsquid/util-internal": "^3.2.0", + "@subsquid/logger": "~1.3.3" }, "peerDependencies": { "typeorm": "^0.3.17", diff --git a/typeorm/typeorm-store/src/database.ts b/typeorm/typeorm-store/src/database.ts index 9cfe5ff82..ea3c3dd28 100644 --- a/typeorm/typeorm-store/src/database.ts +++ b/typeorm/typeorm-store/src/database.ts @@ -1,10 +1,11 @@ import {createOrmConfig} from '@subsquid/typeorm-config' -import {assertNotNull, last, maybeLast} from '@subsquid/util-internal' +import {assertNotNull, def, last, maybeLast} from '@subsquid/util-internal' import assert from 'assert' import {DataSource, EntityManager} from 'typeorm' -import {ChangeTracker, rollbackBlock} from './hot' +import {ChangeWriter, rollbackBlock} from './utils/changeWriter' import {DatabaseState, FinalTxInfo, HashAndHeight, HotTxInfo} from './interfaces' import {Store} from './store' +import {sortMetadatasInCommitOrder} from './utils/commitOrder' export type IsolationLevel = 'SERIALIZABLE' | 'READ COMMITTED' | 'REPEATABLE READ' @@ -179,7 +180,7 @@ export class TypeormDatabase { await this.performUpdates( store => cb(store, i, i + 1), em, - new ChangeTracker(em, this.statusSchema, b.height) + new ChangeWriter(em, this.statusSchema, b.height) ) } } @@ -230,7 +231,7 @@ export class TypeormDatabase { private async performUpdates( cb: (store: Store) => Promise, em: EntityManager, - changeTracker?: ChangeTracker + changeTracker?: ChangeWriter ): Promise { let running = true @@ -239,11 +240,15 @@ export class TypeormDatabase { assert(running, `too late to perform db updates, make sure you haven't forgot to await on db query`) return em }, - changeTracker + { + tracker: changeTracker, + commitOrder: this.getCommitOrder() + } ) try { await cb(store) + await store.flush() } finally { running = false } @@ -270,6 +275,13 @@ export class TypeormDatabase { let con = assertNotNull(this.con) return con.driver.escape(this.statusSchema) } + + @def + private getCommitOrder() { + let con = this.con + assert(con != null, 'not connected') + return sortMetadatasInCommitOrder(con.entityMetadatas) + } } diff --git a/typeorm/typeorm-store/src/decorators/columns/BigDecimalColumn.ts b/typeorm/typeorm-store/src/decorators/columns/BigDecimalColumn.ts index 2f436cc32..8c8f074bd 100644 --- a/typeorm/typeorm-store/src/decorators/columns/BigDecimalColumn.ts +++ b/typeorm/typeorm-store/src/decorators/columns/BigDecimalColumn.ts @@ -1,4 +1,4 @@ -import {bigdecimalTransformer} from '../../transformers' +import {bigdecimalTransformer} from '../../utils/transformers' import {Column} from './Column' import {ColumnCommonOptions} from './common' diff --git a/typeorm/typeorm-store/src/decorators/columns/BigIntColumn.ts b/typeorm/typeorm-store/src/decorators/columns/BigIntColumn.ts index 3c572be4f..d6312fc32 100644 --- a/typeorm/typeorm-store/src/decorators/columns/BigIntColumn.ts +++ b/typeorm/typeorm-store/src/decorators/columns/BigIntColumn.ts @@ -1,4 +1,4 @@ -import {bigintTransformer} from '../../transformers' +import {bigintTransformer} from '../../utils/transformers' import {Column} from './Column' import {ColumnCommonOptions} from './common' diff --git a/typeorm/typeorm-store/src/decorators/columns/FloatColumn.ts b/typeorm/typeorm-store/src/decorators/columns/FloatColumn.ts index fece3d4c9..a11e79532 100644 --- a/typeorm/typeorm-store/src/decorators/columns/FloatColumn.ts +++ b/typeorm/typeorm-store/src/decorators/columns/FloatColumn.ts @@ -1,4 +1,4 @@ -import {floatTransformer} from '../../transformers' +import {floatTransformer} from '../../utils/transformers' import {Column} from './Column' import {ColumnCommonOptions} from './common' diff --git a/typeorm/typeorm-store/src/index.ts b/typeorm/typeorm-store/src/index.ts index 82d4fb142..40e8a0501 100644 --- a/typeorm/typeorm-store/src/index.ts +++ b/typeorm/typeorm-store/src/index.ts @@ -1,4 +1,4 @@ export * from './database' -export {EntityClass, FindManyOptions, FindOneOptions, Store} from './store' +export {FindManyOptions, FindOneOptions, Store, EntityLiteral} from './store' export * from './decorators' -export * from './transformers' \ No newline at end of file +export * from './utils/transformers' diff --git a/typeorm/typeorm-store/src/store.ts b/typeorm/typeorm-store/src/store.ts index d6ead6f58..7bbfb21c3 100644 --- a/typeorm/typeorm-store/src/store.ts +++ b/typeorm/typeorm-store/src/store.ts @@ -1,20 +1,40 @@ import assert from 'assert' -import {EntityManager, FindOptionsOrder, FindOptionsRelations, FindOptionsWhere} from 'typeorm' +import { + EntityManager, + EntityMetadata, + FindOptionsOrder, + FindOptionsRelations, + FindOptionsWhere, + ObjectLiteral, +} from 'typeorm' import {EntityTarget} from 'typeorm/common/EntityTarget' +import {ChangeWriter} from './utils/changeWriter' +import {CacheMap} from './utils/cacheMap' +import {ChangeTracker, ChangeType} from './utils/changeTracker' +import {createLogger, Logger} from '@subsquid/logger' +import {createFuture, def, Future} from '@subsquid/util-internal' +import {copy} from './utils/misc' import {ColumnMetadata} from 'typeorm/metadata/ColumnMetadata' -import {ChangeTracker} from './hot' +export {EntityTarget} -export interface EntityClass { - new (): T +export interface EntityLiteral extends ObjectLiteral { + id: string } +export type ChangeSet = { + metadata: EntityMetadata + inserts: EntityLiteral[] + upserts: EntityLiteral[] + deletes: string[] + extraUpserts: EntityLiteral[] +} -export interface Entity { +export interface GetOptions { id: string + relations?: FindOptionsRelations } - /** * Defines a special criteria to find specific entity. */ @@ -32,43 +52,62 @@ export interface FindOneOptions { /** * Indicates what relations of entity should be loaded (simplified left join form). */ - relations?: FindOptionsRelations; + relations?: FindOptionsRelations /** * Order, in which entities should be ordered. */ order?: FindOptionsOrder -} + cache?: boolean +} export interface FindManyOptions extends FindOneOptions { /** * Offset (paginated) where from entities should be taken. */ - skip?: number; + skip?: number /** * Limit (paginated) - max number of entities should be taken. */ - take?: number; -} + take?: number + cache?: boolean +} /** * Restricted version of TypeORM entity manager for squid data handlers. */ export class Store { - constructor(private em: () => EntityManager, private changes?: ChangeTracker) {} + protected commitOrder: EntityMetadata[] + protected tracker?: ChangeWriter + protected changes: ChangeTracker + protected cache: CacheMap + protected logger: Logger + + protected pendingCommit?: Future + + constructor( + protected em: () => EntityManager, + { + commitOrder, + tracker, + }: { + commitOrder: EntityMetadata[] + tracker?: ChangeWriter + } + ) { + this.logger = createLogger('sqd:typeorm-store') + this.commitOrder = commitOrder + this.tracker = tracker + this.cache = new CacheMap({logger: this.logger}) + this.changes = new ChangeTracker({logger: this.logger}) + } /** * Alias for {@link Store.upsert} */ - save(entity: E): Promise - save(entities: E[]): Promise - save(e: E | E[]): Promise { - if (Array.isArray(e)) { // please the compiler - return this.upsert(e) - } else { - return this.upsert(e) - } + async save(e: E | E[]): Promise { + return this.upsert(e) } /** @@ -76,59 +115,57 @@ export class Store { * * It always executes a primitive operation without cascades, relations, etc. */ - upsert(entity: E): Promise - upsert(entities: E[]): Promise - async upsert(e: E | E[]): Promise { - if (Array.isArray(e)) { - if (e.length == 0) return - let entityClass = e[0].constructor as EntityClass - for (let i = 1; i < e.length; i++) { - assert(entityClass === e[i].constructor, 'mass saving allowed only for entities of the same class') - } - await this.changes?.trackUpsert(entityClass, e) - await this.saveMany(entityClass, e) - } else { - let entityClass = e.constructor as EntityClass - await this.changes?.trackUpsert(entityClass, [e]) - await this.em().upsert(entityClass, e as any, ['id']) + async upsert(e: E | E[]): Promise { + await this.pendingCommit?.promise() + + let entities = Array.isArray(e) ? e : [e] + if (entities.length == 0) return + + for (const entity of entities) { + const md = this.getEntityMetadata(entity.constructor) + + const isNew = this.changes.isDeleted(md, entity.id) + + this.changes.trackUpsert(md, entity.id) + this.cache.add(md, entity, isNew) } } - private async saveMany(entityClass: EntityClass, entities: any[]): Promise { - assert(entities.length > 0) - let em = this.em() - let metadata = em.connection.getMetadata(entityClass) - let fk = metadata.columns.filter(c => c.relationMetadata) - if (fk.length == 0) return this.upsertMany(em, entityClass, entities) + private getFkSignature(fk: ColumnMetadata[], entity: any): bigint { + let sig = 0n + for (let i = 0; i < fk.length; i++) { + let bit = fk[i].getEntityValue(entity) === undefined ? 0n : 1n + sig |= bit << BigInt(i) + } + return sig + } + + private async _upsert(metadata: EntityMetadata, entities: EntityLiteral[]): Promise { + this.logger.debug(`upsert ${entities.length} ${metadata.name} entities`) + await this.tracker?.writeUpsert(metadata, entities) + + let fk = metadata.columns.filter((c) => c.relationMetadata) + if (fk.length == 0) return this.upsertMany(metadata.target, entities) let currentSignature = this.getFkSignature(fk, entities[0]) - let batch = [] + let batch: EntityLiteral[] = [] for (let e of entities) { let sig = this.getFkSignature(fk, e) if (sig === currentSignature) { batch.push(e) } else { - await this.upsertMany(em, entityClass, batch) + await this.upsertMany(metadata.target, batch) currentSignature = sig batch = [e] } } if (batch.length) { - await this.upsertMany(em, entityClass, batch) - } - } - - private getFkSignature(fk: ColumnMetadata[], entity: any): bigint { - let sig = 0n - for (let i = 0; i < fk.length; i++) { - let bit = fk[i].getEntityValue(entity) === undefined ? 0n : 1n - sig |= (bit << BigInt(i)) + await this.upsertMany(metadata.target, batch) } - return sig } - private async upsertMany(em: EntityManager, entityClass: EntityClass, entities: any[]): Promise { + private async upsertMany(target: EntityTarget, entities: EntityLiteral[]) { for (let b of splitIntoBatches(entities, 1000)) { - await em.upsert(entityClass, b as any, ['id']) + await this.em().upsert(target, b as any, ['id']) } } @@ -138,99 +175,446 @@ export class Store { * * Executes a primitive INSERT operation without cascades, relations, etc. */ - insert(entity: E): Promise - insert(entities: E[]): Promise - async insert(e: E | E[]): Promise { - if (Array.isArray(e)) { - if (e.length == 0) return - let entityClass = e[0].constructor as EntityClass - for (let i = 1; i < e.length; i++) { - assert(entityClass === e[i].constructor, 'mass saving allowed only for entities of the same class') - } - await this.changes?.trackInsert(entityClass, e) - for (let b of splitIntoBatches(e, 1000)) { - await this.em().insert(entityClass, b as any) - } - } else { - let entityClass = e.constructor as EntityClass - await this.changes?.trackInsert(entityClass, [e]) - await this.em().insert(entityClass, e as any) + async insert(e: E | E[]): Promise { + await this.pendingCommit?.promise() + + const entities = Array.isArray(e) ? e : [e] + if (entities.length == 0) return + + for (const entity of entities) { + const md = this.getEntityMetadata(entity.constructor) + + this.changes.trackInsert(md, entity.id) + this.cache.add(md, entity, true) + } + } + + private async _insert(metadata: EntityMetadata, entities: EntityLiteral[]) { + this.logger.debug(`insert ${entities.length} ${metadata.name} entities`) + await this.tracker?.writeInsert(metadata, entities) + await this.insertMany(metadata.target, entities) + } + + private async insertMany(target: EntityTarget, entities: EntityLiteral[]) { + for (let b of splitIntoBatches(entities, 1000)) { + await this.em().insert(target, b) } } /** * Deletes a given entity or entities from the database. * - * Unlike {@link EntityManager.remove} executes a primitive DELETE query without cascades, relations, etc. + * Executes a primitive DELETE query without cascades, relations, etc. */ - remove(entity: E): Promise - remove(entities: E[]): Promise - remove(entityClass: EntityClass, id: string | string[]): Promise - async remove(e: E | E[] | EntityClass, id?: string | string[]): Promise{ + async delete(e: E | E[] | EntityTarget, id?: string | string[]): Promise { + await this.pendingCommit?.promise() + if (id == null) { - if (Array.isArray(e)) { - if (e.length == 0) return - let entityClass = e[0].constructor as EntityClass - for (let i = 1; i < e.length; i++) { - assert(entityClass === e[i].constructor, 'mass deletion allowed only for entities of the same class') - } - let ids = e.map(i => i.id) - await this.changes?.trackDelete(entityClass, ids) - await this.em().delete(entityClass, ids) - } else { - let entity = e as E - let entityClass = entity.constructor as EntityClass - await this.changes?.trackDelete(entityClass, [entity.id]) - await this.em().delete(entityClass, entity.id) + const entities = Array.isArray(e) ? e : [e as E] + if (entities.length == 0) return + + for (const entity of entities) { + const md = this.getEntityMetadata(entity.constructor) + + this.changes.trackDelete(md, entity.id) + this.cache.delete(md, entity.id) } } else { - let entityClass = e as EntityClass - await this.changes?.trackDelete(entityClass, Array.isArray(id) ? id : [id]) - await this.em().delete(entityClass, id) + const ids = Array.isArray(id) ? id : [id] + if (ids.length == 0) return + + const md = this.getEntityMetadata(e as EntityTarget) + for (const id of ids) { + this.changes.trackDelete(md, id) + this.cache.delete(md, id) + } } } - async count(entityClass: EntityClass, options?: FindManyOptions): Promise { - return this.em().count(entityClass, options) + private async _delete(metadata: EntityMetadata, ids: string[]) { + this.logger.debug(`delete ${metadata.name} ${ids.length} entities`) + await this.tracker?.writeDelete(metadata, ids) + await this.em().delete(metadata.target, ids) // TODO: should be split by chunks too? } - async countBy(entityClass: EntityClass, where: FindOptionsWhere | FindOptionsWhere[]): Promise { - return this.em().countBy(entityClass, where) + async count(target: EntityTarget, options?: FindManyOptions): Promise { + await this.commit() + return await this.em().count(target, options) } - async find(entityClass: EntityClass, options?: FindManyOptions): Promise { - return this.em().find(entityClass, options) + async countBy( + target: EntityTarget, + where: FindOptionsWhere | FindOptionsWhere[] + ): Promise { + await this.commit() + return await this.em().countBy(target, where) } - async findBy(entityClass: EntityClass, where: FindOptionsWhere | FindOptionsWhere[]): Promise { - return this.em().findBy(entityClass, where) + async find(target: EntityTarget, options: FindManyOptions): Promise { + await this.commit() + + const {cache, ...opts} = options + const res = await this.em().find(target, opts) + if (cache) this.cacheEntities(target, res, options?.relations) + + return res } - async findOne(entityClass: EntityClass, options: FindOneOptions): Promise { - return this.em().findOne(entityClass, options).then(noNull) + async findBy( + target: EntityTarget, + where: FindOptionsWhere | FindOptionsWhere[], + cache?: boolean + ): Promise { + await this.commit() + + const res = await this.em().findBy(target, where) + if (cache) this.cacheEntities(target, res) + + return res } - async findOneBy(entityClass: EntityClass, where: FindOptionsWhere | FindOptionsWhere[]): Promise { - return this.em().findOneBy(entityClass, where).then(noNull) + async findOne( + target: EntityTarget, + options: FindOneOptions + ): Promise { + await this.commit() + + const {cache, ...opts} = options + const res = await this.em().findOne(target, opts).then(noNull) + if (res != null && cache) this.cacheEntities(target, res, options?.relations) + + return res } - async findOneOrFail(entityClass: EntityTarget, options: FindOneOptions): Promise { - return this.em().findOneOrFail(entityClass, options) + async findOneBy( + target: EntityTarget, + where: FindOptionsWhere | FindOptionsWhere[], + cache?: boolean + ): Promise { + await this.commit() + + const res = await this.em().findOneBy(target, where).then(noNull) + if (res != null && cache) this.cacheEntities(target, res) + + return res + } + + async findOneOrFail(target: EntityTarget, options: FindOneOptions): Promise { + await this.commit() + + const {cache, ...opts} = options + const res = await this.em().findOneOrFail(target, opts) + if (cache) this.cacheEntities(target, res, options?.relations) + + return res + } + + async findOneByOrFail( + target: EntityTarget, + where: FindOptionsWhere | FindOptionsWhere[], + cache?: boolean + ): Promise { + await this.commit() + + const res = await this.em().findOneByOrFail(target, where) + if (cache) this.cacheEntities(target, res) + + return res + } + + async get(entityClass: EntityTarget, id: string): Promise + async get(entityClass: EntityTarget, options: GetOptions): Promise + async get( + entityClass: EntityTarget, + idOrOptions: string | GetOptions + ): Promise { + const {id, relations} = parseGetOptions(idOrOptions) + + let entity = this.getFromCache(entityClass, id, relations) + if (entity !== undefined) return entity ?? undefined + + return await this.findOne(entityClass, {where: {id} as any, relations}) } - async findOneByOrFail(entityClass: EntityTarget, where: FindOptionsWhere | FindOptionsWhere[]): Promise { - return this.em().findOneByOrFail(entityClass, where) + async getOrFail(entityClass: EntityTarget, id: string): Promise + async getOrFail(entityClass: EntityTarget, options: GetOptions): Promise + async getOrFail( + entityClass: EntityTarget, + idOrOptions: string | GetOptions + ): Promise { + const options = parseGetOptions(idOrOptions) + let e = await this.get(entityClass, options) + + if (e == null) { + const metadata = this.getEntityMetadata(entityClass) + throw new Error(`Missing entity ${metadata.name} with id "${options.id}"`) + } + + return e } - get(entityClass: EntityClass, optionsOrId: FindOneOptions | string): Promise { - if (typeof optionsOrId == 'string') { - return this.findOneBy(entityClass, {id: optionsOrId} as any) + async commit(): Promise { + await this.pendingCommit?.promise() + + this.pendingCommit = createFuture() + try { + const changeSets = this.computeChangeSets() + + for (const {metadata, upserts} of changeSets) { + if (upserts.length === 0) continue + await this._upsert(metadata, upserts) + } + + for (const {metadata, inserts} of changeSets) { + if (inserts.length === 0) continue + await this._insert(metadata, inserts) + } + + for (const {metadata, deletes} of [...changeSets].reverse()) { + if (deletes.length === 0) continue + await this._delete(metadata, deletes) + } + + for (const {metadata, extraUpserts} of changeSets) { + if (extraUpserts.length === 0) continue + await this._upsert(metadata, extraUpserts) + } + } finally { + this.pendingCommit.resolve() + this.pendingCommit = undefined + } + } + + clear(): void { + this.cache.clear() + this.changes.clear() + } + + async flush(): Promise { + await this.commit() + this.clear() + } + + private getFromCache( + target: EntityTarget, + id: string, + mask?: FindOptionsRelations + ): E | null | undefined { + const metadata = this.getEntityMetadata(target) + const cached = this.cache.get(metadata, id) + + if (cached == null) { + return undefined + } else if (cached.value == null) { + return null } else { - return this.findOne(entityClass, optionsOrId) + const entity = cached.value + + const clonedEntity = metadata.create() + + for (const column of metadata.nonVirtualColumns) { + const objectColumnValue = column.getEntityValue(entity) + if (objectColumnValue !== undefined) { + column.setEntityValue(clonedEntity, copy(objectColumnValue)) + } + } + + if (mask != null) { + for (const relation of metadata.relations) { + const inverseMask = mask[relation.propertyName] + if (!inverseMask) continue + + const inverseEntityMock = relation.getEntityValue(entity) + + if (inverseEntityMock === undefined) { + return undefined // relation is missing, but required + } else if (inverseEntityMock === null) { + relation.setEntityValue(clonedEntity, null) + } else { + const cachedInverseEntity = this.getFromCache( + relation.inverseEntityMetadata.target, + inverseEntityMock.id, + typeof inverseMask === 'boolean' ? undefined : inverseMask + ) + + if (cachedInverseEntity === undefined) { + return undefined // unable to build whole relation chain + } else { + relation.setEntityValue(clonedEntity, cachedInverseEntity) + } + } + } + } + + return clonedEntity + } + } + + private cacheEntities( + target: EntityTarget, + e: E | E[], + mask?: FindOptionsRelations + ) { + const metadata = this.getEntityMetadata(target) + + e = Array.isArray(e) ? e : [e] + for (const entity of e) { + traverseEntity({ + metadata, + entity, + mask: mask || null, + cb: (e, md) => { + this.cache.add(md, e) + }, + }) + } + } + + private computeChangeSets() { + const changes = this.changes.values() + + const changeSets: ChangeSet[] = [] + for (const metadata of this.commitOrder) { + const entityChanges = changes.get(metadata) + if (entityChanges == null) continue + + const changeSet = this.computeChangeSet(metadata, entityChanges) + changeSets.push(changeSet) } + + this.changes.clear() + + return changeSets + } + + private computeChangeSet(metadata: EntityMetadata, changes: Map): ChangeSet { + const inserts: EntityLiteral[] = [] + const upserts: EntityLiteral[] = [] + const deletes: string[] = [] + const extraUpserts: EntityLiteral[] = [] + + for (const [id, type] of changes) { + const cached = this.cache.get(metadata, id) + + switch (type) { + case ChangeType.Insert: { + assert(cached?.value != null, `unable to insert entity ${metadata.name} ${id}`) + + inserts.push(cached.value) + + const extraUpsert = this.extractExtraUpsert(metadata, cached.value) + if (extraUpsert != null) { + extraUpserts.push(extraUpsert) + } + + break + } + case ChangeType.Upsert: { + assert(cached?.value != null, `unable to upsert entity ${metadata.name} ${id}`) + + upserts.push(cached.value) + + const extraUpsert = this.extractExtraUpsert(metadata, cached.value) + if (extraUpsert != null) { + extraUpserts.push(extraUpsert) + } + + break + } + case ChangeType.Delete: { + deletes.push(id) + break + } + } + } + + return {metadata, inserts, upserts, extraUpserts, deletes} + } + + private extractExtraUpsert(metadata: EntityMetadata, entity: E) { + const commitOrderIndex = this.commitOrder.indexOf(metadata) + + let extraUpsert: E | undefined + for (const relation of metadata.relations) { + if (relation.foreignKeys.length == 0) continue + + const inverseEntity = relation.getEntityValue(entity) + if (inverseEntity == null) continue + + const inverseMetadata = relation.inverseEntityMetadata + if (metadata === inverseMetadata && inverseEntity.id === entity.id) continue + + const invCommitOrderIndex = this.commitOrder.indexOf(inverseMetadata) + if (invCommitOrderIndex < commitOrderIndex) continue + + const isInverseInserted = this.changes.isInserted(inverseMetadata, inverseEntity.id) + if (!isInverseInserted) continue + + if (extraUpsert == null) { + extraUpsert = metadata.create() as E + extraUpsert.id = entity.id + Object.assign(extraUpsert, entity) + } + + relation.setEntityValue(entity, undefined) + } + + return extraUpsert + } + + private getEntityMetadata(target: EntityTarget) { + const em = this.em() + return em.connection.getMetadata(target) + } + + @def + private reverseCommitOrder() { + return [...this.commitOrder].reverse() } } +function traverseEntity({ + metadata, + entity, + mask, + cb, +}: { + metadata: EntityMetadata + entity: EntityLiteral | null + mask: FindOptionsRelations | null + cb: (e: EntityLiteral, metadata: EntityMetadata) => void +}) { + if (entity == null) return + + if (mask != null) { + for (const relation of metadata.relations) { + const inverseMask = mask[relation.propertyName] + if (!inverseMask) continue + + const inverseEntity = relation.getEntityValue(entity) + if (relation.isOneToMany || relation.isManyToMany) { + if (!Array.isArray(inverseEntity)) continue + for (const ie of inverseEntity) { + traverseEntity({ + metadata: relation.inverseEntityMetadata, + entity: ie, + mask: inverseMask === true ? null : inverseMask, + cb, + }) + } + } else { + traverseEntity({ + metadata: relation.inverseEntityMetadata, + entity: inverseEntity, + mask: inverseMask === true ? null : inverseMask, + cb, + }) + } + } + } + + cb(entity, metadata) +} function* splitIntoBatches(list: T[], maxBatchSize: number): Generator { if (list.length <= maxBatchSize) { @@ -245,7 +629,14 @@ function* splitIntoBatches(list: T[], maxBatchSize: number): Generator { } } - function noNull(val: null | undefined | T): T | undefined { return val == null ? undefined : val } + +function parseGetOptions(idOrOptions: string | GetOptions): GetOptions { + if (typeof idOrOptions === 'string') { + return {id: idOrOptions} + } else { + return idOrOptions + } +} diff --git a/typeorm/typeorm-store/src/test/database.test.ts b/typeorm/typeorm-store/src/test/database.test.ts index 48ffbf1c4..cb2c38613 100644 --- a/typeorm/typeorm-store/src/test/database.test.ts +++ b/typeorm/typeorm-store/src/test/database.test.ts @@ -206,8 +206,8 @@ describe('TypeormDatabase', function() { ] }, async (store, block) => { expect(block).toEqual({height: 2, hash: 'c-2'}) - expect(await store.find(Data)).toEqual([a1]) - await store.remove(a1) + expect(await store.find(Data, {})).toEqual([a1]) + await store.delete(a1) }) expect(await em.find(Data)).toEqual([]) diff --git a/typeorm/typeorm-store/src/test/store.test.ts b/typeorm/typeorm-store/src/test/store.test.ts index 212f46c73..5f1664697 100644 --- a/typeorm/typeorm-store/src/test/store.test.ts +++ b/typeorm/typeorm-store/src/test/store.test.ts @@ -4,24 +4,25 @@ import {Equal} from 'typeorm' import {Store} from '../store' import {Item, Order} from './lib/model' import {getEntityManager, useDatabase} from './util' +import {sortMetadatasInCommitOrder} from '../utils/commitOrder' describe("Store", function() { - describe(".save()", function() { + describe(".upsert()", function() { useDatabase([ `CREATE TABLE item (id text primary key , name text)` ]) it("saving of a single entity", async function() { let store = await createStore() - await store.save(new Item('1', 'a')) - await expect(getItems()).resolves.toEqual([{id: '1', name: 'a'}]) + await store.upsert(new Item('1', 'a')) + await expect(getItems(store)).resolves.toEqual([{id: '1', name: 'a'}]) }) it("saving of multiple entities", async function() { let store = await createStore() - await store.save([new Item('1', 'a'), new Item('2', 'b')]) - await expect(getItems()).resolves.toEqual([ + await store.upsert([new Item('1', 'a'), new Item('2', 'b')]) + await expect(getItems(store)).resolves.toEqual([ {id: '1', name: 'a'}, {id: '2', name: 'b'} ]) @@ -33,18 +34,18 @@ describe("Store", function() { for (let i = 0; i < 20000; i++) { items.push(new Item(''+i)) } - await store.save(items) + await store.upsert(items) expect(await store.count(Item)).toEqual(items.length) }) it("updates", async function() { let store = await createStore() - await store.save(new Item('1', 'a')) - await store.save([ + await store.upsert(new Item('1', 'a')) + await store.upsert([ new Item('1', 'foo'), new Item('2', 'b') ]) - await expect(getItems()).resolves.toEqual([ + await expect(getItems(store)).resolves.toEqual([ {id: '1', name: 'foo'}, {id: '2', name: 'b'} ]) @@ -61,29 +62,29 @@ describe("Store", function() { it("removal by passing an entity", async function() { let store = await createStore() - await store.remove(new Item('1')) - await expect(getItemIds()).resolves.toEqual(['2', '3']) + await store.delete(new Item('1')) + await expect(getItemIds(store)).resolves.toEqual(['2', '3']) }) it("removal by passing an array of entities", async function() { let store = await createStore() - await store.remove([ + await store.delete([ new Item('1'), new Item('3') ]) - await expect(getItemIds()).resolves.toEqual(['2']) + await expect(getItemIds(store)).resolves.toEqual(['2']) }) it("removal by passing an id", async function() { let store = await createStore() - await store.remove(Item, '1') - await expect(getItemIds()).resolves.toEqual(['2', '3']) + await store.delete(Item, '1') + await expect(getItemIds(store)).resolves.toEqual(['2', '3']) }) it("removal by passing an array of ids", async function() { let store = await createStore() - await store.remove(Item, ['1', '2']) - await expect(getItemIds()).resolves.toEqual(['3']) + await store.delete(Item, ['1', '2']) + await expect(getItemIds(store)).resolves.toEqual(['3']) }) }) @@ -97,11 +98,11 @@ describe("Store", function() { `INSERT INTO "order" (id, item_id, qty) values ('2', '2', 3)` ]) - it(".save() doesn't clear reference (single row update)", async function() { + it(".upsert() doesn't clear reference (single row update)", async function() { let store = await createStore() let order = assertNotNull(await store.get(Order, '1')) order.qty = 5 - await store.save(order) + await store.upsert(order) let newOrder = await store.findOneOrFail(Order, { where: {id: Equal('1')}, relations: { @@ -112,7 +113,7 @@ describe("Store", function() { expect(newOrder.item.id).toEqual('1') }) - it(".save() doesn't clear reference (multi row update)", async function() { + it(".upsert() doesn't clear reference (multi row update)", async function() { let store = await createStore() let orders = await store.find(Order, {order: {id: 'ASC'}}) let items = await store.find(Item, {order: {id: 'ASC'}}) @@ -120,7 +121,7 @@ describe("Store", function() { orders[0].qty = 5 orders[1].qty = 1 orders[1].item = items[0] - await store.save(orders) + await store.upsert(orders) let newOrders = await store.find(Order, { relations: { @@ -154,17 +155,15 @@ describe("Store", function() { export function createStore(): Promise { return getEntityManager().then( - em => new Store(() => em) + em => new Store(() => em, {commitOrder: sortMetadatasInCommitOrder(em.connection.entityMetadatas)}) ) } -export async function getItems(): Promise { - let em = await getEntityManager() - return em.find(Item) +export async function getItems(store: Store): Promise { + return store.find(Item, {where: {}}) } - -export function getItemIds(): Promise { - return getItems().then(items => items.map(it => it.id).sort()) +export function getItemIds(store: Store): Promise { + return getItems(store).then((items) => items.map((it) => it.id).sort()) } diff --git a/typeorm/typeorm-store/src/utils/cacheMap.ts b/typeorm/typeorm-store/src/utils/cacheMap.ts new file mode 100644 index 000000000..1eca570a2 --- /dev/null +++ b/typeorm/typeorm-store/src/utils/cacheMap.ts @@ -0,0 +1,101 @@ +import {EntityMetadata, ObjectLiteral} from 'typeorm' +import {copy} from './misc' +import {Logger} from '@subsquid/logger' + +export class CachedEntity { + constructor(public value: E | null = null) {} +} + +export class CacheMap { + private map: Map> = new Map() + private logger: Logger + + constructor(private opts: {logger: Logger}) { + this.logger = this.opts.logger.child('cache') + } + + exist(metadata: EntityMetadata, id: string): boolean { + const cacheMap = this.getEntityCache(metadata) + const cachedEntity = cacheMap.get(id) + return !!cachedEntity?.value + } + + get(metadata: EntityMetadata, id: string): CachedEntity | undefined { + const cacheMap = this.getEntityCache(metadata) + return cacheMap.get(id) + } + + ensure(metadata: EntityMetadata, id: string): void { + const cacheMap = this.getEntityCache(metadata) + + if (cacheMap.has(id)) return + + cacheMap.set(id, new CachedEntity()) + this.logger.debug(`added empty entity ${metadata.name} ${id}`) + } + + delete(metadata: EntityMetadata, id: string): void { + const cacheMap = this.getEntityCache(metadata) + cacheMap.set(id, new CachedEntity()) + this.logger.debug(`deleted entity ${metadata.name} ${id}`) + } + + clear(): void { + this.logger.debug(`cleared`) + this.map.clear() + } + + add(metadata: EntityMetadata, entity: E, isNew = false): void { + const cacheMap = this.getEntityCache(metadata) + + let cached = cacheMap.get(entity.id) + if (cached == null) { + cached = new CachedEntity() + cacheMap.set(entity.id, cached) + } + + let cachedEntity = cached.value + if (cachedEntity == null) { + cachedEntity = cached.value = metadata.create() as E + cachedEntity.id = entity.id + this.logger.debug(`added entity ${metadata.name} ${entity.id}`) + } + + for (const column of metadata.nonVirtualColumns) { + const objectColumnValue = column.getEntityValue(entity) + if (isNew || objectColumnValue !== undefined) { + column.setEntityValue(cachedEntity, copy(objectColumnValue ?? null)) + } + } + + for (const relation of metadata.relations) { + if (!relation.isOwning) continue + + const inverseEntity = relation.getEntityValue(entity) as ObjectLiteral | null | undefined + const inverseMetadata = relation.inverseEntityMetadata + + if (inverseEntity != null) { + const mockEntity = inverseMetadata.create() + Object.assign(mockEntity, {id: inverseEntity.id}) + + relation.setEntityValue(cachedEntity, mockEntity) + } else if (isNew || inverseEntity === null) { + relation.setEntityValue(cachedEntity, null) + } + } + } + + values(): Map>> { + return new Map(this.map) + } + + private getEntityCache(metadata: EntityMetadata): Map> { + let map = this.map.get(metadata) + if (map == null) { + map = new Map() + this.map.set(metadata, map) + } + + return map as Map> + } +} diff --git a/typeorm/typeorm-store/src/utils/changeTracker.ts b/typeorm/typeorm-store/src/utils/changeTracker.ts new file mode 100644 index 000000000..5cdbd0eb3 --- /dev/null +++ b/typeorm/typeorm-store/src/utils/changeTracker.ts @@ -0,0 +1,105 @@ +import {Logger} from '@subsquid/logger' +import {EntityMetadata} from 'typeorm' + +export enum ChangeType { + Insert = 'insert', + Upsert = 'upsert', + Delete = 'delete', +} + +export class ChangeTracker { + private map: Map> = new Map() + private logger: Logger + + constructor(private opts: {logger: Logger}) { + this.logger = this.opts.logger.child('changes') + } + + trackInsert(metadata: EntityMetadata, id: string): void { + const prevType = this.get(metadata, id) + switch (prevType) { + case undefined: + this.set(metadata, id, ChangeType.Insert) + this.logger.debug(`entity ${metadata.name} ${id} already marked as ${ChangeType.Insert}`) + break + case ChangeType.Delete: + this.set(metadata, id, ChangeType.Upsert) + break + case ChangeType.Insert: + case ChangeType.Upsert: + throw new Error( + `${metadata.name} ${id} is already marked as ${ChangeType.Insert} or ${ChangeType.Upsert}` + ) + } + } + + trackUpsert(metadata: EntityMetadata, id: string): void { + const prevType = this.get(metadata, id) + switch (prevType) { + case ChangeType.Insert: + this.logger.debug(`entity ${metadata.name} ${id} already marked as ${ChangeType.Insert}`) + break + case ChangeType.Upsert: + this.logger.debug(`entity ${metadata.name} ${id} already marked as ${ChangeType.Upsert}`) + break + default: + this.set(metadata, id, ChangeType.Upsert) + break + } + } + + trackDelete(metadata: EntityMetadata, id: string): void { + const prevType = this.get(metadata, id) + switch (prevType) { + case ChangeType.Insert: + this.getChanges(metadata).delete(id) + break + case ChangeType.Delete: + this.logger.debug(`entity ${metadata.name} ${id} already marked as ${ChangeType.Delete}`) + break + default: + this.set(metadata, id, ChangeType.Delete) + } + } + + isInserted(metadata: EntityMetadata, id: string) { + return this.get(metadata, id) === ChangeType.Insert + } + + isUpserted(metadata: EntityMetadata, id: string) { + return this.get(metadata, id) === ChangeType.Upsert + } + + isDeleted(metadata: EntityMetadata, id: string) { + return this.get(metadata, id) === ChangeType.Delete + } + + clear(): void { + this.logger.debug(`cleared`) + this.map.clear() + } + + values(): Map> { + return new Map(this.map) + } + + private set(metadata: EntityMetadata, id: string, type: ChangeType): this { + this.getChanges(metadata).set(id, type) + this.logger.debug(`entity ${metadata.name} ${id} marked as ${type}`) + return this + } + + private get(metadata: EntityMetadata, id: string): ChangeType | undefined { + return this.getChanges(metadata).get(id) + } + + private getChanges(metadata: EntityMetadata): Map { + let map = this.map.get(metadata) + if (map == null) { + map = new Map() + this.map.set(metadata, map) + } + + return map + } +} diff --git a/typeorm/typeorm-store/src/hot.ts b/typeorm/typeorm-store/src/utils/changeWriter.ts similarity index 85% rename from typeorm/typeorm-store/src/hot.ts rename to typeorm/typeorm-store/src/utils/changeWriter.ts index 33030331b..0a1b30c3b 100644 --- a/typeorm/typeorm-store/src/hot.ts +++ b/typeorm/typeorm-store/src/utils/changeWriter.ts @@ -1,7 +1,7 @@ import {assertNotNull} from '@subsquid/util-internal' -import type {EntityManager, EntityMetadata} from 'typeorm' +import type {EntityManager, EntityMetadata, EntityTarget} from 'typeorm' import {ColumnMetadata} from 'typeorm/metadata/ColumnMetadata' -import {Entity, EntityClass} from './store' +import {EntityLiteral} from '../store' export interface RowRef { @@ -37,7 +37,7 @@ export interface ChangeRow { } -export class ChangeTracker { +export class ChangeWriter { private index = 0 constructor( @@ -48,22 +48,19 @@ export class ChangeTracker { this.statusSchema = this.escape(this.statusSchema) } - trackInsert(type: EntityClass, entities: Entity[]): Promise { - let meta = this.getEntityMetadata(type) + writeInsert(metadata: EntityMetadata, entities: EntityLiteral[]): Promise { return this.writeChangeRows(entities.map(e => { return { kind: 'insert', - table: meta.tableName, + table: metadata.tableName, id: e.id } })) } - async trackUpsert(type: EntityClass, entities: Entity[]): Promise { - let meta = this.getEntityMetadata(type) - + async writeUpsert(metadata: EntityMetadata, entities: EntityLiteral[]): Promise { let touchedRows = await this.fetchEntities( - meta, + metadata, entities.map(e => e.id) ).then( entities => new Map( @@ -76,35 +73,34 @@ export class ChangeTracker { if (fields) { return { kind: 'update', - table: meta.tableName, + table: metadata.tableName, id: e.id, fields } } else { return { kind: 'insert', - table: meta.tableName, + table: metadata.tableName, id: e.id, } } })) } - async trackDelete(type: EntityClass, ids: string[]): Promise { - let meta = this.getEntityMetadata(type) - let deletedEntities = await this.fetchEntities(meta, ids) + async writeDelete(metadata: EntityMetadata, ids: string[]): Promise { + let deletedEntities = await this.fetchEntities(metadata, ids) return this.writeChangeRows(deletedEntities.map(e => { let {id, ...fields} = e return { kind: 'delete', - table: meta.tableName, + table: metadata.tableName, id: id, fields } })) } - private async fetchEntities(meta: EntityMetadata, ids: string[]): Promise { + private async fetchEntities(meta: EntityMetadata, ids: string[]): Promise { let entities = await this.em.query( `SELECT * FROM ${this.escape(meta.tableName)} WHERE id = ANY($1::text[])`, [ids] @@ -133,7 +129,7 @@ export class ChangeTracker { return entities } - private writeChangeRows(changes: ChangeRecord[]): Promise { + private async writeChangeRows(changes: ChangeRecord[]): Promise { let height = new Array(changes.length) let index = new Array(changes.length) let change = new Array(changes.length) @@ -149,11 +145,7 @@ export class ChangeTracker { sql += ' SELECT block_height, index, change::jsonb' sql += ' FROM unnest($1::int[], $2::int[], $3::text[]) AS i(block_height, index, change)' - return this.em.query(sql, [height, index, change]).then(() => {}) - } - - private getEntityMetadata(type: EntityClass): EntityMetadata { - return this.em.connection.getMetadata(type) + await this.em.query(sql, [height, index, change]) } private escape(name: string): string { diff --git a/typeorm/typeorm-store/src/utils/commitOrder.ts b/typeorm/typeorm-store/src/utils/commitOrder.ts new file mode 100644 index 000000000..b160b928b --- /dev/null +++ b/typeorm/typeorm-store/src/utils/commitOrder.ts @@ -0,0 +1,62 @@ +import {EntityMetadata} from 'typeorm' +import {RelationMetadata} from 'typeorm/metadata/RelationMetadata' + +enum NodeState { + Unvisited, + Visiting, + Visited, +} + +export function sortMetadatasInCommitOrder(entities: EntityMetadata[]): EntityMetadata[] { + let states: Map = new Map(entities.map((e) => [e.name, NodeState.Unvisited])) + let commitOrder: EntityMetadata[] = [] + + function visit(node: EntityMetadata) { + if (states.get(node.name) !== NodeState.Unvisited) return + + states.set(node.name, NodeState.Visiting) + + for (let edge of node.relations) { + if (edge.foreignKeys.length === 0) continue + + let target = edge.inverseEntityMetadata + let targetState = states.get(target.name) + + if (targetState === NodeState.Unvisited) { + visit(target) + } else if (targetState === NodeState.Visiting) { + let inverseEdge = target.relations.find((r) => r.inverseEntityMetadata === node) + if (inverseEdge != null) { + let edgeWeight = getWeight(edge) + let inverseEdgeWeight = getWeight(inverseEdge) + + if (edgeWeight > inverseEdgeWeight) { + for (let r of target.relations) { + visit(r.inverseEntityMetadata) + } + + states.set(target.name, NodeState.Visited) + commitOrder.push(target) + } + } + } + } + + let nodeState = states.get(node.name) + + if (nodeState !== NodeState.Visited) { + states.set(node.name, NodeState.Visited) + commitOrder.push(node) + } + } + + for (let node of entities) { + visit(node) + } + + return commitOrder +} + +function getWeight(edge: RelationMetadata) { + return edge.isNullable ? 0 : 1 +} diff --git a/typeorm/typeorm-store/src/utils/misc.ts b/typeorm/typeorm-store/src/utils/misc.ts new file mode 100644 index 000000000..e9a5cea68 --- /dev/null +++ b/typeorm/typeorm-store/src/utils/misc.ts @@ -0,0 +1,86 @@ +import {FindOptionsRelations, ObjectLiteral} from 'typeorm' + +export function* splitIntoBatches(list: T[], maxBatchSize: number): Generator { + if (list.length <= maxBatchSize) { + yield list + } else { + let offset = 0 + while (list.length - offset > maxBatchSize) { + yield list.slice(offset, offset + maxBatchSize) + offset += maxBatchSize + } + yield list.slice(offset) + } +} + +const copiedObjects = new WeakMap() + +export function copy(obj: T): T { + if (typeof obj !== 'object' || obj == null) { + return obj + } + + if (copiedObjects.has(obj)) { + return copiedObjects.get(obj) + } else if (obj instanceof Date) { + return new Date(obj) as any + } else if (Array.isArray(obj)) { + const clone = obj.map((i) => copy(i)) + copiedObjects.set(obj, clone) + return clone as any + } else if (obj instanceof Map) { + const clone = new Map(Array.from(obj).map((i) => copy(i))) + copiedObjects.set(obj, clone) + return clone as any + } else if (obj instanceof Set) { + const clone = new Set(Array.from(obj).map((i) => copy(i))) + copiedObjects.set(obj, clone) + return clone as any + } else if (ArrayBuffer.isView(obj)) { + return copyBuffer(obj) + } else { + const clone = Object.create(Object.getPrototypeOf(obj)) + copiedObjects.set(obj, clone) + + for (const k in obj) { + if (obj.hasOwnProperty(k)) { + clone[k] = copy(obj[k]) + } + } + + return clone + } +} + +function copyBuffer(buf: any) { + if (buf instanceof Buffer) { + return Buffer.from(buf) + } else { + return new buf.constructor(buf.buffer.slice(), buf.byteOffset, buf.length) + } +} + +export function mergeRelataions( + a: FindOptionsRelations, + b: FindOptionsRelations +): FindOptionsRelations { + const mergedObject: FindOptionsRelations = {} + + for (const key in a) { + mergedObject[key] = a[key] + } + + for (const key in b) { + const bValue = b[key] + const value = mergedObject[key] + if (typeof bValue === 'object') { + mergedObject[key] = ( + typeof value === 'object' ? mergeRelataions(value as any, bValue as any) : bValue + ) as any + } else { + mergedObject[key] = value || bValue + } + } + + return mergedObject +} diff --git a/typeorm/typeorm-store/src/transformers.ts b/typeorm/typeorm-store/src/utils/transformers.ts similarity index 65% rename from typeorm/typeorm-store/src/transformers.ts rename to typeorm/typeorm-store/src/utils/transformers.ts index 15c3b86f2..0f68d6f41 100644 --- a/typeorm/typeorm-store/src/transformers.ts +++ b/typeorm/typeorm-store/src/utils/transformers.ts @@ -1,3 +1,4 @@ +import {BigDecimal} from '@subsquid/big-decimal' import {ValueTransformer} from 'typeorm' export const bigintTransformer: ValueTransformer = { @@ -18,23 +19,11 @@ export const floatTransformer: ValueTransformer = { }, } -const decimal = { - get BigDecimal(): any { - throw new Error('Package `@subsquid/big-decimal` is not installed') - }, -} - -try { - Object.defineProperty(decimal, 'BigDecimal', { - value: require('@subsquid/big-decimal').BigDecimal, - }) -} catch (e) {} - export const bigdecimalTransformer: ValueTransformer = { to(x?: any) { return x?.toString() }, from(s?: any): any | undefined { - return s == null ? undefined : decimal.BigDecimal(s) + return s == null ? undefined : BigDecimal(s) }, } diff --git a/typeorm/typeorm-store/src/tx.ts b/typeorm/typeorm-store/src/utils/tx.ts similarity index 95% rename from typeorm/typeorm-store/src/tx.ts rename to typeorm/typeorm-store/src/utils/tx.ts index 5c8f204be..2b94bf01e 100644 --- a/typeorm/typeorm-store/src/tx.ts +++ b/typeorm/typeorm-store/src/utils/tx.ts @@ -1,5 +1,5 @@ import type {DataSource, EntityManager} from "typeorm" -import type {IsolationLevel} from "./database" +import type {IsolationLevel} from "../database" export interface Tx { From 5403ef561864ab8795e6575cd36a348fa8ef4187 Mon Sep 17 00:00:00 2001 From: belopash Date: Mon, 27 May 2024 22:28:22 +0500 Subject: [PATCH 02/11] save --- typeorm/typeorm-store/src/database.ts | 151 ++--- typeorm/typeorm-store/src/index.ts | 2 +- typeorm/typeorm-store/src/store.ts | 574 ++++++------------ typeorm/typeorm-store/src/test/lib/model.ts | 2 +- typeorm/typeorm-store/src/test/store.test.ts | 16 +- typeorm/typeorm-store/src/utils/cacheMap.ts | 39 +- .../typeorm-store/src/utils/changeTracker.ts | 105 ---- .../typeorm-store/src/utils/changeWriter.ts | 5 +- .../typeorm-store/src/utils/commitOrder.ts | 26 +- typeorm/typeorm-store/src/utils/misc.ts | 57 +- .../typeorm-store/src/utils/stateManager.ts | 295 +++++++++ 11 files changed, 678 insertions(+), 594 deletions(-) delete mode 100644 typeorm/typeorm-store/src/utils/changeTracker.ts create mode 100644 typeorm/typeorm-store/src/utils/stateManager.ts diff --git a/typeorm/typeorm-store/src/database.ts b/typeorm/typeorm-store/src/database.ts index ea3c3dd28..cfeb813eb 100644 --- a/typeorm/typeorm-store/src/database.ts +++ b/typeorm/typeorm-store/src/database.ts @@ -4,7 +4,9 @@ import assert from 'assert' import {DataSource, EntityManager} from 'typeorm' import {ChangeWriter, rollbackBlock} from './utils/changeWriter' import {DatabaseState, FinalTxInfo, HashAndHeight, HotTxInfo} from './interfaces' -import {Store} from './store' +import {CacheMode, FlushMode, ResetMode, Store} from './store' +import {createLogger} from '@subsquid/logger' +import {StateManager} from './utils/stateManager' import {sortMetadatasInCommitOrder} from './utils/commitOrder' @@ -14,14 +16,21 @@ export type IsolationLevel = 'SERIALIZABLE' | 'READ COMMITTED' | 'REPEATABLE REA export interface TypeormDatabaseOptions { supportHotBlocks?: boolean isolationLevel?: IsolationLevel + flushMode?: FlushMode + resetMode?: ResetMode + cacheMode?: CacheMode stateSchema?: string projectDir?: string } +const STATE_MANAGERS: WeakMap = new WeakMap() export class TypeormDatabase { private statusSchema: string private isolationLevel: IsolationLevel + private flushMode: FlushMode + private resetMode: ResetMode + private cacheMode: CacheMode private con?: DataSource private projectDir: string @@ -30,6 +39,9 @@ export class TypeormDatabase { constructor(options?: TypeormDatabaseOptions) { this.statusSchema = options?.stateSchema || 'squid_processor' this.isolationLevel = options?.isolationLevel || 'SERIALIZABLE' + this.resetMode = options?.resetMode || 'BATCH' + this.flushMode = options?.flushMode || 'AUTO' + this.cacheMode = options?.cacheMode || 'ALL' this.supportsHotBlocks = options?.supportHotBlocks !== false this.projectDir = options?.projectDir || process.cwd() } @@ -43,8 +55,8 @@ export class TypeormDatabase { await this.con.initialize() try { - return await this.con.transaction('SERIALIZABLE', em => this.initTransaction(em)) - } catch(e: any) { + return await this.con.transaction('SERIALIZABLE', (em) => this.initTransaction(em)) + } catch (e: any) { await this.con.destroy().catch(() => {}) // ignore error this.con = undefined throw e @@ -52,39 +64,37 @@ export class TypeormDatabase { } async disconnect(): Promise { - await this.con?.destroy().finally(() => this.con = undefined) + await this.con?.destroy().finally(() => (this.con = undefined)) } private async initTransaction(em: EntityManager): Promise { let schema = this.escapedSchema() - await em.query( - `CREATE SCHEMA IF NOT EXISTS ${schema}` - ) + await em.query(`CREATE SCHEMA IF NOT EXISTS ${schema}`) await em.query( `CREATE TABLE IF NOT EXISTS ${schema}.status (` + - `id int4 primary key, ` + - `height int4 not null, ` + - `hash text DEFAULT '0x', ` + - `nonce int4 DEFAULT 0`+ - `)` + `id int4 primary key, ` + + `height int4 not null, ` + + `hash text DEFAULT '0x', ` + + `nonce int4 DEFAULT 0` + + `)` ) - await em.query( // for databases created by prev version of typeorm store + await em.query( + // for databases created by prev version of typeorm store `ALTER TABLE ${schema}.status ADD COLUMN IF NOT EXISTS hash text DEFAULT '0x'` ) - await em.query( // for databases created by prev version of typeorm store - `ALTER TABLE ${schema}.status ADD COLUMN IF NOT EXISTS nonce int DEFAULT 0` - ) await em.query( - `CREATE TABLE IF NOT EXISTS ${schema}.hot_block (height int4 primary key, hash text not null)` + // for databases created by prev version of typeorm store + `ALTER TABLE ${schema}.status ADD COLUMN IF NOT EXISTS nonce int DEFAULT 0` ) + await em.query(`CREATE TABLE IF NOT EXISTS ${schema}.hot_block (height int4 primary key, hash text not null)`) await em.query( `CREATE TABLE IF NOT EXISTS ${schema}.hot_change_log (` + - `block_height int4 not null references ${schema}.hot_block on delete cascade, ` + - `index int4 not null, ` + - `change jsonb not null, ` + - `PRIMARY KEY (block_height, index)` + - `)` + `block_height int4 not null references ${schema}.hot_block on delete cascade, ` + + `index int4 not null, ` + + `change jsonb not null, ` + + `PRIMARY KEY (block_height, index)` + + `)` ) let status: (HashAndHeight & {nonce: number})[] = await em.query( @@ -95,9 +105,7 @@ export class TypeormDatabase { status.push({height: -1, hash: '0x', nonce: 0}) } - let top: HashAndHeight[] = await em.query( - `SELECT height, hash FROM ${schema}.hot_block ORDER BY height` - ) + let top: HashAndHeight[] = await em.query(`SELECT height, hash FROM ${schema}.hot_block ORDER BY height`) return assertStateInvariants({...status[0], top}) } @@ -111,15 +119,13 @@ export class TypeormDatabase { assert(status.length == 1) - let top: HashAndHeight[] = await em.query( - `SELECT hash, height FROM ${schema}.hot_block ORDER BY height` - ) + let top: HashAndHeight[] = await em.query(`SELECT hash, height FROM ${schema}.hot_block ORDER BY height`) return assertStateInvariants({...status[0], top}) } transact(info: FinalTxInfo, cb: (store: Store) => Promise): Promise { - return this.submit(async em => { + return this.submit(async (em) => { let state = await this.getState(em) let {prevHead: prev, nextHead: next} = info @@ -147,15 +153,21 @@ export class TypeormDatabase { }) } - transactHot2(info: HotTxInfo, cb: (store: Store, sliceBeg: number, sliceEnd: number) => Promise): Promise { - return this.submit(async em => { + transactHot2( + info: HotTxInfo, + cb: (store: Store, sliceBeg: number, sliceEnd: number) => Promise + ): Promise { + return this.submit(async (em) => { let state = await this.getState(em) let chain = [state, ...state.top] assertChainContinuity(info.baseHead, info.newBlocks) assert(info.finalizedHead.height <= (maybeLast(info.newBlocks) ?? info.baseHead).height) - assert(chain.find(b => b.hash === info.baseHead.hash), RACE_MSG) + assert( + chain.find((b) => b.hash === info.baseHead.hash), + RACE_MSG + ) if (info.newBlocks.length == 0) { assert(last(chain).hash === info.baseHead.hash, RACE_MSG) } @@ -170,18 +182,14 @@ export class TypeormDatabase { if (info.newBlocks.length) { let finalizedEnd = info.finalizedHead.height - info.newBlocks[0].height + 1 if (finalizedEnd > 0) { - await this.performUpdates(store => cb(store, 0, finalizedEnd), em) + await this.performUpdates((store) => cb(store, 0, finalizedEnd), em) } else { finalizedEnd = 0 } for (let i = finalizedEnd; i < info.newBlocks.length; i++) { let b = info.newBlocks[i] await this.insertHotBlock(em, b) - await this.performUpdates( - store => cb(store, i, i + 1), - em, - new ChangeWriter(em, this.statusSchema, b.height) - ) + await this.performUpdates((store) => cb(store, i, i + 1), em, new ChangeWriter(em, this.statusSchema, b.height)) } } @@ -196,17 +204,14 @@ export class TypeormDatabase { } private deleteHotBlocks(em: EntityManager, finalizedHeight: number): Promise { - return em.query( - `DELETE FROM ${this.escapedSchema()}.hot_block WHERE height <= $1`, - [finalizedHeight] - ) + return em.query(`DELETE FROM ${this.escapedSchema()}.hot_block WHERE height <= $1`, [finalizedHeight]) } private insertHotBlock(em: EntityManager, block: HashAndHeight): Promise { - return em.query( - `INSERT INTO ${this.escapedSchema()}.hot_block (height, hash) VALUES ($1, $2)`, - [block.height, block.hash] - ) + return em.query(`INSERT INTO ${this.escapedSchema()}.hot_block (height, hash) VALUES ($1, $2)`, [ + block.height, + block.hash, + ]) } private async updateStatus(em: EntityManager, nonce: number, next: HashAndHeight): Promise { @@ -221,36 +226,30 @@ export class TypeormDatabase { // Will never happen if isolation level is SERIALIZABLE or REPEATABLE_READ, // but occasionally people use multiprocessor setups and READ_COMMITTED. - assert.strictEqual( - rowsChanged, - 1, - RACE_MSG - ) + assert.strictEqual(rowsChanged, 1, RACE_MSG) } private async performUpdates( cb: (store: Store) => Promise, em: EntityManager, - changeTracker?: ChangeWriter + changeWriter?: ChangeWriter ): Promise { - let running = true - - let store = new Store( - () => { - assert(running, `too late to perform db updates, make sure you haven't forgot to await on db query`) - return em - }, - { - tracker: changeTracker, - commitOrder: this.getCommitOrder() - } - ) + let store = new Store({ + em, + state: this.getStateManager(), + logger: this.getLogger(), + changes: changeWriter, + cacheMode: this.cacheMode, + flushMode: this.flushMode, + resetMode: this.resetMode, + }) try { await cb(store) await store.flush() + if (this.resetMode === 'BATCH') store.reset() } finally { - running = false + store._close() } } @@ -261,7 +260,7 @@ export class TypeormDatabase { let con = this.con assert(con != null, 'not connected') return await con.transaction(this.isolationLevel, tx) - } catch(e: any) { + } catch (e: any) { if (e.code == '40001' && retries) { retries -= 1 } else { @@ -277,10 +276,22 @@ export class TypeormDatabase { } @def - private getCommitOrder() { - let con = this.con - assert(con != null, 'not connected') - return sortMetadatasInCommitOrder(con.entityMetadatas) + private getLogger() { + return createLogger('sqd:typeorm-db') + } + + private getStateManager() { + let con = assertNotNull(this.con) + let stateManager = STATE_MANAGERS.get(con) + if (stateManager != null) return stateManager + + stateManager = new StateManager({ + commitOrder: sortMetadatasInCommitOrder(con), + logger: this.getLogger(), + }) + STATE_MANAGERS.set(con, stateManager) + + return stateManager } } diff --git a/typeorm/typeorm-store/src/index.ts b/typeorm/typeorm-store/src/index.ts index 40e8a0501..b52d76e34 100644 --- a/typeorm/typeorm-store/src/index.ts +++ b/typeorm/typeorm-store/src/index.ts @@ -1,4 +1,4 @@ export * from './database' -export {FindManyOptions, FindOneOptions, Store, EntityLiteral} from './store' +export {FindManyOptions, FindOneOptions, Store} from './store' export * from './decorators' export * from './utils/transformers' diff --git a/typeorm/typeorm-store/src/store.ts b/typeorm/typeorm-store/src/store.ts index 7bbfb21c3..ccf22ba61 100644 --- a/typeorm/typeorm-store/src/store.ts +++ b/typeorm/typeorm-store/src/store.ts @@ -1,34 +1,20 @@ -import assert from 'assert' -import { - EntityManager, - EntityMetadata, - FindOptionsOrder, - FindOptionsRelations, - FindOptionsWhere, - ObjectLiteral, -} from 'typeorm' +import {EntityManager, EntityMetadata, FindOptionsOrder, FindOptionsRelations, FindOptionsWhere} from 'typeorm' import {EntityTarget} from 'typeorm/common/EntityTarget' import {ChangeWriter} from './utils/changeWriter' -import {CacheMap} from './utils/cacheMap' -import {ChangeTracker, ChangeType} from './utils/changeTracker' +import {StateManager} from './utils/stateManager' import {createLogger, Logger} from '@subsquid/logger' -import {createFuture, def, Future} from '@subsquid/util-internal' -import {copy} from './utils/misc' +import {createFuture, Future} from '@subsquid/util-internal' +import {EntityLiteral, noNull, splitIntoBatches, traverseEntity} from './utils/misc' import {ColumnMetadata} from 'typeorm/metadata/ColumnMetadata' +import assert from 'assert' export {EntityTarget} -export interface EntityLiteral extends ObjectLiteral { - id: string -} +export type FlushMode = 'AUTO' | 'BATCH' | 'ALWAYS' -export type ChangeSet = { - metadata: EntityMetadata - inserts: EntityLiteral[] - upserts: EntityLiteral[] - deletes: string[] - extraUpserts: EntityLiteral[] -} +export type ResetMode = 'BATCH' | 'MANUAL' | 'FLUSH' + +export type CacheMode = 'ALL' | 'MANUAL' export interface GetOptions { id: string @@ -74,33 +60,54 @@ export interface FindManyOptions extends FindOneOptions { cache?: boolean } +export interface StoreOptions { + em: EntityManager + state: StateManager + changes?: ChangeWriter + logger?: Logger + flushMode: FlushMode + resetMode: ResetMode + cacheMode: CacheMode +} + /** * Restricted version of TypeORM entity manager for squid data handlers. */ export class Store { - protected commitOrder: EntityMetadata[] - protected tracker?: ChangeWriter - protected changes: ChangeTracker - protected cache: CacheMap - protected logger: Logger + protected em: EntityManager + protected state: StateManager + protected changes?: ChangeWriter + protected logger?: Logger + + protected flushMode: FlushMode + protected resetMode: ResetMode + protected cacheMode: CacheMode protected pendingCommit?: Future + protected isClosed = false - constructor( - protected em: () => EntityManager, - { - commitOrder, - tracker, - }: { - commitOrder: EntityMetadata[] - tracker?: ChangeWriter - } - ) { - this.logger = createLogger('sqd:typeorm-store') - this.commitOrder = commitOrder - this.tracker = tracker - this.cache = new CacheMap({logger: this.logger}) - this.changes = new ChangeTracker({logger: this.logger}) + constructor({em, changes, logger, state, flushMode, resetMode, cacheMode}: StoreOptions) { + this.em = em + this.changes = changes + this.logger = logger?.child('store') + this.state = state + this.flushMode = flushMode + this.resetMode = resetMode + this.cacheMode = cacheMode + } + + /** + * @internal + */ + get _em() { + return this.em + } + + /** + * @internal + */ + get _state() { + return this.state } /** @@ -116,19 +123,15 @@ export class Store { * It always executes a primitive operation without cascades, relations, etc. */ async upsert(e: E | E[]): Promise { - await this.pendingCommit?.promise() - - let entities = Array.isArray(e) ? e : [e] - if (entities.length == 0) return - - for (const entity of entities) { - const md = this.getEntityMetadata(entity.constructor) - - const isNew = this.changes.isDeleted(md, entity.id) + return await this.performWrite(async () => { + let entities = Array.isArray(e) ? e : [e] + if (entities.length == 0) return - this.changes.trackUpsert(md, entity.id) - this.cache.add(md, entity, isNew) - } + for (const entity of entities) { + const md = this.getEntityMetadata(entity.constructor) + this.state.upsert(md, entity) + } + }) } private getFkSignature(fk: ColumnMetadata[], entity: any): bigint { @@ -141,21 +144,23 @@ export class Store { } private async _upsert(metadata: EntityMetadata, entities: EntityLiteral[]): Promise { - this.logger.debug(`upsert ${entities.length} ${metadata.name} entities`) - await this.tracker?.writeUpsert(metadata, entities) + this.logger?.debug(`upsert ${entities.length} ${metadata.name} entities`) + await this.changes?.writeUpsert(metadata, entities) let fk = metadata.columns.filter((c) => c.relationMetadata) if (fk.length == 0) return this.upsertMany(metadata.target, entities) - let currentSignature = this.getFkSignature(fk, entities[0]) + let signatures = entities + .map((e) => ({entity: e, value: this.getFkSignature(fk, e)})) + .sort((a, b) => (a.value > b.value ? -1 : b.value > a.value ? 1 : 0)) + let currentSignature = signatures[0].value let batch: EntityLiteral[] = [] - for (let e of entities) { - let sig = this.getFkSignature(fk, e) - if (sig === currentSignature) { - batch.push(e) + for (let s of signatures) { + if (s.value === currentSignature) { + batch.push(s.entity) } else { await this.upsertMany(metadata.target, batch) - currentSignature = sig - batch = [e] + currentSignature = s.value + batch = [s.entity] } } if (batch.length) { @@ -165,7 +170,7 @@ export class Store { private async upsertMany(target: EntityTarget, entities: EntityLiteral[]) { for (let b of splitIntoBatches(entities, 1000)) { - await this.em().upsert(target, b as any, ['id']) + await this.em.upsert(target, b as any, ['id']) } } @@ -176,28 +181,26 @@ export class Store { * Executes a primitive INSERT operation without cascades, relations, etc. */ async insert(e: E | E[]): Promise { - await this.pendingCommit?.promise() - - const entities = Array.isArray(e) ? e : [e] - if (entities.length == 0) return - - for (const entity of entities) { - const md = this.getEntityMetadata(entity.constructor) + return await this.performWrite(async () => { + const entities = Array.isArray(e) ? e : [e] + if (entities.length == 0) return - this.changes.trackInsert(md, entity.id) - this.cache.add(md, entity, true) - } + for (const entity of entities) { + const md = this.getEntityMetadata(entity.constructor) + this.state.insert(md, entity) + } + }) } private async _insert(metadata: EntityMetadata, entities: EntityLiteral[]) { - this.logger.debug(`insert ${entities.length} ${metadata.name} entities`) - await this.tracker?.writeInsert(metadata, entities) + this.logger?.debug(`insert ${entities.length} ${metadata.name} entities`) + await this.changes?.writeInsert(metadata, entities) await this.insertMany(metadata.target, entities) } private async insertMany(target: EntityTarget, entities: EntityLiteral[]) { for (let b of splitIntoBatches(entities, 1000)) { - await this.em().insert(target, b) + await this.em.insert(target, b) } } @@ -206,58 +209,59 @@ export class Store { * * Executes a primitive DELETE query without cascades, relations, etc. */ + async delete(e: E | E[]): Promise + async delete(target: EntityTarget, id: string | string[]): Promise async delete(e: E | E[] | EntityTarget, id?: string | string[]): Promise { - await this.pendingCommit?.promise() + return await this.performWrite(async () => { + if (id == null) { + const entities = Array.isArray(e) ? e : [e as E] + if (entities.length == 0) return - if (id == null) { - const entities = Array.isArray(e) ? e : [e as E] - if (entities.length == 0) return + for (const entity of entities) { + const md = this.getEntityMetadata(entity.constructor) - for (const entity of entities) { - const md = this.getEntityMetadata(entity.constructor) + this.state.delete(md, entity.id) + } + } else { + const ids = Array.isArray(id) ? id : [id] + if (ids.length == 0) return - this.changes.trackDelete(md, entity.id) - this.cache.delete(md, entity.id) - } - } else { - const ids = Array.isArray(id) ? id : [id] - if (ids.length == 0) return - - const md = this.getEntityMetadata(e as EntityTarget) - for (const id of ids) { - this.changes.trackDelete(md, id) - this.cache.delete(md, id) + const md = this.getEntityMetadata(e as EntityTarget) + for (const id of ids) { + this.state.delete(md, id) + } } - } + }) } private async _delete(metadata: EntityMetadata, ids: string[]) { - this.logger.debug(`delete ${metadata.name} ${ids.length} entities`) - await this.tracker?.writeDelete(metadata, ids) - await this.em().delete(metadata.target, ids) // TODO: should be split by chunks too? + this.logger?.debug(`delete ${metadata.name} ${ids.length} entities`) + await this.changes?.writeDelete(metadata, ids) + await this.em.delete(metadata.target, ids) // TODO: should be split by chunks too? } async count(target: EntityTarget, options?: FindManyOptions): Promise { - await this.commit() - return await this.em().count(target, options) + return await this.performRead(async () => { + return await this.em.count(target, options) + }) } async countBy( target: EntityTarget, where: FindOptionsWhere | FindOptionsWhere[] ): Promise { - await this.commit() - return await this.em().countBy(target, where) + return await this.performRead(async () => { + return await this.em.countBy(target, where) + }) } async find(target: EntityTarget, options: FindManyOptions): Promise { - await this.commit() - - const {cache, ...opts} = options - const res = await this.em().find(target, opts) - if (cache) this.cacheEntities(target, res, options?.relations) - - return res + return await this.performRead(async () => { + const {cache, ...opts} = options + const res = await this.em.find(target, opts) + if (cache ?? this.cacheMode === 'ALL') this.persistEntities(target, res, options?.relations) + return res + }) } async findBy( @@ -265,25 +269,24 @@ export class Store { where: FindOptionsWhere | FindOptionsWhere[], cache?: boolean ): Promise { - await this.commit() - - const res = await this.em().findBy(target, where) - if (cache) this.cacheEntities(target, res) - - return res + return await this.performRead(async () => { + const res = await this.em.findBy(target, where) + if (cache ?? this.cacheMode === 'ALL') this.persistEntities(target, res) + return res + }) } async findOne( target: EntityTarget, options: FindOneOptions ): Promise { - await this.commit() - - const {cache, ...opts} = options - const res = await this.em().findOne(target, opts).then(noNull) - if (res != null && cache) this.cacheEntities(target, res, options?.relations) - - return res + return await this.performRead(async () => { + const {cache, ...opts} = options + const res = await this.em.findOne(target, opts).then(noNull) + if (res != null && (cache ?? this.cacheMode === 'ALL')) + this.persistEntities(target, res, options?.relations) + return res + }) } async findOneBy( @@ -291,22 +294,22 @@ export class Store { where: FindOptionsWhere | FindOptionsWhere[], cache?: boolean ): Promise { - await this.commit() + return await this.performRead(async () => { + const res = await this.em.findOneBy(target, where).then(noNull) + if (res != null && (cache ?? this.cacheMode === 'ALL')) this.persistEntities(target, res) - const res = await this.em().findOneBy(target, where).then(noNull) - if (res != null && cache) this.cacheEntities(target, res) - - return res + return res + }) } async findOneOrFail(target: EntityTarget, options: FindOneOptions): Promise { - await this.commit() - - const {cache, ...opts} = options - const res = await this.em().findOneOrFail(target, opts) - if (cache) this.cacheEntities(target, res, options?.relations) + return await this.performRead(async () => { + const {cache, ...opts} = options + const res = await this.em.findOneOrFail(target, opts) + if (cache ?? this.cacheMode === 'ALL') this.persistEntities(target, res, options?.relations) - return res + return res + }) } async findOneByOrFail( @@ -314,26 +317,26 @@ export class Store { where: FindOptionsWhere | FindOptionsWhere[], cache?: boolean ): Promise { - await this.commit() - - const res = await this.em().findOneByOrFail(target, where) - if (cache) this.cacheEntities(target, res) - - return res + return await this.performRead(async () => { + const res = await this.em.findOneByOrFail(target, where) + if (cache || this.cacheMode === 'ALL') this.persistEntities(target, res) + return res + }) } - async get(entityClass: EntityTarget, id: string): Promise - async get(entityClass: EntityTarget, options: GetOptions): Promise + async get(target: EntityTarget, id: string): Promise + async get(target: EntityTarget, options: GetOptions): Promise async get( - entityClass: EntityTarget, + target: EntityTarget, idOrOptions: string | GetOptions ): Promise { const {id, relations} = parseGetOptions(idOrOptions) - let entity = this.getFromCache(entityClass, id, relations) - if (entity !== undefined) return entity ?? undefined + const metadata = this.getEntityMetadata(target) + let entity = this.state.get(metadata, id, relations) + if (entity !== undefined) return noNull(entity) - return await this.findOne(entityClass, {where: {id} as any, relations}) + return await this.findOne(target, {where: {id} as any, relations, cache: true}) } async getOrFail(entityClass: EntityTarget, id: string): Promise @@ -353,107 +356,81 @@ export class Store { return e } - async commit(): Promise { + reset(): void { + this.state.reset() + } + + async flush(reset?: boolean): Promise { await this.pendingCommit?.promise() this.pendingCommit = createFuture() try { - const changeSets = this.computeChangeSets() + const {upserts, inserts, deletes, extraUpserts} = this.state.computeChangeSets() - for (const {metadata, upserts} of changeSets) { - if (upserts.length === 0) continue - await this._upsert(metadata, upserts) + for (const {metadata, entities} of upserts) { + await this._upsert(metadata, entities) } - for (const {metadata, inserts} of changeSets) { - if (inserts.length === 0) continue - await this._insert(metadata, inserts) + for (const {metadata, entities} of inserts) { + await this._insert(metadata, entities) } - for (const {metadata, deletes} of [...changeSets].reverse()) { - if (deletes.length === 0) continue - await this._delete(metadata, deletes) + for (const {metadata, ids} of deletes) { + await this._delete(metadata, ids) } - for (const {metadata, extraUpserts} of changeSets) { - if (extraUpserts.length === 0) continue - await this._upsert(metadata, extraUpserts) + for (const {metadata, entities} of extraUpserts) { + await this._upsert(metadata, entities) } + + this.state.clear() } finally { this.pendingCommit.resolve() this.pendingCommit = undefined } - } - clear(): void { - this.cache.clear() - this.changes.clear() + if (this.resetMode === 'FLUSH' || reset) { + this.reset() + } } - async flush(): Promise { - await this.commit() - this.clear() + /** + * @internal + */ + _close() { + this.isClosed = true } - private getFromCache( - target: EntityTarget, - id: string, - mask?: FindOptionsRelations - ): E | null | undefined { - const metadata = this.getEntityMetadata(target) - const cached = this.cache.get(metadata, id) + private async performRead(cb: () => Promise): Promise { + this.assetNotClosed() - if (cached == null) { - return undefined - } else if (cached.value == null) { - return null - } else { - const entity = cached.value + if (this.flushMode === 'AUTO' || this.flushMode === 'ALWAYS') { + await this.flush() + } - const clonedEntity = metadata.create() + return await cb() + } - for (const column of metadata.nonVirtualColumns) { - const objectColumnValue = column.getEntityValue(entity) - if (objectColumnValue !== undefined) { - column.setEntityValue(clonedEntity, copy(objectColumnValue)) - } - } + private async performWrite(cb: () => Promise): Promise { + await this.pendingCommit?.promise() - if (mask != null) { - for (const relation of metadata.relations) { - const inverseMask = mask[relation.propertyName] - if (!inverseMask) continue - - const inverseEntityMock = relation.getEntityValue(entity) - - if (inverseEntityMock === undefined) { - return undefined // relation is missing, but required - } else if (inverseEntityMock === null) { - relation.setEntityValue(clonedEntity, null) - } else { - const cachedInverseEntity = this.getFromCache( - relation.inverseEntityMetadata.target, - inverseEntityMock.id, - typeof inverseMask === 'boolean' ? undefined : inverseMask - ) - - if (cachedInverseEntity === undefined) { - return undefined // unable to build whole relation chain - } else { - relation.setEntityValue(clonedEntity, cachedInverseEntity) - } - } - } - } + this.assetNotClosed() + + await cb() - return clonedEntity + if (this.flushMode === 'ALWAYS') { + await this.flush() } } - private cacheEntities( + private assetNotClosed() { + assert(!this.isClosed, `too late to perform db updates, make sure you haven't forgot to await on db query`) + } + + private persistEntities( target: EntityTarget, e: E | E[], - mask?: FindOptionsRelations + relationMask?: FindOptionsRelations ) { const metadata = this.getEntityMetadata(target) @@ -462,175 +439,16 @@ export class Store { traverseEntity({ metadata, entity, - mask: mask || null, - cb: (e, md) => { - this.cache.add(md, e) - }, + relationMask: relationMask || null, + cb: (e, md) => this.state?.persist(md, e), }) } } - private computeChangeSets() { - const changes = this.changes.values() - - const changeSets: ChangeSet[] = [] - for (const metadata of this.commitOrder) { - const entityChanges = changes.get(metadata) - if (entityChanges == null) continue - - const changeSet = this.computeChangeSet(metadata, entityChanges) - changeSets.push(changeSet) - } - - this.changes.clear() - - return changeSets - } - - private computeChangeSet(metadata: EntityMetadata, changes: Map): ChangeSet { - const inserts: EntityLiteral[] = [] - const upserts: EntityLiteral[] = [] - const deletes: string[] = [] - const extraUpserts: EntityLiteral[] = [] - - for (const [id, type] of changes) { - const cached = this.cache.get(metadata, id) - - switch (type) { - case ChangeType.Insert: { - assert(cached?.value != null, `unable to insert entity ${metadata.name} ${id}`) - - inserts.push(cached.value) - - const extraUpsert = this.extractExtraUpsert(metadata, cached.value) - if (extraUpsert != null) { - extraUpserts.push(extraUpsert) - } - - break - } - case ChangeType.Upsert: { - assert(cached?.value != null, `unable to upsert entity ${metadata.name} ${id}`) - - upserts.push(cached.value) - - const extraUpsert = this.extractExtraUpsert(metadata, cached.value) - if (extraUpsert != null) { - extraUpserts.push(extraUpsert) - } - - break - } - case ChangeType.Delete: { - deletes.push(id) - break - } - } - } - - return {metadata, inserts, upserts, extraUpserts, deletes} - } - - private extractExtraUpsert(metadata: EntityMetadata, entity: E) { - const commitOrderIndex = this.commitOrder.indexOf(metadata) - - let extraUpsert: E | undefined - for (const relation of metadata.relations) { - if (relation.foreignKeys.length == 0) continue - - const inverseEntity = relation.getEntityValue(entity) - if (inverseEntity == null) continue - - const inverseMetadata = relation.inverseEntityMetadata - if (metadata === inverseMetadata && inverseEntity.id === entity.id) continue - - const invCommitOrderIndex = this.commitOrder.indexOf(inverseMetadata) - if (invCommitOrderIndex < commitOrderIndex) continue - - const isInverseInserted = this.changes.isInserted(inverseMetadata, inverseEntity.id) - if (!isInverseInserted) continue - - if (extraUpsert == null) { - extraUpsert = metadata.create() as E - extraUpsert.id = entity.id - Object.assign(extraUpsert, entity) - } - - relation.setEntityValue(entity, undefined) - } - - return extraUpsert - } - private getEntityMetadata(target: EntityTarget) { - const em = this.em() + const em = this.em return em.connection.getMetadata(target) } - - @def - private reverseCommitOrder() { - return [...this.commitOrder].reverse() - } -} - -function traverseEntity({ - metadata, - entity, - mask, - cb, -}: { - metadata: EntityMetadata - entity: EntityLiteral | null - mask: FindOptionsRelations | null - cb: (e: EntityLiteral, metadata: EntityMetadata) => void -}) { - if (entity == null) return - - if (mask != null) { - for (const relation of metadata.relations) { - const inverseMask = mask[relation.propertyName] - if (!inverseMask) continue - - const inverseEntity = relation.getEntityValue(entity) - if (relation.isOneToMany || relation.isManyToMany) { - if (!Array.isArray(inverseEntity)) continue - for (const ie of inverseEntity) { - traverseEntity({ - metadata: relation.inverseEntityMetadata, - entity: ie, - mask: inverseMask === true ? null : inverseMask, - cb, - }) - } - } else { - traverseEntity({ - metadata: relation.inverseEntityMetadata, - entity: inverseEntity, - mask: inverseMask === true ? null : inverseMask, - cb, - }) - } - } - } - - cb(entity, metadata) -} - -function* splitIntoBatches(list: T[], maxBatchSize: number): Generator { - if (list.length <= maxBatchSize) { - yield list - } else { - let offset = 0 - while (list.length - offset > maxBatchSize) { - yield list.slice(offset, offset + maxBatchSize) - offset += maxBatchSize - } - yield list.slice(offset) - } -} - -function noNull(val: null | undefined | T): T | undefined { - return val == null ? undefined : val } function parseGetOptions(idOrOptions: string | GetOptions): GetOptions { diff --git a/typeorm/typeorm-store/src/test/lib/model.ts b/typeorm/typeorm-store/src/test/lib/model.ts index 8cd68bfc3..20cbfd0ef 100644 --- a/typeorm/typeorm-store/src/test/lib/model.ts +++ b/typeorm/typeorm-store/src/test/lib/model.ts @@ -26,7 +26,7 @@ export class Order { @ManyToOne(() => Item, {nullable: true}) item!: Item - @Column({nullable: false}) + @Column('int4') qty!: number } diff --git a/typeorm/typeorm-store/src/test/store.test.ts b/typeorm/typeorm-store/src/test/store.test.ts index 5f1664697..5773b196c 100644 --- a/typeorm/typeorm-store/src/test/store.test.ts +++ b/typeorm/typeorm-store/src/test/store.test.ts @@ -5,6 +5,7 @@ import {Store} from '../store' import {Item, Order} from './lib/model' import {getEntityManager, useDatabase} from './util' import {sortMetadatasInCommitOrder} from '../utils/commitOrder' +import {StateManager} from '../utils/stateManager' describe("Store", function() { @@ -153,10 +154,17 @@ describe("Store", function() { }) -export function createStore(): Promise { - return getEntityManager().then( - em => new Store(() => em, {commitOrder: sortMetadatasInCommitOrder(em.connection.entityMetadatas)}) - ) +export async function createStore(): Promise { + const em = await getEntityManager() + return new Store({ + em, + state: new StateManager({ + commitOrder: sortMetadatasInCommitOrder(em.connection), + }), + cacheMode: 'ALL', + flushMode: 'AUTO', + resetMode: 'BATCH', + }) } diff --git a/typeorm/typeorm-store/src/utils/cacheMap.ts b/typeorm/typeorm-store/src/utils/cacheMap.ts index 1eca570a2..0d7e7e9a9 100644 --- a/typeorm/typeorm-store/src/utils/cacheMap.ts +++ b/typeorm/typeorm-store/src/utils/cacheMap.ts @@ -1,17 +1,21 @@ -import {EntityMetadata, ObjectLiteral} from 'typeorm' -import {copy} from './misc' +import {EntityMetadata} from 'typeorm' +import {copy, EntityLiteral} from './misc' import {Logger} from '@subsquid/logger' -export class CachedEntity { +export class CachedEntity { constructor(public value: E | null = null) {} } export class CacheMap { private map: Map> = new Map() - private logger: Logger + private logger?: Logger - constructor(private opts: {logger: Logger}) { - this.logger = this.opts.logger.child('cache') + constructor(logger?: Logger) { + this.logger = logger?.child('cache') + } + + get(metadata: EntityMetadata, id: string) { + return this.getEntityCache(metadata)?.get(id) } exist(metadata: EntityMetadata, id: string): boolean { @@ -20,32 +24,27 @@ export class CacheMap { return !!cachedEntity?.value } - get(metadata: EntityMetadata, id: string): CachedEntity | undefined { - const cacheMap = this.getEntityCache(metadata) - return cacheMap.get(id) - } - ensure(metadata: EntityMetadata, id: string): void { const cacheMap = this.getEntityCache(metadata) if (cacheMap.has(id)) return cacheMap.set(id, new CachedEntity()) - this.logger.debug(`added empty entity ${metadata.name} ${id}`) + this.logger?.debug(`added empty entity ${metadata.name} ${id}`) } delete(metadata: EntityMetadata, id: string): void { const cacheMap = this.getEntityCache(metadata) cacheMap.set(id, new CachedEntity()) - this.logger.debug(`deleted entity ${metadata.name} ${id}`) + this.logger?.debug(`deleted entity ${metadata.name} ${id}`) } clear(): void { - this.logger.debug(`cleared`) + this.logger?.debug(`cleared`) this.map.clear() } - add(metadata: EntityMetadata, entity: E, isNew = false): void { + add(metadata: EntityMetadata, entity: E, isNew = false): void { const cacheMap = this.getEntityCache(metadata) let cached = cacheMap.get(entity.id) @@ -58,7 +57,7 @@ export class CacheMap { if (cachedEntity == null) { cachedEntity = cached.value = metadata.create() as E cachedEntity.id = entity.id - this.logger.debug(`added entity ${metadata.name} ${entity.id}`) + this.logger?.debug(`added entity ${metadata.name} ${entity.id}`) } for (const column of metadata.nonVirtualColumns) { @@ -71,7 +70,7 @@ export class CacheMap { for (const relation of metadata.relations) { if (!relation.isOwning) continue - const inverseEntity = relation.getEntityValue(entity) as ObjectLiteral | null | undefined + const inverseEntity = relation.getEntityValue(entity) as EntityLiteral | null | undefined const inverseMetadata = relation.inverseEntityMetadata if (inverseEntity != null) { @@ -85,11 +84,7 @@ export class CacheMap { } } - values(): Map>> { - return new Map(this.map) - } - - private getEntityCache(metadata: EntityMetadata): Map> { + private getEntityCache(metadata: EntityMetadata): Map> { let map = this.map.get(metadata) if (map == null) { map = new Map() diff --git a/typeorm/typeorm-store/src/utils/changeTracker.ts b/typeorm/typeorm-store/src/utils/changeTracker.ts deleted file mode 100644 index 5cdbd0eb3..000000000 --- a/typeorm/typeorm-store/src/utils/changeTracker.ts +++ /dev/null @@ -1,105 +0,0 @@ -import {Logger} from '@subsquid/logger' -import {EntityMetadata} from 'typeorm' - -export enum ChangeType { - Insert = 'insert', - Upsert = 'upsert', - Delete = 'delete', -} - -export class ChangeTracker { - private map: Map> = new Map() - private logger: Logger - - constructor(private opts: {logger: Logger}) { - this.logger = this.opts.logger.child('changes') - } - - trackInsert(metadata: EntityMetadata, id: string): void { - const prevType = this.get(metadata, id) - switch (prevType) { - case undefined: - this.set(metadata, id, ChangeType.Insert) - this.logger.debug(`entity ${metadata.name} ${id} already marked as ${ChangeType.Insert}`) - break - case ChangeType.Delete: - this.set(metadata, id, ChangeType.Upsert) - break - case ChangeType.Insert: - case ChangeType.Upsert: - throw new Error( - `${metadata.name} ${id} is already marked as ${ChangeType.Insert} or ${ChangeType.Upsert}` - ) - } - } - - trackUpsert(metadata: EntityMetadata, id: string): void { - const prevType = this.get(metadata, id) - switch (prevType) { - case ChangeType.Insert: - this.logger.debug(`entity ${metadata.name} ${id} already marked as ${ChangeType.Insert}`) - break - case ChangeType.Upsert: - this.logger.debug(`entity ${metadata.name} ${id} already marked as ${ChangeType.Upsert}`) - break - default: - this.set(metadata, id, ChangeType.Upsert) - break - } - } - - trackDelete(metadata: EntityMetadata, id: string): void { - const prevType = this.get(metadata, id) - switch (prevType) { - case ChangeType.Insert: - this.getChanges(metadata).delete(id) - break - case ChangeType.Delete: - this.logger.debug(`entity ${metadata.name} ${id} already marked as ${ChangeType.Delete}`) - break - default: - this.set(metadata, id, ChangeType.Delete) - } - } - - isInserted(metadata: EntityMetadata, id: string) { - return this.get(metadata, id) === ChangeType.Insert - } - - isUpserted(metadata: EntityMetadata, id: string) { - return this.get(metadata, id) === ChangeType.Upsert - } - - isDeleted(metadata: EntityMetadata, id: string) { - return this.get(metadata, id) === ChangeType.Delete - } - - clear(): void { - this.logger.debug(`cleared`) - this.map.clear() - } - - values(): Map> { - return new Map(this.map) - } - - private set(metadata: EntityMetadata, id: string, type: ChangeType): this { - this.getChanges(metadata).set(id, type) - this.logger.debug(`entity ${metadata.name} ${id} marked as ${type}`) - return this - } - - private get(metadata: EntityMetadata, id: string): ChangeType | undefined { - return this.getChanges(metadata).get(id) - } - - private getChanges(metadata: EntityMetadata): Map { - let map = this.map.get(metadata) - if (map == null) { - map = new Map() - this.map.set(metadata, map) - } - - return map - } -} diff --git a/typeorm/typeorm-store/src/utils/changeWriter.ts b/typeorm/typeorm-store/src/utils/changeWriter.ts index 0a1b30c3b..09606ca0e 100644 --- a/typeorm/typeorm-store/src/utils/changeWriter.ts +++ b/typeorm/typeorm-store/src/utils/changeWriter.ts @@ -1,8 +1,7 @@ import {assertNotNull} from '@subsquid/util-internal' import type {EntityManager, EntityMetadata, EntityTarget} from 'typeorm' import {ColumnMetadata} from 'typeorm/metadata/ColumnMetadata' -import {EntityLiteral} from '../store' - +import {EntityLiteral} from './misc' export interface RowRef { table: string @@ -41,7 +40,7 @@ export class ChangeWriter { private index = 0 constructor( - private em: EntityManager, + protected em: EntityManager, private statusSchema: string, private blockHeight: number ) { diff --git a/typeorm/typeorm-store/src/utils/commitOrder.ts b/typeorm/typeorm-store/src/utils/commitOrder.ts index b160b928b..84fce502c 100644 --- a/typeorm/typeorm-store/src/utils/commitOrder.ts +++ b/typeorm/typeorm-store/src/utils/commitOrder.ts @@ -1,4 +1,4 @@ -import {EntityMetadata} from 'typeorm' +import {DataSource, EntityMetadata} from 'typeorm' import {RelationMetadata} from 'typeorm/metadata/RelationMetadata' enum NodeState { @@ -7,9 +7,13 @@ enum NodeState { Visited, } -export function sortMetadatasInCommitOrder(entities: EntityMetadata[]): EntityMetadata[] { - let states: Map = new Map(entities.map((e) => [e.name, NodeState.Unvisited])) - let commitOrder: EntityMetadata[] = [] +const COMMIT_ORDERS: WeakMap = new WeakMap() + +export function sortMetadatasInCommitOrder(connection: DataSource): EntityMetadata[] { + let commitOrder = COMMIT_ORDERS.get(connection) + if (commitOrder != null) return commitOrder + + let states: Map = new Map() function visit(node: EntityMetadata) { if (states.get(node.name) !== NodeState.Unvisited) return @@ -36,7 +40,7 @@ export function sortMetadatasInCommitOrder(entities: EntityMetadata[]): EntityMe } states.set(target.name, NodeState.Visited) - commitOrder.push(target) + commitOrder?.push(target) } } } @@ -46,14 +50,22 @@ export function sortMetadatasInCommitOrder(entities: EntityMetadata[]): EntityMe if (nodeState !== NodeState.Visited) { states.set(node.name, NodeState.Visited) - commitOrder.push(node) + commitOrder?.push(node) } } - for (let node of entities) { + commitOrder = [] + + for (let node of connection.entityMetadatas) { + if (!states.has(node.name)) { + states.set(node.name, NodeState.Unvisited) + } + visit(node) } + COMMIT_ORDERS.set(connection, commitOrder) + return commitOrder } diff --git a/typeorm/typeorm-store/src/utils/misc.ts b/typeorm/typeorm-store/src/utils/misc.ts index e9a5cea68..37b7a0956 100644 --- a/typeorm/typeorm-store/src/utils/misc.ts +++ b/typeorm/typeorm-store/src/utils/misc.ts @@ -1,4 +1,8 @@ -import {FindOptionsRelations, ObjectLiteral} from 'typeorm' +import {EntityMetadata, FindOptionsRelations, ObjectLiteral} from 'typeorm' + +export interface EntityLiteral extends ObjectLiteral { + id: string +} export function* splitIntoBatches(list: T[], maxBatchSize: number): Generator { if (list.length <= maxBatchSize) { @@ -60,7 +64,7 @@ function copyBuffer(buf: any) { } } -export function mergeRelataions( +export function mergeRelations( a: FindOptionsRelations, b: FindOptionsRelations ): FindOptionsRelations { @@ -75,7 +79,7 @@ export function mergeRelataions( const value = mergedObject[key] if (typeof bValue === 'object') { mergedObject[key] = ( - typeof value === 'object' ? mergeRelataions(value as any, bValue as any) : bValue + typeof value === 'object' ? mergeRelations(value as any, bValue as any) : bValue ) as any } else { mergedObject[key] = value || bValue @@ -84,3 +88,50 @@ export function mergeRelataions( return mergedObject } + +export function traverseEntity({ + metadata, + entity, + relationMask, + cb, +}: { + metadata: EntityMetadata + entity: EntityLiteral + relationMask: FindOptionsRelations | null + cb: (e: EntityLiteral, metadata: EntityMetadata) => void +}) { + if (relationMask != null) { + for (const relation of metadata.relations) { + const inverseRelationMask = relationMask[relation.propertyName] + if (!inverseRelationMask) continue + + const inverseEntity = relation.getEntityValue(entity) + if (inverseEntity == null) continue + + if (relation.isOneToMany || relation.isManyToMany) { + if (!Array.isArray(inverseEntity)) continue + for (const ie of inverseEntity) { + traverseEntity({ + metadata: relation.inverseEntityMetadata, + entity: ie, + relationMask: inverseRelationMask === true ? null : inverseRelationMask, + cb, + }) + } + } else { + traverseEntity({ + metadata: relation.inverseEntityMetadata, + entity: inverseEntity, + relationMask: inverseRelationMask === true ? null : inverseRelationMask, + cb, + }) + } + } + } + + cb(entity, metadata) +} + +export function noNull(val: null | undefined | T): T | undefined { + return val == null ? undefined : val +} diff --git a/typeorm/typeorm-store/src/utils/stateManager.ts b/typeorm/typeorm-store/src/utils/stateManager.ts new file mode 100644 index 000000000..28d067436 --- /dev/null +++ b/typeorm/typeorm-store/src/utils/stateManager.ts @@ -0,0 +1,295 @@ +import {Logger} from '@subsquid/logger' +import {EntityManager, EntityMetadata, FindOptionsRelations} from 'typeorm' +import {CacheMap} from './cacheMap' +import assert from 'assert' +import {copy, EntityLiteral} from './misc' +import {sortMetadatasInCommitOrder} from './commitOrder' +import {unexpectedCase} from '@subsquid/util-internal' + +export enum ChangeType { + Insert = 'insert', + Upsert = 'upsert', + Delete = 'delete', +} + +export type ChangeSets = { + upserts: {metadata: EntityMetadata; entities: EntityLiteral[]}[] + inserts: {metadata: EntityMetadata; entities: EntityLiteral[]}[] + deletes: {metadata: EntityMetadata; ids: string[]}[] + extraUpserts: {metadata: EntityMetadata; entities: EntityLiteral[]}[] +} + +export class StateManager { + protected cacheMap: CacheMap + protected stateMap: Map> + protected commitOrder: EntityMetadata[] + protected logger?: Logger + + constructor({commitOrder, logger}: {commitOrder: EntityMetadata[]; logger?: Logger}) { + this.cacheMap = new CacheMap(this.logger) + this.stateMap = new Map() + this.commitOrder = commitOrder + this.logger = logger?.child('state') + } + + get( + metadata: EntityMetadata, + id: string, + relationMask?: FindOptionsRelations + ): E | null | undefined { + const cached = this.cacheMap.get(metadata, id) + + if (cached == null) { + return undefined + } else if (cached.value == null) { + return null + } else { + const entity = cached.value + const clonedEntity = metadata.create() + + for (const column of metadata.nonVirtualColumns) { + const objectColumnValue = column.getEntityValue(entity) + if (objectColumnValue !== undefined) { + column.setEntityValue(clonedEntity, copy(objectColumnValue)) + } + } + + if (relationMask != null) { + for (const relation of metadata.relations) { + const inverseMask = relationMask[relation.propertyName] + if (!inverseMask) continue + + const inverseEntityMock = relation.getEntityValue(entity) as EntityLiteral + + if (inverseEntityMock === null) { + relation.setEntityValue(clonedEntity, null) + } else { + const cachedInverseEntity = + inverseEntityMock != null + ? this.get( + relation.inverseEntityMetadata, + inverseEntityMock.id, + typeof inverseMask === 'boolean' ? undefined : inverseMask + ) + : undefined + + if (cachedInverseEntity === undefined) { + return undefined // unable to build whole relation chain + } else { + relation.setEntityValue(clonedEntity, cachedInverseEntity) + } + } + } + } + + return clonedEntity + } + } + + insert(metadata: EntityMetadata, entity: EntityLiteral): void { + const prevType = this.getState(metadata, entity.id) + switch (prevType) { + case undefined: + this.setState(metadata, entity.id, ChangeType.Insert) + this.cacheMap.add(metadata, entity, true) + break + case ChangeType.Insert: + case ChangeType.Upsert: + throw new Error(`Entity ${metadata.name} ${entity.id} is already marked as ${prevType}`) + case ChangeType.Delete: + this.setState(metadata, entity.id, ChangeType.Upsert) + this.cacheMap.add(metadata, entity, true) + break + default: + throw unexpectedCase(prevType) + } + } + + upsert(metadata: EntityMetadata, entity: EntityLiteral): void { + const prevType = this.getState(metadata, entity.id) + switch (prevType) { + case undefined: + case ChangeType.Insert: + case ChangeType.Upsert: + this.setState(metadata, entity.id, ChangeType.Upsert) + this.cacheMap.add(metadata, entity) + break + case ChangeType.Delete: + this.setState(metadata, entity.id, ChangeType.Upsert) + this.cacheMap.add(metadata, entity, true) + default: + throw unexpectedCase(prevType) + } + } + + delete(metadata: EntityMetadata, id: string): void { + const prevType = this.getState(metadata, id) + switch (prevType) { + case undefined: + case ChangeType.Upsert: + this.setState(metadata, id, ChangeType.Delete) + this.cacheMap.delete(metadata, id) + break + case ChangeType.Insert: + this.getChanges(metadata).delete(id) + this.cacheMap.delete(metadata, id) + break + case ChangeType.Delete: + this.logger?.debug(`entity ${metadata.name} ${id} is already marked as ${ChangeType.Delete}`) + break + default: + throw unexpectedCase(prevType) + } + } + + persist(metadata: EntityMetadata, entity: EntityLiteral) { + this.getChanges(metadata).delete(entity.id) // reset state + this.cacheMap.add(metadata, entity) + } + + isInserted(metadata: EntityMetadata, id: string) { + return this.getState(metadata, id) === ChangeType.Insert + } + + isUpserted(metadata: EntityMetadata, id: string) { + return this.getState(metadata, id) === ChangeType.Upsert + } + + isDeleted(metadata: EntityMetadata, id: string) { + return this.getState(metadata, id) === ChangeType.Delete + } + + clear(): void { + this.logger?.debug(`clear states`) + this.stateMap.clear() + } + + reset(): void { + this.clear() + this.logger?.debug(`reset cache`) + this.cacheMap.clear() + } + + computeChangeSets() { + const changeSets: ChangeSets = { + inserts: [], + upserts: [], + deletes: [], + extraUpserts: [], + } + + for (const metadata of this.commitOrder) { + const entityChanges = this.stateMap.get(metadata) + if (entityChanges == null || entityChanges.size == 0) continue + + const inserts: EntityLiteral[] = [] + const upserts: EntityLiteral[] = [] + const deletes: string[] = [] + const extraUpserts: EntityLiteral[] = [] + + for (const [id, type] of entityChanges) { + const cached = this.cacheMap.get(metadata, id) + + switch (type) { + case ChangeType.Insert: { + assert(cached?.value != null, `unable to insert entity ${metadata.name} ${id}`) + + inserts.push(cached.value) + + const extraUpsert = this.extractExtraUpsert(metadata, cached.value) + if (extraUpsert != null) { + extraUpserts.push(extraUpsert) + } + + break + } + case ChangeType.Upsert: { + assert(cached?.value != null, `unable to upsert entity ${metadata.name} ${id}`) + + upserts.push(cached.value) + + const extraUpsert = this.extractExtraUpsert(metadata, cached.value) + if (extraUpsert != null) { + extraUpserts.push(extraUpsert) + } + + break + } + case ChangeType.Delete: { + deletes.push(id) + break + } + } + } + + if (upserts.length) { + changeSets.upserts.push({metadata, entities: upserts}) + } + + if (inserts.length) { + changeSets.inserts.push({metadata, entities: inserts}) + } + + if (deletes.length) { + changeSets.deletes.push({metadata, ids: deletes}) + } + + if (extraUpserts.length) { + changeSets.extraUpserts.push({metadata, entities: extraUpserts}) + } + } + + return changeSets + } + + private extractExtraUpsert(metadata: EntityMetadata, entity: E) { + const commitOrderIndex = this.commitOrder.indexOf(metadata) + + let extraUpsert: E | undefined + for (const relation of metadata.relations) { + if (relation.foreignKeys.length == 0) continue + + const inverseEntity = relation.getEntityValue(entity) + if (inverseEntity == null) continue + + const inverseMetadata = relation.inverseEntityMetadata + if (metadata === inverseMetadata && inverseEntity.id === entity.id) continue + + const invCommitOrderIndex = this.commitOrder.indexOf(inverseMetadata) + if (invCommitOrderIndex < commitOrderIndex) continue + + const isInverseInserted = this.isInserted(inverseMetadata, inverseEntity.id) + if (!isInverseInserted) continue + + if (extraUpsert == null) { + extraUpsert = metadata.create() as E + extraUpsert.id = entity.id + Object.assign(extraUpsert, entity) + } + + relation.setEntityValue(entity, undefined) + } + + return extraUpsert + } + + private setState(metadata: EntityMetadata, id: string, type: ChangeType): this { + this.getChanges(metadata).set(id, type) + this.logger?.debug(`entity ${metadata.name} ${id} marked as ${type}`) + return this + } + + private getState(metadata: EntityMetadata, id: string): ChangeType | undefined { + return this.getChanges(metadata).get(id) + } + + private getChanges(metadata: EntityMetadata): Map { + let map = this.stateMap.get(metadata) + if (map == null) { + map = new Map() + this.stateMap.set(metadata, map) + } + + return map + } +} From 87edb3354e7166f6ace4e0b50df73a9d76e1a3c5 Mon Sep 17 00:00:00 2001 From: belopash Date: Tue, 28 May 2024 00:51:05 +0500 Subject: [PATCH 03/11] save --- typeorm/typeorm-store/package.json | 2 +- typeorm/typeorm-store/src/database.ts | 2 +- .../decorators/columns/BigDecimalColumn.ts | 2 +- .../src/decorators/columns/BigIntColumn.ts | 2 +- .../src/decorators/columns/FloatColumn.ts | 2 +- typeorm/typeorm-store/src/index.ts | 2 +- typeorm/typeorm-store/src/store.ts | 167 +++++++----------- .../src/{utils => }/transformers.ts | 0 typeorm/typeorm-store/src/utils/cacheMap.ts | 4 +- typeorm/typeorm-store/src/utils/misc.ts | 48 ++--- .../typeorm-store/src/utils/stateManager.ts | 19 +- 11 files changed, 98 insertions(+), 152 deletions(-) rename typeorm/typeorm-store/src/{utils => }/transformers.ts (100%) diff --git a/typeorm/typeorm-store/package.json b/typeorm/typeorm-store/package.json index 4424dce6b..f467708cd 100644 --- a/typeorm/typeorm-store/package.json +++ b/typeorm/typeorm-store/package.json @@ -20,7 +20,7 @@ "dependencies": { "@subsquid/typeorm-config": "^4.1.1", "@subsquid/util-internal": "^3.2.0", - "@subsquid/logger": "~1.3.3" + "@subsquid/logger": "^1.3.3" }, "peerDependencies": { "typeorm": "^0.3.17", diff --git a/typeorm/typeorm-store/src/database.ts b/typeorm/typeorm-store/src/database.ts index cfeb813eb..183a261cf 100644 --- a/typeorm/typeorm-store/src/database.ts +++ b/typeorm/typeorm-store/src/database.ts @@ -249,7 +249,7 @@ export class TypeormDatabase { await store.flush() if (this.resetMode === 'BATCH') store.reset() } finally { - store._close() + store['isClosed'] = true } } diff --git a/typeorm/typeorm-store/src/decorators/columns/BigDecimalColumn.ts b/typeorm/typeorm-store/src/decorators/columns/BigDecimalColumn.ts index 8c8f074bd..2f436cc32 100644 --- a/typeorm/typeorm-store/src/decorators/columns/BigDecimalColumn.ts +++ b/typeorm/typeorm-store/src/decorators/columns/BigDecimalColumn.ts @@ -1,4 +1,4 @@ -import {bigdecimalTransformer} from '../../utils/transformers' +import {bigdecimalTransformer} from '../../transformers' import {Column} from './Column' import {ColumnCommonOptions} from './common' diff --git a/typeorm/typeorm-store/src/decorators/columns/BigIntColumn.ts b/typeorm/typeorm-store/src/decorators/columns/BigIntColumn.ts index d6312fc32..3c572be4f 100644 --- a/typeorm/typeorm-store/src/decorators/columns/BigIntColumn.ts +++ b/typeorm/typeorm-store/src/decorators/columns/BigIntColumn.ts @@ -1,4 +1,4 @@ -import {bigintTransformer} from '../../utils/transformers' +import {bigintTransformer} from '../../transformers' import {Column} from './Column' import {ColumnCommonOptions} from './common' diff --git a/typeorm/typeorm-store/src/decorators/columns/FloatColumn.ts b/typeorm/typeorm-store/src/decorators/columns/FloatColumn.ts index a11e79532..fece3d4c9 100644 --- a/typeorm/typeorm-store/src/decorators/columns/FloatColumn.ts +++ b/typeorm/typeorm-store/src/decorators/columns/FloatColumn.ts @@ -1,4 +1,4 @@ -import {floatTransformer} from '../../utils/transformers' +import {floatTransformer} from '../../transformers' import {Column} from './Column' import {ColumnCommonOptions} from './common' diff --git a/typeorm/typeorm-store/src/index.ts b/typeorm/typeorm-store/src/index.ts index b52d76e34..7db198d25 100644 --- a/typeorm/typeorm-store/src/index.ts +++ b/typeorm/typeorm-store/src/index.ts @@ -1,4 +1,4 @@ export * from './database' export {FindManyOptions, FindOneOptions, Store} from './store' export * from './decorators' -export * from './utils/transformers' +export * from './transformers' diff --git a/typeorm/typeorm-store/src/store.ts b/typeorm/typeorm-store/src/store.ts index ccf22ba61..73fb6a745 100644 --- a/typeorm/typeorm-store/src/store.ts +++ b/typeorm/typeorm-store/src/store.ts @@ -1,4 +1,11 @@ -import {EntityManager, EntityMetadata, FindOptionsOrder, FindOptionsRelations, FindOptionsWhere} from 'typeorm' +import { + EntityManager, + EntityMetadata, + EntityNotFoundError, + FindOptionsOrder, + FindOptionsRelations, + FindOptionsWhere, +} from 'typeorm' import {EntityTarget} from 'typeorm/common/EntityTarget' import {ChangeWriter} from './utils/changeWriter' import {StateManager} from './utils/stateManager' @@ -96,16 +103,10 @@ export class Store { this.cacheMode = cacheMode } - /** - * @internal - */ get _em() { return this.em } - /** - * @internal - */ get _state() { return this.state } @@ -237,7 +238,7 @@ export class Store { private async _delete(metadata: EntityMetadata, ids: string[]) { this.logger?.debug(`delete ${metadata.name} ${ids.length} entities`) await this.changes?.writeDelete(metadata, ids) - await this.em.delete(metadata.target, ids) // TODO: should be split by chunks too? + await this.em.delete(metadata.target, ids) // NOTE: should be split by chunks too? } async count(target: EntityTarget, options?: FindManyOptions): Promise { @@ -250,16 +251,19 @@ export class Store { target: EntityTarget, where: FindOptionsWhere | FindOptionsWhere[] ): Promise { - return await this.performRead(async () => { - return await this.em.countBy(target, where) - }) + return await this.count(target, {where}) } async find(target: EntityTarget, options: FindManyOptions): Promise { return await this.performRead(async () => { const {cache, ...opts} = options const res = await this.em.find(target, opts) - if (cache ?? this.cacheMode === 'ALL') this.persistEntities(target, res, options?.relations) + if (cache ?? this.cacheMode === 'ALL') { + const metadata = this.getEntityMetadata(target) + for (const e of res) { + this.cacheEntity(metadata, e) + } + } return res }) } @@ -269,11 +273,7 @@ export class Store { where: FindOptionsWhere | FindOptionsWhere[], cache?: boolean ): Promise { - return await this.performRead(async () => { - const res = await this.em.findBy(target, where) - if (cache ?? this.cacheMode === 'ALL') this.persistEntities(target, res) - return res - }) + return await this.find(target, {where, cache}) } async findOne( @@ -283,8 +283,11 @@ export class Store { return await this.performRead(async () => { const {cache, ...opts} = options const res = await this.em.findOne(target, opts).then(noNull) - if (res != null && (cache ?? this.cacheMode === 'ALL')) - this.persistEntities(target, res, options?.relations) + if (cache ?? this.cacheMode === 'ALL') { + const metadata = this.getEntityMetadata(target) + const idOrEntity = res || getIdFromWhere(options.where) + this.cacheEntity(metadata, idOrEntity) + } return res }) } @@ -294,22 +297,13 @@ export class Store { where: FindOptionsWhere | FindOptionsWhere[], cache?: boolean ): Promise { - return await this.performRead(async () => { - const res = await this.em.findOneBy(target, where).then(noNull) - if (res != null && (cache ?? this.cacheMode === 'ALL')) this.persistEntities(target, res) - - return res - }) + return await this.findOne(target, {where, cache}) } async findOneOrFail(target: EntityTarget, options: FindOneOptions): Promise { - return await this.performRead(async () => { - const {cache, ...opts} = options - const res = await this.em.findOneOrFail(target, opts) - if (cache ?? this.cacheMode === 'ALL') this.persistEntities(target, res, options?.relations) - - return res - }) + const res = await this.findOne(target, options) + if (res == null) throw new EntityNotFoundError(target, options.where) + return res } async findOneByOrFail( @@ -317,11 +311,9 @@ export class Store { where: FindOptionsWhere | FindOptionsWhere[], cache?: boolean ): Promise { - return await this.performRead(async () => { - const res = await this.em.findOneByOrFail(target, where) - if (cache || this.cacheMode === 'ALL') this.persistEntities(target, res) - return res - }) + const res = await this.findOneBy(target, where, cache) + if (res == null) throw new EntityNotFoundError(target, where) + return res } async get(target: EntityTarget, id: string): Promise @@ -331,28 +323,18 @@ export class Store { idOrOptions: string | GetOptions ): Promise { const {id, relations} = parseGetOptions(idOrOptions) - const metadata = this.getEntityMetadata(target) let entity = this.state.get(metadata, id, relations) if (entity !== undefined) return noNull(entity) - return await this.findOne(target, {where: {id} as any, relations, cache: true}) } - async getOrFail(entityClass: EntityTarget, id: string): Promise - async getOrFail(entityClass: EntityTarget, options: GetOptions): Promise - async getOrFail( - entityClass: EntityTarget, - idOrOptions: string | GetOptions - ): Promise { + async getOrFail(target: EntityTarget, id: string): Promise + async getOrFail(target: EntityTarget, options: GetOptions): Promise + async getOrFail(target: EntityTarget, idOrOptions: string | GetOptions): Promise { const options = parseGetOptions(idOrOptions) - let e = await this.get(entityClass, options) - - if (e == null) { - const metadata = this.getEntityMetadata(entityClass) - throw new Error(`Missing entity ${metadata.name} with id "${options.id}"`) - } - + let e = await this.get(target, options) + if (e == null) throw new EntityNotFoundError(target, options.id) return e } @@ -365,89 +347,66 @@ export class Store { this.pendingCommit = createFuture() try { - const {upserts, inserts, deletes, extraUpserts} = this.state.computeChangeSets() + await this.state.performUpdate(async ({upserts, inserts, deletes, extraUpserts}) => { + for (const {metadata, entities} of upserts) { + await this._upsert(metadata, entities) + } - for (const {metadata, entities} of upserts) { - await this._upsert(metadata, entities) - } + for (const {metadata, entities} of inserts) { + await this._insert(metadata, entities) + } - for (const {metadata, entities} of inserts) { - await this._insert(metadata, entities) - } + for (const {metadata, ids} of deletes) { + await this._delete(metadata, ids) + } - for (const {metadata, ids} of deletes) { - await this._delete(metadata, ids) - } + for (const {metadata, entities} of extraUpserts) { + await this._upsert(metadata, entities) + } + }) - for (const {metadata, entities} of extraUpserts) { - await this._upsert(metadata, entities) + if (this.resetMode === 'FLUSH' || reset) { + this.reset() } - - this.state.clear() } finally { this.pendingCommit.resolve() this.pendingCommit = undefined } - - if (this.resetMode === 'FLUSH' || reset) { - this.reset() - } - } - - /** - * @internal - */ - _close() { - this.isClosed = true } private async performRead(cb: () => Promise): Promise { - this.assetNotClosed() - + this.assertNotClosed() if (this.flushMode === 'AUTO' || this.flushMode === 'ALWAYS') { await this.flush() } - return await cb() } private async performWrite(cb: () => Promise): Promise { + this.assertNotClosed() await this.pendingCommit?.promise() - - this.assetNotClosed() - await cb() - if (this.flushMode === 'ALWAYS') { await this.flush() } } - private assetNotClosed() { + private assertNotClosed() { assert(!this.isClosed, `too late to perform db updates, make sure you haven't forgot to await on db query`) } - private persistEntities( - target: EntityTarget, - e: E | E[], - relationMask?: FindOptionsRelations - ) { - const metadata = this.getEntityMetadata(target) - - e = Array.isArray(e) ? e : [e] - for (const entity of e) { - traverseEntity({ - metadata, - entity, - relationMask: relationMask || null, - cb: (e, md) => this.state?.persist(md, e), - }) + private cacheEntity(metadata: EntityMetadata, entityOrId?: E | string) { + if (entityOrId == null) { + return + } else if (typeof entityOrId === 'string') { + this.state.settle(metadata, entityOrId) + } else { + traverseEntity(metadata, entityOrId, (e, md) => this.state.persist(md, e)) } } private getEntityMetadata(target: EntityTarget) { - const em = this.em - return em.connection.getMetadata(target) + return this.em.connection.getMetadata(target) } } @@ -458,3 +417,7 @@ function parseGetOptions(idOrOptions: string | GetOptions): GetOptions return idOrOptions } } + +function getIdFromWhere(where?: FindOptionsWhere) { + return typeof where?.id === 'string' ? where.id : undefined +} diff --git a/typeorm/typeorm-store/src/utils/transformers.ts b/typeorm/typeorm-store/src/transformers.ts similarity index 100% rename from typeorm/typeorm-store/src/utils/transformers.ts rename to typeorm/typeorm-store/src/transformers.ts diff --git a/typeorm/typeorm-store/src/utils/cacheMap.ts b/typeorm/typeorm-store/src/utils/cacheMap.ts index 0d7e7e9a9..7a9af0233 100644 --- a/typeorm/typeorm-store/src/utils/cacheMap.ts +++ b/typeorm/typeorm-store/src/utils/cacheMap.ts @@ -18,13 +18,13 @@ export class CacheMap { return this.getEntityCache(metadata)?.get(id) } - exist(metadata: EntityMetadata, id: string): boolean { + has(metadata: EntityMetadata, id: string): boolean { const cacheMap = this.getEntityCache(metadata) const cachedEntity = cacheMap.get(id) return !!cachedEntity?.value } - ensure(metadata: EntityMetadata, id: string): void { + settle(metadata: EntityMetadata, id: string): void { const cacheMap = this.getEntityCache(metadata) if (cacheMap.has(id)) return diff --git a/typeorm/typeorm-store/src/utils/misc.ts b/typeorm/typeorm-store/src/utils/misc.ts index 37b7a0956..c293f9baa 100644 --- a/typeorm/typeorm-store/src/utils/misc.ts +++ b/typeorm/typeorm-store/src/utils/misc.ts @@ -89,43 +89,21 @@ export function mergeRelations( return mergedObject } -export function traverseEntity({ - metadata, - entity, - relationMask, - cb, -}: { - metadata: EntityMetadata - entity: EntityLiteral - relationMask: FindOptionsRelations | null +export function traverseEntity( + metadata: EntityMetadata, + entity: EntityLiteral, cb: (e: EntityLiteral, metadata: EntityMetadata) => void -}) { - if (relationMask != null) { - for (const relation of metadata.relations) { - const inverseRelationMask = relationMask[relation.propertyName] - if (!inverseRelationMask) continue - - const inverseEntity = relation.getEntityValue(entity) - if (inverseEntity == null) continue - - if (relation.isOneToMany || relation.isManyToMany) { - if (!Array.isArray(inverseEntity)) continue - for (const ie of inverseEntity) { - traverseEntity({ - metadata: relation.inverseEntityMetadata, - entity: ie, - relationMask: inverseRelationMask === true ? null : inverseRelationMask, - cb, - }) - } - } else { - traverseEntity({ - metadata: relation.inverseEntityMetadata, - entity: inverseEntity, - relationMask: inverseRelationMask === true ? null : inverseRelationMask, - cb, - }) +) { + for (const relation of metadata.relations) { + const inverseEntity = relation.getEntityValue(entity) + if (inverseEntity == null) continue + + if (relation.isOneToMany || relation.isManyToMany) { + for (const ie of inverseEntity) { + traverseEntity(relation.inverseEntityMetadata, ie, cb) } + } else { + traverseEntity(relation.inverseEntityMetadata, inverseEntity, cb) } } diff --git a/typeorm/typeorm-store/src/utils/stateManager.ts b/typeorm/typeorm-store/src/utils/stateManager.ts index 28d067436..53ca38d4c 100644 --- a/typeorm/typeorm-store/src/utils/stateManager.ts +++ b/typeorm/typeorm-store/src/utils/stateManager.ts @@ -147,6 +147,10 @@ export class StateManager { this.cacheMap.add(metadata, entity) } + settle(metadata: EntityMetadata, id: string) { + this.cacheMap.settle(metadata, id) + } + isInserted(metadata: EntityMetadata, id: string) { return this.getState(metadata, id) === ChangeType.Insert } @@ -159,18 +163,17 @@ export class StateManager { return this.getState(metadata, id) === ChangeType.Delete } - clear(): void { - this.logger?.debug(`clear states`) - this.stateMap.clear() + isExists(metadata: EntityMetadata, id: string) { + return this.cacheMap.has(metadata, id) } reset(): void { - this.clear() - this.logger?.debug(`reset cache`) + this.logger?.debug(`reset`) + this.stateMap.clear() this.cacheMap.clear() } - computeChangeSets() { + async performUpdate(cb: (cs: ChangeSets) => Promise) { const changeSets: ChangeSets = { inserts: [], upserts: [], @@ -239,7 +242,9 @@ export class StateManager { } } - return changeSets + await cb(changeSets) + + this.stateMap.clear() } private extractExtraUpsert(metadata: EntityMetadata, entity: E) { From 7452f6f114a6ea9d9886e128a4197ca6ef3efba1 Mon Sep 17 00:00:00 2001 From: belopash Date: Tue, 28 May 2024 00:58:47 +0500 Subject: [PATCH 04/11] save --- typeorm/typeorm-store/src/database.ts | 5 ++++- typeorm/typeorm-store/src/index.ts | 2 +- typeorm/typeorm-store/src/utils/tx.ts | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/typeorm/typeorm-store/src/database.ts b/typeorm/typeorm-store/src/database.ts index 183a261cf..ecf18bfa5 100644 --- a/typeorm/typeorm-store/src/database.ts +++ b/typeorm/typeorm-store/src/database.ts @@ -8,9 +8,10 @@ import {CacheMode, FlushMode, ResetMode, Store} from './store' import {createLogger} from '@subsquid/logger' import {StateManager} from './utils/stateManager' import {sortMetadatasInCommitOrder} from './utils/commitOrder' +import {IsolationLevel} from './utils/tx' -export type IsolationLevel = 'SERIALIZABLE' | 'READ COMMITTED' | 'REPEATABLE READ' +export {IsolationLevel} export interface TypeormDatabaseOptions { @@ -23,8 +24,10 @@ export interface TypeormDatabaseOptions { projectDir?: string } + const STATE_MANAGERS: WeakMap = new WeakMap() + export class TypeormDatabase { private statusSchema: string private isolationLevel: IsolationLevel diff --git a/typeorm/typeorm-store/src/index.ts b/typeorm/typeorm-store/src/index.ts index 7db198d25..ccb47792d 100644 --- a/typeorm/typeorm-store/src/index.ts +++ b/typeorm/typeorm-store/src/index.ts @@ -1,4 +1,4 @@ export * from './database' -export {FindManyOptions, FindOneOptions, Store} from './store' +export * from './store' export * from './decorators' export * from './transformers' diff --git a/typeorm/typeorm-store/src/utils/tx.ts b/typeorm/typeorm-store/src/utils/tx.ts index 2b94bf01e..d2ae10f4d 100644 --- a/typeorm/typeorm-store/src/utils/tx.ts +++ b/typeorm/typeorm-store/src/utils/tx.ts @@ -1,5 +1,7 @@ import type {DataSource, EntityManager} from "typeorm" -import type {IsolationLevel} from "../database" + + +export type IsolationLevel = 'SERIALIZABLE' | 'READ COMMITTED' | 'REPEATABLE READ' export interface Tx { From 0080b0135350848c50fc92c2ebfc364c146551c0 Mon Sep 17 00:00:00 2001 From: belopash Date: Tue, 28 May 2024 14:30:40 +0500 Subject: [PATCH 05/11] save --- typeorm/typeorm-store/src/database.ts | 19 ++++++++++--------- typeorm/typeorm-store/src/store.ts | 6 +++--- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/typeorm/typeorm-store/src/database.ts b/typeorm/typeorm-store/src/database.ts index ecf18bfa5..a64b494c9 100644 --- a/typeorm/typeorm-store/src/database.ts +++ b/typeorm/typeorm-store/src/database.ts @@ -58,7 +58,7 @@ export class TypeormDatabase { await this.con.initialize() try { - return await this.con.transaction('SERIALIZABLE', (em) => this.initTransaction(em)) + return await this.con.transaction('SERIALIZABLE', em => this.initTransaction(em)) } catch (e: any) { await this.con.destroy().catch(() => {}) // ignore error this.con = undefined @@ -128,7 +128,7 @@ export class TypeormDatabase { } transact(info: FinalTxInfo, cb: (store: Store) => Promise): Promise { - return this.submit(async (em) => { + return this.submit(async em => { let state = await this.getState(em) let {prevHead: prev, nextHead: next} = info @@ -160,7 +160,7 @@ export class TypeormDatabase { info: HotTxInfo, cb: (store: Store, sliceBeg: number, sliceEnd: number) => Promise ): Promise { - return this.submit(async (em) => { + return this.submit(async em => { let state = await this.getState(em) let chain = [state, ...state.top] @@ -168,7 +168,7 @@ export class TypeormDatabase { assert(info.finalizedHead.height <= (maybeLast(info.newBlocks) ?? info.baseHead).height) assert( - chain.find((b) => b.hash === info.baseHead.hash), + chain.find(b => b.hash === info.baseHead.hash), RACE_MSG ) if (info.newBlocks.length == 0) { @@ -185,14 +185,18 @@ export class TypeormDatabase { if (info.newBlocks.length) { let finalizedEnd = info.finalizedHead.height - info.newBlocks[0].height + 1 if (finalizedEnd > 0) { - await this.performUpdates((store) => cb(store, 0, finalizedEnd), em) + await this.performUpdates(store => cb(store, 0, finalizedEnd), em) } else { finalizedEnd = 0 } for (let i = finalizedEnd; i < info.newBlocks.length; i++) { let b = info.newBlocks[i] await this.insertHotBlock(em, b) - await this.performUpdates((store) => cb(store, i, i + 1), em, new ChangeWriter(em, this.statusSchema, b.height)) + await this.performUpdates( + store => cb(store, i, i + 1), + em, + new ChangeWriter(em, this.statusSchema, b.height) + ) } } @@ -298,10 +302,8 @@ export class TypeormDatabase { } } - const RACE_MSG = 'status table was updated by foreign process, make sure no other processor is running' - function assertStateInvariants(state: DatabaseState): DatabaseState { let height = state.height @@ -313,7 +315,6 @@ function assertStateInvariants(state: DatabaseState): DatabaseState { return state } - function assertChainContinuity(base: HashAndHeight, chain: HashAndHeight[]) { let prev = base for (let b of chain) { diff --git a/typeorm/typeorm-store/src/store.ts b/typeorm/typeorm-store/src/store.ts index 73fb6a745..477996241 100644 --- a/typeorm/typeorm-store/src/store.ts +++ b/typeorm/typeorm-store/src/store.ts @@ -148,10 +148,10 @@ export class Store { this.logger?.debug(`upsert ${entities.length} ${metadata.name} entities`) await this.changes?.writeUpsert(metadata, entities) - let fk = metadata.columns.filter((c) => c.relationMetadata) + let fk = metadata.columns.filter(c => c.relationMetadata) if (fk.length == 0) return this.upsertMany(metadata.target, entities) let signatures = entities - .map((e) => ({entity: e, value: this.getFkSignature(fk, e)})) + .map(e => ({entity: e, value: this.getFkSignature(fk, e)})) .sort((a, b) => (a.value > b.value ? -1 : b.value > a.value ? 1 : 0)) let currentSignature = signatures[0].value let batch: EntityLiteral[] = [] @@ -365,7 +365,7 @@ export class Store { } }) - if (this.resetMode === 'FLUSH' || reset) { + if (reset ?? this.resetMode === 'FLUSH') { this.reset() } } finally { From 47e97dfcc3a43f5f32b1480ddaeafb0d8182d970 Mon Sep 17 00:00:00 2001 From: abernatskiy Date: Fri, 7 Jun 2024 05:56:30 +0900 Subject: [PATCH 06/11] Inline docs modified: src/database.ts modified: src/store.ts modified: src/test/store.test.ts --- typeorm/typeorm-store/src/database.ts | 57 +++++++++++++++-- typeorm/typeorm-store/src/store.ts | 65 +++++++++++++++++--- typeorm/typeorm-store/src/test/store.test.ts | 8 +-- 3 files changed, 114 insertions(+), 16 deletions(-) diff --git a/typeorm/typeorm-store/src/database.ts b/typeorm/typeorm-store/src/database.ts index a64b494c9..21f9484fb 100644 --- a/typeorm/typeorm-store/src/database.ts +++ b/typeorm/typeorm-store/src/database.ts @@ -15,12 +15,61 @@ export {IsolationLevel} export interface TypeormDatabaseOptions { + + /** + * Support for storing the data on unfinalized / hot + * blocks and the related rollbacks. + * See {@link https://docs.subsquid.io/sdk/resources/basics/unfinalized-blocks/} + * + * @defaultValue true + */ supportHotBlocks?: boolean + + /** + * PostgreSQL ransaction isolation level + * See {@link https://www.postgresql.org/docs/current/transaction-iso.html} + * + * @defaultValue 'SERIALIZABLE' + */ isolationLevel?: IsolationLevel + + /** + * When the queries should be sent to the database? + * + * @defaultValue FlushMode.AUTO + */ flushMode?: FlushMode + + /** + * When the cache should be dropped? + * + * @defaultValue ResetMode.BATCH + */ resetMode?: ResetMode + + /** + * Which database reads should be cached? + * + * @defaultValue CacheMode.ALL + */ cacheMode?: CacheMode + + /** + * Name of the database schema that the processor + * will use to track its state (height and hash of + * the highest indexed block). Set this if you run + * more than one processor against the same DB. + * + * @defaultValue 'squid_processor' + */ stateSchema?: string + + /** + * Directory with model definitions (at lib/model) + * and migrations (at db/migrations). + * + * @defaultValue process.cwd() + */ projectDir?: string } @@ -42,9 +91,9 @@ export class TypeormDatabase { constructor(options?: TypeormDatabaseOptions) { this.statusSchema = options?.stateSchema || 'squid_processor' this.isolationLevel = options?.isolationLevel || 'SERIALIZABLE' - this.resetMode = options?.resetMode || 'BATCH' - this.flushMode = options?.flushMode || 'AUTO' - this.cacheMode = options?.cacheMode || 'ALL' + this.resetMode = options?.resetMode || ResetMode.BATCH + this.flushMode = options?.flushMode || FlushMode.AUTO + this.cacheMode = options?.cacheMode || CacheMode.ALL this.supportsHotBlocks = options?.supportHotBlocks !== false this.projectDir = options?.projectDir || process.cwd() } @@ -254,7 +303,7 @@ export class TypeormDatabase { try { await cb(store) await store.flush() - if (this.resetMode === 'BATCH') store.reset() + if (this.resetMode === ResetMode.BATCH) store.reset() } finally { store['isClosed'] = true } diff --git a/typeorm/typeorm-store/src/store.ts b/typeorm/typeorm-store/src/store.ts index 477996241..a52167c86 100644 --- a/typeorm/typeorm-store/src/store.ts +++ b/typeorm/typeorm-store/src/store.ts @@ -17,11 +17,60 @@ import assert from 'assert' export {EntityTarget} -export type FlushMode = 'AUTO' | 'BATCH' | 'ALWAYS' +export const enum FlushMode { -export type ResetMode = 'BATCH' | 'MANUAL' | 'FLUSH' + /** + * Send queries to the database transaction at every + * direct database read (all read methods besides + * .get()) and at the end of the batch. + */ + AUTO, + + /** + * Send queries to the database transaction strictly + * at the end of the batch. + */ + BATCH, + + /** + * Send queries to the database transaction whenever + * the data is read or written (including .get(), + * .insert(), .upsert(), .delete()) + */ + ALWAYS +} -export type CacheMode = 'ALL' | 'MANUAL' +export const enum ResetMode { + + /** + * Clear cache at the end of each batch or manually. + */ + BATCH, + + /** + * Clear cache only manually. + */ + MANUAL, + + /** + * Clear cache whenever any queries are sent to the + * database transaction. + */ + FLUSH +} + +export const enum CacheMode { + + /** + * Data from all database reads is cached. + */ + ALL, + + /** + * Only the data from flagged database reads is cached. + */ + MANUAL +} export interface GetOptions { id: string @@ -258,7 +307,7 @@ export class Store { return await this.performRead(async () => { const {cache, ...opts} = options const res = await this.em.find(target, opts) - if (cache ?? this.cacheMode === 'ALL') { + if (cache ?? this.cacheMode === CacheMode.ALL) { const metadata = this.getEntityMetadata(target) for (const e of res) { this.cacheEntity(metadata, e) @@ -283,7 +332,7 @@ export class Store { return await this.performRead(async () => { const {cache, ...opts} = options const res = await this.em.findOne(target, opts).then(noNull) - if (cache ?? this.cacheMode === 'ALL') { + if (cache ?? this.cacheMode === CacheMode.ALL) { const metadata = this.getEntityMetadata(target) const idOrEntity = res || getIdFromWhere(options.where) this.cacheEntity(metadata, idOrEntity) @@ -365,7 +414,7 @@ export class Store { } }) - if (reset ?? this.resetMode === 'FLUSH') { + if (reset ?? this.resetMode === ResetMode.FLUSH) { this.reset() } } finally { @@ -376,7 +425,7 @@ export class Store { private async performRead(cb: () => Promise): Promise { this.assertNotClosed() - if (this.flushMode === 'AUTO' || this.flushMode === 'ALWAYS') { + if (this.flushMode === FlushMode.AUTO || this.flushMode === FlushMode.ALWAYS) { await this.flush() } return await cb() @@ -386,7 +435,7 @@ export class Store { this.assertNotClosed() await this.pendingCommit?.promise() await cb() - if (this.flushMode === 'ALWAYS') { + if (this.flushMode === FlushMode.ALWAYS) { await this.flush() } } diff --git a/typeorm/typeorm-store/src/test/store.test.ts b/typeorm/typeorm-store/src/test/store.test.ts index 5773b196c..5841d4cfb 100644 --- a/typeorm/typeorm-store/src/test/store.test.ts +++ b/typeorm/typeorm-store/src/test/store.test.ts @@ -1,7 +1,7 @@ import {assertNotNull} from '@subsquid/util-internal' import expect from 'expect' import {Equal} from 'typeorm' -import {Store} from '../store' +import {Store, CacheMode, FlushMode, ResetMode} from '../store' import {Item, Order} from './lib/model' import {getEntityManager, useDatabase} from './util' import {sortMetadatasInCommitOrder} from '../utils/commitOrder' @@ -161,9 +161,9 @@ export async function createStore(): Promise { state: new StateManager({ commitOrder: sortMetadatasInCommitOrder(em.connection), }), - cacheMode: 'ALL', - flushMode: 'AUTO', - resetMode: 'BATCH', + cacheMode: CacheMode.ALL, + flushMode: FlushMode.AUTO, + resetMode: ResetMode.BATCH, }) } From a6753ada2ccfd3f0ad42b9e69e3a774a5e83da28 Mon Sep 17 00:00:00 2001 From: belopash Date: Tue, 11 Jun 2024 16:57:19 +0500 Subject: [PATCH 07/11] fix --- typeorm/typeorm-store/src/utils/stateManager.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/typeorm/typeorm-store/src/utils/stateManager.ts b/typeorm/typeorm-store/src/utils/stateManager.ts index 53ca38d4c..2f023a846 100644 --- a/typeorm/typeorm-store/src/utils/stateManager.ts +++ b/typeorm/typeorm-store/src/utils/stateManager.ts @@ -117,6 +117,7 @@ export class StateManager { case ChangeType.Delete: this.setState(metadata, entity.id, ChangeType.Upsert) this.cacheMap.add(metadata, entity, true) + break default: throw unexpectedCase(prevType) } From e1b51d82f3a4d51ecf339bd97a77f0ff5eab316e Mon Sep 17 00:00:00 2001 From: belopash Date: Thu, 13 Jun 2024 15:18:28 +0500 Subject: [PATCH 08/11] do not modify original entity if extra upsert is needed --- .../typeorm-store/src/utils/stateManager.ts | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/typeorm/typeorm-store/src/utils/stateManager.ts b/typeorm/typeorm-store/src/utils/stateManager.ts index 2f023a846..86de1e102 100644 --- a/typeorm/typeorm-store/src/utils/stateManager.ts +++ b/typeorm/typeorm-store/src/utils/stateManager.ts @@ -198,9 +198,8 @@ export class StateManager { case ChangeType.Insert: { assert(cached?.value != null, `unable to insert entity ${metadata.name} ${id}`) - inserts.push(cached.value) - - const extraUpsert = this.extractExtraUpsert(metadata, cached.value) + const {entity, extraUpsert} = this.extractExtraUpsert(metadata, cached.value) + inserts.push(entity) if (extraUpsert != null) { extraUpserts.push(extraUpsert) } @@ -210,9 +209,9 @@ export class StateManager { case ChangeType.Upsert: { assert(cached?.value != null, `unable to upsert entity ${metadata.name} ${id}`) - upserts.push(cached.value) - - const extraUpsert = this.extractExtraUpsert(metadata, cached.value) + + const {entity, extraUpsert} = this.extractExtraUpsert(metadata, cached.value) + upserts.push(entity) if (extraUpsert != null) { extraUpserts.push(extraUpsert) } @@ -268,15 +267,18 @@ export class StateManager { if (!isInverseInserted) continue if (extraUpsert == null) { - extraUpsert = metadata.create() as E - extraUpsert.id = entity.id - Object.assign(extraUpsert, entity) + extraUpsert = entity + entity = metadata.create() as E + Object.assign(entity, extraUpsert) } relation.setEntityValue(entity, undefined) } - return extraUpsert + return { + entity, + extraUpsert + } } private setState(metadata: EntityMetadata, id: string, type: ChangeType): this { From 77ace58943fde32fa2f73f48689188ffb1f4b413 Mon Sep 17 00:00:00 2001 From: belopash Date: Thu, 13 Jun 2024 22:24:11 +0500 Subject: [PATCH 09/11] use fast-copy package --- common/config/rush/pnpm-lock.yaml | 111 +++++++++++------- typeorm/typeorm-store/package.json | 4 +- typeorm/typeorm-store/src/utils/cacheMap.ts | 5 +- typeorm/typeorm-store/src/utils/misc.ts | 47 -------- .../typeorm-store/src/utils/stateManager.ts | 6 +- 5 files changed, 78 insertions(+), 95 deletions(-) diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 017975960..b20efde2a 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -302,6 +302,9 @@ dependencies: '@substrate/calc': specifier: ^0.2.8 version: 0.2.8 + '@types/clone': + specifier: ^2.1.4 + version: 2.1.4 '@types/deep-equal': specifier: ^1.0.4 version: 1.0.4 @@ -365,6 +368,9 @@ dependencies: camelcase: specifier: ^6.3.0 version: 6.3.0 + clone: + specifier: ^2.1.2 + version: 2.1.2 commander: specifier: ^11.1.0 version: 11.1.0 @@ -2548,6 +2554,10 @@ packages: '@types/node': 18.19.31 dev: false + /@types/clone@2.1.4: + resolution: {integrity: sha512-NKRWaEGaVGVLnGLB2GazvDaZnyweW9FJLLFL5LhywGJB3aqGMT9R/EUoJoSRP4nzofYnZysuDmrEJtJdAqUOtQ==} + dev: false + /@types/connect@3.4.38: resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} dependencies: @@ -3383,6 +3393,11 @@ packages: mimic-response: 1.0.1 dev: false + /clone@2.1.2: + resolution: {integrity: sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==} + engines: {node: '>=0.8'} + dev: false + /cluster-key-slot@1.1.2: resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} engines: {node: '>=0.10.0'} @@ -3937,6 +3952,10 @@ packages: pure-rand: 6.1.0 dev: false + /fast-copy@3.0.2: + resolution: {integrity: sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ==} + dev: false + /fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} dev: false @@ -5502,6 +5521,10 @@ packages: engines: {node: '>= 4'} dev: false + /rfdc@1.4.1: + resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + dev: false + /rollup@4.14.2: resolution: {integrity: sha512-WkeoTWvuBoFjFAhsEOHKRoZ3r9GfTyhh7Vff1zwebEFLEFjT1lG3784xEgKiTa7E+e70vsC81roVL2MP4tgEEQ==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} @@ -6532,7 +6555,7 @@ packages: dev: false file:projects/astar-erc20.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-fqUQYw2QOvqPIa3TEHmaQSQIzf/TRnu92b/cwDihlmlfYV+UwTc/bLpCnBTfDbJWlQ5rPJmY7FJYTzK0zGYIXA==, tarball: file:projects/astar-erc20.tgz} + resolution: {integrity: sha512-nM8FzKINL+LRE3dJYPRcmDUYQylhViG8sv0ITeHvNrThiTwoRw3Is01eRofsmCanLrxKP7sedWQK7+LKLph1gQ==, tarball: file:projects/astar-erc20.tgz} id: file:projects/astar-erc20.tgz name: '@rush-temp/astar-erc20' version: 0.0.0 @@ -6566,7 +6589,7 @@ packages: dev: false file:projects/balances.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-SOIUJD5nBTiOJbTTG9/nx3HAQooJusXGXe6BI7vv1DeSc1UclmhexzZV3VYQim5NRST0DwbsNciDxgh6TqQMEQ==, tarball: file:projects/balances.tgz} + resolution: {integrity: sha512-sVXMA5Q+CTTaWSY0dP6Jy4wuAmKFA7YYb6Phg+Giep32Wo4MhuVqkzfHf2awB1pf+G3A6b0Yn/LmeQsyT6OCLQ==, tarball: file:projects/balances.tgz} id: file:projects/balances.tgz name: '@rush-temp/balances' version: 0.0.0 @@ -6617,7 +6640,7 @@ packages: dev: false file:projects/borsh-bench.tgz: - resolution: {integrity: sha512-4thNgEQETqf3DVQzlFeKdaEBDQDU4C+hLDMDCpzCcBwsbfcTGrh4bqxYZjKHKba2Sl0ZVg2MCiBUuOK6Ujo+wA==, tarball: file:projects/borsh-bench.tgz} + resolution: {integrity: sha512-0xnTzo1dLljgeDpBDXNZwF8r3oM/EyDvJVrdGQq4dq2QPCswCKdFCMubp071rmcJzeZJqbZ7e/FH3tb4bFCgBg==, tarball: file:projects/borsh-bench.tgz} name: '@rush-temp/borsh-bench' version: 0.0.0 dependencies: @@ -6655,7 +6678,7 @@ packages: dev: false file:projects/data-test.tgz: - resolution: {integrity: sha512-XOjgnQzgt0Awd6ZhQK3peOHMglZ9SsH+AHiiREw9Hfds8LCeTp4OhlflLe/QHK3JZInGTdKBHtyNBqkYfReq+g==, tarball: file:projects/data-test.tgz} + resolution: {integrity: sha512-DGiTWToHQch/bYReq5oI0mNGC0LM/WuMe8uwkU1a5sADzS+aX68lIypYVvIkRgpJEfcmOfsa1RgL2mata3AbJg==, tarball: file:projects/data-test.tgz} name: '@rush-temp/data-test' version: 0.0.0 dependencies: @@ -6669,7 +6692,7 @@ packages: dev: false file:projects/erc20-transfers.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-ch12dYQwXZ0AJw/99dxAR9eUhAMZXkOX4f20B7hqoEQavSFASWMOPZ9l6LB1lH3YnS5fBgrgn6dyIjLc9McCSA==, tarball: file:projects/erc20-transfers.tgz} + resolution: {integrity: sha512-AN4uyAnmQuT4N8ns6IAYjw2p6f1suGPLwX/Sj1CrIJ69zUuAK5qPd+dYEDfwKm0FiEbB5/pJya/CQCl3M3tvpw==, tarball: file:projects/erc20-transfers.tgz} id: file:projects/erc20-transfers.tgz name: '@rush-temp/erc20-transfers' version: 0.0.0 @@ -6703,7 +6726,7 @@ packages: dev: false file:projects/evm-abi.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-KGidYnAntwDgonJGlDFpUjhlPahyM1xgkj4cHOnpxVypF7wnahUo3AYG8A++9koD2KQBSmxsFsNaljj2Lgn5rA==, tarball: file:projects/evm-abi.tgz} + resolution: {integrity: sha512-YOB7H12nuCatlWP8NECCnDDpRL7l8wBxH2BauIZQ/4rGNf5zgemknH8DemVjO1wsk5aqx2Nm2lPUw6oF29e4OQ==, tarball: file:projects/evm-abi.tgz} id: file:projects/evm-abi.tgz name: '@rush-temp/evm-abi' version: 0.0.0 @@ -6733,7 +6756,7 @@ packages: dev: false file:projects/evm-codec.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-w1XelVKp/Sb6coGaQYXQw1ho/4UxFlZ2bc9xidnxatDY7yUNGn0TgBOI49OFEPvThDFYCuutY0j9Qre2aWRI4w==, tarball: file:projects/evm-codec.tgz} + resolution: {integrity: sha512-lcRV7X5lEKL8SNPHDvavJN1SAqUq/x+I6b2ArX49I0fLdmmxia0Hw5zKtA4TrbxTFnq9hIfQSgsLrHlzOS8VNg==, tarball: file:projects/evm-codec.tgz} id: file:projects/evm-codec.tgz name: '@rush-temp/evm-codec' version: 0.0.0 @@ -6762,7 +6785,7 @@ packages: dev: false file:projects/evm-processor.tgz: - resolution: {integrity: sha512-QjnluQmW9FI9zI51Kl3xpjK91z7O8taHttj2mMHO3J61beFmSbFgO1iIlUaXOm5f6AQecuxbXAN5+hBKkqm36w==, tarball: file:projects/evm-processor.tgz} + resolution: {integrity: sha512-RoOv+nclQlx/zDAnZmmyDfvG//UkB7b7zkir+M19TX944SsX4BbskPMYBqGX4oIweT+62cOYGD2GYPEBt+Nr9g==, tarball: file:projects/evm-processor.tgz} name: '@rush-temp/evm-processor' version: 0.0.0 dependencies: @@ -6771,7 +6794,7 @@ packages: dev: false file:projects/evm-typegen.tgz: - resolution: {integrity: sha512-HsCgpWkqSwWLDXiToIldaqNXQzUWuC1zviMI7Beb81eRL6Nur5KPU+JJkdRf24EkxOf2XNTCl0evhr8gwVLcdg==, tarball: file:projects/evm-typegen.tgz} + resolution: {integrity: sha512-12v7+7DuFV5DfWMjr7F9EUlsd3Fi2VbiZ8Br9WgclQP5Gg1UpZiZVF7J7fTV9reX1U+I7In+lHEKHh8yRx1DlA==, tarball: file:projects/evm-typegen.tgz} name: '@rush-temp/evm-typegen' version: 0.0.0 dependencies: @@ -6798,7 +6821,7 @@ packages: dev: false file:projects/fuel-data.tgz: - resolution: {integrity: sha512-dNmf9p1d7ilXI4tRNCaUHdDxXbIk3PyT+8bCidHbgofS8t3ZKVw+YdQ2HvRp5slbJ6NnHWAb1YcjYFLGlPp1RQ==, tarball: file:projects/fuel-data.tgz} + resolution: {integrity: sha512-VuXVfyKxlcvaYVTCI0RBJ9qYRwGT/XEGDXqzc3Pa4TttUquQFO0bWB47slxgPiBHYAIgd9/ZejW0DO+UM9k5Fw==, tarball: file:projects/fuel-data.tgz} name: '@rush-temp/fuel-data' version: 0.0.0 dependencies: @@ -6807,7 +6830,7 @@ packages: dev: false file:projects/fuel-dump.tgz: - resolution: {integrity: sha512-uYwx3hc4Vc/pskvTl2RhFp49KzH4YztbdoN+yPoYPzfQdWUgvOrX1/ZDWcWkR9ccjbOvbDBSr6St1EeLeigqjw==, tarball: file:projects/fuel-dump.tgz} + resolution: {integrity: sha512-9j6rLrBfJ5ko3as3QmZH5HnzGM05+CksWCTgZi0FOLMoB336JOgzLCGNTgR/BMTdgcccDjgKqIsvWB/OOSQFpg==, tarball: file:projects/fuel-dump.tgz} name: '@rush-temp/fuel-dump' version: 0.0.0 dependencies: @@ -6816,7 +6839,7 @@ packages: dev: false file:projects/fuel-indexer.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-Lpgs5YWKEUwa4RNULHVdGAkwD1ls4hig8fs18fS4Fs6w776hw7v0W1H1Pp+fByqX1o677tITg06nUULpc7pvhA==, tarball: file:projects/fuel-indexer.tgz} + resolution: {integrity: sha512-5/wI8rl+4dPJDnKYxprkRahZQBggGZ8PFIb++U/Ux8iviuzSgrcTp5QXpbBP0hNSMRItg+EXlEuZ9soH7a+7zg==, tarball: file:projects/fuel-indexer.tgz} id: file:projects/fuel-indexer.tgz name: '@rush-temp/fuel-indexer' version: 0.0.0 @@ -6847,7 +6870,7 @@ packages: dev: false file:projects/fuel-ingest.tgz: - resolution: {integrity: sha512-J4qwkpMsR9v3Tffp05tQWXnRkVKXz0Z22nItSqsz58H8xCWhAXOXs5vuc4AfSvwn/J7wDlR8P+PX1kyT7husjQ==, tarball: file:projects/fuel-ingest.tgz} + resolution: {integrity: sha512-S0MBibhUnPHGEZDmPXeg7Ty0n9KUy8unbwc0KwyFsM6wZPTtLxY7vvEI/fyRE776Zwv1PQC0v05AB7HCNDvrvQ==, tarball: file:projects/fuel-ingest.tgz} name: '@rush-temp/fuel-ingest' version: 0.0.0 dependencies: @@ -6856,7 +6879,7 @@ packages: dev: false file:projects/fuel-normalization.tgz: - resolution: {integrity: sha512-zBSLvem3QcfHry+1tXc/dxvGX/0lL6MtwYjIw8P3mtENMychDho9FwQvOwNt5gAg2qgvJEgSef3GW98ESUjspg==, tarball: file:projects/fuel-normalization.tgz} + resolution: {integrity: sha512-5OC0ntR4l4na/fJyUnjuN7pGx2eF5ice4qg/Ct9VtcqKt3oCVlO6hU2JORMwU1xQRFYq1QQK1pujEucrsnj+pQ==, tarball: file:projects/fuel-normalization.tgz} name: '@rush-temp/fuel-normalization' version: 0.0.0 dependencies: @@ -6865,7 +6888,7 @@ packages: dev: false file:projects/fuel-objects.tgz: - resolution: {integrity: sha512-GWl3yKfrwUeqbl/ov+0AXha8FTkye7hl/uD0jK4pEh9+8d8xGX4fkMclBx/YLNy9JCryPiCtX6kEXNE9sH/74w==, tarball: file:projects/fuel-objects.tgz} + resolution: {integrity: sha512-OJ8Vr2eQP+TmuJ3L2Asj8hxegmNn8aT21XGf+FChQ7SWYEj+VqMtMtpYAqjtG8U5Gyk7wuobahNHuVMVwZFBjg==, tarball: file:projects/fuel-objects.tgz} name: '@rush-temp/fuel-objects' version: 0.0.0 dependencies: @@ -6874,7 +6897,7 @@ packages: dev: false file:projects/fuel-stream.tgz: - resolution: {integrity: sha512-ETASw7qHqdaZlb3lVXkwLg/hnuH8BNlisS5DtM/BE/f2jk6s6fD+ruqa6XKPonNI0NbcJE+ajfUaODjBRNaP+g==, tarball: file:projects/fuel-stream.tgz} + resolution: {integrity: sha512-Mh5LbXoIpmruRVgdRvjZsWUBL6Rws5EMnloykA7f/GgTyqPNbpr1PAw8rqxN7BnD1wqJv38BptByeu3A2dA5pw==, tarball: file:projects/fuel-stream.tgz} name: '@rush-temp/fuel-stream' version: 0.0.0 dependencies: @@ -6901,7 +6924,7 @@ packages: dev: false file:projects/graphql-server.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-Em4HlNc1XsbxZlBDWgi/kyzr3bKi+aey58DMFrJIhpdbZWuUnFVWQUx2MEH5Gd8G+7CJUNOnM8sLI3Y5rcghPg==, tarball: file:projects/graphql-server.tgz} + resolution: {integrity: sha512-yQm2KmgdGJjxe5XLuqx2vUc9DJFc7+WIXiZR81nCke0HlCjZrlrZ5IV6TreN7TQdBNeQ2iAQEnyHxnCGmpFyOQ==, tarball: file:projects/graphql-server.tgz} id: file:projects/graphql-server.tgz name: '@rush-temp/graphql-server' version: 0.0.0 @@ -7005,7 +7028,7 @@ packages: dev: false file:projects/openreader.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-1puqNG9tXqqbQFfGZ7BlcGGLqKiEd6QyjBVyjzkvaW+OQIe1Bwauo/bGl12rBmo/dmREuXfKcMRu3d38noDNdA==, tarball: file:projects/openreader.tgz} + resolution: {integrity: sha512-1yTnDvAUuWzyPC2bH2BZUtYhRMgK7Yh1MDnWwEyBjEhkhrJp/oqZ+8LbN8GpWREaQusvqQn+0U5PLiWqSjfNrw==, tarball: file:projects/openreader.tgz} id: file:projects/openreader.tgz name: '@rush-temp/openreader' version: 0.0.0 @@ -7050,7 +7073,7 @@ packages: dev: false file:projects/raw-archive-validator.tgz: - resolution: {integrity: sha512-q+OGPdLgI3fMid3VGwbwha6STqbMNcC51DlCT89DHIN5L8yJCeT6J9J3xhYRZcjdi/y5FDHH+q/X1VBnI74V2g==, tarball: file:projects/raw-archive-validator.tgz} + resolution: {integrity: sha512-cNBWqHaHWG16Os44defUv2B8FunEseiZFZU3OiNCh6FQXMi2lDWfIVUZ1ls920y/ZImJKBeSWiJ3SAM2b7tp7Q==, tarball: file:projects/raw-archive-validator.tgz} name: '@rush-temp/raw-archive-validator' version: 0.0.0 dependencies: @@ -7089,7 +7112,7 @@ packages: dev: false file:projects/shibuya-psp22.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-UrElinCriyBs2rDud5OJBuRr5TSOQEPviv/aaVCOiiPomNkWReqxRZ1dfKpC2Ab6WaTo+FJ2Kp7Vs48Uz/bF/Q==, tarball: file:projects/shibuya-psp22.tgz} + resolution: {integrity: sha512-1t8Hsg3mW5H7WUZGfcgmUAO08W1QcSQr3t8fkH9E1QhlngZFRh3zCmnETKAcINB5O5qqvzPXEqo9rR7cZv27cQ==, tarball: file:projects/shibuya-psp22.tgz} id: file:projects/shibuya-psp22.tgz name: '@rush-temp/shibuya-psp22' version: 0.0.0 @@ -7120,7 +7143,7 @@ packages: dev: false file:projects/solana-dump.tgz: - resolution: {integrity: sha512-j0NXfhUpCwqsp+WJN5lPFcKUCQNLg1sO48TlJiVRh3oqbrqbk/2YAL1HpbT7IYiUTwjBTDH80t/kq12iIWr0Og==, tarball: file:projects/solana-dump.tgz} + resolution: {integrity: sha512-YzB/1GR4S3BsjfuX2DrapDSm4fimLkLyx2oOxatzkl5Pwgl7IWNNY1MnGcKkvFnRhLZoV8QnCjS+R49BYtiwHw==, tarball: file:projects/solana-dump.tgz} name: '@rush-temp/solana-dump' version: 0.0.0 dependencies: @@ -7129,7 +7152,7 @@ packages: dev: false file:projects/solana-example.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-weRsJJ5qlielUcNWVf7Y/cbsnB1lPnwD3teFDjELKZaehmqAMX3C0rMNp1yLjokpgPk9z4avMmR0QcyjNQ1zZQ==, tarball: file:projects/solana-example.tgz} + resolution: {integrity: sha512-Ns7M8TC//cSLfJybpLFsop3TVKD33gDaCcfjs6VUh/UDsMpEPOmxyGEXmYzujPpmpc/7GD7iBXJ52kZ/po/fHg==, tarball: file:projects/solana-example.tgz} id: file:projects/solana-example.tgz name: '@rush-temp/solana-example' version: 0.0.0 @@ -7160,7 +7183,7 @@ packages: dev: false file:projects/solana-ingest.tgz: - resolution: {integrity: sha512-7cgYeih8AyL9y/y8nq61M44dK9XjxJwQPC1Ydb4Fp12AzwYL8T7ZhJmCVXBr+LIIYLi3USr4hHBgxg3fgxeZMw==, tarball: file:projects/solana-ingest.tgz} + resolution: {integrity: sha512-jfOG8LHbabxvfyr66/F9IvuwLO7XdSE4dEdlPZhrjoq+m7k0gtJ16QVTKDuhAA4U8qOqwtQicuzfPKy+df3iPQ==, tarball: file:projects/solana-ingest.tgz} name: '@rush-temp/solana-ingest' version: 0.0.0 dependencies: @@ -7169,7 +7192,7 @@ packages: dev: false file:projects/solana-normalization.tgz: - resolution: {integrity: sha512-O//+YPMmLqlNLdWYmga/ACWv49TMqzDMJB2iiJZRZzicUODmqFhhq7aY/ZQQe8NV/HrCS19cNomP7YQXHKfVHg==, tarball: file:projects/solana-normalization.tgz} + resolution: {integrity: sha512-y3093dWykFo9bW4AiRNqNSi1DFFvPjEnrr6aMN7Xgwrg86DmvVLhTLJRYk0MUiGp/nD4+WJKOIOd9qkzAy283w==, tarball: file:projects/solana-normalization.tgz} name: '@rush-temp/solana-normalization' version: 0.0.0 dependencies: @@ -7178,7 +7201,7 @@ packages: dev: false file:projects/solana-objects.tgz: - resolution: {integrity: sha512-LZrDlFBde7ywNHYsqlBnWktjLa4aPw2z8ARdoyfh3P3cMoBwtONGnUc33cVb2Jhp+tMlQ1v4mo4RJVqYXuHoGg==, tarball: file:projects/solana-objects.tgz} + resolution: {integrity: sha512-69rX5eue8et5dXGe4uInoFAfW/D5liK8cfcuQX34asOQXKP53i921uSBQDLvbvoxi4JkwGeQur7AjTeKXfLmWQ==, tarball: file:projects/solana-objects.tgz} name: '@rush-temp/solana-objects' version: 0.0.0 dependencies: @@ -7187,7 +7210,7 @@ packages: dev: false file:projects/solana-rpc-data.tgz: - resolution: {integrity: sha512-6wx7ff9zAv3F9wPMB24SuL4EEQYKASBuhMp/nQZvISkEBOFTFnQ1l5N9y+vUY1aMU7+MM6fsvHlLcmUSpd4p9Q==, tarball: file:projects/solana-rpc-data.tgz} + resolution: {integrity: sha512-OjV0jXCULMC0mll8sJlnUEs/WLflfyg8H6RdS0bw1qSzlUrxPWzEKYrP+axvlLwvdnCpZnceYHi4ePa0lqzFfA==, tarball: file:projects/solana-rpc-data.tgz} name: '@rush-temp/solana-rpc-data' version: 0.0.0 dependencies: @@ -7196,7 +7219,7 @@ packages: dev: false file:projects/solana-rpc.tgz: - resolution: {integrity: sha512-WIGmFLuy0IfwhHSITTAvY4ayHzUzgCDci2JvCf1OrOSRTvcOHvswF/Ubmh4BGz2hsIw1/vOndBvCTg2wQx3Pcg==, tarball: file:projects/solana-rpc.tgz} + resolution: {integrity: sha512-+UyXUQwPC520Mt3cwmiJ5CclgON5Fvc7Zw2HVyI/uhME8d2/iSNwjaEsFvYlSZanqfK2mp6IORum0QOZN/d+Tg==, tarball: file:projects/solana-rpc.tgz} name: '@rush-temp/solana-rpc' version: 0.0.0 dependencies: @@ -7205,7 +7228,7 @@ packages: dev: false file:projects/solana-stream.tgz: - resolution: {integrity: sha512-DAb2WWlb+uDB6ze1prkOsHKWXH9cAzkWxh2LubZIDlQTfi2cYkiwYBdvA6yYCUraW1177YVffKzN8I+9gGV6qA==, tarball: file:projects/solana-stream.tgz} + resolution: {integrity: sha512-jNA0VI23u7Z8mdSYSATMKKz5d8Y/ckdo2oRgON+a22cQSvIqRXGbX8yp2RGYK47YkL2pEhfZ6o1GX9il4bxmeA==, tarball: file:projects/solana-stream.tgz} name: '@rush-temp/solana-stream' version: 0.0.0 dependencies: @@ -7215,7 +7238,7 @@ packages: dev: false file:projects/solana-typegen.tgz: - resolution: {integrity: sha512-370CDKZNivGBMfcnWSmTnEuy4qEi1ZetBEtM9eJrWMBMfLkbcQ3oeKdPWpzzl0isSPy8YJxe1/rYKPfkymv0IQ==, tarball: file:projects/solana-typegen.tgz} + resolution: {integrity: sha512-LSlfR1IVH+qzkPzeDnDL7VHsbCNSf6ClQKsMxn4lXcwjYI6dbX2RcWJX+/ImZnWCYJ1k+RtRSN+n6akiAsKcBA==, tarball: file:projects/solana-typegen.tgz} name: '@rush-temp/solana-typegen' version: 0.0.0 dependencies: @@ -7248,7 +7271,7 @@ packages: dev: false file:projects/substrate-data-raw.tgz: - resolution: {integrity: sha512-o7h09auR6YZ/ZQViKLAM8fuEq+PFlk/DCQWH2ShJ1AwsGuZzAesha2oGdUNt56RxFtdAQk9hM6ySbDwuNI8aFw==, tarball: file:projects/substrate-data-raw.tgz} + resolution: {integrity: sha512-T9oA/jEt/x9WA4O38M/74iZ+IbAFzbhCfjX3dVThJIV66mpXhvhXHfN8Fj9kNe4jIjphtU7rWHZE6jvtNcfvGw==, tarball: file:projects/substrate-data-raw.tgz} name: '@rush-temp/substrate-data-raw' version: 0.0.0 dependencies: @@ -7257,7 +7280,7 @@ packages: dev: false file:projects/substrate-data.tgz: - resolution: {integrity: sha512-Omt2Tp505it+SDM9tReb6zEjWGvOzEab4mbbs6FpKiPSupM5+eMkCA1trn9rp/Pk+QUvkuv0CNn7r/HXb7sU2A==, tarball: file:projects/substrate-data.tgz} + resolution: {integrity: sha512-qQHwx9m0DMmQPIwS+4hqTdxToovLJAEukLYTqZQCyF/qnAkFC9ozmyfkuOSkz+chcAbrryouJHwxQgn6xdKMYQ==, tarball: file:projects/substrate-data.tgz} name: '@rush-temp/substrate-data' version: 0.0.0 dependencies: @@ -7268,7 +7291,7 @@ packages: dev: false file:projects/substrate-dump.tgz: - resolution: {integrity: sha512-Tn+o7bwXWDzCZbWQv7qAlxkpcNVjW6Sz7kKk7CXIKPwPwd3QunYTw60IasBuq0rJHTVVCew3GGCb5ipapXMjdQ==, tarball: file:projects/substrate-dump.tgz} + resolution: {integrity: sha512-FUJ19MipnL80R+xikPlYWNL85EWS5DFFMnvL09LgAxi2JjvfoBfvY2jUuUkezEpn0hOFyFrGFKozEnc3kEcAUw==, tarball: file:projects/substrate-dump.tgz} name: '@rush-temp/substrate-dump' version: 0.0.0 dependencies: @@ -7277,7 +7300,7 @@ packages: dev: false file:projects/substrate-ingest.tgz: - resolution: {integrity: sha512-N0gpgIiC5rj1kQIIwMqPMrIGwbtvQ3kMzX3s548N1ZMw7usv+MjgbYJSw0744o54ETZ8KOH2YTI1rZHcimmn7A==, tarball: file:projects/substrate-ingest.tgz} + resolution: {integrity: sha512-XmCLuG8FqnRe5pUUb1gM23NSJbtZxrryH1ubHztSHGhQwZPYRAC6kysLH2srxF99OkuxVCdWPywDtNCW0i0Ddg==, tarball: file:projects/substrate-ingest.tgz} name: '@rush-temp/substrate-ingest' version: 0.0.0 dependencies: @@ -7305,7 +7328,7 @@ packages: dev: false file:projects/substrate-processor.tgz: - resolution: {integrity: sha512-3fvboYXqkBYCSH13tnzyK522VPy+3hhHtSSYJPFFDVuSokA2/flhEciSiPsxc2vemX8BVdvcuanPaFP2dJG1eQ==, tarball: file:projects/substrate-processor.tgz} + resolution: {integrity: sha512-wZi9NLCMCfG3NgxjRylCKqUkV9tZ/re41pzUYpo+srZ+G/PQIz202lM5hcyi3QBvg9Z3/s/ix/18xI2iNrYoIA==, tarball: file:projects/substrate-processor.tgz} name: '@rush-temp/substrate-processor' version: 0.0.0 dependencies: @@ -7314,7 +7337,7 @@ packages: dev: false file:projects/substrate-runtime.tgz: - resolution: {integrity: sha512-ndbIQwyTh3t+Q5AiXk/oawuGVHqQWsCZcTfuPYx++EGYNRcFfYvM3kf9AvFYV0xuptoXTew+VnTHeWlnz0i+pQ==, tarball: file:projects/substrate-runtime.tgz} + resolution: {integrity: sha512-BQmKHPLpPHo90qwSymGADy8dhzW4n+upTa5fynp5DnRKiGm5cy/KYEI0bHj9h6mXfxidsNCpPauSmD6PzwPhmw==, tarball: file:projects/substrate-runtime.tgz} name: '@rush-temp/substrate-runtime' version: 0.0.0 dependencies: @@ -7327,7 +7350,7 @@ packages: dev: false file:projects/substrate-typegen.tgz: - resolution: {integrity: sha512-qXKZIoyz64qLhPMU063hgMeRtGvknlgU7T33O85IY1Fh4KKjaXz9xj8RbHVVso94EB0oktedgqT6JZyzXGtJYw==, tarball: file:projects/substrate-typegen.tgz} + resolution: {integrity: sha512-q6W2NzSV3iKpBeBoQ9qA4z1wawx10JZZ2wT5zIpYnl38NjD+1cn5Di3arUbWHsAo6FU33poRVP83Hs0ctH0Z/Q==, tarball: file:projects/substrate-typegen.tgz} name: '@rush-temp/substrate-typegen' version: 0.0.0 dependencies: @@ -7337,7 +7360,7 @@ packages: dev: false file:projects/typeorm-codegen.tgz: - resolution: {integrity: sha512-NHbOI/JYD7iz4i03UWvMKIZ0d7VbLZB5adSLmkK7nFKjkBFUlDjGoGrMI3CkVO3ToL4VA6s5oujD3nKAykCccA==, tarball: file:projects/typeorm-codegen.tgz} + resolution: {integrity: sha512-6cdUKkYHaSNAZtjWrOQCYCKWnHdWEDzy3GV3jxO1PpDd6ySXiujA5ZsPWOAMSINVs3bfu9tpwvTnvXRZTjjTqw==, tarball: file:projects/typeorm-codegen.tgz} name: '@rush-temp/typeorm-codegen' version: 0.0.0 dependencies: @@ -7347,7 +7370,7 @@ packages: dev: false file:projects/typeorm-config.tgz(pg@8.11.5)(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-wKT5CJd1nmlelB6/N9XlgRr89zsY3ZUZ0hxeWxflFK0CXHEpoCAuH9NOue6qpVdTFDDCSVuzgW2pHJhcbHLRwQ==, tarball: file:projects/typeorm-config.tgz} + resolution: {integrity: sha512-QfGFyFFGo+xz1wg5RR7Ez1YZ632lb960Jza+E40D+qHkGjs3R6/6nd4n5AVbPA/hseDGLFFgdYAGCSEYPzdtDQ==, tarball: file:projects/typeorm-config.tgz} id: file:projects/typeorm-config.tgz name: '@rush-temp/typeorm-config' version: 0.0.0 @@ -7377,7 +7400,7 @@ packages: dev: false file:projects/typeorm-migration.tgz(pg@8.11.5)(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-1qIAhun15pIWQR53FR4r9vGfopSAqEkhZwOYC1rlNoIzHd+szSGv5+T9VhrT3qd4nnIJsnTNMnu6D2aRRDEI5Q==, tarball: file:projects/typeorm-migration.tgz} + resolution: {integrity: sha512-pMPW0qPmab6ZlrE5VibdnG+OeyQWdrFy81mL+YkoiMhqHbvwLjGRauthM5Bj7hiGcrXSgRU74FUHQWaY5gC87Q==, tarball: file:projects/typeorm-migration.tgz} id: file:projects/typeorm-migration.tgz name: '@rush-temp/typeorm-migration' version: 0.0.0 @@ -7409,17 +7432,21 @@ packages: dev: false file:projects/typeorm-store.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-sXhxNVktY6PiOA0HiRCYo8dJlsT4AH259wOZ3ledM0FnVRV+6XxQ6Btbd2Yw/9N74raYn5FeRyhSYcewH0Zjmg==, tarball: file:projects/typeorm-store.tgz} + resolution: {integrity: sha512-Uts/GfKPO3MjUUHEfH9Ilu4WW2MvcDUlTU/7dPLEB0JzEssKzczEzXo0w6gKYej4abSQ58m82SqWVVHyMIBD9w==, tarball: file:projects/typeorm-store.tgz} id: file:projects/typeorm-store.tgz name: '@rush-temp/typeorm-store' version: 0.0.0 dependencies: + '@types/clone': 2.1.4 '@types/mocha': 10.0.6 '@types/node': 18.19.31 '@types/pg': 8.11.5 + clone: 2.1.2 expect: 29.7.0 + fast-copy: 3.0.2 mocha: 10.4.0 pg: 8.11.5 + rfdc: 1.4.1 typeorm: 0.3.20(pg@8.11.5)(supports-color@8.1.1)(ts-node@10.9.2) typescript: 5.3.3 transitivePeerDependencies: @@ -7515,7 +7542,7 @@ packages: dev: false file:projects/util-internal-dump-cli.tgz: - resolution: {integrity: sha512-RSNg1RNaDTjBXo1gBwOQrhwSB2t1yHPPm6Dtt7dORMcWod2AubPtG+JRXodw5FsulKMKL+EQ2CYGjTNycboxrA==, tarball: file:projects/util-internal-dump-cli.tgz} + resolution: {integrity: sha512-xgG4eXDo6rW+fJukgdy7UofH0WOSwoJBYu894LYJq7BscGyFgA5vg4hPB7Oyngt1WOQUalKiolKFsIpPCXaqZQ==, tarball: file:projects/util-internal-dump-cli.tgz} name: '@rush-temp/util-internal-dump-cli' version: 0.0.0 dependencies: @@ -7559,7 +7586,7 @@ packages: dev: false file:projects/util-internal-ingest-cli.tgz: - resolution: {integrity: sha512-ACgY+iOdRYoLQhT1B1ys26kxz8WNBf1TP8trLw+yXUyM8tSZ6rg5VXM3raB89TO38vGBTOVC1Xw/2V9/lmiwzA==, tarball: file:projects/util-internal-ingest-cli.tgz} + resolution: {integrity: sha512-aUnjqkCZ1K036D1WsGDalurA8VO5LOChQxgqrTXNHZwIsNU92wERfGdHQ8IkfXDQ/n7TcL/MRd1fNjS6HzftWg==, tarball: file:projects/util-internal-ingest-cli.tgz} name: '@rush-temp/util-internal-ingest-cli' version: 0.0.0 dependencies: diff --git a/typeorm/typeorm-store/package.json b/typeorm/typeorm-store/package.json index f467708cd..f4e7ac3dc 100644 --- a/typeorm/typeorm-store/package.json +++ b/typeorm/typeorm-store/package.json @@ -20,13 +20,15 @@ "dependencies": { "@subsquid/typeorm-config": "^4.1.1", "@subsquid/util-internal": "^3.2.0", - "@subsquid/logger": "^1.3.3" + "@subsquid/logger": "^1.3.3", + "fast-copy": "^3.0.2" }, "peerDependencies": { "typeorm": "^0.3.17", "@subsquid/big-decimal": "^1.0.0" }, "devDependencies": { + "@types/clone": "^2.1.4", "@types/mocha": "^10.0.6", "@types/node": "^18.18.14", "@types/pg": "^8.10.9", diff --git a/typeorm/typeorm-store/src/utils/cacheMap.ts b/typeorm/typeorm-store/src/utils/cacheMap.ts index 7a9af0233..bdb74bc62 100644 --- a/typeorm/typeorm-store/src/utils/cacheMap.ts +++ b/typeorm/typeorm-store/src/utils/cacheMap.ts @@ -1,6 +1,7 @@ import {EntityMetadata} from 'typeorm' -import {copy, EntityLiteral} from './misc' +import {EntityLiteral} from './misc' import {Logger} from '@subsquid/logger' +import clone from 'fast-copy' export class CachedEntity { constructor(public value: E | null = null) {} @@ -63,7 +64,7 @@ export class CacheMap { for (const column of metadata.nonVirtualColumns) { const objectColumnValue = column.getEntityValue(entity) if (isNew || objectColumnValue !== undefined) { - column.setEntityValue(cachedEntity, copy(objectColumnValue ?? null)) + column.setEntityValue(cachedEntity, clone(objectColumnValue ?? null)) } } diff --git a/typeorm/typeorm-store/src/utils/misc.ts b/typeorm/typeorm-store/src/utils/misc.ts index c293f9baa..353fd50b2 100644 --- a/typeorm/typeorm-store/src/utils/misc.ts +++ b/typeorm/typeorm-store/src/utils/misc.ts @@ -17,53 +17,6 @@ export function* splitIntoBatches(list: T[], maxBatchSize: number): Generator } } -const copiedObjects = new WeakMap() - -export function copy(obj: T): T { - if (typeof obj !== 'object' || obj == null) { - return obj - } - - if (copiedObjects.has(obj)) { - return copiedObjects.get(obj) - } else if (obj instanceof Date) { - return new Date(obj) as any - } else if (Array.isArray(obj)) { - const clone = obj.map((i) => copy(i)) - copiedObjects.set(obj, clone) - return clone as any - } else if (obj instanceof Map) { - const clone = new Map(Array.from(obj).map((i) => copy(i))) - copiedObjects.set(obj, clone) - return clone as any - } else if (obj instanceof Set) { - const clone = new Set(Array.from(obj).map((i) => copy(i))) - copiedObjects.set(obj, clone) - return clone as any - } else if (ArrayBuffer.isView(obj)) { - return copyBuffer(obj) - } else { - const clone = Object.create(Object.getPrototypeOf(obj)) - copiedObjects.set(obj, clone) - - for (const k in obj) { - if (obj.hasOwnProperty(k)) { - clone[k] = copy(obj[k]) - } - } - - return clone - } -} - -function copyBuffer(buf: any) { - if (buf instanceof Buffer) { - return Buffer.from(buf) - } else { - return new buf.constructor(buf.buffer.slice(), buf.byteOffset, buf.length) - } -} - export function mergeRelations( a: FindOptionsRelations, b: FindOptionsRelations diff --git a/typeorm/typeorm-store/src/utils/stateManager.ts b/typeorm/typeorm-store/src/utils/stateManager.ts index 86de1e102..5386050db 100644 --- a/typeorm/typeorm-store/src/utils/stateManager.ts +++ b/typeorm/typeorm-store/src/utils/stateManager.ts @@ -2,9 +2,9 @@ import {Logger} from '@subsquid/logger' import {EntityManager, EntityMetadata, FindOptionsRelations} from 'typeorm' import {CacheMap} from './cacheMap' import assert from 'assert' -import {copy, EntityLiteral} from './misc' -import {sortMetadatasInCommitOrder} from './commitOrder' +import {EntityLiteral} from './misc' import {unexpectedCase} from '@subsquid/util-internal' +import clone from 'fast-copy' export enum ChangeType { Insert = 'insert', @@ -50,7 +50,7 @@ export class StateManager { for (const column of metadata.nonVirtualColumns) { const objectColumnValue = column.getEntityValue(entity) if (objectColumnValue !== undefined) { - column.setEntityValue(clonedEntity, copy(objectColumnValue)) + column.setEntityValue(clonedEntity, clone(objectColumnValue)) } } From 60904492565d1b543663208db73c2560a19138bd Mon Sep 17 00:00:00 2001 From: belopash Date: Mon, 17 Jun 2024 17:24:14 +0500 Subject: [PATCH 10/11] use flags instead of mods --- typeorm/typeorm-store/src/database.ts | 78 +++++++------ typeorm/typeorm-store/src/store.ts | 109 ++++++------------- typeorm/typeorm-store/src/test/store.test.ts | 8 +- 3 files changed, 72 insertions(+), 123 deletions(-) diff --git a/typeorm/typeorm-store/src/database.ts b/typeorm/typeorm-store/src/database.ts index 21f9484fb..b189e07f4 100644 --- a/typeorm/typeorm-store/src/database.ts +++ b/typeorm/typeorm-store/src/database.ts @@ -4,20 +4,17 @@ import assert from 'assert' import {DataSource, EntityManager} from 'typeorm' import {ChangeWriter, rollbackBlock} from './utils/changeWriter' import {DatabaseState, FinalTxInfo, HashAndHeight, HotTxInfo} from './interfaces' -import {CacheMode, FlushMode, ResetMode, Store} from './store' +import {Store} from './store' import {createLogger} from '@subsquid/logger' import {StateManager} from './utils/stateManager' import {sortMetadatasInCommitOrder} from './utils/commitOrder' import {IsolationLevel} from './utils/tx' - export {IsolationLevel} - export interface TypeormDatabaseOptions { - /** - * Support for storing the data on unfinalized / hot + * Support for storing the data on unfinalized * blocks and the related rollbacks. * See {@link https://docs.subsquid.io/sdk/resources/basics/unfinalized-blocks/} * @@ -26,7 +23,7 @@ export interface TypeormDatabaseOptions { supportHotBlocks?: boolean /** - * PostgreSQL ransaction isolation level + * PostgreSQL transaction isolation level * See {@link https://www.postgresql.org/docs/current/transaction-iso.html} * * @defaultValue 'SERIALIZABLE' @@ -34,25 +31,25 @@ export interface TypeormDatabaseOptions { isolationLevel?: IsolationLevel /** - * When the queries should be sent to the database? - * - * @defaultValue FlushMode.AUTO + * @defaultValue true */ - flushMode?: FlushMode + batchWriteOperations?: boolean /** - * When the cache should be dropped? - * - * @defaultValue ResetMode.BATCH + * @defaultValue true */ - resetMode?: ResetMode + cacheEntitiesByDefault?: boolean + // FIXME: needs better name, means if we check db if entity is not found in the state /** - * Which database reads should be cached? - * - * @defaultValue CacheMode.ALL + * @defaultValue true */ - cacheMode?: CacheMode + syncOnGet?: boolean + + /** + * @defaultValue true + */ + resetOnCommit?: boolean /** * Name of the database schema that the processor @@ -73,27 +70,27 @@ export interface TypeormDatabaseOptions { projectDir?: string } - const STATE_MANAGERS: WeakMap = new WeakMap() - export class TypeormDatabase { - private statusSchema: string - private isolationLevel: IsolationLevel - private flushMode: FlushMode - private resetMode: ResetMode - private cacheMode: CacheMode - private con?: DataSource - private projectDir: string + protected statusSchema: string + protected isolationLevel: IsolationLevel + protected batchWriteOperations: boolean + protected cacheEntitiesByDefault: boolean + protected syncOnGet: boolean + protected resetOnCommit: boolean + protected con?: DataSource + protected projectDir: string public readonly supportsHotBlocks: boolean constructor(options?: TypeormDatabaseOptions) { this.statusSchema = options?.stateSchema || 'squid_processor' this.isolationLevel = options?.isolationLevel || 'SERIALIZABLE' - this.resetMode = options?.resetMode || ResetMode.BATCH - this.flushMode = options?.flushMode || FlushMode.AUTO - this.cacheMode = options?.cacheMode || CacheMode.ALL + this.batchWriteOperations = options?.batchWriteOperations ?? true + this.cacheEntitiesByDefault = options?.cacheEntitiesByDefault ?? true + this.syncOnGet = options?.syncOnGet ?? true + this.resetOnCommit = options?.resetOnCommit ?? true this.supportsHotBlocks = options?.supportHotBlocks !== false this.projectDir = options?.projectDir || process.cwd() } @@ -107,7 +104,7 @@ export class TypeormDatabase { await this.con.initialize() try { - return await this.con.transaction('SERIALIZABLE', em => this.initTransaction(em)) + return await this.con.transaction('SERIALIZABLE', (em) => this.initTransaction(em)) } catch (e: any) { await this.con.destroy().catch(() => {}) // ignore error this.con = undefined @@ -177,7 +174,7 @@ export class TypeormDatabase { } transact(info: FinalTxInfo, cb: (store: Store) => Promise): Promise { - return this.submit(async em => { + return this.submit(async (em) => { let state = await this.getState(em) let {prevHead: prev, nextHead: next} = info @@ -209,7 +206,7 @@ export class TypeormDatabase { info: HotTxInfo, cb: (store: Store, sliceBeg: number, sliceEnd: number) => Promise ): Promise { - return this.submit(async em => { + return this.submit(async (em) => { let state = await this.getState(em) let chain = [state, ...state.top] @@ -217,7 +214,7 @@ export class TypeormDatabase { assert(info.finalizedHead.height <= (maybeLast(info.newBlocks) ?? info.baseHead).height) assert( - chain.find(b => b.hash === info.baseHead.hash), + chain.find((b) => b.hash === info.baseHead.hash), RACE_MSG ) if (info.newBlocks.length == 0) { @@ -234,7 +231,7 @@ export class TypeormDatabase { if (info.newBlocks.length) { let finalizedEnd = info.finalizedHead.height - info.newBlocks[0].height + 1 if (finalizedEnd > 0) { - await this.performUpdates(store => cb(store, 0, finalizedEnd), em) + await this.performUpdates((store) => cb(store, 0, finalizedEnd), em) } else { finalizedEnd = 0 } @@ -242,7 +239,7 @@ export class TypeormDatabase { let b = info.newBlocks[i] await this.insertHotBlock(em, b) await this.performUpdates( - store => cb(store, i, i + 1), + (store) => cb(store, i, i + 1), em, new ChangeWriter(em, this.statusSchema, b.height) ) @@ -295,15 +292,14 @@ export class TypeormDatabase { state: this.getStateManager(), logger: this.getLogger(), changes: changeWriter, - cacheMode: this.cacheMode, - flushMode: this.flushMode, - resetMode: this.resetMode, + batchWriteOperations: this.batchWriteOperations, + syncOnGet: this.syncOnGet, + cacheEntitiesByDefault: this.cacheEntitiesByDefault, }) try { await cb(store) - await store.flush() - if (this.resetMode === ResetMode.BATCH) store.reset() + await store.sync(this.resetOnCommit) } finally { store['isClosed'] = true } diff --git a/typeorm/typeorm-store/src/store.ts b/typeorm/typeorm-store/src/store.ts index a52167c86..f986c89b8 100644 --- a/typeorm/typeorm-store/src/store.ts +++ b/typeorm/typeorm-store/src/store.ts @@ -9,7 +9,7 @@ import { import {EntityTarget} from 'typeorm/common/EntityTarget' import {ChangeWriter} from './utils/changeWriter' import {StateManager} from './utils/stateManager' -import {createLogger, Logger} from '@subsquid/logger' +import {Logger} from '@subsquid/logger' import {createFuture, Future} from '@subsquid/util-internal' import {EntityLiteral, noNull, splitIntoBatches, traverseEntity} from './utils/misc' import {ColumnMetadata} from 'typeorm/metadata/ColumnMetadata' @@ -17,61 +17,6 @@ import assert from 'assert' export {EntityTarget} -export const enum FlushMode { - - /** - * Send queries to the database transaction at every - * direct database read (all read methods besides - * .get()) and at the end of the batch. - */ - AUTO, - - /** - * Send queries to the database transaction strictly - * at the end of the batch. - */ - BATCH, - - /** - * Send queries to the database transaction whenever - * the data is read or written (including .get(), - * .insert(), .upsert(), .delete()) - */ - ALWAYS -} - -export const enum ResetMode { - - /** - * Clear cache at the end of each batch or manually. - */ - BATCH, - - /** - * Clear cache only manually. - */ - MANUAL, - - /** - * Clear cache whenever any queries are sent to the - * database transaction. - */ - FLUSH -} - -export const enum CacheMode { - - /** - * Data from all database reads is cached. - */ - ALL, - - /** - * Only the data from flagged database reads is cached. - */ - MANUAL -} - export interface GetOptions { id: string relations?: FindOptionsRelations @@ -121,9 +66,9 @@ export interface StoreOptions { state: StateManager changes?: ChangeWriter logger?: Logger - flushMode: FlushMode - resetMode: ResetMode - cacheMode: CacheMode + batchWriteOperations: boolean + cacheEntitiesByDefault: boolean + syncOnGet: boolean } /** @@ -135,21 +80,21 @@ export class Store { protected changes?: ChangeWriter protected logger?: Logger - protected flushMode: FlushMode - protected resetMode: ResetMode - protected cacheMode: CacheMode + protected batchWriteOperations: boolean + protected cacheEntitiesByDefault: boolean + protected syncOnGet: boolean protected pendingCommit?: Future protected isClosed = false - constructor({em, changes, logger, state, flushMode, resetMode, cacheMode}: StoreOptions) { + constructor({em, changes, logger, state, ...opts}: StoreOptions) { this.em = em this.changes = changes this.logger = logger?.child('store') this.state = state - this.flushMode = flushMode - this.resetMode = resetMode - this.cacheMode = cacheMode + this.batchWriteOperations = opts.batchWriteOperations + this.cacheEntitiesByDefault = opts.cacheEntitiesByDefault + this.syncOnGet = opts.syncOnGet } get _em() { @@ -197,10 +142,10 @@ export class Store { this.logger?.debug(`upsert ${entities.length} ${metadata.name} entities`) await this.changes?.writeUpsert(metadata, entities) - let fk = metadata.columns.filter(c => c.relationMetadata) + let fk = metadata.columns.filter((c) => c.relationMetadata) if (fk.length == 0) return this.upsertMany(metadata.target, entities) let signatures = entities - .map(e => ({entity: e, value: this.getFkSignature(fk, e)})) + .map((e) => ({entity: e, value: this.getFkSignature(fk, e)})) .sort((a, b) => (a.value > b.value ? -1 : b.value > a.value ? 1 : 0)) let currentSignature = signatures[0].value let batch: EntityLiteral[] = [] @@ -306,13 +251,15 @@ export class Store { async find(target: EntityTarget, options: FindManyOptions): Promise { return await this.performRead(async () => { const {cache, ...opts} = options + const res = await this.em.find(target, opts) - if (cache ?? this.cacheMode === CacheMode.ALL) { + if (cache ?? this.cacheEntitiesByDefault) { const metadata = this.getEntityMetadata(target) for (const e of res) { this.cacheEntity(metadata, e) } } + return res }) } @@ -331,12 +278,14 @@ export class Store { ): Promise { return await this.performRead(async () => { const {cache, ...opts} = options + const res = await this.em.findOne(target, opts).then(noNull) - if (cache ?? this.cacheMode === CacheMode.ALL) { + if (cache ?? this.cacheEntitiesByDefault) { const metadata = this.getEntityMetadata(target) const idOrEntity = res || getIdFromWhere(options.where) this.cacheEntity(metadata, idOrEntity) } + return res }) } @@ -352,6 +301,7 @@ export class Store { async findOneOrFail(target: EntityTarget, options: FindOneOptions): Promise { const res = await this.findOne(target, options) if (res == null) throw new EntityNotFoundError(target, options.where) + return res } @@ -362,6 +312,7 @@ export class Store { ): Promise { const res = await this.findOneBy(target, where, cache) if (res == null) throw new EntityNotFoundError(target, where) + return res } @@ -373,8 +324,10 @@ export class Store { ): Promise { const {id, relations} = parseGetOptions(idOrOptions) const metadata = this.getEntityMetadata(target) + let entity = this.state.get(metadata, id, relations) - if (entity !== undefined) return noNull(entity) + if (entity !== undefined || !this.syncOnGet) return noNull(entity) + return await this.findOne(target, {where: {id} as any, relations, cache: true}) } @@ -382,8 +335,10 @@ export class Store { async getOrFail(target: EntityTarget, options: GetOptions): Promise async getOrFail(target: EntityTarget, idOrOptions: string | GetOptions): Promise { const options = parseGetOptions(idOrOptions) + let e = await this.get(target, options) if (e == null) throw new EntityNotFoundError(target, options.id) + return e } @@ -391,7 +346,7 @@ export class Store { this.state.reset() } - async flush(reset?: boolean): Promise { + async sync(reset?: boolean): Promise { await this.pendingCommit?.promise() this.pendingCommit = createFuture() @@ -414,7 +369,7 @@ export class Store { } }) - if (reset ?? this.resetMode === ResetMode.FLUSH) { + if (reset) { this.reset() } } finally { @@ -425,9 +380,7 @@ export class Store { private async performRead(cb: () => Promise): Promise { this.assertNotClosed() - if (this.flushMode === FlushMode.AUTO || this.flushMode === FlushMode.ALWAYS) { - await this.flush() - } + await this.sync() return await cb() } @@ -435,8 +388,8 @@ export class Store { this.assertNotClosed() await this.pendingCommit?.promise() await cb() - if (this.flushMode === FlushMode.ALWAYS) { - await this.flush() + if (!this.batchWriteOperations) { + await this.sync() } } diff --git a/typeorm/typeorm-store/src/test/store.test.ts b/typeorm/typeorm-store/src/test/store.test.ts index 5841d4cfb..99eac4689 100644 --- a/typeorm/typeorm-store/src/test/store.test.ts +++ b/typeorm/typeorm-store/src/test/store.test.ts @@ -1,7 +1,7 @@ import {assertNotNull} from '@subsquid/util-internal' import expect from 'expect' import {Equal} from 'typeorm' -import {Store, CacheMode, FlushMode, ResetMode} from '../store' +import {Store} from '../store' import {Item, Order} from './lib/model' import {getEntityManager, useDatabase} from './util' import {sortMetadatasInCommitOrder} from '../utils/commitOrder' @@ -161,9 +161,9 @@ export async function createStore(): Promise { state: new StateManager({ commitOrder: sortMetadatasInCommitOrder(em.connection), }), - cacheMode: CacheMode.ALL, - flushMode: FlushMode.AUTO, - resetMode: ResetMode.BATCH, + batchWriteOperations: true, + cacheEntitiesByDefault: true, + syncOnGet: true, }) } From b990aab92e3bc3ed42bdcde1bc30925262aebf89 Mon Sep 17 00:00:00 2001 From: belopash Date: Tue, 18 Jun 2024 15:28:34 +0500 Subject: [PATCH 11/11] cache -> cacheEntities --- typeorm/typeorm-store/src/store.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/typeorm/typeorm-store/src/store.ts b/typeorm/typeorm-store/src/store.ts index f986c89b8..727b681f7 100644 --- a/typeorm/typeorm-store/src/store.ts +++ b/typeorm/typeorm-store/src/store.ts @@ -45,7 +45,7 @@ export interface FindOneOptions { */ order?: FindOptionsOrder - cache?: boolean + cacheEntities?: boolean } export interface FindManyOptions extends FindOneOptions { @@ -277,10 +277,10 @@ export class Store { options: FindOneOptions ): Promise { return await this.performRead(async () => { - const {cache, ...opts} = options + const {cacheEntities, ...opts} = options const res = await this.em.findOne(target, opts).then(noNull) - if (cache ?? this.cacheEntitiesByDefault) { + if (cacheEntities ?? this.cacheEntitiesByDefault) { const metadata = this.getEntityMetadata(target) const idOrEntity = res || getIdFromWhere(options.where) this.cacheEntity(metadata, idOrEntity) @@ -293,9 +293,9 @@ export class Store { async findOneBy( target: EntityTarget, where: FindOptionsWhere | FindOptionsWhere[], - cache?: boolean + cacheEntities?: boolean ): Promise { - return await this.findOne(target, {where, cache}) + return await this.findOne(target, {where, cacheEntities}) } async findOneOrFail(target: EntityTarget, options: FindOneOptions): Promise { @@ -328,7 +328,7 @@ export class Store { let entity = this.state.get(metadata, id, relations) if (entity !== undefined || !this.syncOnGet) return noNull(entity) - return await this.findOne(target, {where: {id} as any, relations, cache: true}) + return await this.findOne(target, {where: {id} as any, relations, cacheEntities: true}) } async getOrFail(target: EntityTarget, id: string): Promise