From d85eb73ca5adc96f995e7628cc127e32aa27b2c0 Mon Sep 17 00:00:00 2001 From: Derek Guenther Date: Sat, 10 Aug 2024 16:17:57 -0400 Subject: [PATCH] Update account head and add transactions in a single DB transaction --- packages/mobile-app/data/wallet/db.ts | 347 ++++++++++-------- packages/mobile-app/data/wallet/wallet.ts | 28 +- packages/mobile-app/data/wallet/writeCache.ts | 96 ++--- 3 files changed, 264 insertions(+), 207 deletions(-) diff --git a/packages/mobile-app/data/wallet/db.ts b/packages/mobile-app/data/wallet/db.ts index 1447e6f..6ba9d3f 100644 --- a/packages/mobile-app/data/wallet/db.ts +++ b/packages/mobile-app/data/wallet/db.ts @@ -181,6 +181,10 @@ export class WalletDb { col.notNull(), ) .addColumn("hash", SQLiteType.Blob, (col) => col.notNull()) + .addUniqueConstraint("accountnetworkheads_accountId_network", [ + "accountId", + "network", + ]) .execute(); console.log("created accountNetworkHeads"); }, @@ -419,7 +423,13 @@ export class WalletDb { } const result = await this.db.transaction().execute(async (db) => { - db.deleteFrom("accountNetworkHeads") + await db + .deleteFrom("transactionBalanceDeltas") + .where("accountId", "=", account.id) + .executeTakeFirstOrThrow(); + + await db + .deleteFrom("accountNetworkHeads") .where("accountId", "=", account.id) .executeTakeFirstOrThrow(); @@ -520,21 +530,48 @@ export class WalletDb { hash: hash, sequence: sequence, }) + .onConflict((oc) => + oc.columns(["accountId", "network"]).doUpdateSet({ + hash: hash, + sequence: sequence, + }), + ) .executeTakeFirst(); } } - async saveTransaction(values: { - hash: Uint8Array; + async saveBlock(values: { accountId: number; network: Network; - blockSequence: number | null; - blockHash: Uint8Array | null; - ownerNotes: { position: number | null; note: Note; nullifier: string }[]; - foundNullifiers: Uint8Array[]; - spenderNotes: { note: Note }[]; - timestamp: Date; + blockSequence: number; + blockHash: Uint8Array; + transactions: { + hash: Uint8Array; + ownerNotes: { position: number | null; note: Note; nullifier: string }[]; + foundNullifiers: Uint8Array[]; + spenderNotes: { note: Note }[]; + timestamp: Date; + }[]; }) { + if (values.transactions.length === 0) { + await this.db + .insertInto("accountNetworkHeads") + .values({ + accountId: values.accountId, + network: values.network, + hash: values.blockHash, + sequence: values.blockSequence, + }) + .onConflict((oc) => + oc.columns(["accountId", "network"]).doUpdateSet({ + hash: values.blockHash, + sequence: values.blockSequence, + }), + ) + .executeTakeFirstOrThrow(); + return; + } + const address = ( await this.db .selectFrom("accounts") @@ -546,168 +583,186 @@ export class WalletDb { throw new Error(`Account with ID ${values.accountId} not found`); } - const balanceDeltas = new BalanceDeltas(); - - // Note that we can't rely on spenderNotes alone for subtracting from balance deltas - // because they don't include transaction fees. - if (values.foundNullifiers.length > 0) { - const spentNotes = await this.db - .selectFrom("notes") - .innerJoin("nullifiers", "nullifiers.noteId", "notes.id") - .selectAll() - .where((eb) => - eb.and([ - eb("nullifiers.nullifier", "in", values.foundNullifiers), - eb("notes.accountId", "=", values.accountId), - ]), - ) - .execute(); - - if (spentNotes.length !== values.foundNullifiers.length) { - console.error("Some nullifiers were not found in the database"); - } - - for (const note of spentNotes) { - balanceDeltas.subtract(note.assetId, BigInt(note.value)); - } - } - for (const note of values.ownerNotes) { - balanceDeltas.add(note.note.assetId(), note.note.value()); - } - - // Intended to match the logic in ironfish sdk's getTransactionType - const allNotes = [...values.ownerNotes, ...values.spenderNotes]; - const transactionType = - values.foundNullifiers.length !== 0 || - allNotes[0].note.sender() === address - ? TransactionType.SEND - : TransactionType.RECEIVE; - await this.db.transaction().execute(async (db) => { - // One transaction could apply to multiple accounts await db - .insertInto("transactions") + .insertInto("accountNetworkHeads") .values({ - hash: values.hash, + accountId: values.accountId, network: values.network, - blockSequence: values.blockSequence, - blockHash: values.blockHash, - timestamp: values.timestamp, + hash: values.blockHash, + sequence: values.blockSequence, }) - .onConflict((oc) => oc.column("hash").doNothing()) + .onConflict((oc) => + oc.columns(["accountId", "network"]).doUpdateSet({ + hash: values.blockHash, + sequence: values.blockSequence, + }), + ) .executeTakeFirstOrThrow(); - await db - .insertInto("accountTransactions") - .values({ - accountId: values.accountId, - transactionHash: values.hash, - type: transactionType, - }) - .executeTakeFirst(); + for (const txn of values.transactions) { + // Intended to match the logic in ironfish sdk's getTransactionType + const allNotes = [...txn.ownerNotes, ...txn.spenderNotes]; + const transactionType = + txn.foundNullifiers.length !== 0 || + allNotes[0].note.sender() === address + ? TransactionType.SEND + : TransactionType.RECEIVE; + + const balanceDeltas = new BalanceDeltas(); + + // Note that we can't rely on spenderNotes alone for subtracting from balance deltas + // because they don't include transaction fees. + if (txn.foundNullifiers.length > 0) { + const spentNotes = await db + .selectFrom("notes") + .innerJoin("nullifiers", "nullifiers.noteId", "notes.id") + .selectAll() + .where((eb) => + eb.and([ + eb("nullifiers.nullifier", "in", txn.foundNullifiers), + eb("notes.accountId", "=", values.accountId), + ]), + ) + .execute(); - for (const note of values.ownerNotes) { - const result = await db - .insertInto("notes") - .values({ - accountId: values.accountId, - network: values.network, - transactionHash: values.hash, - note: new Uint8Array(note.note.serialize()), - assetId: new Uint8Array(note.note.assetId()), - owner: Uint8ArrayUtils.fromHex(note.note.owner()), - sender: Uint8ArrayUtils.fromHex(note.note.sender()), - value: note.note.value().toString(), - memo: new Uint8Array(note.note.memo()), - position: note.position, - }) - .executeTakeFirst(); + if (spentNotes.length !== txn.foundNullifiers.length) { + console.error("Some nullifiers were not found in the database"); + } - if (!result.insertId) { - throw new Error(); + for (const note of spentNotes) { + balanceDeltas.subtract(note.assetId, BigInt(note.value)); + } + } + for (const note of txn.ownerNotes) { + balanceDeltas.add(note.note.assetId(), note.note.value()); } + // One transaction could apply to multiple accounts await db - .insertInto("nullifiers") + .insertInto("transactions") .values({ - noteId: Number(result.insertId), - nullifier: Uint8ArrayUtils.fromHex(note.nullifier), - accountId: values.accountId, + hash: txn.hash, network: values.network, - transactionHash: null, + blockSequence: values.blockSequence, + blockHash: values.blockHash, + timestamp: txn.timestamp, }) - .executeTakeFirst(); - } - - for (const nullifier of values.foundNullifiers) { - await db - .updateTable("nullifiers") - .set("transactionHash", values.hash) - .where("nullifier", "=", nullifier) + .onConflict((oc) => oc.column("hash").doNothing()) .executeTakeFirstOrThrow(); - } - for (const note of values.spenderNotes) { await db - .insertInto("notes") + .insertInto("accountTransactions") .values({ accountId: values.accountId, - network: values.network, - transactionHash: values.hash, - note: new Uint8Array(note.note.serialize()), - assetId: new Uint8Array(note.note.assetId()), - owner: Uint8ArrayUtils.fromHex(note.note.owner()), - sender: Uint8ArrayUtils.fromHex(note.note.sender()), - value: note.note.value().toString(), - memo: new Uint8Array(note.note.memo()), - position: null, + transactionHash: txn.hash, + type: transactionType, }) .executeTakeFirstOrThrow(); - } - for (const delta of balanceDeltas) { - // This could be done with the SQLite decimal extension, but it's not available - // in the Expo SQLite driver. - const existingBalance = BigInt( - ( - await db - .selectFrom("balances") - .select("value") - .where((eb) => - eb.and([ - eb("accountId", "=", values.accountId), - eb("network", "=", values.network), - eb("assetId", "=", Uint8ArrayUtils.fromHex(delta[0])), - ]), - ) - .executeTakeFirst() - )?.value ?? "0", - ); + for (const note of txn.ownerNotes) { + const result = await db + .insertInto("notes") + .values({ + accountId: values.accountId, + network: values.network, + transactionHash: txn.hash, + note: new Uint8Array(note.note.serialize()), + assetId: new Uint8Array(note.note.assetId()), + owner: Uint8ArrayUtils.fromHex(note.note.owner()), + sender: Uint8ArrayUtils.fromHex(note.note.sender()), + value: note.note.value().toString(), + memo: new Uint8Array(note.note.memo()), + position: note.position, + }) + .executeTakeFirst(); + + if (!result.insertId) { + throw new Error(); + } + + await db + .insertInto("nullifiers") + .values({ + noteId: Number(result.insertId), + nullifier: Uint8ArrayUtils.fromHex(note.nullifier), + accountId: values.accountId, + network: values.network, + transactionHash: null, + }) + .executeTakeFirst(); + } - await db - .insertInto("balances") - .values({ - accountId: values.accountId, - network: values.network, - assetId: Uint8ArrayUtils.fromHex(delta[0]), - value: (existingBalance + delta[1]).toString(), - }) - .onConflict((oc) => - oc - .columns(["accountId", "network", "assetId"]) - .doUpdateSet({ value: (existingBalance + delta[1]).toString() }), - ) - .executeTakeFirstOrThrow(); + for (const nullifier of txn.foundNullifiers) { + await db + .updateTable("nullifiers") + .set("transactionHash", txn.hash) + .where("nullifier", "=", nullifier) + .executeTakeFirstOrThrow(); + } - await db - .insertInto("transactionBalanceDeltas") - .values({ - accountId: values.accountId, - assetId: Uint8ArrayUtils.fromHex(delta[0]), - transactionHash: values.hash, - value: delta[1].toString(), - }) - .executeTakeFirstOrThrow(); + for (const note of txn.spenderNotes) { + await db + .insertInto("notes") + .values({ + accountId: values.accountId, + network: values.network, + transactionHash: txn.hash, + note: new Uint8Array(note.note.serialize()), + assetId: new Uint8Array(note.note.assetId()), + owner: Uint8ArrayUtils.fromHex(note.note.owner()), + sender: Uint8ArrayUtils.fromHex(note.note.sender()), + value: note.note.value().toString(), + memo: new Uint8Array(note.note.memo()), + position: null, + }) + .executeTakeFirstOrThrow(); + } + + for (const delta of balanceDeltas) { + // This could be done with the SQLite decimal extension, but it's not available + // in the Expo SQLite driver. + const existingBalance = BigInt( + ( + await db + .selectFrom("balances") + .select("value") + .where((eb) => + eb.and([ + eb("accountId", "=", values.accountId), + eb("network", "=", values.network), + eb("assetId", "=", Uint8ArrayUtils.fromHex(delta[0])), + ]), + ) + .executeTakeFirst() + )?.value ?? "0", + ); + + await db + .insertInto("balances") + .values({ + accountId: values.accountId, + network: values.network, + assetId: Uint8ArrayUtils.fromHex(delta[0]), + value: (existingBalance + delta[1]).toString(), + }) + .onConflict((oc) => + oc.columns(["accountId", "network", "assetId"]).doUpdateSet({ + value: (existingBalance + delta[1]).toString(), + }), + ) + .executeTakeFirstOrThrow(); + + await db + .insertInto("transactionBalanceDeltas") + .values({ + accountId: values.accountId, + assetId: Uint8ArrayUtils.fromHex(delta[0]), + transactionHash: txn.hash, + value: delta[1].toString(), + }) + .executeTakeFirstOrThrow(); + } } }); } diff --git a/packages/mobile-app/data/wallet/wallet.ts b/packages/mobile-app/data/wallet/wallet.ts index d982b1d..e41926c 100644 --- a/packages/mobile-app/data/wallet/wallet.ts +++ b/packages/mobile-app/data/wallet/wallet.ts @@ -541,23 +541,17 @@ class Wallet { } } - for (const transaction of transactionMap.values()) { - cache.pushTransaction( - account.id, - block.hash, - block.sequence, - new Date(block.timestamp), - transaction.transaction, - transaction.ownerNotes, - transaction.spenderNotes, - transaction.foundNullifiers, - ); - } - - cache.setHead(account.id, { - hash: block.hash, - sequence: block.sequence, - }); + cache.writeBlock( + account.id, + { hash: block.hash, sequence: block.sequence }, + [...transactionMap.values()].map((txn) => ({ + hash: txn.transaction.hash, + timestamp: new Date(block.timestamp), + ownerNotes: txn.ownerNotes, + spenderNotes: txn.spenderNotes, + foundNullifiers: txn.foundNullifiers, + })), + ); } }); }, diff --git a/packages/mobile-app/data/wallet/writeCache.ts b/packages/mobile-app/data/wallet/writeCache.ts index 585de66..7601ff1 100644 --- a/packages/mobile-app/data/wallet/writeCache.ts +++ b/packages/mobile-app/data/wallet/writeCache.ts @@ -1,5 +1,4 @@ import { Note } from "@ironfish/sdk"; -import { LightTransaction } from "../api/lightstreamer"; import { Network } from "../constants"; import { WalletDb } from "./db"; import * as UInt8ArrayUtils from "../../utils/uint8Array"; @@ -13,15 +12,17 @@ export class WriteCache { readonly heads: Map = new Map(); - readonly transactions: { + private writeQueue: { accountId: number; - hash: Uint8Array; sequence: number; - timestamp: Date; - transaction: LightTransaction; - ownerNotes: { position: number; note: Note; nullifier: string }[]; - spenderNotes: { note: Note }[]; - foundNullifiers: Uint8Array[]; + hash: Uint8Array; + transactions: { + hash: Uint8Array; + timestamp: Date; + ownerNotes: { position: number; note: Note; nullifier: string }[]; + spenderNotes: { note: Note }[]; + foundNullifiers: Uint8Array[]; + }[]; }[] = []; readonly nullifierSet = new Set(); @@ -39,28 +40,38 @@ export class WriteCache { this.heads.set(id, head); } - pushTransaction( + writeBlock( accountId: number, - hash: Uint8Array, - sequence: number, - timestamp: Date, - transaction: LightTransaction, - ownerNotes: { position: number; note: Note; nullifier: string }[], - spenderNotes: { note: Note }[], - foundNullifiers: Uint8Array[], + block: { hash: Uint8Array; sequence: number }, + transactions: { + hash: Uint8Array; + timestamp: Date; + ownerNotes: { position: number; note: Note; nullifier: string }[]; + spenderNotes: { note: Note }[]; + foundNullifiers: Uint8Array[]; + }[], ) { - this.transactions.push({ + this.heads.set(accountId, block); + + const existing = this.writeQueue.findLast((w) => w.accountId === accountId); + if ( + transactions.length === 0 && + existing && + existing.transactions.length === 0 + ) { + existing.hash = block.hash; + existing.sequence = block.sequence; + return; + } + + this.writeQueue.push({ accountId, - hash, - sequence, - transaction, - timestamp, - ownerNotes, - spenderNotes, - foundNullifiers, + sequence: block.sequence, + hash: block.hash, + transactions, }); - for (const n of ownerNotes) { + for (const n of transactions.flatMap((t) => t.ownerNotes)) { this.nullifierSet.add(n.nullifier); } } @@ -69,29 +80,26 @@ export class WriteCache { * Writes the cache to the database. */ async write() { - for (const [k, v] of this.heads) { - console.log(`updating account ID ${k} head to ${v.sequence}`); - await this.db.updateAccountHead(k, this.network, v.sequence, v.hash); - } + if (!this.writeQueue.length) return; + + // Reset the write queue so new cached writes aren't added while + // we're writing to the database. + // TODO: Handle errors if the write fails. The account head should probably be frozen + // until the write is successful. + const writeQueue = this.writeQueue; + this.writeQueue = []; - let txn; - while ((txn = this.transactions.shift())) { - console.log( - `saving transaction ${UInt8ArrayUtils.toHex(txn.transaction.hash)} for account ID ${txn.accountId}`, - ); - await this.db.saveTransaction({ - hash: txn.transaction.hash, - accountId: txn.accountId, - blockHash: txn.hash, - blockSequence: txn.sequence, - timestamp: txn.timestamp, + let w; + while ((w = writeQueue.shift())) { + await this.db.saveBlock({ + accountId: w.accountId, network: this.network, - ownerNotes: txn.ownerNotes, - foundNullifiers: txn.foundNullifiers, - spenderNotes: txn.spenderNotes, + blockHash: w.hash, + blockSequence: w.sequence, + transactions: w.transactions, }); - for (const n of txn.ownerNotes) { + for (const n of w.transactions.flatMap((t) => t.ownerNotes)) { this.nullifierSet.delete(n.nullifier); } }