From 3c17ee0831e07dcef3a19387dc17ee59638e871a Mon Sep 17 00:00:00 2001 From: "elessar.eth" Date: Thu, 26 Sep 2024 11:39:57 +0200 Subject: [PATCH] feat(db): add tx index to `BlobsOnTransactions` model (#584) * chore: add changeset * chore: merge changesets * chore(db): refactor database generator * feat(db): add tx index to `BlobsOnTransactions` model * feat: make tx index mandatory * perf(api): add block timestamp and transaction index as ordering clauses when fetching blobs * chore: add changeset * fix(api): return tx index when fetching a single blob * perf(db): improve tx index backfilling migration --- .changeset/sixty-pillows-melt.md | 5 + .changeset/yellow-peas-search.md | 5 + .../api/src/routers/blob/common/selects.ts | 2 + .../src/routers/blob/common/serializers.ts | 6 + packages/api/src/routers/blob/getAll.ts | 8 +- .../src/routers/indexer/indexData.utils.ts | 26 +- .../api/test/__snapshots__/blob.test.ts.snap | 122 +++++ packages/api/test/indexer.test.ts | 6 + packages/db/prisma/extensions/base.ts | 6 +- packages/db/prisma/index.ts | 2 +- .../migration.sql | 2 + .../migration.sql | 32 ++ .../migration.sql | 8 + .../migration.sql | 11 + packages/db/prisma/schema.prisma | 5 +- packages/db/prisma/seed/DataGenerator.ts | 516 +++++++++++------- packages/db/prisma/seed/main.ts | 289 +++++----- packages/db/prisma/seed/params.ts | 65 ++- packages/db/prisma/seed/utils.ts | 48 +- packages/db/prisma/seed/web3.ts | 99 ++++ .../db/test/extensions/base-extension.test.ts | 2 + .../stats-extension.test.fixtures.ts | 4 + .../test/DailyStatsSyncer.test.fixtures.ts | 6 + packages/test/src/fixtures/postgres/data.json | 33 ++ 24 files changed, 909 insertions(+), 399 deletions(-) create mode 100644 .changeset/sixty-pillows-melt.md create mode 100644 .changeset/yellow-peas-search.md create mode 100644 packages/db/prisma/migrations/20240925230913_add_tx_index_to_blobs_on_transactions_model/migration.sql create mode 100644 packages/db/prisma/migrations/20240925232017_backfill_tx_index_column/migration.sql create mode 100644 packages/db/prisma/migrations/20240925232521_make_blobs_on_transactions_tx_index_mandatory/migration.sql create mode 100644 packages/db/prisma/migrations/20240926003524_include_tx_index_on_blobs_on_transactions_model_indexes/migration.sql create mode 100644 packages/db/prisma/seed/web3.ts diff --git a/.changeset/sixty-pillows-melt.md b/.changeset/sixty-pillows-melt.md new file mode 100644 index 000000000..b8a2a3269 --- /dev/null +++ b/.changeset/sixty-pillows-melt.md @@ -0,0 +1,5 @@ +--- +"@blobscan/db": minor +--- + +Added tx index to `BlobsOnTransactions` model diff --git a/.changeset/yellow-peas-search.md b/.changeset/yellow-peas-search.md new file mode 100644 index 000000000..87b7ab8d4 --- /dev/null +++ b/.changeset/yellow-peas-search.md @@ -0,0 +1,5 @@ +--- +"@blobscan/api": patch +--- + +Enhanced blob fetching performance by adding sorting based on block timestamps and transaction index diff --git a/packages/api/src/routers/blob/common/selects.ts b/packages/api/src/routers/blob/common/selects.ts index 523c5b360..a62becbb2 100644 --- a/packages/api/src/routers/blob/common/selects.ts +++ b/packages/api/src/routers/blob/common/selects.ts @@ -23,6 +23,7 @@ export function createBlobSelect(expands: Expands) { blockTimestamp: true, index: true, txHash: true, + txIndex: true, transaction: { select: { rollup: true, @@ -48,6 +49,7 @@ export function createBlobsOnTransactionsSelect(expands: Expands) { blockNumber: true, blockTimestamp: true, txHash: true, + txIndex: true, blob: { select: baseBlobSelect, }, diff --git a/packages/api/src/routers/blob/common/serializers.ts b/packages/api/src/routers/blob/common/serializers.ts index bd56f536b..f42960af9 100644 --- a/packages/api/src/routers/blob/common/serializers.ts +++ b/packages/api/src/routers/blob/common/serializers.ts @@ -67,6 +67,7 @@ export const serializedBlobOnTransactionSchema = serializedBaseBlobSchema.merge( z.object({ index: blobIndexSchema, txHash: z.string(), + txIndex: z.number().nonnegative(), blockHash: z.string(), blockNumber: blockNumberSchema, blockTimestamp: z.string(), @@ -86,6 +87,7 @@ export const serializedBlobSchema = serializedBaseBlobSchema.merge( z .object({ hash: z.string(), + txIndex: z.number().nonnegative(), index: blobIndexSchema, blockHash: z.string(), blockNumber: z.number().nonnegative(), @@ -127,6 +129,7 @@ export function serializeBlobOnTransaction( blockTimestamp, index, txHash, + txIndex, block, transaction, } = blobOnTransaction; @@ -136,6 +139,7 @@ export function serializeBlobOnTransaction( blockNumber, blockTimestamp: serializeDate(blockTimestamp), txHash, + txIndex, index, }; @@ -167,6 +171,7 @@ export function serializeBlob(blob: QueriedBlob): SerializedBlob { blockHash, blockNumber, blockTimestamp, + txIndex, index, txHash, block, @@ -174,6 +179,7 @@ export function serializeBlob(blob: QueriedBlob): SerializedBlob { }) => { return { index, + txIndex, hash: txHash, blockHash, blockNumber, diff --git a/packages/api/src/routers/blob/getAll.ts b/packages/api/src/routers/blob/getAll.ts index 4637dd6a4..4aecdac42 100644 --- a/packages/api/src/routers/blob/getAll.ts +++ b/packages/api/src/routers/blob/getAll.ts @@ -70,12 +70,8 @@ export const getAll = publicProcedure : undefined, }, orderBy: [ - { blockNumber: filters.sort }, - { - transaction: { - index: filters.sort, - }, - }, + { blockTimestamp: filters.sort }, + { txIndex: filters.sort }, { index: filters.sort, }, diff --git a/packages/api/src/routers/indexer/indexData.utils.ts b/packages/api/src/routers/indexer/indexData.utils.ts index 6584de110..dcd58be51 100644 --- a/packages/api/src/routers/indexer/indexData.utils.ts +++ b/packages/api/src/routers/indexer/indexData.utils.ts @@ -171,14 +171,24 @@ export function createDBBlobs({ export function createDBBlobsOnTransactions({ block, + transactions, blobs, }: IndexDataFormattedInput): BlobsOnTransactions[] { - return blobs.map(({ versionedHash, txHash, index }) => ({ - blobHash: versionedHash, - blockHash: block.hash, - blockNumber: block.number, - blockTimestamp: timestampToDate(block.timestamp), - txHash: txHash, - index, - })); + return blobs.map(({ versionedHash, txHash, index }) => { + const tx = transactions.find((t) => t.hash === txHash); + + if (!tx) { + throw new Error(`Transaction ${txHash} not found`); + } + + return { + blobHash: versionedHash, + blockHash: block.hash, + blockNumber: block.number, + blockTimestamp: timestampToDate(block.timestamp), + txHash: txHash, + txIndex: tx.index, + index, + }; + }); } diff --git a/packages/api/test/__snapshots__/blob.test.ts.snap b/packages/api/test/__snapshots__/blob.test.ts.snap index ae5696896..d8318ec8f 100644 --- a/packages/api/test/__snapshots__/blob.test.ts.snap +++ b/packages/api/test/__snapshots__/blob.test.ts.snap @@ -28,6 +28,7 @@ exports[`Blob router > getAll > when getting expanded blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash016", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -59,6 +60,7 @@ exports[`Blob router > getAll > when getting expanded blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, ] @@ -101,6 +103,7 @@ exports[`Blob router > getAll > when getting expanded blob results > should retu "to": "address4", }, "txHash": "txHash016", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -139,6 +142,7 @@ exports[`Blob router > getAll > when getting expanded blob results > should retu "to": "address4", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, ] @@ -174,6 +178,7 @@ exports[`Blob router > getAll > when getting expanded blob results > should retu "to": "address4", }, "txHash": "txHash016", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -205,6 +210,7 @@ exports[`Blob router > getAll > when getting expanded blob results > should retu "to": "address4", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, ] @@ -230,6 +236,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash006", }, { @@ -250,6 +257,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash005", }, { @@ -274,6 +282,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -298,6 +307,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash003", }, { @@ -318,6 +328,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash002", }, { @@ -342,6 +353,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -362,6 +374,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash003", + "txIndex": 2, "versionedHash": "blobHash002", }, { @@ -386,6 +399,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash003", + "txIndex": 2, "versionedHash": "blobHash001", }, { @@ -407,6 +421,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash002", + "txIndex": 1, "versionedHash": "blobHash001", }, { @@ -428,6 +443,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof003", "size": 1200, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash003", }, { @@ -445,6 +461,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof002", "size": 1100, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash002", }, { @@ -466,6 +483,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash001", }, ] @@ -492,6 +510,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash016", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -516,6 +535,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -540,6 +560,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, ] @@ -566,6 +587,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -583,6 +605,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof002", "size": 1100, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash002", }, { @@ -604,6 +627,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof003", "size": 1200, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash003", }, ] @@ -633,6 +657,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -657,6 +682,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -681,6 +707,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -705,6 +732,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash012", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -729,6 +757,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash012", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -753,6 +782,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash012", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -777,6 +807,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash007", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -797,6 +828,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash003", + "txIndex": 2, "versionedHash": "blobHash002", }, { @@ -821,6 +853,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash003", + "txIndex": 2, "versionedHash": "blobHash001", }, ] @@ -846,6 +879,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "0xd80214f2e7c7271114f372b6a8baaf39bcb364448788f6d8229d2a903edf9272", + "txIndex": 0, "versionedHash": "0x01d5cc28986f58db309e0fae63b60ade81a01667721e190ec142051240b5d436", }, ] @@ -872,6 +906,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "89cf91c4c8be6f2a390d4262425f79dffb74c174fb15a210182184543bf7394e5a7970a774ee8e0dabc315424c22df0f", "size": 1500, "txHash": "0x5be77167b05f39ea8950f11b0da2bdfec6e04055030068b051ac5a43aaf251e9", + "txIndex": 0, "versionedHash": "0x010001c79d78a76fb9b4bab3896ee3ea32f3e2607da7801eb1a92da39d6c1368", }, { @@ -893,6 +928,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash013", + "txIndex": 1, "versionedHash": "blobHash001", }, { @@ -917,6 +953,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash012", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -941,6 +978,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash012", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -965,6 +1003,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash012", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -985,6 +1024,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash011", + "txIndex": 4, "versionedHash": "blobHash005", }, { @@ -1006,6 +1046,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof003", "size": 1200, "txHash": "txHash010", + "txIndex": 3, "versionedHash": "blobHash003", }, { @@ -1027,6 +1068,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof003", "size": 1200, "txHash": "txHash009", + "txIndex": 2, "versionedHash": "blobHash003", }, { @@ -1047,6 +1089,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "arbitrum", }, "txHash": "txHash008", + "txIndex": 1, "versionedHash": "blobHash002", }, { @@ -1071,6 +1114,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash007", + "txIndex": 0, "versionedHash": "blobHash001", }, ] @@ -1096,6 +1140,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash011", + "txIndex": 4, "versionedHash": "blobHash005", }, { @@ -1117,6 +1162,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof003", "size": 1200, "txHash": "txHash010", + "txIndex": 3, "versionedHash": "blobHash003", }, { @@ -1138,6 +1184,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof003", "size": 1200, "txHash": "txHash009", + "txIndex": 2, "versionedHash": "blobHash003", }, { @@ -1158,6 +1205,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "arbitrum", }, "txHash": "txHash008", + "txIndex": 1, "versionedHash": "blobHash002", }, { @@ -1182,6 +1230,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash007", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -1203,6 +1252,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash006", + "txIndex": 1, "versionedHash": "blobHash001", }, { @@ -1224,6 +1274,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash006", + "txIndex": 1, "versionedHash": "blobHash001", }, { @@ -1245,6 +1296,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash006", + "txIndex": 1, "versionedHash": "blobHash001", }, { @@ -1266,6 +1318,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash005", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -1283,6 +1336,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof002", "size": 1100, "txHash": "txHash005", + "txIndex": 0, "versionedHash": "blobHash002", }, { @@ -1300,6 +1354,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof002", "size": 1100, "txHash": "txHash005", + "txIndex": 0, "versionedHash": "blobHash002", }, ] @@ -1329,6 +1384,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash007", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -1350,6 +1406,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof003", "size": 1200, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash003", }, { @@ -1367,6 +1424,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof002", "size": 1100, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash002", }, { @@ -1388,6 +1446,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash001", }, ] @@ -1417,6 +1476,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash007", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -1437,6 +1497,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash006", }, { @@ -1457,6 +1518,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash005", }, { @@ -1481,6 +1543,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -1505,6 +1568,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash003", }, { @@ -1525,6 +1589,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash002", }, { @@ -1549,6 +1614,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -1570,6 +1636,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof003", "size": 1200, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash003", }, { @@ -1587,6 +1654,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof002", "size": 1100, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash002", }, { @@ -1608,6 +1676,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash001", }, ] @@ -1637,6 +1706,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash007", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -1657,6 +1727,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash006", }, { @@ -1677,6 +1748,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash005", }, { @@ -1701,6 +1773,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -1725,6 +1798,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash003", }, { @@ -1745,6 +1819,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash002", }, { @@ -1769,6 +1844,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -1790,6 +1866,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof003", "size": 1200, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash003", }, { @@ -1807,6 +1884,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof002", "size": 1100, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash002", }, { @@ -1828,6 +1906,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash001", }, ] @@ -1853,6 +1932,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash006", }, { @@ -1873,6 +1953,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash005", }, { @@ -1897,6 +1978,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -1921,6 +2003,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash003", }, { @@ -1941,6 +2024,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash002", }, { @@ -1965,6 +2049,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "base", }, "txHash": "txHash004", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -1985,6 +2070,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash003", + "txIndex": 2, "versionedHash": "blobHash002", }, { @@ -2009,6 +2095,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash003", + "txIndex": 2, "versionedHash": "blobHash001", }, { @@ -2030,6 +2117,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash002", + "txIndex": 1, "versionedHash": "blobHash001", }, { @@ -2051,6 +2139,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof003", "size": 1200, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash003", }, { @@ -2068,6 +2157,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof002", "size": 1100, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash002", }, { @@ -2089,6 +2179,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash001", }, ] @@ -2114,6 +2205,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash003", + "txIndex": 2, "versionedHash": "blobHash002", }, { @@ -2138,6 +2230,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash003", + "txIndex": 2, "versionedHash": "blobHash001", }, { @@ -2159,6 +2252,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash002", + "txIndex": 1, "versionedHash": "blobHash001", }, { @@ -2180,6 +2274,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof003", "size": 1200, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash003", }, { @@ -2197,6 +2292,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof002", "size": 1100, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash002", }, { @@ -2218,6 +2314,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash001", + "txIndex": 0, "versionedHash": "blobHash001", }, ] @@ -2244,6 +2341,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash016", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -2268,6 +2366,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -2292,6 +2391,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -2316,6 +2416,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, ] @@ -2342,6 +2443,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash016", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -2366,6 +2468,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -2390,6 +2493,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -2414,6 +2518,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -2435,6 +2540,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "89cf91c4c8be6f2a390d4262425f79dffb74c174fb15a210182184543bf7394e5a7970a774ee8e0dabc315424c22df0f", "size": 1500, "txHash": "0x5be77167b05f39ea8950f11b0da2bdfec6e04055030068b051ac5a43aaf251e9", + "txIndex": 0, "versionedHash": "0x010001c79d78a76fb9b4bab3896ee3ea32f3e2607da7801eb1a92da39d6c1368", }, ] @@ -2461,6 +2567,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "proof": "proof001", "size": 1000, "txHash": "txHash016", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -2485,6 +2592,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -2509,6 +2617,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, { @@ -2533,6 +2642,7 @@ exports[`Blob router > getAll > when getting filtered blob results > should retu "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, ] @@ -2559,6 +2669,7 @@ exports[`Blob router > getAll > when getting paginated blob results > should def "proof": "proof001", "size": 1000, "txHash": "txHash016", + "txIndex": 0, "versionedHash": "blobHash001", }, { @@ -2583,6 +2694,7 @@ exports[`Blob router > getAll > when getting paginated blob results > should def "rollup": "optimism", }, "txHash": "txHash015", + "txIndex": 0, "versionedHash": "blobHash004", }, ] @@ -2609,6 +2721,7 @@ exports[`Blob router > getAll > when getting paginated blob results > should ret "proof": "89cf91c4c8be6f2a390d4262425f79dffb74c174fb15a210182184543bf7394e5a7970a774ee8e0dabc315424c22df0f", "size": 1500, "txHash": "0x5be77167b05f39ea8950f11b0da2bdfec6e04055030068b051ac5a43aaf251e9", + "txIndex": 0, "versionedHash": "0x010001c79d78a76fb9b4bab3896ee3ea32f3e2607da7801eb1a92da39d6c1368", }, { @@ -2630,6 +2743,7 @@ exports[`Blob router > getAll > when getting paginated blob results > should ret "proof": "proof001", "size": 1000, "txHash": "txHash013", + "txIndex": 1, "versionedHash": "blobHash001", }, ] @@ -2707,6 +2821,7 @@ exports[`Blob router > getByBlobId > should get a blob by kzg commitment 1`] = ` "hash": "txHash004", "index": 3, "rollup": "base", + "txIndex": 0, }, { "blockHash": "blockHash007", @@ -2716,6 +2831,7 @@ exports[`Blob router > getByBlobId > should get a blob by kzg commitment 1`] = ` "hash": "txHash015", "index": 0, "rollup": "optimism", + "txIndex": 0, }, { "blockHash": "blockHash007", @@ -2725,6 +2841,7 @@ exports[`Blob router > getByBlobId > should get a blob by kzg commitment 1`] = ` "hash": "txHash015", "index": 1, "rollup": "optimism", + "txIndex": 0, }, { "blockHash": "blockHash007", @@ -2734,6 +2851,7 @@ exports[`Blob router > getByBlobId > should get a blob by kzg commitment 1`] = ` "hash": "txHash015", "index": 2, "rollup": "optimism", + "txIndex": 0, }, ], "versionedHash": "blobHash004", @@ -2765,6 +2883,7 @@ exports[`Blob router > getByBlobId > should get a blob by versioned hash 1`] = ` "hash": "txHash004", "index": 3, "rollup": "base", + "txIndex": 0, }, { "blockHash": "blockHash007", @@ -2774,6 +2893,7 @@ exports[`Blob router > getByBlobId > should get a blob by versioned hash 1`] = ` "hash": "txHash015", "index": 0, "rollup": "optimism", + "txIndex": 0, }, { "blockHash": "blockHash007", @@ -2783,6 +2903,7 @@ exports[`Blob router > getByBlobId > should get a blob by versioned hash 1`] = ` "hash": "txHash015", "index": 1, "rollup": "optimism", + "txIndex": 0, }, { "blockHash": "blockHash007", @@ -2792,6 +2913,7 @@ exports[`Blob router > getByBlobId > should get a blob by versioned hash 1`] = ` "hash": "txHash015", "index": 2, "rollup": "optimism", + "txIndex": 0, }, ], "versionedHash": "blobHash004", diff --git a/packages/api/test/indexer.test.ts b/packages/api/test/indexer.test.ts index 4fe026616..72095d3f9 100644 --- a/packages/api/test/indexer.test.ts +++ b/packages/api/test/indexer.test.ts @@ -284,6 +284,7 @@ describe("Indexer router", async () => { "blockTimestamp": 2023-09-01T13:50:21.000Z, "index": 0, "txHash": "txHash1000", + "txIndex": 1, }, { "blobHash": "blobHash1001", @@ -292,6 +293,7 @@ describe("Indexer router", async () => { "blockTimestamp": 2023-09-01T13:50:21.000Z, "index": 1, "txHash": "txHash1000", + "txIndex": 1, }, { "blobHash": "blobHash999", @@ -300,6 +302,7 @@ describe("Indexer router", async () => { "blockTimestamp": 2023-09-01T13:50:21.000Z, "index": 0, "txHash": "txHash999", + "txIndex": 0, }, ] `); @@ -502,6 +505,7 @@ describe("Indexer router", async () => { "blockTimestamp": 2023-09-01T13:50:21.000Z, "index": 0, "txHash": "txHash1000", + "txIndex": 1, }, { "blobHash": "blobHash1001", @@ -510,6 +514,7 @@ describe("Indexer router", async () => { "blockTimestamp": 2023-09-01T13:50:21.000Z, "index": 1, "txHash": "txHash1000", + "txIndex": 1, }, { "blobHash": "blobHash999", @@ -518,6 +523,7 @@ describe("Indexer router", async () => { "blockTimestamp": 2023-09-01T13:50:21.000Z, "index": 0, "txHash": "txHash999", + "txIndex": 0, }, ] `); diff --git a/packages/db/prisma/extensions/base.ts b/packages/db/prisma/extensions/base.ts index be7e4d469..e8dd8a932 100644 --- a/packages/db/prisma/extensions/base.ts +++ b/packages/db/prisma/extensions/base.ts @@ -177,6 +177,7 @@ export const baseExtension = Prisma.defineExtension((prisma) => blockTimestamp, index, txHash, + txIndex, }) => Prisma.join([ blobHash, @@ -185,6 +186,7 @@ export const baseExtension = Prisma.defineExtension((prisma) => Prisma.sql`${blockTimestamp}::timestamp`, index, txHash, + txIndex, ]) ) .map((rowColumnsSql) => Prisma.sql`(${rowColumnsSql})`); @@ -196,12 +198,14 @@ export const baseExtension = Prisma.defineExtension((prisma) => block_number, block_timestamp, index, - tx_hash + tx_hash, + tx_index ) VALUES ${Prisma.join(sqlValues)} ON CONFLICT (tx_hash, index) DO UPDATE SET block_hash = EXCLUDED.block_hash, block_number = EXCLUDED.block_number, block_timestamp = EXCLUDED.block_timestamp, + tx_index = EXCLUDED.tx_index, index = EXCLUDED.index `; }, diff --git a/packages/db/prisma/index.ts b/packages/db/prisma/index.ts index 92a5356cc..ad0cb4877 100644 --- a/packages/db/prisma/index.ts +++ b/packages/db/prisma/index.ts @@ -28,7 +28,7 @@ if (!prisma_) { // eslint-disable-next-line turbo/no-undeclared-env-vars if (process.env.MODE !== "test") { p.$on("query", (e) => { - logger.debug(`${e.query}\nDuration=${e.duration}ms`); + logger.info(`${e.query}\nDuration=${e.duration}ms`); }); p.$on("error", (e) => { diff --git a/packages/db/prisma/migrations/20240925230913_add_tx_index_to_blobs_on_transactions_model/migration.sql b/packages/db/prisma/migrations/20240925230913_add_tx_index_to_blobs_on_transactions_model/migration.sql new file mode 100644 index 000000000..9b9cdfe78 --- /dev/null +++ b/packages/db/prisma/migrations/20240925230913_add_tx_index_to_blobs_on_transactions_model/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "blobs_on_transactions" ADD COLUMN "tx_index" INTEGER; diff --git a/packages/db/prisma/migrations/20240925232017_backfill_tx_index_column/migration.sql b/packages/db/prisma/migrations/20240925232017_backfill_tx_index_column/migration.sql new file mode 100644 index 000000000..3fbd1c869 --- /dev/null +++ b/packages/db/prisma/migrations/20240925232017_backfill_tx_index_column/migration.sql @@ -0,0 +1,32 @@ +DO $$ +DECLARE + batch_size INT := 10000; -- Define the size of each batch + update_count INT := 0; -- Variable to store the count of updated rows +BEGIN + -- Loop indefinitely until no rows are updated + LOOP + WITH candidate_rows AS ( + -- Select rows for update in batches of 1000, using NOWAIT to avoid locking conflicts + SELECT blobs_on_transactions.tx_hash as tx_hash, transaction.index as new_index + FROM blobs_on_transactions JOIN transaction ON blobs_on_transactions.tx_hash = transaction.hash + WHERE blobs_on_transactions.tx_index IS NULL + LIMIT batch_size + FOR UPDATE NOWAIT + ), update_rows AS ( + -- Perform the update for the selected rows + UPDATE blobs_on_transactions + SET tx_index = candidate_rows.new_index + FROM candidate_rows + WHERE candidate_rows.tx_hash = blobs_on_transactions.tx_hash + RETURNING blobs_on_transactions.tx_hash + ) + -- Count the number of updated rows in this batch + SELECT count(1) INTO update_count FROM update_rows; + + -- Exit the loop if no rows were updated in this batch + EXIT WHEN update_count = 0; + + -- Optional: Sleep briefly to reduce load on the database + PERFORM pg_sleep(0.1); + END LOOP; +END $$; \ No newline at end of file diff --git a/packages/db/prisma/migrations/20240925232521_make_blobs_on_transactions_tx_index_mandatory/migration.sql b/packages/db/prisma/migrations/20240925232521_make_blobs_on_transactions_tx_index_mandatory/migration.sql new file mode 100644 index 000000000..69b02f7be --- /dev/null +++ b/packages/db/prisma/migrations/20240925232521_make_blobs_on_transactions_tx_index_mandatory/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - Made the column `tx_index` on table `blobs_on_transactions` required. This step will fail if there are existing NULL values in that column. + +*/ +-- AlterTable +ALTER TABLE "blobs_on_transactions" ALTER COLUMN "tx_index" SET NOT NULL; diff --git a/packages/db/prisma/migrations/20240926003524_include_tx_index_on_blobs_on_transactions_model_indexes/migration.sql b/packages/db/prisma/migrations/20240926003524_include_tx_index_on_blobs_on_transactions_model_indexes/migration.sql new file mode 100644 index 000000000..50e51024e --- /dev/null +++ b/packages/db/prisma/migrations/20240926003524_include_tx_index_on_blobs_on_transactions_model_indexes/migration.sql @@ -0,0 +1,11 @@ +-- DropIndex +DROP INDEX "blobs_on_transactions_block_number_idx"; + +-- DropIndex +DROP INDEX "blobs_on_transactions_block_timestamp_idx"; + +-- CreateIndex +CREATE INDEX "blobs_on_transactions_block_timestamp_tx_index_index_idx" ON "blobs_on_transactions"("block_timestamp", "tx_index", "index"); + +-- CreateIndex +CREATE INDEX "blobs_on_transactions_block_number_tx_index_index_idx" ON "blobs_on_transactions"("block_number", "tx_index", "index"); diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index c9fd52312..339851deb 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -183,6 +183,7 @@ model BlobsOnTransactions { blockNumber Int @map("block_number") blockTimestamp DateTime @map("block_timestamp") txHash String @map("tx_hash") + txIndex Int @map("tx_index") index Int blob Blob @relation(fields: [blobHash], references: [versionedHash]) @@ -190,10 +191,10 @@ model BlobsOnTransactions { transaction Transaction @relation(fields: [txHash], references: [hash]) @@id([txHash, index]) + @@index([blockTimestamp, txIndex, index]) + @@index([blockNumber, txIndex, index]) @@index([blobHash]) @@index([blockHash]) - @@index([blockNumber]) - @@index([blockTimestamp]) @@index([txHash]) @@map("blobs_on_transactions") } diff --git a/packages/db/prisma/seed/DataGenerator.ts b/packages/db/prisma/seed/DataGenerator.ts index 999ec8b7c..9c8f353a3 100644 --- a/packages/db/prisma/seed/DataGenerator.ts +++ b/packages/db/prisma/seed/DataGenerator.ts @@ -1,16 +1,32 @@ import { faker } from "@faker-js/faker"; -import type { Address, Blob } from "@prisma/client"; -import { $Enums, Prisma } from "@prisma/client"; +import type { + Address, + Blob, + BlobDataStorageReference, + Block, + Transaction, + TransactionFork, +} from "@prisma/client"; +import { Category, Prisma } from "@prisma/client"; import { sha256 } from "js-sha256"; import dayjs from "@blobscan/dayjs"; +import { BlobStorage } from "../enums"; import type { SeedParams } from "./params"; - -function bigintToDecimal(bigint: bigint): Prisma.Decimal { - return new Prisma.Decimal(bigint.toString()); -} - +import { + BLOB_GAS_PER_BLOB, + calculateBlobGasPrice, + calculateExcessBlobGas, + COMMON_MAX_FEE_PER_BLOB_GAS, + ROLLUP_ADDRESSES, +} from "./web3"; + +export type FullBlock = Block & { + transactions: (Transaction & { + blobs: (Blob & { storageRefs: BlobDataStorageReference[] })[]; + })[]; +}; export class DataGenerator { #seedParams: SeedParams; @@ -18,10 +34,12 @@ export class DataGenerator { this.#seedParams = seedParams; } - generateUniqueAddresses(): string[] { - return Array.from({ length: this.#seedParams.maxUniqueAddresses }).map(() => - faker.finance.ethereumAddress() - ); + generateAddresses(): string[] { + return Array.from({ + length: + this.#seedParams.maxUniqueAddresses - + Object.keys(ROLLUP_ADDRESSES).length, + }).map(() => faker.finance.ethereumAddress()); } generateDBAddresses( @@ -45,250 +63,332 @@ export class DataGenerator { })); } - generateDBBlobs(): Blob[] { + generateDBAddress(tx: Transaction): Address[] { const now = new Date(); - const blobs: Blob[] = []; + const addresses = [tx.fromId, tx.toId]; - for (let i = 0; i < this.#seedParams.maxUniqueBlobs; i++) { - const commitment = faker.string.hexadecimal({ - length: 96, - }); - const proof = faker.string.hexadecimal({ length: 96 }); - const versionedHash = `0x01${sha256(commitment).slice(2)}`; - const dataLength = faker.number.int({ - min: this.#seedParams.maxBlobBytesSize, - max: this.#seedParams.maxBlobBytesSize * 2, - }); - const size = dataLength % 2 === 0 ? dataLength : dataLength + 1; - - blobs.push({ - commitment, - proof, - size, - versionedHash, - firstBlockNumber: Infinity, + return addresses.map((address) => { + return { + address, + firstBlockNumberAsReceiver: tx.blockNumber, + firstBlockNumberAsSender: tx.blockNumber, insertedAt: now, updatedAt: now, - }); - } + }; + }); + } - return blobs; + generateBlob(): Blob { + const now = new Date(); + const commitment = faker.string.hexadecimal({ + length: 96, + }); + const proof = faker.string.hexadecimal({ length: 96 }); + const versionedHash = `0x01${sha256(commitment).slice(2)}`; + const size = Number(BLOB_GAS_PER_BLOB); + + return { + commitment, + proof, + size, + versionedHash, + firstBlockNumber: Infinity, + insertedAt: now, + updatedAt: now, + }; } - generateBlobData(sizes: number[]): string[] { - return sizes.map((s) => faker.string.hexadecimal({ length: s })); + generateBlobDataStorageRef(blob: Blob): BlobDataStorageReference[] { + return [BlobStorage.GOOGLE, BlobStorage.POSTGRES].map((blobStorage) => ({ + blobHash: blob.versionedHash, + blobStorage, + dataReference: blob.versionedHash, + })); } - generateDBBlobOnTxs( - blocks: Prisma.BlockCreateManyInput[], - blocksTxs: Prisma.TransactionCreateManyInput[][], - uniqueBlobs: Blob[] - ) { - return blocksTxs.flatMap((blockTxs) => { - let blockBlobsRemaining = faker.number.int({ - min: 1, - max: this.#seedParams.maxBlobsPerBlock, - }); - return blockTxs.flatMap((tx, i) => { - const remainingTxs = blockTxs.length - i + 1; - const txBlobs = faker.number.int({ - min: 1, - max: Math.max(1, blockBlobsRemaining - remainingTxs), - }); + generateBlock({ + parentBlock, + blockInterval, + }: { + parentBlock: Pick< + Block, + "number" | "slot" | "blobGasUsed" | "excessBlobGas" | "timestamp" + >; + blockInterval: number; + }): Block { + const now = new Date(); - blockBlobsRemaining -= txBlobs; + const number = parentBlock.number + faker.number.int({ min: 1, max: 4 }); + const timestamp = dayjs(parentBlock.timestamp.toISOString()).add( + blockInterval, + "second" + ); + const slot = parentBlock.slot + faker.number.int({ min: 1, max: 5 }); + const txsCount = faker.number.int({ + min: 1, + max: 6, + }); + const blobGasUsed = BLOB_GAS_PER_BLOB * BigInt(txsCount); + const excessBlobGas = calculateExcessBlobGas( + BigInt(parentBlock.excessBlobGas.toString()), + BigInt( + number - parentBlock.number === 1 + ? parentBlock.blobGasUsed.toString() + : 0 + ) + ).toString(); + const blobGasPrice = calculateBlobGasPrice( + BigInt(excessBlobGas) + ).toString(); + + return { + hash: faker.string.hexadecimal({ length: 64 }), + number, + timestamp: timestamp.toDate(), + slot, + blobAsCalldataGasUsed: new Prisma.Decimal(0), + blobGasUsed: new Prisma.Decimal(blobGasUsed.toString()), + blobGasPrice: new Prisma.Decimal(blobGasPrice), + excessBlobGas: new Prisma.Decimal(excessBlobGas), + insertedAt: now, + updatedAt: now, + }; + } - const blobsOnTxs: Prisma.BlobsOnTransactionsCreateManyInput[] = []; + generateBlockTransactions( + block: Block, + uniqueAddresses: string[] + ): Transaction[] { + const now = new Date(); + const maxBlobs = Number(block.blobGasUsed) / Number(BLOB_GAS_PER_BLOB); - for (let i = 0; i < txBlobs; i++) { - const blobIndex = faker.number.int({ - min: 0, - max: uniqueBlobs.length - 1, - }); + const txCount = faker.number.int({ + min: 1, + max: maxBlobs, + }); - const blob = - uniqueBlobs.find((b) => b.firstBlockNumber === Infinity) ?? - uniqueBlobs[blobIndex]; - - if (!blob) { - throw new Error("Blob not found"); - } - - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const block = blocks.find((b) => b.hash === tx.blockHash)!; - blob.firstBlockNumber = Math.min( - blob.firstBlockNumber ?? Infinity, - block.number ?? Infinity - ); - - blobsOnTxs.push({ - blobHash: blob.versionedHash, - index: i, - txHash: tx.hash, - blockHash: block.hash, - blockNumber: block.number, - blockTimestamp: new Date(block.timestamp), - }); - } + let remainingBlobs = 6 - txCount; - return blobsOnTxs; - }); - }); - } + return Array.from({ + length: txCount, + }).map((_, i) => { + const txHash = faker.string.hexadecimal({ length: 64 }); + const category = faker.helpers.weightedArrayElement( + this.#seedParams.categoryWeights + ); + const rollup = + category === Category.ROLLUP + ? faker.helpers.weightedArrayElement(this.#seedParams.rollupWeights) + : null; - generateDBBlocks() { - const now = new Date(); - const timestamps = this.#generateUniqueTimestamps( - this.#seedParams.totalDays, - this.#seedParams.minBlocksPerDay, - this.#seedParams.maxBlocksPerDay - ); + let fromId: string; - let prevBlockNumber = 0; - let prevSlot = 0; + if (rollup) { + fromId = ROLLUP_ADDRESSES[rollup]; + } else { + fromId = faker.helpers.arrayElement(uniqueAddresses); + } - return timestamps.map((timestamp) => { - const number = prevBlockNumber + faker.number.int({ min: 1, max: 20 }); - const slot = prevSlot + faker.number.int({ min: 1, max: 10 }); - const txsCount = faker.number.int({ - min: 1, - max: this.#seedParams.maxBlobsPerBlock, - }); - const blobGasUsed = faker.number.int({ - min: this.#seedParams.gasPerBlob * txsCount, - }); - const blobGasPrice = faker.number.bigInt({ - min: this.#seedParams.minBlobGasPrice, - max: this.#seedParams.maxBlobGasPrice, - }); - const excessBlobGas = faker.number.int({ - min: 15_000_000, - max: 20_000_000, - }); + let toId = faker.helpers.arrayElement(uniqueAddresses); + + while (toId === fromId) { + toId = faker.helpers.arrayElement(uniqueAddresses); + } - prevBlockNumber = number; - prevSlot = slot; + const gasPrice = faker.number + .bigInt({ + min: 1770074991, + max: 6323447039, + }) + .toString(); + const maxFeePerBlobGas = faker.helpers + .arrayElement(COMMON_MAX_FEE_PER_BLOB_GAS) + .toString(); + const extraBlobs = faker.number.int({ min: 0, max: remainingBlobs }); + const blobGasUsed = ( + BigInt(1 + extraBlobs) * BLOB_GAS_PER_BLOB + ).toString(); + + remainingBlobs -= extraBlobs; return { - hash: faker.string.hexadecimal({ length: 64 }), - number, - timestamp, - slot, - blobAsCalldataGasUsed: 0, + hash: txHash, + index: i, + fromId, + toId, + blockHash: block.hash, + blockNumber: block.number, + blockTimestamp: block.timestamp, + blobAsCalldataGasUsed: new Prisma.Decimal(0), blobGasUsed: new Prisma.Decimal(blobGasUsed), - blobGasPrice: bigintToDecimal(blobGasPrice), - excessBlobGas: new Prisma.Decimal(excessBlobGas), + gasPrice: new Prisma.Decimal(gasPrice), + maxFeePerBlobGas: new Prisma.Decimal(maxFeePerBlobGas), + category, + rollup, insertedAt: now, updatedAt: now, }; }); } - #generateUniqueTimestamps( - days: number, - minTimestamps: number, - maxTimestamps: number - ) { - const uniqueTimestamps: Date[] = []; - - let startDay = dayjs().subtract(days, "day"); - - Array.from({ length: days }).forEach(() => { - const dayTimestamps = faker.number.int({ - min: minTimestamps, - max: maxTimestamps, - }); - const timestamps = new Set(); + generateTransactionBlobs(tx: Transaction, prevBlobs: Blob[]): Blob[] { + return Array.from({ + length: Number(tx.blobGasUsed) / Number(BLOB_GAS_PER_BLOB), + }).map(() => { + const isUnique = tx.rollup + ? true + : faker.datatype.boolean({ + probability: this.#seedParams.uniqueBlobsRatio, + }); - let previousTimestamp: dayjs.Dayjs = startDay; + if (isUnique || !prevBlobs.length) { + return this.generateBlob(); + } else { + return faker.helpers.arrayElement(prevBlobs); + } + }); + } - Array.from({ length: dayTimestamps }).forEach(() => { - const blocksUntilNextTimestamp = faker.number.int({ min: 1, max: 3 }); - const blockValidationTime = faker.number.int({ min: 10, max: 15 }); - const timestamp = previousTimestamp.add( - blocksUntilNextTimestamp * blockValidationTime, - "second" - ); + generateBlobData(sizes: number[]): string[] { + return sizes.map((s) => faker.string.hexadecimal({ length: s })); + } - timestamps.add(timestamp.toDate()); + generateDBFullBlocks({ + initialBlock, + uniqueAddresses, + prevBlobs, + }: { + initialBlock?: Block; + uniqueAddresses: string[]; + prevBlobs: Blob[]; + }): FullBlock[] { + const totalBlocks = faker.number.int({ + min: this.#seedParams.minBlocksPerDay, + max: this.#seedParams.maxBlocksPerDay, + }); + const blockInterval = 86400 / totalBlocks; + + const prevDay = initialBlock + ? dayjs(initialBlock.timestamp) + : dayjs().subtract(this.#seedParams.totalDays, "day"); + const nextDay = prevDay.add(1, "day"); + const initialTimestamp = nextDay.startOf("day"); + + let parentBlock: Pick< + Block, + "number" | "slot" | "excessBlobGas" | "blobGasUsed" | "timestamp" + > = initialBlock + ? { + number: initialBlock.number, + slot: initialBlock.slot, + excessBlobGas: initialBlock.excessBlobGas, + blobGasUsed: initialBlock.blobGasUsed, + timestamp: initialBlock.timestamp, + } + : { + number: 0, + slot: 0, + excessBlobGas: new Prisma.Decimal(0), + blobGasUsed: new Prisma.Decimal(0), + timestamp: initialTimestamp.toDate(), + }; - previousTimestamp = timestamp; + return Array.from({ length: totalBlocks }).map((_, __) => { + const block = this.generateBlock({ + parentBlock, + blockInterval, }); - uniqueTimestamps.push(...Array.from(timestamps).sort()); + parentBlock = block; - startDay = startDay.add(1, "day"); + return { + ...block, + transactions: this.generateBlockTransactions( + block, + uniqueAddresses + ).map((tx) => { + const blobs = this.generateTransactionBlobs(tx, prevBlobs); + + prevBlobs.push(...blobs); + + return { + ...tx, + + blobs: blobs.map((b) => ({ + ...b, + storageRefs: this.generateBlobDataStorageRef(b), + })), + }; + }), + }; }); - - return uniqueTimestamps; } generateDBBlockTransactions( - blocks: Prisma.BlockCreateManyInput[], + blocks: Block[], uniqueAddresses: string[] - ) { - const now = new Date(); - + ): Transaction[][] { return blocks.map((block) => { - const txCount = faker.number.int({ - min: 1, - max: this.#seedParams.maxBlobsPerBlock, + return this.generateBlockTransactions(block, uniqueAddresses); + }); + } + + generateDBTransactionForks(blocks: FullBlock[]): { + blocks: Block[]; + txs: TransactionFork[]; + } { + const dbForkTxs: TransactionFork[] = []; + const forkBlocks: Block[] = []; + + blocks.forEach((b, i) => { + const isReorg = faker.datatype.boolean({ + probability: this.#seedParams.reorgRatio, }); + const parentBlock = blocks[i - 1]; - return Array.from({ - length: txCount, - }).map(() => { - const txHash = faker.string.hexadecimal({ length: 64 }); - const from = - uniqueAddresses[faker.number.int(uniqueAddresses.length - 1)]; - const to = - uniqueAddresses[faker.number.int(uniqueAddresses.length - 1)]; - const gasPrice = faker.number.bigInt({ - min: this.#seedParams.minGasPrice, - max: this.#seedParams.maxGasPrice, + if (!parentBlock) { + return; + } + + const availableSlots = b.slot - parentBlock.slot > 1; + + if (isReorg && availableSlots) { + const forkBlock = this.generateBlock({ + parentBlock, + blockInterval: 2, }); - const maxFeePerBlobGas = faker.number.bigInt({ - min: 0, - max: this.#seedParams.maxFeePerBlobGas, + forkBlock.number = b.number; + forkBlock.slot = faker.number.int({ + min: parentBlock.slot + 1, + max: b.slot - 1, }); - const rollup = [faker.helpers.enumValue($Enums.Rollup), null][ - faker.number.int({ min: 0, max: 1 }) - ]; - - // Unreachable code, done only for type checking - if (!from || !to) throw new Error("Address not found"); - - return { - hash: txHash, - fromId: from, - toId: to, - blockHash: block.hash, - blockNumber: block.number, - blockTimestamp: block.timestamp, - blobAsCalldataGasUsed: new Prisma.Decimal(0), - blobGasUsed: new Prisma.Decimal(0), - gasPrice: bigintToDecimal(gasPrice), - maxFeePerBlobGas: bigintToDecimal(maxFeePerBlobGas), - rollup, - insertedAt: now, - updatedAt: now, - }; - }); - }); - } - calculateEIP2028GasCost(hexData: string): number { - const bytes = Buffer.from(hexData.slice(2), "hex"); - let gasCost = 0; + forkBlock.timestamp = faker.date.between({ + from: dayjs(parentBlock.timestamp).add(1, "millisecond").toDate(), + to: dayjs(b.timestamp).subtract(1, "millisecond").toDate(), + }); - for (const byte of bytes.entries()) { - if (byte[1] === 0) { - gasCost += 4; - } else { - gasCost += 16; + const nextTxs = blocks.slice(i, i + 4).flatMap((b) => b.transactions); + const forkedTxsCount = faker.number.int({ + min: 1, + max: 6, + }); + + const forkedTxs = faker.helpers + .arrayElements(nextTxs, forkedTxsCount) + .map((tx) => ({ + blockHash: forkBlock.hash, + hash: tx.hash, + insertedAt: new Date(), + updatedAt: new Date(), + })); + + forkBlocks.push(forkBlock); + + dbForkTxs.push(...forkedTxs); } - } + }); - return gasCost; + return { blocks: forkBlocks, txs: dbForkTxs }; } } diff --git a/packages/db/prisma/seed/main.ts b/packages/db/prisma/seed/main.ts index a941067c2..3aba44546 100644 --- a/packages/db/prisma/seed/main.ts +++ b/packages/db/prisma/seed/main.ts @@ -1,173 +1,194 @@ -import type { BlobDataStorageReference } from "@prisma/client"; +import type { Address, Blob, Block, Prisma } from "@prisma/client"; import dayjs from "@blobscan/dayjs"; import { prisma } from ".."; import { DataGenerator } from "./DataGenerator"; import { seedParams } from "./params"; -import { performPrismaOpInBatches } from "./utils"; - -const BATCH_SIZE = 1000; -const STORAGE_BATCH_SIZE = 100; +import { performPrismaUpsertManyInBatches } from "./utils"; async function main() { const dataGenerator = new DataGenerator(seedParams); - - // 1. Generate mock data - const dataGenerationStart = performance.now(); - - const addresses = dataGenerator.generateUniqueAddresses(); - const dbBlocks = dataGenerator.generateDBBlocks(); - const dbBlockTxs = dataGenerator.generateDBBlockTransactions( - dbBlocks, - addresses - ); - const dbTxs = dbBlockTxs.flat(); - const dbAddresses = dataGenerator.generateDBAddresses( - addresses, - dbTxs, - dbBlocks - ); - const dbBlobs = dataGenerator.generateDBBlobs(); - const dbBlobsOnTxs = dataGenerator.generateDBBlobOnTxs( - dbBlocks, - dbBlockTxs, - dbBlobs - ); - - const dataGenerationEnd = performance.now(); - - const batches = Math.ceil(dbBlobs.length / STORAGE_BATCH_SIZE); - const dbBlobDataStorageRefs: BlobDataStorageReference[] = []; - - const blobsUploadStart = performance.now(); - // 2. Store blobs' data in storages - for (let i = 0; i < batches; i++) { - const blobsBatch = dbBlobs.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE); - const blobsDataBatch = dataGenerator.generateBlobData( - blobsBatch.map((b) => b.size) - ); - - const blobDataStorageRefs: BlobDataStorageReference[][] = await Promise.all( - blobsDataBatch.map(async (blobData, i) => { - const blob = blobsBatch[i]; - if (!blob) { - throw new Error("Blob not found"); + const addresses = dataGenerator.generateAddresses(); + let lastBlock: Block | undefined; + const blobs: Blob[] = []; + for (let i = seedParams.totalDays; i > 0; i--) { + const fullBlocks = dataGenerator.generateDBFullBlocks({ + initialBlock: lastBlock, + uniqueAddresses: addresses, + prevBlobs: blobs, + }); + const { blocks: forkBlocks, txs: forkTxs } = + dataGenerator.generateDBTransactionForks(fullBlocks); + const dbAddresses: Record = {}; + const dbBlockInsertions: Prisma.BlockCreateManyInput[] = []; + const dbTxInsertions: Prisma.TransactionCreateManyInput[] = []; + const dbBlobInsertions: Blob[] = []; + const dbBlobsOnTxsInsertions: Prisma.BlobsOnTransactionsCreateManyInput[] = + []; + const dbBlobDataStorageRefs: Prisma.BlobDataStorageReferenceCreateManyInput[] = + []; + + fullBlocks.forEach(({ transactions, ...block }) => { + dbBlockInsertions.push(block); + transactions.forEach(({ blobs, ...tx }) => { + dbTxInsertions.push(tx); + const from = dbAddresses[tx.fromId]; + const to = dbAddresses[tx.toId]; + if (from) { + from.firstBlockNumberAsSender = Math.min( + from.firstBlockNumberAsSender ?? tx.blockNumber, + tx.blockNumber + ); + } else { + dbAddresses[tx.fromId] = { + address: tx.fromId, + firstBlockNumberAsReceiver: null, + firstBlockNumberAsSender: tx.blockNumber, + insertedAt: new Date(), + updatedAt: new Date(), + }; } - - await prisma.blobData.create({ - data: { - id: blob.versionedHash, - data: Buffer.from(blobData.slice(2), "hex"), - }, - }); - - return [ - { + if (to) { + to.firstBlockNumberAsReceiver = Math.min( + to.firstBlockNumberAsReceiver ?? tx.blockNumber, + tx.blockNumber + ); + } else { + dbAddresses[tx.toId] = { + address: tx.toId, + firstBlockNumberAsReceiver: tx.blockNumber, + firstBlockNumberAsSender: null, + insertedAt: new Date(), + updatedAt: new Date(), + }; + } + blobs.forEach(({ storageRefs, ...blob }, i) => { + dbBlobInsertions.push(blob); + dbBlobsOnTxsInsertions.push({ blobHash: blob.versionedHash, - blobStorage: "POSTGRES", - dataReference: blob.versionedHash, - }, - ]; - }) + blockHash: tx.blockHash, + blockNumber: tx.blockNumber, + blockTimestamp: tx.blockTimestamp, + index: i, + txHash: tx.hash, + txIndex: tx.index as number, + }); + storageRefs.forEach((storageRef) => { + dbBlobDataStorageRefs.push(storageRef); + }); + }); + }); + }); + console.log( + `Day ${i} of ${seedParams.totalDays} generated (blocks: ${ + fullBlocks.length + }, txs: ${dbTxInsertions.length}, blobs: ${ + dbBlobInsertions.length + }, addresses: ${Object.keys(dbAddresses).length})` + ); + // await performPrismaOpInBatches(dbBlockInsertions, prisma.block.createMany); + const blockPerformance = performance.now(); + await prisma.block.createMany({ + data: dbBlockInsertions, + }); + const blockPerformanceEnd = performance.now(); + const addressPerformance = performance.now(); + await performPrismaUpsertManyInBatches( + Object.values(dbAddresses), + prisma.address.upsertMany + ); + const addressPerformanceEnd = performance.now(); + const txPerformance = performance.now(); + await prisma.transaction.createMany({ + data: dbTxInsertions, + }); + const txPerformanceEnd = performance.now(); + const blobsPerformance = performance.now(); + await prisma.blob.createMany({ + data: dbBlobInsertions, + skipDuplicates: true, + }); + const blobsPerformanceEnd = performance.now(); + const blobsOnTxsPerformance = performance.now(); + await prisma.blobsOnTransactions.createMany({ + data: dbBlobsOnTxsInsertions, + skipDuplicates: true, + }); + const blobsOnTxsPerformanceEnd = performance.now(); + const blobStoragePerformance = performance.now(); + await prisma.blobDataStorageReference.createMany({ + data: dbBlobDataStorageRefs, + skipDuplicates: true, + }); + const blobStoragePerformanceEnd = performance.now(); + + // Generate forks + await prisma.$transaction([ + prisma.block.createMany({ + data: forkBlocks, + }), + prisma.transactionFork.createMany({ + data: forkTxs, + }), + ]); + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const { transactions: _, ...block } = fullBlocks[fullBlocks.length - 1]!; + lastBlock = block; + console.log( + `Day ${i} of ${seedParams.totalDays} inserted. Times: blocks: ${ + (blockPerformanceEnd - blockPerformance) / 1000 + }, addresses: ${ + (addressPerformanceEnd - addressPerformance) / 1000 + }, txs: ${(txPerformanceEnd - txPerformance) / 1000}, blobs: ${ + (blobsPerformanceEnd - blobsPerformance) / 1000 + }, blobsOnTxs: ${ + (blobsOnTxsPerformanceEnd - blobsOnTxsPerformance) / 1000 + }, blobStorage: ${ + (blobStoragePerformanceEnd - blobStoragePerformance) / 1000 + }` + ); + console.log( + "========================================================================" ); - - dbBlobDataStorageRefs.push(...blobDataStorageRefs.flat()); } - const blobsUploadEnd = performance.now(); - - // 3. Insert data into the database - - const dataInsertionStart = performance.now(); - - console.log( - "========================================================================" - ); - - await performPrismaOpInBatches(dbBlocks, prisma.block.createMany); - console.log(`Blocks inserted: ${dbBlocks.length}`); - - await performPrismaOpInBatches(dbAddresses, prisma.address.createMany); - console.log(`Addresses inserted: ${dbAddresses.length}`); - - await performPrismaOpInBatches(dbTxs, prisma.transaction.createMany); - console.log(`Transactions inserted: ${dbBlockTxs.length}`); - - await performPrismaOpInBatches(dbBlobs, prisma.blob.createMany); - console.log(`Blobs inserted: ${dbBlobs.length}`); - - await performPrismaOpInBatches( - dbBlobDataStorageRefs, - prisma.blobDataStorageReference.createMany - ); - console.log( - `Blob storage references inserted: ${dbBlobDataStorageRefs.length}` - ); - - await performPrismaOpInBatches( - dbBlobsOnTxs, - prisma.blobsOnTransactions.createMany - ); - console.log(`Blobs on transactions inserted: ${dbBlobsOnTxs.length}`); - - const dataInsertionEnd = performance.now(); - - console.log( - "========================================================================" - ); - - console.log( - `Data generation took ${ - (dataGenerationEnd - dataGenerationStart) / 1000 - } seconds` - ); - console.log( - `Data insertion took ${ - (dataInsertionEnd - dataInsertionStart) / 1000 - } seconds` - ); - console.log( - `Blobs upload took ${(blobsUploadEnd - blobsUploadStart) / 1000} seconds` - ); - - console.log(`Data inserted for the last ${seedParams.totalDays} days`); - + const overallPerformance = performance.now(); await Promise.all([ prisma.blobOverallStats.increment({ from: 0, - to: 9999, + to: Number.MAX_SAFE_INTEGER, }), prisma.blockOverallStats.increment({ from: 0, - to: 9999, + to: Number.MAX_SAFE_INTEGER, }), prisma.transactionOverallStats.increment({ from: 0, - to: 9999, + to: Number.MAX_SAFE_INTEGER, }), ]); - - console.log("Overall stats created."); - + const overallPerformanceEnd = performance.now(); console.log( - "========================================================================" + `Overall stats created. Time: ${ + (overallPerformanceEnd - overallPerformance) / 1000 + }` ); - + const dailyStatsPerformance = performance.now(); const yesterdayPeriod = { to: dayjs().subtract(1, "day").startOf("day").toISOString(), }; - await Promise.all([ prisma.blobDailyStats.populate(yesterdayPeriod), prisma.blockDailyStats.populate(yesterdayPeriod), prisma.transactionDailyStats.populate(yesterdayPeriod), ]); - - console.log("Daily stats created"); - + const dailyStatsPerformanceEnd = performance.now(); console.log( - "========================================================================" + `Daily stats created. Time: ${ + (dailyStatsPerformanceEnd - dailyStatsPerformance) / 1000 + }` ); } diff --git a/packages/db/prisma/seed/params.ts b/packages/db/prisma/seed/params.ts index 8782e2319..223454f4d 100644 --- a/packages/db/prisma/seed/params.ts +++ b/packages/db/prisma/seed/params.ts @@ -1,40 +1,57 @@ +import { Category, Rollup } from "../enums"; + export type SeedParams = { totalDays: number; minBlocksPerDay: number; maxBlocksPerDay: number; - maxBlobsPerBlock: number; - maxBlobsPerTx: number; - maxUniqueBlobs: number; maxUniqueAddresses: number; - maxBlobBytesSize: number; - minGasPrice: bigint; - maxGasPrice: bigint; - minBlobGasPrice: bigint; - maxBlobGasPrice: bigint; - maxFeePerBlobGas: bigint; + categoryWeights: { value: Category; weight: number }[]; + rollupWeights: { value: Rollup; weight: number }[]; + uniqueBlobsRatio: number; + reorgRatio: number; gasPerBlob: number; }; export const seedParams: SeedParams = { - totalDays: 20, - minBlocksPerDay: 300, - maxBlocksPerDay: 500, - - maxBlobsPerBlock: 6, - maxBlobsPerTx: 3, + totalDays: 10, + minBlocksPerDay: 1000, + maxBlocksPerDay: 1500, - maxUniqueBlobs: 100, maxUniqueAddresses: 1000, - maxBlobBytesSize: 1024, - - minGasPrice: BigInt(30000000000), - maxGasPrice: BigInt(100000000000), - minBlobGasPrice: BigInt(30000000000), - maxBlobGasPrice: BigInt(100000000000), - maxFeePerBlobGas: BigInt(1000), - + categoryWeights: [ + { + value: Category.OTHER, + weight: 0.2, + }, + { + value: Category.ROLLUP, + weight: 0.8, + }, + ], + rollupWeights: [ + { value: Rollup.ARBITRUM, weight: 0.08096648 }, + { value: Rollup.BASE, weight: 0.10522268 }, + { value: Rollup.OPTIMISM, weight: 0.03785166 }, + { value: Rollup.SCROLL, weight: 0.10230571 }, + { value: Rollup.STARKNET, weight: 0.043719 }, + { value: Rollup.ZKSYNC, weight: 0.02146207 }, + { value: Rollup.MODE, weight: 0.00489422 }, + { value: Rollup.ZORA, weight: 0.00515352 }, + { value: Rollup.LINEA, weight: 0.04148197 }, + { value: Rollup.PARADEX, weight: 0.03303769 }, + { value: Rollup.BOBA, weight: 0.00515216 }, + { value: Rollup.CAMP, weight: 0.00147192 }, + { value: Rollup.KROMA, weight: 0.01695014 }, + { value: Rollup.METAL, weight: 0.02825735 }, + { value: Rollup.PGN, weight: 0.00021592 }, + { value: Rollup.BLAST, weight: 0.02580827 }, + { value: Rollup.OPTOPIA, weight: 0.006698 }, + { value: Rollup.TAIKO, weight: 0.27215873 }, + ], + uniqueBlobsRatio: 0.04, + reorgRatio: 0.01, gasPerBlob: 2 ** 17, // 131_072 }; diff --git a/packages/db/prisma/seed/utils.ts b/packages/db/prisma/seed/utils.ts index 084e8137c..e48c92d5d 100644 --- a/packages/db/prisma/seed/utils.ts +++ b/packages/db/prisma/seed/utils.ts @@ -1,4 +1,5 @@ -export const BATCH_SIZE = 10_000; +export const BATCH_SIZE = 30_000; +export const UPSERT_MANY_BATCH_SIZE = 5_000; export const STORAGE_BATCH_SIZE = 100; export function buildGoogleStorageUri(hash: string): string { @@ -8,21 +9,15 @@ export function buildGoogleStorageUri(hash: string): string { )}/${hash.slice(6, 8)}/${hash.slice(2)}.txt`; } -// export async function performBlobStorageOpInBatches(blobs: SeedBlob[]) { -// const batches = Math.ceil(blobs.length / STORAGE_BATCH_SIZE); - -// for (let i = 0; i < batches; i++) { -// const start = i * STORAGE_BATCH_SIZE; -// const end = start + STORAGE_BATCH_SIZE; -// const batchBlobs = blobs.slice(start, end); - -// await Promise.all(batchBlobs.map((b) => blobStorageManager.storeBlob(b))); -// } -// } - export function performPrismaOpInBatches( elements: T[], - prismaOp: ({ data }: { data: T[] }) => Promise<{ count: number }> + prismaOp: ({ + data, + skipDuplicates, + }: { + data: T[]; + skipDuplicates: boolean; + }) => Promise<{ count: number }> ) { const batches = Math.ceil(elements.length / BATCH_SIZE); const operations: Promise<{ count: number }>[] = []; @@ -31,7 +26,30 @@ export function performPrismaOpInBatches( const start = index * BATCH_SIZE; const end = start + BATCH_SIZE; - operations.push(prismaOp({ data: elements.slice(start, end) })); + operations.push( + prismaOp({ data: elements.slice(start, end), skipDuplicates: true }) + ); + }); + + return Promise.all(operations); +} + +export function performPrismaUpsertManyInBatches( + elements: T[], + prismaOp: (data: T[]) => Promise +) { + const batches = Math.ceil(elements.length / UPSERT_MANY_BATCH_SIZE); + const operations: Promise[] = []; + + Array.from({ length: batches }).forEach((_, index) => { + const start = index * UPSERT_MANY_BATCH_SIZE; + const end = start + UPSERT_MANY_BATCH_SIZE; + + const elementsSlice = elements.slice(start, end); + + if (elementsSlice.length) { + operations.push(prismaOp(elementsSlice)); + } }); return Promise.all(operations); diff --git a/packages/db/prisma/seed/web3.ts b/packages/db/prisma/seed/web3.ts new file mode 100644 index 000000000..f0c17c3ef --- /dev/null +++ b/packages/db/prisma/seed/web3.ts @@ -0,0 +1,99 @@ +import { Rollup } from "../enums"; + +const MIN_BLOB_BASE_FEE = BigInt(1); +const BLOB_BASE_FEE_UPDATE_FRACTION = BigInt(3_338_477); +export const BLOB_GAS_PER_BLOB = BigInt(131_072); +export const TARGET_BLOB_GAS_PER_BLOCK = BigInt(393216); + +export const COMMON_MAX_FEE_PER_BLOB_GAS = [ + 1000000000, 2, 150000000000, 10, 2000000000, 26000000000, 1, 4000000000, 4, + 10000000000, 80000000000, 50000000000, 56000000000, 8, 65000000000, 20, + 8000000000, 60000000000, 20000000000, 16, 15000000000, 6, 119443524432, + 90741658012, 30, 77553845946, 102084366682, 98154088832, 30000000000, + 5000000000, 176876867046, 134373966794, 129200529966, 139754557944, + 71000000000, 25000000000, 13253037170, 145350598142, 106172020274, + 61277111208, 32, 40, 110423351454, 87248077832, 12000000000, 94375127918, + 31436483250, 151170714642, 41379939894, 74568000282, 58917924626, 29062453216, + 83889001508, 50, 12, 71697110620, 124226272074, 24838702154, 114844914084, + 14200000000, 40000000000, 56649567354, 52371487834, 54468542552, 44760150790, +].map((x) => BigInt(x)); + +export const ROLLUP_ADDRESSES: Record = { + [Rollup.ARBITRUM]: "0xc1b634853cb333d3ad8663715b08f41a3aec47cc", + [Rollup.BASE]: "0x5050f69a9786f081509234f1a7f4684b5e5b76c9", + [Rollup.BLAST]: "0x415c8893d514f9bc5211d36eeda4183226b84aa7", + [Rollup.BOBA]: "0xe1b64045351b0b6e9821f19b39f81bc4711d2230", + [Rollup.CAMP]: "0x08f9f14ff43e112b18c96f0986f28cb1878f1d11", + [Rollup.KROMA]: "0x41b8cd6791de4d8f9e0eaf7861ac506822adce12", + [Rollup.LINEA]: "0xa9268341831efa4937537bc3e9eb36dbece83c7e", + [Rollup.METAL]: "0xc94c243f8fb37223f3eb2f7961f7072602a51b8b", + [Rollup.OPTIMISM]: "0x6887246668a3b87f54deb3b94ba47a6f63f32985", + [Rollup.OPTOPIA]: "0x3d0bf26e60a689a7da5ea3ddad7371f27f7671a5", + [Rollup.PARADEX]: "0xc70ae19b5feaa5c19f576e621d2bad9771864fe2", + [Rollup.PGN]: "0x5ead389b57d533a94a0eacd570dc1cc59c25f2d4", + [Rollup.SCROLL]: "0xcf2898225ed05be911d3709d9417e86e0b4cfc8f", + [Rollup.STARKNET]: "0x2c169dfe5fbba12957bdd0ba47d9cedbfe260ca7", + [Rollup.TAIKO]: "0x000000633b68f5d8d3a86593ebb815b4663bcbe0", + [Rollup.ZKSYNC]: "0x0d3250c3d5facb74ac15834096397a3ef790ec99", + [Rollup.MODE]: "0x99199a22125034c808ff20f377d91187e8050f2e", + [Rollup.ZORA]: "0x625726c858dbf78c0125436c943bf4b4be9d9033", +}; + +export function fakeExponential( + factor: bigint, + numerator: bigint, + denominator: bigint +): bigint { + let i = BigInt(1); + let output = BigInt(0); + let numerator_accumulator = factor * denominator; + + while (numerator_accumulator > 0) { + output += numerator_accumulator; + numerator_accumulator = + (numerator_accumulator * numerator) / (denominator * i); + + i++; + } + + return output / denominator; +} + +export function getEIP2028CalldataGas(hexData: string) { + const bytes = Buffer.from(hexData.slice(2), "hex"); + let gasCost = BigInt(0); + + for (const byte of bytes.entries()) { + if (byte[1] === 0) { + gasCost += BigInt(4); + } else { + gasCost += BigInt(16); + } + } + + return gasCost; +} + +export function calculateBlobGasPrice(excessBlobGas: bigint): bigint { + return BigInt( + fakeExponential( + MIN_BLOB_BASE_FEE, + excessBlobGas, + BLOB_BASE_FEE_UPDATE_FRACTION + ) + ); +} + +export function calculateExcessBlobGas( + parentExcessBlobGas: bigint, + parentBlobGasUsed: bigint +) { + const excessBlobGas = BigInt(parentExcessBlobGas.toString()); + const blobGasUsed = BigInt(parentBlobGasUsed.toString()); + + if (excessBlobGas + blobGasUsed < TARGET_BLOB_GAS_PER_BLOCK) { + return BigInt(0); + } else { + return excessBlobGas + blobGasUsed - TARGET_BLOB_GAS_PER_BLOCK; + } +} diff --git a/packages/db/test/extensions/base-extension.test.ts b/packages/db/test/extensions/base-extension.test.ts index acba920f4..4a9287b6d 100644 --- a/packages/db/test/extensions/base-extension.test.ts +++ b/packages/db/test/extensions/base-extension.test.ts @@ -513,6 +513,7 @@ describe("Base Extension", () => { blockNumber: 1008, blockTimestamp: new Date("2023-08-31T16:00:00Z"), txHash: "txHash016", + txIndex: 0, index: 1, }, { @@ -521,6 +522,7 @@ describe("Base Extension", () => { blockNumber: 1008, blockTimestamp: new Date("2023-08-31T16:00:00Z"), txHash: "txHash016", + txIndex: 0, index: 2, }, ]; diff --git a/packages/db/test/extensions/stats-extension.test.fixtures.ts b/packages/db/test/extensions/stats-extension.test.fixtures.ts index e232717f8..8e49e81c3 100644 --- a/packages/db/test/extensions/stats-extension.test.fixtures.ts +++ b/packages/db/test/extensions/stats-extension.test.fixtures.ts @@ -32,6 +32,7 @@ export const NEW_DATA = { hash: "newTxHash001", fromId: "address6", toId: "address5", + index: 0, blockHash: "newBlockHash001", blockNumber: 2001, blockTimestamp: "2023-09-01T10:00:00Z", @@ -47,6 +48,7 @@ export const NEW_DATA = { hash: "newTxHash002", fromId: "address4", toId: "address3", + index: 0, blockHash: "newBlockHash002", blockNumber: 2002, blockTimestamp: "2023-09-01T12:00:00Z", @@ -86,6 +88,7 @@ export const NEW_DATA = { blockHash: "newBlockHash001", blockNumber: 2001, blockTimestamp: "2023-09-01T10:00:00Z", + txIndex: 0, index: 0, }, { @@ -94,6 +97,7 @@ export const NEW_DATA = { blockHash: "newBlockHash002", blockNumber: 2002, blockTimestamp: "2023-09-01T12:00:00Z", + txIndex: 0, index: 0, }, ], diff --git a/packages/syncers/test/DailyStatsSyncer.test.fixtures.ts b/packages/syncers/test/DailyStatsSyncer.test.fixtures.ts index 6e95e6679..9163cbb78 100644 --- a/packages/syncers/test/DailyStatsSyncer.test.fixtures.ts +++ b/packages/syncers/test/DailyStatsSyncer.test.fixtures.ts @@ -20,6 +20,7 @@ export const CURRENT_DAY_DATA = { hash: "newTxHash001", fromId: "address1", toId: "address2", + index: 0, blockHash: "newBlockHash", blockNumber: 1009, blockTimestamp: "2023-09-01T17:00:00Z", @@ -34,6 +35,7 @@ export const CURRENT_DAY_DATA = { hash: "newTxHash002", fromId: "address5", toId: "address2", + index: 1, blockHash: "newBlockHash", blockNumber: 1009, blockTimestamp: "2023-09-01T17:00:00Z", @@ -52,6 +54,7 @@ export const CURRENT_DAY_DATA = { blockHash: "newBlockHash", blockNumber: 1009, blockTimestamp: "2023-09-01T17:00:00Z", + txIndex: 0, index: 0, }, { @@ -60,6 +63,7 @@ export const CURRENT_DAY_DATA = { blockHash: "newBlockHash", blockNumber: 1009, blockTimestamp: "2023-09-01T17:00:00Z", + txIndex: 0, index: 1, }, { @@ -68,6 +72,7 @@ export const CURRENT_DAY_DATA = { blockHash: "newBlockHash", blockNumber: 1009, blockTimestamp: "2023-09-01T17:00:00Z", + txIndex: 1, index: 0, }, { @@ -76,6 +81,7 @@ export const CURRENT_DAY_DATA = { blockHash: "newBlockHash", blockNumber: 1009, blockTimestamp: "2023-09-01T17:00:00Z", + txIndex: 1, index: 1, }, ], diff --git a/packages/test/src/fixtures/postgres/data.json b/packages/test/src/fixtures/postgres/data.json index 6976ea424..88c6a49df 100644 --- a/packages/test/src/fixtures/postgres/data.json +++ b/packages/test/src/fixtures/postgres/data.json @@ -639,6 +639,7 @@ "blockHash": "blockHash001", "blockNumber": 1001, "blockTimestamp": "2022-10-16T12:00:00Z", + "txIndex": 0, "index": 0 }, { @@ -647,6 +648,7 @@ "blockHash": "blockHash001", "blockNumber": 1001, "blockTimestamp": "2022-10-16T12:00:00Z", + "txIndex": 0, "index": 1 }, { @@ -655,6 +657,7 @@ "blockHash": "blockHash001", "blockNumber": 1001, "blockTimestamp": "2022-10-16T12:00:00Z", + "txIndex": 0, "index": 2 }, { @@ -663,6 +666,7 @@ "blockHash": "blockHash001", "blockNumber": 1001, "blockTimestamp": "2022-10-16T12:00:00Z", + "txIndex": 1, "index": 0 }, { @@ -671,6 +675,7 @@ "blockHash": "blockHash001", "blockNumber": 1001, "blockTimestamp": "2022-10-16T12:00:00Z", + "txIndex": 2, "index": 0 }, { @@ -679,6 +684,7 @@ "blockHash": "blockHash001", "blockNumber": 1001, "blockTimestamp": "2022-10-16T12:00:00Z", + "txIndex": 2, "index": 1 }, { @@ -687,6 +693,7 @@ "blockHash": "blockHash002", "blockNumber": 1002, "blockTimestamp": "2023-05-10T12:00:00Z", + "txIndex": 0, "index": 0 }, { @@ -695,6 +702,7 @@ "blockHash": "blockHash002", "blockNumber": 1002, "blockTimestamp": "2023-05-10T12:00:00Z", + "txIndex": 0, "index": 1 }, { @@ -703,6 +711,7 @@ "blockHash": "blockHash002", "blockNumber": 1002, "blockTimestamp": "2023-05-10T12:00:00Z", + "txIndex": 0, "index": 2 }, { @@ -711,6 +720,7 @@ "blockHash": "blockHash002", "blockNumber": 1002, "blockTimestamp": "2023-05-10T12:00:00Z", + "txIndex": 0, "index": 3 }, { @@ -719,6 +729,7 @@ "blockHash": "blockHash002", "blockNumber": 1002, "blockTimestamp": "2023-05-10T12:00:00Z", + "txIndex": 0, "index": 4 }, { @@ -727,6 +738,7 @@ "blockHash": "blockHash002", "blockNumber": 1002, "blockTimestamp": "2023-05-10T12:00:00Z", + "txIndex": 0, "index": 5 }, { @@ -735,6 +747,7 @@ "blockHash": "blockHash003", "blockNumber": 1003, "blockTimestamp": "2023-08-03T12:00:00Z", + "txIndex": 0, "index": 0 }, { @@ -743,6 +756,7 @@ "blockHash": "blockHash003", "blockNumber": 1003, "blockTimestamp": "2023-08-03T12:00:00Z", + "txIndex": 0, "index": 1 }, { @@ -751,6 +765,7 @@ "blockHash": "blockHash003", "blockNumber": 1003, "blockTimestamp": "2023-08-03T12:00:00Z", + "txIndex": 0, "index": 2 }, { @@ -759,6 +774,7 @@ "blockHash": "blockHash003", "blockNumber": 1003, "blockTimestamp": "2023-08-03T12:00:00Z", + "txIndex": 1, "index": 0 }, { @@ -767,6 +783,7 @@ "blockHash": "blockHash003", "blockNumber": 1003, "blockTimestamp": "2023-08-03T12:00:00Z", + "txIndex": 1, "index": 1 }, { @@ -775,6 +792,7 @@ "blockHash": "blockHash003", "blockNumber": 1003, "blockTimestamp": "2023-08-03T12:00:00Z", + "txIndex": 1, "index": 2 }, { @@ -783,6 +801,7 @@ "blockHash": "blockHash004", "blockNumber": 1004, "blockTimestamp": "2023-08-20T12:00:00Z", + "txIndex": 0, "index": 0 }, { @@ -791,6 +810,7 @@ "blockHash": "blockHash004", "blockNumber": 1004, "blockTimestamp": "2023-08-20T12:00:00Z", + "txIndex": 1, "index": 0 }, { @@ -799,6 +819,7 @@ "blockHash": "blockHash004", "blockNumber": 1004, "blockTimestamp": "2023-08-20T12:00:00Z", + "txIndex": 2, "index": 0 }, { @@ -807,6 +828,7 @@ "blockHash": "blockHash004", "blockNumber": 1004, "blockTimestamp": "2023-08-20T12:00:00Z", + "txIndex": 3, "index": 0 }, { @@ -815,6 +837,7 @@ "blockHash": "blockHash004", "blockNumber": 1004, "blockTimestamp": "2023-08-20T12:00:00Z", + "txIndex": 4, "index": 0 }, { @@ -823,6 +846,7 @@ "blockHash": "blockHash005", "blockNumber": 1005, "blockTimestamp": "2023-08-28T10:00:00Z", + "txIndex": 0, "index": 0 }, { @@ -831,6 +855,7 @@ "blockHash": "blockHash005", "blockNumber": 1005, "blockTimestamp": "2023-08-28T10:00:00Z", + "txIndex": 0, "index": 1 }, { @@ -839,6 +864,7 @@ "blockHash": "blockHash005", "blockNumber": 1005, "blockTimestamp": "2023-08-28T10:00:00Z", + "txIndex": 0, "index": 2 }, { @@ -847,6 +873,7 @@ "blockHash": "blockHash005", "blockNumber": 1005, "blockTimestamp": "2023-08-28T10:00:00Z", + "txIndex": 1, "index": 0 }, { @@ -855,6 +882,7 @@ "blockHash": "blockHash006", "blockNumber": 1006, "blockTimestamp": "2023-08-31T12:00:00Z", + "txIndex": 0, "index": 0 }, { @@ -863,6 +891,7 @@ "blockHash": "blockHash007", "blockNumber": 1007, "blockTimestamp": "2023-08-31T14:00:00Z", + "txIndex": 0, "index": 0 }, { @@ -871,6 +900,7 @@ "blockHash": "blockHash007", "blockNumber": 1007, "blockTimestamp": "2023-08-31T14:00:00Z", + "txIndex": 0, "index": 1 }, { @@ -879,6 +909,7 @@ "blockHash": "blockHash007", "blockNumber": 1007, "blockTimestamp": "2023-08-31T14:00:00Z", + "txIndex": 0, "index": 2 }, { @@ -887,6 +918,7 @@ "blockHash": "blockHash008", "blockNumber": 1008, "blockTimestamp": "2023-08-31T16:00:00Z", + "txIndex": 0, "index": 0 }, { @@ -895,6 +927,7 @@ "blockHash": "0x00903f147f44929cdb385b595b2e745566fe50658362b4e3821fa52b5ebe8f06", "blockNumber": 1008, "blockTimestamp": "2023-08-31T15:50:00Z", + "txIndex": 0, "index": 0 } ],