Use map instead of array

multiversx · Nov 16, 2023 · dcf1e8c · dcf1e8c
1 parent a1d47cd
commit dcf1e8c
Show file tree

Hide file tree

Showing 13 changed files with 184 additions and 309 deletions.
diff --git a/src/db/auctions/auctions.repository.ts b/src/db/auctions/auctions.repository.ts
@@ -32,6 +32,8 @@ import {
   getLowestAuctionForIdentifiersAndMarketplace,
   getOnSaleAssetsCountForCollection,
 } from './sql.queries';
+import { CpuProfiler } from '@multiversx/sdk-nestjs-monitoring';
+import { OrderEntity } from '../orders';
 
 @Injectable()
 export class AuctionsRepository {
@@ -434,58 +436,78 @@ export class AuctionsRepository {
   }
 
   async saveBulkAuctionsOrUpdateAndFillId(auctions: AuctionEntity[]): Promise<void> {
+    const batchSize = 1000;
     if (auctions.length === 0) {
       return;
     }
-    const response = await this.auctionsRepository.upsert(auctions, { conflictPaths: ['marketplaceAuctionId', 'marketplaceKey'] });
-
-    // const saveOrUpdateResponse = await this.auctionsRepository
-    //   .createQueryBuilder()
-    //   .insert()
-    //   .into('auctions')
-    //   .values(auctions)
-    //   .orUpdate({
-    //     overwrite: [
-    //       'creationDate',
-    //       'modifiedDate',
-    //       'collection',
-    //       'nrAuctionedTokens',
-    //       'identifier',
-    //       'nonce',
-    //       'status',
-    //       'type',
-    //       'paymentToken',
-    //       'paymentNonce',
-    //       'ownerAddress',
-    //       'minBidDiff',
-    //       'minBid',
-    //       'minBidDenominated',
-    //       'maxBid',
-    //       'maxBidDenominated',
-    //       'startDate',
-    //       'endDate',
-    //       'tags',
-    //       'blockHash',
-    //     ],
-    //     conflict_target: ['marketplaceAuctionId', 'marketplaceKey'],
-    //   })
-    //   .updateEntity(false)
-    //   .execute();
-    // if (saveOrUpdateResponse.identifiers.length === 0 || auctions.findIndex((a) => a.id === undefined) !== -1) {
-    //   const dbAuctions = await this.getBulkAuctionsByMarketplaceAndAuctionIds(
-    //     auctions?.[0]?.marketplaceKey,
-    //     auctions?.map((a) => a.marketplaceAuctionId),
-    //   );
-    //   for (let i = 0; i < dbAuctions.length; i++) {
-    //     const auctionIndex = auctions.findIndex((a) => a.marketplaceAuctionId === dbAuctions[i].marketplaceAuctionId);
-    //     auctions[auctionIndex].id = dbAuctions[i].id;
-    //   }
-    // }
-    // if (auctions.findIndex((a) => a.id === undefined) !== -1) {
-    //   const wtf = auctions.filter((a) => a.id === undefined).map((a) => a.marketplaceAuctionId);
-    //   const duplicate = auctions.filter((a) => a.marketplaceAuctionId === 31434);
-    //   throw new Error(`oooppps ${JSON.stringify(duplicate)}`);
-    // }
+
+    const connection = this.auctionsRepository.manager.connection;
+
+    await connection.transaction(async (transactionalEntityManager) => {
+      const queryBuilder = transactionalEntityManager.createQueryBuilder().from(AuctionEntity, 'auction');
+
+      for (let i = 0; i < auctions.length; i += batchSize) {
+        const currentQueryBuilder = queryBuilder.clone();
+        const batch = auctions.slice(i, i + batchSize);
+        console.log('Processing batch number', i);
+
+        const cpu = new CpuProfiler();
+        for (const item of batch) {
+          currentQueryBuilder.insert().values(item).onConflict(`("marketplaceKey", "marketplaceAuctionId") DO UPDATE SET
+              "collection" = EXCLUDED."collection",
+              "nrAuctionedTokens" = EXCLUDED."nrAuctionedTokens",
+              "identifier" = EXCLUDED."identifier",
+              "nonce" = EXCLUDED."nonce",
+              "status" = EXCLUDED."status",
+              "type" = EXCLUDED."type",
+              "paymentToken" = EXCLUDED."paymentToken",
+              "paymentNonce" = EXCLUDED."paymentNonce",
+              "ownerAddress" = EXCLUDED."ownerAddress",
+              "minBidDiff" = EXCLUDED."minBidDiff",
+              "minBid" = EXCLUDED."minBid",
+              "minBidDenominated" = EXCLUDED."minBidDenominated",
+              "maxBid" = EXCLUDED."maxBid",
+              "maxBidDenominated" = EXCLUDED."maxBidDenominated",
+              "startDate" = EXCLUDED."startDate",
+              "endDate" = EXCLUDED."endDate",
+              "tags" = EXCLUDED."tags",
+              "blockHash" = EXCLUDED."blockHash"
+            `);
+
+          // Include related orders
+          if (item.orders && item.orders.length > 0) {
+            for (const order of item.orders) {
+              currentQueryBuilder.insert().into(OrderEntity).values({
+                priceToken: order.priceToken,
+                priceAmount: order.priceAmount,
+                priceAmountDenominated: order.priceAmountDenominated,
+                priceNonce: order.priceNonce,
+                status: order.status,
+                ownerAddress: order.ownerAddress,
+                boughtTokensNo: order.boughtTokensNo,
+                auctionId: order.auctionId,
+                blockHash: order.blockHash,
+                marketplaceKey: order.marketplaceKey,
+              }).onConflict(`("auctionId", "marketplaceKey") DO UPDATE SET
+                "priceToken" = EXCLUDED."priceToken",
+                "priceAmount" = EXCLUDED."priceAmount",
+                "priceAmountDenominated" = EXCLUDED."priceAmountDenominated",
+                "priceNonce" = EXCLUDED."priceNonce",
+                "status" = EXCLUDED."status",
+                "ownerAddress" = EXCLUDED."ownerAddress",
+                "boughtTokensNo" = EXCLUDED."boughtTokensNo",
+                "blockHash" = EXCLUDED."blockHash"
+              `);
+            }
+          }
+        }
+
+        await currentQueryBuilder.execute();
+        cpu.stop(`batch ${i}`);
+      }
+
+      console.log('Bulk insert or update completed successfully.');
+    });
   }
 
   async rollbackAuctionAndOrdersByHash(blockHash: string): Promise<any> {

diff --git a/src/modules/auctions/auctions-setter.service.ts b/src/modules/auctions/auctions-setter.service.ts
@@ -89,12 +89,8 @@ export class AuctionsSetterService {
     }
   }
 
-  async saveBulkAuctionsOrUpdateAndFillId(
-    auctions: AuctionEntity[],
-  ): Promise<void> {
-    return await this.persistenceService.saveBulkAuctionsOrUpdateAndFillId(
-      auctions,
-    );
+  async saveBulkAuctionsOrUpdateAndFillId(auctions: AuctionEntity[]): Promise<void> {
+    return await this.persistenceService.saveBulkAuctionsOrUpdateAndFillId(auctions);
   }
 
   async saveAuctionEntity(auctionEntity: AuctionEntity, assetTags: string[]): Promise<AuctionEntity> {

diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bid.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bid.handler.ts
@@ -10,18 +10,18 @@ export class ReindexAuctionBidHandler {
   constructor() {}
 
   handle(marketplaceReindexState: MarketplaceReindexState, input: AuctionBidSummary, paymentToken: Token, paymentNonce: number): void {
-    const auctionIndex = marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId);
+    const auction = marketplaceReindexState.auctionMap.get(input.auctionId);
 
-    if (auctionIndex === -1) {
+    if (!auction) {
       return;
     }
 
-    let order = marketplaceReindexState.createOrder(auctionIndex, input, OrderStatusEnum.Active, paymentToken, paymentNonce);
-    if (order.priceAmount === marketplaceReindexState.auctions[auctionIndex].maxBid) {
+    let order = marketplaceReindexState.createOrder(auction, input, OrderStatusEnum.Active, paymentToken, paymentNonce);
+    if (order.priceAmount === auction.maxBid) {
       order.status = OrderStatusEnum.Bought;
-      marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Ended, input.timestamp);
+      marketplaceReindexState.updateAuctionStatus(auction, input.blockHash, AuctionStatusEnum.Ended, input.timestamp);
     }
 
-    marketplaceReindexState.updateOrderListForAuction(auctionIndex, order);
+    marketplaceReindexState.updateOrderListForAuction(auction, order);
   }
 }
diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts
@@ -12,27 +12,26 @@ export class ReindexAuctionBoughtHandler {
   constructor() {}
 
   handle(marketplaceReindexState: MarketplaceReindexState, input: AuctionBuySummary, paymentToken: Token, paymentNonce: number): void {
-    const auctionIndex =
+    const auction =
       marketplaceReindexState.marketplace.key !== ELRONDNFTSWAP_KEY
-        ? marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId)
-        : marketplaceReindexState.getAuctionIndexByIdentifier(input.identifier);
+        ? marketplaceReindexState.auctionMap.get(input.auctionId)
+        : marketplaceReindexState.auctionMap.get(input.auctionId); //de scris
 
-    if (auctionIndex === -1) {
+    if (!auction) {
       return;
     }
 
-    const order = marketplaceReindexState.createOrder(auctionIndex, input, OrderStatusEnum.Bought, paymentToken, paymentNonce);
-    const auction = marketplaceReindexState.auctions[auctionIndex];
+    const order = marketplaceReindexState.createOrder(auction, input, OrderStatusEnum.Bought, paymentToken, paymentNonce);
 
     if (auction.nrAuctionedTokens > 1) {
       const totalBought = this.getTotalBoughtTokensForAuction(auction.id, auction.orders);
 
       if (auction.nrAuctionedTokens === totalBought) {
-        marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Ended, input.timestamp);
+        marketplaceReindexState.updateAuctionStatus(auction, input.blockHash, AuctionStatusEnum.Ended, input.timestamp);
       }
     }
 
-    marketplaceReindexState.updateOrderListForAuction(auctionIndex, order);
+    marketplaceReindexState.updateOrderListForAuction(auction, order);
   }
 
   private getTotalBoughtTokensForAuction(auctionId: number, orders: OrderEntity[]): number {

diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-closed.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-closed.handler.ts
@@ -10,18 +10,18 @@ export class ReindexAuctionClosedHandler {
   constructor() {}
 
   handle(marketplaceReindexState: MarketplaceReindexState, input: ReindexAuctionClosedSummary): void {
-    const auctionIndex =
+    const auction =
       marketplaceReindexState.marketplace.key !== ELRONDNFTSWAP_KEY
-        ? marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId)
-        : marketplaceReindexState.getAuctionIndexByIdentifier(input.identifier);
+        ? marketplaceReindexState.auctionMap.get(input.auctionId)
+        : marketplaceReindexState.auctionMap.get(input.auctionId); //de scris
     const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp);
 
-    if (auctionIndex === -1) {
+    if (!auction) {
       return;
     }
 
-    marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Closed, input.timestamp);
+    marketplaceReindexState.updateAuctionStatus(auction, input.blockHash, AuctionStatusEnum.Closed, input.timestamp);
 
-    marketplaceReindexState.setInactiveOrdersForAuction(auctionIndex, modifiedDate);
+    marketplaceReindexState.setInactiveOrdersForAuction(auction, modifiedDate);
   }
 }
diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-ended.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-ended.handler.ts
@@ -11,30 +11,25 @@ export class ReindexAuctionEndedHandler {
   constructor() {}
 
   handle(marketplaceReindexState: MarketplaceReindexState, input: AuctionEndedSummary, paymentToken: Token): void {
-    const auctionIndex = marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId);
+    const auction = marketplaceReindexState.auctionMap.get(input.auctionId);
     const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp);
 
-    if (auctionIndex === -1) {
+    if (!auction) {
       return;
     }
 
-    marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Ended, input.timestamp);
-    const selectedAuction = marketplaceReindexState.auctions[auctionIndex];
+    marketplaceReindexState.updateAuctionStatus(auction, input.blockHash, AuctionStatusEnum.Ended, input.timestamp);
 
-    const winnerOrderId = marketplaceReindexState.setAuctionOrderWinnerStatusAndReturnId(
-      selectedAuction.id,
-      OrderStatusEnum.Bought,
-      modifiedDate,
-    );
+    const winnerOrderId = marketplaceReindexState.setAuctionOrderWinnerStatusAndReturnId(auction, OrderStatusEnum.Bought, modifiedDate);
 
     if (winnerOrderId !== -1) {
-      marketplaceReindexState.setInactiveOrdersForAuction(selectedAuction.id, modifiedDate, winnerOrderId);
+      marketplaceReindexState.setInactiveOrdersForAuction(auction, modifiedDate, winnerOrderId);
     } else if (input.currentBid !== '0') {
-      const order = marketplaceReindexState.createOrder(auctionIndex, input, OrderStatusEnum.Bought, paymentToken);
-      if (marketplaceReindexState.auctions[auctionIndex].orders) {
-        marketplaceReindexState.auctions[auctionIndex].orders.push(order);
+      const order = marketplaceReindexState.createOrder(auction, input, OrderStatusEnum.Bought, paymentToken);
+      if (auction.orders) {
+        marketplaceReindexState.auctionMap.get(auction.marketplaceAuctionId).orders.push(order);
       } else {
-        marketplaceReindexState.auctions[auctionIndex].orders = [order];
+        marketplaceReindexState.auctionMap.get(auction.marketplaceAuctionId).orders = [order];
       }
     }
   }

diff --git a/...dules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts b/...dules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts
@@ -10,14 +10,13 @@ export class ReindexAuctionPriceUpdatedHandler {
   constructor() {}
 
   handle(marketplaceReindexState: MarketplaceReindexState, input: AuctionPriceUpdatedSummary, decimals: number): void {
-    const auctionIndex = marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId);
+    const auction = marketplaceReindexState.auctionMap.get(input.auctionId);
 
-    if (auctionIndex === -1) {
+    if (!auction) {
       return;
     }
 
     const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp);
-    const auction = marketplaceReindexState.auctions[auctionIndex];
 
     auction.blockHash = auction.blockHash ?? input.blockHash;
     auction.modifiedDate = modifiedDate;

diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts
@@ -23,13 +23,11 @@ export class ReindexAuctionStartedHandler {
     const minBidDenominated = BigNumberUtils.denominateAmount(input.minBid, paymentToken.decimals);
     const maxBidDenominated = BigNumberUtils.denominateAmount(input.maxBid !== 'NaN' ? input.maxBid : '0', paymentToken.decimals);
 
-    marketplaceReindexState.deleteAuctionIfDuplicates(input.auctionId);
-
     const auction = new AuctionEntity({
       creationDate: modifiedDate,
       modifiedDate,
       id: marketplaceReindexState.getNewAuctionId(),
-      marketplaceAuctionId: input.auctionId !== 0 ? input.auctionId : marketplaceReindexState.auctions.length + 1,
+      marketplaceAuctionId: input.auctionId !== 0 ? input.auctionId : marketplaceReindexState.auctionMap.size + 1,
       identifier: input.identifier,
       collection: input.collection,
       nonce: nonce,
@@ -51,6 +49,6 @@ export class ReindexAuctionStartedHandler {
       marketplaceKey: marketplaceReindexState.marketplace.key,
     });
 
-    marketplaceReindexState.auctions.push(auction);
+    marketplaceReindexState.auctionMap.set(auction.marketplaceAuctionId, auction);
   }
 }
diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-updated.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-updated.handler.ts
@@ -11,34 +11,28 @@ export class ReindexAuctionUpdatedHandler {
   constructor() {}
 
   handle(marketplaceReindexState: MarketplaceReindexState, input: AuctionUpdatedSummary, decimals: number): void {
-    const auctionIndex = marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId);
+    const auction = marketplaceReindexState.auctionMap.get(input.auctionId);
     const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp);
 
-    if (auctionIndex === -1) {
+    if (!auction) {
       return;
     }
 
-    marketplaceReindexState.auctions[auctionIndex].blockHash = marketplaceReindexState.auctions[auctionIndex].blockHash ?? input.blockHash;
-    marketplaceReindexState.auctions[auctionIndex].modifiedDate = modifiedDate;
+    auction.blockHash = auction.blockHash ?? input.blockHash;
+    auction.modifiedDate = modifiedDate;
 
-    marketplaceReindexState.auctions[auctionIndex].minBid = input.minBid;
-    marketplaceReindexState.auctions[auctionIndex].minBidDenominated = Math.min(
-      BigNumberUtils.denominateAmount(input.minBid, decimals),
-      constants.dbMaxDenominatedValue,
-    );
-    marketplaceReindexState.auctions[auctionIndex].maxBid = input.minBid;
-    marketplaceReindexState.auctions[auctionIndex].maxBidDenominated = Math.min(
-      marketplaceReindexState.auctions[auctionIndex].minBidDenominated,
-      constants.dbMaxDenominatedValue,
-    );
+    auction.minBid = input.minBid;
+    auction.minBidDenominated = Math.min(BigNumberUtils.denominateAmount(input.minBid, decimals), constants.dbMaxDenominatedValue);
+    auction.maxBid = input.minBid;
+    auction.maxBidDenominated = Math.min(auction.minBidDenominated, constants.dbMaxDenominatedValue);
 
     if (input.paymentToken) {
-      marketplaceReindexState.auctions[auctionIndex].paymentToken = input.paymentToken;
-      marketplaceReindexState.auctions[auctionIndex].paymentNonce = BinaryUtils.hexToNumber(input.paymentNonce);
+      auction.paymentToken = input.paymentToken;
+      auction.paymentNonce = BinaryUtils.hexToNumber(input.paymentNonce);
     }
 
     if (input.deadline > 0) {
-      marketplaceReindexState.auctions[auctionIndex].endDate = input.deadline;
+      auction.endDate = input.deadline;
     }
   }
 }
diff --git a/...dules/marketplaces/marketplaces-reindex-handlers/reindex-global-offer-accepted.handler.ts b/...dules/marketplaces/marketplaces-reindex-handlers/reindex-global-offer-accepted.handler.ts
@@ -10,13 +10,13 @@ export class ReindexGlobalOfferAcceptedHandler {
 
   handle(marketplaceReindexState: MarketplaceReindexState, input: GlobalOfferAcceptedSummary): void {
     const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp);
-    const auctionIndex = marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId);
+    const auction = marketplaceReindexState.auctionMap.get(input.auctionId);
 
-    if (auctionIndex === -1) {
+    if (!auction) {
       return;
     }
 
-    marketplaceReindexState.auctions[auctionIndex].status = AuctionStatusEnum.Closed;
-    marketplaceReindexState.auctions[auctionIndex].modifiedDate = modifiedDate;
+    auction.status = AuctionStatusEnum.Closed;
+    auction.modifiedDate = modifiedDate;
   }
 }