Merge pull request #1056 from multiversx/development

Development

src/common/persistence/persistence.service.ts

@@ -732,11 +732,15 @@ export class PersistenceService {
   }
 
   async getMarketplaceEventsAsc(
-    marketplaceKey: string,
+    marketplaceAddress: string,
     afterTimestamp?: number,
     beforeTimestamp?: number,
   ): Promise<MarketplaceEventsEntity[]> {
-    return await this.marketplaceEventsRepository.getEventsByMarketplaceAndTimestampsAsc(marketplaceKey, afterTimestamp, beforeTimestamp);
+    return await this.marketplaceEventsRepository.getEventsByMarketplaceAndTimestampsAsc(
+      marketplaceAddress,
+      afterTimestamp,
+      beforeTimestamp,
+    );
   }
 
   async getBlacklistedCollections(): Promise<[BlacklistedCollectionEntity[], number]> {

src/config/default.json

@@ -74,7 +74,9 @@
     "getLogsFromElasticBatchSize": 1000,
     "dbMaxTimestamp": 2147483647,
     "defaultPageOffset": 0,
-    "defaultPageSize": 10
+    "defaultPageSize": 10,
+    "dbMaxDenominatedValue": 99999999999999999,
+    "dbMaxTagLength": 20
   },
   "elasticDictionary": {
     "scamInfo": {

src/modules/campaigns/campaigns-caching.service.ts

@@ -1,14 +1,15 @@
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 import '../../utils/extensions';
 import { Constants } from '@multiversx/sdk-nestjs-common';
 import { CacheService } from '@multiversx/sdk-nestjs-cache';
 import { CollectionType } from '../assets/models/Collection.type';
 import { CacheInfo } from 'src/common/services/caching/entities/cache.info';
 import { Campaign } from './models';
+import { ClientProxy } from '@nestjs/microservices';
 
 @Injectable()
 export class CampaignsCachingService {
-  constructor(private cacheService: CacheService) {}
+  constructor(private cacheService: CacheService, @Inject('PUBSUB_SERVICE') private clientProxy: ClientProxy) {}
 
   public async getAllMarketplaces(getCampaignsFromDb: () => any): Promise<CollectionType<Campaign>> {
     return await this.cacheService.getOrSet(CacheInfo.Campaigns.key, () => getCampaignsFromDb(), Constants.oneHour());
@@ -24,5 +25,16 @@ export class CampaignsCachingService {
 
   public async invalidateCache() {
     await this.cacheService.deleteInCache(CacheInfo.Campaigns.key);
+    await this.refreshCacheKey(CacheInfo.Campaigns.key, CacheInfo.MarketplaceAddressCollection.ttl);
+  }
+
+  private async refreshCacheKey(key: string, ttl: number) {
+    this.clientProxy.emit<{
+      key: string;
+      ttl: number;
+    }>('refreshCacheKey', {
+      key,
+      ttl,
+    });
   }
 }

src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts

@@ -1,4 +1,5 @@
 import { Injectable } from '@nestjs/common';
+import { constants } from 'src/config';
 import { BigNumberUtils } from 'src/utils/bigNumber-utils';
 import { DateUtils } from 'src/utils/date-utils';
 import { MarketplaceReindexState } from '../models/MarketplaceReindexState';
@@ -19,11 +20,17 @@ export class ReindexAuctionPriceUpdatedHandler {
     marketplaceReindexState.auctions[auctionIndex].modifiedDate = modifiedDate;
 
     marketplaceReindexState.auctions[auctionIndex].minBid = input.minBid;
-    marketplaceReindexState.auctions[auctionIndex].minBidDenominated = BigNumberUtils.denominateAmount(input.minBid, decimals);
+    marketplaceReindexState.auctions[auctionIndex].minBidDenominated = Math.min(
+      BigNumberUtils.denominateAmount(input.minBid, decimals),
+      constants.dbMaxDenominatedValue,
+    );
 
     if (input.maxBid) {
       marketplaceReindexState.auctions[auctionIndex].maxBid = input.maxBid;
-      marketplaceReindexState.auctions[auctionIndex].maxBidDenominated = BigNumberUtils.denominateAmount(input.maxBid, decimals);
+      marketplaceReindexState.auctions[auctionIndex].maxBidDenominated = Math.min(
+        BigNumberUtils.denominateAmount(input.maxBid, decimals),
+        constants.dbMaxDenominatedValue,
+      );
     } else {
       marketplaceReindexState.auctions[auctionIndex].maxBid = input.minBid;
       marketplaceReindexState.auctions[auctionIndex].maxBidDenominated = marketplaceReindexState.auctions[auctionIndex].minBidDenominated;

src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts

@@ -1,5 +1,6 @@
 import { BinaryUtils } from '@multiversx/sdk-nestjs-common';
 import { Injectable } from '@nestjs/common';
+import { constants } from 'src/config';
 import { AuctionEntity } from 'src/db/auctions';
 import { AuctionStatusEnum } from 'src/modules/auctions/models';
 import { Token } from 'src/modules/usdPrice/Token.model';
@@ -17,6 +18,10 @@ export class ReindexAuctionStartedHandler {
     const itemsCount = parseInt(input.itemsCount);
     const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp);
     const startTime = Number.isNaN(input.startTime) ? input.timestamp : input.startTime;
+    const endTime = input.endTime > 0 ? input.endTime : 0;
+    const minBidDenominated = BigNumberUtils.denominateAmount(input.minBid, paymentToken.decimals);
+    const maxBidDenominated = BigNumberUtils.denominateAmount(input.maxBid !== 'NaN' ? input.maxBid : '0', paymentToken.decimals);
+
     const auction = new AuctionEntity({
       creationDate: modifiedDate,
       modifiedDate,
@@ -33,11 +38,11 @@ export class ReindexAuctionStartedHandler {
       ownerAddress: input.sender,
       minBid: input.minBid,
       maxBid: input.maxBid !== 'NaN' ? input.maxBid : '0',
-      minBidDenominated: BigNumberUtils.denominateAmount(input.minBid, paymentToken.decimals),
-      maxBidDenominated: BigNumberUtils.denominateAmount(input.maxBid !== 'NaN' ? input.maxBid : '0', paymentToken.decimals),
+      minBidDenominated: Math.min(minBidDenominated, constants.dbMaxDenominatedValue),
+      maxBidDenominated: Math.min(maxBidDenominated, constants.dbMaxDenominatedValue),
       minBidDiff: input.minBidDiff ?? '0',
-      startDate: startTime,
-      endDate: input.endTime > 0 ? input.endTime : 0,
+      startDate: Math.min(startTime, constants.dbMaxTimestamp),
+      endDate: Math.min(endTime, constants.dbMaxTimestamp),
       tags: '',
       blockHash: input.blockHash ?? '',
       marketplaceKey: marketplaceReindexState.marketplace.key,

src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-updated.handler.ts

@@ -1,5 +1,6 @@
 import { BinaryUtils } from '@multiversx/sdk-nestjs-common';
 import { Injectable } from '@nestjs/common';
+import { constants } from 'src/config';
 import { BigNumberUtils } from 'src/utils/bigNumber-utils';
 import { DateUtils } from 'src/utils/date-utils';
 import { MarketplaceReindexState } from '../models/MarketplaceReindexState';
@@ -21,9 +22,15 @@ export class ReindexAuctionUpdatedHandler {
     marketplaceReindexState.auctions[auctionIndex].modifiedDate = modifiedDate;
 
     marketplaceReindexState.auctions[auctionIndex].minBid = input.minBid;
-    marketplaceReindexState.auctions[auctionIndex].minBidDenominated = BigNumberUtils.denominateAmount(input.minBid, decimals);
+    marketplaceReindexState.auctions[auctionIndex].minBidDenominated = Math.min(
+      BigNumberUtils.denominateAmount(input.minBid, decimals),
+      constants.dbMaxDenominatedValue,
+    );
     marketplaceReindexState.auctions[auctionIndex].maxBid = input.minBid;
-    marketplaceReindexState.auctions[auctionIndex].maxBidDenominated = marketplaceReindexState.auctions[auctionIndex].minBidDenominated;
+    marketplaceReindexState.auctions[auctionIndex].maxBidDenominated = Math.min(
+      marketplaceReindexState.auctions[auctionIndex].minBidDenominated,
+      constants.dbMaxDenominatedValue,
+    );
 
     if (input.paymentToken) {
       marketplaceReindexState.auctions[auctionIndex].paymentToken = input.paymentToken;

src/modules/marketplaces/marketplaces-reindex-handlers/reindex-offer-created.hander.ts

@@ -1,4 +1,5 @@
 import { Injectable } from '@nestjs/common';
+import { constants } from 'src/config';
 import { OfferEntity } from 'src/db/offers';
 import { OfferStatusEnum } from 'src/modules/offers/models';
 import { BigNumberUtils } from 'src/utils/bigNumber-utils';
@@ -12,6 +13,7 @@ export class ReindexOfferCreatedHandler {
 
   handle(marketplaceReindexState: MarketplaceReindexState, input: OfferCreatedSummary, decimals: number): void {
     const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp);
+    const priceAmountDenominated = BigNumberUtils.denominateAmount(input.price, decimals);
     const offer = new OfferEntity({
       id: marketplaceReindexState.offers.length,
       creationDate: modifiedDate,
@@ -23,9 +25,9 @@ export class ReindexOfferCreatedHandler {
       priceToken: input.paymentToken,
       priceNonce: input.paymentNonce,
       priceAmount: input.price,
-      priceAmountDenominated: BigNumberUtils.denominateAmount(input.price, decimals),
+      priceAmountDenominated: Math.min(priceAmountDenominated, constants.dbMaxDenominatedValue),
       ownerAddress: input.address,
-      endDate: input.endTime,
+      endDate: Math.min(input.endTime, constants.dbMaxTimestamp),
       boughtTokensNo: input.itemsCount,
       marketplaceKey: marketplaceReindexState.marketplace.key,
       status: OfferStatusEnum.Active,

src/modules/marketplaces/marketplaces-reindex.service.ts

@@ -93,7 +93,7 @@ export class MarketplacesReindexService {
       return [
         new MarketplaceReindexState({
           marketplace,
-          isReindexFromTheBeginning: input.afterTimestamp ? false : true,
+          isFullStateInMemory: input.afterTimestamp ? false : true,
         }),
       ];
     }
@@ -106,7 +106,7 @@ export class MarketplacesReindexService {
       marketplaceReindexStates.push(
         new MarketplaceReindexState({
           marketplace: internalMarketplaces[i],
-          isReindexFromTheBeginning: input.afterTimestamp ? false : true,
+          isFullStateInMemory: input.afterTimestamp ? false : true,
           listedCollections: marketplaceCollections,
         }),
       );
@@ -138,6 +138,13 @@ export class MarketplacesReindexService {
         marketplaceReindexStates,
         processInNextBatch.concat(batch),
       );
+
+      processInNextBatch = await this.processEventsBatchAndReturnUnprocessedEvents(
+        marketplaceReindexStates,
+        processInNextBatch.concat(batch),
+      );
+
+      await this.addInactiveStateItemsToDb(marketplaceReindexStates);
     } while (input.beforeTimestamp ? afterTimestamp < input.beforeTimestamp : true);
 
     const isFinalBatch = true;
@@ -209,7 +216,7 @@ export class MarketplacesReindexService {
       eventOrdersAndTx,
     );
 
-    if (!marketplaceReindexStates[0].isReindexFromTheBeginning) {
+    if (!marketplaceReindexStates[0].isFullStateInMemory) {
       await Promise.all(
         marketplaceReindexStates.map((state) => {
           return this.getStateFromDbIfMissing(state, eventsSetSummaries);
@@ -402,15 +409,60 @@ export class MarketplacesReindexService {
     }
   }
 
+  private async addInactiveStateItemsToDb(marketplaceReindexStates: MarketplaceReindexState[]): Promise<void> {
+    for (const marketplaceReindexState of marketplaceReindexStates) {
+      marketplaceReindexState.setStateItemsToExpiredIfOlderThanTimestamp(DateUtils.getCurrentTimestamp());
+
+      const [inactiveAuctions, inactiveOrders, inactiveOffers] = marketplaceReindexState.popInactiveItems();
+
+      await this.populateAuctionAssetTags(inactiveAuctions);
+
+      for (let i = 0; i < inactiveOrders.length; i++) {
+        inactiveOrders[i].auctionId = inactiveAuctions.findIndex((a) => a.id === inactiveOrders[i].auctionId);
+        delete inactiveOrders[i].id;
+      }
+
+      for (let i = 0; i < inactiveAuctions.length; i++) {
+        delete inactiveAuctions[i].id;
+      }
+
+      await this.auctionSetterService.saveBulkAuctions(inactiveAuctions);
+
+      let tags: TagEntity[] = [];
+      inactiveAuctions.map((auction) => {
+        const assetTags = auction.tags.split(',');
+        assetTags.map((assetTag) => {
+          if (assetTag !== '') {
+            tags.push(
+              new TagEntity({
+                auctionId: auction.id,
+                tag: assetTag.trim().slice(0, constants.dbMaxTagLength),
+                auction: auction,
+              }),
+            );
+          }
+        });
+      });
+
+      const saveTagsPromise = this.persistenceService.saveTags(tags);
+
+      for (let i = 0; i < inactiveOrders.length; i++) {
+        inactiveOrders[i].auction = inactiveAuctions[inactiveOrders[i].auctionId];
+        inactiveOrders[i].auctionId = inactiveAuctions[inactiveOrders[i].auctionId].id;
+      }
+      inactiveOffers.map((o) => delete o.id);
+
+      await Promise.all([
+        saveTagsPromise,
+        this.persistenceService.saveBulkOrders(inactiveOrders),
+        this.persistenceService.saveBulkOffers(inactiveOffers),
+      ]);
+    }
+  }
+
   private async addMarketplaceStateToDb(marketplaceReindexState: MarketplaceReindexState): Promise<void> {
     marketplaceReindexState.auctions.map((a) => {
       delete a.id;
-      if (a.startDate > constants.dbMaxTimestamp) {
-        a.startDate = constants.dbMaxTimestamp;
-      }
-      if (a.endDate > constants.dbMaxTimestamp) {
-        a.endDate = constants.dbMaxTimestamp;
-      }
     });
     marketplaceReindexState.orders.map((o) => delete o.id);
     marketplaceReindexState.offers.map((o) => delete o.id);
@@ -430,17 +482,19 @@ export class MarketplacesReindexService {
           tags.push(
             new TagEntity({
               auctionId: auction.id,
-              tag: assetTag.trim().slice(0, 20),
+              tag: assetTag.trim().slice(0, constants.dbMaxTagLength),
               auction: auction,
             }),
           );
         }
       });
     });
 
-    await this.persistenceService.saveTags(tags);
-    await this.persistenceService.saveBulkOrders(marketplaceReindexState.orders);
-    await this.persistenceService.saveBulkOffers(marketplaceReindexState.offers);
+    await Promise.all([
+      this.persistenceService.saveBulkOrders(marketplaceReindexState.orders),
+      this.persistenceService.saveBulkOffers(marketplaceReindexState.offers),
+      this.persistenceService.saveTags(tags),
+    ]);
   }
 
   private async populateAuctionAssetTags(auctions: AuctionEntity[]): Promise<void> {

src/modules/marketplaces/models/MarketplaceReindexState.ts

@@ -14,7 +14,7 @@ import { Marketplace } from './Marketplace.dto';
 @ObjectType()
 export class MarketplaceReindexState {
   marketplace: Marketplace;
-  isReindexFromTheBeginning: boolean;
+  isFullStateInMemory: boolean;
   listedCollections: string[] = [];
   auctions: AuctionEntity[] = [];
   orders: OrderEntity[] = [];
@@ -94,6 +94,41 @@ export class MarketplaceReindexState {
     this.setOffersToExpiredIfOlderThanTimestamp(timestamp);
   }
 
+  popInactiveItems(): [AuctionEntity[], OrderEntity[], OfferEntity[]] {
+    const inactiveAuctionStatuses = [AuctionStatusEnum.Closed, AuctionStatusEnum.Ended];
+    const inactiveOfferStatuses = [OfferStatusEnum.Accepted, OfferStatusEnum.Closed, OfferStatusEnum.Expired];
+
+    let inactiveAuctions = [];
+    let inactiveOrders = [];
+    let inactiveOffers = [];
+
+    for (let i = 0; i < this.auctions.length; i++) {
+      if (inactiveAuctionStatuses.includes(this.auctions[i].status)) {
+        inactiveAuctions.push(this.auctions[i]);
+        delete this.auctions[i];
+      }
+    }
+    this.auctions = this.auctions.filter((a) => a);
+
+    for (let i = 0; i < this.orders.length; i++) {
+      if (inactiveAuctions.findIndex((a) => a.id === this.orders[i].auctionId) !== -1) {
+        inactiveOrders.push(this.orders[i]);
+        delete this.orders[i];
+      }
+    }
+    this.orders = this.orders.filter((o) => o);
+
+    for (let i = 0; i < this.offers.length; i++) {
+      if (inactiveOfferStatuses.includes(this.offers[i].status)) {
+        inactiveOffers.push(this.offers[i]);
+        delete this.offers[i];
+      }
+    }
+    this.offers = this.offers.filter((o) => o);
+
+    return [inactiveAuctions, inactiveOrders, inactiveOffers];
+  }
+
   private setAuctionsAndOrdersToExpiredIfOlderThanTimestamp(timestamp: number): void {
     const runningAuctions = this.auctions?.filter((a) => a.status === AuctionStatusEnum.Running);
     for (let i = 0; i < runningAuctions.length; i++) {

src/modules/marketplaces/models/marketplaces-reindex-events-summaries/AuctionStartedSummary.ts

@@ -42,7 +42,7 @@ export class AuctionStartedSummary extends ReindexGenericSummary {
     const address = event.data.eventData?.address ?? tx.receiver;
     const topics = this.getTopics(event);
 
-    if (!topics || (!topics.price && !topics.minBid)) {
+    if (!topics || (!topics.price && !topics.minBid) || !topics.auctionType) {
       return;
     }
 
@@ -60,7 +60,7 @@ export class AuctionStartedSummary extends ReindexGenericSummary {
       minBid: topics.minBid ?? topics.price,
       maxBid: topics.maxBid ?? '0',
       minBidDiff: minBidDiff,
-      startTime: topics.startTime,
+      startTime: topics.startTime ?? event.timestamp,
       endTime: topics.endTime ?? topics.deadline ?? 0,
       paymentToken: topics.paymentToken,
       paymentNonce: topics.paymentNonce ?? topics.paymentTokenNonce ?? 0,