diff --git a/src/common/persistence/persistence.service.ts b/src/common/persistence/persistence.service.ts index ea159ab96..64061323d 100644 --- a/src/common/persistence/persistence.service.ts +++ b/src/common/persistence/persistence.service.ts @@ -732,11 +732,15 @@ export class PersistenceService { } async getMarketplaceEventsAsc( - marketplaceKey: string, + marketplaceAddress: string, afterTimestamp?: number, beforeTimestamp?: number, ): Promise { - return await this.marketplaceEventsRepository.getEventsByMarketplaceAndTimestampsAsc(marketplaceKey, afterTimestamp, beforeTimestamp); + return await this.marketplaceEventsRepository.getEventsByMarketplaceAndTimestampsAsc( + marketplaceAddress, + afterTimestamp, + beforeTimestamp, + ); } async getBlacklistedCollections(): Promise<[BlacklistedCollectionEntity[], number]> { diff --git a/src/config/default.json b/src/config/default.json index fe4c3ec89..caaf03bfa 100644 --- a/src/config/default.json +++ b/src/config/default.json @@ -74,7 +74,9 @@ "getLogsFromElasticBatchSize": 1000, "dbMaxTimestamp": 2147483647, "defaultPageOffset": 0, - "defaultPageSize": 10 + "defaultPageSize": 10, + "dbMaxDenominatedValue": 99999999999999999, + "dbMaxTagLength": 20 }, "elasticDictionary": { "scamInfo": { diff --git a/src/modules/campaigns/campaigns-caching.service.ts b/src/modules/campaigns/campaigns-caching.service.ts index f107e7dc4..d3b080ca2 100644 --- a/src/modules/campaigns/campaigns-caching.service.ts +++ b/src/modules/campaigns/campaigns-caching.service.ts @@ -1,14 +1,15 @@ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import '../../utils/extensions'; import { Constants } from '@multiversx/sdk-nestjs-common'; import { CacheService } from '@multiversx/sdk-nestjs-cache'; import { CollectionType } from '../assets/models/Collection.type'; import { CacheInfo } from 'src/common/services/caching/entities/cache.info'; import { Campaign } from './models'; +import { ClientProxy } from '@nestjs/microservices'; @Injectable() export class CampaignsCachingService { - constructor(private cacheService: CacheService) {} + constructor(private cacheService: CacheService, @Inject('PUBSUB_SERVICE') private clientProxy: ClientProxy) {} public async getAllMarketplaces(getCampaignsFromDb: () => any): Promise> { return await this.cacheService.getOrSet(CacheInfo.Campaigns.key, () => getCampaignsFromDb(), Constants.oneHour()); @@ -24,5 +25,16 @@ export class CampaignsCachingService { public async invalidateCache() { await this.cacheService.deleteInCache(CacheInfo.Campaigns.key); + await this.refreshCacheKey(CacheInfo.Campaigns.key, CacheInfo.MarketplaceAddressCollection.ttl); + } + + private async refreshCacheKey(key: string, ttl: number) { + this.clientProxy.emit<{ + key: string; + ttl: number; + }>('refreshCacheKey', { + key, + ttl, + }); } } diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts index 72c37b4b4..8b2f513a1 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts @@ -1,4 +1,5 @@ import { Injectable } from '@nestjs/common'; +import { constants } from 'src/config'; import { BigNumberUtils } from 'src/utils/bigNumber-utils'; import { DateUtils } from 'src/utils/date-utils'; import { MarketplaceReindexState } from '../models/MarketplaceReindexState'; @@ -19,11 +20,17 @@ export class ReindexAuctionPriceUpdatedHandler { marketplaceReindexState.auctions[auctionIndex].modifiedDate = modifiedDate; marketplaceReindexState.auctions[auctionIndex].minBid = input.minBid; - marketplaceReindexState.auctions[auctionIndex].minBidDenominated = BigNumberUtils.denominateAmount(input.minBid, decimals); + marketplaceReindexState.auctions[auctionIndex].minBidDenominated = Math.min( + BigNumberUtils.denominateAmount(input.minBid, decimals), + constants.dbMaxDenominatedValue, + ); if (input.maxBid) { marketplaceReindexState.auctions[auctionIndex].maxBid = input.maxBid; - marketplaceReindexState.auctions[auctionIndex].maxBidDenominated = BigNumberUtils.denominateAmount(input.maxBid, decimals); + marketplaceReindexState.auctions[auctionIndex].maxBidDenominated = Math.min( + BigNumberUtils.denominateAmount(input.maxBid, decimals), + constants.dbMaxDenominatedValue, + ); } else { marketplaceReindexState.auctions[auctionIndex].maxBid = input.minBid; marketplaceReindexState.auctions[auctionIndex].maxBidDenominated = marketplaceReindexState.auctions[auctionIndex].minBidDenominated; diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts index 9f39d20f3..a93351c5f 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts @@ -1,5 +1,6 @@ import { BinaryUtils } from '@multiversx/sdk-nestjs-common'; import { Injectable } from '@nestjs/common'; +import { constants } from 'src/config'; import { AuctionEntity } from 'src/db/auctions'; import { AuctionStatusEnum } from 'src/modules/auctions/models'; import { Token } from 'src/modules/usdPrice/Token.model'; @@ -17,6 +18,10 @@ export class ReindexAuctionStartedHandler { const itemsCount = parseInt(input.itemsCount); const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); const startTime = Number.isNaN(input.startTime) ? input.timestamp : input.startTime; + const endTime = input.endTime > 0 ? input.endTime : 0; + const minBidDenominated = BigNumberUtils.denominateAmount(input.minBid, paymentToken.decimals); + const maxBidDenominated = BigNumberUtils.denominateAmount(input.maxBid !== 'NaN' ? input.maxBid : '0', paymentToken.decimals); + const auction = new AuctionEntity({ creationDate: modifiedDate, modifiedDate, @@ -33,11 +38,11 @@ export class ReindexAuctionStartedHandler { ownerAddress: input.sender, minBid: input.minBid, maxBid: input.maxBid !== 'NaN' ? input.maxBid : '0', - minBidDenominated: BigNumberUtils.denominateAmount(input.minBid, paymentToken.decimals), - maxBidDenominated: BigNumberUtils.denominateAmount(input.maxBid !== 'NaN' ? input.maxBid : '0', paymentToken.decimals), + minBidDenominated: Math.min(minBidDenominated, constants.dbMaxDenominatedValue), + maxBidDenominated: Math.min(maxBidDenominated, constants.dbMaxDenominatedValue), minBidDiff: input.minBidDiff ?? '0', - startDate: startTime, - endDate: input.endTime > 0 ? input.endTime : 0, + startDate: Math.min(startTime, constants.dbMaxTimestamp), + endDate: Math.min(endTime, constants.dbMaxTimestamp), tags: '', blockHash: input.blockHash ?? '', marketplaceKey: marketplaceReindexState.marketplace.key, diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-updated.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-updated.handler.ts index 0317e66bd..8b0350795 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-updated.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-updated.handler.ts @@ -1,5 +1,6 @@ import { BinaryUtils } from '@multiversx/sdk-nestjs-common'; import { Injectable } from '@nestjs/common'; +import { constants } from 'src/config'; import { BigNumberUtils } from 'src/utils/bigNumber-utils'; import { DateUtils } from 'src/utils/date-utils'; import { MarketplaceReindexState } from '../models/MarketplaceReindexState'; @@ -21,9 +22,15 @@ export class ReindexAuctionUpdatedHandler { marketplaceReindexState.auctions[auctionIndex].modifiedDate = modifiedDate; marketplaceReindexState.auctions[auctionIndex].minBid = input.minBid; - marketplaceReindexState.auctions[auctionIndex].minBidDenominated = BigNumberUtils.denominateAmount(input.minBid, decimals); + marketplaceReindexState.auctions[auctionIndex].minBidDenominated = Math.min( + BigNumberUtils.denominateAmount(input.minBid, decimals), + constants.dbMaxDenominatedValue, + ); marketplaceReindexState.auctions[auctionIndex].maxBid = input.minBid; - marketplaceReindexState.auctions[auctionIndex].maxBidDenominated = marketplaceReindexState.auctions[auctionIndex].minBidDenominated; + marketplaceReindexState.auctions[auctionIndex].maxBidDenominated = Math.min( + marketplaceReindexState.auctions[auctionIndex].minBidDenominated, + constants.dbMaxDenominatedValue, + ); if (input.paymentToken) { marketplaceReindexState.auctions[auctionIndex].paymentToken = input.paymentToken; diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-offer-created.hander.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-offer-created.hander.ts index fac2faeb8..7e45e29c6 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-offer-created.hander.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-offer-created.hander.ts @@ -1,4 +1,5 @@ import { Injectable } from '@nestjs/common'; +import { constants } from 'src/config'; import { OfferEntity } from 'src/db/offers'; import { OfferStatusEnum } from 'src/modules/offers/models'; import { BigNumberUtils } from 'src/utils/bigNumber-utils'; @@ -12,6 +13,7 @@ export class ReindexOfferCreatedHandler { handle(marketplaceReindexState: MarketplaceReindexState, input: OfferCreatedSummary, decimals: number): void { const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); + const priceAmountDenominated = BigNumberUtils.denominateAmount(input.price, decimals); const offer = new OfferEntity({ id: marketplaceReindexState.offers.length, creationDate: modifiedDate, @@ -23,9 +25,9 @@ export class ReindexOfferCreatedHandler { priceToken: input.paymentToken, priceNonce: input.paymentNonce, priceAmount: input.price, - priceAmountDenominated: BigNumberUtils.denominateAmount(input.price, decimals), + priceAmountDenominated: Math.min(priceAmountDenominated, constants.dbMaxDenominatedValue), ownerAddress: input.address, - endDate: input.endTime, + endDate: Math.min(input.endTime, constants.dbMaxTimestamp), boughtTokensNo: input.itemsCount, marketplaceKey: marketplaceReindexState.marketplace.key, status: OfferStatusEnum.Active, diff --git a/src/modules/marketplaces/marketplaces-reindex.service.ts b/src/modules/marketplaces/marketplaces-reindex.service.ts index 7e6eef3e2..7ec907d54 100644 --- a/src/modules/marketplaces/marketplaces-reindex.service.ts +++ b/src/modules/marketplaces/marketplaces-reindex.service.ts @@ -93,7 +93,7 @@ export class MarketplacesReindexService { return [ new MarketplaceReindexState({ marketplace, - isReindexFromTheBeginning: input.afterTimestamp ? false : true, + isFullStateInMemory: input.afterTimestamp ? false : true, }), ]; } @@ -106,7 +106,7 @@ export class MarketplacesReindexService { marketplaceReindexStates.push( new MarketplaceReindexState({ marketplace: internalMarketplaces[i], - isReindexFromTheBeginning: input.afterTimestamp ? false : true, + isFullStateInMemory: input.afterTimestamp ? false : true, listedCollections: marketplaceCollections, }), ); @@ -138,6 +138,13 @@ export class MarketplacesReindexService { marketplaceReindexStates, processInNextBatch.concat(batch), ); + + processInNextBatch = await this.processEventsBatchAndReturnUnprocessedEvents( + marketplaceReindexStates, + processInNextBatch.concat(batch), + ); + + await this.addInactiveStateItemsToDb(marketplaceReindexStates); } while (input.beforeTimestamp ? afterTimestamp < input.beforeTimestamp : true); const isFinalBatch = true; @@ -209,7 +216,7 @@ export class MarketplacesReindexService { eventOrdersAndTx, ); - if (!marketplaceReindexStates[0].isReindexFromTheBeginning) { + if (!marketplaceReindexStates[0].isFullStateInMemory) { await Promise.all( marketplaceReindexStates.map((state) => { return this.getStateFromDbIfMissing(state, eventsSetSummaries); @@ -402,15 +409,60 @@ export class MarketplacesReindexService { } } + private async addInactiveStateItemsToDb(marketplaceReindexStates: MarketplaceReindexState[]): Promise { + for (const marketplaceReindexState of marketplaceReindexStates) { + marketplaceReindexState.setStateItemsToExpiredIfOlderThanTimestamp(DateUtils.getCurrentTimestamp()); + + const [inactiveAuctions, inactiveOrders, inactiveOffers] = marketplaceReindexState.popInactiveItems(); + + await this.populateAuctionAssetTags(inactiveAuctions); + + for (let i = 0; i < inactiveOrders.length; i++) { + inactiveOrders[i].auctionId = inactiveAuctions.findIndex((a) => a.id === inactiveOrders[i].auctionId); + delete inactiveOrders[i].id; + } + + for (let i = 0; i < inactiveAuctions.length; i++) { + delete inactiveAuctions[i].id; + } + + await this.auctionSetterService.saveBulkAuctions(inactiveAuctions); + + let tags: TagEntity[] = []; + inactiveAuctions.map((auction) => { + const assetTags = auction.tags.split(','); + assetTags.map((assetTag) => { + if (assetTag !== '') { + tags.push( + new TagEntity({ + auctionId: auction.id, + tag: assetTag.trim().slice(0, constants.dbMaxTagLength), + auction: auction, + }), + ); + } + }); + }); + + const saveTagsPromise = this.persistenceService.saveTags(tags); + + for (let i = 0; i < inactiveOrders.length; i++) { + inactiveOrders[i].auction = inactiveAuctions[inactiveOrders[i].auctionId]; + inactiveOrders[i].auctionId = inactiveAuctions[inactiveOrders[i].auctionId].id; + } + inactiveOffers.map((o) => delete o.id); + + await Promise.all([ + saveTagsPromise, + this.persistenceService.saveBulkOrders(inactiveOrders), + this.persistenceService.saveBulkOffers(inactiveOffers), + ]); + } + } + private async addMarketplaceStateToDb(marketplaceReindexState: MarketplaceReindexState): Promise { marketplaceReindexState.auctions.map((a) => { delete a.id; - if (a.startDate > constants.dbMaxTimestamp) { - a.startDate = constants.dbMaxTimestamp; - } - if (a.endDate > constants.dbMaxTimestamp) { - a.endDate = constants.dbMaxTimestamp; - } }); marketplaceReindexState.orders.map((o) => delete o.id); marketplaceReindexState.offers.map((o) => delete o.id); @@ -430,7 +482,7 @@ export class MarketplacesReindexService { tags.push( new TagEntity({ auctionId: auction.id, - tag: assetTag.trim().slice(0, 20), + tag: assetTag.trim().slice(0, constants.dbMaxTagLength), auction: auction, }), ); @@ -438,9 +490,11 @@ export class MarketplacesReindexService { }); }); - await this.persistenceService.saveTags(tags); - await this.persistenceService.saveBulkOrders(marketplaceReindexState.orders); - await this.persistenceService.saveBulkOffers(marketplaceReindexState.offers); + await Promise.all([ + this.persistenceService.saveBulkOrders(marketplaceReindexState.orders), + this.persistenceService.saveBulkOffers(marketplaceReindexState.offers), + this.persistenceService.saveTags(tags), + ]); } private async populateAuctionAssetTags(auctions: AuctionEntity[]): Promise { diff --git a/src/modules/marketplaces/models/MarketplaceReindexState.ts b/src/modules/marketplaces/models/MarketplaceReindexState.ts index 44f48891b..9205231a5 100644 --- a/src/modules/marketplaces/models/MarketplaceReindexState.ts +++ b/src/modules/marketplaces/models/MarketplaceReindexState.ts @@ -14,7 +14,7 @@ import { Marketplace } from './Marketplace.dto'; @ObjectType() export class MarketplaceReindexState { marketplace: Marketplace; - isReindexFromTheBeginning: boolean; + isFullStateInMemory: boolean; listedCollections: string[] = []; auctions: AuctionEntity[] = []; orders: OrderEntity[] = []; @@ -94,6 +94,41 @@ export class MarketplaceReindexState { this.setOffersToExpiredIfOlderThanTimestamp(timestamp); } + popInactiveItems(): [AuctionEntity[], OrderEntity[], OfferEntity[]] { + const inactiveAuctionStatuses = [AuctionStatusEnum.Closed, AuctionStatusEnum.Ended]; + const inactiveOfferStatuses = [OfferStatusEnum.Accepted, OfferStatusEnum.Closed, OfferStatusEnum.Expired]; + + let inactiveAuctions = []; + let inactiveOrders = []; + let inactiveOffers = []; + + for (let i = 0; i < this.auctions.length; i++) { + if (inactiveAuctionStatuses.includes(this.auctions[i].status)) { + inactiveAuctions.push(this.auctions[i]); + delete this.auctions[i]; + } + } + this.auctions = this.auctions.filter((a) => a); + + for (let i = 0; i < this.orders.length; i++) { + if (inactiveAuctions.findIndex((a) => a.id === this.orders[i].auctionId) !== -1) { + inactiveOrders.push(this.orders[i]); + delete this.orders[i]; + } + } + this.orders = this.orders.filter((o) => o); + + for (let i = 0; i < this.offers.length; i++) { + if (inactiveOfferStatuses.includes(this.offers[i].status)) { + inactiveOffers.push(this.offers[i]); + delete this.offers[i]; + } + } + this.offers = this.offers.filter((o) => o); + + return [inactiveAuctions, inactiveOrders, inactiveOffers]; + } + private setAuctionsAndOrdersToExpiredIfOlderThanTimestamp(timestamp: number): void { const runningAuctions = this.auctions?.filter((a) => a.status === AuctionStatusEnum.Running); for (let i = 0; i < runningAuctions.length; i++) { diff --git a/src/modules/marketplaces/models/marketplaces-reindex-events-summaries/AuctionStartedSummary.ts b/src/modules/marketplaces/models/marketplaces-reindex-events-summaries/AuctionStartedSummary.ts index 5a17ba21e..7391acf76 100644 --- a/src/modules/marketplaces/models/marketplaces-reindex-events-summaries/AuctionStartedSummary.ts +++ b/src/modules/marketplaces/models/marketplaces-reindex-events-summaries/AuctionStartedSummary.ts @@ -42,7 +42,7 @@ export class AuctionStartedSummary extends ReindexGenericSummary { const address = event.data.eventData?.address ?? tx.receiver; const topics = this.getTopics(event); - if (!topics || (!topics.price && !topics.minBid)) { + if (!topics || (!topics.price && !topics.minBid) || !topics.auctionType) { return; } @@ -60,7 +60,7 @@ export class AuctionStartedSummary extends ReindexGenericSummary { minBid: topics.minBid ?? topics.price, maxBid: topics.maxBid ?? '0', minBidDiff: minBidDiff, - startTime: topics.startTime, + startTime: topics.startTime ?? event.timestamp, endTime: topics.endTime ?? topics.deadline ?? 0, paymentToken: topics.paymentToken, paymentNonce: topics.paymentNonce ?? topics.paymentTokenNonce ?? 0,