Development to main (#984)

* SERVICES1086 rebuild marketplace data/state endpoint, service and functions (part 1)

* SERVICES-1085 extract confusing logic in a separete function; renaming; remove comment

* SERVICES-1085 code splitting, renamings & improvements

* SERVICES-1085 rename param

* SERVICES-1085 add marketplace reindex process event logic

* SERVICES-1085 fix batch slice function; code improvements for get event category function

* SERVICES-1085 remove console log

* SERVICES-1085 extract total bought tokens in a separate function

* SERVICES-1085 split reindex marketplace data code

* SERVICES-1085 use handlers and summary classes for rebuilding marketplace state

* SERVICES-1085 fix marketplace elastic query time range filter

* SERVICES-1085 create MarketplaceReindexState class with common methods and remove common handlers

* SERVICES-1085 add handlers for XOXNO specfic events

* set default paymentNonce if not already set

* SERVICES-1085 ignore not used marketplace events for better performance\

* undo default paymentNonce

* SERVICES-1085 marketplace events summaries

* SERVICES-1085 update marketplace event summaries and add DeadRare updatePrice topics

* SERVICES-1085 marketplace events summary service

* SERVICES-1085 fix marketplace reindex summaries and handle krogan acceptOffer

* SERVICES-1085 marketplace reindex state handlers

* SERVICES-1085 remove confusing method

* SERVICES-1085 remove confusing method

* SERVICES-1085 extract create new order from event summary logic into a separate method

* SERVICES-1085 improve createOrder from summary function

* SERVICES-1085 renamings

* SERVICES-1085 renamings

* SERVICES-1085 update marketplace reindex state service

* SERVICES-1085 process event even if no payment token found

* SERVICES-1085 xoxno user deposit

* SERVICES-1085 fix elrondnftswap offerId type

* SERVICES-1085 reindex marketplace state before & after time range

* SERVICES-1085 reindex marketplace state logs & fixes

* SERVICES-1085 renaming

* SERVICES-1085 fix duplicate bid for internal marketplace

* SERVICES-1085 try-catch and better logging

* SERVICES-1085 fix xoxno no startTime

* SERVICES-1085 fix xoxno no startTime

* SERVICES-1085 fix endAuction with no winning order

* SERVICES-1085 fix case when undefined beforeTimestamp

* SERVICES-1085 save auction tags

* SERVICES-1085 admin endpoints fire and forget

* SERVICES-1085 fix internal auction timestamp topic

* SERVICES-1085 marketplace reindex events locker & logs

* SERVICES-1085 handle multiple internal marketplaces with same address case

* fix admin resolver catch

* SERVICES-1085 exclude collection filtering when unique marketplace address

* SERVICES-1085 fix case when timestamp not spported by DB

* Add plugins module

* Update scam cronjob

* Update scam service

* Add none scam info

* Update workflow

* Update cronjob time

* Remode delete by pattern where not needed

* Fix scam report

* Remove unused code

* trigger cache invalidation

* Change delete to update

* Remove delete by pattern for asset history

* Remove delete by pattern from cache invalidation

* Clear to none nft scam info

* Update scam update cache invalidation

* Add null check

* Add collections with no volum

* Fix trending order by

* Fix trending return

* Fix order trending

* Move blacklist filter before order

* Update filter collection

* Remove clear by pattern in auctions

* Add Scam Collection setting

* Add cache invalidation

* Remove scam info from api

* clean up code

* Upgrade testing packages

* Remove data api calls

* Update mapping

* Refactor Token model

* Add historical price retrieve

* Add elrond tools back

* Add create nft with multiple files

* Extract duplicated code

* Fix typo

* Decrease caching time

* Decrease caching time

* Add ici handling

* Update key

* Update key

* Fix Scam Mapping

* Fix bug for event processing

* Add tickets type on collections

* Add page consts and remove extra pagination

* Remove extra line

* Add isTicket field on asset

* Update redis handler for is ticket

* Add debuging logging

* Fix tickets assets return

* Remove logging

* Update length check

* Update collection length checking

* Fix undefined error

* Allow nullable for role address

---------

Co-authored-by: johnykes <[email protected]>
Co-authored-by: johnykes <[email protected]>

.github/workflows/node.js.yml

@@ -26,6 +26,5 @@ jobs:
       with:
         node-version: ${{ matrix.node-version }}
         cache: 'npm'
-    - run: npm install
-    - run: npm run build --if-present
+    - run: npm ci
     - run: npm test

.gitignore

@@ -400,3 +400,4 @@ build.yaml
 
 # PEM file(s)
 *.pem
+/src/plugins

package.json

@@ -5,6 +5,8 @@
   "description": "Nest TypeScript starter repository",
   "license": "MIT",
   "scripts": {
+    "init": "run-script-os",
+    "init:nix": "rimraf src/plugins && mkdir src/plugins && cp src/plugins.template/plugin.module.ts.template src/plugins/plugin.module.ts",
     "prebuild": "rimraf dist",
     "build": "nest build && npm run copy-pem",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
@@ -90,11 +92,11 @@
   "devDependencies": {
     "@nestjs/cli": "9.2.0",
     "@nestjs/schematics": "9.0.4",
-    "@nestjs/testing": "9.1.4",
+    "@nestjs/testing": "9.3.12",
     "@types/express": "^4.17.11",
     "@types/graphql-relay": "^0.6.0",
-    "@types/jest": "^27.4.1",
-    "@types/node": "^14.14.36",
+    "@types/jest": "29.5.0",
+    "@types/node": "^14.14.42",
     "@types/passport-local": "^1.0.33",
     "@types/supertest": "^2.0.12",
     "@types/tiny-async-pool": "^1.0.0",
@@ -103,7 +105,7 @@
     "eslint": "8.28.0",
     "eslint-config-prettier": "8.5.0",
     "eslint-plugin-prettier": "3.4.1",
-    "jest": "29.4.2",
+    "jest": "29.5.0",
     "prettier": "^2.2.1",
     "supertest": "6.3.3",
     "ts-jest": "29.0.5",

schema.gql

@@ -88,6 +88,7 @@ type Asset {
   identifier: String!
   isLiked(byAddress: String!): Boolean
   isNsfw: Boolean
+  isTicket: Boolean
   isWhitelistedStorage: Boolean!
   likesCount: Int
   lowestAuction(filters: MarketplaceFilters): Auction
@@ -115,12 +116,15 @@ type Asset {
 
 enum AssetActionEnum {
   Added
+  Bid
   Bought
   ClosedAuction
   Created
   EndedAuction
+  PriceUpdated
   Received
   StartedAuction
+  Updated
 }
 
 type AssetEdge {
@@ -181,6 +185,7 @@ input AssetsFilter {
   """This will work only with an owner address"""
   collections: [String!]
   creatorAddress: String
+  customFilters: CustomFiltersEnum
   identifier: String
   identifiers: [String!]
   likedByAddress: String
@@ -508,7 +513,7 @@ type CollectionResponse {
 }
 
 type CollectionRole {
-  address: String!
+  address: String
   canAddQuantity: Boolean
   canAddUri: Boolean
   canBurn: Boolean
@@ -623,6 +628,10 @@ input CurrentPaymentTokensFilters {
   marketplaceKey: String
 }
 
+enum CustomFiltersEnum {
+  Tickets
+}
+
 type ExploreCollectionsStats {
   activeLast30DaysCount: Int!
   allCollectionsCount: Int!
@@ -645,6 +654,7 @@ type ExploreStats {
 enum FeaturedCollectionTypeEnum {
   Featured
   Hero
+  Tickets
 }
 
 input FeaturedCollectionsArgs {
@@ -768,6 +778,12 @@ type MarketplacePageInfo {
   startCursor: String
 }
 
+input MarketplaceReindexDataArgs {
+  afterTimestamp: Float
+  beforeTimestamp: Float
+  marketplaceAddress: String!
+}
+
 enum MarketplaceTypeEnum {
   External
   Internal
@@ -838,13 +854,15 @@ type Mutation {
   clearReportNft(input: ClearReportInput!): Boolean!
   createAuction(input: CreateAuctionArgs!): TransactionNode!
   createNft(file: Upload!, input: CreateNftArgs!): TransactionNode!
+  createNftWithMultipleFiles(files: [Upload!]!, input: CreateNftArgs!): TransactionNode!
   endAuction(auctionId: Int!): TransactionNode!
   flagCollection(input: FlagCollectionInput!): Boolean!
   flagNft(input: FlagNftInput!): Boolean!
   indexTrandingCollections(forTheLastHours: Float!): Boolean!
   issueCampaign(input: IssueCampaignArgs!): TransactionNode!
   issueNftCollection(input: IssueCollectionArgs!): TransactionNode!
   issueSftCollection(input: IssueCollectionArgs!): TransactionNode!
+  reindexMarketplaceData(input: MarketplaceReindexDataArgs!): Boolean!
   reindexMarketplaceEvents(input: MarketplaceEventsIndexingArgs!): Boolean!
   removeBlacklistedCollection(collection: String!): Boolean!
   removeFeaturedCollection(input: FeaturedCollectionsArgs!): Boolean!
@@ -1201,6 +1219,7 @@ type ScamInfo {
 }
 
 enum ScamInfoTypeEnum {
+  none
   potentialScam
   scam
 }

src/common/persistence/persistence.service.ts

@@ -84,7 +84,7 @@ export class PersistenceService {
     private readonly auctionsRepository: AuctionsRepository,
     private readonly marketplaceEventsRepository: MarketplaceEventsRepository,
     private readonly offersRepository: OffersRepository,
-  ) {}
+  ) { }
 
   private async execute<T>(key: string, action: Promise<T>): Promise<T> {
     const profiler = new PerformanceProfiler();
@@ -338,6 +338,13 @@ export class PersistenceService {
     );
   }
 
+  async getFeaturedCollectionsByIdentifiers(collections: string[]): Promise<FeaturedCollectionEntity[]> {
+    return await this.execute(
+      this.getFeaturedCollections.name,
+      this.featuredCollectionsRepository.getFeaturedCollectionsByIdentifiers(collections),
+    );
+  }
+
   async addFeaturedCollection(
     collection: string,
     type: FeaturedCollectionTypeEnum,
@@ -729,6 +736,13 @@ export class PersistenceService {
     );
   }
 
+  async saveBulkOrders(orders: OrderEntity[]) {
+    return await this.execute(
+      this.saveBulkOrders.name,
+      this.ordersRepository.saveBulkOrders(orders),
+    );
+  }
+
   async updateOrderWithStatus(order: OrderEntity, status: OrderStatusEnum) {
     return await this.execute(
       this.updateOrderWithStatus.name,
@@ -878,6 +892,18 @@ export class PersistenceService {
       this.auctionsRepository.getBulkAuctions(auctionsIds),
     );
   }
+  async getBulkAuctionsByAuctionIdsAndMarketplace(
+    auctionsIds: number[],
+    marketplaceKey: string,
+  ): Promise<AuctionEntity[]> {
+    return await this.execute(
+      this.getBulkAuctionsByAuctionIdsAndMarketplace.name,
+      this.auctionsRepository.getBulkAuctionsByAuctionIdsAndMarketplace(
+        auctionsIds,
+        marketplaceKey,
+      ),
+    );
+  }
 
   async getAuctionByMarketplace(
     id: number,
@@ -918,6 +944,19 @@ export class PersistenceService {
     );
   }
 
+  async getBulkAuctionsByIdentifierAndMarketplace(
+    identifiers: string[],
+    marletplaceKey: string,
+  ): Promise<any[]> {
+    return await this.execute(
+      this.getBulkAuctionsByIdentifierAndMarketplace.name,
+      this.auctionsRepository.getBulkAuctionsByIdentifiersAndMarketplace(
+        identifiers,
+        marletplaceKey,
+      ),
+    );
+  }
+
   async getAvailableTokensForIdentifiers(identifiers: string[]): Promise<any> {
     return await this.execute(
       this.getAvailableTokensForIdentifiers.name,
@@ -978,6 +1017,13 @@ export class PersistenceService {
     );
   }
 
+  async saveBulkAuctions(auctions: AuctionEntity[]): Promise<void> {
+    return await this.execute(
+      this.saveBulkAuctions.name,
+      this.auctionsRepository.saveBulkAuctions(auctions),
+    );
+  }
+
   async rollbackAuctionAndOrdersByHash(blockHash: string): Promise<any> {
     return await this.execute(
       this.rollbackAuctionAndOrdersByHash.name,
@@ -1078,6 +1124,13 @@ export class PersistenceService {
     );
   }
 
+  async saveBulkOffers(offers: OfferEntity[]): Promise<void> {
+    return await this.execute(
+      this.saveBulkOffers.name,
+      this.offersRepository.saveBulkOffers(offers),
+    );
+  }
+
   async getOffersThatReachedDeadline(): Promise<OfferEntity[]> {
     return await this.execute(
       this.getOffersThatReachedDeadline.name,
@@ -1092,6 +1145,19 @@ export class PersistenceService {
     );
   }
 
+  async getBulkOffersByOfferIdsAndMarketplace(
+    offerIds: number[],
+    marketplaceKey: string,
+  ): Promise<OfferEntity[]> {
+    return await this.execute(
+      this.getBulkOffersByOfferIdsAndMarketplace.name,
+      this.offersRepository.getBulkOffersByOfferIdsAndMarketplace(
+        offerIds,
+        marketplaceKey,
+      ),
+    );
+  }
+
   async getMostLikedAssetsIdentifiers(
     offset?: number,
     limit?: number,

src/common/pluggins/plugin.service.ts

@@ -0,0 +1,7 @@
+import { Injectable } from '@nestjs/common';
+import { Asset } from 'src/modules/assets/models/Asset.dto';
+
+@Injectable()
+export class PluginService {
+  async computeScamInfo(_nft: Asset[]): Promise<void> {}
+}

src/common/services/caching/entities/cache.info.ts

@@ -129,6 +129,16 @@ export class CacheInfo {
     ttl: 10 * Constants.oneMinute(),
   };
 
+  static CexTokens: CacheInfo = {
+    key: 'cexTokens',
+    ttl: 10 * Constants.oneMinute(),
+  };
+
+  static xExchangeTokens: CacheInfo = {
+    key: 'xExchangeTokens',
+    ttl: 10 * Constants.oneMinute(),
+  };
+
   static AllTokens: CacheInfo = {
     key: 'allTokens',
     ttl: Constants.oneMinute(),
@@ -156,7 +166,7 @@ export class CacheInfo {
 
   static AssetHistory: CacheInfo = {
     key: 'assetHistory',
-    ttl: Constants.oneDay(),
+    ttl: 5 * Constants.oneMinute(),
   };
 
   static CollectionTypes: CacheInfo = {

src/common/services/mx-communication/index.ts

@@ -1,7 +1,6 @@
 export * from './mx-api.service';
 export * from './mx-stats.service';
 export * from './mx-communication.module';
-export * from './mx-tools.service';
 export * from './mx-elastic.service';
 export * from './mx-identity.service';
 export * from './mx-proxy.service';

src/common/services/mx-communication/models/account.info.ts

@@ -0,0 +1,19 @@
+import { ScamInfo } from 'src/modules/assets/models/ScamInfo.dto';
+
+export class AccountInfo {
+  address: string = '';
+  nonce: number = 0;
+  balance: string = '';
+  username: string = '';
+  code: string = '';
+  codeHash: string | undefined;
+  rootHash: string = '';
+  codeMetadata: string = '';
+  developerReward: string = '';
+  ownerAddress: string = '';
+  scamInfo: ScamInfo | undefined = undefined;
+
+  constructor(init?: Partial<AccountInfo>) {
+    Object.assign(this, init);
+  }
+}

src/common/services/mx-communication/models/api-token.model.ts

@@ -0,0 +1,23 @@
+export class ApiToken {
+  identifier: string;
+  ticker: string;
+  name: string;
+  price: string;
+  decimals: number;
+
+  constructor(init?: Partial<ApiToken>) {
+    Object.assign(this, init);
+  }
+}
+
+export class DexToken {
+  id: string;
+  symbol: string;
+  name: string;
+  price: string;
+  decimals: number;
+
+  constructor(init?: Partial<DexToken>) {
+    Object.assign(this, init);
+  }
+}

src/common/services/mx-communication/models/nft.dto.ts

@@ -31,6 +31,7 @@ export interface Nft {
   nonce: number;
   royalties: string;
   timestamp: number;
+  url: string;
   uris: string[];
   tags: string[];
   metadata: NftMetadata;

src/common/services/mx-communication/models/scam-info.dto.ts

@@ -4,6 +4,7 @@ export interface ScamInfoApi {
 }
 
 export enum ScamInfoTypeApiEnum {
+  none = 'none',
   scam = 'scam',
   potentialScam = 'potentialScam',
 }