diff --git a/src/crons/elastic.updater/nsfw-queries.ts b/src/crons/elastic.updater/nsfw-queries.ts index ef09ff334..1c57f77a4 100644 --- a/src/crons/elastic.updater/nsfw-queries.ts +++ b/src/crons/elastic.updater/nsfw-queries.ts @@ -1,16 +1,17 @@ import { NftTypeEnum } from 'src/modules/assets/models'; import { ElasticQuery, QueryType } from '@multiversx/sdk-nestjs-elastic'; import { constants } from 'src/config'; +import { ELASTIC_NFT_NSFW } from 'src/utils/constants'; export const getNsfwNotMarkedQuery = ElasticQuery.create() - .withMustNotExistCondition('nft_nsfw_mark') + .withMustNotExistCondition(ELASTIC_NFT_NSFW) .withMustExistCondition('identifier') .withMustMultiShouldCondition([NftTypeEnum.NonFungibleESDT, NftTypeEnum.SemiFungibleESDT], (type) => QueryType.Match('type', type)) .withMustCondition(QueryType.Nested('data', { 'data.nonEmptyURIs': true })) .withPagination({ from: 0, size: constants.elasticMaxBatch }); export const getNsfwMarkedQuery = ElasticQuery.create() - .withFields(['nft_nsfw_mark']) + .withFields([ELASTIC_NFT_NSFW]) .withMustExistCondition('identifier') .withMustMultiShouldCondition([NftTypeEnum.NonFungibleESDT, NftTypeEnum.SemiFungibleESDT], (type) => QueryType.Match('type', type)) .withMustCondition(QueryType.Nested('data', { 'data.nonEmptyURIs': true })) diff --git a/src/crons/elastic.updater/nsfw.updater.service.ts b/src/crons/elastic.updater/nsfw.updater.service.ts index e388bd2aa..2d562b9de 100644 --- a/src/crons/elastic.updater/nsfw.updater.service.ts +++ b/src/crons/elastic.updater/nsfw.updater.service.ts @@ -7,6 +7,7 @@ import { CacheEventsPublisherService } from 'src/modules/rabbitmq/cache-invalida import { ChangedEvent, CacheEventTypeEnum } from 'src/modules/rabbitmq/cache-invalidation/events/changed.event'; import { getNsfwMarkedQuery, getNsfwNotMarkedQuery } from './nsfw-queries'; import { constants } from 'src/config'; +import { ELASTIC_NFT_NSFW, ELASTIC_TOKENS_INDEX } from 'src/utils/constants'; type NsfwType = { identifier: string; @@ -24,7 +25,7 @@ export class NsfwUpdaterService { ) {} public async cleanReindexing() { - await this.elasticService.getScrollableList('tokens', 'identifier', getNsfwNotMarkedQuery, async (items) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'identifier', getNsfwNotMarkedQuery, async (items) => { const nsfwItems = items.map((item) => ({ identifier: item.identifier, nsfw: item.nft_nsfw_mark, @@ -35,7 +36,7 @@ export class NsfwUpdaterService { } public async validateNsfwFlags() { - await this.elasticService.getScrollableList('tokens', 'identifier', getNsfwMarkedQuery, async (items) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'identifier', getNsfwMarkedQuery, async (items) => { const nsfwItems = items.map((item) => ({ identifier: item.identifier, nsfw: item.nft_nsfw_mark, @@ -46,7 +47,7 @@ export class NsfwUpdaterService { } public async updateNsfwWhereNone() { - await this.elasticService.getScrollableList('tokens', 'identifier', getNsfwNotMarkedQuery, async (items) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'identifier', getNsfwNotMarkedQuery, async (items) => { const nsfwItems = items.map((item) => ({ identifier: item.identifier, nsfw: item.nft_nsfw_mark, @@ -118,9 +119,9 @@ export class NsfwUpdaterService { try { this.logger.log(`Setting nsfw for '${identifier}' with value ${nsfw}`); await this.elasticService.setCustomValue( - 'tokens', + ELASTIC_TOKENS_INDEX, identifier, - this.elasticService.buildUpdateBody('nft_nsfw_mark', nsfw), + this.elasticService.buildUpdateBody(ELASTIC_NFT_NSFW, nsfw), '?retry_on_conflict=2', ); } catch (error) { @@ -136,7 +137,7 @@ export class NsfwUpdaterService { try { if (items && items.length > 0) { this.logger.log(`Updating NSFW flag`); - await this.elasticService.bulkRequest('tokens', this.buildNsfwBulkUpdate(items)); + await this.elasticService.bulkRequest(ELASTIC_TOKENS_INDEX, this.buildNsfwBulkUpdate(items)); } } catch (error) { this.logger.error('Unexpected error when updating nsfw with bulk request', { @@ -149,7 +150,7 @@ export class NsfwUpdaterService { private buildNsfwBulkUpdate(items: { identifier: string; nsfw: number }[]): string[] { let updates: string[] = []; items.forEach((r) => { - updates.push(this.buildBulkUpdate('tokens', r.identifier, 'nft_nsfw_mark', parseFloat(r.nsfw.toString()))); + updates.push(this.buildBulkUpdate(ELASTIC_TOKENS_INDEX, r.identifier, ELASTIC_NFT_NSFW, parseFloat(r.nsfw.toString()))); }); return updates; } diff --git a/src/crons/elastic.updater/rarity.updater.service.ts b/src/crons/elastic.updater/rarity.updater.service.ts index a36ba45bc..7c50ec626 100644 --- a/src/crons/elastic.updater/rarity.updater.service.ts +++ b/src/crons/elastic.updater/rarity.updater.service.ts @@ -5,6 +5,7 @@ import { generateCacheKeyFromParams } from 'src/utils/generate-cache-key'; import { NftRarityElasticService } from 'src/modules/nft-rarity/nft-rarity.elastic.service'; import { RedisCacheService } from '@multiversx/sdk-nestjs-cache'; import { Locker } from '@multiversx/sdk-nestjs-common'; +import { ELASTIC_TOKENS_INDEX } from 'src/utils/constants'; @Injectable() export class RarityUpdaterService { @@ -33,7 +34,7 @@ export class RarityUpdaterService { const query = this.nftRarityElasticService.getAllNftsWhereRarityNotComputedFromElasticQuery(); - await this.elasticService.getScrollableList('tokens', 'token', query, async (items) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'token', query, async (items) => { const collections = [...new Set(items.map((i) => i.token))]; collectionsToUpdate = collectionsToUpdate.concat(collections.filter((c) => collectionsToUpdate.indexOf(c) === -1)); if (collectionsToUpdate.length >= maxCollectionsToUpdate) { diff --git a/src/crons/elastic.updater/traits.updater.service.ts b/src/crons/elastic.updater/traits.updater.service.ts index fd302584d..f24bf1490 100644 --- a/src/crons/elastic.updater/traits.updater.service.ts +++ b/src/crons/elastic.updater/traits.updater.service.ts @@ -10,6 +10,7 @@ import { getCollectionsWhereTraitsFlagNotSetFromElasticQuery, getCollectionsWithTraitSummaryFromElasticQuery, } from 'src/modules/nft-traits/nft-traits.elastic.queries'; +import { ELASTIC_TOKENS_INDEX } from 'src/utils/constants'; @Injectable() export class TraitsUpdaterService { @@ -30,7 +31,7 @@ export class TraitsUpdaterService { const lastIndex = await this.getLastValidatedCollectionIndex(); let collections: string[] = []; - await this.elasticService.getScrollableList('tokens', 'token', query, async (items) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'token', query, async (items) => { collections = collections.concat([...new Set(items.map((i) => i.token))]); if (collections.length > lastIndex + maxCollectionsToValidate) { return undefined; @@ -65,7 +66,7 @@ export class TraitsUpdaterService { const query = getCollectionsWhereTraitsFlagNotSetFromElasticQuery(maxCollectionsToUpdate); - await this.elasticService.getScrollableList('tokens', 'token', query, async (items) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'token', query, async (items) => { const collections = [...new Set(items.map((i) => i.token))]; collectionsToUpdate = collectionsToUpdate.concat(collections.filter((c) => collectionsToUpdate.indexOf(c) === -1)); if (collectionsToUpdate.length >= maxCollectionsToUpdate) { diff --git a/src/modules/admins/flag-nft.service.ts b/src/modules/admins/flag-nft.service.ts index 10adc9098..4fe1584bc 100644 --- a/src/modules/admins/flag-nft.service.ts +++ b/src/modules/admins/flag-nft.service.ts @@ -10,6 +10,7 @@ import { Asset, NftTypeEnum } from '../assets/models'; import { VerifyContentService } from '../assets/verify-content.service'; import { CacheEventsPublisherService } from '../rabbitmq/cache-invalidation/cache-invalidation-publisher/change-events-publisher.service'; import { CacheEventTypeEnum, ChangedEvent } from '../rabbitmq/cache-invalidation/events/changed.event'; +import { ELASTIC_NFT_NSFW, ELASTIC_TOKENS_INDEX } from 'src/utils/constants'; type NsfwType = { identifier: string; nsfw: any; @@ -68,7 +69,7 @@ export class FlagNftService { .withMustMultiShouldCondition([NftTypeEnum.NonFungibleESDT, NftTypeEnum.SemiFungibleESDT], (type) => QueryType.Match('type', type)) .withMustCondition(QueryType.Nested('data', { 'data.nonEmptyURIs': true })) .withPagination({ from: 0, size: 10000 }); - await this.elasticUpdater.getScrollableList('tokens', 'identifier', query, async (items) => { + await this.elasticUpdater.getScrollableList(ELASTIC_TOKENS_INDEX, 'identifier', query, async (items) => { const nsfwItems = items.map((item) => ({ identifier: item.identifier, nsfw: item.nft_nsfw_mark, @@ -124,9 +125,9 @@ export class FlagNftService { }), ); await this.elasticUpdater.setCustomValue( - 'tokens', + ELASTIC_TOKENS_INDEX, identifier, - this.elasticUpdater.buildUpdateBody('nft_nsfw_mark', savedValue), + this.elasticUpdater.buildUpdateBody(ELASTIC_NFT_NSFW, savedValue), '?retry_on_conflict=2', ); } @@ -158,9 +159,9 @@ export class FlagNftService { }), ); await this.elasticUpdater.setCustomValue( - 'tokens', + ELASTIC_TOKENS_INDEX, identifier, - this.elasticUpdater.buildUpdateBody('nft_nsfw_mark', value.toRounded(2)), + this.elasticUpdater.buildUpdateBody(ELASTIC_NFT_NSFW, value.toRounded(2)), '?retry_on_conflict=2', ); @@ -219,10 +220,10 @@ export class FlagNftService { async () => { try { await this.elasticUpdater.putMappings( - 'tokens', + ELASTIC_TOKENS_INDEX, this.elasticUpdater.buildPutMultipleMappingsBody([ { - key: 'nft_nsfw_mark', + key: ELASTIC_NFT_NSFW, value: 'float', }, ]), diff --git a/src/modules/assets/assets-getter.service.ts b/src/modules/assets/assets-getter.service.ts index 776b7dbb7..6469b4da4 100644 --- a/src/modules/assets/assets-getter.service.ts +++ b/src/modules/assets/assets-getter.service.ts @@ -23,6 +23,7 @@ import { FeaturedService } from '../featured/featured.service'; import { FeaturedCollectionsFilter } from '../featured/Featured-Collections.Filter'; import { FeaturedCollectionTypeEnum } from '../featured/FeatureCollectionType.enum'; import { constants } from 'src/config'; +import { ELASTIC_TOKENS_INDEX } from 'src/utils/constants'; @Injectable() export class AssetsGetterService { @@ -200,7 +201,7 @@ export class AssetsGetterService { if (artistCollections) { const batch = artistCollections?.collections?.slice(0, 100); let elasticQuery = this.getCollectionsElasticQuery(batch, offset, size); - let elasticNfts = await this.elasticService.getList('tokens', 'identifier', elasticQuery); + let elasticNfts = await this.elasticService.getList(ELASTIC_TOKENS_INDEX, 'identifier', elasticQuery); const returnAssets = await this.mapElasticNfts(elasticNfts); return new CollectionType({ diff --git a/src/modules/nft-rarity/nft-rarity.elastic.service.ts b/src/modules/nft-rarity/nft-rarity.elastic.service.ts index 9111aacee..d4f6550a6 100644 --- a/src/modules/nft-rarity/nft-rarity.elastic.service.ts +++ b/src/modules/nft-rarity/nft-rarity.elastic.service.ts @@ -8,6 +8,20 @@ import { constants } from 'src/config'; import { CustomRank } from './models/custom-rank.model'; import { CollectionFromElastic } from './models/collection-from-elastic.model'; import { Rarity } from '../assets/models/Rarity'; +import { + ELASTIC_NFT_HASRARITY, + ELASTIC_NFT_RANK_CUSTOM, + ELASTIC_NFT_RANK_HASH, + ELASTIC_NFT_RANK_JACCARD, + ELASTIC_NFT_RANK_OPENRARITY, + ELASTIC_NFT_RANK_STATISTICAL, + ELASTIC_NFT_RANK_TRAIT, + ELASTIC_NFT_SCORE_JACCARD, + ELASTIC_NFT_SCORE_OPENRARITY, + ELASTIC_NFT_SCORE_STATISTICAL, + ELASTIC_NFT_SCORE_TRAIT, + ELASTIC_TOKENS_INDEX, +} from 'src/utils/constants'; @Injectable() export class NftRarityElasticService { @@ -20,7 +34,7 @@ export class NftRarityElasticService { try { const updateBody = this.elasticService.buildUpdateBody('nft_hasRarities', hasRarities); - await this.elasticService.setCustomValue('tokens', collection, updateBody, '?retry_on_conflict=2&timeout=1m'); + await this.elasticService.setCustomValue(ELASTIC_TOKENS_INDEX, collection, updateBody, '?retry_on_conflict=2&timeout=1m'); } catch (error) { this.logger.error('Error when setting collection rarity flag', { path: 'NftRarityService.updateRarities', @@ -49,7 +63,11 @@ export class NftRarityElasticService { } try { - await this.elasticService.bulkRequest('tokens', this.buildNftRaritiesBulkUpdate(outdatedNfts, hasRarities), '?timeout=1m'); + await this.elasticService.bulkRequest( + ELASTIC_TOKENS_INDEX, + this.buildNftRaritiesBulkUpdate(outdatedNfts, hasRarities), + '?timeout=1m', + ); } catch (error) { this.logger.error('Error when bulk updating nft rarities Elastic', { path: 'NftRarityService.setNftRaritiesInElastic', @@ -73,7 +91,11 @@ export class NftRarityElasticService { if (customRanks.length > 0) { try { - await this.elasticService.bulkRequest('tokens', this.buildNftCustomRanksBulkUpdate(collection, outdatedRanks), '?timeout=1m'); + await this.elasticService.bulkRequest( + ELASTIC_TOKENS_INDEX, + this.buildNftCustomRanksBulkUpdate(collection, outdatedRanks), + '?timeout=1m', + ); } catch (error) { this.logger.error('Error when bulk updating nft custom ranks inElastic', { path: 'NftRarityService.setNftCustomRanksInElastic', @@ -91,13 +113,13 @@ export class NftRarityElasticService { .withMustExistCondition('token') .withMustMultiShouldCondition([NftTypeEnum.NonFungibleESDT, NftTypeEnum.SemiFungibleESDT], (type) => QueryType.Match('type', type)) .withMustCondition(QueryType.Match('token', collectionTicker, QueryOperator.AND)) - .withFields(['nft_custom_ranks_hash']) + .withFields([ELASTIC_NFT_RANK_HASH]) .withPagination({ from: 0, size: 1, }); - await this.elasticService.getScrollableList('tokens', 'identifier', query, async (items) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'identifier', query, async (items) => { hash = items[0].nft_custom_ranks_hash; return undefined; }); @@ -126,7 +148,7 @@ export class NftRarityElasticService { size: 1, }); - await this.elasticService.getScrollableList('tokens', 'identifier', query, async (items) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'identifier', query, async (items) => { hasRarities = items.length === 1 ? items[0].nft_hasRarities || false : undefined; return undefined; }); @@ -156,16 +178,16 @@ export class NftRarityElasticService { // ) .withFields([ 'nonce', - 'nft_hasRarity', - 'nft_rank_custom', - 'nft_score_openRarity', - 'nft_rank_openRarity', - 'nft_score_jaccardDistances', - 'nft_rank_jaccardDistances', - 'nft_score_trait', - 'nft_rank_trait', - 'nft_score_statistical', - 'nft_rank_statistical', + ELASTIC_NFT_HASRARITY, + ELASTIC_NFT_RANK_CUSTOM, + ELASTIC_NFT_SCORE_OPENRARITY, + ELASTIC_NFT_RANK_OPENRARITY, + ELASTIC_NFT_SCORE_JACCARD, + ELASTIC_NFT_RANK_JACCARD, + ELASTIC_NFT_SCORE_TRAIT, + ELASTIC_NFT_RANK_TRAIT, + ELASTIC_NFT_SCORE_STATISTICAL, + ELASTIC_NFT_RANK_STATISTICAL, ]) .withMustMultiShouldCondition([NftTypeEnum.NonFungibleESDT, NftTypeEnum.SemiFungibleESDT], (type) => QueryType.Match('type', type)) .withSort([{ name: 'nonce', order: ElasticSortOrder.descending }]) @@ -174,7 +196,7 @@ export class NftRarityElasticService { size: constants.getNftsFromElasticBatchSize, }); - await this.elasticService.getScrollableList('tokens', 'identifier', query, async (newNfts) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'identifier', query, async (newNfts) => { nfts = nfts.concat(NftRarityData.fromElasticNfts(newNfts)); return undefined; }); @@ -197,46 +219,46 @@ export class NftRarityElasticService { async () => { try { await this.elasticService.putMappings( - 'tokens', + ELASTIC_TOKENS_INDEX, this.elasticService.buildPutMultipleMappingsBody([ { - key: 'nft_rank_custom', + key: ELASTIC_NFT_RANK_CUSTOM, value: 'long', }, { - key: 'nft_custom_ranks_hash', + key: ELASTIC_NFT_RANK_HASH, value: 'text', }, { - key: 'nft_score_openRarity', + key: ELASTIC_NFT_SCORE_OPENRARITY, value: 'float', }, { - key: 'nft_rank_openRarity', + key: ELASTIC_NFT_RANK_OPENRARITY, value: 'long', }, { - key: 'nft_score_jaccardDistances', + key: ELASTIC_NFT_SCORE_JACCARD, value: 'float', }, { - key: 'nft_rank_jaccardDistances', + key: ELASTIC_NFT_RANK_JACCARD, value: 'long', }, { - key: 'nft_score_trait', + key: ELASTIC_NFT_SCORE_TRAIT, value: 'float', }, { - key: 'nft_rank_trait', + key: ELASTIC_NFT_RANK_TRAIT, value: 'long', }, { - key: 'nft_score_statistical', + key: ELASTIC_NFT_SCORE_STATISTICAL, value: 'float', }, { - key: 'nft_rank_statistical', + key: ELASTIC_NFT_RANK_STATISTICAL, value: 'long', }, ]), @@ -255,7 +277,7 @@ export class NftRarityElasticService { async getAllCollectionsFromElastic(): Promise { const query = this.getAllCollectionsFromElasticQuery(); let collections: string[] = []; - await this.elasticService.getScrollableList('tokens', 'token', query, async (items) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'token', query, async (items) => { collections = collections.concat([...new Set(items.map((i) => i.token))]); }); return collections; @@ -264,7 +286,7 @@ export class NftRarityElasticService { async getAllCollectionsWithRarityFlagsFromElastic(): Promise { const query = this.getAllCollectionsWithRarityFlagsFromElasticQuery(); let collections: CollectionFromElastic[] = []; - await this.elasticService.getScrollableList('tokens', 'token', query, async (items) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'token', query, async (items) => { const collectionsWithRarityFlags = items.map((collection) => CollectionFromElastic.fromElastic(collection)); collections = collections.concat([...new Set(collectionsWithRarityFlags)]); }); @@ -276,33 +298,66 @@ export class NftRarityElasticService { nfts.forEach((r) => { if (hasRarities) { if (r.rarities.customRank) { - updates.push(this.elasticService.buildBulkUpdate('tokens', r.identifier, 'nft_rank_custom', r.rarities.customRank)); + updates.push( + this.elasticService.buildBulkUpdate(ELASTIC_TOKENS_INDEX, r.identifier, ELASTIC_NFT_RANK_CUSTOM, r.rarities.customRank), + ); } updates.push( - this.elasticService.buildBulkUpdate('tokens', r.identifier, 'nft_score_openRarity', r.rarities.openRarityScore), + this.elasticService.buildBulkUpdate( + ELASTIC_TOKENS_INDEX, + r.identifier, + ELASTIC_NFT_SCORE_OPENRARITY, + r.rarities.openRarityScore, + ), + ); + updates.push( + this.elasticService.buildBulkUpdate( + ELASTIC_TOKENS_INDEX, + r.identifier, + ELASTIC_NFT_RANK_OPENRARITY, + r.rarities.openRarityRank, + ), ); - updates.push(this.elasticService.buildBulkUpdate('tokens', r.identifier, 'nft_rank_openRarity', r.rarities.openRarityRank)); updates.push( this.elasticService.buildBulkUpdate( - 'tokens', + ELASTIC_TOKENS_INDEX, r.identifier, - 'nft_score_jaccardDistances', + ELASTIC_NFT_SCORE_JACCARD, r.rarities.jaccardDistancesScore, ), ); updates.push( - this.elasticService.buildBulkUpdate('tokens', r.identifier, 'nft_rank_jaccardDistances', r.rarities.jaccardDistancesRank), + this.elasticService.buildBulkUpdate( + ELASTIC_TOKENS_INDEX, + r.identifier, + ELASTIC_NFT_RANK_JACCARD, + r.rarities.jaccardDistancesRank, + ), + ); + updates.push( + this.elasticService.buildBulkUpdate(ELASTIC_TOKENS_INDEX, r.identifier, ELASTIC_NFT_SCORE_TRAIT, r.rarities.traitScore), ); - updates.push(this.elasticService.buildBulkUpdate('tokens', r.identifier, 'nft_score_trait', r.rarities.traitScore)); - updates.push(this.elasticService.buildBulkUpdate('tokens', r.identifier, 'nft_rank_trait', r.rarities.traitRank)); updates.push( - this.elasticService.buildBulkUpdate('tokens', r.identifier, 'nft_score_statistical', r.rarities.statisticalScore), + this.elasticService.buildBulkUpdate(ELASTIC_TOKENS_INDEX, r.identifier, ELASTIC_NFT_RANK_TRAIT, r.rarities.traitRank), ); updates.push( - this.elasticService.buildBulkUpdate('tokens', r.identifier, 'nft_rank_statistical', r.rarities.statisticalRank), + this.elasticService.buildBulkUpdate( + ELASTIC_TOKENS_INDEX, + r.identifier, + ELASTIC_NFT_SCORE_STATISTICAL, + r.rarities.statisticalScore, + ), + ); + updates.push( + this.elasticService.buildBulkUpdate( + ELASTIC_TOKENS_INDEX, + r.identifier, + ELASTIC_NFT_RANK_STATISTICAL, + r.rarities.statisticalRank, + ), ); } - updates.push(this.elasticService.buildBulkUpdate('tokens', r.identifier, 'nft_hasRarity', hasRarities)); + updates.push(this.elasticService.buildBulkUpdate(ELASTIC_TOKENS_INDEX, r.identifier, ELASTIC_NFT_HASRARITY, hasRarities)); }); return updates; } @@ -310,10 +365,15 @@ export class NftRarityElasticService { buildNftCustomRanksBulkUpdate(collection, customRanks: CustomRank[]): string[] { let updates: string[] = []; customRanks.forEach((cr) => { - updates.push(this.elasticService.buildBulkUpdate('tokens', cr.identifier, 'nft_rank_custom', cr.rank)); + updates.push(this.elasticService.buildBulkUpdate(ELASTIC_TOKENS_INDEX, cr.identifier, ELASTIC_NFT_RANK_CUSTOM, cr.rank)); }); updates.push( - this.elasticService.buildBulkUpdate('tokens', collection, 'nft_custom_ranks_hash', CustomRank.generateHash(customRanks)), + this.elasticService.buildBulkUpdate( + ELASTIC_TOKENS_INDEX, + collection, + ELASTIC_NFT_RANK_HASH, + CustomRank.generateHash(customRanks), + ), ); return updates; } @@ -345,7 +405,7 @@ export class NftRarityElasticService { getAllNftsWhereRarityNotComputedFromElasticQuery(): ElasticQuery { return ElasticQuery.create() .withMustExistCondition('nonce') - .withMustNotCondition(QueryType.Exists('nft_hasRarity')) + .withMustNotCondition(QueryType.Exists(ELASTIC_NFT_HASRARITY)) .withMustCondition(QueryType.Nested('data', { 'data.nonEmptyURIs': true })) .withMustMultiShouldCondition([NftTypeEnum.NonFungibleESDT, NftTypeEnum.SemiFungibleESDT], (type) => QueryType.Match('type', type)) .withFields(['token']) diff --git a/src/modules/nft-traits/nft-traits.elastic.queries.ts b/src/modules/nft-traits/nft-traits.elastic.queries.ts index 298b13951..f2f1f9a3b 100644 --- a/src/modules/nft-traits/nft-traits.elastic.queries.ts +++ b/src/modules/nft-traits/nft-traits.elastic.queries.ts @@ -8,12 +8,13 @@ import { } from '@multiversx/sdk-nestjs-elastic'; import { constants } from 'src/config'; import { NftTypeEnum } from '../assets/models'; +import { ELASTIC_NFT_HASTRAITSUMMARY, ELASTIC_NFT_TRAITS } from 'src/utils/constants'; export const getAllEncodedNftValuesFromElasticQuery = (collection: string, startNonce?: number, endNonce?: number): ElasticQuery => { let query = ElasticQuery.create() .withMustExistCondition('nonce') .withMustCondition(QueryType.Match('token', collection, QueryOperator.AND)) - .withFields(['nft_traitValues', 'nonce']) + .withFields([ELASTIC_NFT_TRAITS, 'nonce']) .withMustMultiShouldCondition([NftTypeEnum.NonFungibleESDT, NftTypeEnum.SemiFungibleESDT], (type) => QueryType.Match('type', type)) .withPagination({ from: 0, @@ -28,7 +29,7 @@ export const getAllEncodedNftValuesFromElasticQuery = (collection: string, start export const getAllEncodedNftValuesFromElasticBeforeTimestampQuery = (beforeTimestamp?: number): ElasticQuery => { let query = ElasticQuery.create() .withMustExistCondition('nonce') - .withFields(['nft_traitValues', 'timestamp', 'identifier']) + .withFields([ELASTIC_NFT_TRAITS, 'timestamp', 'identifier']) .withRangeFilter('timestamp', new RangeLowerThanOrEqual(beforeTimestamp)) .withSort([{ name: 'timestamp', order: ElasticSortOrder.descending }]) .withMustMultiShouldCondition([NftTypeEnum.NonFungibleESDT, NftTypeEnum.SemiFungibleESDT], (type) => QueryType.Match('type', type)) @@ -57,7 +58,7 @@ export const getAllCollectionsWithTraitsFlagFromElasticQuery = (): ElasticQuery .withMustExistCondition('token') .withMustNotExistCondition('nonce') .withMustMultiShouldCondition([NftTypeEnum.NonFungibleESDT, NftTypeEnum.SemiFungibleESDT], (type) => QueryType.Match('type', type)) - .withFields(['token', 'nft_hasTraitSummary']) + .withFields(['token', ELASTIC_NFT_HASTRAITSUMMARY]) .withPagination({ from: 0, size: constants.getCollectionsFromElasticBatchSize, @@ -68,7 +69,7 @@ export const getNftWithTraitValuesFromElasticQuery = (identifier: string): Elast return ElasticQuery.create() .withMustExistCondition('nonce') .withMustCondition(QueryType.Match('identifier', identifier, QueryOperator.AND)) - .withFields(['nft_traitValues']) + .withFields([ELASTIC_NFT_TRAITS]) .withMustMultiShouldCondition([NftTypeEnum.NonFungibleESDT, NftTypeEnum.SemiFungibleESDT], (type) => QueryType.Match('type', type)) .withPagination({ from: 0, @@ -80,7 +81,7 @@ export const getCollectionsWhereTraitsFlagNotSetFromElasticQuery = (maxCollectio return ElasticQuery.create() .withMustExistCondition('token') .withMustNotExistCondition('nonce') - .withMustNotCondition(QueryType.Match('nft_hasTraitSummary', true)) + .withMustNotCondition(QueryType.Match(ELASTIC_NFT_HASTRAITSUMMARY, true)) .withMustMultiShouldCondition([NftTypeEnum.NonFungibleESDT, NftTypeEnum.SemiFungibleESDT], (type) => QueryType.Match('type', type)) .withFields(['token']) .withPagination({ @@ -93,7 +94,7 @@ export const getCollectionsWithTraitSummaryFromElasticQuery = (maxCollectionsToV return ElasticQuery.create() .withMustExistCondition('token') .withMustNotExistCondition('nonce') - .withMustCondition(QueryType.Match('nft_hasTraitSummary', true)) + .withMustCondition(QueryType.Match(ELASTIC_NFT_HASTRAITSUMMARY, true)) .withMustMultiShouldCondition([NftTypeEnum.NonFungibleESDT, NftTypeEnum.SemiFungibleESDT], (type) => QueryType.Match('type', type)) .withPagination({ from: 0, diff --git a/src/modules/nft-traits/nft-traits.elastic.service.ts b/src/modules/nft-traits/nft-traits.elastic.service.ts index 5ec4b4b39..2d0c20dca 100644 --- a/src/modules/nft-traits/nft-traits.elastic.service.ts +++ b/src/modules/nft-traits/nft-traits.elastic.service.ts @@ -9,6 +9,7 @@ import { getAllEncodedNftValuesFromElasticQuery, getNftWithTraitValuesFromElasticQuery, } from './nft-traits.elastic.queries'; +import { ELASTIC_NFT_HASTRAITSUMMARY, ELASTIC_NFT_TRAITS, ELASTIC_TOKENS_INDEX } from 'src/utils/constants'; @Injectable() export class NftTraitsElasticService { @@ -17,7 +18,7 @@ export class NftTraitsElasticService { async setNftsTraitsInElastic(nfts: NftTraits[]): Promise { if (nfts.length > 0) { try { - await this.elasticService.bulkRequest('tokens', this.buildNftTraitsBulkUpdate(nfts), '?timeout=1m'); + await this.elasticService.bulkRequest(ELASTIC_TOKENS_INDEX, this.buildNftTraitsBulkUpdate(nfts), '?timeout=1m'); } catch (error) { this.logger.error('Error when bulk updating nft traits in Elastic', { path: `${NftTraitsElasticService.name}.${this.setNftsTraitsInElastic.name}`, @@ -30,7 +31,7 @@ export class NftTraitsElasticService { async setNftsValuesInElastic(encodedNftValues: EncodedNftValues[]): Promise { if (encodedNftValues.length > 0) { try { - await this.elasticService.bulkRequest('tokens', this.buildNftEncodedValuesBulkUpdate(encodedNftValues), '?timeout=1m'); + await this.elasticService.bulkRequest(ELASTIC_TOKENS_INDEX, this.buildNftEncodedValuesBulkUpdate(encodedNftValues), '?timeout=1m'); } catch (error) { this.logger.error('Error when bulk updating nft traits in Elastic', { path: `${NftTraitsElasticService.name}.${this.setNftsValuesInElastic.name}`, @@ -46,7 +47,7 @@ export class NftTraitsElasticService { try { const query = getNftWithTraitValuesFromElasticQuery(identifier); - await this.elasticService.getScrollableList('tokens', 'identifier', query, async (items) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'identifier', query, async (items) => { nftValues = items[0]?.nft_traitValues ?? []; return undefined; }); @@ -70,7 +71,7 @@ export class NftTraitsElasticService { let maxNonce: number = 0; let minNonce: number = Number.POSITIVE_INFINITY; - await this.elasticService.getScrollableList('tokens', 'identifier', query, async (items) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'identifier', query, async (items) => { const nonces: number[] = items.map((nft) => Number(nft.nonce)); maxNonce = Math.max(...nonces.concat([maxNonce])); minNonce = Math.min(...nonces.concat([maxNonce])); @@ -100,8 +101,8 @@ export class NftTraitsElasticService { async setCollectionTraitsFlagInElastic(collection: string): Promise { try { let updates: string[] = []; - updates.push(this.elasticService.buildBulkUpdate('tokens', collection, 'nft_hasTraitSummary', true)); - await this.elasticService.bulkRequest('tokens', updates, '?timeout=1m'); + updates.push(this.elasticService.buildBulkUpdate(ELASTIC_TOKENS_INDEX, collection, ELASTIC_NFT_HASTRAITSUMMARY, true)); + await this.elasticService.bulkRequest(ELASTIC_TOKENS_INDEX, updates, '?timeout=1m'); } catch (error) { this.logger.error('Error when setting collection traits flag', { path: `${NftTraitsElasticService.name}.${this.setCollectionTraitsFlagInElastic.name}`, @@ -117,7 +118,7 @@ export class NftTraitsElasticService { const query = getAllEncodedNftValuesFromElasticBeforeTimestampQuery(beforeTimestamp); await this.elasticService.getScrollableList( - 'tokens', + ELASTIC_TOKENS_INDEX, 'identifier', query, async (items) => { @@ -137,7 +138,7 @@ export class NftTraitsElasticService { async getAllCollectionsFromElastic(): Promise { const query = getAllCollectionsFromElasticQuery(); let collections: string[] = []; - await this.elasticService.getScrollableList('tokens', 'token', query, async (items) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'token', query, async (items) => { collections = collections.concat([...new Set(items.map((i) => i.token))]); }); return collections; @@ -146,7 +147,7 @@ export class NftTraitsElasticService { async getAllCollectionsWithTraitsFlagFromElastic(): Promise { const query = getAllCollectionsWithTraitsFlagFromElasticQuery(); let collections: CollectionWithTraitsFlag[] = []; - await this.elasticService.getScrollableList('tokens', 'token', query, async (items) => { + await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'token', query, async (items) => { collections = collections.concat([ ...new Set( items.map( @@ -166,9 +167,9 @@ export class NftTraitsElasticService { let updates: string[] = []; nfts.forEach((nft) => { const payload = this.elasticService.buildBulkUpdate( - 'tokens', + ELASTIC_TOKENS_INDEX, nft.identifier, - 'nft_traitValues', + ELASTIC_NFT_TRAITS, nft.traits.map((t) => EncodedNftValues.encode(t)), ); updates.push(payload); @@ -179,7 +180,12 @@ export class NftTraitsElasticService { private buildNftEncodedValuesBulkUpdate(encodedNftValues: EncodedNftValues[]): string[] { let updates: string[] = []; encodedNftValues.forEach((nft) => { - const payload = this.elasticService.buildBulkUpdate('tokens', nft.identifier, 'nft_traitValues', nft.encodedValues ?? []); + const payload = this.elasticService.buildBulkUpdate( + ELASTIC_TOKENS_INDEX, + nft.identifier, + ELASTIC_NFT_TRAITS, + nft.encodedValues ?? [], + ); updates.push(payload); }); return updates; diff --git a/src/modules/scam/nft-scam.elastic.service.ts b/src/modules/scam/nft-scam.elastic.service.ts index 0a59fb5d6..d89682eaa 100644 --- a/src/modules/scam/nft-scam.elastic.service.ts +++ b/src/modules/scam/nft-scam.elastic.service.ts @@ -5,6 +5,7 @@ import { Asset } from '../assets/models'; import { ScamInfo } from '../assets/models/ScamInfo.dto'; import { NftScamInfoModel } from './models/nft-scam-info.model'; import { getAllCollectionsFromElasticQuery, getNftWithScamInfoFromElasticQuery } from './nft-scam.queries'; +import { ELASTIC_TOKENS_INDEX } from 'src/utils/constants'; @Injectable() export class NftScamElasticService { @@ -15,7 +16,7 @@ export class NftScamElasticService { try { const query = getNftWithScamInfoFromElasticQuery(identifier); - await this.mxService.getScrollableList('tokens', 'identifier', query, async (items) => { + await this.mxService.getScrollableList(ELASTIC_TOKENS_INDEX, 'identifier', query, async (items) => { nft = items[0]; return undefined; }); @@ -32,7 +33,7 @@ export class NftScamElasticService { async setBulkNftScamInfoInElastic(nfts: Asset[], clearScamInfoIfEmpty?: boolean): Promise { if (nfts.length > 0) { try { - await this.mxService.bulkRequest('tokens', this.buildNftScamInfoBulkUpdate(nfts, clearScamInfoIfEmpty)); + await this.mxService.bulkRequest(ELASTIC_TOKENS_INDEX, this.buildNftScamInfoBulkUpdate(nfts, clearScamInfoIfEmpty)); } catch (error) { this.logger.error('Error when bulk updating nft scam info in Elastic', { path: `${NftScamElasticService.name}.${this.setBulkNftScamInfoInElastic.name}`, @@ -45,7 +46,7 @@ export class NftScamElasticService { async updateBulkNftScamInfoInElastic(updates: string[]): Promise { if (updates.length > 0) { try { - await this.mxService.bulkRequest('tokens', updates, '?timeout=1m'); + await this.mxService.bulkRequest(ELASTIC_TOKENS_INDEX, updates, '?timeout=1m'); } catch (error) { this.logger.error('Error when bulk updating nft scam info in Elastic', { path: `${NftScamElasticService.name}.${this.updateBulkNftScamInfoInElastic.name}`, @@ -58,10 +59,20 @@ export class NftScamElasticService { async setNftScamInfoManuallyInElastic(identifier: string, scamInfo: ScamInfo): Promise { try { const updates = [ - this.mxService.buildBulkUpdate('tokens', identifier, elasticDictionary.scamInfo.typeKey, scamInfo?.type ?? null), - this.mxService.buildBulkUpdate('tokens', identifier, elasticDictionary.scamInfo.infoKey, scamInfo?.info ?? null), + this.mxService.buildBulkUpdate( + ELASTIC_TOKENS_INDEX, + identifier, + elasticDictionary.scamInfo.typeKey, + scamInfo?.type ?? null, + ), + this.mxService.buildBulkUpdate( + ELASTIC_TOKENS_INDEX, + identifier, + elasticDictionary.scamInfo.infoKey, + scamInfo?.info ?? null, + ), ]; - await this.mxService.bulkRequest('tokens', updates); + await this.mxService.bulkRequest(ELASTIC_TOKENS_INDEX, updates); } catch (error) { this.logger.error('Error when manually setting nft scam info in Elastic', { path: `${NftScamElasticService.name}.${this.setNftScamInfoManuallyInElastic.name}`, @@ -73,10 +84,20 @@ export class NftScamElasticService { async setCollectionScamInfoManuallyInElastic(collection: string, scamInfo: ScamInfo): Promise { try { const updates = [ - this.mxService.buildBulkUpdate('tokens', collection, elasticDictionary.scamInfo.typeKey, scamInfo?.type ?? null), - this.mxService.buildBulkUpdate('tokens', collection, elasticDictionary.scamInfo.infoKey, scamInfo?.info ?? null), + this.mxService.buildBulkUpdate( + ELASTIC_TOKENS_INDEX, + collection, + elasticDictionary.scamInfo.typeKey, + scamInfo?.type ?? null, + ), + this.mxService.buildBulkUpdate( + ELASTIC_TOKENS_INDEX, + collection, + elasticDictionary.scamInfo.infoKey, + scamInfo?.info ?? null, + ), ]; - await this.mxService.bulkRequest('tokens', updates); + await this.mxService.bulkRequest(ELASTIC_TOKENS_INDEX, updates); } catch (error) { this.logger.error('Error when manually setting collection scam info in Elastic', { path: `${NftScamElasticService.name}.${this.setCollectionScamInfoManuallyInElastic.name}`, @@ -88,7 +109,7 @@ export class NftScamElasticService { async getAllCollectionsFromElastic(): Promise { const query = getAllCollectionsFromElasticQuery(); let collections: string[] = []; - await this.mxService.getScrollableList('tokens', 'token', query, async (items) => { + await this.mxService.getScrollableList(ELASTIC_TOKENS_INDEX, 'token', query, async (items) => { collections = collections.concat([...new Set(items.map((i) => i.token))]); }); return collections; @@ -99,14 +120,28 @@ export class NftScamElasticService { for (const nft of nfts) { if (nft.scamInfo) { updates.push( - this.mxService.buildBulkUpdate('tokens', nft.identifier, elasticDictionary.scamInfo.typeKey, nft.scamInfo.type), + this.mxService.buildBulkUpdate( + ELASTIC_TOKENS_INDEX, + nft.identifier, + elasticDictionary.scamInfo.typeKey, + nft.scamInfo.type, + ), ); updates.push( - this.mxService.buildBulkUpdate('tokens', nft.identifier, elasticDictionary.scamInfo.infoKey, nft.scamInfo.info), + this.mxService.buildBulkUpdate( + ELASTIC_TOKENS_INDEX, + nft.identifier, + elasticDictionary.scamInfo.infoKey, + nft.scamInfo.info, + ), ); } else if (clearScamInfo) { - updates.push(this.mxService.buildBulkUpdate('tokens', nft.identifier, elasticDictionary.scamInfo.typeKey, null)); - updates.push(this.mxService.buildBulkUpdate('tokens', nft.identifier, elasticDictionary.scamInfo.infoKey, null)); + updates.push( + this.mxService.buildBulkUpdate(ELASTIC_TOKENS_INDEX, nft.identifier, elasticDictionary.scamInfo.typeKey, null), + ); + updates.push( + this.mxService.buildBulkUpdate(ELASTIC_TOKENS_INDEX, nft.identifier, elasticDictionary.scamInfo.infoKey, null), + ); } } return updates; @@ -117,12 +152,16 @@ export class NftScamElasticService { for (let i = 0; i < nftsFromDb.length; i++) { const nftFromElastic = nftsFromElastic.find((nft) => nft.identifier === nftsFromDb[i].identifier); if (!nftsFromDb[i] && nftFromElastic?.[elasticDictionary.scamInfo.typeKey]) { - updates.push(this.mxService.buildBulkUpdate('tokens', nftsFromDb[i].identifier, elasticDictionary.scamInfo.typeKey, null)); - updates.push(this.mxService.buildBulkUpdate('tokens', nftsFromDb[i].identifier, elasticDictionary.scamInfo.infoKey, null)); + updates.push( + this.mxService.buildBulkUpdate(ELASTIC_TOKENS_INDEX, nftsFromDb[i].identifier, elasticDictionary.scamInfo.typeKey, null), + ); + updates.push( + this.mxService.buildBulkUpdate(ELASTIC_TOKENS_INDEX, nftsFromDb[i].identifier, elasticDictionary.scamInfo.infoKey, null), + ); } else { updates.push( this.mxService.buildBulkUpdate( - 'tokens', + ELASTIC_TOKENS_INDEX, nftsFromDb[i].identifier, elasticDictionary.scamInfo.typeKey, nftsFromDb[i].type, @@ -130,7 +169,7 @@ export class NftScamElasticService { ); updates.push( this.mxService.buildBulkUpdate( - 'tokens', + ELASTIC_TOKENS_INDEX, nftsFromDb[i].identifier, elasticDictionary.scamInfo.infoKey, nftsFromDb[i].info, diff --git a/src/modules/scam/nft-scam.service.ts b/src/modules/scam/nft-scam.service.ts index 9b05e9618..76723b7c9 100644 --- a/src/modules/scam/nft-scam.service.ts +++ b/src/modules/scam/nft-scam.service.ts @@ -13,6 +13,7 @@ import { getCollectionNftsQuery } from './nft-scam.queries'; import { AssetByIdentifierService } from '../assets'; import { Locker } from '@multiversx/sdk-nestjs-common'; import { PluginService } from 'src/common/pluggins/plugin.service'; +import { ELASTIC_TOKENS_INDEX } from 'src/utils/constants'; @Injectable() export class NftScamService { @@ -73,7 +74,7 @@ export class NftScamService { async validateOrUpdateAllNftsScamInfoForCollection(collection: string, scamEngineVersion: string): Promise { this.logger.log(`Processing scamInfo for ${collection}...`); const nftsQuery = getCollectionNftsQuery(collection); - await this.mxElasticService.getScrollableList('tokens', 'identifier', nftsQuery, async (nftsBatch) => { + await this.mxElasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'identifier', nftsQuery, async (nftsBatch) => { await this.validateOrUpdateNftsScamInfoBatch(nftsBatch, scamEngineVersion); }); } @@ -81,7 +82,7 @@ export class NftScamService { async markAllNftsForCollection(collection: string, scamEngineVersion: string, scamInfo: ScamInfo): Promise { this.logger.log(`Processing scamInfo for ${collection}...`); const nftsQuery = getCollectionNftsQuery(collection); - await this.mxElasticService.getScrollableList('tokens', 'identifier', nftsQuery, async (nftsBatch) => { + await this.mxElasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'identifier', nftsQuery, async (nftsBatch) => { await this.markNftsScamInfoBatch(nftsBatch, scamEngineVersion, scamInfo); }); } diff --git a/src/utils/constants/index.ts b/src/utils/constants/index.ts index 0cfff384d..ddff0c011 100644 --- a/src/utils/constants/index.ts +++ b/src/utils/constants/index.ts @@ -18,3 +18,19 @@ export const ELRONDNFTSWAP_KEY = 'elrondnftswap'; export const ENEFTOR_KEY = 'eneftor'; export const FRAMEIT_KEY = 'frameit'; export const ICI_KEY = 'ici'; + +export const ELASTIC_TOKENS_INDEX = 'tokens'; +export const ELASTIC_NFT_NSFW = 'nft_nsfw_mark'; +export const ELASTIC_NFT_RANK_CUSTOM = 'nft_rank_custom'; +export const ELASTIC_NFT_RANK_HASH = 'nft_custom_ranks_hash'; +export const ELASTIC_NFT_HASRARITY = 'nft_hasRarity'; +export const ELASTIC_NFT_SCORE_OPENRARITY = 'nft_score_openRarity'; +export const ELASTIC_NFT_RANK_OPENRARITY = 'nft_rank_openRarity'; +export const ELASTIC_NFT_SCORE_JACCARD = 'nft_score_jaccardDistances'; +export const ELASTIC_NFT_RANK_JACCARD = 'nft_rank_jaccardDistances'; +export const ELASTIC_NFT_SCORE_TRAIT = 'nft_score_trait'; +export const ELASTIC_NFT_RANK_TRAIT = 'nft_rank_trait'; +export const ELASTIC_NFT_SCORE_STATISTICAL = 'nft_score_statistical'; +export const ELASTIC_NFT_RANK_STATISTICAL = 'nft_rank_statistical'; +export const ELASTIC_NFT_TRAITS = 'nft_traitValues'; +export const ELASTIC_NFT_HASTRAITSUMMARY = 'nft_hasTraitSummary';