From 33037b62ab4de20f562f75b94ca22fbdba39fadc Mon Sep 17 00:00:00 2001 From: danielailie Date: Thu, 21 Sep 2023 13:20:36 +0300 Subject: [PATCH 1/9] Add marketplace info on volumeDatas --- schema.gql | 1 + .../analytics-data.getter.service.ts | 15 +++-- .../timescaledb/entities/sum-daily.entity.ts | 61 +++++++++++++++++++ .../1695284191069-AddNewMaterializedView.ts | 44 +++++++++++++ .../timescaledb/timescaledb.module.ts | 4 +- .../analytics/analytics.getter.service.ts | 2 +- .../models/analytics-aggregate-value.ts | 24 +++++++- 7 files changed, 141 insertions(+), 10 deletions(-) create mode 100644 src/common/persistence/timescaledb/migrations/1695284191069-AddNewMaterializedView.ts diff --git a/schema.gql b/schema.gql index 126ad4ae6..bc9a27386 100644 --- a/schema.gql +++ b/schema.gql @@ -57,6 +57,7 @@ input AddLikeArgs { type AnalyticsAggregateValue { avg: Float count: Float + marketplacesData: [KeyValueType] max: Float min: Float series: String diff --git a/src/common/persistence/timescaledb/analytics-data.getter.service.ts b/src/common/persistence/timescaledb/analytics-data.getter.service.ts index 5d37c40e4..a39346775 100644 --- a/src/common/persistence/timescaledb/analytics-data.getter.service.ts +++ b/src/common/persistence/timescaledb/analytics-data.getter.service.ts @@ -3,7 +3,7 @@ import { Repository } from 'typeorm'; import { InjectRepository } from '@nestjs/typeorm'; import { computeTimeInterval } from 'src/utils/analytics.utils'; import { AnalyticsArgs } from './entities/analytics.query'; -import { FloorPriceDaily, SumDaily } from './entities/sum-daily.entity'; +import { FloorPriceDaily, SumDaily, SumMarketplaceDaily } from './entities/sum-daily.entity'; import { AnalyticsAggregateValue } from 'src/modules/analytics/models/analytics-aggregate-value'; @Injectable() @@ -11,6 +11,8 @@ export class AnalyticsDataGetterService { constructor( @InjectRepository(SumDaily, 'timescaledb') private readonly sumDaily: Repository, + @InjectRepository(SumMarketplaceDaily, 'timescaledb') + private readonly sumMarketplaceDaily: Repository, @InjectRepository(FloorPriceDaily, 'timescaledb') private readonly floorPriceDaily: Repository, ) {} @@ -37,15 +39,16 @@ export class AnalyticsDataGetterService { return [[new AnalyticsAggregateValue({ value: 0, series: series })], 1]; } - return [response?.map((row) => AnalyticsAggregateValue.fromTimescaleObjext(row)) ?? [], count ?? 0]; + return [response?.map((row) => AnalyticsAggregateValue.fromTimescaleObject(row)) ?? [], count ?? 0]; } - async getVolumeData({ time, series, metric, start }: AnalyticsArgs): Promise { + async getVolumeDataWithMarketplaces({ time, series, metric, start }: AnalyticsArgs): Promise { const [startDate, endDate] = computeTimeInterval(time, start); - const query = await this.sumDaily + const query = await this.sumMarketplaceDaily .createQueryBuilder() .select("time_bucket_gapfill('1 day', time) as timestamp") .addSelect('sum(sum) as value') + .addSelect('xoxno, frameit, elrondapes, deadrare, hoghomies, elrondnftswap, aquaverse, holoride, eneftor, ici') .where('key = :metric', { metric }) .andWhere('series = :series', { series }) .andWhere(endDate ? 'time BETWEEN :startDate AND :endDate' : 'time >= :startDate', { startDate, endDate }) @@ -53,7 +56,7 @@ export class AnalyticsDataGetterService { .groupBy('timestamp') .getRawMany(); - return query?.map((row) => AnalyticsAggregateValue.fromTimescaleObjext(row)) ?? []; + return query?.map((row) => AnalyticsAggregateValue.fromTimescaleObjectWithMarketplaces(row)) ?? []; } async getFloorPriceData({ time, series, metric, start }: AnalyticsArgs): Promise { @@ -69,6 +72,6 @@ export class AnalyticsDataGetterService { .groupBy('timestamp') .getRawMany(); - return query?.map((row) => AnalyticsAggregateValue.fromTimescaleObjext(row)) ?? []; + return query?.map((row) => AnalyticsAggregateValue.fromTimescaleObject(row)) ?? []; } } diff --git a/src/common/persistence/timescaledb/entities/sum-daily.entity.ts b/src/common/persistence/timescaledb/entities/sum-daily.entity.ts index f608bc38a..e244fefee 100644 --- a/src/common/persistence/timescaledb/entities/sum-daily.entity.ts +++ b/src/common/persistence/timescaledb/entities/sum-daily.entity.ts @@ -31,6 +31,67 @@ export class SumDaily { } } +@ViewEntity({ + expression: ` + SELECT + time_bucket('1 day', timestamp) AS time, series, key, + (select SUM(CASE WHEN "marketplaceKey" ='frameit' THEN value ELSE 0 END)) as frameit, + (select SUM(CASE WHEN "marketplaceKey" ='xoxno' THEN value ELSE 0 END)) as xoxno, + (select SUM(CASE WHEN "marketplaceKey" ='elrondapes' THEN value ELSE 0 END)) as elrondapes, + (select SUM(CASE WHEN "marketplaceKey" ='deadrare' THEN value ELSE 0 END)) as deadrare, + (select SUM(CASE WHEN "marketplaceKey" ='hoghomies' THEN value ELSE 0 END)) as hoghomies, + (select SUM(CASE WHEN "marketplaceKey" ='elrondnftswap' THEN value ELSE 0 END)) as elrondnftswap, + (select SUM(CASE WHEN "marketplaceKey" ='aquaverse' THEN value ELSE 0 END)) as aquaverse, + (select SUM(CASE WHEN "marketplaceKey" ='holoride' THEN value ELSE 0 END)) as holoride, + (select SUM(CASE WHEN "marketplaceKey" ='eneftor' THEN value ELSE 0 END)) as eneftor, + (select SUM(CASE WHEN "marketplaceKey" ='ici' THEN value ELSE 0 END)) as ici, + sum(value) AS sum + FROM "hyper_nfts_analytics" + WHERE key = 'volumeUSD' + GROUP BY time, series, key; + `, + materialized: true, + name: 'nfts_sum_marketplace_daily', +}) +export class SumMarketplaceDaily { + @ViewColumn() + @PrimaryColumn() + time: Date = new Date(); + + @ViewColumn() + sum = '0'; + @ViewColumn() + frameit = '0'; + @ViewColumn() + xoxno = '0'; + @ViewColumn() + elrondapes = '0'; + @ViewColumn() + deadrare = '0'; + @ViewColumn() + hoghomies = '0'; + @ViewColumn() + elrondnftswap = '0'; + @ViewColumn() + aquaverse = '0'; + @ViewColumn() + holoride = '0'; + @ViewColumn() + eneftor = '0'; + @ViewColumn() + ici = '0'; + + @ViewColumn() + series?: string; + + @ViewColumn() + key?: string; + + constructor(init?: Partial) { + Object.assign(this, init); + } +} + @ViewEntity({ expression: ` SELECT diff --git a/src/common/persistence/timescaledb/migrations/1695284191069-AddNewMaterializedView.ts b/src/common/persistence/timescaledb/migrations/1695284191069-AddNewMaterializedView.ts new file mode 100644 index 000000000..3c459a91d --- /dev/null +++ b/src/common/persistence/timescaledb/migrations/1695284191069-AddNewMaterializedView.ts @@ -0,0 +1,44 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddNewMaterializedView1695284191069 implements MigrationInterface { + name = 'AddNewMaterializedView1695284191069'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE MATERIALIZED VIEW "nfts_sum_marketplace_daily" WITH (timescaledb.continuous) AS + SELECT + time_bucket('1 day', timestamp) AS time, series, key, + (select SUM(CASE WHEN "marketplaceKey" ='frameit' THEN value ELSE 0 END)) as frameit, + (select SUM(CASE WHEN "marketplaceKey" ='xoxno' THEN value ELSE 0 END)) as xoxno, + (select SUM(CASE WHEN "marketplaceKey" ='elrondapes' THEN value ELSE 0 END)) as elrondapes, + (select SUM(CASE WHEN "marketplaceKey" ='deadrare' THEN value ELSE 0 END)) as deadrare, + (select SUM(CASE WHEN "marketplaceKey" ='hoghomies' THEN value ELSE 0 END)) as hoghomies, + (select SUM(CASE WHEN "marketplaceKey" ='elrondnftswap' THEN value ELSE 0 END)) as elrondnftswap, + (select SUM(CASE WHEN "marketplaceKey" ='aquaverse' THEN value ELSE 0 END)) as aquaverse, + (select SUM(CASE WHEN "marketplaceKey" ='holoride' THEN value ELSE 0 END)) as holoride, + (select SUM(CASE WHEN "marketplaceKey" ='eneftor' THEN value ELSE 0 END)) as eneftor, + (select SUM(CASE WHEN "marketplaceKey" ='ici' THEN value ELSE 0 END)) as ici, + sum(value) AS sum + FROM "hyper_nfts_analytics" + WHERE key = 'volumeUSD' + GROUP BY time, series, key; + `); + await queryRunner.query( + `INSERT INTO "typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'nfts_sum_marketplace_daily', + "SELECT\n time_bucket('1 day', timestamp) AS time, series, key,\n (select SUM(CASE WHEN \"marketplaceKey\" ='frameit' THEN value ELSE 0 END)) as frameit,\n (select SUM(CASE WHEN \"marketplaceKey\" ='xoxno' THEN value ELSE 0 END)) as xoxno,\n (select SUM(CASE WHEN \"marketplaceKey\" ='elrondapes' THEN value ELSE 0 END)) as elrondapes,\n (select SUM(CASE WHEN \"marketplaceKey\" ='deadrare' THEN value ELSE 0 END)) as deadrare,\n (select SUM(CASE WHEN \"marketplaceKey\" ='hoghomies' THEN value ELSE 0 END)) as hoghomies,\n (select SUM(CASE WHEN \"marketplaceKey\" ='elrondnftswap' THEN value ELSE 0 END)) as elrondnftswap,\n (select SUM(CASE WHEN \"marketplaceKey\" ='aquaverse' THEN value ELSE 0 END)) as aquaverse,\n (select SUM(CASE WHEN \"marketplaceKey\" ='holoride' THEN value ELSE 0 END)) as holoride,\n (select SUM(CASE WHEN \"marketplaceKey\" ='eneftor' THEN value ELSE 0 END)) as eneftor,\n (select SUM(CASE WHEN \"marketplaceKey\" ='ici' THEN value ELSE 0 END)) as ici,\n sum(value) AS sum\n FROM \"hyper_nfts_analytics\"\n WHERE key = 'volumeUSD'\n GROUP BY time, series, key;", + ], + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DELETE FROM "typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, [ + 'MATERIALIZED_VIEW', + 'nfts_sum_marketplace_daily', + 'public', + ]); + await queryRunner.query(`DROP MATERIALIZED VIEW "nfts_sum_marketplace_daily"`); + } +} diff --git a/src/common/persistence/timescaledb/timescaledb.module.ts b/src/common/persistence/timescaledb/timescaledb.module.ts index cdb06828b..d934e6a40 100644 --- a/src/common/persistence/timescaledb/timescaledb.module.ts +++ b/src/common/persistence/timescaledb/timescaledb.module.ts @@ -6,7 +6,7 @@ import { ApiConfigService } from 'src/modules/common/api-config/api.config.servi import { AnalyticsDataGetterService } from './analytics-data.getter.service'; import { AnalyticsDataSetterService } from './analytics-data.setter.service'; import { XNftsAnalyticsEntity } from './entities/analytics.entity'; -import { FloorPriceDaily, SumDaily } from './entities/sum-daily.entity'; +import { FloorPriceDaily, SumDaily, SumMarketplaceDaily } from './entities/sum-daily.entity'; import { SumWeekly } from './entities/sum-weekly.entity'; @Module({ @@ -34,7 +34,7 @@ import { SumWeekly } from './entities/sum-weekly.entity'; }), inject: [ApiConfigService], }), - TypeOrmModule.forFeature([XNftsAnalyticsEntity, SumDaily, SumWeekly, FloorPriceDaily], 'timescaledb'), + TypeOrmModule.forFeature([XNftsAnalyticsEntity, SumDaily, SumMarketplaceDaily, SumWeekly, FloorPriceDaily], 'timescaledb'), ], providers: [AnalyticsDataGetterService, AnalyticsDataSetterService], exports: [AnalyticsDataGetterService, AnalyticsDataSetterService], diff --git a/src/modules/analytics/analytics.getter.service.ts b/src/modules/analytics/analytics.getter.service.ts index 0152a10f7..119ad2578 100644 --- a/src/modules/analytics/analytics.getter.service.ts +++ b/src/modules/analytics/analytics.getter.service.ts @@ -19,7 +19,7 @@ export class AnalyticsGetterService { return await this.cacheService.getOrSet( cacheKey, () => - this.analyticsQuery.getVolumeData({ + this.analyticsQuery.getVolumeDataWithMarketplaces({ series, metric, time, diff --git a/src/modules/analytics/models/analytics-aggregate-value.ts b/src/modules/analytics/models/analytics-aggregate-value.ts index 9108ce720..6c17bd6d4 100644 --- a/src/modules/analytics/models/analytics-aggregate-value.ts +++ b/src/modules/analytics/models/analytics-aggregate-value.ts @@ -1,5 +1,6 @@ import { Field, Float, ObjectType } from '@nestjs/graphql'; import * as moment from 'moment'; +import { KeyValueType } from 'src/modules/assets/models'; @ObjectType() export class AnalyticsAggregateValue { @@ -23,6 +24,8 @@ export class AnalyticsAggregateValue { @Field(() => Float, { nullable: true }) avg?: number; + @Field(() => [KeyValueType], { nullable: 'itemsAndList' }) + marketplacesData: KeyValueType[]; constructor(init?: Partial) { Object.assign(this, init); @@ -39,12 +42,31 @@ export class AnalyticsAggregateValue { avg: row.avg ?? 0, }); } + static fromTimescaleObject(row: any) { + return new AnalyticsAggregateValue({ + series: row.series, + time: moment.utc(row.timestamp ?? row.time).format('yyyy-MM-DD HH:mm:ss'), + value: row.value ?? 0, + }); + } - static fromTimescaleObjext(row: any) { + static fromTimescaleObjectWithMarketplaces(row: any) { return new AnalyticsAggregateValue({ series: row.series, time: moment.utc(row.timestamp ?? row.time).format('yyyy-MM-DD HH:mm:ss'), value: row.value ?? 0, + marketplacesData: [ + new KeyValueType({ key: row.xoxno.name, value: row.xoxno }), + new KeyValueType({ key: row.frameit.name, value: row.frameit }), + new KeyValueType({ key: row.frameit.name, value: row.deadrare }), + new KeyValueType({ key: row.frameit.name, value: row.elrondapes }), + new KeyValueType({ key: row.frameit.name, value: row.elrondnftswap }), + new KeyValueType({ key: row.frameit.name, value: row.eneftor }), + new KeyValueType({ key: row.frameit.name, value: row.hoghomies }), + new KeyValueType({ key: row.frameit.name, value: row.holoride }), + new KeyValueType({ key: row.frameit.name, value: row.aquaverse }), + new KeyValueType({ key: row.frameit.name, value: row.ici }), + ], }); } } From 21f04b0d23644772d50558b5355b661da13e7725 Mon Sep 17 00:00:00 2001 From: danielailie Date: Thu, 21 Sep 2023 14:16:15 +0300 Subject: [PATCH 2/9] Update materialied view --- .../timescaledb/entities/sum-daily.entity.ts | 20 ++++----- .../1695284191069-AddNewMaterializedView.ts | 44 ------------------- ...4912071-AddDailyViewWithMarketplaceInfo.ts | 44 +++++++++++++++++++ 3 files changed, 54 insertions(+), 54 deletions(-) delete mode 100644 src/common/persistence/timescaledb/migrations/1695284191069-AddNewMaterializedView.ts create mode 100644 src/common/persistence/timescaledb/migrations/1695294912071-AddDailyViewWithMarketplaceInfo.ts diff --git a/src/common/persistence/timescaledb/entities/sum-daily.entity.ts b/src/common/persistence/timescaledb/entities/sum-daily.entity.ts index e244fefee..5dbdb6557 100644 --- a/src/common/persistence/timescaledb/entities/sum-daily.entity.ts +++ b/src/common/persistence/timescaledb/entities/sum-daily.entity.ts @@ -35,16 +35,16 @@ export class SumDaily { expression: ` SELECT time_bucket('1 day', timestamp) AS time, series, key, - (select SUM(CASE WHEN "marketplaceKey" ='frameit' THEN value ELSE 0 END)) as frameit, - (select SUM(CASE WHEN "marketplaceKey" ='xoxno' THEN value ELSE 0 END)) as xoxno, - (select SUM(CASE WHEN "marketplaceKey" ='elrondapes' THEN value ELSE 0 END)) as elrondapes, - (select SUM(CASE WHEN "marketplaceKey" ='deadrare' THEN value ELSE 0 END)) as deadrare, - (select SUM(CASE WHEN "marketplaceKey" ='hoghomies' THEN value ELSE 0 END)) as hoghomies, - (select SUM(CASE WHEN "marketplaceKey" ='elrondnftswap' THEN value ELSE 0 END)) as elrondnftswap, - (select SUM(CASE WHEN "marketplaceKey" ='aquaverse' THEN value ELSE 0 END)) as aquaverse, - (select SUM(CASE WHEN "marketplaceKey" ='holoride' THEN value ELSE 0 END)) as holoride, - (select SUM(CASE WHEN "marketplaceKey" ='eneftor' THEN value ELSE 0 END)) as eneftor, - (select SUM(CASE WHEN "marketplaceKey" ='ici' THEN value ELSE 0 END)) as ici, + SUM(CASE WHEN "marketplaceKey" ='frameit' THEN value ELSE 0 END) as frameit, + SUM(CASE WHEN "marketplaceKey" ='xoxno' THEN value ELSE 0 END) as xoxno, + SUM(CASE WHEN "marketplaceKey" ='elrondapes' THEN value ELSE 0 END) as elrondapes, + SUM(CASE WHEN "marketplaceKey" ='deadrare' THEN value ELSE 0 END) as deadrare, + SUM(CASE WHEN "marketplaceKey" ='hoghomies' THEN value ELSE 0 END) as hoghomies, + SUM(CASE WHEN "marketplaceKey" ='elrondnftswap' THEN value ELSE 0 END) as elrondnftswap, + SUM(CASE WHEN "marketplaceKey" ='aquaverse' THEN value ELSE 0 END) as aquaverse, + SUM(CASE WHEN "marketplaceKey" ='holoride' THEN value ELSE 0 END) as holoride, + SUM(CASE WHEN "marketplaceKey" ='eneftor' THEN value ELSE 0 END) as eneftor, + SUM(CASE WHEN "marketplaceKey" ='ici' THEN value ELSE 0 END) as ici, sum(value) AS sum FROM "hyper_nfts_analytics" WHERE key = 'volumeUSD' diff --git a/src/common/persistence/timescaledb/migrations/1695284191069-AddNewMaterializedView.ts b/src/common/persistence/timescaledb/migrations/1695284191069-AddNewMaterializedView.ts deleted file mode 100644 index 3c459a91d..000000000 --- a/src/common/persistence/timescaledb/migrations/1695284191069-AddNewMaterializedView.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { MigrationInterface, QueryRunner } from 'typeorm'; - -export class AddNewMaterializedView1695284191069 implements MigrationInterface { - name = 'AddNewMaterializedView1695284191069'; - - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(`CREATE MATERIALIZED VIEW "nfts_sum_marketplace_daily" WITH (timescaledb.continuous) AS - SELECT - time_bucket('1 day', timestamp) AS time, series, key, - (select SUM(CASE WHEN "marketplaceKey" ='frameit' THEN value ELSE 0 END)) as frameit, - (select SUM(CASE WHEN "marketplaceKey" ='xoxno' THEN value ELSE 0 END)) as xoxno, - (select SUM(CASE WHEN "marketplaceKey" ='elrondapes' THEN value ELSE 0 END)) as elrondapes, - (select SUM(CASE WHEN "marketplaceKey" ='deadrare' THEN value ELSE 0 END)) as deadrare, - (select SUM(CASE WHEN "marketplaceKey" ='hoghomies' THEN value ELSE 0 END)) as hoghomies, - (select SUM(CASE WHEN "marketplaceKey" ='elrondnftswap' THEN value ELSE 0 END)) as elrondnftswap, - (select SUM(CASE WHEN "marketplaceKey" ='aquaverse' THEN value ELSE 0 END)) as aquaverse, - (select SUM(CASE WHEN "marketplaceKey" ='holoride' THEN value ELSE 0 END)) as holoride, - (select SUM(CASE WHEN "marketplaceKey" ='eneftor' THEN value ELSE 0 END)) as eneftor, - (select SUM(CASE WHEN "marketplaceKey" ='ici' THEN value ELSE 0 END)) as ici, - sum(value) AS sum - FROM "hyper_nfts_analytics" - WHERE key = 'volumeUSD' - GROUP BY time, series, key; - `); - await queryRunner.query( - `INSERT INTO "typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, - [ - 'public', - 'MATERIALIZED_VIEW', - 'nfts_sum_marketplace_daily', - "SELECT\n time_bucket('1 day', timestamp) AS time, series, key,\n (select SUM(CASE WHEN \"marketplaceKey\" ='frameit' THEN value ELSE 0 END)) as frameit,\n (select SUM(CASE WHEN \"marketplaceKey\" ='xoxno' THEN value ELSE 0 END)) as xoxno,\n (select SUM(CASE WHEN \"marketplaceKey\" ='elrondapes' THEN value ELSE 0 END)) as elrondapes,\n (select SUM(CASE WHEN \"marketplaceKey\" ='deadrare' THEN value ELSE 0 END)) as deadrare,\n (select SUM(CASE WHEN \"marketplaceKey\" ='hoghomies' THEN value ELSE 0 END)) as hoghomies,\n (select SUM(CASE WHEN \"marketplaceKey\" ='elrondnftswap' THEN value ELSE 0 END)) as elrondnftswap,\n (select SUM(CASE WHEN \"marketplaceKey\" ='aquaverse' THEN value ELSE 0 END)) as aquaverse,\n (select SUM(CASE WHEN \"marketplaceKey\" ='holoride' THEN value ELSE 0 END)) as holoride,\n (select SUM(CASE WHEN \"marketplaceKey\" ='eneftor' THEN value ELSE 0 END)) as eneftor,\n (select SUM(CASE WHEN \"marketplaceKey\" ='ici' THEN value ELSE 0 END)) as ici,\n sum(value) AS sum\n FROM \"hyper_nfts_analytics\"\n WHERE key = 'volumeUSD'\n GROUP BY time, series, key;", - ], - ); - } - - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`DELETE FROM "typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, [ - 'MATERIALIZED_VIEW', - 'nfts_sum_marketplace_daily', - 'public', - ]); - await queryRunner.query(`DROP MATERIALIZED VIEW "nfts_sum_marketplace_daily"`); - } -} diff --git a/src/common/persistence/timescaledb/migrations/1695294912071-AddDailyViewWithMarketplaceInfo.ts b/src/common/persistence/timescaledb/migrations/1695294912071-AddDailyViewWithMarketplaceInfo.ts new file mode 100644 index 000000000..e567f4084 --- /dev/null +++ b/src/common/persistence/timescaledb/migrations/1695294912071-AddDailyViewWithMarketplaceInfo.ts @@ -0,0 +1,44 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddDailyViewWithMarketplaceInfo1695294912071 implements MigrationInterface { + name = 'AddDailyViewWithMarketplaceInfo1695294912071'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE MATERIALIZED VIEW "nfts_sum_marketplace_daily" WITH (timescaledb.continuous) AS + SELECT + time_bucket('1 day', timestamp) AS time, series, key, + SUM(CASE WHEN "marketplaceKey" ='frameit' THEN value ELSE 0 END) as frameit, + SUM(CASE WHEN "marketplaceKey" ='xoxno' THEN value ELSE 0 END) as xoxno, + SUM(CASE WHEN "marketplaceKey" ='elrondapes' THEN value ELSE 0 END) as elrondapes, + SUM(CASE WHEN "marketplaceKey" ='deadrare' THEN value ELSE 0 END) as deadrare, + SUM(CASE WHEN "marketplaceKey" ='hoghomies' THEN value ELSE 0 END) as hoghomies, + SUM(CASE WHEN "marketplaceKey" ='elrondnftswap' THEN value ELSE 0 END) as elrondnftswap, + SUM(CASE WHEN "marketplaceKey" ='aquaverse' THEN value ELSE 0 END) as aquaverse, + SUM(CASE WHEN "marketplaceKey" ='holoride' THEN value ELSE 0 END) as holoride, + SUM(CASE WHEN "marketplaceKey" ='eneftor' THEN value ELSE 0 END) as eneftor, + SUM(CASE WHEN "marketplaceKey" ='ici' THEN value ELSE 0 END) as ici, + sum(value) AS sum + FROM "hyper_nfts_analytics" + WHERE key = 'volumeUSD' + GROUP BY time, series, key; + `); + await queryRunner.query( + `INSERT INTO "typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'nfts_sum_marketplace_daily', + "SELECT\n time_bucket('1 day', timestamp) AS time, series, key,\n SUM(CASE WHEN \"marketplaceKey\" ='frameit' THEN value ELSE 0 END) as frameit,\n SUM(CASE WHEN \"marketplaceKey\" ='xoxno' THEN value ELSE 0 END) as xoxno,\n SUM(CASE WHEN \"marketplaceKey\" ='elrondapes' THEN value ELSE 0 END) as elrondapes,\n SUM(CASE WHEN \"marketplaceKey\" ='deadrare' THEN value ELSE 0 END) as deadrare,\n SUM(CASE WHEN \"marketplaceKey\" ='hoghomies' THEN value ELSE 0 END) as hoghomies,\n SUM(CASE WHEN \"marketplaceKey\" ='elrondnftswap' THEN value ELSE 0 END) as elrondnftswap,\n SUM(CASE WHEN \"marketplaceKey\" ='aquaverse' THEN value ELSE 0 END) as aquaverse,\n SUM(CASE WHEN \"marketplaceKey\" ='holoride' THEN value ELSE 0 END) as holoride,\n SUM(CASE WHEN \"marketplaceKey\" ='eneftor' THEN value ELSE 0 END) as eneftor,\n SUM(CASE WHEN \"marketplaceKey\" ='ici' THEN value ELSE 0 END) as ici,\n sum(value) AS sum\n FROM \"hyper_nfts_analytics\"\n WHERE key = 'volumeUSD'\n GROUP BY time, series, key;", + ], + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DELETE FROM "typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, [ + 'MATERIALIZED_VIEW', + 'nfts_sum_marketplace_daily', + 'public', + ]); + await queryRunner.query(`DROP MATERIALIZED VIEW "nfts_sum_marketplace_daily"`); + } +} From 14bd58c0872687e11965224dc363cbae44da1e5a Mon Sep 17 00:00:00 2001 From: danielailie Date: Thu, 21 Sep 2023 14:53:59 +0300 Subject: [PATCH 3/9] Fix query --- .../persistence/timescaledb/analytics-data.getter.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/persistence/timescaledb/analytics-data.getter.service.ts b/src/common/persistence/timescaledb/analytics-data.getter.service.ts index a39346775..e8cbbeca9 100644 --- a/src/common/persistence/timescaledb/analytics-data.getter.service.ts +++ b/src/common/persistence/timescaledb/analytics-data.getter.service.ts @@ -53,7 +53,7 @@ export class AnalyticsDataGetterService { .andWhere('series = :series', { series }) .andWhere(endDate ? 'time BETWEEN :startDate AND :endDate' : 'time >= :startDate', { startDate, endDate }) .orderBy('timestamp', 'ASC') - .groupBy('timestamp') + .groupBy('timestamp,xoxno, frameit, elrondapes, deadrare, hoghomies, elrondnftswap, aquaverse, holoride, eneftor, ici') .getRawMany(); return query?.map((row) => AnalyticsAggregateValue.fromTimescaleObjectWithMarketplaces(row)) ?? []; From 0e72dab1cac2d4e2588099658bde1f18d35c3202 Mon Sep 17 00:00:00 2001 From: danielailie Date: Thu, 21 Sep 2023 15:33:48 +0300 Subject: [PATCH 4/9] Fix get properties name --- schema.gql | 4 +-- .../models/analytics-aggregate-value.ts | 33 +++++++++++++------ src/modules/assets/models/Metadata.dto.ts | 4 +-- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/schema.gql b/schema.gql index bc9a27386..4db4266ec 100644 --- a/schema.gql +++ b/schema.gql @@ -823,8 +823,8 @@ input IssueCollectionArgs { } type KeyValueType { - key: String! - value: String! + key: String + value: String } type Marketplace { diff --git a/src/modules/analytics/models/analytics-aggregate-value.ts b/src/modules/analytics/models/analytics-aggregate-value.ts index 6c17bd6d4..c650a248a 100644 --- a/src/modules/analytics/models/analytics-aggregate-value.ts +++ b/src/modules/analytics/models/analytics-aggregate-value.ts @@ -51,22 +51,35 @@ export class AnalyticsAggregateValue { } static fromTimescaleObjectWithMarketplaces(row: any) { + const rowProperties = proxiedPropertiesOf(row); + return new AnalyticsAggregateValue({ series: row.series, time: moment.utc(row.timestamp ?? row.time).format('yyyy-MM-DD HH:mm:ss'), value: row.value ?? 0, marketplacesData: [ - new KeyValueType({ key: row.xoxno.name, value: row.xoxno }), - new KeyValueType({ key: row.frameit.name, value: row.frameit }), - new KeyValueType({ key: row.frameit.name, value: row.deadrare }), - new KeyValueType({ key: row.frameit.name, value: row.elrondapes }), - new KeyValueType({ key: row.frameit.name, value: row.elrondnftswap }), - new KeyValueType({ key: row.frameit.name, value: row.eneftor }), - new KeyValueType({ key: row.frameit.name, value: row.hoghomies }), - new KeyValueType({ key: row.frameit.name, value: row.holoride }), - new KeyValueType({ key: row.frameit.name, value: row.aquaverse }), - new KeyValueType({ key: row.frameit.name, value: row.ici }), + new KeyValueType({ key: rowProperties.xoxno, value: row.xoxno }), + new KeyValueType({ key: rowProperties.frameit, value: row.frameit }), + new KeyValueType({ key: rowProperties.deadrare, value: row.deadrare }), + new KeyValueType({ key: rowProperties.elrondapes, value: row.elrondapes }), + new KeyValueType({ key: rowProperties.elrondnftswap, value: row.elrondnftswap }), + new KeyValueType({ key: rowProperties.eneftor, value: row.eneftor }), + new KeyValueType({ key: rowProperties.hoghomies, value: row.hoghomies }), + new KeyValueType({ key: rowProperties.holoride, value: row.holoride }), + new KeyValueType({ key: rowProperties.aquaverse, value: row.aquaverse }), + new KeyValueType({ key: rowProperties.ici, value: row.ici }), ], }); } } + +export function proxiedPropertiesOf(obj?: TObj) { + return new Proxy( + {}, + { + get: (_, prop) => prop, + }, + ) as { + [P in keyof TObj]?: P; + }; +} diff --git a/src/modules/assets/models/Metadata.dto.ts b/src/modules/assets/models/Metadata.dto.ts index 3f6fbb9de..77edb9ba8 100644 --- a/src/modules/assets/models/Metadata.dto.ts +++ b/src/modules/assets/models/Metadata.dto.ts @@ -76,9 +76,9 @@ export class AttributeType { @ObjectType() export class KeyValueType { - @Field() + @Field({ nullable: true }) key: String; - @Field() + @Field({ nullable: true }) value: String; constructor(init?: Partial) { Object.assign(this, init); From 712c1f6f63f659de0b4de822052ac4cda2bb036b Mon Sep 17 00:00:00 2001 From: danielailie Date: Thu, 21 Sep 2023 15:41:35 +0300 Subject: [PATCH 5/9] Update marketplace data --- schema.gql | 9 ++++- .../models/analytics-aggregate-value.ts | 37 ++++++++++++------- src/modules/assets/models/Metadata.dto.ts | 2 +- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/schema.gql b/schema.gql index 4db4266ec..6ed86c4c1 100644 --- a/schema.gql +++ b/schema.gql @@ -57,7 +57,7 @@ input AddLikeArgs { type AnalyticsAggregateValue { avg: Float count: Float - marketplacesData: [KeyValueType] + marketplacesData: [MarketplaceData] max: Float min: Float series: String @@ -823,7 +823,7 @@ input IssueCollectionArgs { } type KeyValueType { - key: String + key: String! value: String } @@ -840,6 +840,11 @@ type Marketplace { url: String! } +type MarketplaceData { + key: String! + value: Float +} + type MarketplaceEdge { cursor: String node: Marketplace diff --git a/src/modules/analytics/models/analytics-aggregate-value.ts b/src/modules/analytics/models/analytics-aggregate-value.ts index c650a248a..300ab6254 100644 --- a/src/modules/analytics/models/analytics-aggregate-value.ts +++ b/src/modules/analytics/models/analytics-aggregate-value.ts @@ -1,4 +1,4 @@ -import { Field, Float, ObjectType } from '@nestjs/graphql'; +import { Field, Float, Int, ObjectType } from '@nestjs/graphql'; import * as moment from 'moment'; import { KeyValueType } from 'src/modules/assets/models'; @@ -24,8 +24,8 @@ export class AnalyticsAggregateValue { @Field(() => Float, { nullable: true }) avg?: number; - @Field(() => [KeyValueType], { nullable: 'itemsAndList' }) - marketplacesData: KeyValueType[]; + @Field(() => [MarketplaceData], { nullable: 'itemsAndList' }) + marketplacesData: MarketplaceData[]; constructor(init?: Partial) { Object.assign(this, init); @@ -58,21 +58,32 @@ export class AnalyticsAggregateValue { time: moment.utc(row.timestamp ?? row.time).format('yyyy-MM-DD HH:mm:ss'), value: row.value ?? 0, marketplacesData: [ - new KeyValueType({ key: rowProperties.xoxno, value: row.xoxno }), - new KeyValueType({ key: rowProperties.frameit, value: row.frameit }), - new KeyValueType({ key: rowProperties.deadrare, value: row.deadrare }), - new KeyValueType({ key: rowProperties.elrondapes, value: row.elrondapes }), - new KeyValueType({ key: rowProperties.elrondnftswap, value: row.elrondnftswap }), - new KeyValueType({ key: rowProperties.eneftor, value: row.eneftor }), - new KeyValueType({ key: rowProperties.hoghomies, value: row.hoghomies }), - new KeyValueType({ key: rowProperties.holoride, value: row.holoride }), - new KeyValueType({ key: rowProperties.aquaverse, value: row.aquaverse }), - new KeyValueType({ key: rowProperties.ici, value: row.ici }), + new MarketplaceData({ key: rowProperties.xoxno, value: row.xoxno }), + new MarketplaceData({ key: rowProperties.frameit, value: row.frameit }), + new MarketplaceData({ key: rowProperties.deadrare, value: row.deadrare }), + new MarketplaceData({ key: rowProperties.elrondapes, value: row.elrondapes }), + new MarketplaceData({ key: rowProperties.elrondnftswap, value: row.elrondnftswap }), + new MarketplaceData({ key: rowProperties.eneftor, value: row.eneftor }), + new MarketplaceData({ key: rowProperties.hoghomies, value: row.hoghomies }), + new MarketplaceData({ key: rowProperties.holoride, value: row.holoride }), + new MarketplaceData({ key: rowProperties.aquaverse, value: row.aquaverse }), + new MarketplaceData({ key: rowProperties.ici, value: row.ici }), ], }); } } +@ObjectType() +export class MarketplaceData { + @Field() + key: String; + @Field(() => Float, { nullable: true }) + value: number; + constructor(init?: Partial) { + Object.assign(this, init); + } +} + export function proxiedPropertiesOf(obj?: TObj) { return new Proxy( {}, diff --git a/src/modules/assets/models/Metadata.dto.ts b/src/modules/assets/models/Metadata.dto.ts index 77edb9ba8..b275e3099 100644 --- a/src/modules/assets/models/Metadata.dto.ts +++ b/src/modules/assets/models/Metadata.dto.ts @@ -76,7 +76,7 @@ export class AttributeType { @ObjectType() export class KeyValueType { - @Field({ nullable: true }) + @Field() key: String; @Field({ nullable: true }) value: String; From def7228f2b30dc5df031ffdee7ef3e8415e281f4 Mon Sep 17 00:00:00 2001 From: danielailie Date: Thu, 21 Sep 2023 15:44:48 +0300 Subject: [PATCH 6/9] cleanup --- schema.gql | 2 +- src/modules/analytics/models/analytics-aggregate-value.ts | 1 - src/modules/assets/models/Metadata.dto.ts | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/schema.gql b/schema.gql index 6ed86c4c1..5b7e84b37 100644 --- a/schema.gql +++ b/schema.gql @@ -824,7 +824,7 @@ input IssueCollectionArgs { type KeyValueType { key: String! - value: String + value: String! } type Marketplace { diff --git a/src/modules/analytics/models/analytics-aggregate-value.ts b/src/modules/analytics/models/analytics-aggregate-value.ts index 300ab6254..c59983011 100644 --- a/src/modules/analytics/models/analytics-aggregate-value.ts +++ b/src/modules/analytics/models/analytics-aggregate-value.ts @@ -1,6 +1,5 @@ import { Field, Float, Int, ObjectType } from '@nestjs/graphql'; import * as moment from 'moment'; -import { KeyValueType } from 'src/modules/assets/models'; @ObjectType() export class AnalyticsAggregateValue { diff --git a/src/modules/assets/models/Metadata.dto.ts b/src/modules/assets/models/Metadata.dto.ts index b275e3099..3f6fbb9de 100644 --- a/src/modules/assets/models/Metadata.dto.ts +++ b/src/modules/assets/models/Metadata.dto.ts @@ -78,7 +78,7 @@ export class AttributeType { export class KeyValueType { @Field() key: String; - @Field({ nullable: true }) + @Field() value: String; constructor(init?: Partial) { Object.assign(this, init); From 14b9933fad157c7b6d1e9572bec5f195f9374590 Mon Sep 17 00:00:00 2001 From: danielailie Date: Thu, 21 Sep 2023 16:19:43 +0300 Subject: [PATCH 7/9] Fix query --- .../analytics-data.getter.service.ts | 6 ++++-- .../models/analytics-aggregate-value.ts | 20 +++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/common/persistence/timescaledb/analytics-data.getter.service.ts b/src/common/persistence/timescaledb/analytics-data.getter.service.ts index e8cbbeca9..4ca106b0f 100644 --- a/src/common/persistence/timescaledb/analytics-data.getter.service.ts +++ b/src/common/persistence/timescaledb/analytics-data.getter.service.ts @@ -48,12 +48,14 @@ export class AnalyticsDataGetterService { .createQueryBuilder() .select("time_bucket_gapfill('1 day', time) as timestamp") .addSelect('sum(sum) as value') - .addSelect('xoxno, frameit, elrondapes, deadrare, hoghomies, elrondnftswap, aquaverse, holoride, eneftor, ici') + .addSelect( + 'sum(sum) as value, sum(xoxno) as xoxno, sum(frameit) as frameit, sum(elrondapes) as elrondapes, sum(deadrare) as deadrare, sum(hoghomies) as hoghomies, sum(elrondnftswap) as elrondnftswap, sum(aquaverse) as aquaverse, sum(holoride) as holoride, sum(eneftor) as eneftor,sum(ici) as ici ', + ) .where('key = :metric', { metric }) .andWhere('series = :series', { series }) .andWhere(endDate ? 'time BETWEEN :startDate AND :endDate' : 'time >= :startDate', { startDate, endDate }) .orderBy('timestamp', 'ASC') - .groupBy('timestamp,xoxno, frameit, elrondapes, deadrare, hoghomies, elrondnftswap, aquaverse, holoride, eneftor, ici') + .groupBy('timestamp') .getRawMany(); return query?.map((row) => AnalyticsAggregateValue.fromTimescaleObjectWithMarketplaces(row)) ?? []; diff --git a/src/modules/analytics/models/analytics-aggregate-value.ts b/src/modules/analytics/models/analytics-aggregate-value.ts index c59983011..7836e37c1 100644 --- a/src/modules/analytics/models/analytics-aggregate-value.ts +++ b/src/modules/analytics/models/analytics-aggregate-value.ts @@ -57,16 +57,16 @@ export class AnalyticsAggregateValue { time: moment.utc(row.timestamp ?? row.time).format('yyyy-MM-DD HH:mm:ss'), value: row.value ?? 0, marketplacesData: [ - new MarketplaceData({ key: rowProperties.xoxno, value: row.xoxno }), - new MarketplaceData({ key: rowProperties.frameit, value: row.frameit }), - new MarketplaceData({ key: rowProperties.deadrare, value: row.deadrare }), - new MarketplaceData({ key: rowProperties.elrondapes, value: row.elrondapes }), - new MarketplaceData({ key: rowProperties.elrondnftswap, value: row.elrondnftswap }), - new MarketplaceData({ key: rowProperties.eneftor, value: row.eneftor }), - new MarketplaceData({ key: rowProperties.hoghomies, value: row.hoghomies }), - new MarketplaceData({ key: rowProperties.holoride, value: row.holoride }), - new MarketplaceData({ key: rowProperties.aquaverse, value: row.aquaverse }), - new MarketplaceData({ key: rowProperties.ici, value: row.ici }), + new MarketplaceData({ key: rowProperties.xoxno, value: row.xoxno ?? 0 }), + new MarketplaceData({ key: rowProperties.frameit, value: row.frameit ?? 0 }), + new MarketplaceData({ key: rowProperties.deadrare, value: row.deadrare ?? 0 }), + new MarketplaceData({ key: rowProperties.elrondapes, value: row.elrondapes ?? 0 }), + new MarketplaceData({ key: rowProperties.elrondnftswap, value: row.elrondnftswap ?? 0 }), + new MarketplaceData({ key: rowProperties.eneftor, value: row.eneftor ?? 0 }), + new MarketplaceData({ key: rowProperties.hoghomies, value: row.hoghomies ?? 0 }), + new MarketplaceData({ key: rowProperties.holoride, value: row.holoride ?? 0 }), + new MarketplaceData({ key: rowProperties.aquaverse, value: row.aquaverse ?? 0 }), + new MarketplaceData({ key: rowProperties.ici, value: row.ici ?? 0 }), ], }); } From 0f3dc8f6c09e94c39c7f8dd88c83c4d13a62d271 Mon Sep 17 00:00:00 2001 From: danielailie Date: Mon, 25 Sep 2023 15:19:57 +0300 Subject: [PATCH 8/9] Fix tests references --- .../timescaledb/tests/analytics-data.getter.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/persistence/timescaledb/tests/analytics-data.getter.spec.ts b/src/common/persistence/timescaledb/tests/analytics-data.getter.spec.ts index 922fe2629..5ef2dfeb2 100644 --- a/src/common/persistence/timescaledb/tests/analytics-data.getter.spec.ts +++ b/src/common/persistence/timescaledb/tests/analytics-data.getter.spec.ts @@ -190,7 +190,7 @@ describe('Analytics Data Getter Service', () => { orderBy, groupBy, })); - const result = await service.getVolumeData({ + const result = await service.getVolumeDataWithMarketplaces({ time: '10d', metric: 'test', series: 'test', @@ -222,7 +222,7 @@ describe('Analytics Data Getter Service', () => { groupBy, })); - const response = await service.getVolumeData({ + const response = await service.getVolumeDataWithMarketplaces({ time: '10d', metric: 'test', series: 'test', From 00d689b5eeaddd0e4c468a204a8577833a399350 Mon Sep 17 00:00:00 2001 From: danielailie Date: Mon, 25 Sep 2023 15:43:14 +0300 Subject: [PATCH 9/9] Add tests missing dependencies --- .../timescaledb/tests/analytics-data.getter.spec.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/common/persistence/timescaledb/tests/analytics-data.getter.spec.ts b/src/common/persistence/timescaledb/tests/analytics-data.getter.spec.ts index 5ef2dfeb2..134b4d8ca 100644 --- a/src/common/persistence/timescaledb/tests/analytics-data.getter.spec.ts +++ b/src/common/persistence/timescaledb/tests/analytics-data.getter.spec.ts @@ -1,6 +1,6 @@ import { Test, TestingModule } from '@nestjs/testing'; import { AnalyticsDataGetterService } from '../analytics-data.getter.service'; -import { FloorPriceDaily, SumDaily } from '../entities/sum-daily.entity'; +import { FloorPriceDaily, SumDaily, SumMarketplaceDaily } from '../entities/sum-daily.entity'; import { getRepositoryToken } from '@nestjs/typeorm'; import { AnalyticsAggregateValue } from 'src/modules/analytics/models/analytics-aggregate-value'; @@ -24,6 +24,11 @@ describe('Analytics Data Getter Service', () => { provide: getRepositoryToken(SumDaily, 'timescaledb'), useFactory: () => ({}), }, + { + provide: getRepositoryToken(SumMarketplaceDaily, 'timescaledb'), + useFactory: () => ({}), + }, + { provide: getRepositoryToken(FloorPriceDaily, 'timescaledb'), useFactory: () => ({}), @@ -175,10 +180,10 @@ describe('Analytics Data Getter Service', () => { }); }); - describe('getVolumeData', () => { + describe('getVolumeDataWithMarketplaces', () => { it('returns empty list when no data for specific collection', async () => { const getRawMany = jest.fn().mockReturnValueOnce([]); - const sumDailyRepository = module.get(getRepositoryToken(SumDaily, 'timescaledb')); + const sumDailyRepository = module.get(getRepositoryToken(SumMarketplaceDaily, 'timescaledb')); sumDailyRepository.createQueryBuilder = jest.fn(() => ({ select, offset, @@ -209,7 +214,7 @@ describe('Analytics Data Getter Service', () => { new AnalyticsAggregateValue({ value: 2, series: 'test' }), new AnalyticsAggregateValue({ value: 121, series: 'test' }), ]; - const sumDailyRepository = module.get(getRepositoryToken(SumDaily, 'timescaledb')); + const sumDailyRepository = module.get(getRepositoryToken(SumMarketplaceDaily, 'timescaledb')); sumDailyRepository.createQueryBuilder = jest.fn(() => ({ select, offset,