From daee6890a27bfe3dc5340db56b213b605ed7da4e Mon Sep 17 00:00:00 2001 From: hschiau Date: Tue, 13 Aug 2024 19:30:21 +0300 Subject: [PATCH 01/13] SERVICES-2541: get single token price candles for the past 7 days --- src/modules/analytics/analytics.resolver.ts | 15 ++ .../services/analytics.compute.service.ts | 29 ++++ .../interfaces/analytics.query.interface.ts | 8 ++ .../mocks/analytics.query.service.mock.ts | 8 ++ .../services/analytics.query.service.ts | 17 +++ .../timescaledb/timescaledb.query.service.ts | 128 ++++++++++++++++++ 6 files changed, 205 insertions(+) diff --git a/src/modules/analytics/analytics.resolver.ts b/src/modules/analytics/analytics.resolver.ts index defc2e5db..2529a2ee1 100644 --- a/src/modules/analytics/analytics.resolver.ts +++ b/src/modules/analytics/analytics.resolver.ts @@ -4,6 +4,7 @@ import { Args, Resolver } from '@nestjs/graphql'; import { CandleDataModel, HistoricDataModel, + OhlcvDataModel, } from 'src/modules/analytics/models/analytics.model'; import { AnalyticsQueryArgs, PriceCandlesQueryArgs } from './models/query.args'; import { AnalyticsAWSGetterService } from './services/analytics.aws.getter.service'; @@ -197,4 +198,18 @@ export class AnalyticsResolver { args.resolution, ); } + + @Query(() => [OhlcvDataModel]) + @UsePipes( + new ValidationPipe({ + skipNullProperties: true, + skipMissingProperties: true, + skipUndefinedProperties: true, + }), + ) + async tokenPast7dPrice( + @Args('tokenID') tokenID: string, + ): Promise { + return await this.analyticsCompute.tokenPast7dPrice(tokenID); + } } diff --git a/src/modules/analytics/services/analytics.compute.service.ts b/src/modules/analytics/services/analytics.compute.service.ts index 26e19b2ba..050218a21 100644 --- a/src/modules/analytics/services/analytics.compute.service.ts +++ b/src/modules/analytics/services/analytics.compute.service.ts @@ -21,6 +21,8 @@ import { FarmAbiFactory } from 'src/modules/farm/farm.abi.factory'; import { TokenComputeService } from 'src/modules/tokens/services/token.compute.service'; import { TokenService } from 'src/modules/tokens/services/token.service'; import { ErrorLoggerAsync } from '@multiversx/sdk-nestjs-common'; +import { OhlcvDataModel } from '../models/analytics.model'; +import moment from 'moment'; @Injectable() export class AnalyticsComputeService { @@ -253,6 +255,33 @@ export class AnalyticsComputeService { }); } + @ErrorLoggerAsync() + @GetOrSetCache({ + baseKey: 'analytics', + remoteTtl: Constants.oneMinute() * 30, + localTtl: Constants.oneMinute() * 10, + }) + async tokenPast7dPrice(tokenID: string): Promise { + return await this.computeTokenPast7dPrice(tokenID); + } + + async computeTokenPast7dPrice(tokenID: string): Promise { + const endDate = moment().utc().unix(); + const startDate = moment() + .subtract(7, 'days') + .utc() + .startOf('hour') + .unix(); + + return await this.analyticsQuery.getCandlesWithGapfilling({ + series: tokenID, + metric: 'priceUSD', + start: startDate, + end: endDate, + resolution: '4 hours', + }); + } + private async fiterPairsByIssuedLpToken( pairsAddress: string[], ): Promise { diff --git a/src/services/analytics/interfaces/analytics.query.interface.ts b/src/services/analytics/interfaces/analytics.query.interface.ts index c08ff9f84..5b3bfb1bc 100644 --- a/src/services/analytics/interfaces/analytics.query.interface.ts +++ b/src/services/analytics/interfaces/analytics.query.interface.ts @@ -46,5 +46,13 @@ export interface AnalyticsQueryInterface { end, }): Promise; + getCandlesWithGapfilling({ + series, + metric, + resolution, + start, + end, + }): Promise; + getStartDate(series: string): Promise; } diff --git a/src/services/analytics/mocks/analytics.query.service.mock.ts b/src/services/analytics/mocks/analytics.query.service.mock.ts index 545cbe7ea..85213bcc0 100644 --- a/src/services/analytics/mocks/analytics.query.service.mock.ts +++ b/src/services/analytics/mocks/analytics.query.service.mock.ts @@ -52,6 +52,14 @@ export class AnalyticsQueryServiceMock implements AnalyticsQueryInterface { getCandles({ series, metric, start, end }): Promise { throw new Error('Method not implemented.'); } + getCandlesWithGapfilling({ + series, + metric, + start, + end, + }): Promise { + throw new Error('Method not implemented.'); + } getStartDate(series: string): Promise { throw new Error('Method not implemented.'); } diff --git a/src/services/analytics/services/analytics.query.service.ts b/src/services/analytics/services/analytics.query.service.ts index 8251f0d17..2188a210a 100644 --- a/src/services/analytics/services/analytics.query.service.ts +++ b/src/services/analytics/services/analytics.query.service.ts @@ -116,6 +116,23 @@ export class AnalyticsQueryService implements AnalyticsQueryInterface { }); } + async getCandlesWithGapfilling({ + series, + metric, + resolution, + start, + end, + }): Promise { + const service = await this.getService(); + return await service.getCandlesWithGapfilling({ + series, + metric, + resolution, + start, + end, + }); + } + async getStartDate(series: string): Promise { const service = await this.getService(); return await service.getStartDate(series); diff --git a/src/services/analytics/timescaledb/timescaledb.query.service.ts b/src/services/analytics/timescaledb/timescaledb.query.service.ts index 0cbc377b8..e66ef8057 100644 --- a/src/services/analytics/timescaledb/timescaledb.query.service.ts +++ b/src/services/analytics/timescaledb/timescaledb.query.service.ts @@ -534,6 +534,58 @@ export class TimescaleDBQueryService implements AnalyticsQueryInterface { ); } + @TimescaleDBQuery() + async getCandlesWithGapfilling({ + series, + metric, + resolution, + start, + end, + }): Promise { + try { + const candleRepository = + this.getCandleRepositoryByResolutionAndMetric( + resolution, + metric, + ); + const startDate = moment.unix(start).utc().toDate(); + const endDate = moment.unix(end).utc().toDate(); + + const queryResult = await candleRepository + .createQueryBuilder() + .select(`time_bucket_gapfill('${resolution}', time) as bucket`) + .addSelect('locf(first(open, time)) as open') + .addSelect('locf(max(high)) as high') + .addSelect('locf(min(low)) as low') + .addSelect('locf(last(close, time)) as close') + .addSelect('locf(sum(volume)) as volume') + .where('series = :series', { series }) + .andWhere('time between :startDate and :endDate', { + startDate, + endDate, + }) + .groupBy('bucket') + .getRawMany(); + + return await this.gapfillCandleData( + queryResult, + series, + metric, + startDate, + candleRepository, + ); + } catch (error) { + this.logger.error('getTokenPriceCandles', { + series, + resolution, + start, + end, + error, + }); + return []; + } + } + @TimescaleDBQuery() async getCandles({ series, @@ -721,4 +773,80 @@ export class TimescaleDBQueryService implements AnalyticsQueryInterface { }), ); } + + private async gapfillCandleData( + data: any[], + series: string, + metric: string, + previousStartDate: Date, + repository: Repository< + | TokenCandlesMinute + | TokenCandlesHourly + | TokenCandlesDaily + | PairFirstTokenCandlesMinute + | PairFirstTokenCandlesHourly + | PairFirstTokenCandlesDaily + | PairSecondTokenCandlesMinute + | PairSecondTokenCandlesHourly + | PairSecondTokenCandlesDaily + >, + ): Promise { + if (!data || data.length === 0) { + return []; + } + + if (data[0].open) { + return this.formatCandleData(data); + } + + const startDate = await this.getStartDate(series); + const endDate = previousStartDate; + + if (!startDate) { + return this.formatCandleData(data); + } + + const previousValue = await repository + .createQueryBuilder() + .select('open, high, low, close, volume') + .where('series = :series', { series }) + .andWhere('time between :start and :end', { + start: moment(startDate).utc().toDate(), + end: endDate, + }) + .orderBy('time', 'DESC') + .limit(1) + .getRawOne(); + + if (!previousValue) { + return this.formatCandleData(data); + } + + return this.formatCandleData(data, previousValue); + } + + private formatCandleData( + data: any[], + gapfillValue = { + open: '0', + high: '0', + low: '0', + close: '0', + volume: '0', + }, + ): OhlcvDataModel[] { + return data.map( + (row) => + new OhlcvDataModel({ + time: row.bucket, + ohlcv: [ + row.open ?? gapfillValue.open, + row.high ?? gapfillValue.high, + row.low ?? gapfillValue.low, + row.close ?? gapfillValue.close, + row.volume ?? gapfillValue.volume, + ], + }), + ); + } } From 4328b2e5e9c066f001a9c567c791cc6ef0f45e6e Mon Sep 17 00:00:00 2001 From: hschiau Date: Wed, 14 Aug 2024 11:37:31 +0300 Subject: [PATCH 02/13] SERVICES-2541: refactor to allow fetching candles for an array of series --- src/modules/analytics/analytics.resolver.ts | 17 +- .../analytics/models/analytics.model.ts | 13 ++ src/modules/analytics/models/query.args.ts | 6 + .../services/analytics.compute.service.ts | 119 ++++++++++-- .../interfaces/analytics.query.interface.ts | 16 +- .../mocks/analytics.query.service.mock.ts | 19 +- .../services/analytics.query.service.ts | 30 ++- .../timescaledb/timescaledb.query.service.ts | 183 ++++++++++++++++-- 8 files changed, 344 insertions(+), 59 deletions(-) diff --git a/src/modules/analytics/analytics.resolver.ts b/src/modules/analytics/analytics.resolver.ts index 2529a2ee1..587286d3a 100644 --- a/src/modules/analytics/analytics.resolver.ts +++ b/src/modules/analytics/analytics.resolver.ts @@ -5,8 +5,13 @@ import { CandleDataModel, HistoricDataModel, OhlcvDataModel, + TokenCandlesModel, } from 'src/modules/analytics/models/analytics.model'; -import { AnalyticsQueryArgs, PriceCandlesQueryArgs } from './models/query.args'; +import { + AnalyticsQueryArgs, + PriceCandlesQueryArgs, + TokenPriceCandlesQueryArgs, +} from './models/query.args'; import { AnalyticsAWSGetterService } from './services/analytics.aws.getter.service'; import { AnalyticsComputeService } from './services/analytics.compute.service'; import { PairComputeService } from '../pair/services/pair.compute.service'; @@ -199,7 +204,7 @@ export class AnalyticsResolver { ); } - @Query(() => [OhlcvDataModel]) + @Query(() => [TokenCandlesModel]) @UsePipes( new ValidationPipe({ skipNullProperties: true, @@ -207,9 +212,9 @@ export class AnalyticsResolver { skipUndefinedProperties: true, }), ) - async tokenPast7dPrice( - @Args('tokenID') tokenID: string, - ): Promise { - return await this.analyticsCompute.tokenPast7dPrice(tokenID); + async tokensLast7dPrice( + @Args() args: TokenPriceCandlesQueryArgs, + ): Promise { + return await this.analyticsCompute.tokensLast7dPrice(args.identifiers); } } diff --git a/src/modules/analytics/models/analytics.model.ts b/src/modules/analytics/models/analytics.model.ts index 37c65a041..1f6d6686f 100644 --- a/src/modules/analytics/models/analytics.model.ts +++ b/src/modules/analytics/models/analytics.model.ts @@ -95,3 +95,16 @@ export class OhlcvDataModel { Object.assign(this, init); } } + +@ObjectType() +export class TokenCandlesModel { + @Field() + identifier: string; + + @Field(() => [OhlcvDataModel]) + candles: OhlcvDataModel[]; + + constructor(init?: Partial) { + Object.assign(this, init); + } +} diff --git a/src/modules/analytics/models/query.args.ts b/src/modules/analytics/models/query.args.ts index 396e446c2..372282127 100644 --- a/src/modules/analytics/models/query.args.ts +++ b/src/modules/analytics/models/query.args.ts @@ -53,3 +53,9 @@ export class PriceCandlesQueryArgs { @Field(() => PriceCandlesResolutions) resolution: PriceCandlesResolutions; } + +@ArgsType() +export class TokenPriceCandlesQueryArgs { + @Field(() => [String]) + identifiers: string[]; +} diff --git a/src/modules/analytics/services/analytics.compute.service.ts b/src/modules/analytics/services/analytics.compute.service.ts index 050218a21..df4a4fb96 100644 --- a/src/modules/analytics/services/analytics.compute.service.ts +++ b/src/modules/analytics/services/analytics.compute.service.ts @@ -21,7 +21,7 @@ import { FarmAbiFactory } from 'src/modules/farm/farm.abi.factory'; import { TokenComputeService } from 'src/modules/tokens/services/token.compute.service'; import { TokenService } from 'src/modules/tokens/services/token.service'; import { ErrorLoggerAsync } from '@multiversx/sdk-nestjs-common'; -import { OhlcvDataModel } from '../models/analytics.model'; +import { OhlcvDataModel, TokenCandlesModel } from '../models/analytics.model'; import moment from 'moment'; @Injectable() @@ -256,30 +256,111 @@ export class AnalyticsComputeService { } @ErrorLoggerAsync() - @GetOrSetCache({ - baseKey: 'analytics', - remoteTtl: Constants.oneMinute() * 30, - localTtl: Constants.oneMinute() * 10, - }) - async tokenPast7dPrice(tokenID: string): Promise { - return await this.computeTokenPast7dPrice(tokenID); + // @GetOrSetCache({ + // baseKey: 'analytics', + // remoteTtl: Constants.oneMinute() * 30, + // localTtl: Constants.oneMinute() * 10, + // }) + async tokensLast7dPrice( + identifiers: string[], + ): Promise { + return await this.computeTokensLast7dPrice(identifiers); } - async computeTokenPast7dPrice(tokenID: string): Promise { - const endDate = moment().utc().unix(); - const startDate = moment() - .subtract(7, 'days') - .utc() - .startOf('hour') - .unix(); - - return await this.analyticsQuery.getCandlesWithGapfilling({ - series: tokenID, - metric: 'priceUSD', + async computeTokensLast7dPrice( + identifiers: string[], + ): Promise { + // const endDate = moment().utc().unix(); + // const startDate = moment() + // .subtract(2, 'days') + // .utc() + // .startOf('hour') + // .unix(); + + const startDate = 1723449600; + const endDate = 1723620357; + + const tokenCandles = await this.analyticsQuery.getCandlesForTokens({ + identifiers, start: startDate, end: endDate, resolution: '4 hours', }); + + const tokensNeedingGapfilling = []; + for (let i = 0; i < identifiers.length; i++) { + const tokenID = identifiers[i]; + + const tokenData = tokenCandles.find( + (elem) => elem.identifier === tokenID, + ); + + if (!tokenData) { + tokensNeedingGapfilling.push(tokenID); + continue; + } + + const needsGapfilling = tokenData.candles.some((candle) => + candle.ohlcv.includes(-1), + ); + + if (needsGapfilling) { + tokensNeedingGapfilling.push(tokenID); + } + } + + if (tokensNeedingGapfilling.length === 0) { + return tokenCandles; + } + + console.log('Start gapfilling', tokensNeedingGapfilling); + + const earliestStartDate = + await this.analyticsQuery.getEarliestStartDate( + tokensNeedingGapfilling, + ); + + console.log('Min start date', earliestStartDate); + + // TODO : handle case where startDate == undefined. + // No activity for any of the tokens -> return array with 0 for all tokens ?? + + const lastCandles = await this.analyticsQuery.getLastCandleForTokens({ + identifiers: tokensNeedingGapfilling, + start: moment(earliestStartDate).utc().unix(), + end: startDate, + }); + console.log(lastCandles); + + // TODO : handle case where lastCandles == []. + + // TODO : perform manual gapfilling on tokenCandles with the data in 'lastCandles' + const result: TokenCandlesModel[] = []; + // for (let i = 0; i < identifiers.length; i++) { + // const tokenID = identifiers[i]; + + // const tokenData = tokenCandles.find( + // (elem) => elem.identifier === tokenID, + // ); + + // if (!tokenData) { + // const missingCandle = lastCandles.find( + // (elem) => elem.identifier === tokenID, + // ); + + // continue; + // } + + // const needsGapfilling = tokenData.candles.some((candle) => + // candle.ohlcv.includes(-1), + // ); + + // if (needsGapfilling) { + // tokensNeedingGapfilling.push(tokenID); + // } + // } + + return tokenCandles; } private async fiterPairsByIssuedLpToken( diff --git a/src/services/analytics/interfaces/analytics.query.interface.ts b/src/services/analytics/interfaces/analytics.query.interface.ts index 5b3bfb1bc..bb750ebfd 100644 --- a/src/services/analytics/interfaces/analytics.query.interface.ts +++ b/src/services/analytics/interfaces/analytics.query.interface.ts @@ -2,6 +2,7 @@ import { CandleDataModel, HistoricDataModel, OhlcvDataModel, + TokenCandlesModel, } from 'src/modules/analytics/models/analytics.model'; import { AnalyticsQueryArgs } from '../entities/analytics.query.args'; @@ -46,13 +47,20 @@ export interface AnalyticsQueryInterface { end, }): Promise; - getCandlesWithGapfilling({ - series, - metric, + getCandlesForTokens({ + identifiers, resolution, start, end, - }): Promise; + }): Promise; + + getLastCandleForTokens({ + identifiers, + start, + end, + }): Promise; getStartDate(series: string): Promise; + + getEarliestStartDate(series: string[]): Promise; } diff --git a/src/services/analytics/mocks/analytics.query.service.mock.ts b/src/services/analytics/mocks/analytics.query.service.mock.ts index 85213bcc0..9a8b660de 100644 --- a/src/services/analytics/mocks/analytics.query.service.mock.ts +++ b/src/services/analytics/mocks/analytics.query.service.mock.ts @@ -2,6 +2,7 @@ import { CandleDataModel, HistoricDataModel, OhlcvDataModel, + TokenCandlesModel, } from 'src/modules/analytics/models/analytics.model'; import { AnalyticsQueryArgs } from '../entities/analytics.query.args'; import { AnalyticsQueryInterface } from '../interfaces/analytics.query.interface'; @@ -52,17 +53,27 @@ export class AnalyticsQueryServiceMock implements AnalyticsQueryInterface { getCandles({ series, metric, start, end }): Promise { throw new Error('Method not implemented.'); } - getCandlesWithGapfilling({ - series, - metric, + getCandlesForTokens({ + identifiers, + resolution, + start, + end, + }): Promise { + throw new Error('Method not implemented.'); + } + getLastCandleForTokens({ + identifiers, start, end, - }): Promise { + }): Promise { throw new Error('Method not implemented.'); } getStartDate(series: string): Promise { throw new Error('Method not implemented.'); } + getEarliestStartDate(series: string[]): Promise { + throw new Error('Method not implemented.'); + } } export const AnalyticsQueryServiceProvider = { diff --git a/src/services/analytics/services/analytics.query.service.ts b/src/services/analytics/services/analytics.query.service.ts index 2188a210a..cc28ab954 100644 --- a/src/services/analytics/services/analytics.query.service.ts +++ b/src/services/analytics/services/analytics.query.service.ts @@ -3,6 +3,7 @@ import { CandleDataModel, HistoricDataModel, OhlcvDataModel, + TokenCandlesModel, } from 'src/modules/analytics/models/analytics.model'; import { TimescaleDBQueryService } from '../timescaledb/timescaledb.query.service'; import { AnalyticsQueryInterface } from '../interfaces/analytics.query.interface'; @@ -116,28 +117,43 @@ export class AnalyticsQueryService implements AnalyticsQueryInterface { }); } - async getCandlesWithGapfilling({ - series, - metric, + async getCandlesForTokens({ + identifiers, resolution, start, end, - }): Promise { + }): Promise { const service = await this.getService(); - return await service.getCandlesWithGapfilling({ - series, - metric, + return await service.getCandlesForTokens({ + identifiers, resolution, start, end, }); } + async getLastCandleForTokens({ + identifiers, + start, + end, + }): Promise { + const service = await this.getService(); + return await service.getLastCandleForTokens({ + identifiers, + start, + end, + }); + } async getStartDate(series: string): Promise { const service = await this.getService(); return await service.getStartDate(series); } + async getEarliestStartDate(series: string[]): Promise { + const service = await this.getService(); + return await service.getEarliestStartDate(series); + } + private async getService(): Promise { return this.timescaleDBQuery; } diff --git a/src/services/analytics/timescaledb/timescaledb.query.service.ts b/src/services/analytics/timescaledb/timescaledb.query.service.ts index e66ef8057..d8d3b640f 100644 --- a/src/services/analytics/timescaledb/timescaledb.query.service.ts +++ b/src/services/analytics/timescaledb/timescaledb.query.service.ts @@ -4,6 +4,7 @@ import { CandleDataModel, HistoricDataModel, OhlcvDataModel, + TokenCandlesModel, } from 'src/modules/analytics/models/analytics.model'; import { computeTimeInterval } from 'src/utils/analytics.utils'; import { AnalyticsQueryArgs } from '../entities/analytics.query.args'; @@ -423,6 +424,40 @@ export class TimescaleDBQueryService implements AnalyticsQueryInterface { return moment.min(filteredTimestamps).toISOString(); } + async getEarliestStartDate(series: string[]): Promise { + const allStartDates = await this.allStartDates(); + + const filteredTimestamps = []; + + for (const currentSeries of series) { + if (!currentSeries.includes('%')) { + if (allStartDates[currentSeries] !== undefined) { + filteredTimestamps.push( + moment(allStartDates[currentSeries]), + ); + } + continue; + } + + const seriesWithoutWildcard = currentSeries.replace( + new RegExp('%', 'g'), + '', + ); + for (const [key, value] of Object.entries(allStartDates)) { + if (!key.includes(seriesWithoutWildcard)) { + continue; + } + filteredTimestamps.push(moment(value)); + } + } + + if (filteredTimestamps.length === 0) { + return undefined; + } + + return moment.min(filteredTimestamps).toISOString(); + } + @ErrorLoggerAsync({ logArgs: true, }) @@ -535,48 +570,87 @@ export class TimescaleDBQueryService implements AnalyticsQueryInterface { } @TimescaleDBQuery() - async getCandlesWithGapfilling({ - series, - metric, + async getCandlesForTokens({ + identifiers, resolution, start, end, - }): Promise { + }): Promise { try { const candleRepository = this.getCandleRepositoryByResolutionAndMetric( resolution, - metric, + 'priceUSD', ); const startDate = moment.unix(start).utc().toDate(); const endDate = moment.unix(end).utc().toDate(); - const queryResult = await candleRepository + const query = candleRepository .createQueryBuilder() - .select(`time_bucket_gapfill('${resolution}', time) as bucket`) + .select( + `time_bucket_gapfill('${resolution}', time) as bucket, series`, + ) .addSelect('locf(first(open, time)) as open') .addSelect('locf(max(high)) as high') .addSelect('locf(min(low)) as low') .addSelect('locf(last(close, time)) as close') .addSelect('locf(sum(volume)) as volume') - .where('series = :series', { series }) + .where('series in (:...identifiers)', { identifiers }) .andWhere('time between :startDate and :endDate', { startDate, endDate, }) - .groupBy('bucket') - .getRawMany(); + .groupBy('series') + .addGroupBy('bucket'); + // .getRawMany(); - return await this.gapfillCandleData( - queryResult, - series, - metric, - startDate, - candleRepository, - ); + // console.log(query.getQueryAndParameters()); + + const queryResult = await query.getRawMany(); + + // console.log(queryResult.length); + + if (!queryResult || queryResult.length === 0) { + return []; + } + + const result: TokenCandlesModel[] = []; + + for (let i = 0; i < queryResult.length; i++) { + const row = queryResult[i]; + + let tokenIndex = result.findIndex( + (elem) => elem.identifier === row.series, + ); + + if (tokenIndex === -1) { + result.push( + new TokenCandlesModel({ + identifier: row.series, + candles: [], + }), + ); + tokenIndex = result.length - 1; + } + + result[tokenIndex].candles.push( + new OhlcvDataModel({ + time: row.bucket, + ohlcv: [ + row.open ?? -1, + row.high ?? -1, + row.low ?? -1, + row.close ?? -1, + row.volume ?? -1, + ], + }), + ); + } + + return result; } catch (error) { - this.logger.error('getTokenPriceCandles', { - series, + this.logger.error('getCandlesForTokens', { + identifiers, resolution, start, end, @@ -586,6 +660,77 @@ export class TimescaleDBQueryService implements AnalyticsQueryInterface { } } + @TimescaleDBQuery() + async getLastCandleForTokens({ + identifiers, + start, + end, + }): Promise { + const startDate = moment.unix(start).utc().toDate(); + const endDate = moment.unix(end).utc().toDate(); + + const query = this.tokenCandlesMinute + .createQueryBuilder() + .select(`series`) + .addSelect('last(time, time) as time') + .addSelect('last(open, time) as open') + .addSelect('last(high, time) as high') + .addSelect('last(low, time) as low') + .addSelect('last(close, time) as close') + .addSelect('last(volume, time) as volume') + .where('series in (:...identifiers)', { identifiers }) + .andWhere('time between :startDate and :endDate', { + startDate, + endDate, + }) + .groupBy('series'); + + // console.log(query.getQueryAndParameters()); + + const queryResult = await query.getRawMany(); + + if (!queryResult || queryResult.length === 0) { + return []; + } + + // console.log(queryResult); + + // TODO: refactor result format. duplicated code from getCandlesForTokens + const result: TokenCandlesModel[] = []; + for (let i = 0; i < queryResult.length; i++) { + const row = queryResult[i]; + + let tokenIndex = result.findIndex( + (elem) => elem.identifier === row.series, + ); + + if (tokenIndex === -1) { + result.push( + new TokenCandlesModel({ + identifier: row.series, + candles: [], + }), + ); + tokenIndex = result.length - 1; + } + + result[tokenIndex].candles.push( + new OhlcvDataModel({ + time: row.bucket, + ohlcv: [ + row.open ?? 0, + row.high ?? 0, + row.low ?? 0, + row.close ?? 0, + row.volume ?? 0, + ], + }), + ); + } + + return result; + } + @TimescaleDBQuery() async getCandles({ series, From 940933557946eec586bd5a53f58049a935bcc5dd Mon Sep 17 00:00:00 2001 From: hschiau Date: Wed, 14 Aug 2024 11:43:25 +0300 Subject: [PATCH 03/13] SERVICES-2541: remove unused code --- .../services/analytics.compute.service.ts | 42 ++-------- .../timescaledb/timescaledb.query.service.ts | 76 ------------------- 2 files changed, 7 insertions(+), 111 deletions(-) diff --git a/src/modules/analytics/services/analytics.compute.service.ts b/src/modules/analytics/services/analytics.compute.service.ts index df4a4fb96..eb4896e25 100644 --- a/src/modules/analytics/services/analytics.compute.service.ts +++ b/src/modules/analytics/services/analytics.compute.service.ts @@ -270,15 +270,12 @@ export class AnalyticsComputeService { async computeTokensLast7dPrice( identifiers: string[], ): Promise { - // const endDate = moment().utc().unix(); - // const startDate = moment() - // .subtract(2, 'days') - // .utc() - // .startOf('hour') - // .unix(); - - const startDate = 1723449600; - const endDate = 1723620357; + const endDate = moment().utc().unix(); + const startDate = moment() + .subtract(7, 'days') + .utc() + .startOf('hour') + .unix(); const tokenCandles = await this.analyticsQuery.getCandlesForTokens({ identifiers, @@ -332,35 +329,10 @@ export class AnalyticsComputeService { }); console.log(lastCandles); - // TODO : handle case where lastCandles == []. - // TODO : perform manual gapfilling on tokenCandles with the data in 'lastCandles' const result: TokenCandlesModel[] = []; - // for (let i = 0; i < identifiers.length; i++) { - // const tokenID = identifiers[i]; - - // const tokenData = tokenCandles.find( - // (elem) => elem.identifier === tokenID, - // ); - - // if (!tokenData) { - // const missingCandle = lastCandles.find( - // (elem) => elem.identifier === tokenID, - // ); - - // continue; - // } - - // const needsGapfilling = tokenData.candles.some((candle) => - // candle.ohlcv.includes(-1), - // ); - - // if (needsGapfilling) { - // tokensNeedingGapfilling.push(tokenID); - // } - // } - return tokenCandles; + return result; } private async fiterPairsByIssuedLpToken( diff --git a/src/services/analytics/timescaledb/timescaledb.query.service.ts b/src/services/analytics/timescaledb/timescaledb.query.service.ts index d8d3b640f..0042a76fc 100644 --- a/src/services/analytics/timescaledb/timescaledb.query.service.ts +++ b/src/services/analytics/timescaledb/timescaledb.query.service.ts @@ -918,80 +918,4 @@ export class TimescaleDBQueryService implements AnalyticsQueryInterface { }), ); } - - private async gapfillCandleData( - data: any[], - series: string, - metric: string, - previousStartDate: Date, - repository: Repository< - | TokenCandlesMinute - | TokenCandlesHourly - | TokenCandlesDaily - | PairFirstTokenCandlesMinute - | PairFirstTokenCandlesHourly - | PairFirstTokenCandlesDaily - | PairSecondTokenCandlesMinute - | PairSecondTokenCandlesHourly - | PairSecondTokenCandlesDaily - >, - ): Promise { - if (!data || data.length === 0) { - return []; - } - - if (data[0].open) { - return this.formatCandleData(data); - } - - const startDate = await this.getStartDate(series); - const endDate = previousStartDate; - - if (!startDate) { - return this.formatCandleData(data); - } - - const previousValue = await repository - .createQueryBuilder() - .select('open, high, low, close, volume') - .where('series = :series', { series }) - .andWhere('time between :start and :end', { - start: moment(startDate).utc().toDate(), - end: endDate, - }) - .orderBy('time', 'DESC') - .limit(1) - .getRawOne(); - - if (!previousValue) { - return this.formatCandleData(data); - } - - return this.formatCandleData(data, previousValue); - } - - private formatCandleData( - data: any[], - gapfillValue = { - open: '0', - high: '0', - low: '0', - close: '0', - volume: '0', - }, - ): OhlcvDataModel[] { - return data.map( - (row) => - new OhlcvDataModel({ - time: row.bucket, - ohlcv: [ - row.open ?? gapfillValue.open, - row.high ?? gapfillValue.high, - row.low ?? gapfillValue.low, - row.close ?? gapfillValue.close, - row.volume ?? gapfillValue.volume, - ], - }), - ); - } } From 583ce743b3eb53ced7d3e1a689bfba279d9c6cc8 Mon Sep 17 00:00:00 2001 From: hschiau Date: Wed, 4 Sep 2024 17:47:19 +0300 Subject: [PATCH 04/13] SERVICES-2541: manual gapfilling - handle scenario when price data is missing completely from timescale (gapfill with 0) - refactoring --- .../services/analytics.compute.service.ts | 157 ++++++++++++++++-- .../timescaledb/timescaledb.query.service.ts | 8 +- 2 files changed, 144 insertions(+), 21 deletions(-) diff --git a/src/modules/analytics/services/analytics.compute.service.ts b/src/modules/analytics/services/analytics.compute.service.ts index eb4896e25..1cf3ca190 100644 --- a/src/modules/analytics/services/analytics.compute.service.ts +++ b/src/modules/analytics/services/analytics.compute.service.ts @@ -256,11 +256,6 @@ export class AnalyticsComputeService { } @ErrorLoggerAsync() - // @GetOrSetCache({ - // baseKey: 'analytics', - // remoteTtl: Constants.oneMinute() * 30, - // localTtl: Constants.oneMinute() * 10, - // }) async tokensLast7dPrice( identifiers: string[], ): Promise { @@ -270,12 +265,8 @@ export class AnalyticsComputeService { async computeTokensLast7dPrice( identifiers: string[], ): Promise { - const endDate = moment().utc().unix(); - const startDate = moment() - .subtract(7, 'days') - .utc() - .startOf('hour') - .unix(); + const endDate = moment().unix(); + const startDate = moment().subtract(7, 'days').startOf('hour').unix(); const tokenCandles = await this.analyticsQuery.getCandlesForTokens({ identifiers, @@ -310,31 +301,161 @@ export class AnalyticsComputeService { return tokenCandles; } - console.log('Start gapfilling', tokensNeedingGapfilling); - const earliestStartDate = await this.analyticsQuery.getEarliestStartDate( tokensNeedingGapfilling, ); - console.log('Min start date', earliestStartDate); + // No activity for any of the tokens -> gapfill with 0 candles for all tokens + if (!earliestStartDate) { + tokensNeedingGapfilling.forEach((tokenID) => { + let missingTokenData = new TokenCandlesModel({ + identifier: tokenID, + candles: [], + }); + + missingTokenData = this.gapfillTokenCandles( + missingTokenData, + startDate, + endDate, + 4, + [0, 0, 0, 0], + ); + tokenCandles.push(missingTokenData); + }); - // TODO : handle case where startDate == undefined. - // No activity for any of the tokens -> return array with 0 for all tokens ?? + return tokenCandles; + } const lastCandles = await this.analyticsQuery.getLastCandleForTokens({ identifiers: tokensNeedingGapfilling, start: moment(earliestStartDate).utc().unix(), end: startDate, }); - console.log(lastCandles); - // TODO : perform manual gapfilling on tokenCandles with the data in 'lastCandles' const result: TokenCandlesModel[] = []; + for (let i = 0; i < identifiers.length; i++) { + const tokenID = identifiers[i]; + + let tokenData = tokenCandles.find( + (elem) => elem.identifier === tokenID, + ); + + const lastCandle = lastCandles.find( + (elem) => elem.identifier === tokenID, + ); + + const gapfillOhlc = !lastCandle + ? [0, 0, 0, 0] + : lastCandle.candles[0].ohlcv; + + if (!tokenData) { + tokenData = new TokenCandlesModel({ + identifier: tokenID, + candles: [], + }); + } + + tokenData = this.gapfillTokenCandles( + tokenData, + startDate, + endDate, + 4, + gapfillOhlc, + ); + + result.push(tokenData); + } + return result; } + private gapfillTokenCandles( + tokenData: TokenCandlesModel, + startTimestamp: number, + endTimestamp: number, + hoursResolution: number, + gapfillOhlc: number[], + ): TokenCandlesModel { + if (tokenData.candles.length === 0) { + const intervalTimestamps = this.generateTimestampsForHoursInterval( + startTimestamp, + endTimestamp, + hoursResolution, + ); + intervalTimestamps.forEach((timestamp) => { + tokenData.candles.push( + new OhlcvDataModel({ + time: (timestamp * 1000).toString(), + ohlcv: [ + gapfillOhlc[0], + gapfillOhlc[1], + gapfillOhlc[2], + gapfillOhlc[3], + 0, + ], + }), + ); + }); + + return tokenData; + } + + const needsGapfilling = tokenData.candles.some((candle) => + candle.ohlcv.includes(-1), + ); + + if (!needsGapfilling) { + return tokenData; + } + + for (let i = 0; i < tokenData.candles.length; i++) { + if (!tokenData.candles[i].ohlcv.includes(-1)) { + continue; + } + + tokenData.candles[i].ohlcv = [ + gapfillOhlc[0], + gapfillOhlc[1], + gapfillOhlc[2], + gapfillOhlc[3], + 0, + ]; + } + + return tokenData; + } + + private generateTimestampsForHoursInterval( + startTimestamp: number, + endTimestamp: number, + intervalHours: number, + ): number[] { + const timestamps: number[] = []; + + let start = moment.unix(startTimestamp); + const end = moment.unix(endTimestamp); + + // Align the start time with the next 4-hour boundary + const remainder = start.hour() % intervalHours; + if (remainder !== 0) { + start = start + .add(intervalHours - remainder, 'hours') + .startOf('hour'); + } else { + start = start.startOf('hour'); // Align exactly to the hour if already aligned + } + + // Generate timestamps at the specified interval until we reach the end time + while (start.isSameOrBefore(end)) { + timestamps.push(start.unix()); // Store the Unix timestamp + start = start.add(intervalHours, 'hours'); // Add the interval + } + + return timestamps; + } + private async fiterPairsByIssuedLpToken( pairsAddress: string[], ): Promise { diff --git a/src/services/analytics/timescaledb/timescaledb.query.service.ts b/src/services/analytics/timescaledb/timescaledb.query.service.ts index 02ed2d5d5..9079e5c47 100644 --- a/src/services/analytics/timescaledb/timescaledb.query.service.ts +++ b/src/services/analytics/timescaledb/timescaledb.query.service.ts @@ -594,7 +594,7 @@ export class TimescaleDBQueryService implements AnalyticsQueryInterface { .addSelect('locf(max(high)) as high') .addSelect('locf(min(low)) as low') .addSelect('locf(last(close, time)) as close') - .addSelect('locf(sum(volume)) as volume') + .addSelect('sum(volume) as volume') .where('series in (:...identifiers)', { identifiers }) .andWhere('time between :startDate and :endDate', { startDate, @@ -604,6 +604,8 @@ export class TimescaleDBQueryService implements AnalyticsQueryInterface { .addGroupBy('bucket'); // .getRawMany(); + // console.log('start', startDate); + // console.log('end', endDate); // console.log(query.getQueryAndParameters()); const queryResult = await query.getRawMany(); @@ -641,7 +643,7 @@ export class TimescaleDBQueryService implements AnalyticsQueryInterface { row.high ?? -1, row.low ?? -1, row.close ?? -1, - row.volume ?? -1, + row.volume ?? 0, ], }), ); @@ -716,7 +718,7 @@ export class TimescaleDBQueryService implements AnalyticsQueryInterface { result[tokenIndex].candles.push( new OhlcvDataModel({ - time: row.bucket, + time: row.time, ohlcv: [ row.open ?? 0, row.high ?? 0, From 72da884f76443fe0a8482689442eddc84a1471c5 Mon Sep 17 00:00:00 2001 From: hschiau Date: Wed, 4 Sep 2024 18:28:44 +0300 Subject: [PATCH 05/13] SERVICES-2541: extract price analytics to separate service + refactor --- src/modules/analytics/analytics.module.ts | 2 + src/modules/analytics/analytics.resolver.ts | 6 +- .../services/analytics.compute.service.ts | 203 ---------------- .../services/analytics.token.service.ts | 222 ++++++++++++++++++ 4 files changed, 229 insertions(+), 204 deletions(-) create mode 100644 src/modules/analytics/services/analytics.token.service.ts diff --git a/src/modules/analytics/analytics.module.ts b/src/modules/analytics/analytics.module.ts index b63015239..b2457ce10 100644 --- a/src/modules/analytics/analytics.module.ts +++ b/src/modules/analytics/analytics.module.ts @@ -21,6 +21,7 @@ import { RemoteConfigModule } from '../remote-config/remote-config.module'; import { AnalyticsModule as AnalyticsServicesModule } from 'src/services/analytics/analytics.module'; import { WeeklyRewardsSplittingModule } from 'src/submodules/weekly-rewards-splitting/weekly-rewards-splitting.module'; import { AnalyticsSetterService } from './services/analytics.setter.service'; +import { AnalyticsTokenService } from './services/analytics.token.service'; @Module({ imports: [ @@ -48,6 +49,7 @@ import { AnalyticsSetterService } from './services/analytics.setter.service'; AnalyticsSetterService, AnalyticsPairService, PairDayDataResolver, + AnalyticsTokenService, ], exports: [ AnalyticsAWSGetterService, diff --git a/src/modules/analytics/analytics.resolver.ts b/src/modules/analytics/analytics.resolver.ts index 587286d3a..6d56aa5e0 100644 --- a/src/modules/analytics/analytics.resolver.ts +++ b/src/modules/analytics/analytics.resolver.ts @@ -18,6 +18,7 @@ import { PairComputeService } from '../pair/services/pair.compute.service'; import { TokenService } from '../tokens/services/token.service'; import { AnalyticsPairService } from './services/analytics.pair.service'; import { PriceCandlesArgsValidationPipe } from './validators/price.candles.args.validator'; +import { AnalyticsTokenService } from './services/analytics.token.service'; @Resolver() export class AnalyticsResolver { @@ -27,6 +28,7 @@ export class AnalyticsResolver { private readonly tokenService: TokenService, private readonly pairCompute: PairComputeService, private readonly analyticsPairService: AnalyticsPairService, + private readonly analyticsTokenService: AnalyticsTokenService, ) {} @Query(() => String) @@ -215,6 +217,8 @@ export class AnalyticsResolver { async tokensLast7dPrice( @Args() args: TokenPriceCandlesQueryArgs, ): Promise { - return await this.analyticsCompute.tokensLast7dPrice(args.identifiers); + return await this.analyticsTokenService.tokensLast7dPrice( + args.identifiers, + ); } } diff --git a/src/modules/analytics/services/analytics.compute.service.ts b/src/modules/analytics/services/analytics.compute.service.ts index 1cf3ca190..26e19b2ba 100644 --- a/src/modules/analytics/services/analytics.compute.service.ts +++ b/src/modules/analytics/services/analytics.compute.service.ts @@ -21,8 +21,6 @@ import { FarmAbiFactory } from 'src/modules/farm/farm.abi.factory'; import { TokenComputeService } from 'src/modules/tokens/services/token.compute.service'; import { TokenService } from 'src/modules/tokens/services/token.service'; import { ErrorLoggerAsync } from '@multiversx/sdk-nestjs-common'; -import { OhlcvDataModel, TokenCandlesModel } from '../models/analytics.model'; -import moment from 'moment'; @Injectable() export class AnalyticsComputeService { @@ -255,207 +253,6 @@ export class AnalyticsComputeService { }); } - @ErrorLoggerAsync() - async tokensLast7dPrice( - identifiers: string[], - ): Promise { - return await this.computeTokensLast7dPrice(identifiers); - } - - async computeTokensLast7dPrice( - identifiers: string[], - ): Promise { - const endDate = moment().unix(); - const startDate = moment().subtract(7, 'days').startOf('hour').unix(); - - const tokenCandles = await this.analyticsQuery.getCandlesForTokens({ - identifiers, - start: startDate, - end: endDate, - resolution: '4 hours', - }); - - const tokensNeedingGapfilling = []; - for (let i = 0; i < identifiers.length; i++) { - const tokenID = identifiers[i]; - - const tokenData = tokenCandles.find( - (elem) => elem.identifier === tokenID, - ); - - if (!tokenData) { - tokensNeedingGapfilling.push(tokenID); - continue; - } - - const needsGapfilling = tokenData.candles.some((candle) => - candle.ohlcv.includes(-1), - ); - - if (needsGapfilling) { - tokensNeedingGapfilling.push(tokenID); - } - } - - if (tokensNeedingGapfilling.length === 0) { - return tokenCandles; - } - - const earliestStartDate = - await this.analyticsQuery.getEarliestStartDate( - tokensNeedingGapfilling, - ); - - // No activity for any of the tokens -> gapfill with 0 candles for all tokens - if (!earliestStartDate) { - tokensNeedingGapfilling.forEach((tokenID) => { - let missingTokenData = new TokenCandlesModel({ - identifier: tokenID, - candles: [], - }); - - missingTokenData = this.gapfillTokenCandles( - missingTokenData, - startDate, - endDate, - 4, - [0, 0, 0, 0], - ); - tokenCandles.push(missingTokenData); - }); - - return tokenCandles; - } - - const lastCandles = await this.analyticsQuery.getLastCandleForTokens({ - identifiers: tokensNeedingGapfilling, - start: moment(earliestStartDate).utc().unix(), - end: startDate, - }); - - const result: TokenCandlesModel[] = []; - - for (let i = 0; i < identifiers.length; i++) { - const tokenID = identifiers[i]; - - let tokenData = tokenCandles.find( - (elem) => elem.identifier === tokenID, - ); - - const lastCandle = lastCandles.find( - (elem) => elem.identifier === tokenID, - ); - - const gapfillOhlc = !lastCandle - ? [0, 0, 0, 0] - : lastCandle.candles[0].ohlcv; - - if (!tokenData) { - tokenData = new TokenCandlesModel({ - identifier: tokenID, - candles: [], - }); - } - - tokenData = this.gapfillTokenCandles( - tokenData, - startDate, - endDate, - 4, - gapfillOhlc, - ); - - result.push(tokenData); - } - - return result; - } - - private gapfillTokenCandles( - tokenData: TokenCandlesModel, - startTimestamp: number, - endTimestamp: number, - hoursResolution: number, - gapfillOhlc: number[], - ): TokenCandlesModel { - if (tokenData.candles.length === 0) { - const intervalTimestamps = this.generateTimestampsForHoursInterval( - startTimestamp, - endTimestamp, - hoursResolution, - ); - intervalTimestamps.forEach((timestamp) => { - tokenData.candles.push( - new OhlcvDataModel({ - time: (timestamp * 1000).toString(), - ohlcv: [ - gapfillOhlc[0], - gapfillOhlc[1], - gapfillOhlc[2], - gapfillOhlc[3], - 0, - ], - }), - ); - }); - - return tokenData; - } - - const needsGapfilling = tokenData.candles.some((candle) => - candle.ohlcv.includes(-1), - ); - - if (!needsGapfilling) { - return tokenData; - } - - for (let i = 0; i < tokenData.candles.length; i++) { - if (!tokenData.candles[i].ohlcv.includes(-1)) { - continue; - } - - tokenData.candles[i].ohlcv = [ - gapfillOhlc[0], - gapfillOhlc[1], - gapfillOhlc[2], - gapfillOhlc[3], - 0, - ]; - } - - return tokenData; - } - - private generateTimestampsForHoursInterval( - startTimestamp: number, - endTimestamp: number, - intervalHours: number, - ): number[] { - const timestamps: number[] = []; - - let start = moment.unix(startTimestamp); - const end = moment.unix(endTimestamp); - - // Align the start time with the next 4-hour boundary - const remainder = start.hour() % intervalHours; - if (remainder !== 0) { - start = start - .add(intervalHours - remainder, 'hours') - .startOf('hour'); - } else { - start = start.startOf('hour'); // Align exactly to the hour if already aligned - } - - // Generate timestamps at the specified interval until we reach the end time - while (start.isSameOrBefore(end)) { - timestamps.push(start.unix()); // Store the Unix timestamp - start = start.add(intervalHours, 'hours'); // Add the interval - } - - return timestamps; - } - private async fiterPairsByIssuedLpToken( pairsAddress: string[], ): Promise { diff --git a/src/modules/analytics/services/analytics.token.service.ts b/src/modules/analytics/services/analytics.token.service.ts new file mode 100644 index 000000000..c11efe2be --- /dev/null +++ b/src/modules/analytics/services/analytics.token.service.ts @@ -0,0 +1,222 @@ +import { ErrorLoggerAsync } from '@multiversx/sdk-nestjs-common'; +import { Injectable } from '@nestjs/common'; +import moment from 'moment'; +import { AnalyticsQueryService } from 'src/services/analytics/services/analytics.query.service'; +import { OhlcvDataModel, TokenCandlesModel } from '../models/analytics.model'; + +@Injectable() +export class AnalyticsTokenService { + constructor(private readonly analyticsQuery: AnalyticsQueryService) {} + + @ErrorLoggerAsync() + async tokensLast7dPrice( + identifiers: string[], + ): Promise { + return await this.computeTokensLast7dPrice(identifiers); + } + + async computeTokensLast7dPrice( + identifiers: string[], + hoursResolution = 4, + ): Promise { + const endDate = moment().unix(); + const startDate = moment().subtract(7, 'days').startOf('hour').unix(); + + const tokenCandles = await this.analyticsQuery.getCandlesForTokens({ + identifiers, + start: startDate, + end: endDate, + resolution: `${hoursResolution} hours`, + }); + + const tokensNeedingGapfilling = this.identifyTokensNeedingGapfilling( + identifiers, + tokenCandles, + ); + + if (tokensNeedingGapfilling.length === 0) { + return tokenCandles; + } + + return this.handleGapFilling( + tokenCandles, + tokensNeedingGapfilling, + startDate, + endDate, + hoursResolution, + ); + } + + private identifyTokensNeedingGapfilling( + identifiers: string[], + tokenCandles: TokenCandlesModel[], + ): string[] { + return identifiers.filter((tokenID) => { + const tokenData = tokenCandles.find( + (elem) => elem.identifier === tokenID, + ); + return ( + !tokenData || + tokenData.candles.some((candle) => candle.ohlcv.includes(-1)) + ); + }); + } + + private async handleGapFilling( + tokenCandles: TokenCandlesModel[], + tokensNeedingGapfilling: string[], + startDate: number, + endDate: number, + hoursResolution: number, + ): Promise { + const earliestStartDate = + await this.analyticsQuery.getEarliestStartDate( + tokensNeedingGapfilling, + ); + + if (!earliestStartDate) { + return this.gapfillTokensWithEmptyData( + tokenCandles, + tokensNeedingGapfilling, + startDate, + endDate, + hoursResolution, + ); + } + + const lastCandles = await this.analyticsQuery.getLastCandleForTokens({ + identifiers: tokensNeedingGapfilling, + start: moment(earliestStartDate).utc().unix(), + end: startDate, + }); + + return this.gapfillTokens( + tokenCandles, + tokensNeedingGapfilling, + lastCandles, + startDate, + endDate, + hoursResolution, + ); + } + + private gapfillTokensWithEmptyData( + tokenCandles: TokenCandlesModel[], + tokensNeedingGapfilling: string[], + startTimestamp: number, + endTimestamp: number, + hoursResolution: number, + ): TokenCandlesModel[] { + tokensNeedingGapfilling.forEach((tokenID) => { + const emptyTokenData = this.gapfillTokenCandles( + new TokenCandlesModel({ identifier: tokenID, candles: [] }), + startTimestamp, + endTimestamp, + hoursResolution, + [0, 0, 0, 0, 0], + ); + tokenCandles.push(emptyTokenData); + }); + return tokenCandles; + } + + private gapfillTokens( + tokenCandles: TokenCandlesModel[], + tokensNeedingGapfilling: string[], + lastCandles: TokenCandlesModel[], + startTimestamp: number, + endTimestamp: number, + hoursResolution: number, + ): TokenCandlesModel[] { + return tokensNeedingGapfilling.map((tokenID) => { + let tokenData = tokenCandles.find( + (elem) => elem.identifier === tokenID, + ); + const lastCandle = lastCandles.find( + (elem) => elem.identifier === tokenID, + ); + + // TODO: replace gapfilled volume value with 0 + const gapfillOhlc = lastCandle + ? lastCandle.candles[0].ohlcv + : [0, 0, 0, 0, 0]; + + if (!tokenData) { + tokenData = new TokenCandlesModel({ + identifier: tokenID, + candles: [], + }); + } + + return this.gapfillTokenCandles( + tokenData, + startTimestamp, + endTimestamp, + hoursResolution, + gapfillOhlc, + ); + }); + } + + private gapfillTokenCandles( + tokenData: TokenCandlesModel, + startTimestamp: number, + endTimestamp: number, + hoursResolution: number, + gapfillOhlc: number[], + ): TokenCandlesModel { + if (tokenData.candles.length === 0) { + const timestamps = this.generateTimestampsForHoursInterval( + startTimestamp, + endTimestamp, + hoursResolution, + ); + + timestamps.forEach((timestamp) => { + tokenData.candles.push( + new OhlcvDataModel({ + time: (timestamp * 1000).toString(), + ohlcv: [...gapfillOhlc], + }), + ); + }); + + return tokenData; + } + + tokenData.candles.forEach((candle) => { + if (candle.ohlcv.includes(-1)) { + candle.ohlcv = [...gapfillOhlc]; + } + }); + + return tokenData; + } + + private generateTimestampsForHoursInterval( + startTimestamp: number, + endTimestamp: number, + intervalHours: number, + ): number[] { + const timestamps: number[] = []; + + let start = moment.unix(startTimestamp); + const end = moment.unix(endTimestamp); + + // Align the start time with the next 4-hour boundary + const remainder = start.hour() % intervalHours; + if (remainder !== 0) { + start = start.add(intervalHours - remainder, 'hours'); + } + + start = start.startOf('hour'); + + // Generate timestamps at the specified interval until we reach the end time + while (start.isSameOrBefore(end)) { + timestamps.push(start.unix()); + start = start.add(intervalHours, 'hours'); + } + + return timestamps; + } +} From 63028ae256b896a0b4908e8f53470d0c2bb3da0f Mon Sep 17 00:00:00 2001 From: hschiau Date: Thu, 5 Sep 2024 17:23:14 +0300 Subject: [PATCH 06/13] SERVICES-2541: cache warm token prices + resolve using cache getter --- src/modules/analytics/analytics.module.ts | 1 + src/modules/analytics/analytics.resolver.ts | 2 +- .../services/analytics.aws.getter.service.ts | 29 +++++++++++- .../services/analytics.setter.service.ts | 13 ++++++ .../services/analytics.token.service.ts | 28 +++++++----- .../crons/analytics.cache.warmer.service.ts | 44 +++++++++++++++++++ src/utils/get.many.utils.ts | 2 +- 7 files changed, 105 insertions(+), 14 deletions(-) diff --git a/src/modules/analytics/analytics.module.ts b/src/modules/analytics/analytics.module.ts index b2457ce10..eca81650a 100644 --- a/src/modules/analytics/analytics.module.ts +++ b/src/modules/analytics/analytics.module.ts @@ -56,6 +56,7 @@ import { AnalyticsTokenService } from './services/analytics.token.service'; AnalyticsAWSSetterService, AnalyticsComputeService, AnalyticsSetterService, + AnalyticsTokenService, ], }) export class AnalyticsModule {} diff --git a/src/modules/analytics/analytics.resolver.ts b/src/modules/analytics/analytics.resolver.ts index 6d56aa5e0..ece3366a6 100644 --- a/src/modules/analytics/analytics.resolver.ts +++ b/src/modules/analytics/analytics.resolver.ts @@ -217,7 +217,7 @@ export class AnalyticsResolver { async tokensLast7dPrice( @Args() args: TokenPriceCandlesQueryArgs, ): Promise { - return await this.analyticsTokenService.tokensLast7dPrice( + return await this.analyticsAWSGetter.getTokensLast7dPrices( args.identifiers, ); } diff --git a/src/modules/analytics/services/analytics.aws.getter.service.ts b/src/modules/analytics/services/analytics.aws.getter.service.ts index 6145ba51e..6b6e83601 100644 --- a/src/modules/analytics/services/analytics.aws.getter.service.ts +++ b/src/modules/analytics/services/analytics.aws.getter.service.ts @@ -1,9 +1,14 @@ import { Injectable } from '@nestjs/common'; import { generateCacheKeyFromParams } from '../../../utils/generate-cache-key'; import { CacheService } from '@multiversx/sdk-nestjs-cache'; -import { HistoricDataModel } from '../models/analytics.model'; +import { + HistoricDataModel, + OhlcvDataModel, + TokenCandlesModel, +} from '../models/analytics.model'; import moment from 'moment'; import { ErrorLoggerAsync } from '@multiversx/sdk-nestjs-common'; +import { getMany } from 'src/utils/get.many.utils'; @Injectable() export class AnalyticsAWSGetterService { @@ -108,6 +113,28 @@ export class AnalyticsAWSGetterService { return data !== undefined ? data.slice(1) : []; } + @ErrorLoggerAsync() + async getTokensLast7dPrices( + identifiers: string[], + ): Promise { + const cacheKeys = identifiers.map((tokenID) => + this.getAnalyticsCacheKey('tokenLast7dPrices', tokenID), + ); + + const candles = await getMany( + this.cachingService, + cacheKeys, + ); + + return candles.map( + (tokenCandles, index) => + new TokenCandlesModel({ + identifier: identifiers[index], + candles: tokenCandles ?? [], + }), + ); + } + private getAnalyticsCacheKey(...args: any) { return generateCacheKeyFromParams('analytics', ...args); } diff --git a/src/modules/analytics/services/analytics.setter.service.ts b/src/modules/analytics/services/analytics.setter.service.ts index 477e11a62..c1ed369a6 100644 --- a/src/modules/analytics/services/analytics.setter.service.ts +++ b/src/modules/analytics/services/analytics.setter.service.ts @@ -4,6 +4,7 @@ import { Constants } from '@multiversx/sdk-nestjs-common'; import { CacheService } from '@multiversx/sdk-nestjs-cache'; import { GenericSetterService } from 'src/services/generics/generic.setter.service'; import { Logger } from 'winston'; +import { OhlcvDataModel } from '../models/analytics.model'; export class AnalyticsSetterService extends GenericSetterService { constructor( @@ -84,4 +85,16 @@ export class AnalyticsSetterService extends GenericSetterService { Constants.oneMinute() * 5, ); } + + async setTokenLast7dPrices( + tokenID: string, + values: OhlcvDataModel[], + ): Promise { + return await this.setData( + this.getCacheKey('tokenLast7dPrices', tokenID), + values, + Constants.oneMinute() * 10, + Constants.oneMinute() * 6, + ); + } } diff --git a/src/modules/analytics/services/analytics.token.service.ts b/src/modules/analytics/services/analytics.token.service.ts index c11efe2be..a332bbb84 100644 --- a/src/modules/analytics/services/analytics.token.service.ts +++ b/src/modules/analytics/services/analytics.token.service.ts @@ -9,12 +9,6 @@ export class AnalyticsTokenService { constructor(private readonly analyticsQuery: AnalyticsQueryService) {} @ErrorLoggerAsync() - async tokensLast7dPrice( - identifiers: string[], - ): Promise { - return await this.computeTokensLast7dPrice(identifiers); - } - async computeTokensLast7dPrice( identifiers: string[], hoursResolution = 4, @@ -128,7 +122,12 @@ export class AnalyticsTokenService { endTimestamp: number, hoursResolution: number, ): TokenCandlesModel[] { - return tokensNeedingGapfilling.map((tokenID) => { + const result = tokenCandles.filter( + (tokenData) => + !tokensNeedingGapfilling.includes(tokenData.identifier), + ); + + const gapfilledTokens = tokensNeedingGapfilling.map((tokenID) => { let tokenData = tokenCandles.find( (elem) => elem.identifier === tokenID, ); @@ -136,10 +135,15 @@ export class AnalyticsTokenService { (elem) => elem.identifier === tokenID, ); - // TODO: replace gapfilled volume value with 0 - const gapfillOhlc = lastCandle - ? lastCandle.candles[0].ohlcv - : [0, 0, 0, 0, 0]; + let gapfillOhlc = [0, 0, 0, 0, 0]; + if (lastCandle) { + // remove volume (last value in array) - not suitable for gapfilling + const adjustedCandle = lastCandle.candles[0].ohlcv; + adjustedCandle.pop(); + adjustedCandle.push(0); + + gapfillOhlc = adjustedCandle; + } if (!tokenData) { tokenData = new TokenCandlesModel({ @@ -156,6 +160,8 @@ export class AnalyticsTokenService { gapfillOhlc, ); }); + + return [...gapfilledTokens, ...result]; } private gapfillTokenCandles( diff --git a/src/services/crons/analytics.cache.warmer.service.ts b/src/services/crons/analytics.cache.warmer.service.ts index 999d1d94a..575889b31 100644 --- a/src/services/crons/analytics.cache.warmer.service.ts +++ b/src/services/crons/analytics.cache.warmer.service.ts @@ -7,6 +7,12 @@ import { RedisPubSub } from 'graphql-redis-subscriptions'; import { PUB_SUB } from '../redis.pubSub.module'; import { ApiConfigService } from 'src/helpers/api.config.service'; import { AnalyticsSetterService } from 'src/modules/analytics/services/analytics.setter.service'; +import { Lock } from '@multiversx/sdk-nestjs-common'; +import { TokenService } from 'src/modules/tokens/services/token.service'; +import { WINSTON_MODULE_PROVIDER } from 'nest-winston'; +import { Logger } from 'winston'; +import { PerformanceProfiler } from '@multiversx/sdk-nestjs-monitoring'; +import { AnalyticsTokenService } from 'src/modules/analytics/services/analytics.token.service'; @Injectable() export class AnalyticsCacheWarmerService { @@ -14,7 +20,10 @@ export class AnalyticsCacheWarmerService { private readonly analyticsCompute: AnalyticsComputeService, private readonly analyticsSetter: AnalyticsSetterService, private readonly apiConfig: ApiConfigService, + private readonly tokenService: TokenService, + private readonly analyticsTokenService: AnalyticsTokenService, @Inject(PUB_SUB) private pubSub: RedisPubSub, + @Inject(WINSTON_MODULE_PROVIDER) protected readonly logger: Logger, ) {} @Cron(CronExpression.EVERY_MINUTE) @@ -73,6 +82,41 @@ export class AnalyticsCacheWarmerService { await this.deleteCacheKeys(cachedKeys); } + @Cron(CronExpression.EVERY_5_MINUTES) + @Lock({ name: 'cacheTokensLast7dPrice', verbose: true }) + async cacheTokensLast7dPrice(): Promise { + const tokens = await this.tokenService.getUniqueTokenIDs(false); + this.logger.info('Start refresh tokens last 7 days price'); + const profiler = new PerformanceProfiler(); + + for (let i = 0; i < tokens.length; i += 10) { + const batch = tokens.slice(i, i + 10); + + const tokensCandles = + await this.analyticsTokenService.computeTokensLast7dPrice( + batch, + ); + + const promises = []; + tokensCandles.forEach((elem) => { + promises.push( + this.analyticsSetter.setTokenLast7dPrices( + elem.identifier, + elem.candles, + ), + ); + }); + const cachedKeys = await Promise.all(promises); + + await this.deleteCacheKeys(cachedKeys); + } + + profiler.stop(); + this.logger.info( + `Finish refresh tokens last 7 days price in ${profiler.duration}`, + ); + } + private async deleteCacheKeys(invalidatedKeys: string[]) { await this.pubSub.publish('deleteCacheKeys', invalidatedKeys); } diff --git a/src/utils/get.many.utils.ts b/src/utils/get.many.utils.ts index 98449e6ac..ae389b724 100644 --- a/src/utils/get.many.utils.ts +++ b/src/utils/get.many.utils.ts @@ -1,7 +1,7 @@ import { CacheService } from '@multiversx/sdk-nestjs-cache'; import { parseCachedNullOrUndefined } from './cache.utils'; -async function getMany( +export async function getMany( cacheService: CacheService, keys: string[], ): Promise<(T | undefined)[]> { From 3c13ea480e8db89f6f4f0692cadb63927ba0504a Mon Sep 17 00:00:00 2001 From: hschiau Date: Thu, 5 Sep 2024 19:07:33 +0300 Subject: [PATCH 07/13] SERVICES-2541: fix timestamp bug --- .../services/analytics.token.service.ts | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/modules/analytics/services/analytics.token.service.ts b/src/modules/analytics/services/analytics.token.service.ts index a332bbb84..052cee0ef 100644 --- a/src/modules/analytics/services/analytics.token.service.ts +++ b/src/modules/analytics/services/analytics.token.service.ts @@ -3,6 +3,7 @@ import { Injectable } from '@nestjs/common'; import moment from 'moment'; import { AnalyticsQueryService } from 'src/services/analytics/services/analytics.query.service'; import { OhlcvDataModel, TokenCandlesModel } from '../models/analytics.model'; +import { isValidUnixTimestamp } from 'src/helpers/helpers'; @Injectable() export class AnalyticsTokenService { @@ -29,7 +30,7 @@ export class AnalyticsTokenService { ); if (tokensNeedingGapfilling.length === 0) { - return tokenCandles; + return tokenCandles.map((tokenData) => this.formatData(tokenData)); } return this.handleGapFilling( @@ -111,7 +112,7 @@ export class AnalyticsTokenService { ); tokenCandles.push(emptyTokenData); }); - return tokenCandles; + return tokenCandles.map((tokenData) => this.formatData(tokenData)); } private gapfillTokens( @@ -161,7 +162,9 @@ export class AnalyticsTokenService { ); }); - return [...gapfilledTokens, ...result]; + return [...gapfilledTokens, ...result].map((tokenData) => + this.formatData(tokenData), + ); } private gapfillTokenCandles( @@ -181,7 +184,7 @@ export class AnalyticsTokenService { timestamps.forEach((timestamp) => { tokenData.candles.push( new OhlcvDataModel({ - time: (timestamp * 1000).toString(), + time: timestamp, ohlcv: [...gapfillOhlc], }), ); @@ -203,8 +206,8 @@ export class AnalyticsTokenService { startTimestamp: number, endTimestamp: number, intervalHours: number, - ): number[] { - const timestamps: number[] = []; + ): string[] { + const timestamps: string[] = []; let start = moment.unix(startTimestamp); const end = moment.unix(endTimestamp); @@ -219,10 +222,21 @@ export class AnalyticsTokenService { // Generate timestamps at the specified interval until we reach the end time while (start.isSameOrBefore(end)) { - timestamps.push(start.unix()); + timestamps.push(start.unix().toString()); start = start.add(intervalHours, 'hours'); } return timestamps; } + + private formatData(tokenData: TokenCandlesModel): TokenCandlesModel { + tokenData.candles.forEach((candle) => { + const candleTime = isValidUnixTimestamp(candle.time) + ? candle.time + : moment(candle.time).unix().toString(); + + candle.time = candleTime; + }); + return tokenData; + } } From 3d0bbf48ea15e782f92ce8bbcf4f04e6b126b963 Mon Sep 17 00:00:00 2001 From: hschiau Date: Thu, 5 Sep 2024 19:09:20 +0300 Subject: [PATCH 08/13] SERVICES-2541: refactor timescaledb query --- src/helpers/helpers.ts | 20 +++ src/modules/analytics/analytics.resolver.ts | 3 - .../timescaledb/timescaledb.query.service.ts | 125 +++++++----------- 3 files changed, 65 insertions(+), 83 deletions(-) diff --git a/src/helpers/helpers.ts b/src/helpers/helpers.ts index 608451bbd..c84d37db1 100644 --- a/src/helpers/helpers.ts +++ b/src/helpers/helpers.ts @@ -1,6 +1,7 @@ import { Address } from '@multiversx/sdk-core'; import { BigNumber } from 'bignumber.js'; import { BinaryUtils } from '@multiversx/sdk-nestjs-common'; +import moment from 'moment'; export function encodeTransactionData(data: string): string { const delimiter = '@'; @@ -60,3 +61,22 @@ export function awsOneYear(): string { export function delay(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } + +export function isValidUnixTimestamp(value: string) { + const timestamp = Number(value); + if (isNaN(timestamp)) { + return false; + } + + // If the timestamp is in seconds (10 digits) + if (value.length === 10) { + return moment.unix(timestamp).isValid(); + } + + // If the timestamp is in milliseconds (13 digits) + if (value.length === 13) { + return moment(timestamp).isValid(); + } + + return false; +} diff --git a/src/modules/analytics/analytics.resolver.ts b/src/modules/analytics/analytics.resolver.ts index ece3366a6..7b5ae8121 100644 --- a/src/modules/analytics/analytics.resolver.ts +++ b/src/modules/analytics/analytics.resolver.ts @@ -4,7 +4,6 @@ import { Args, Resolver } from '@nestjs/graphql'; import { CandleDataModel, HistoricDataModel, - OhlcvDataModel, TokenCandlesModel, } from 'src/modules/analytics/models/analytics.model'; import { @@ -18,7 +17,6 @@ import { PairComputeService } from '../pair/services/pair.compute.service'; import { TokenService } from '../tokens/services/token.service'; import { AnalyticsPairService } from './services/analytics.pair.service'; import { PriceCandlesArgsValidationPipe } from './validators/price.candles.args.validator'; -import { AnalyticsTokenService } from './services/analytics.token.service'; @Resolver() export class AnalyticsResolver { @@ -28,7 +26,6 @@ export class AnalyticsResolver { private readonly tokenService: TokenService, private readonly pairCompute: PairComputeService, private readonly analyticsPairService: AnalyticsPairService, - private readonly analyticsTokenService: AnalyticsTokenService, ) {} @Query(() => String) diff --git a/src/services/analytics/timescaledb/timescaledb.query.service.ts b/src/services/analytics/timescaledb/timescaledb.query.service.ts index 9079e5c47..106979b8a 100644 --- a/src/services/analytics/timescaledb/timescaledb.query.service.ts +++ b/src/services/analytics/timescaledb/timescaledb.query.service.ts @@ -585,7 +585,7 @@ export class TimescaleDBQueryService implements AnalyticsQueryInterface { const startDate = moment.unix(start).utc().toDate(); const endDate = moment.unix(end).utc().toDate(); - const query = candleRepository + const queryResult = await candleRepository .createQueryBuilder() .select( `time_bucket_gapfill('${resolution}', time) as bucket, series`, @@ -601,55 +601,10 @@ export class TimescaleDBQueryService implements AnalyticsQueryInterface { endDate, }) .groupBy('series') - .addGroupBy('bucket'); - // .getRawMany(); - - // console.log('start', startDate); - // console.log('end', endDate); - // console.log(query.getQueryAndParameters()); - - const queryResult = await query.getRawMany(); - - // console.log(queryResult.length); - - if (!queryResult || queryResult.length === 0) { - return []; - } - - const result: TokenCandlesModel[] = []; - - for (let i = 0; i < queryResult.length; i++) { - const row = queryResult[i]; - - let tokenIndex = result.findIndex( - (elem) => elem.identifier === row.series, - ); - - if (tokenIndex === -1) { - result.push( - new TokenCandlesModel({ - identifier: row.series, - candles: [], - }), - ); - tokenIndex = result.length - 1; - } - - result[tokenIndex].candles.push( - new OhlcvDataModel({ - time: row.bucket, - ohlcv: [ - row.open ?? -1, - row.high ?? -1, - row.low ?? -1, - row.close ?? -1, - row.volume ?? 0, - ], - }), - ); - } + .addGroupBy('bucket') + .getRawMany(); - return result; + return this.processTokenCandles(queryResult, -1); } catch (error) { this.logger.error('getCandlesForTokens', { identifiers, @@ -668,40 +623,50 @@ export class TimescaleDBQueryService implements AnalyticsQueryInterface { start, end, }): Promise { - const startDate = moment.unix(start).utc().toDate(); - const endDate = moment.unix(end).utc().toDate(); - - const query = this.tokenCandlesMinute - .createQueryBuilder() - .select(`series`) - .addSelect('last(time, time) as time') - .addSelect('last(open, time) as open') - .addSelect('last(high, time) as high') - .addSelect('last(low, time) as low') - .addSelect('last(close, time) as close') - .addSelect('last(volume, time) as volume') - .where('series in (:...identifiers)', { identifiers }) - .andWhere('time between :startDate and :endDate', { - startDate, - endDate, - }) - .groupBy('series'); + try { + const startDate = moment.unix(start).utc().toDate(); + const endDate = moment.unix(end).utc().toDate(); - // console.log(query.getQueryAndParameters()); + const queryResult = await this.tokenCandlesMinute + .createQueryBuilder() + .select(`series`) + .addSelect('last(time, time) as time') + .addSelect('last(open, time) as open') + .addSelect('last(high, time) as high') + .addSelect('last(low, time) as low') + .addSelect('last(close, time) as close') + .addSelect('last(volume, time) as volume') + .where('series in (:...identifiers)', { identifiers }) + .andWhere('time between :startDate and :endDate', { + startDate, + endDate, + }) + .groupBy('series') + .getRawMany(); - const queryResult = await query.getRawMany(); + return this.processTokenCandles(queryResult, 0); + } catch (error) { + this.logger.error('getLastCandleForTokens', { + identifiers, + start, + end, + error, + }); + return []; + } + } + private processTokenCandles( + queryResult: any[], + defaultValue: number, + ): TokenCandlesModel[] { if (!queryResult || queryResult.length === 0) { return []; } - // console.log(queryResult); - - // TODO: refactor result format. duplicated code from getCandlesForTokens const result: TokenCandlesModel[] = []; - for (let i = 0; i < queryResult.length; i++) { - const row = queryResult[i]; + queryResult.forEach((row) => { let tokenIndex = result.findIndex( (elem) => elem.identifier === row.series, ); @@ -718,17 +683,17 @@ export class TimescaleDBQueryService implements AnalyticsQueryInterface { result[tokenIndex].candles.push( new OhlcvDataModel({ - time: row.time, + time: row.bucket ?? row.time, ohlcv: [ - row.open ?? 0, - row.high ?? 0, - row.low ?? 0, - row.close ?? 0, + row.open ?? defaultValue, + row.high ?? defaultValue, + row.low ?? defaultValue, + row.close ?? defaultValue, row.volume ?? 0, ], }), ); - } + }); return result; } From 68c14964f5059034337a5be4dcf46d01b1d07501 Mon Sep 17 00:00:00 2001 From: hschiau Date: Thu, 5 Sep 2024 19:25:58 +0300 Subject: [PATCH 09/13] SERVICES-2541: fix unit test --- src/modules/farm/specs/farm.compute.service.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/farm/specs/farm.compute.service.spec.ts b/src/modules/farm/specs/farm.compute.service.spec.ts index 7db6dccf4..2f4cd3203 100644 --- a/src/modules/farm/specs/farm.compute.service.spec.ts +++ b/src/modules/farm/specs/farm.compute.service.spec.ts @@ -67,7 +67,7 @@ describe('FarmService', () => { const service = module.get( FarmComputeServiceV1_2, ); - const farmedTokenPriceUSD = await service.computeFarmedTokenPriceUSD( + const farmedTokenPriceUSD = await service.farmedTokenPriceUSD( Address.fromHex( '0000000000000000000000000000000000000000000000000000000000000021', ).bech32(), From 33176e748d7ad7c76a459713d1bc47ee55f0c382 Mon Sep 17 00:00:00 2001 From: hschiau Date: Fri, 6 Sep 2024 10:31:12 +0300 Subject: [PATCH 10/13] SERVICES-2541: mark old query as deprecated --- src/modules/analytics/analytics.resolver.ts | 5 +++-- src/modules/analytics/models/query.args.ts | 12 +++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/modules/analytics/analytics.resolver.ts b/src/modules/analytics/analytics.resolver.ts index 7b5ae8121..55b07415c 100644 --- a/src/modules/analytics/analytics.resolver.ts +++ b/src/modules/analytics/analytics.resolver.ts @@ -182,7 +182,7 @@ export class AnalyticsResolver { return []; } - @Query(() => [CandleDataModel]) + @Query(() => [CandleDataModel], { @UsePipes( new ValidationPipe({ skipNullProperties: true, @@ -212,7 +212,8 @@ export class AnalyticsResolver { }), ) async tokensLast7dPrice( - @Args() args: TokenPriceCandlesQueryArgs, + @Args({ type: () => TokenPriceCandlesQueryArgs }) + args: TokenPriceCandlesQueryArgs, ): Promise { return await this.analyticsAWSGetter.getTokensLast7dPrices( args.identifiers, diff --git a/src/modules/analytics/models/query.args.ts b/src/modules/analytics/models/query.args.ts index 372282127..d6dd4fbab 100644 --- a/src/modules/analytics/models/query.args.ts +++ b/src/modules/analytics/models/query.args.ts @@ -1,5 +1,12 @@ import { ArgsType, Field, registerEnumType } from '@nestjs/graphql'; -import { IsNotEmpty, Matches } from 'class-validator'; +import { + ArrayMaxSize, + ArrayMinSize, + IsArray, + IsNotEmpty, + Matches, + Min, +} from 'class-validator'; import { IsValidMetric } from 'src/helpers/validators/metric.validator'; import { IsValidSeries } from 'src/helpers/validators/series.validator'; import { IsValidUnixTime } from 'src/helpers/validators/unix.time.validator'; @@ -57,5 +64,8 @@ export class PriceCandlesQueryArgs { @ArgsType() export class TokenPriceCandlesQueryArgs { @Field(() => [String]) + @IsArray() + @ArrayMinSize(1, { message: 'At least 1 token ID is required' }) + @ArrayMaxSize(10, { message: 'At most 10 token IDs can be provided' }) identifiers: string[]; } From 77320add184dc66780a21ea78b1bbb8695b57b2e Mon Sep 17 00:00:00 2001 From: hschiau Date: Fri, 6 Sep 2024 10:32:57 +0300 Subject: [PATCH 11/13] SERVICES-2541: mark old query as deprecated --- src/modules/analytics/analytics.resolver.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/modules/analytics/analytics.resolver.ts b/src/modules/analytics/analytics.resolver.ts index 55b07415c..a9c66f4d6 100644 --- a/src/modules/analytics/analytics.resolver.ts +++ b/src/modules/analytics/analytics.resolver.ts @@ -183,6 +183,10 @@ export class AnalyticsResolver { } @Query(() => [CandleDataModel], { + deprecationReason: + 'New optimized query is now available (tokensLast7dPrice).' + + 'It allows fetching price data for multiple tokens in a single request', + }) @UsePipes( new ValidationPipe({ skipNullProperties: true, From d46a377f421cd565f85fe02772454b6df30d283c Mon Sep 17 00:00:00 2001 From: hschiau Date: Tue, 10 Sep 2024 09:23:08 +0300 Subject: [PATCH 12/13] SERVICES-2541: fixes after review - use regex for numeric string validation - replace magic numbers with constants --- src/helpers/helpers.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/helpers/helpers.ts b/src/helpers/helpers.ts index c84d37db1..b86b3b882 100644 --- a/src/helpers/helpers.ts +++ b/src/helpers/helpers.ts @@ -3,6 +3,9 @@ import { BigNumber } from 'bignumber.js'; import { BinaryUtils } from '@multiversx/sdk-nestjs-common'; import moment from 'moment'; +export const SECONDS_TIMESTAMP_LENGTH = 10; +export const MILLISECONDS_TIMESTAMP_LENGTH = 13; + export function encodeTransactionData(data: string): string { const delimiter = '@'; @@ -63,18 +66,17 @@ export function delay(ms: number) { } export function isValidUnixTimestamp(value: string) { - const timestamp = Number(value); - if (isNaN(timestamp)) { + if (/^\d+$/.test(value) === false) { return false; } - // If the timestamp is in seconds (10 digits) - if (value.length === 10) { + const timestamp = Number(value); + + if (value.length === SECONDS_TIMESTAMP_LENGTH) { return moment.unix(timestamp).isValid(); } - // If the timestamp is in milliseconds (13 digits) - if (value.length === 13) { + if (value.length === MILLISECONDS_TIMESTAMP_LENGTH) { return moment(timestamp).isValid(); } From 37fed91a6cec873132cc9981724189f317171224 Mon Sep 17 00:00:00 2001 From: hschiau Date: Tue, 10 Sep 2024 12:52:57 +0300 Subject: [PATCH 13/13] SERVICES-2541: add return type --- src/helpers/helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/helpers.ts b/src/helpers/helpers.ts index b86b3b882..93cd7c604 100644 --- a/src/helpers/helpers.ts +++ b/src/helpers/helpers.ts @@ -65,7 +65,7 @@ export function delay(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } -export function isValidUnixTimestamp(value: string) { +export function isValidUnixTimestamp(value: string): boolean { if (/^\d+$/.test(value) === false) { return false; }