From bce5a112d31717b1d89fb0bdf2b679fc9cf39623 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Mon, 16 Oct 2023 16:29:56 -0600 Subject: [PATCH 01/11] add new agg function --- .../explorer/service/TokenService.kt | 63 +++++++++++++++++-- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt index fe597d5a..d299b9d4 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt @@ -31,6 +31,7 @@ import io.provenance.explorer.domain.extensions.toCoinStr import io.provenance.explorer.domain.extensions.toOffset import io.provenance.explorer.domain.extensions.toPercentage import io.provenance.explorer.domain.models.explorer.DlobHistBase +import io.provenance.explorer.domain.models.explorer.DlobHistorical import io.provenance.explorer.domain.models.explorer.TokenHistoricalDataRequest import io.provenance.explorer.grpc.v1.AccountGrpcClient import io.provenance.explorer.model.AssetHolder @@ -145,7 +146,8 @@ class TokenService(private val accountClient: AccountGrpcClient) { } fun getTokenBreakdown() = runBlocking { - val bonded = accountClient.getStakingPool().pool.bondedTokens.toBigDecimal().roundWhole().toCoinStr(UTILITY_TOKEN) + val bonded = + accountClient.getStakingPool().pool.bondedTokens.toBigDecimal().roundWhole().toCoinStr(UTILITY_TOKEN) TokenSupply( maxSupply().toCoinStr(UTILITY_TOKEN), totalSupply().toCoinStr(UTILITY_TOKEN), @@ -157,16 +159,20 @@ class TokenService(private val accountClient: AccountGrpcClient) { } fun nhashMarkerAddr() = MarkerCacheRecord.findByDenom(UTILITY_TOKEN)?.markerAddress!! - fun burnedSupply() = runBlocking { accountClient.getMarkerBalance(nhashMarkerAddr(), UTILITY_TOKEN).toBigDecimal().roundWhole() } + fun burnedSupply() = + runBlocking { accountClient.getMarkerBalance(nhashMarkerAddr(), UTILITY_TOKEN).toBigDecimal().roundWhole() } + fun moduleAccounts() = AccountRecord.findAccountsByType(listOf(Auth.ModuleAccount::class.java.simpleName)) fun zeroSeqAccounts() = AccountRecord.findZeroSequenceAccounts() fun vestingAccounts() = AccountRecord.findAccountsByType(vestingAccountTypes) fun contractAccounts() = AccountRecord.findContractAccounts() fun allAccounts() = transaction { AccountRecord.all().toMutableList() } - fun communityPoolSupply() = runBlocking { accountClient.getCommunityPoolAmount(UTILITY_TOKEN).toBigDecimal().roundWhole() } + fun communityPoolSupply() = + runBlocking { accountClient.getCommunityPoolAmount(UTILITY_TOKEN).toBigDecimal().roundWhole() } + fun richListAccounts() = allAccounts().addressList() - zeroSeqAccounts().toSet() - moduleAccounts().addressList() - - contractAccounts().addressList() - setOf(nhashMarkerAddr()) + contractAccounts().addressList() - setOf(nhashMarkerAddr()) fun totalBalanceForList(addresses: Set) = runBlocking { TokenDistributionPaginatedResultsRecord.findByAddresses(addresses).asFlow() @@ -234,6 +240,55 @@ class TokenService(private val accountClient: AccountGrpcClient) { } } + suspend fun getHistoricalFromDlobNew(startTime: DateTime): DlobHistBase? { + val dlobContractUrls = listOf( + "https://www.dlob.io/gecko/external/api/v1/order-books/pb1w6ul64t5fjcg65mmscec758dgyml6xmmw5fy2vyxxc9dhq3tmhusyzcj3r/aggregate?unit=YEAR", + "https://www.dlob.io/gecko/external/api/v1/order-books/pb18vd8fpwxzck93qlwghaj6arh4p7c5n894vnu5g/aggregate?unit=YEAR" + ) + var dlobHistorBase: DlobHistBase? = null + for (url in dlobContractUrls) { + try { + val data = fetchDataFromUrl(url, startTime) + if (data != null) { + if (dlobHistorBase == null) { + dlobHistorBase = data + }else { + val combinedBuy = dlobHistorBase.buy + data.buy + dlobHistorBase = dlobHistorBase.copy(buy = combinedBuy) + } + } + } catch (e: ResponseException) { + logger.error("Error fetching from Dlob: ${e.response}") + } catch (e: Exception) { + logger.error("Error fetching from Dlob: ${e.message}") + } catch (e: Throwable) { + logger.error("Error fetching from Dlob: ${e.message}") + } + } + + return dlobHistorBase + } + + private suspend fun fetchDataFromUrl(url: String, startTime: DateTime): DlobHistBase? { + return try { + KTOR_CLIENT_JAVA.get(url) { + parameter("ticker_id", "HASH_USD") + parameter("type", "buy") + parameter("start_time", DateTimeFormat.forPattern("dd-MM-yyyy").print(startTime)) + accept(ContentType.Application.Json) + }.body() + } catch (e: ResponseException) { + logger.error("Error fetching from Dlob: ${e.response}") + null + } catch (e: Exception) { + logger.error("Error fetching from Dlob: ${e.message}") + null + } catch (e: Throwable) { + logger.error("Error fetching from Dlob: ${e.message}") + null + } + } + fun getTokenHistorical(fromDate: DateTime?, toDate: DateTime?) = TokenHistoricalDailyRecord.findForDates(fromDate?.startOfDay(), toDate?.startOfDay()) From aa8de911fd9a95ee7781126c3e5f8bfa4325383a Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Tue, 24 Oct 2023 09:48:37 -0600 Subject: [PATCH 02/11] update historical service to pull from multiple tickers, update start time calculation --- .../explorer/domain/entities/ExplorerCache.kt | 17 ++++-- .../explorer/service/TokenService.kt | 57 +++---------------- .../explorer/service/async/AsyncService.kt | 43 +++++++++----- 3 files changed, 50 insertions(+), 67 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt index ece23e6f..fe6efe1d 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt @@ -90,7 +90,8 @@ class ValidatorMarketRateStatsRecord(id: EntityID) : IntEntity(id) { } fun findByAddress(address: String, fromDate: DateTime?, toDate: DateTime?, count: Int) = transaction { - val query = ValidatorMarketRateStatsTable.select { ValidatorMarketRateStatsTable.operatorAddress eq address } + val query = + ValidatorMarketRateStatsTable.select { ValidatorMarketRateStatsTable.operatorAddress eq address } if (fromDate != null) { query.andWhere { ValidatorMarketRateStatsTable.date greaterEq fromDate } } @@ -214,7 +215,7 @@ class ChainAumHourlyRecord(id: EntityID) : IntEntity(id) { fun getAumForPeriod(fromDate: DateTime, toDate: DateTime) = transaction { ChainAumHourlyRecord.find { (ChainAumHourlyTable.datetime greaterEq fromDate) and - (ChainAumHourlyTable.datetime lessEq toDate.plusDays(1)) + (ChainAumHourlyTable.datetime lessEq toDate.plusDays(1)) } .orderBy(Pair(ChainAumHourlyTable.datetime, SortOrder.ASC)) .toList() @@ -272,6 +273,14 @@ class TokenHistoricalDailyRecord(id: EntityID) : Entity(id) .orderBy(Pair(TokenHistoricalDailyTable.timestamp, SortOrder.DESC)) .firstOrNull()?.data?.quote?.get(USD_UPPER)?.close ?: BigDecimal.ZERO } + + fun getLatestDateEntry(): TokenHistoricalDailyRecord? = transaction { + return@transaction TokenHistoricalDailyRecord + .all() + .orderBy(Pair(TokenHistoricalDailyTable.timestamp, SortOrder.DESC)) + .limit(1) + .firstOrNull() + } } var timestamp by TokenHistoricalDailyTable.timestamp @@ -292,7 +301,7 @@ class ProcessQueueRecord(id: EntityID) : IntEntity(id) { fun findByType(processType: ProcessQueueType) = transaction { ProcessQueueRecord.find { (ProcessQueueTable.processType eq processType.name) and - (ProcessQueueTable.processing eq false) + (ProcessQueueTable.processing eq false) }.toList() } @@ -305,7 +314,7 @@ class ProcessQueueRecord(id: EntityID) : IntEntity(id) { fun delete(processType: ProcessQueueType, value: String) = transaction { ProcessQueueTable.deleteWhere { (ProcessQueueTable.processType eq processType.name) and - (ProcessQueueTable.processValue eq value) + (ProcessQueueTable.processValue eq value) } } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt index d299b9d4..b08f32bc 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt @@ -223,10 +223,10 @@ class TokenService(private val accountClient: AccountGrpcClient) { } } - fun getHistoricalFromDlob(startTime: DateTime): DlobHistBase? = runBlocking { + fun getHistoricalFromDlob(startTime: DateTime, tickerId: String): DlobHistBase? = runBlocking { try { - KTOR_CLIENT_JAVA.get("https://www.dlob.io:443/gecko/external/api/v1/exchange/historical_trades") { - parameter("ticker_id", "HASH_USD") + KTOR_CLIENT_JAVA.get("https://test.dlob.io:443/gecko/external/api/v1/exchange/historical_trades") { + parameter("ticker_id", tickerId) parameter("type", "buy") parameter("start_time", DateTimeFormat.forPattern("dd-MM-yyyy").print(startTime)) accept(ContentType.Application.Json) @@ -240,53 +240,14 @@ class TokenService(private val accountClient: AccountGrpcClient) { } } - suspend fun getHistoricalFromDlobNew(startTime: DateTime): DlobHistBase? { - val dlobContractUrls = listOf( - "https://www.dlob.io/gecko/external/api/v1/order-books/pb1w6ul64t5fjcg65mmscec758dgyml6xmmw5fy2vyxxc9dhq3tmhusyzcj3r/aggregate?unit=YEAR", - "https://www.dlob.io/gecko/external/api/v1/order-books/pb18vd8fpwxzck93qlwghaj6arh4p7c5n894vnu5g/aggregate?unit=YEAR" - ) - var dlobHistorBase: DlobHistBase? = null - for (url in dlobContractUrls) { - try { - val data = fetchDataFromUrl(url, startTime) - if (data != null) { - if (dlobHistorBase == null) { - dlobHistorBase = data - }else { - val combinedBuy = dlobHistorBase.buy + data.buy - dlobHistorBase = dlobHistorBase.copy(buy = combinedBuy) - } - } - } catch (e: ResponseException) { - logger.error("Error fetching from Dlob: ${e.response}") - } catch (e: Exception) { - logger.error("Error fetching from Dlob: ${e.message}") - } catch (e: Throwable) { - logger.error("Error fetching from Dlob: ${e.message}") - } - } + fun getHistoricalFromDlob(startTime: DateTime): DlobHistBase { + val tickerIds = listOf("HASH_USD", "HASH_USDOMNI") - return dlobHistorBase - } + val dlobHistorical = tickerIds + .mapNotNull { getHistoricalFromDlob(startTime, it)?.buy } + .flatten() - private suspend fun fetchDataFromUrl(url: String, startTime: DateTime): DlobHistBase? { - return try { - KTOR_CLIENT_JAVA.get(url) { - parameter("ticker_id", "HASH_USD") - parameter("type", "buy") - parameter("start_time", DateTimeFormat.forPattern("dd-MM-yyyy").print(startTime)) - accept(ContentType.Application.Json) - }.body() - } catch (e: ResponseException) { - logger.error("Error fetching from Dlob: ${e.response}") - null - } catch (e: Exception) { - logger.error("Error fetching from Dlob: ${e.message}") - null - } catch (e: Throwable) { - logger.error("Error fetching from Dlob: ${e.message}") - null - } + return DlobHistBase(dlobHistorical) } fun getTokenHistorical(fromDate: DateTime?, toDate: DateTime?) = diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt index 5c68cd75..45f259d6 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt @@ -104,7 +104,7 @@ class AsyncService( protected val logger = logger(AsyncService::class) protected var collectHistorical = true - @Scheduled(initialDelay = 0L, fixedDelay = 5000L) + // @Scheduled(initialDelay = 0L, fixedDelay = 5000L) fun updateLatestBlockHeightJob() { val index = getBlockIndex() val startHeight = blockService.getLatestBlockHeight() @@ -277,10 +277,15 @@ class AsyncService( @Scheduled(cron = "0 0 0/1 * * ?") // Every hour fun saveChainAum() = explorerService.saveChainAum() - @Scheduled(cron = "0 0 1 * * ?") // Every day at 1 am + @Scheduled(cron = "0 0 1 * * *") // Every day at 1 am fun updateTokenHistorical() { val today = DateTime.now().startOfDay() - val startDate = today.minusMonths(1) + var startDate = today.minusMonths(1) + var latest = TokenHistoricalDailyRecord.getLatestDateEntry() + if (latest != null) { + startDate = latest.timestamp.minusDays(1) + } + val dlobRes = tokenService.getHistoricalFromDlob(startDate) ?: return val baseMap = Interval(startDate, today) .let { int -> generateSequence(int.start) { dt -> dt.plusDays(1) }.takeWhile { dt -> dt < int.end } } @@ -306,22 +311,26 @@ class AsyncService( time_low = if (low != null) DateTime(low.trade_timestamp * 1000) else k, quote = mapOf( USD_UPPER to - CmcQuote( - open = open, - high = high?.price ?: prevPrice, - low = low?.price ?: prevPrice, - close = close, - volume = usdVolume, - market_cap = close.multiply(tokenService.totalSupply().divide(UTILITY_TOKEN_BASE_MULTIPLIER)), - timestamp = closeDate - ) + CmcQuote( + open = open, + high = high?.price ?: prevPrice, + low = low?.price ?: prevPrice, + close = close, + volume = usdVolume, + market_cap = close.multiply( + tokenService.totalSupply().divide(UTILITY_TOKEN_BASE_MULTIPLIER) + ), + timestamp = closeDate + ) ) ).also { prevPrice = close } TokenHistoricalDailyRecord.save(record.time_open.startOfDay(), record) } } - @Scheduled(cron = "0 0/5 * * * ?") // Every 5 minutes + // @Scheduled(cron = "0 0/5 * * * ?") // Every 5 minutes + @Scheduled(initialDelay = 0L, fixedDelay = 5000L) + fun updateTokenLatest() { val today = DateTime.now().withZone(DateTimeZone.UTC) val startDate = today.minusDays(7) @@ -346,7 +355,10 @@ class AsyncService( today, mapOf(USD_UPPER to CmcLatestQuoteAbbrev(price, percentChg, vol24Hr, marketCap, today)) ) - CacheUpdateRecord.updateCacheByKey(CacheKeys.UTILITY_TOKEN_LATEST.key, VANILLA_MAPPER.writeValueAsString(rec)) + CacheUpdateRecord.updateCacheByKey( + CacheKeys.UTILITY_TOKEN_LATEST.key, + VANILLA_MAPPER.writeValueAsString(rec) + ) } } @@ -443,7 +455,8 @@ class AsyncService( try { transaction { it.apply { this.processing = true } } send(it.processValue) - } catch (_: Exception) { } + } catch (_: Exception) { + } } } } From 497002dd0d511cb89f7ed9347b8e72ff897c70ba Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Tue, 24 Oct 2023 09:51:29 -0600 Subject: [PATCH 03/11] add back scheduled jobs --- .../io/provenance/explorer/service/async/AsyncService.kt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt index 45f259d6..a6e0abd2 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt @@ -104,7 +104,7 @@ class AsyncService( protected val logger = logger(AsyncService::class) protected var collectHistorical = true - // @Scheduled(initialDelay = 0L, fixedDelay = 5000L) + @Scheduled(initialDelay = 0L, fixedDelay = 5000L) fun updateLatestBlockHeightJob() { val index = getBlockIndex() val startHeight = blockService.getLatestBlockHeight() @@ -328,9 +328,7 @@ class AsyncService( } } - // @Scheduled(cron = "0 0/5 * * * ?") // Every 5 minutes - @Scheduled(initialDelay = 0L, fixedDelay = 5000L) - + @Scheduled(cron = "0 0/5 * * * ?") // Every 5 minutes fun updateTokenLatest() { val today = DateTime.now().withZone(DateTimeZone.UTC) val startDate = today.minusDays(7) From 042c54f1237c14c49b56e86e70c63c9ae5523e13 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Wed, 25 Oct 2023 10:02:27 -0600 Subject: [PATCH 04/11] add delayed init start for dlob historical cron job, various other refactors --- .../explorer/service/TokenService.kt | 7 +++--- .../explorer/service/async/AsyncService.kt | 24 +++++++++++++++---- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt index b08f32bc..c1821666 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt @@ -240,14 +240,13 @@ class TokenService(private val accountClient: AccountGrpcClient) { } } - fun getHistoricalFromDlob(startTime: DateTime): DlobHistBase { + fun getHistoricalFromDlob(startTime: DateTime): DlobHistBase? { val tickerIds = listOf("HASH_USD", "HASH_USDOMNI") val dlobHistorical = tickerIds - .mapNotNull { getHistoricalFromDlob(startTime, it)?.buy } - .flatten() + .flatMap { getHistoricalFromDlob(startTime, it)?.buy.orEmpty() } - return DlobHistBase(dlobHistorical) + return if (dlobHistorical.isNotEmpty()) DlobHistBase(dlobHistorical) else null } fun getTokenHistorical(fromDate: DateTime?, toDate: DateTime?) = diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt index a6e0abd2..d26c7558 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt @@ -84,6 +84,8 @@ import tendermint.types.BlockOuterClass import java.math.BigDecimal import java.time.OffsetDateTime import java.time.ZoneOffset +import javax.annotation.PostConstruct + @Service class AsyncService( @@ -104,7 +106,19 @@ class AsyncService( protected val logger = logger(AsyncService::class) protected var collectHistorical = true - @Scheduled(initialDelay = 0L, fixedDelay = 5000L) + @PostConstruct + fun asyncServiceOnStartInit() { + Thread { + try { + Thread.sleep(5000) + updateTokenHistorical() + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + } + }.start() + } + + @Scheduled(initialDelay = 0L, fixedDelay = 5000L) fun updateLatestBlockHeightJob() { val index = getBlockIndex() val startHeight = blockService.getLatestBlockHeight() @@ -145,7 +159,7 @@ class AsyncService( } fun getBlockIndex() = blockService.getBlockIndexFromCache()?.let { - Pair(it.maxHeightRead, it.minHeightRead) + Pair(it.maxHeightRead, it.minHeightRead) } fun startCollectingHistoricalBlocks(blockIndex: Pair?) = @@ -248,11 +262,11 @@ class AsyncService( val now = OffsetDateTime.now().withOffsetSameInstant(ZoneOffset.UTC).toString() cacheService.getCacheValue(key)!!.let { cache -> pricingService.getPricingAsync(cache.cacheValue!!, "async pricing update").forEach { price -> - // dont set price from PE + // don't set price from PE if (price.markerDenom != UTILITY_TOKEN) { assetService.getAssetRaw(price.markerDenom).let { pricingService.insertAssetPricing(it, price) } } else { - // Pull price from CMC, calced to the true base denom price + // Pull price from CMC, calculate to the true base denom price val cmcPrice = tokenService.getTokenLatest()?.quote?.get(USD_UPPER)?.price ?.let { @@ -281,7 +295,7 @@ class AsyncService( fun updateTokenHistorical() { val today = DateTime.now().startOfDay() var startDate = today.minusMonths(1) - var latest = TokenHistoricalDailyRecord.getLatestDateEntry() + val latest = TokenHistoricalDailyRecord.getLatestDateEntry() if (latest != null) { startDate = latest.timestamp.minusDays(1) } From 2d647e6ab683e2cb0715a27a328c32e12fd44a34 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Wed, 25 Oct 2023 10:26:24 -0600 Subject: [PATCH 05/11] fix ktlint items --- .../io/provenance/explorer/service/TokenService.kt | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt index c1821666..a3ea7d65 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt @@ -31,7 +31,6 @@ import io.provenance.explorer.domain.extensions.toCoinStr import io.provenance.explorer.domain.extensions.toOffset import io.provenance.explorer.domain.extensions.toPercentage import io.provenance.explorer.domain.models.explorer.DlobHistBase -import io.provenance.explorer.domain.models.explorer.DlobHistorical import io.provenance.explorer.domain.models.explorer.TokenHistoricalDataRequest import io.provenance.explorer.grpc.v1.AccountGrpcClient import io.provenance.explorer.model.AssetHolder @@ -70,7 +69,7 @@ class TokenService(private val accountClient: AccountGrpcClient) { AccountRecord.saveAccount( it.ownerAddress, PROV_ACC_PREFIX, - accountClient.getAccountInfo(it.ownerAddress) + accountClient.getAccountInfo(it.ownerAddress), ) }.collect() TokenDistributionPaginatedResultsRecord.savePaginatedResults(records) @@ -96,7 +95,7 @@ class TokenService(private val accountClient: AccountGrpcClient) { denom: String, page: Int, count: Int, - retryCount: Int = 3 + retryCount: Int = 3, ): PagedResults { var hasSucceeded = false var numberOfTriesRemaining = retryCount @@ -132,7 +131,7 @@ class TokenService(private val accountClient: AccountGrpcClient) { Triple(50, 50, "51-100"), Triple(500, 100, "101-500"), Triple(500, 500, "501-1000"), - Triple("ALL", 1000, "1001-") + Triple("ALL", 1000, "1001-"), ).map { (limit, offset, range) -> val results = TokenDistributionPaginatedResultsRecord.findByLimitOffset(richListAccounts(), limit, offset) val denom = results[0].data.denom @@ -154,7 +153,7 @@ class TokenService(private val accountClient: AccountGrpcClient) { circulatingSupply().toCoinStr(UTILITY_TOKEN), communityPoolSupply().toCoinStr(UTILITY_TOKEN), bonded, - burnedSupply().toCoinStr(UTILITY_TOKEN) + burnedSupply().toCoinStr(UTILITY_TOKEN), ) } @@ -171,8 +170,7 @@ class TokenService(private val accountClient: AccountGrpcClient) { runBlocking { accountClient.getCommunityPoolAmount(UTILITY_TOKEN).toBigDecimal().roundWhole() } fun richListAccounts() = - allAccounts().addressList() - zeroSeqAccounts().toSet() - moduleAccounts().addressList() - - contractAccounts().addressList() - setOf(nhashMarkerAddr()) + allAccounts().addressList() - zeroSeqAccounts().toSet() - moduleAccounts().addressList() - contractAccounts().addressList() - setOf(nhashMarkerAddr()) fun totalBalanceForList(addresses: Set) = runBlocking { TokenDistributionPaginatedResultsRecord.findByAddresses(addresses).asFlow() @@ -218,7 +216,7 @@ class TokenService(private val accountClient: AccountGrpcClient) { RichAccount( it.ownerAddress, CoinStr(it.data.count, it.data.denom), - it.data.count.toPercentage(BigDecimal(100), totalSupply, 4) + it.data.count.toPercentage(BigDecimal(100), totalSupply, 4), ) } } From ededff07bfeba1a6aabbd710f6104c79c9791398 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Wed, 25 Oct 2023 10:37:27 -0600 Subject: [PATCH 06/11] fix more ktlint --- .../explorer/domain/entities/ExplorerCache.kt | 6 ++--- .../explorer/service/TokenService.kt | 10 ++++---- .../explorer/service/async/AsyncService.kt | 23 +++++++++---------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt index fe6efe1d..b267bfd2 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt @@ -215,7 +215,7 @@ class ChainAumHourlyRecord(id: EntityID) : IntEntity(id) { fun getAumForPeriod(fromDate: DateTime, toDate: DateTime) = transaction { ChainAumHourlyRecord.find { (ChainAumHourlyTable.datetime greaterEq fromDate) and - (ChainAumHourlyTable.datetime lessEq toDate.plusDays(1)) + (ChainAumHourlyTable.datetime lessEq toDate.plusDays(1)) } .orderBy(Pair(ChainAumHourlyTable.datetime, SortOrder.ASC)) .toList() @@ -301,7 +301,7 @@ class ProcessQueueRecord(id: EntityID) : IntEntity(id) { fun findByType(processType: ProcessQueueType) = transaction { ProcessQueueRecord.find { (ProcessQueueTable.processType eq processType.name) and - (ProcessQueueTable.processing eq false) + (ProcessQueueTable.processing eq false) }.toList() } @@ -314,7 +314,7 @@ class ProcessQueueRecord(id: EntityID) : IntEntity(id) { fun delete(processType: ProcessQueueType, value: String) = transaction { ProcessQueueTable.deleteWhere { (ProcessQueueTable.processType eq processType.name) and - (ProcessQueueTable.processValue eq value) + (processValue eq value) } } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt index a3ea7d65..6d53627e 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt @@ -69,7 +69,7 @@ class TokenService(private val accountClient: AccountGrpcClient) { AccountRecord.saveAccount( it.ownerAddress, PROV_ACC_PREFIX, - accountClient.getAccountInfo(it.ownerAddress), + accountClient.getAccountInfo(it.ownerAddress) ) }.collect() TokenDistributionPaginatedResultsRecord.savePaginatedResults(records) @@ -95,7 +95,7 @@ class TokenService(private val accountClient: AccountGrpcClient) { denom: String, page: Int, count: Int, - retryCount: Int = 3, + retryCount: Int = 3 ): PagedResults { var hasSucceeded = false var numberOfTriesRemaining = retryCount @@ -131,7 +131,7 @@ class TokenService(private val accountClient: AccountGrpcClient) { Triple(50, 50, "51-100"), Triple(500, 100, "101-500"), Triple(500, 500, "501-1000"), - Triple("ALL", 1000, "1001-"), + Triple("ALL", 1000, "1001-") ).map { (limit, offset, range) -> val results = TokenDistributionPaginatedResultsRecord.findByLimitOffset(richListAccounts(), limit, offset) val denom = results[0].data.denom @@ -153,7 +153,7 @@ class TokenService(private val accountClient: AccountGrpcClient) { circulatingSupply().toCoinStr(UTILITY_TOKEN), communityPoolSupply().toCoinStr(UTILITY_TOKEN), bonded, - burnedSupply().toCoinStr(UTILITY_TOKEN), + burnedSupply().toCoinStr(UTILITY_TOKEN) ) } @@ -216,7 +216,7 @@ class TokenService(private val accountClient: AccountGrpcClient) { RichAccount( it.ownerAddress, CoinStr(it.data.count, it.data.denom), - it.data.count.toPercentage(BigDecimal(100), totalSupply, 4), + it.data.count.toPercentage(BigDecimal(100), totalSupply, 4) ) } } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt index d26c7558..62d20925 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt @@ -86,7 +86,6 @@ import java.time.OffsetDateTime import java.time.ZoneOffset import javax.annotation.PostConstruct - @Service class AsyncService( private val props: ExplorerProperties, @@ -325,17 +324,17 @@ class AsyncService( time_low = if (low != null) DateTime(low.trade_timestamp * 1000) else k, quote = mapOf( USD_UPPER to - CmcQuote( - open = open, - high = high?.price ?: prevPrice, - low = low?.price ?: prevPrice, - close = close, - volume = usdVolume, - market_cap = close.multiply( - tokenService.totalSupply().divide(UTILITY_TOKEN_BASE_MULTIPLIER) - ), - timestamp = closeDate - ) + CmcQuote( + open = open, + high = high?.price ?: prevPrice, + low = low?.price ?: prevPrice, + close = close, + volume = usdVolume, + market_cap = close.multiply( + tokenService.totalSupply().divide(UTILITY_TOKEN_BASE_MULTIPLIER) + ), + timestamp = closeDate + ) ) ).also { prevPrice = close } TokenHistoricalDailyRecord.save(record.time_open.startOfDay(), record) From 4429244bbf9686bdf65f14c6d242af192a61346c Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Thu, 26 Oct 2023 15:12:45 -0600 Subject: [PATCH 07/11] add some logging --- .../io/provenance/explorer/service/async/AsyncService.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt index 62d20925..d36cd199 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt @@ -298,8 +298,9 @@ class AsyncService( if (latest != null) { startDate = latest.timestamp.minusDays(1) } - val dlobRes = tokenService.getHistoricalFromDlob(startDate) ?: return + logger.info("Updating token historical data starting from $startDate with ${dlobRes.buy.size} buy record for roll-up.") + val baseMap = Interval(startDate, today) .let { int -> generateSequence(int.start) { dt -> dt.plusDays(1) }.takeWhile { dt -> dt < int.end } } .map { it to emptyList() }.toMap().toMutableMap() From 01fdc2fd3076afbff782da81fd9a6d125a029798 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Thu, 26 Oct 2023 15:24:11 -0600 Subject: [PATCH 08/11] add mainnet dlob address --- .../main/kotlin/io/provenance/explorer/service/TokenService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt index 6d53627e..0de1ef68 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt @@ -223,7 +223,7 @@ class TokenService(private val accountClient: AccountGrpcClient) { fun getHistoricalFromDlob(startTime: DateTime, tickerId: String): DlobHistBase? = runBlocking { try { - KTOR_CLIENT_JAVA.get("https://test.dlob.io:443/gecko/external/api/v1/exchange/historical_trades") { + KTOR_CLIENT_JAVA.get("https://www.dlob.io:443/gecko/external/api/v1/exchange/historical_trades") { parameter("ticker_id", tickerId) parameter("type", "buy") parameter("start_time", DateTimeFormat.forPattern("dd-MM-yyyy").print(startTime)) From 490ccfbd39a1be1ae2bc2986714e72a10c0f4aa1 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Fri, 27 Oct 2023 12:38:24 -0600 Subject: [PATCH 09/11] Add deletion of historical daily database version --- .../db/migration/V1_89__Delete_from_token_historical_daily.sql | 3 +++ .../io/provenance/explorer/service/async/AsyncService.kt | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 database/src/main/resources/db/migration/V1_89__Delete_from_token_historical_daily.sql diff --git a/database/src/main/resources/db/migration/V1_89__Delete_from_token_historical_daily.sql b/database/src/main/resources/db/migration/V1_89__Delete_from_token_historical_daily.sql new file mode 100644 index 00000000..08826c58 --- /dev/null +++ b/database/src/main/resources/db/migration/V1_89__Delete_from_token_historical_daily.sql @@ -0,0 +1,3 @@ +SELECT 'Deleting from token_historical_daily dates after 8/23/2023' AS comment; + +DELETE FROM token_historical_daily WHERE historical_timestamp > '2023-08-23 00:00:00'; diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt index d36cd199..99263ba5 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt @@ -299,7 +299,7 @@ class AsyncService( startDate = latest.timestamp.minusDays(1) } val dlobRes = tokenService.getHistoricalFromDlob(startDate) ?: return - logger.info("Updating token historical data starting from $startDate with ${dlobRes.buy.size} buy record for roll-up.") + logger.info("Updating token historical data starting from $startDate with ${dlobRes.buy.size} buy records for roll-up.") val baseMap = Interval(startDate, today) .let { int -> generateSequence(int.start) { dt -> dt.plusDays(1) }.takeWhile { dt -> dt < int.end } } From 775f76b3501d47da55e82b471e2f3fc891365146 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Fri, 27 Oct 2023 12:48:38 -0600 Subject: [PATCH 10/11] Update delete timestamp --- .../migration/V1_89__Delete_from_token_historical_daily.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/src/main/resources/db/migration/V1_89__Delete_from_token_historical_daily.sql b/database/src/main/resources/db/migration/V1_89__Delete_from_token_historical_daily.sql index 08826c58..9a838524 100644 --- a/database/src/main/resources/db/migration/V1_89__Delete_from_token_historical_daily.sql +++ b/database/src/main/resources/db/migration/V1_89__Delete_from_token_historical_daily.sql @@ -1,3 +1,3 @@ -SELECT 'Deleting from token_historical_daily dates after 8/23/2023' AS comment; +SELECT 'Deleting from token_historical_daily dates after 8/03/2023' AS comment; -DELETE FROM token_historical_daily WHERE historical_timestamp > '2023-08-23 00:00:00'; +DELETE FROM token_historical_daily WHERE historical_timestamp > '2023-08-03 00:00:00'; From 3132c7b95d60f722cb97c4ef41579bb6a2c263d3 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Fri, 27 Oct 2023 12:50:54 -0600 Subject: [PATCH 11/11] revert cron change --- .../kotlin/io/provenance/explorer/service/async/AsyncService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt index 99263ba5..7ffe1c46 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt @@ -290,7 +290,7 @@ class AsyncService( @Scheduled(cron = "0 0 0/1 * * ?") // Every hour fun saveChainAum() = explorerService.saveChainAum() - @Scheduled(cron = "0 0 1 * * *") // Every day at 1 am + @Scheduled(cron = "0 0 1 * * ?") // Every day at 1 am fun updateTokenHistorical() { val today = DateTime.now().startOfDay() var startDate = today.minusMonths(1)