From e696de88b107f5c300b9b10b0f2f8cf956aa24c6 Mon Sep 17 00:00:00 2001 From: Carlton Hanna Date: Mon, 28 Oct 2024 07:11:05 -0600 Subject: [PATCH] Remove spotlight from caching to the database (#559) * remove caching of spot light to the database * add change log --- CHANGELOG.md | 1 + .../explorer/config/ExplorerProperties.kt | 3 -- .../explorer/domain/entities/ExplorerCache.kt | 29 ------------------- .../explorer/service/CacheService.kt | 13 ++++++--- .../explorer/service/ExplorerService.kt | 23 ++++++--------- .../explorer/service/ValidatorService.kt | 3 +- .../service/async/ScheduledTaskService.kt | 3 -- .../application-container.properties | 1 - .../application-development.properties | 1 - .../explorer/service/ExplorerServiceTest.kt | 9 ------ .../resources/application-test.properties | 1 - 11 files changed, 20 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e8d5251..e7eea78d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * Remove update block latency procedure call [#550](https://github.com/provenance-io/explorer-service/pull/550) * Docker images at tag `latest` for main branch merges [#551](https://github.com/provenance-io/explorer-service/pull/551) * Refactor account processing implementation to be more efficient [#552](https://github.com/provenance-io/explorer-service/issues/552) +* Remove spotlight from caching to the database [#532](https://github.com/provenance-io/explorer-service/issues/532) * Update keep alive times for flow api grpc calls [#558](https://github.com/provenance-io/explorer-service/pull/558) * Updates to asset pricing table will set data column to null [#562](https://github.com/provenance-io/explorer-service/pull/562) diff --git a/service/src/main/kotlin/io/provenance/explorer/config/ExplorerProperties.kt b/service/src/main/kotlin/io/provenance/explorer/config/ExplorerProperties.kt index dd4af046..a8ea49ee 100644 --- a/service/src/main/kotlin/io/provenance/explorer/config/ExplorerProperties.kt +++ b/service/src/main/kotlin/io/provenance/explorer/config/ExplorerProperties.kt @@ -15,7 +15,6 @@ class ExplorerProperties( val pbUrl: String, val flowApiUrl: String, val initialHistoricalDayCount: String, - val spotlightTtlMs: String, val genesisVersionUrl: String, val upgradeVersionRegex: String, val upgradeGithubRepo: String, @@ -28,8 +27,6 @@ class ExplorerProperties( fun initialHistoricalDays() = initialHistoricalDayCount.toInt() - fun spotlightTtlMs() = spotlightTtlMs.toLong() - fun hiddenApis() = hiddenApis.toBoolean() fun oneElevenBugRange() = diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt index 2dcc6165..824170eb 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt @@ -5,7 +5,6 @@ import io.provenance.explorer.domain.core.sql.jsonb import io.provenance.explorer.model.ChainAum import io.provenance.explorer.model.ChainMarketRate import io.provenance.explorer.model.CmcHistoricalQuote -import io.provenance.explorer.model.Spotlight import io.provenance.explorer.model.ValidatorMarketRate import io.provenance.explorer.model.base.USD_UPPER import org.jetbrains.exposed.dao.Entity @@ -30,33 +29,6 @@ import org.jetbrains.exposed.sql.update import org.joda.time.DateTime import java.math.BigDecimal -object SpotlightCacheTable : IntIdTable(name = "spotlight_cache") { - val spotlight = jsonb("spotlight", OBJECT_MAPPER) - val lastHit = datetime("last_hit") -} - -class SpotlightCacheRecord(id: EntityID) : IntEntity(id) { - companion object : IntEntityClass(SpotlightCacheTable) { - fun getSpotlight() = transaction { - SpotlightCacheRecord.all() - .orderBy(Pair(SpotlightCacheTable.id, SortOrder.DESC)) - .limit(1) - .firstOrNull() - ?.spotlight - } - - fun insertIgnore(json: Spotlight) = transaction { - SpotlightCacheTable.insertIgnore { - it[this.spotlight] = json - it[this.lastHit] = DateTime.now() - } - } - } - - var spotlight by SpotlightCacheTable.spotlight - var lastHit by SpotlightCacheTable.lastHit -} - object ValidatorMarketRateStatsTable : IntIdTable(name = "validator_market_rate_stats") { val date = date("date") val operatorAddress = varchar("operator_address", 96) @@ -177,7 +149,6 @@ object CacheUpdateTable : IntIdTable(name = "cache_update") { enum class CacheKeys(val key: String) { PRICING_UPDATE("pricing_update"), CHAIN_RELEASES("chain_releases"), - SPOTLIGHT_PROCESSING("spotlight_processing"), STANDARD_BLOCK_TIME("standard_block_time"), UTILITY_TOKEN_LATEST("utility_token_latest"), FEE_BUG_ONE_ELEVEN_START_BLOCK("fee_bug_one_eleven_start_block"), diff --git a/service/src/main/kotlin/io/provenance/explorer/service/CacheService.kt b/service/src/main/kotlin/io/provenance/explorer/service/CacheService.kt index 20cee39e..0459dd22 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/CacheService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/CacheService.kt @@ -3,19 +3,24 @@ package io.provenance.explorer.service import io.provenance.explorer.domain.core.logger import io.provenance.explorer.domain.entities.CacheKeys import io.provenance.explorer.domain.entities.CacheUpdateRecord -import io.provenance.explorer.domain.entities.SpotlightCacheRecord import io.provenance.explorer.model.Spotlight -import org.jetbrains.exposed.sql.transactions.transaction import org.springframework.stereotype.Service @Service class CacheService { + @Volatile + private var cachedSpotlight: Spotlight? = null + protected val logger = logger(CacheService::class) - fun addSpotlightToCache(spotlightResponse: Spotlight) = SpotlightCacheRecord.insertIgnore(spotlightResponse) + fun updateSpotlight(spotlightResponse: Spotlight) { + cachedSpotlight = spotlightResponse + } - fun getSpotlight() = transaction { SpotlightCacheRecord.getSpotlight() } + fun getSpotlight(): Spotlight? { + return cachedSpotlight + } fun getCacheValue(key: String) = CacheUpdateRecord.fetchCacheByKey(key) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/ExplorerService.kt b/service/src/main/kotlin/io/provenance/explorer/service/ExplorerService.kt index db7d319c..b2852961 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/ExplorerService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/ExplorerService.kt @@ -166,20 +166,15 @@ class ExplorerService( fun getSpotlightStatistics() = cacheService.getSpotlight() - fun createSpotlight() = - when { - cacheService.getCacheValue(CacheKeys.SPOTLIGHT_PROCESSING.key)?.cacheValue.toBoolean() -> - getBondedTokenRatio().let { - Spotlight( - latestBlock = getBlockAtHeight(blockService.getMaxBlockCacheHeight() - 1), - avgBlockTime = BlockProposerRecord.findAvgBlockCreation(100), - bondedTokens = CountStrTotal(it.first.toString(), it.second, UTILITY_TOKEN), - totalTxCount = BlockCacheHourlyTxCountsRecord.getTotalTxCount().toBigInteger(), - totalAum = pricingService.getTotalAum().toCoinStr(USD_UPPER) - ) - }.let { cacheService.addSpotlightToCache(it) } - else -> null - } + fun createSpotlight() = getBondedTokenRatio().let { + Spotlight( + latestBlock = getBlockAtHeight(blockService.getMaxBlockCacheHeight() - 1), + avgBlockTime = BlockProposerRecord.findAvgBlockCreation(100), + bondedTokens = CountStrTotal(it.first.toString(), it.second, UTILITY_TOKEN), + totalTxCount = BlockCacheHourlyTxCountsRecord.getTotalTxCount().toBigInteger(), + totalAum = pricingService.getTotalAum().toCoinStr(USD_UPPER) + ) + }.let { cacheService.updateSpotlight(it) } fun getBondedTokenRatio() = let { val totalBlockChainTokens = assetService.getCurrentSupply(UTILITY_TOKEN) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt b/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt index bb8ef676..b46657bb 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt @@ -14,7 +14,6 @@ import io.provenance.explorer.config.ResourceNotFoundException import io.provenance.explorer.domain.core.logger import io.provenance.explorer.domain.entities.AddressImageRecord import io.provenance.explorer.domain.entities.MissedBlocksRecord -import io.provenance.explorer.domain.entities.SpotlightCacheRecord import io.provenance.explorer.domain.entities.StakingValidatorCacheRecord import io.provenance.explorer.domain.entities.ValidatorMarketRateRecord import io.provenance.explorer.domain.entities.ValidatorMarketRateStatsRecord @@ -542,7 +541,7 @@ class ValidatorService( } } - private fun getLatestHeight() = SpotlightCacheRecord.getSpotlight()?.latestBlock?.height ?: blockService.getMaxBlockCacheHeight() + private fun getLatestHeight() = cacheService.getSpotlight()?.latestBlock?.height ?: blockService.getMaxBlockCacheHeight() fun getDistinctValidatorsWithMissedBlocksInTimeframe(timeframe: Timeframe) = transaction { val currentHeight = getLatestHeight() diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt index 9f55f9a1..9172fa9c 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt @@ -128,9 +128,6 @@ class ScheduledTaskService( } BlockTxCountsCacheRecord.updateTxCounts() - if (!cacheService.getCacheValue(CacheKeys.SPOTLIGHT_PROCESSING.key)!!.cacheValue.toBoolean()) { - cacheService.updateCacheValue(CacheKeys.SPOTLIGHT_PROCESSING.key, true.toString()) - } } fun getBlockIndex() = blockService.getBlockIndexFromCache()?.let { diff --git a/service/src/main/resources/application-container.properties b/service/src/main/resources/application-container.properties index 83d77da5..00d01de6 100644 --- a/service/src/main/resources/application-container.properties +++ b/service/src/main/resources/application-container.properties @@ -6,7 +6,6 @@ spring.datasource.url=${SPRING_DATASOURCE_URL} spring.datasource.hikari.schema=${DB_SCHEMA} spring.datasource.hikari.maximum-pool-size=${DB_CONNECTION_POOL_SIZE} -explorer.spotlight-ttl-ms=${SPOTLIGHT_TTL_MS} explorer.initial-historical-day-count=${INITIAL_HIST_DAY_COUNT} explorer.mainnet=${EXPLORER_MAINNET} explorer.pb-url=${EXPLORER_PB_URL} diff --git a/service/src/main/resources/application-development.properties b/service/src/main/resources/application-development.properties index d08fd66f..3fcaf009 100644 --- a/service/src/main/resources/application-development.properties +++ b/service/src/main/resources/application-development.properties @@ -9,7 +9,6 @@ spring.datasource.hikari.maximum-pool-size=40 #explorer.pb-url=http://localhost:9090 explorer.flow-api-url=http://localhost:50051 explorer.initial-historical-day-count=14 -explorer.spotlight-ttl-ms=5000 explorer.upgrade-version-regex=(v[0-9]+.[0-9]+.[0-9]+[-\\w\\d]*) explorer.upgrade-github-repo=provenance-io/provenance explorer.hidden-apis=false diff --git a/service/src/test/kotlin/io/provenance/explorer/service/ExplorerServiceTest.kt b/service/src/test/kotlin/io/provenance/explorer/service/ExplorerServiceTest.kt index b52a6205..9d1fb5ae 100644 --- a/service/src/test/kotlin/io/provenance/explorer/service/ExplorerServiceTest.kt +++ b/service/src/test/kotlin/io/provenance/explorer/service/ExplorerServiceTest.kt @@ -140,15 +140,6 @@ // } // // @Test -// // @Ignore("TODO turn into integration tests") -// fun `should return spotlight object with current block and average block creation time`() { -// Thread.sleep(5000) -// val result = explorerService.getSpotlightStatistics() -// Assert.assertNotNull(result) -// Assert.assertTrue(result.avgBlockTime > BigDecimal("0.0")) -// } -// -// @Test // // @Ignore // fun `test get transaction history`() { // // https://test.provenance.io/explorer/secured/api/v1/txs/history?toDate=2020-11-24&fromDate=2020-11-11&granulatiry=day diff --git a/service/src/test/resources/application-test.properties b/service/src/test/resources/application-test.properties index 0eb387d0..a1561f6e 100644 --- a/service/src/test/resources/application-test.properties +++ b/service/src/test/resources/application-test.properties @@ -27,6 +27,5 @@ explorer.tendermint-url=https://test.provenance.io/explorer/sentinel/ explorer.pb-client-timeout-ms=120000 explorer.pb-url=http://localhost:1317/ explorer.initial-historical-day-count=2 -explorer.spotlight-ttl-ms=5000 explorer.staking-validator-ttl-ms=5000 explorer.staking-validator-delegations-ttl-ms=5000