From 12e7e43ff81011a1d313b72744d59ee6c9f6afd2 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Tue, 17 Sep 2024 15:53:24 -0600 Subject: [PATCH 1/8] remove logic to update running count and total count --- .../explorer/domain/entities/Blocks.kt | 77 +------------------ .../explorer/service/ValidatorService.kt | 2 +- 2 files changed, 5 insertions(+), 74 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt index 01512d59..5bd7e218 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt @@ -2,7 +2,6 @@ package io.provenance.explorer.domain.entities import cosmos.base.tendermint.v1beta1.Query import io.provenance.explorer.OBJECT_MAPPER -import io.provenance.explorer.domain.core.logger import io.provenance.explorer.domain.core.sql.DateTrunc import io.provenance.explorer.domain.core.sql.Distinct import io.provenance.explorer.domain.core.sql.ExtractDOW @@ -58,9 +57,7 @@ import org.jetbrains.exposed.sql.jodatime.datetime import org.jetbrains.exposed.sql.leftJoin import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.selectAll -import org.jetbrains.exposed.sql.statements.BatchUpdateStatement import org.jetbrains.exposed.sql.sum -import org.jetbrains.exposed.sql.transactions.TransactionManager import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update import org.joda.time.DateTime @@ -263,79 +260,13 @@ class MissedBlocksRecord(id: EntityID) : IntEntity(id) { query.exec(arguments).map { it.getString("val_cons_address") } } - fun findLatestForVal(valconsAddr: String) = transaction { - MissedBlocksRecord.find { MissedBlocksTable.valConsAddr eq valconsAddr } - .orderBy(Pair(MissedBlocksTable.blockHeight, SortOrder.DESC)) - .limit(1) - .firstOrNull() - } - - fun findForValFirstUnderHeight(valconsAddr: String, height: Int) = transaction { - MissedBlocksRecord - .find { (MissedBlocksTable.valConsAddr eq valconsAddr) and (MissedBlocksTable.blockHeight lessEq height) } - .orderBy(Pair(MissedBlocksTable.blockHeight, SortOrder.DESC)) - .limit(1) - .firstOrNull() - } - - fun calculateMissedAndInsert(height: Int, valconsAddr: String) = transaction { - val startTime = System.currentTimeMillis() - - val (running, total, updateFromHeight) = findLatestForVal(valconsAddr)?.let { rec -> - when { - rec.blockHeight == height - 1 -> listOf(rec.runningCount, rec.totalCount, null) - rec.blockHeight > height -> - when (val last = findForValFirstUnderHeight(valconsAddr, height - 1)) { - null -> listOf(0, 0, height) - else -> listOf( - if (last.blockHeight == height - 1) last.runningCount else 0, - last.totalCount, - height - ) - } - else -> listOf(0, rec.totalCount, null) - } - } ?: listOf(0, 0, null) - + fun insert(height: Int, valconsAddr: String) = transaction { MissedBlocksTable.insertIgnore { it[this.blockHeight] = height it[this.valConsAddr] = valconsAddr - it[this.runningCount] = running!! + 1 - it[this.totalCount] = total!! + 1 - } - - if (updateFromHeight != null) { - updateRecords(updateFromHeight, valconsAddr, running!! + 1, total!! + 1) - } - - val endTime = System.currentTimeMillis() - val duration = endTime - startTime - - // TODO: remove warning if problem is fixed after adding limit to queries See: https://github.com/provenance-io/explorer-service/issues/546 - if (duration > 1000) { - logger().warn("Processing calculateMissedAndInsert took ${duration}ms for height: $height and valconsAddr: $valconsAddr") - } - } - - fun updateRecords(height: Int, valconsAddr: String, currRunning: Int, currTotal: Int) = transaction { - val records = MissedBlocksRecord - .find { (MissedBlocksTable.valConsAddr eq valconsAddr) and (MissedBlocksTable.blockHeight greater height) } - .orderBy(Pair(MissedBlocksTable.blockHeight, SortOrder.ASC)) - - BatchUpdateStatement(MissedBlocksTable).apply { - var lastHeight = height - var lastRunning = currRunning - var lastTotal = currTotal - records.forEach { - addBatch(it.id) - val running = if (lastHeight == it.blockHeight - 1) lastRunning + 1 else 1 - this[MissedBlocksTable.runningCount] = running - this[MissedBlocksTable.totalCount] = lastTotal + 1 - lastHeight = it.blockHeight - lastRunning = running - lastTotal += 1 - } - execute(TransactionManager.current()) + // TODO: remove these column from database + it[this.runningCount] = -1 + it[this.totalCount] = -1 } } } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt b/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt index 27942e04..61c3f3b8 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt @@ -502,7 +502,7 @@ class ValidatorService( currentVals.validatorsList.forEach { vali -> if (!signatures.contains(vali.address)) { - MissedBlocksRecord.calculateMissedAndInsert(lastBlock.height.toInt(), vali.address) + MissedBlocksRecord.insert(lastBlock.height.toInt(), vali.address) } } } From 0f6bd5e0ed8954db6a4a6fd54f135716b3177802 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Tue, 17 Sep 2024 19:54:36 -0600 Subject: [PATCH 2/8] add temp logging --- .../io/provenance/explorer/service/async/ScheduledTaskService.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt index 845b7e08..22b2866a 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt @@ -123,6 +123,7 @@ class ScheduledTaskService( val index = getBlockIndex() val startHeight = blockService.getLatestBlockHeight() var indexHeight = startHeight + logger.info("Starting updateLatestBlockHeightJob startHeight $startHeight") if (startCollectingHistoricalBlocks(index) || continueCollectingHistoricalBlocks(index!!.first!!, index.second!!) ) { From a9dd789f15c1d7ca297aa49027fafdcf7a972a00 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Wed, 18 Sep 2024 07:38:20 -0600 Subject: [PATCH 3/8] add temp counter --- .../provenance/explorer/service/async/ScheduledTaskService.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt index 22b2866a..a4eefacd 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt @@ -120,6 +120,7 @@ class ScheduledTaskService( @Scheduled(initialDelay = 0L, fixedDelay = 5000L) fun updateLatestBlockHeightJob() { + var count = 0 val index = getBlockIndex() val startHeight = blockService.getLatestBlockHeight() var indexHeight = startHeight @@ -139,6 +140,7 @@ class ScheduledTaskService( blockAndTxProcessor.saveBlockEtc(it) indexHeight = it.block.height() - 1 } + count++ blockService.updateBlockMinHeightIndex(indexHeight + 1) } blockService.updateBlockMaxHeightIndex(startHeight) @@ -147,6 +149,7 @@ class ScheduledTaskService( blockService.getBlockAtHeightFromChain(indexHeight)?.let { blockAndTxProcessor.saveBlockEtc(it) indexHeight = it.block.height() - 1 + count++ } } blockService.updateBlockMaxHeightIndex(startHeight) @@ -157,6 +160,7 @@ class ScheduledTaskService( if (!cacheService.getCacheValue(CacheKeys.SPOTLIGHT_PROCESSING.key)!!.cacheValue.toBoolean()) { cacheService.updateCacheValue(CacheKeys.SPOTLIGHT_PROCESSING.key, true.toString()) } + logger.info("Finished updateLatestBlockHeightJob added: $count") } fun getBlockIndex() = blockService.getBlockIndexFromCache()?.let { From 0415649e3c3c39276af8239adff8b163f0f51de3 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Wed, 18 Sep 2024 08:46:33 -0600 Subject: [PATCH 4/8] comment out calc latency function --- .../explorer/service/async/ScheduledTaskService.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt index a4eefacd..ea308899 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt @@ -143,6 +143,7 @@ class ScheduledTaskService( count++ blockService.updateBlockMinHeightIndex(indexHeight + 1) } + logger.info("Finished updateLatestBlockHeightJob historical added: $count") blockService.updateBlockMaxHeightIndex(startHeight) } else { while (indexHeight > index.first!!) { @@ -152,15 +153,16 @@ class ScheduledTaskService( count++ } } + logger.info("Finished updateLatestBlockHeightJob added: $count") blockService.updateBlockMaxHeightIndex(startHeight) } BlockTxCountsCacheRecord.updateTxCounts() - BlockProposerRecord.calcLatency() + // BlockProposerRecord.calcLatency() if (!cacheService.getCacheValue(CacheKeys.SPOTLIGHT_PROCESSING.key)!!.cacheValue.toBoolean()) { cacheService.updateCacheValue(CacheKeys.SPOTLIGHT_PROCESSING.key, true.toString()) } - logger.info("Finished updateLatestBlockHeightJob added: $count") + logger.info("Finished updateLatestBlockHeightJob") } fun getBlockIndex() = blockService.getBlockIndexFromCache()?.let { From 468259dc3047aae71df7bacaf2e7da4473aacd7e Mon Sep 17 00:00:00 2001 From: Carlton Hanna Date: Thu, 19 Sep 2024 09:37:18 -0600 Subject: [PATCH 5/8] remove tmp logging, uncomment latency calc --- .../explorer/service/async/ScheduledTaskService.kt | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt index ea308899..2d0e3f47 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt @@ -120,11 +120,9 @@ class ScheduledTaskService( @Scheduled(initialDelay = 0L, fixedDelay = 5000L) fun updateLatestBlockHeightJob() { - var count = 0 val index = getBlockIndex() val startHeight = blockService.getLatestBlockHeight() var indexHeight = startHeight - logger.info("Starting updateLatestBlockHeightJob startHeight $startHeight") if (startCollectingHistoricalBlocks(index) || continueCollectingHistoricalBlocks(index!!.first!!, index.second!!) ) { @@ -140,29 +138,24 @@ class ScheduledTaskService( blockAndTxProcessor.saveBlockEtc(it) indexHeight = it.block.height() - 1 } - count++ blockService.updateBlockMinHeightIndex(indexHeight + 1) } - logger.info("Finished updateLatestBlockHeightJob historical added: $count") blockService.updateBlockMaxHeightIndex(startHeight) } else { while (indexHeight > index.first!!) { blockService.getBlockAtHeightFromChain(indexHeight)?.let { blockAndTxProcessor.saveBlockEtc(it) indexHeight = it.block.height() - 1 - count++ } } - logger.info("Finished updateLatestBlockHeightJob added: $count") blockService.updateBlockMaxHeightIndex(startHeight) } BlockTxCountsCacheRecord.updateTxCounts() - // BlockProposerRecord.calcLatency() + BlockProposerRecord.calcLatency() if (!cacheService.getCacheValue(CacheKeys.SPOTLIGHT_PROCESSING.key)!!.cacheValue.toBoolean()) { cacheService.updateCacheValue(CacheKeys.SPOTLIGHT_PROCESSING.key, true.toString()) } - logger.info("Finished updateLatestBlockHeightJob") } fun getBlockIndex() = blockService.getBlockIndexFromCache()?.let { From bc0d02af614632ff24dc3fbda0e7c866421430da Mon Sep 17 00:00:00 2001 From: Carlton Hanna Date: Thu, 19 Sep 2024 09:40:48 -0600 Subject: [PATCH 6/8] fix formatting --- .../provenance/explorer/service/async/ScheduledTaskService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt index 2d0e3f47..845b7e08 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt @@ -152,7 +152,7 @@ class ScheduledTaskService( } BlockTxCountsCacheRecord.updateTxCounts() - BlockProposerRecord.calcLatency() + BlockProposerRecord.calcLatency() if (!cacheService.getCacheValue(CacheKeys.SPOTLIGHT_PROCESSING.key)!!.cacheValue.toBoolean()) { cacheService.updateCacheValue(CacheKeys.SPOTLIGHT_PROCESSING.key, true.toString()) } From 95a9982f7dd96cad98e9f54dab7a84db0914d840 Mon Sep 17 00:00:00 2001 From: Carlton Hanna Date: Thu, 19 Sep 2024 09:50:13 -0600 Subject: [PATCH 7/8] add issue link to todo --- .../kotlin/io/provenance/explorer/domain/entities/Blocks.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt index 5bd7e218..5eff818a 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt @@ -264,7 +264,7 @@ class MissedBlocksRecord(id: EntityID) : IntEntity(id) { MissedBlocksTable.insertIgnore { it[this.blockHeight] = height it[this.valConsAddr] = valconsAddr - // TODO: remove these column from database + // TODO: remove these column from database See: https://github.com/provenance-io/explorer-service/issues/549 it[this.runningCount] = -1 it[this.totalCount] = -1 } From a663a3612e9a9f2c9dbd08abf0112cebec634db9 Mon Sep 17 00:00:00 2001 From: Carlton Hanna Date: Thu, 19 Sep 2024 10:00:35 -0600 Subject: [PATCH 8/8] add change log entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86a15720..43c785d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ ### Improvements * Add limit of 1 to missing block queries [#544](https://github.com/provenance-io/explorer-service/pull/544) +* Removes the logic for calculating `running_count` and `total_count` in the `missed_blocks` [#548](https://github.com/provenance-io/explorer-service/pull/548) ## [v5.11.0](https://github.com/provenance-io/explorer-service/releases/tag/v5.11.0) - 2024-08-27