Remove update block latency procedure call (#550)

* remove latency calculation, update block time lag function to use query, comment out validator lag for now

* drop update_block_latency procedure, few lints

* fix query

* add a temporary logging statement

* remove latency api for validator address

* remove typo

* use execAndMap

* refactor to use execAndMap properly, add temp logging

* remove temporary logging

* fix lint

* remove unused import

* add change log

* fix lint

CHANGELOG.md

@@ -37,6 +37,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 * Add limit of 1 to missing block queries [#544](https://github.com/provenance-io/explorer-service/pull/544)
 * Removes the logic for calculating `running_count` and `total_count` in the `missed_blocks` [#548](https://github.com/provenance-io/explorer-service/pull/548)
+* Remove update block latency procedure call [#550](https://github.com/provenance-io/explorer-service/pull/550)
 
 ## [v5.11.0](https://github.com/provenance-io/explorer-service/releases/tag/v5.11.0) - 2024-08-27
 

database/src/main/resources/db/migration/V1_92__Drop_update_block_latency_procedure.sql

@@ -0,0 +1,3 @@
+SELECT 'Drop update_block_latency procedure' AS comment;
+
+DROP PROCEDURE IF EXISTS update_block_latency();

service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt

@@ -9,7 +9,6 @@ import io.provenance.explorer.domain.core.sql.ExtractDay
 import io.provenance.explorer.domain.core.sql.ExtractHour
 import io.provenance.explorer.domain.core.sql.jsonb
 import io.provenance.explorer.domain.core.sql.toProcedureObject
-import io.provenance.explorer.domain.extensions.average
 import io.provenance.explorer.domain.extensions.exec
 import io.provenance.explorer.domain.extensions.execAndMap
 import io.provenance.explorer.domain.extensions.map
@@ -62,6 +61,7 @@ import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
 import org.joda.time.DateTime
 import org.joda.time.DateTimeZone
+import java.math.BigDecimal
 import java.sql.ResultSet
 
 object BlockCacheTable : CacheIdTable<Int>(name = "block_cache") {
@@ -168,7 +168,6 @@ object BlockProposerTable : IdTable<Int>(name = "block_proposer") {
     override val id = blockHeight.entityId()
     val proposerOperatorAddress = varchar("proposer_operator_address", 96)
     val blockTimestamp = datetime("block_timestamp")
-    val blockLatency = decimal("block_latency", 50, 25).nullable()
 }
 
 fun BlockProposer.buildInsert() = listOf(
@@ -184,18 +183,27 @@ class BlockProposerRecord(id: EntityID<Int>) : IntEntity(id) {
         fun buildInsert(height: Int, minGasFee: Double, timestamp: DateTime, proposer: String) =
             listOf(height, proposer, minGasFee, timestamp, null).toProcedureObject()
 
-        fun calcLatency() = transaction {
-            val query = "CALL update_block_latency();"
-            this.exec(query)
-        }
-
-        fun findAvgBlockCreation(limit: Int) = transaction {
-            BlockProposerTable.slice(BlockProposerTable.blockLatency)
-                .select { BlockProposerTable.blockLatency.isNotNull() }
-                .orderBy(BlockProposerTable.blockHeight, SortOrder.DESC)
-                .limit(limit)
-                .mapNotNull { it[BlockProposerTable.blockLatency] }
-                .average()
+        fun findAvgBlockCreation(limit: Int): BigDecimal = transaction {
+            val sqlQuery =
+"""
+    WITH limited_blocks AS (
+        SELECT block_height, block_timestamp
+        FROM block_proposer
+        WHERE block_timestamp IS NOT NULL
+        ORDER BY block_height DESC
+        LIMIT $limit
+    )
+    SELECT AVG(diff_in_seconds) AS avg_block_creation_time
+    FROM (
+        SELECT EXTRACT(EPOCH FROM (block_timestamp - LAG(block_timestamp) 
+            OVER (ORDER BY block_height))) AS diff_in_seconds
+        FROM limited_blocks
+    ) AS time_differences
+    WHERE diff_in_seconds IS NOT NULL;
+""".trimIndent()
+            sqlQuery.execAndMap {
+                it.getBigDecimal("avg_block_creation_time")
+            }.firstOrNull() ?: BigDecimal.ZERO
         }
 
         fun findMissingRecords(min: Int, max: Int, limit: Int) = transaction {
@@ -210,7 +218,7 @@ class BlockProposerRecord(id: EntityID<Int>) : IntEntity(id) {
 
         fun getRecordsForProposer(address: String, limit: Int) = transaction {
             BlockProposerRecord.find {
-                (BlockProposerTable.proposerOperatorAddress eq address) and (BlockProposerTable.blockLatency.isNotNull())
+                (BlockProposerTable.proposerOperatorAddress eq address)
             }.orderBy(Pair(BlockProposerTable.blockHeight, SortOrder.DESC))
                 .limit(limit)
                 .toList()
@@ -220,7 +228,6 @@ class BlockProposerRecord(id: EntityID<Int>) : IntEntity(id) {
     var blockHeight by BlockProposerTable.blockHeight
     var proposerOperatorAddress by BlockProposerTable.proposerOperatorAddress
     var blockTimestamp by BlockProposerTable.blockTimestamp
-    var blockLatency by BlockProposerTable.blockLatency
 }
 
 object MissedBlocksTable : IntIdTable(name = "missed_blocks") {

service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt

@@ -13,7 +13,6 @@ import io.provenance.explorer.config.ExplorerProperties.Companion.UTILITY_TOKEN
 import io.provenance.explorer.config.ResourceNotFoundException
 import io.provenance.explorer.domain.core.logger
 import io.provenance.explorer.domain.entities.AddressImageRecord
-import io.provenance.explorer.domain.entities.BlockProposerRecord
 import io.provenance.explorer.domain.entities.MissedBlocksRecord
 import io.provenance.explorer.domain.entities.SpotlightCacheRecord
 import io.provenance.explorer.domain.entities.StakingValidatorCacheRecord
@@ -46,7 +45,6 @@ import io.provenance.explorer.domain.models.explorer.hourlyBlockCount
 import io.provenance.explorer.domain.models.explorer.zeroOutValidatorObj
 import io.provenance.explorer.grpc.v1.AttributeGrpcClient
 import io.provenance.explorer.grpc.v1.ValidatorGrpcClient
-import io.provenance.explorer.model.BlockLatencyData
 import io.provenance.explorer.model.CommissionList
 import io.provenance.explorer.model.CommissionRate
 import io.provenance.explorer.model.Delegation
@@ -544,15 +542,6 @@ class ValidatorService(
         }
     }
 
-    fun getBlockLatencyData(address: String, blockCount: Int) =
-        getValidatorOperatorAddress(address)?.let { addr ->
-            BlockProposerRecord.getRecordsForProposer(addr.operatorAddress, blockCount).let { res ->
-                val average = res.map { it.blockLatency!! }.average()
-                val data = res.associate { it.blockHeight to it.blockLatency!! }
-                BlockLatencyData(addr.operatorAddress, data, average)
-            }
-        } ?: throw ResourceNotFoundException("Invalid validator address: '$address'")
-
     private fun getLatestHeight() = SpotlightCacheRecord.getSpotlight()?.latestBlock?.height ?: blockService.getMaxBlockCacheHeight()
 
     fun getDistinctValidatorsWithMissedBlocksInTimeframe(timeframe: Timeframe) = transaction {

service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt

@@ -10,7 +10,6 @@ import io.provenance.explorer.config.ExplorerProperties.Companion.UTILITY_TOKEN_
 import io.provenance.explorer.config.ExplorerProperties.Companion.UTILITY_TOKEN_BASE_MULTIPLIER
 import io.provenance.explorer.domain.core.logger
 import io.provenance.explorer.domain.entities.BlockCacheRecord
-import io.provenance.explorer.domain.entities.BlockProposerRecord
 import io.provenance.explorer.domain.entities.BlockTxCountsCacheRecord
 import io.provenance.explorer.domain.entities.BlockTxRetryRecord
 import io.provenance.explorer.domain.entities.CacheKeys
@@ -152,7 +151,6 @@ class ScheduledTaskService(
         }
 
         BlockTxCountsCacheRecord.updateTxCounts()
-        BlockProposerRecord.calcLatency()
         if (!cacheService.getCacheValue(CacheKeys.SPOTLIGHT_PROCESSING.key)!!.cacheValue.toBoolean()) {
             cacheService.updateCacheValue(CacheKeys.SPOTLIGHT_PROCESSING.key, true.toString())
         }

service/src/main/kotlin/io/provenance/explorer/web/v2/ValidatorControllerV2.kt

@@ -165,17 +165,6 @@ class ValidatorControllerV2(
         dayCount: Int
     ) = validatorService.getValidatorMarketRateStats(address, fromDate, toDate, dayCount)
 
-    @ApiOperation("Returns block latency data for the validator")
-    @GetMapping("/{address}/latency")
-    fun blockLatency(
-        @ApiParam(value = "The Validator's operator, owning account, or consensus address")
-        @PathVariable
-        address: String,
-        @ApiParam(value = "The number of blocks of data returned", defaultValue = "100", required = false)
-        @RequestParam(defaultValue = "100")
-        blockCount: Int
-    ) = validatorService.getBlockLatencyData(address, blockCount)
-
     @ApiOperation("Returns distinct validators with missed blocks for the timeframe")
     @GetMapping("/missed_blocks/distinct")
     fun missedBlocksDistinct(