From 57ad2c9cefd83726c3abc79d7fedaabb84be12c4 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Fri, 6 Sep 2024 12:55:41 -0600 Subject: [PATCH 1/6] add logging for saving txs and blocks --- .../service/async/BlockAndTxProcessor.kt | 129 ++++++++++++++---- 1 file changed, 101 insertions(+), 28 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt index f31ff861..e3132fde 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt @@ -157,41 +157,66 @@ class BlockAndTxProcessor( fun saveBlockEtc( blockRes: Query.GetBlockByHeightResponse?, - rerunTxs: Pair = Pair(false, false) // rerun txs, pull from db + rerunTxs: Pair = Pair(false, false) ): Query.GetBlockByHeightResponse? { if (blockRes == null) return null - logger.info("saving block ${blockRes.block.height()}") + + val startTime = System.currentTimeMillis() + val blockHeight = blockRes.block.height() + logger.info("Block saving started. blockHeight=$blockHeight") + val blockTimestamp = blockRes.block.header.time.toDateTime() - val block = - BlockCacheRecord.buildInsert( - blockRes.block.height(), - blockRes.block.data.txsCount, - blockTimestamp, - blockRes - ) - val proposerRec = validatorService.buildProposerInsert(blockRes, blockTimestamp, blockRes.block.height()) - val valsAtHeight = validatorService.buildValidatorsAtHeight(blockRes.block.height()) + val block = BlockCacheRecord.buildInsert( + blockHeight, + blockRes.block.data.txsCount, + blockTimestamp, + blockRes + ) + + var stepTime = startTime + + val proposerRec = validatorService.buildProposerInsert(blockRes, blockTimestamp, blockHeight) + logTimeElapsed("buildProposerInsert", stepTime, blockHeight) + stepTime = System.currentTimeMillis() + + val valsAtHeight = validatorService.buildValidatorsAtHeight(blockHeight) + logTimeElapsed("buildValidatorsAtHeight", stepTime, blockHeight) + stepTime = System.currentTimeMillis() + validatorService.saveMissedBlocks(blockRes) - val txs = - if (blockRes.block.data.txsCount > 0) { - saveTxs( - blockRes, - proposerRec, - rerunTxs - ).map { it.toProcedureObject() } - } else { - listOf() - } + logTimeElapsed("saveMissedBlocks", stepTime, blockHeight) + stepTime = System.currentTimeMillis() + + val txs = if (blockRes.block.data.txsCount > 0) { + saveTxs(blockRes, proposerRec, rerunTxs).map { it.toProcedureObject() } + } else { + listOf() + } + logTimeElapsed("saveTxs", stepTime, blockHeight) + stepTime = System.currentTimeMillis() + val blockUpdate = BlockUpdate(block, proposerRec.buildInsert(), valsAtHeight, txs) + try { BlockCacheRecord.insertToProcedure(blockUpdate) } catch (e: Exception) { - logger.error("Failed to save block: ${blockRes.block.height()}", e) - BlockTxRetryRecord.insertOrUpdate(blockRes.block.height(), e) + logger.error("Failed to save block: $blockHeight", e) + BlockTxRetryRecord.insertOrUpdate(blockHeight, e) } + logTimeElapsed("insertToProcedure", stepTime, blockHeight) + + val totalTime = System.currentTimeMillis() - startTime + logger.info("Block saving completed. blockHeight=$blockHeight, totalDuration=${totalTime}ms") + return blockRes } + fun logTimeElapsed(stepName: String, startTime: Long, blockHeight: Int) { + val currentTime = System.currentTimeMillis() + val elapsed = currentTime - startTime + logger.info("$stepName completed. blockHeight=$blockHeight, duration=${elapsed}ms") + } + data class TxUpdatedItems( val addresses: Map>, val markers: List, @@ -267,42 +292,90 @@ class BlockAndTxProcessor( listOf() } + + // TODO: See: https://github.com/provenance-io/explorer-service/issues/538 fun processAndSaveTransactionData( res: ServiceOuterClass.GetTxResponse, blockTime: DateTime, proposerRec: BlockProposer ): TxUpdatedItems { + val startTime = System.currentTimeMillis() + val txHash = res.txResponse.txhash + val blockHeight = proposerRec.blockHeight + logger.info("Transaction processing started. blockHeight=$blockHeight, txHash=$txHash") + val tx = TxCacheRecord.buildInsert(res, blockTime) val txUpdate = TxUpdate(tx) - val txInfo = TxData(proposerRec.blockHeight, null, res.txResponse.txhash, blockTime) + val txInfo = TxData(blockHeight, null, txHash, blockTime) + + var stepTime = startTime - // TODO: See: https://github.com/provenance-io/explorer-service/issues/538 saveMessages(txInfo, res, txUpdate) + logTimeElapsed("saveMessages", stepTime, blockHeight, txHash) + stepTime = System.currentTimeMillis() + saveTxFees(res, txInfo, txUpdate, proposerRec) + logTimeElapsed("saveTxFees", stepTime, blockHeight, txHash) + stepTime = System.currentTimeMillis() + val addrs = saveAddresses(txInfo, res, txUpdate) + logTimeElapsed("saveAddresses", stepTime, blockHeight, txHash) + stepTime = System.currentTimeMillis() + val markers = saveMarkers(txInfo, res, txUpdate) + logTimeElapsed("saveMarkers", stepTime, blockHeight, txHash) + stepTime = System.currentTimeMillis() + saveNftData(txInfo, res, txUpdate) + logTimeElapsed("saveNftData", stepTime, blockHeight, txHash) + stepTime = System.currentTimeMillis() + saveGovData(res, txInfo, txUpdate) + logTimeElapsed("saveGovData", stepTime, blockHeight, txHash) + stepTime = System.currentTimeMillis() + try { saveIbcChannelData(res, txInfo, txUpdate) } catch (e: Exception) { - logger.error("Failed to process IBC channel data for tx ${txInfo.txHash} at height ${txInfo.blockHeight}. Error: ${e.message}") + logger.error("Failed to process IBC channel data for tx $txHash at height $blockHeight. Error: ${e.message}") TxProcessingFailureRecord.insertOrUpdate( - txInfo.blockHeight, - txInfo.txHash, + blockHeight, + txHash, "ibc_channel_data", e.stackTraceToString(), false ) } + logTimeElapsed("saveIbcChannelData", stepTime, blockHeight, txHash) + stepTime = System.currentTimeMillis() + saveSmartContractData(res, txInfo, txUpdate) + logTimeElapsed("saveSmartContractData", stepTime, blockHeight, txHash) + stepTime = System.currentTimeMillis() + saveNameData(res, txInfo) + logTimeElapsed("saveNameData", stepTime, blockHeight, txHash) + stepTime = System.currentTimeMillis() + groupService.saveGroups(res, txInfo, txUpdate) + logTimeElapsed("saveGroups", stepTime, blockHeight, txHash) + stepTime = System.currentTimeMillis() + saveSignaturesTx(res, txInfo, txUpdate) + logTimeElapsed("saveSignaturesTx", stepTime, blockHeight, txHash) + + val totalTime = System.currentTimeMillis() - startTime + logger.info("Transaction processing completed. blockHeight=$blockHeight, txHash=$txHash, totalDuration=${totalTime}ms") return TxUpdatedItems(addrs, markers, txUpdate) } + fun logTimeElapsed(method: String, startTime: Long, blockHeight: Int, txHash: String) { + val currentTime = System.currentTimeMillis() + val elapsed = currentTime - startTime + logger.info("$method completed. blockHeight=$blockHeight, txHash=$txHash, duration=${elapsed}ms") + } + private fun saveTxFees( tx: ServiceOuterClass.GetTxResponse, txInfo: TxData, From 0802ff7fec98ebe610e2886a9c9160e847ab0cc5 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Fri, 6 Sep 2024 13:54:09 -0600 Subject: [PATCH 2/6] only log tx processing if happens --- .../explorer/service/async/BlockAndTxProcessor.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt index e3132fde..ec9edc31 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt @@ -188,12 +188,13 @@ class BlockAndTxProcessor( stepTime = System.currentTimeMillis() val txs = if (blockRes.block.data.txsCount > 0) { - saveTxs(blockRes, proposerRec, rerunTxs).map { it.toProcedureObject() } + val savedTxs = saveTxs(blockRes, proposerRec, rerunTxs).map { it.toProcedureObject() } + logTimeElapsed("saveTxs", stepTime, blockHeight) + stepTime = System.currentTimeMillis() + savedTxs } else { listOf() } - logTimeElapsed("saveTxs", stepTime, blockHeight) - stepTime = System.currentTimeMillis() val blockUpdate = BlockUpdate(block, proposerRec.buildInsert(), valsAtHeight, txs) From f4cb5a2bf22eb147eea963068976d22ce3db273b Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Fri, 6 Sep 2024 13:55:50 -0600 Subject: [PATCH 3/6] add tx count --- .../io/provenance/explorer/service/async/BlockAndTxProcessor.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt index ec9edc31..cada1cda 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt @@ -189,7 +189,7 @@ class BlockAndTxProcessor( val txs = if (blockRes.block.data.txsCount > 0) { val savedTxs = saveTxs(blockRes, proposerRec, rerunTxs).map { it.toProcedureObject() } - logTimeElapsed("saveTxs", stepTime, blockHeight) + logTimeElapsed("saveTxs count=${savedTxs.size}", stepTime, blockHeight) stepTime = System.currentTimeMillis() savedTxs } else { From a6d4ee5f4e4e5766606c8cd7c8c3abc390afb7c4 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Fri, 6 Sep 2024 14:45:07 -0600 Subject: [PATCH 4/6] rename application to ExplrerServiceApplication --- .../explorer/{Application.kt => ExplorerServiceApplication.kt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename service/src/main/kotlin/io/provenance/explorer/{Application.kt => ExplorerServiceApplication.kt} (100%) diff --git a/service/src/main/kotlin/io/provenance/explorer/Application.kt b/service/src/main/kotlin/io/provenance/explorer/ExplorerServiceApplication.kt similarity index 100% rename from service/src/main/kotlin/io/provenance/explorer/Application.kt rename to service/src/main/kotlin/io/provenance/explorer/ExplorerServiceApplication.kt From cf6c2cb8ed03f7de98cbfde2917d4bcf4c6a00f8 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Fri, 6 Sep 2024 14:50:42 -0600 Subject: [PATCH 5/6] fix lint --- .../io/provenance/explorer/service/async/BlockAndTxProcessor.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt index cada1cda..d7519399 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt @@ -293,7 +293,6 @@ class BlockAndTxProcessor( listOf() } - // TODO: See: https://github.com/provenance-io/explorer-service/issues/538 fun processAndSaveTransactionData( res: ServiceOuterClass.GetTxResponse, From 8c3762b2c139996964925474cd1edfb5523d4719 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Fri, 6 Sep 2024 15:14:13 -0600 Subject: [PATCH 6/6] update the main class path in gradle file --- service/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/build.gradle.kts b/service/build.gradle.kts index e53e1926..31f5608d 100644 --- a/service/build.gradle.kts +++ b/service/build.gradle.kts @@ -90,7 +90,7 @@ tasks.getByName("bootRun") { args = mutableListOf("--spring.profiles.active=$profiles") } -springBoot.mainClass.set("io.provenance.explorer.ApplicationKt") +springBoot.mainClass.set("io.provenance.explorer.ExplorerServiceApplicationKt") println("\nExclude Spring Boot Dev tools? " + version.toString().contains("main")) tasks.getByName("bootJar") {