From 3ed68ca1bd27e795f4554b00829c74d37933c6b4 Mon Sep 17 00:00:00 2001 From: Carlton Hanna Date: Fri, 23 Aug 2024 10:27:45 -0600 Subject: [PATCH 01/16] rename classes for clarification of their function --- .../async/{AsyncCachingV2.kt => BlockAndTxProcessor.kt} | 0 .../async/{AsyncService.kt => ScheduledTaskService.kt} | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename service/src/main/kotlin/io/provenance/explorer/service/async/{AsyncCachingV2.kt => BlockAndTxProcessor.kt} (100%) rename service/src/main/kotlin/io/provenance/explorer/service/async/{AsyncService.kt => ScheduledTaskService.kt} (99%) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncCachingV2.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt similarity index 100% rename from service/src/main/kotlin/io/provenance/explorer/service/async/AsyncCachingV2.kt rename to service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt similarity index 99% rename from service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt rename to service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt index cfb6c18e..eb397b75 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt @@ -88,7 +88,7 @@ import java.time.ZoneOffset import javax.annotation.PostConstruct @Service -class AsyncService( +class ScheduledTaskService( private val props: ExplorerProperties, private val blockService: BlockService, private val assetService: AssetService, @@ -103,7 +103,7 @@ class AsyncService( private val metricsService: MetricsService ) { - protected val logger = logger(AsyncService::class) + protected val logger = logger(ScheduledTaskService::class) protected var collectHistorical = true @PostConstruct From 586152179a6531b5d0dec232a4186e454c94862a Mon Sep 17 00:00:00 2001 From: Carlton Hanna Date: Fri, 23 Aug 2024 10:43:58 -0600 Subject: [PATCH 02/16] remove unneeded pass through function --- .../explorer/service/async/BlockAndTxProcessor.kt | 15 +++++---------- .../explorer/service/utility/UtilityService.kt | 2 +- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt index b632670c..2f98f509 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt @@ -254,11 +254,11 @@ class AsyncCachingV2( if (pullFromDb) { transaction { TxCacheRecord.findByHeight(blockHeight) - .map { addTxToCacheWithTimestamp(it.txV2, blockTime, proposerRec) } + .map { processAndSaveTransactionData(it.txV2, blockTime.toDateTime(), proposerRec) } } } else { runBlocking { txClient.getTxsByHeight(blockHeight, txCount) } - .map { addTxToCacheWithTimestamp(it, blockTime, proposerRec) } + .map { processAndSaveTransactionData(it, blockTime.toDateTime(), proposerRec) } } } catch (e: Exception) { logger.error("Failed to retrieve transactions at block: $blockHeight error: ${e.message}", e) @@ -266,15 +266,8 @@ class AsyncCachingV2( listOf() } - fun addTxToCacheWithTimestamp( - res: ServiceOuterClass.GetTxResponse, - blockTime: Timestamp, - proposerRec: BlockProposer - ) = - addTxToCache(res, blockTime.toDateTime(), proposerRec) - // Function that saves all the things under a transaction - fun addTxToCache( + fun processAndSaveTransactionData( res: ServiceOuterClass.GetTxResponse, blockTime: DateTime, proposerRec: BlockProposer @@ -282,6 +275,7 @@ class AsyncCachingV2( val tx = TxCacheRecord.buildInsert(res, blockTime) val txUpdate = TxUpdate(tx) val txInfo = TxData(proposerRec.blockHeight, null, res.txResponse.txhash, blockTime) + saveMessages(txInfo, res, txUpdate) saveTxFees(res, txInfo, txUpdate, proposerRec) val addrs = saveAddresses(txInfo, res, txUpdate) @@ -293,6 +287,7 @@ class AsyncCachingV2( saveNameData(res, txInfo) groupService.saveGroups(res, txInfo, txUpdate) saveSignaturesTx(res, txInfo, txUpdate) + return TxUpdatedItems(addrs, markers, txUpdate) } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/utility/UtilityService.kt b/service/src/main/kotlin/io/provenance/explorer/service/utility/UtilityService.kt index e4323bf9..f9d8a58b 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/utility/UtilityService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/utility/UtilityService.kt @@ -96,7 +96,7 @@ class UtilityService( fun parseRawTxJson(rawJson: String, blockHeight: Int = 1, timestamp: DateTime = DateTime.now()) = transaction { val builder = ServiceOuterClass.GetTxResponse.newBuilder() protoParser.ignoringUnknownFields().merge(rawJson, builder) - async.addTxToCache(builder.build(), DateTime.now(), BlockProposer(blockHeight, "", timestamp)) + async.processAndSaveTransactionData(builder.build(), DateTime.now(), BlockProposer(blockHeight, "", timestamp)) } fun saveRawTxJson(rawJson: String, blockHeight: Int = 1, timestamp: DateTime = DateTime.now()) = transaction { From 19e9bcc32d0a7b257158fd76131dac4ae06f2b4d Mon Sep 17 00:00:00 2001 From: Carlton Hanna Date: Fri, 23 Aug 2024 11:20:09 -0600 Subject: [PATCH 03/16] add new database create script, entity, and insertOrUpdate, add TODO for issue 538 --- ...1__Create_tx_processing_failures_table.sql | 27 ++++++++ .../explorer/domain/entities/Blocks.kt | 67 +++++++++++++++++++ .../service/async/BlockAndTxProcessor.kt | 50 ++++---------- 3 files changed, 108 insertions(+), 36 deletions(-) create mode 100644 database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql diff --git a/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql b/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql new file mode 100644 index 00000000..e9511adc --- /dev/null +++ b/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql @@ -0,0 +1,27 @@ +SELECT 'Create tx_processing_failures table' AS comment; +CREATE TABLE IF NOT EXISTS tx_processing_failures +( + block_height + INT + NOT + NULL, + tx_hash + VARCHAR +( + 128 +) NOT NULL, + process_type VARCHAR +( + 64 +) NOT NULL, + failure_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + error_message TEXT DEFAULT NULL, + retried BOOLEAN NOT NULL DEFAULT FALSE, + success BOOLEAN NOT NULL DEFAULT FALSE, + PRIMARY KEY +( + block_height, + tx_hash, + process_type +) + ); \ No newline at end of file diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt index e51ecf02..1f0238e1 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt @@ -582,3 +582,70 @@ class BlockTxRetryRecord(id: EntityID) : IntEntity(id) { var success by BlockTxRetryTable.success var errorBlock by BlockTxRetryTable.errorBlock } + +object TxProcessingFailuresTable : IdTable(name = "tx_processing_failures") { + val blockHeight = integer("block_height") + val txHash = varchar("tx_hash", 128) + val processType = varchar("process_type", 64) + val failureTime = datetime("failure_time") + val errorMessage = text("error_message").nullable() + val retried = bool("retried").default(false) + val success = bool("success").default(false) + + override val id = integer("id").entityId() + + init { + index(true, blockHeight, txHash, processType) + } +} + +class TxProcessingFailureRecord(id: EntityID) : IntEntity(id) { + companion object : IntEntityClass(TxProcessingFailuresTable) { + + fun insertOrUpdate( + blockHeight: Int, + txHash: String, + processType: String, + errorMessage: String?, + success: Boolean + ) = transaction { + val existingRecord = TxProcessingFailureRecord.find { + (TxProcessingFailuresTable.blockHeight eq blockHeight) and + (TxProcessingFailuresTable.txHash eq txHash) and + (TxProcessingFailuresTable.processType eq processType) + }.firstOrNull() + + if (existingRecord == null) { + TxProcessingFailuresTable.insertIgnore { + it[this.blockHeight] = blockHeight + it[this.txHash] = txHash + it[this.processType] = processType + it[this.errorMessage] = errorMessage + it[this.success] = success + } + } else { + existingRecord.apply { + this.errorMessage = errorMessage + this.success = success + this.retried = true + this.failureTime = DateTime.now() + } + } + } + + fun deleteProcessedRecords() = transaction { + TxProcessingFailuresTable.deleteWhere { + (TxProcessingFailuresTable.retried eq true) and + (TxProcessingFailuresTable.success eq true) + } + } + } + + var blockHeight by TxProcessingFailuresTable.blockHeight + var txHash by TxProcessingFailuresTable.txHash + var processType by TxProcessingFailuresTable.processType + var failureTime by TxProcessingFailuresTable.failureTime + var errorMessage by TxProcessingFailuresTable.errorMessage + var retried by TxProcessingFailuresTable.retried + var success by TxProcessingFailuresTable.success +} \ No newline at end of file diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt index 2f98f509..580c718a 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt @@ -8,40 +8,7 @@ import io.provenance.explorer.config.ExplorerProperties import io.provenance.explorer.domain.core.logger import io.provenance.explorer.domain.core.sql.toArray import io.provenance.explorer.domain.core.sql.toObject -import io.provenance.explorer.domain.entities.AccountRecord -import io.provenance.explorer.domain.entities.BlockCacheRecord -import io.provenance.explorer.domain.entities.BlockTxRetryRecord -import io.provenance.explorer.domain.entities.FeePayer -import io.provenance.explorer.domain.entities.IbcAckType -import io.provenance.explorer.domain.entities.IbcLedgerRecord -import io.provenance.explorer.domain.entities.IbcRelayerRecord -import io.provenance.explorer.domain.entities.NameRecord -import io.provenance.explorer.domain.entities.ProcessQueueRecord -import io.provenance.explorer.domain.entities.ProcessQueueType -import io.provenance.explorer.domain.entities.SignatureRecord -import io.provenance.explorer.domain.entities.SignatureTxRecord -import io.provenance.explorer.domain.entities.TxAddressJoinRecord -import io.provenance.explorer.domain.entities.TxAddressJoinType -import io.provenance.explorer.domain.entities.TxCacheRecord -import io.provenance.explorer.domain.entities.TxEventAttrRecord -import io.provenance.explorer.domain.entities.TxEventAttrTable -import io.provenance.explorer.domain.entities.TxEventRecord -import io.provenance.explorer.domain.entities.TxFeeRecord -import io.provenance.explorer.domain.entities.TxFeepayerRecord -import io.provenance.explorer.domain.entities.TxGasCacheRecord -import io.provenance.explorer.domain.entities.TxIbcRecord -import io.provenance.explorer.domain.entities.TxMarkerJoinRecord -import io.provenance.explorer.domain.entities.TxMessageRecord -import io.provenance.explorer.domain.entities.TxMsgTypeSubtypeRecord -import io.provenance.explorer.domain.entities.TxMsgTypeSubtypeTable -import io.provenance.explorer.domain.entities.TxNftJoinRecord -import io.provenance.explorer.domain.entities.TxSingleMessageCacheRecord -import io.provenance.explorer.domain.entities.TxSmCodeRecord -import io.provenance.explorer.domain.entities.TxSmContractRecord -import io.provenance.explorer.domain.entities.ValidatorMarketRateRecord -import io.provenance.explorer.domain.entities.ValidatorStateRecord -import io.provenance.explorer.domain.entities.buildInsert -import io.provenance.explorer.domain.entities.updateHitCount +import io.provenance.explorer.domain.entities.* import io.provenance.explorer.domain.exceptions.InvalidArgumentException import io.provenance.explorer.domain.extensions.TX_ACC_SEQ import io.provenance.explorer.domain.extensions.TX_EVENT @@ -266,7 +233,6 @@ class AsyncCachingV2( listOf() } - // Function that saves all the things under a transaction fun processAndSaveTransactionData( res: ServiceOuterClass.GetTxResponse, blockTime: DateTime, @@ -276,13 +242,25 @@ class AsyncCachingV2( val txUpdate = TxUpdate(tx) val txInfo = TxData(proposerRec.blockHeight, null, res.txResponse.txhash, blockTime) + // TODO: See: https://github.com/provenance-io/explorer-service/issues/538 saveMessages(txInfo, res, txUpdate) saveTxFees(res, txInfo, txUpdate, proposerRec) val addrs = saveAddresses(txInfo, res, txUpdate) val markers = saveMarkers(txInfo, res, txUpdate) saveNftData(txInfo, res, txUpdate) saveGovData(res, txInfo, txUpdate) - saveIbcChannelData(res, txInfo, txUpdate) + try { + saveIbcChannelData(res, txInfo, txUpdate) + } catch (e: Exception) { + logger.error("Failed to process IBC channel data for tx ${txInfo.txHash} at height ${txInfo.blockHeight}. Error: ${e.message}") + TxProcessingFailureRecord.insertOrUpdate( + txInfo.blockHeight, + txInfo.txHash, + "ibc_channel_data", + e.stackTraceToString(), + false + ) + } saveSmartContractData(res, txInfo, txUpdate) saveNameData(res, txInfo) groupService.saveGroups(res, txInfo, txUpdate) From 758a457f7c132771327529827e85d282b5ffe254 Mon Sep 17 00:00:00 2001 From: Carlton Hanna Date: Fri, 23 Aug 2024 11:25:25 -0600 Subject: [PATCH 04/16] fix lint imports --- .../service/async/BlockAndTxProcessor.kt | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt index 580c718a..ccfac358 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt @@ -8,7 +8,41 @@ import io.provenance.explorer.config.ExplorerProperties import io.provenance.explorer.domain.core.logger import io.provenance.explorer.domain.core.sql.toArray import io.provenance.explorer.domain.core.sql.toObject -import io.provenance.explorer.domain.entities.* +import io.provenance.explorer.domain.entities.AccountRecord +import io.provenance.explorer.domain.entities.BlockCacheRecord +import io.provenance.explorer.domain.entities.BlockTxRetryRecord +import io.provenance.explorer.domain.entities.FeePayer +import io.provenance.explorer.domain.entities.IbcAckType +import io.provenance.explorer.domain.entities.IbcLedgerRecord +import io.provenance.explorer.domain.entities.IbcRelayerRecord +import io.provenance.explorer.domain.entities.NameRecord +import io.provenance.explorer.domain.entities.ProcessQueueRecord +import io.provenance.explorer.domain.entities.ProcessQueueType +import io.provenance.explorer.domain.entities.SignatureRecord +import io.provenance.explorer.domain.entities.SignatureTxRecord +import io.provenance.explorer.domain.entities.TxAddressJoinRecord +import io.provenance.explorer.domain.entities.TxAddressJoinType +import io.provenance.explorer.domain.entities.TxCacheRecord +import io.provenance.explorer.domain.entities.TxEventAttrRecord +import io.provenance.explorer.domain.entities.TxEventAttrTable +import io.provenance.explorer.domain.entities.TxEventRecord +import io.provenance.explorer.domain.entities.TxFeeRecord +import io.provenance.explorer.domain.entities.TxFeepayerRecord +import io.provenance.explorer.domain.entities.TxGasCacheRecord +import io.provenance.explorer.domain.entities.TxIbcRecord +import io.provenance.explorer.domain.entities.TxMarkerJoinRecord +import io.provenance.explorer.domain.entities.TxMessageRecord +import io.provenance.explorer.domain.entities.TxMsgTypeSubtypeRecord +import io.provenance.explorer.domain.entities.TxMsgTypeSubtypeTable +import io.provenance.explorer.domain.entities.TxNftJoinRecord +import io.provenance.explorer.domain.entities.TxProcessingFailureRecord +import io.provenance.explorer.domain.entities.TxSingleMessageCacheRecord +import io.provenance.explorer.domain.entities.TxSmCodeRecord +import io.provenance.explorer.domain.entities.TxSmContractRecord +import io.provenance.explorer.domain.entities.ValidatorMarketRateRecord +import io.provenance.explorer.domain.entities.ValidatorStateRecord +import io.provenance.explorer.domain.entities.buildInsert +import io.provenance.explorer.domain.entities.updateHitCount import io.provenance.explorer.domain.exceptions.InvalidArgumentException import io.provenance.explorer.domain.extensions.TX_ACC_SEQ import io.provenance.explorer.domain.extensions.TX_EVENT From 3ba3eef69eb519d464507c7e04eede63ba36ca5b Mon Sep 17 00:00:00 2001 From: Carlton Hanna Date: Fri, 23 Aug 2024 11:34:54 -0600 Subject: [PATCH 05/16] fix lint --- .../explorer/domain/entities/Blocks.kt | 58 ++++++++++--------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt index 1f0238e1..1d553a3f 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt @@ -213,7 +213,7 @@ class BlockProposerRecord(id: EntityID) : IntEntity(id) { fun getRecordsForProposer(address: String, limit: Int) = transaction { BlockProposerRecord.find { (BlockProposerTable.proposerOperatorAddress eq address) and - (BlockProposerTable.blockLatency.isNotNull()) + (BlockProposerTable.blockLatency.isNotNull()) }.orderBy(Pair(BlockProposerTable.blockHeight, SortOrder.DESC)) .limit(limit) .toList() @@ -353,13 +353,14 @@ class BlockCacheHourlyTxCountsRecord(id: EntityID) : Entity( BlockCacheHourlyTxCountsTable.slice(txSum).selectAll().map { it[txSum] }.first()!! } - fun getTxCountsForParams(fromDate: DateTime, toDate: DateTime, granularity: DateTruncGranularity) = transaction { - when (granularity) { - DAY, MONTH -> getGranularityCounts(fromDate, toDate, granularity) - HOUR -> getHourlyCounts(fromDate, toDate) - MINUTE -> emptyList() + fun getTxCountsForParams(fromDate: DateTime, toDate: DateTime, granularity: DateTruncGranularity) = + transaction { + when (granularity) { + DAY, MONTH -> getGranularityCounts(fromDate, toDate, granularity) + HOUR -> getHourlyCounts(fromDate, toDate) + MINUTE -> emptyList() + } } - } fun getTxHeatmap(fromDate: DateTime? = null, toDate: DateTime? = null) = transaction { val blockTimestamp = BlockCacheHourlyTxCountsTable.blockTimestamp @@ -399,22 +400,23 @@ class BlockCacheHourlyTxCountsRecord(id: EntityID) : Entity( TxHeatmapRes(result, dayTotals, hourTotals) } - private fun getGranularityCounts(fromDate: DateTime, toDate: DateTime, granularity: DateTruncGranularity) = transaction { - val dateTrunc = DateTrunc(granularity.name, BlockCacheHourlyTxCountsTable.blockTimestamp) - val txSum = BlockCacheHourlyTxCountsTable.txCount.sum() - BlockCacheHourlyTxCountsTable.slice(dateTrunc, txSum) - .select { - dateTrunc.between(fromDate.startOfDay(), toDate.startOfDay()) - } - .groupBy(dateTrunc) - .orderBy(dateTrunc, SortOrder.DESC) - .map { - TxHistory( - it[dateTrunc]!!.withZone(DateTimeZone.UTC).toString("yyyy-MM-dd HH:mm:ss"), - it[txSum]!! - ) - } - } + private fun getGranularityCounts(fromDate: DateTime, toDate: DateTime, granularity: DateTruncGranularity) = + transaction { + val dateTrunc = DateTrunc(granularity.name, BlockCacheHourlyTxCountsTable.blockTimestamp) + val txSum = BlockCacheHourlyTxCountsTable.txCount.sum() + BlockCacheHourlyTxCountsTable.slice(dateTrunc, txSum) + .select { + dateTrunc.between(fromDate.startOfDay(), toDate.startOfDay()) + } + .groupBy(dateTrunc) + .orderBy(dateTrunc, SortOrder.DESC) + .map { + TxHistory( + it[dateTrunc]!!.withZone(DateTimeZone.UTC).toString("yyyy-MM-dd HH:mm:ss"), + it[txSum]!! + ) + } + } private fun getHourlyCounts(fromDate: DateTime, toDate: DateTime) = transaction { BlockCacheHourlyTxCountsRecord.find { @@ -536,7 +538,7 @@ class BlockTxRetryRecord(id: EntityID) : IntEntity(id) { it[this.success] = false it[this.errorBlock] = "NON BLOCKING ERROR: Logged to know what happened, but didnt stop processing.\n " + - e.stackTraceToString() + e.stackTraceToString() } } @@ -545,7 +547,7 @@ class BlockTxRetryRecord(id: EntityID) : IntEntity(id) { it[this.height] = height it[this.errorBlock] = "NON BLOCKING ERROR: Logged to know what happened, but didnt stop processing.\n " + - e.stackTraceToString() + e.stackTraceToString() } } @@ -571,8 +573,8 @@ class BlockTxRetryRecord(id: EntityID) : IntEntity(id) { BlockTxRetryTable .deleteWhere { (BlockTxRetryTable.retried eq true) and - (BlockTxRetryTable.success eq true) and - (BlockTxRetryTable.height inList heights) + (BlockTxRetryTable.success eq true) and + (BlockTxRetryTable.height inList heights) } } } @@ -648,4 +650,4 @@ class TxProcessingFailureRecord(id: EntityID) : IntEntity(id) { var errorMessage by TxProcessingFailuresTable.errorMessage var retried by TxProcessingFailuresTable.retried var success by TxProcessingFailuresTable.success -} \ No newline at end of file +} From bb9a31fa97c350ae57c8c85d81b6d4d9f6e810a3 Mon Sep 17 00:00:00 2001 From: Carlton Hanna Date: Fri, 23 Aug 2024 11:46:05 -0600 Subject: [PATCH 06/16] format sql --- ...1__Create_tx_processing_failures_table.sql | 29 +++++-------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql b/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql index e9511adc..2adc25c1 100644 --- a/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql +++ b/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql @@ -1,27 +1,12 @@ SELECT 'Create tx_processing_failures table' AS comment; -CREATE TABLE IF NOT EXISTS tx_processing_failures -( - block_height - INT - NOT - NULL, - tx_hash - VARCHAR -( - 128 -) NOT NULL, - process_type VARCHAR -( - 64 -) NOT NULL, + +CREATE TABLE IF NOT EXISTS tx_processing_failures ( + block_height INT NOT NULL, + tx_hash VARCHAR(128) NOT NULL, + process_type VARCHAR(64) NOT NULL, failure_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, error_message TEXT DEFAULT NULL, retried BOOLEAN NOT NULL DEFAULT FALSE, success BOOLEAN NOT NULL DEFAULT FALSE, - PRIMARY KEY -( - block_height, - tx_hash, - process_type -) - ); \ No newline at end of file + PRIMARY KEY (block_height, tx_hash, process_type) +); From e66d1e9a8baf76c00326dcd37beed21522f8f19c Mon Sep 17 00:00:00 2001 From: Carlton Hanna Date: Mon, 26 Aug 2024 07:16:30 -0600 Subject: [PATCH 07/16] update entitiy, start test --- ...1__Create_tx_processing_failures_table.sql | 3 +- .../entities/TxProcessingFailuresTableTest.kt | 61 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt diff --git a/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql b/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql index 2adc25c1..dfc92a02 100644 --- a/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql +++ b/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql @@ -1,6 +1,7 @@ SELECT 'Create tx_processing_failures table' AS comment; CREATE TABLE IF NOT EXISTS tx_processing_failures ( + id SERIAL PRIMARY KEY, block_height INT NOT NULL, tx_hash VARCHAR(128) NOT NULL, process_type VARCHAR(64) NOT NULL, @@ -8,5 +9,5 @@ CREATE TABLE IF NOT EXISTS tx_processing_failures ( error_message TEXT DEFAULT NULL, retried BOOLEAN NOT NULL DEFAULT FALSE, success BOOLEAN NOT NULL DEFAULT FALSE, - PRIMARY KEY (block_height, tx_hash, process_type) + UNIQUE (block_height, tx_hash, process_type) ); diff --git a/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt b/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt new file mode 100644 index 00000000..fb3731a1 --- /dev/null +++ b/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt @@ -0,0 +1,61 @@ +import io.provenance.explorer.domain.entities.TxProcessingFailureRecord +import io.provenance.explorer.domain.entities.TxProcessingFailuresTable +import org.jetbrains.exposed.sql.transactions.transaction +import org.jetbrains.exposed.sql.and +import org.jetbrains.exposed.sql.Database +import org.junit.jupiter.api.Test + +class TxProcessingFailureRecordTest { + + @Test + fun testInsertOrUpdate_newRecord() { + Database.connect("jdbc:h2:mem:test;MODE=PostgreSQL;DB_CLOSE_DELAY=-1;", driver = "org.h2.Driver") + transaction { + val sql = this::class.java.getResource("/db/migration/V1_91__Create_tx_processing_failures_table.sql")!!.readText() + exec(sql) + } + transaction { + TxProcessingFailureRecord.insertOrUpdate( + blockHeight = 100, + txHash = "testHash", + processType = "testProcess", + errorMessage = "testError", + success = false + ) + + val record = TxProcessingFailureRecord.find { + (TxProcessingFailuresTable.blockHeight eq 100) and + (TxProcessingFailuresTable.txHash eq "testHash") and + (TxProcessingFailuresTable.processType eq "testProcess") + }.firstOrNull() + } + } + +// @Test +// fun testInsertOrUpdate_updateRecord() { +// transaction { +// TxProcessingFailureRecord.insertOrUpdate( +// blockHeight = 100, +// txHash = "testHash", +// processType = "testProcess", +// errorMessage = "testError", +// success = false +// ) +// +// TxProcessingFailureRecord.insertOrUpdate( +// blockHeight = 100, +// txHash = "testHash", +// processType = "testProcess", +// errorMessage = "updatedError", +// success = true +// ) +// +// val record = TxProcessingFailureRecord.find { +// (TxProcessingFailuresTable.blockHeight eq 100) and +// (TxProcessingFailuresTable.txHash eq "testHash") and +// (TxProcessingFailuresTable.processType eq "testProcess") +// }.firstOrNull() +// +// } +// } +} From b1b83ad004a57bc17a07edc679d5d35d45559196 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Mon, 26 Aug 2024 07:54:27 -0600 Subject: [PATCH 08/16] update tests --- .../entities/TxProcessingFailuresTableTest.kt | 64 ++++++++++--------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt b/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt index fb3731a1..d1bd2a58 100644 --- a/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt +++ b/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt @@ -3,17 +3,21 @@ import io.provenance.explorer.domain.entities.TxProcessingFailuresTable import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.Database +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Test class TxProcessingFailureRecordTest { @Test - fun testInsertOrUpdate_newRecord() { + fun `test tx_processing_failures table insertOrUpdate`() { Database.connect("jdbc:h2:mem:test;MODE=PostgreSQL;DB_CLOSE_DELAY=-1;", driver = "org.h2.Driver") + transaction { val sql = this::class.java.getResource("/db/migration/V1_91__Create_tx_processing_failures_table.sql")!!.readText() exec(sql) } + transaction { TxProcessingFailureRecord.insertOrUpdate( blockHeight = 100, @@ -23,39 +27,41 @@ class TxProcessingFailureRecordTest { success = false ) + var record = TxProcessingFailureRecord.find { + (TxProcessingFailuresTable.blockHeight eq 100) and + (TxProcessingFailuresTable.txHash eq "testHash") and + (TxProcessingFailuresTable.processType eq "testProcess") + }.firstOrNull() + + assertNotNull(record, "Record should not be null") + assertEquals(100, record?.blockHeight) + assertEquals("testHash", record?.txHash) + assertEquals("testProcess", record?.processType) + assertEquals("testError", record?.errorMessage) + assertEquals(false, record?.success) + } + + transaction { + TxProcessingFailureRecord.insertOrUpdate( + blockHeight = 100, + txHash = "testHash", + processType = "testProcess", + errorMessage = "updatedError", + success = true + ) + val record = TxProcessingFailureRecord.find { (TxProcessingFailuresTable.blockHeight eq 100) and (TxProcessingFailuresTable.txHash eq "testHash") and (TxProcessingFailuresTable.processType eq "testProcess") }.firstOrNull() + + assertNotNull(record, "Record should not be null") + assertEquals(100, record?.blockHeight) + assertEquals("testHash", record?.txHash) + assertEquals("testProcess", record?.processType) + assertEquals("updatedError", record?.errorMessage) + assertEquals(true, record?.success) } } - -// @Test -// fun testInsertOrUpdate_updateRecord() { -// transaction { -// TxProcessingFailureRecord.insertOrUpdate( -// blockHeight = 100, -// txHash = "testHash", -// processType = "testProcess", -// errorMessage = "testError", -// success = false -// ) -// -// TxProcessingFailureRecord.insertOrUpdate( -// blockHeight = 100, -// txHash = "testHash", -// processType = "testProcess", -// errorMessage = "updatedError", -// success = true -// ) -// -// val record = TxProcessingFailureRecord.find { -// (TxProcessingFailuresTable.blockHeight eq 100) and -// (TxProcessingFailuresTable.txHash eq "testHash") and -// (TxProcessingFailuresTable.processType eq "testProcess") -// }.firstOrNull() -// -// } -// } } From 3f5e105a934ba918b91a2640e17b86d93c51e1b8 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Mon, 26 Aug 2024 07:54:49 -0600 Subject: [PATCH 09/16] refactor name of block processing service --- .../kotlin/io/provenance/explorer/service/ExplorerService.kt | 4 ++-- .../io/provenance/explorer/service/TransactionService.kt | 4 ++-- .../provenance/explorer/service/async/BlockAndTxProcessor.kt | 4 ++-- .../provenance/explorer/service/async/ScheduledTaskService.kt | 2 +- .../provenance/explorer/service/utility/MigrationService.kt | 4 ++-- .../io/provenance/explorer/service/utility/UtilityService.kt | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/ExplorerService.kt b/service/src/main/kotlin/io/provenance/explorer/service/ExplorerService.kt index 72a8befe..db7d319c 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/ExplorerService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/ExplorerService.kt @@ -69,7 +69,7 @@ import io.provenance.explorer.model.base.DateTruncGranularity import io.provenance.explorer.model.base.PREFIX_SCOPE import io.provenance.explorer.model.base.PagedResults import io.provenance.explorer.model.base.USD_UPPER -import io.provenance.explorer.service.async.AsyncCachingV2 +import io.provenance.explorer.service.async.BlockAndTxProcessor import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.runBlocking @@ -88,7 +88,7 @@ class ExplorerService( private val blockService: BlockService, private val validatorService: ValidatorService, private val assetService: AssetService, - private val asyncV2: AsyncCachingV2, + private val asyncV2: BlockAndTxProcessor, private val govClient: GovGrpcClient, private val accountClient: AccountGrpcClient, private val ibcClient: IbcGrpcClient, diff --git a/service/src/main/kotlin/io/provenance/explorer/service/TransactionService.kt b/service/src/main/kotlin/io/provenance/explorer/service/TransactionService.kt index 8038e512..ac11f7e7 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/TransactionService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/TransactionService.kt @@ -54,7 +54,7 @@ import io.provenance.explorer.model.base.isMAddress import io.provenance.explorer.model.base.toMAddress import io.provenance.explorer.model.base.toMAddressScope import io.provenance.explorer.model.download.TxHistoryChartData -import io.provenance.explorer.service.async.AsyncCachingV2 +import io.provenance.explorer.service.async.BlockAndTxProcessor import io.provenance.explorer.service.async.getAddressType import org.jetbrains.exposed.dao.id.EntityID import org.jetbrains.exposed.sql.SizedIterable @@ -68,7 +68,7 @@ import javax.servlet.ServletOutputStream @Service class TransactionService( private val protoPrinter: JsonFormat.Printer, - private val asyncV2: AsyncCachingV2, + private val asyncV2: BlockAndTxProcessor, private val nftService: NftService, private val valService: ValidatorService, private val ibcService: IbcService diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt index ccfac358..f31ff861 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt @@ -125,7 +125,7 @@ import org.joda.time.DateTime import org.springframework.stereotype.Service @Service -class AsyncCachingV2( +class BlockAndTxProcessor( private val txClient: TransactionGrpcClient, private val blockService: BlockService, private val validatorService: ValidatorService, @@ -140,7 +140,7 @@ class AsyncCachingV2( private val groupService: GroupService ) { - protected val logger = logger(AsyncCachingV2::class) + protected val logger = logger(BlockAndTxProcessor::class) protected var chainId: String = "" diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt index eb397b75..110eb9a5 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt @@ -93,7 +93,7 @@ class ScheduledTaskService( private val blockService: BlockService, private val assetService: AssetService, private val govService: GovService, - private val asyncCache: AsyncCachingV2, + private val asyncCache: BlockAndTxProcessor, private val explorerService: ExplorerService, private val cacheService: CacheService, private val tokenService: TokenService, diff --git a/service/src/main/kotlin/io/provenance/explorer/service/utility/MigrationService.kt b/service/src/main/kotlin/io/provenance/explorer/service/utility/MigrationService.kt index c6f4294e..75c0e807 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/utility/MigrationService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/utility/MigrationService.kt @@ -6,14 +6,14 @@ import io.provenance.explorer.domain.entities.BlockCacheTable import io.provenance.explorer.service.AccountService import io.provenance.explorer.service.BlockService import io.provenance.explorer.service.ValidatorService -import io.provenance.explorer.service.async.AsyncCachingV2 +import io.provenance.explorer.service.async.BlockAndTxProcessor import org.jetbrains.exposed.sql.SortOrder import org.jetbrains.exposed.sql.transactions.transaction import org.springframework.stereotype.Service @Service class MigrationService( - private val asyncCaching: AsyncCachingV2, + private val asyncCaching: BlockAndTxProcessor, private val validatorService: ValidatorService, private val accountService: AccountService, private val blockService: BlockService diff --git a/service/src/main/kotlin/io/provenance/explorer/service/utility/UtilityService.kt b/service/src/main/kotlin/io/provenance/explorer/service/utility/UtilityService.kt index f9d8a58b..6f0bc22f 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/utility/UtilityService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/utility/UtilityService.kt @@ -18,7 +18,7 @@ import io.provenance.explorer.domain.models.explorer.BlockProposer import io.provenance.explorer.domain.models.explorer.getCategoryForType import io.provenance.explorer.grpc.v1.AccountGrpcClient import io.provenance.explorer.service.AssetService -import io.provenance.explorer.service.async.AsyncCachingV2 +import io.provenance.explorer.service.async.BlockAndTxProcessor import io.provenance.explorer.service.firstMatchLabel import kotlinx.coroutines.runBlocking import net.pearx.kasechange.toSnakeCase @@ -33,7 +33,7 @@ class UtilityService( private val protoParser: JsonFormat.Parser, private val accountClient: AccountGrpcClient, private val assetService: AssetService, - private val async: AsyncCachingV2 + private val async: BlockAndTxProcessor ) { protected val logger = logger(UtilityService::class) From 564f32740a25ee3cedd120f1bd7096f35d5e4a11 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Mon, 26 Aug 2024 11:01:25 -0600 Subject: [PATCH 10/16] minor refactors --- .../explorer/domain/entities/Blocks.kt | 17 ++++++++--------- .../service/async/ScheduledTaskService.kt | 10 +++++----- .../service/async/BlockAndTxProcessorTest.kt | 5 +++++ 3 files changed, 18 insertions(+), 14 deletions(-) create mode 100644 service/src/test/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessorTest.kt diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt index 1d553a3f..f9c6d4d5 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt @@ -212,8 +212,7 @@ class BlockProposerRecord(id: EntityID) : IntEntity(id) { fun getRecordsForProposer(address: String, limit: Int) = transaction { BlockProposerRecord.find { - (BlockProposerTable.proposerOperatorAddress eq address) and - (BlockProposerTable.blockLatency.isNotNull()) + (BlockProposerTable.proposerOperatorAddress eq address) and (BlockProposerTable.blockLatency.isNotNull()) }.orderBy(Pair(BlockProposerTable.blockHeight, SortOrder.DESC)) .limit(limit) .toList() @@ -538,7 +537,7 @@ class BlockTxRetryRecord(id: EntityID) : IntEntity(id) { it[this.success] = false it[this.errorBlock] = "NON BLOCKING ERROR: Logged to know what happened, but didnt stop processing.\n " + - e.stackTraceToString() + e.stackTraceToString() } } @@ -547,7 +546,7 @@ class BlockTxRetryRecord(id: EntityID) : IntEntity(id) { it[this.height] = height it[this.errorBlock] = "NON BLOCKING ERROR: Logged to know what happened, but didnt stop processing.\n " + - e.stackTraceToString() + e.stackTraceToString() } } @@ -573,8 +572,8 @@ class BlockTxRetryRecord(id: EntityID) : IntEntity(id) { BlockTxRetryTable .deleteWhere { (BlockTxRetryTable.retried eq true) and - (BlockTxRetryTable.success eq true) and - (BlockTxRetryTable.height inList heights) + (BlockTxRetryTable.success eq true) and + (BlockTxRetryTable.height inList heights) } } } @@ -613,8 +612,8 @@ class TxProcessingFailureRecord(id: EntityID) : IntEntity(id) { ) = transaction { val existingRecord = TxProcessingFailureRecord.find { (TxProcessingFailuresTable.blockHeight eq blockHeight) and - (TxProcessingFailuresTable.txHash eq txHash) and - (TxProcessingFailuresTable.processType eq processType) + (TxProcessingFailuresTable.txHash eq txHash) and + (TxProcessingFailuresTable.processType eq processType) }.firstOrNull() if (existingRecord == null) { @@ -638,7 +637,7 @@ class TxProcessingFailureRecord(id: EntityID) : IntEntity(id) { fun deleteProcessedRecords() = transaction { TxProcessingFailuresTable.deleteWhere { (TxProcessingFailuresTable.retried eq true) and - (TxProcessingFailuresTable.success eq true) + (TxProcessingFailuresTable.success eq true) } } } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt index 110eb9a5..845b7e08 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt @@ -93,7 +93,7 @@ class ScheduledTaskService( private val blockService: BlockService, private val assetService: AssetService, private val govService: GovService, - private val asyncCache: BlockAndTxProcessor, + private val blockAndTxProcessor: BlockAndTxProcessor, private val explorerService: ExplorerService, private val cacheService: CacheService, private val tokenService: TokenService, @@ -135,7 +135,7 @@ class ScheduledTaskService( shouldContinue = false return } - asyncCache.saveBlockEtc(it) + blockAndTxProcessor.saveBlockEtc(it) indexHeight = it.block.height() - 1 } blockService.updateBlockMinHeightIndex(indexHeight + 1) @@ -144,7 +144,7 @@ class ScheduledTaskService( } else { while (indexHeight > index.first!!) { blockService.getBlockAtHeightFromChain(indexHeight)?.let { - asyncCache.saveBlockEtc(it) + blockAndTxProcessor.saveBlockEtc(it) indexHeight = it.block.height() - 1 } } @@ -246,7 +246,7 @@ class ScheduledTaskService( logger.info("Retrying block/tx record at $height.") var retryException: Exception? = null val block = try { - asyncCache.saveBlockEtc(blockService.getBlockAtHeightFromChain(height), Pair(true, false))!! + blockAndTxProcessor.saveBlockEtc(blockService.getBlockAtHeightFromChain(height), Pair(true, false))!! } catch (e: Exception) { retryException = e logger.error("Error saving block $height on retry.", e) @@ -409,7 +409,7 @@ class ScheduledTaskService( (startBlock.toInt()..minOf(props.oneElevenBugRange()!!.last, startBlock.toInt().plus(100))).toList() .let { BlockCacheRecord.getBlocksForRange(it.first(), it.last()) } .forEach { block -> - if (block.txCount > 0) asyncCache.saveBlockEtc(block.block, Pair(true, false)) + if (block.txCount > 0) blockAndTxProcessor.saveBlockEtc(block.block, Pair(true, false)) // Check if the last processed block equals the end of the fee bug range if (block.height == props.oneElevenBugRange()!!.last) { cacheService.updateCacheValue(CacheKeys.FEE_BUG_ONE_ELEVEN_START_BLOCK.key, done) diff --git a/service/src/test/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessorTest.kt b/service/src/test/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessorTest.kt new file mode 100644 index 00000000..f2fca876 --- /dev/null +++ b/service/src/test/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessorTest.kt @@ -0,0 +1,5 @@ +package io.provenance.explorer.service.async + +import org.junit.jupiter.api.Assertions.* + +class BlockAndTxProcessorTest \ No newline at end of file From a1711514a581b83cb5bbe4605f6f585dee252f4c Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Mon, 26 Aug 2024 11:03:11 -0600 Subject: [PATCH 11/16] test refactor --- .../entities/TxProcessingFailuresTableTest.kt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt b/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt index d1bd2a58..2a1dcd90 100644 --- a/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt +++ b/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt @@ -1,8 +1,8 @@ import io.provenance.explorer.domain.entities.TxProcessingFailureRecord import io.provenance.explorer.domain.entities.TxProcessingFailuresTable -import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.Database +import org.jetbrains.exposed.sql.transactions.transaction import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Test @@ -14,7 +14,8 @@ class TxProcessingFailureRecordTest { Database.connect("jdbc:h2:mem:test;MODE=PostgreSQL;DB_CLOSE_DELAY=-1;", driver = "org.h2.Driver") transaction { - val sql = this::class.java.getResource("/db/migration/V1_91__Create_tx_processing_failures_table.sql")!!.readText() + val sql = this::class.java.getResource("/db/migration/V1_91__Create_tx_processing_failures_table.sql")!! + .readText() exec(sql) } @@ -29,8 +30,8 @@ class TxProcessingFailureRecordTest { var record = TxProcessingFailureRecord.find { (TxProcessingFailuresTable.blockHeight eq 100) and - (TxProcessingFailuresTable.txHash eq "testHash") and - (TxProcessingFailuresTable.processType eq "testProcess") + (TxProcessingFailuresTable.txHash eq "testHash") and + (TxProcessingFailuresTable.processType eq "testProcess") }.firstOrNull() assertNotNull(record, "Record should not be null") @@ -52,8 +53,8 @@ class TxProcessingFailureRecordTest { val record = TxProcessingFailureRecord.find { (TxProcessingFailuresTable.blockHeight eq 100) and - (TxProcessingFailuresTable.txHash eq "testHash") and - (TxProcessingFailuresTable.processType eq "testProcess") + (TxProcessingFailuresTable.txHash eq "testHash") and + (TxProcessingFailuresTable.processType eq "testProcess") }.firstOrNull() assertNotNull(record, "Record should not be null") From f1862c05a3fea379c731c8ab0283d9fe8ca25d7c Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Mon, 26 Aug 2024 11:11:40 -0600 Subject: [PATCH 12/16] fix lints --- .../io/provenance/explorer/domain/entities/Blocks.kt | 8 ++------ .../domain/entities/TxProcessingFailuresTableTest.kt | 6 +++--- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt index f9c6d4d5..cafe9ca1 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt @@ -535,18 +535,14 @@ class BlockTxRetryRecord(id: EntityID) : IntEntity(id) { it[this.height] = height it[this.retried] = true it[this.success] = false - it[this.errorBlock] = - "NON BLOCKING ERROR: Logged to know what happened, but didnt stop processing.\n " + - e.stackTraceToString() + it[this.errorBlock] = "NON BLOCKING ERROR: Logged to know what happened, but didnt stop processing.\n " + e.stackTraceToString() } } fun insertNonBlockingRetry(height: Int, e: Exception) = transaction { BlockTxRetryTable.insertIgnore { it[this.height] = height - it[this.errorBlock] = - "NON BLOCKING ERROR: Logged to know what happened, but didnt stop processing.\n " + - e.stackTraceToString() + it[this.errorBlock] = "NON BLOCKING ERROR: Logged to know what happened, but didnt stop processing.\n " + e.stackTraceToString() } } diff --git a/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt b/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt index 2a1dcd90..1c798f7a 100644 --- a/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt +++ b/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt @@ -1,11 +1,11 @@ import io.provenance.explorer.domain.entities.TxProcessingFailureRecord import io.provenance.explorer.domain.entities.TxProcessingFailuresTable -import org.jetbrains.exposed.sql.and -import org.jetbrains.exposed.sql.Database -import org.jetbrains.exposed.sql.transactions.transaction import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Test +import org.jetbrains.exposed.sql.and +import org.jetbrains.exposed.sql.Database +import org.jetbrains.exposed.sql.transactions.transaction class TxProcessingFailureRecordTest { From 53739711e2ec39426be541456688467d274dd8eb Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Mon, 26 Aug 2024 11:14:26 -0600 Subject: [PATCH 13/16] remove test file --- .../explorer/service/async/BlockAndTxProcessorTest.kt | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 service/src/test/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessorTest.kt diff --git a/service/src/test/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessorTest.kt b/service/src/test/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessorTest.kt deleted file mode 100644 index f2fca876..00000000 --- a/service/src/test/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessorTest.kt +++ /dev/null @@ -1,5 +0,0 @@ -package io.provenance.explorer.service.async - -import org.junit.jupiter.api.Assertions.* - -class BlockAndTxProcessorTest \ No newline at end of file From d4412949336326b064f962c0299a496a18d9fd0f Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Mon, 26 Aug 2024 11:32:22 -0600 Subject: [PATCH 14/16] refactor table name --- ...1__Create_tx_processing_failures_table.sql | 4 +-- .../explorer/domain/entities/Blocks.kt | 32 +++++++++---------- .../entities/TxProcessingFailuresTableTest.kt | 14 ++++---- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql b/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql index dfc92a02..d09d275e 100644 --- a/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql +++ b/database/src/main/resources/db/migration/V1_91__Create_tx_processing_failures_table.sql @@ -1,6 +1,6 @@ -SELECT 'Create tx_processing_failures table' AS comment; +SELECT 'Create tx_processing_failure table' AS comment; -CREATE TABLE IF NOT EXISTS tx_processing_failures ( +CREATE TABLE IF NOT EXISTS tx_processing_failure ( id SERIAL PRIMARY KEY, block_height INT NOT NULL, tx_hash VARCHAR(128) NOT NULL, diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt index cafe9ca1..68219c6c 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt @@ -580,7 +580,7 @@ class BlockTxRetryRecord(id: EntityID) : IntEntity(id) { var errorBlock by BlockTxRetryTable.errorBlock } -object TxProcessingFailuresTable : IdTable(name = "tx_processing_failures") { +object TxProcessingFailureTable : IdTable(name = "tx_processing_failure") { val blockHeight = integer("block_height") val txHash = varchar("tx_hash", 128) val processType = varchar("process_type", 64) @@ -597,7 +597,7 @@ object TxProcessingFailuresTable : IdTable(name = "tx_processing_failures") } class TxProcessingFailureRecord(id: EntityID) : IntEntity(id) { - companion object : IntEntityClass(TxProcessingFailuresTable) { + companion object : IntEntityClass(TxProcessingFailureTable) { fun insertOrUpdate( blockHeight: Int, @@ -607,13 +607,13 @@ class TxProcessingFailureRecord(id: EntityID) : IntEntity(id) { success: Boolean ) = transaction { val existingRecord = TxProcessingFailureRecord.find { - (TxProcessingFailuresTable.blockHeight eq blockHeight) and - (TxProcessingFailuresTable.txHash eq txHash) and - (TxProcessingFailuresTable.processType eq processType) + (TxProcessingFailureTable.blockHeight eq blockHeight) and + (TxProcessingFailureTable.txHash eq txHash) and + (TxProcessingFailureTable.processType eq processType) }.firstOrNull() if (existingRecord == null) { - TxProcessingFailuresTable.insertIgnore { + TxProcessingFailureTable.insertIgnore { it[this.blockHeight] = blockHeight it[this.txHash] = txHash it[this.processType] = processType @@ -631,18 +631,18 @@ class TxProcessingFailureRecord(id: EntityID) : IntEntity(id) { } fun deleteProcessedRecords() = transaction { - TxProcessingFailuresTable.deleteWhere { - (TxProcessingFailuresTable.retried eq true) and - (TxProcessingFailuresTable.success eq true) + TxProcessingFailureTable.deleteWhere { + (TxProcessingFailureTable.retried eq true) and + (TxProcessingFailureTable.success eq true) } } } - var blockHeight by TxProcessingFailuresTable.blockHeight - var txHash by TxProcessingFailuresTable.txHash - var processType by TxProcessingFailuresTable.processType - var failureTime by TxProcessingFailuresTable.failureTime - var errorMessage by TxProcessingFailuresTable.errorMessage - var retried by TxProcessingFailuresTable.retried - var success by TxProcessingFailuresTable.success + var blockHeight by TxProcessingFailureTable.blockHeight + var txHash by TxProcessingFailureTable.txHash + var processType by TxProcessingFailureTable.processType + var failureTime by TxProcessingFailureTable.failureTime + var errorMessage by TxProcessingFailureTable.errorMessage + var retried by TxProcessingFailureTable.retried + var success by TxProcessingFailureTable.success } diff --git a/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt b/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt index 1c798f7a..1f0e01a1 100644 --- a/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt +++ b/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt @@ -1,5 +1,5 @@ import io.provenance.explorer.domain.entities.TxProcessingFailureRecord -import io.provenance.explorer.domain.entities.TxProcessingFailuresTable +import io.provenance.explorer.domain.entities.TxProcessingFailureTable import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Test @@ -29,9 +29,9 @@ class TxProcessingFailureRecordTest { ) var record = TxProcessingFailureRecord.find { - (TxProcessingFailuresTable.blockHeight eq 100) and - (TxProcessingFailuresTable.txHash eq "testHash") and - (TxProcessingFailuresTable.processType eq "testProcess") + (TxProcessingFailureTable.blockHeight eq 100) and + (TxProcessingFailureTable.txHash eq "testHash") and + (TxProcessingFailureTable.processType eq "testProcess") }.firstOrNull() assertNotNull(record, "Record should not be null") @@ -52,9 +52,9 @@ class TxProcessingFailureRecordTest { ) val record = TxProcessingFailureRecord.find { - (TxProcessingFailuresTable.blockHeight eq 100) and - (TxProcessingFailuresTable.txHash eq "testHash") and - (TxProcessingFailuresTable.processType eq "testProcess") + (TxProcessingFailureTable.blockHeight eq 100) and + (TxProcessingFailureTable.txHash eq "testHash") and + (TxProcessingFailureTable.processType eq "testProcess") }.firstOrNull() assertNotNull(record, "Record should not be null") From 8e9e28a5c296cd87a070c9a5ac1ca1f700913274 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Mon, 26 Aug 2024 11:33:41 -0600 Subject: [PATCH 15/16] fix lint --- .../domain/entities/TxProcessingFailuresTableTest.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt b/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt index 1f0e01a1..955ee243 100644 --- a/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt +++ b/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt @@ -1,11 +1,11 @@ import io.provenance.explorer.domain.entities.TxProcessingFailureRecord import io.provenance.explorer.domain.entities.TxProcessingFailureTable -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNotNull -import org.junit.jupiter.api.Test import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.Database import org.jetbrains.exposed.sql.transactions.transaction +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Test class TxProcessingFailureRecordTest { From 2cfd8945334210ac0a12c6374f4a8e0981524b74 Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Mon, 26 Aug 2024 11:38:06 -0600 Subject: [PATCH 16/16] fix import lint --- .../explorer/domain/entities/TxProcessingFailuresTableTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt b/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt index 955ee243..0d3d8092 100644 --- a/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt +++ b/service/src/test/kotlin/io/provenance/explorer/domain/entities/TxProcessingFailuresTableTest.kt @@ -1,7 +1,7 @@ import io.provenance.explorer.domain.entities.TxProcessingFailureRecord import io.provenance.explorer.domain.entities.TxProcessingFailureTable -import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.Database +import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.transactions.transaction import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull