Fix group save tx processing (#531)

* move group save to group service * setup test * add test setup, start moving away from using logs * add backwards compatibility to extracting attribute values * disable a test until complete * remove wildcard imports * update logging * add update to retry blocks table to update error message, it might have changed * change retry tx update to update error message, change scheduled time from 1 min to 5 min * throw an actual meaningful error here when policy not found * look up group policy later in process * add an error when member address not found, instead of null pointer exception * temp comment out ibc * retry error every 10 seconds * remove unused import * fix lints * refactor code for easier debugging, update testing * fix lints * add better logging on retries * fix lint * fix another lint * more lints * lint * remove blank line * add changelog
provenance-io · Aug 20, 2024 · 240ff77 · 240ff77
1 parent 5f0bf5b
commit 240ff77
Show file tree

Hide file tree

Showing 20 changed files with 3,488 additions and 299 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -43,6 +43,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 * Fixed issue with Proto deserialization incorrectly matching `cosmos.upgrade.v1beta1.CancelSoftwareUpgradeProposal` as `cosmos.upgrade.v1beta1.SoftwareUpgradeProposal`, ensuring accurate type URL handling. [#524](https://github.com/provenance-io/explorer-service/pull/524)
 * Update osmosis pricing query with new required field `coinMinimalDenom` [#526](https://github.com/provenance-io/explorer-service/pull/526)
+* Fix group save tx processing [#526](https://github.com/provenance-io/explorer-service/pull/528), [#526](https://github.com/provenance-io/explorer-service/pull/529)
 
 ## [v5.10.0](https://github.com/provenance-io/explorer-service/releases/tag/v5.10.0) - 2024-06-11
 ### Release Name: Fridtjof Nansen

diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt
@@ -60,6 +60,7 @@ object Versions {
     const val Jupiter = "5.9.1"
     const val SpringMockk = "3.1.1"
     const val Kotest = "5.5.4"
+    const val H2Database = "1.4.200"
 }
 
 object Libraries {
@@ -113,4 +114,5 @@ object Libraries {
     const val JunitJupiterEngine = "org.junit.jupiter:junit-jupiter-engine:${Versions.Jupiter}"
     const val SpringMockk = "com.ninja-squad:springmockk:${Versions.SpringMockk}"
     const val KotestAssert = "io.kotest:kotest-assertions-core:${Versions.Kotest}"
+    const val H2Database = "com.h2database:h2:${Versions.H2Database}"
 }
diff --git a/service/build.gradle.kts b/service/build.gradle.kts
@@ -69,10 +69,10 @@ dependencies {
 
     testImplementation(Libraries.SpringBootStarterTest) {
         exclude(module = "junit")
-        exclude(module = "mockito-core")
         exclude(module = "assertj-core")
     }
     testImplementation(Libraries.JunitJupiterApi)
+    testImplementation(Libraries.H2Database)
     testRuntimeOnly(Libraries.JunitJupiterEngine)
     testImplementation(Libraries.SpringMockk)
     testImplementation(Libraries.KotestAssert)

diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt
@@ -61,6 +61,7 @@ import org.jetbrains.exposed.sql.statements.BatchUpdateStatement
 import org.jetbrains.exposed.sql.sum
 import org.jetbrains.exposed.sql.transactions.TransactionManager
 import org.jetbrains.exposed.sql.transactions.transaction
+import org.jetbrains.exposed.sql.update
 import org.joda.time.DateTime
 import org.joda.time.DateTimeZone
 import java.sql.ResultSet
@@ -515,11 +516,17 @@ object BlockTxRetryTable : IdTable<Int>(name = "block_tx_retry") {
 class BlockTxRetryRecord(id: EntityID<Int>) : IntEntity(id) {
     companion object : IntEntityClass<BlockTxRetryRecord>(BlockTxRetryTable) {
 
-        fun insert(height: Int, e: Exception) = transaction {
-            BlockTxRetryTable.insertIgnore {
-                it[this.height] = height
+        fun insertOrUpdate(height: Int, e: Exception) = transaction {
+            val rowsUpdated = BlockTxRetryTable.update({ BlockTxRetryTable.height eq height }) {
                 it[this.errorBlock] = e.stackTraceToString()
             }
+
+            if (rowsUpdated == 0) {
+                BlockTxRetryTable.insertIgnore {
+                    it[this.height] = height
+                    it[this.errorBlock] = e.stackTraceToString()
+                }
+            }
         }
 
         fun insertNonRetry(height: Int, e: Exception) = transaction {
@@ -550,10 +557,13 @@ class BlockTxRetryRecord(id: EntityID<Int>) : IntEntity(id) {
                 .map { it.height }
         }
 
-        fun updateRecord(height: Int, success: Boolean) = transaction {
+        fun updateRecord(height: Int, success: Boolean, e: Exception?) = transaction {
             BlockTxRetryRecord.find { BlockTxRetryTable.height eq height }.first().apply {
                 this.retried = true
                 this.success = success
+                if (e != null) {
+                    this.errorBlock = e.stackTraceToString()
+                }
             }
         }
 

diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Groups.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Groups.kt
@@ -208,11 +208,13 @@ class GroupsProposalRecord(id: EntityID<Int>) : IntEntity(id) {
             proposalData: GroupsProposalInsertData,
             txInfo: TxData
         ) = transaction {
+            val policyId = GroupsPolicyRecord.findByPolicyAddr(proposalData.policyAddress)?.id?.value
+                ?: throw IllegalStateException("Policy not found for address: ${proposalData.policyAddress}")
             listOf(
                 -1,
                 proposalData.groupId,
-                proposalData.policy.id.value,
-                proposalData.policy.policyAddress,
+                policyId,
+                proposalData.policyAddress,
                 proposalData.proposalId,
                 proposalData.data.stringify(),
                 proposalData.nodeData,

diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/GroupModels.kt b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/GroupModels.kt
@@ -6,7 +6,6 @@ import cosmos.group.v1.Types.Proposal
 import cosmos.group.v1.Types.ProposalExecutorResult
 import cosmos.group.v1.Types.ProposalStatus
 import cosmos.group.v1.Types.VoteOption
-import io.provenance.explorer.domain.entities.GroupsPolicyRecord
 import org.joda.time.DateTime
 
 data class GroupMembers(val list: MutableList<Types.Member>)
@@ -25,7 +24,7 @@ data class GroupsProposalData(
 
 data class GroupsProposalInsertData(
     val groupId: Long,
-    val policy: GroupsPolicyRecord,
+    val policyAddress: String,
     val proposalId: Long,
     val data: GroupsProposalData,
     val nodeData: Proposal? = null,

diff --git a/service/src/main/kotlin/io/provenance/explorer/grpc/extensions/Helper.kt b/service/src/main/kotlin/io/provenance/explorer/grpc/extensions/Helper.kt
@@ -2,16 +2,33 @@ package io.provenance.explorer.grpc.extensions
 
 import cosmos.tx.v1beta1.ServiceOuterClass
 
-fun ServiceOuterClass.GetTxResponse.mapEventAttrValues(idx: Int, event: String, attrList: List<String>) =
-    this.txResponse.logsList[idx].eventsList.firstOrNull { it.type == event }
+fun ServiceOuterClass.GetTxResponse.mapEventAttrValues(idx: Int, event: String, attrList: List<String>): Map<String, String> {
+    return if (this.txResponse.logsList == null || this.txResponse.logsList.size <= idx) {
+        mapEventAttrValuesByMsgIndex(idx, event, attrList)
+    } else {
+        mapEventAttrValuesFromLogs(idx, event, attrList)
+    }
+}
+
+fun ServiceOuterClass.GetTxResponse.mapEventAttrValuesFromLogs(idx: Int, event: String, attrList: List<String>): Map<String, String> {
+    return this.txResponse.logsList[idx].eventsList.firstOrNull { it.type == event }
         ?.attributesList?.let { list -> attrList.map { attr -> attr to list.first { it.key == attr }.value.scrubQuotes() } }
         ?.toMap() ?: mapOf()
+}
 
-fun ServiceOuterClass.GetTxResponse.findEvent(idx: Int, event: String) =
-    this.txResponse.logsList[idx].eventsList.firstOrNull { it.type == event }
+fun ServiceOuterClass.GetTxResponse.mapEventAttrValuesByMsgIndex(idx: Int, event: String, attrList: List<String>): Map<String, String> {
+    return this.txResponse.eventsList
+        .firstOrNull { eventEntry ->
+            eventEntry.type == event && eventEntry.attributesList.any { it.key == "msg_index" && it.value == idx.toString() }
+        }
+        ?.attributesList
+        ?.filter { it.key in attrList }
+        ?.associate { it.key to it.value.scrubQuotes() }
+        ?: mapOf()
+}
 
 fun ServiceOuterClass.GetTxResponse.findAllMatchingEvents(eventList: List<String>) =
-    this.txResponse.logsList.flatMap { log -> log.eventsList }.filter { it.type in eventList }
+    this.txResponse.eventsList.filter { it.type in eventList }
 
 fun String.removeFirstSlash() = this.split("/")[1]
 

diff --git a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/TransactionGrpcClient.kt b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/TransactionGrpcClient.kt
@@ -2,14 +2,12 @@ package io.provenance.explorer.grpc.v1
 
 import cosmos.tx.v1beta1.ServiceGrpcKt
 import cosmos.tx.v1beta1.ServiceOuterClass
-import cosmos.tx.v1beta1.getTxRequest
 import cosmos.tx.v1beta1.getTxResponse
 import cosmos.tx.v1beta1.getTxsEventRequest
 import io.grpc.ManagedChannelBuilder
 import io.provenance.explorer.config.interceptor.GrpcLoggingInterceptor
 import io.provenance.explorer.domain.core.logger
 import io.provenance.explorer.domain.exceptions.TendermintApiException
-import kotlinx.coroutines.runBlocking
 import org.springframework.stereotype.Component
 import java.net.URI
 import java.util.concurrent.TimeUnit
@@ -40,8 +38,6 @@ class TransactionGrpcClient(channelUri: URI) {
         txClient = ServiceGrpcKt.ServiceCoroutineStub(channel)
     }
 
-    fun getTxByHash(hash: String) = runBlocking { txClient.getTx(getTxRequest { this.hash = hash }) }
-
     suspend fun getTxsByHeight(height: Int, total: Int): List<ServiceOuterClass.GetTxResponse> {
         var page = 1
         val limit = 10