diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 46bb7d8e..1b7ca59b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -34,7 +34,7 @@ jobs: VERSION=latest fi fi - TAGS="${DOCKER_IMAGE}:${VERSION}" + TAGS="${DOCKER_IMAGE}:${VERSION}-${{github.run_number}}" if [[ $VERSION =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then TAGS="$TAGS,${DOCKER_IMAGE}:${VERSION}" fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 411bc5a8..d3735dff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,11 @@ Ref: https://keepachangelog.com/en/1.0.0/ ## Unreleased +### Features +* Validator Delegation Program metrics #482 + * Added support to calculate and return validator metrics used by the Validator Delegation Program + * Added client support + ### Improvements * Add date parameters to `/api/v3/txs/heatmap` #462 * Added `fromDate`, `toDate`, `timeframe` @@ -41,6 +46,11 @@ Ref: https://keepachangelog.com/en/1.0.0/ ### Bug Fixes * Fixed `/api/v3/validators` url * Now handling no fee amount in the tx +* Now handling Exec'd governance msgs, and properly handling weights from v1.Gov msgs + +### Data +* Migration 1.88 - Add `block_time_spread`, `validator_metrics` #482 + * Adds view, table to support calculating and storing metrics for the Validator Delegation Program ## [v5.4.0](https://github.com/provenance-io/explorer-service/releases/tag/v5.4.0) - 2022-12-15 ### Release Name: Jean de Béthencourt diff --git a/api-client/src/main/kotlin/io/provenance/explorer/client/AccountClient.kt b/api-client/src/main/kotlin/io/provenance/explorer/client/AccountClient.kt index 037113cc..548468f7 100644 --- a/api-client/src/main/kotlin/io/provenance/explorer/client/AccountClient.kt +++ b/api-client/src/main/kotlin/io/provenance/explorer/client/AccountClient.kt @@ -9,12 +9,12 @@ import io.provenance.explorer.model.AccountRewards import io.provenance.explorer.model.AccountVestingInfo import io.provenance.explorer.model.Delegation import io.provenance.explorer.model.DenomBalanceBreakdown -import io.provenance.explorer.model.TxHistoryChartData import io.provenance.explorer.model.UnpaginatedDelegation import io.provenance.explorer.model.base.CoinStr import io.provenance.explorer.model.base.DateTruncGranularity import io.provenance.explorer.model.base.PagedResults import io.provenance.explorer.model.base.PeriodInSeconds +import io.provenance.explorer.model.download.TxHistoryChartData import org.joda.time.DateTime object AccountRoutes { diff --git a/api-client/src/main/kotlin/io/provenance/explorer/client/TransactionClient.kt b/api-client/src/main/kotlin/io/provenance/explorer/client/TransactionClient.kt index aff799ab..248a7796 100644 --- a/api-client/src/main/kotlin/io/provenance/explorer/client/TransactionClient.kt +++ b/api-client/src/main/kotlin/io/provenance/explorer/client/TransactionClient.kt @@ -7,13 +7,14 @@ import io.provenance.explorer.model.MsgTypeSet import io.provenance.explorer.model.TxDetails import io.provenance.explorer.model.TxGov import io.provenance.explorer.model.TxHeatmapRes -import io.provenance.explorer.model.TxHistoryChartData import io.provenance.explorer.model.TxMessage import io.provenance.explorer.model.TxStatus import io.provenance.explorer.model.TxSummary import io.provenance.explorer.model.TxType import io.provenance.explorer.model.base.DateTruncGranularity import io.provenance.explorer.model.base.PagedResults +import io.provenance.explorer.model.base.Timeframe +import io.provenance.explorer.model.download.TxHistoryChartData import org.joda.time.DateTime object TransactionRoutes { @@ -26,7 +27,7 @@ object TransactionRoutes { const val TX_JSON = "$TX_V2/{hash}/json" const val TX_TYPES = "$TX_V3/{hash}/types" const val TXS_AT_HEIGHT = "$TX_V2/height/{height}" - const val HEATMAP = "$TX_V2/heatmap" + const val HEATMAP = "$TX_V3/heatmap" const val TYPES = "$TX_V2/types" const val TYPES_BY_MODULE = "$TX_V2/types/{module}" const val TXS_BY_MODULE = "$TX_V2/module/{module}" @@ -83,7 +84,11 @@ interface TransactionClient : BaseClient { ): PagedResults @RequestLine("GET ${TransactionRoutes.HEATMAP}") - fun heatmap(): TxHeatmapRes + fun heatmap( + @Param("fromDate") fromDate: DateTime? = null, + @Param("toDate") toDate: DateTime? = null, + @Param("timeframe") timeframe: Timeframe = Timeframe.FOREVER + ): TxHeatmapRes @RequestLine("GET ${TransactionRoutes.TYPES}") fun types(): List diff --git a/api-client/src/main/kotlin/io/provenance/explorer/client/ValidatorClient.kt b/api-client/src/main/kotlin/io/provenance/explorer/client/ValidatorClient.kt index b25cd0b6..0b629750 100644 --- a/api-client/src/main/kotlin/io/provenance/explorer/client/ValidatorClient.kt +++ b/api-client/src/main/kotlin/io/provenance/explorer/client/ValidatorClient.kt @@ -6,6 +6,7 @@ import feign.RequestLine import io.provenance.explorer.model.BlockLatencyData import io.provenance.explorer.model.Delegation import io.provenance.explorer.model.MarketRateAvg +import io.provenance.explorer.model.MetricPeriod import io.provenance.explorer.model.MissedBlocksTimeframe import io.provenance.explorer.model.UnpaginatedDelegation import io.provenance.explorer.model.UptimeDataSet @@ -14,6 +15,7 @@ import io.provenance.explorer.model.ValidatorCommission import io.provenance.explorer.model.ValidatorCommissionHistory import io.provenance.explorer.model.ValidatorDetails import io.provenance.explorer.model.ValidatorMarketRate +import io.provenance.explorer.model.ValidatorMetrics import io.provenance.explorer.model.ValidatorState import io.provenance.explorer.model.ValidatorSummary import io.provenance.explorer.model.ValidatorSummaryAbbrev @@ -38,6 +40,9 @@ object ValidatorRoutes { const val MISSED_BLOCKS_DISTINCT = "$VALIDATOR_V2/missed_blocks/distinct" const val MISSED_BLOCKS = "$VALIDATOR_V2/missed_blocks" const val UPTIME = "$VALIDATOR_V2/uptime" + const val VALIDATOR_METRICS = "$VALIDATOR_V3/{address}/metrics" + const val VALIDATOR_METRICS_PERIODS = "$VALIDATOR_V3/{address}/metrics/periods" + const val ALL_METRICS_PERIODS = "$VALIDATOR_V3/metrics/periods" } @Headers(BaseClient.CT_JSON) @@ -112,4 +117,17 @@ interface ValidatorClient : BaseClient { @RequestLine("GET ${ValidatorRoutes.UPTIME}") fun uptimeData(): UptimeDataSet + + @RequestLine("GET ${ValidatorRoutes.VALIDATOR_METRICS}") + fun validatorMetrics( + @Param("address") address: String, + @Param("year") year: Int, + @Param("quarter") quarter: Int + ): ValidatorMetrics + + @RequestLine("GET ${ValidatorRoutes.VALIDATOR_METRICS_PERIODS}") + fun validatorMetricsPeriods(@Param("address") address: String): MutableList + + @RequestLine("GET ${ValidatorRoutes.ALL_METRICS_PERIODS}") + fun allMetricsPeriods(): MutableList } diff --git a/api-model/src/main/kotlin/io/provenance/explorer/model/TokenModels.kt b/api-model/src/main/kotlin/io/provenance/explorer/model/TokenModels.kt index 6e6c3f21..30ac6b6f 100644 --- a/api-model/src/main/kotlin/io/provenance/explorer/model/TokenModels.kt +++ b/api-model/src/main/kotlin/io/provenance/explorer/model/TokenModels.kt @@ -3,6 +3,8 @@ package io.provenance.explorer.model import io.provenance.explorer.model.base.CoinStr import io.provenance.explorer.model.base.DateTruncGranularity import io.provenance.explorer.model.base.USD_UPPER +import io.provenance.explorer.model.download.currFormat +import io.provenance.explorer.model.download.customFormat import org.joda.time.DateTime import org.joda.time.DateTimeZone import java.math.BigDecimal diff --git a/api-model/src/main/kotlin/io/provenance/explorer/model/ValidatorModels.kt b/api-model/src/main/kotlin/io/provenance/explorer/model/ValidatorModels.kt index c9a38f44..40d79fb6 100644 --- a/api-model/src/main/kotlin/io/provenance/explorer/model/ValidatorModels.kt +++ b/api-model/src/main/kotlin/io/provenance/explorer/model/ValidatorModels.kt @@ -46,7 +46,8 @@ data class ValidatorDetails( val status: String, val unbondingHeight: Long?, val jailedUntil: DateTime?, - val removed: Boolean + val removed: Boolean, + val isVerified: Boolean ) data class ValidatorCommission( @@ -144,3 +145,20 @@ data class UptimeDataSet( val avgUptimeCountPercentage: String, val validatorsAtRisk: List ) + +data class ValidatorMetrics( + val operatorAddr: String, + val moniker: String, + val year: Int, + val quarter: Int, + val isActive: Boolean, + val isVerified: Boolean, + val votingMetric: CountTotal, + val uptimeMetrics: CountTotal +) + +data class MetricPeriod( + val label: String, + val year: Int, + val quarter: Int +) diff --git a/api-model/src/main/kotlin/io/provenance/explorer/model/ChartModels.kt b/api-model/src/main/kotlin/io/provenance/explorer/model/download/ChartModels.kt similarity index 96% rename from api-model/src/main/kotlin/io/provenance/explorer/model/ChartModels.kt rename to api-model/src/main/kotlin/io/provenance/explorer/model/download/ChartModels.kt index 9854a79b..e3e7ea4c 100644 --- a/api-model/src/main/kotlin/io/provenance/explorer/model/ChartModels.kt +++ b/api-model/src/main/kotlin/io/provenance/explorer/model/download/ChartModels.kt @@ -1,4 +1,4 @@ -package io.provenance.explorer.model +package io.provenance.explorer.model.download import io.provenance.explorer.model.base.DateTruncGranularity import io.provenance.explorer.model.base.stringfy @@ -15,9 +15,9 @@ fun BigDecimal.currFormat() = currFormat.format(this) fun DateTime.customFormat(granularity: DateTruncGranularity) = when (granularity) { DateTruncGranularity.HOUR, - DateTruncGranularity.MINUTE -> DateTimeFormat.forPattern("yyy-MM-dd hh:mm:ss").print(this) - DateTruncGranularity.DAY -> DateTimeFormat.forPattern("yyy-MM-dd").print(this) - DateTruncGranularity.MONTH -> DateTimeFormat.forPattern("yyy-MM").print(this) + DateTruncGranularity.MINUTE -> DateTimeFormat.forPattern("yyyy-MM-dd hh:mm:ss").print(this) + DateTruncGranularity.DAY -> DateTimeFormat.forPattern("yyyy-MM-dd").print(this) + DateTruncGranularity.MONTH -> DateTimeFormat.forPattern("yyyy-MM").print(this) } data class TxHistoryChartData( diff --git a/api-model/src/main/kotlin/io/provenance/explorer/model/download/ValidatorMetricModels.kt b/api-model/src/main/kotlin/io/provenance/explorer/model/download/ValidatorMetricModels.kt new file mode 100644 index 00000000..9512133a --- /dev/null +++ b/api-model/src/main/kotlin/io/provenance/explorer/model/download/ValidatorMetricModels.kt @@ -0,0 +1,32 @@ +package io.provenance.explorer.model.download + +data class ValidatorMetricData( + val year: Int, + val quarter: Int, + val moniker: String, + val operatorAddress: String, + val isActive: Boolean, + val isVerified: Boolean, + val govVotes: Int, + val govProposals: Int, + val govPercentage: String, + val blocksUp: Int, + val blocksTotal: Int, + val uptimePercentage: String +) { + fun toCsv() = + mutableListOf( + this.year, + this.quarter, + this.moniker, + this.operatorAddress, + this.isActive, + this.isVerified, + this.govVotes, + this.govProposals, + this.govPercentage, + this.blocksUp, + this.blocksTotal, + this.uptimePercentage + ) +} diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt index 9212d5be..91032f41 100644 --- a/buildSrc/src/main/kotlin/Dependencies.kt +++ b/buildSrc/src/main/kotlin/Dependencies.kt @@ -51,7 +51,7 @@ object Versions { const val SpringBoot = PluginVersions.SpringBoot const val Swagger = "3.0.0" const val Grpc = "1.50.2" - const val ProvProto = "1.13.0" + const val ProvProto = "1.14.0-rc2" const val Postgres = "42.2.23" const val Protobuf = "3.21.9" diff --git a/database/src/main/resources/db/migration/V1_88__Validator_delegation_updates.sql b/database/src/main/resources/db/migration/V1_88__Validator_delegation_updates.sql new file mode 100644 index 00000000..6bc337ef --- /dev/null +++ b/database/src/main/resources/db/migration/V1_88__Validator_delegation_updates.sql @@ -0,0 +1,58 @@ +SELECT 'Validator Delegation Program updates' AS comment; + +CREATE INDEX IF NOT EXISTS block_tx_count_cache_block_timestamp_idx on block_tx_count_cache (block_timestamp); + +drop materialized view if exists block_time_spread; +Create MATERIALIZED VIEW if not exists block_time_spread as +select extract(year from block_timestamp) as year, + extract(quarter from block_timestamp) as quarter, + extract(month from block_timestamp) as month, + min(block_height) as min_height, + max(block_height) as max_height, + min(block_timestamp) as min_time, + max(block_timestamp) as max_time, + count(*) as total_blocks +from block_tx_count_cache +group by extract(year from block_timestamp), + extract(quarter from block_timestamp), + extract(month from block_timestamp) +with data; + +create table if not exists validator_metrics +( + id serial primary key, + oper_addr_id integer, + operator_address varchar(128), + year integer, + quarter integer, + data jsonb +); + +create unique index if not exists validator_metrics_unique_idx on validator_metrics (oper_addr_id, year, quarter); + +create index if not exists validator_metrics_oper_id_idx on validator_metrics (oper_addr_id); +create index if not exists validator_metrics_oper_addr_idx on validator_metrics (operator_address); +create index if not exists validator_metrics_period_idx on validator_metrics (year, quarter); + + +-- Fix for +create or replace procedure insert_proposal_monitor(proposalmonitors proposal_monitor[]) + language plpgsql as +$$ +DECLARE + pm proposal_monitor; +BEGIN + FOREACH pm IN ARRAY proposalMonitors + LOOP + INSERT INTO proposal_monitor(proposal_id, submitted_height, proposed_completion_height, voting_end_time, + proposal_type, matching_data_hash) + VALUES (pm.proposal_id, + pm.submitted_height, + pm.proposed_completion_height, + pm.voting_end_time, + pm.proposal_type, + pm.matching_data_hash) + ON CONFLICT (proposal_id, matching_data_hash) DO NOTHING; + END LOOP; +END; +$$; diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt index df6137df..2453ef5e 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt @@ -14,7 +14,9 @@ import io.provenance.explorer.domain.extensions.exec import io.provenance.explorer.domain.extensions.execAndMap import io.provenance.explorer.domain.extensions.map import io.provenance.explorer.domain.extensions.startOfDay +import io.provenance.explorer.domain.extensions.toDateTime import io.provenance.explorer.domain.models.explorer.BlockProposer +import io.provenance.explorer.domain.models.explorer.BlockTimeSpread import io.provenance.explorer.domain.models.explorer.BlockUpdate import io.provenance.explorer.domain.models.explorer.MissedBlockPeriod import io.provenance.explorer.domain.models.explorer.TxHeatmapRaw @@ -42,9 +44,7 @@ import org.jetbrains.exposed.sql.Max import org.jetbrains.exposed.sql.Op import org.jetbrains.exposed.sql.SortOrder import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq -import org.jetbrains.exposed.sql.SqlExpressionBuilder.greaterEq import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList -import org.jetbrains.exposed.sql.SqlExpressionBuilder.lessEq import org.jetbrains.exposed.sql.Sum import org.jetbrains.exposed.sql.VarCharColumnType import org.jetbrains.exposed.sql.and @@ -63,6 +63,7 @@ import org.jetbrains.exposed.sql.transactions.TransactionManager import org.jetbrains.exposed.sql.transactions.transaction import org.joda.time.DateTime import org.joda.time.DateTimeZone +import java.sql.ResultSet object BlockCacheTable : CacheIdTable(name = "block_cache") { val height = integer("height") @@ -457,6 +458,32 @@ class BlockTxCountsCacheRecord(id: EntityID) : IntEntity(id) { val query = "CALL update_block_cache_hourly_tx_counts()" this.exec(query) } + + fun updateSpreadView() = transaction { + val query = "REFRESH MATERIALIZED VIEW block_time_spread" + this.exec(query) + } + + fun getBlockTimeSpread(year: Int, quarter: Int) = transaction { + val query = """ + SELECT + year, + quarter, + min(min_height) AS min_height, + max(max_height) AS max_height, + min(min_time) AS min_time, + max(max_time) AS max_time, + sum(total_blocks) AS total_blocks + FROM block_time_spread + WHERE year = ? AND quarter = ? + GROUP BY year, quarter; + """.trimIndent() + val arguments = mutableListOf>( + Pair(IntegerColumnType(), year), + Pair(IntegerColumnType(), quarter) + ) + query.execAndMap(arguments) { it.toBlockTimeSpread() }.firstOrNull() + } } var blockHeight by BlockTxCountsCacheTable.blockHeight @@ -465,6 +492,16 @@ class BlockTxCountsCacheRecord(id: EntityID) : IntEntity(id) { var processed by BlockTxCountsCacheTable.processed } +fun ResultSet.toBlockTimeSpread() = BlockTimeSpread( + this.getInt("year"), + this.getInt("quarter"), + this.getInt("min_height"), + this.getInt("max_height"), + this.getTimestamp("min_time").toDateTime(), + this.getTimestamp("max_time").toDateTime(), + this.getInt("total_blocks") +) + object BlockTxRetryTable : IdTable(name = "block_tx_retry") { val height = integer("height") val retried = bool("retried").default(false) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Governance.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Governance.kt index 1f8eabca..39a4aba1 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Governance.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Governance.kt @@ -9,7 +9,7 @@ import io.provenance.explorer.domain.core.sql.ArrayAgg import io.provenance.explorer.domain.core.sql.jsonb import io.provenance.explorer.domain.core.sql.toProcedureObject import io.provenance.explorer.domain.extensions.stringify -import io.provenance.explorer.domain.extensions.toDecimal +import io.provenance.explorer.domain.extensions.toDecimalNew import io.provenance.explorer.domain.extensions.toObjectNodeList import io.provenance.explorer.domain.models.explorer.AddrData import io.provenance.explorer.domain.models.explorer.GovContentV1List @@ -96,6 +96,14 @@ class GovProposalRecord(id: EntityID) : IntEntity(id) { GovProposalRecord.find { GovProposalTable.status notInList completeStatuses }.toList() } + fun getCompletedProposalsForPeriod(startDate: DateTime, endDate: DateTime) = transaction { + GovProposalRecord.find { + (GovProposalTable.txTimestamp.between(startDate, endDate)) and + (GovProposalTable.status inList completeStatuses) and + (GovProposalTable.votingParamCheckHeight neq -1) + }.toMutableList() + } + fun buildInsert( proposal: GovV1.Proposal, protoPrinter: JsonFormat.Printer, @@ -288,6 +296,11 @@ class GovVoteRecord(id: EntityID) : IntEntity(id) { GovVoteRecord.find { GovVoteTable.addressId eq addrId }.groupBy { GovVoteTable.proposalId }.count() } + fun getAddressVotesForProposalList(addrId: Int, list: List) = transaction { + GovVoteRecord.find { (GovVoteTable.proposalId inList list) and (GovVoteTable.addressId eq addrId) } + .toMutableList() + } + fun buildInsert( txInfo: TxData, votes: List, @@ -307,7 +320,7 @@ class GovVoteRecord(id: EntityID) : IntEntity(id) { txInfo.txHash, txInfo.txTimestamp, 0, - it.weight.toDecimal(), + it.weight.toDecimalNew(), justification ).toProcedureObject() } diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Name.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Name.kt index f59b6829..eff7b6a2 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Name.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Name.kt @@ -55,6 +55,10 @@ class NameRecord(id: EntityID) : IntEntity(id) { NameRecord.find { (NameTable.fullName eq fullName) and (NameTable.owner eq owner) }.firstOrNull() } + fun getAllNamesForParent(parent: String) = transaction { + NameRecord.find { NameTable.parent like "%$parent" }.toList() + } + fun getNameSet() = transaction { val query = """ with data as ( diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Transactions.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Transactions.kt index 32af41a4..7bdccb69 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Transactions.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Transactions.kt @@ -407,53 +407,43 @@ class TxMessageRecord(id: EntityID) : IntEntity(id) { private fun findByQueryParams(tqp: TxQueryParams, distinctQuery: List>?) = transaction { var join: ColumnSet = TxMessageTable - if (tqp.msgTypes.isNotEmpty()) { - join = + if (tqp.msgTypes.isNotEmpty()) + join = if (tqp.primaryTypesOnly) + join.innerJoin(TxMsgTypeSubtypeTable, { TxMessageTable.txHashId }, { TxMsgTypeSubtypeTable.txHashId }) + else join.innerJoin(TxMsgTypeQueryTable, { TxMessageTable.txHashId }, { TxMsgTypeQueryTable.txHashId }) - } - if (tqp.txStatus != null) { + if (tqp.txStatus != null) join = join.innerJoin(TxCacheTable, { TxMessageTable.txHashId }, { TxCacheTable.id }) - } - if ((tqp.addressId != null && tqp.addressType != null) || tqp.address != null) { + if ((tqp.addressId != null && tqp.addressType != null) || tqp.address != null) join = join.innerJoin(TxAddressJoinTable, { TxMessageTable.txHashId }, { TxAddressJoinTable.txHashId }) - } - if (tqp.smCodeId != null) { + if (tqp.smCodeId != null) join = join.innerJoin(TxSmCodeTable, { TxMessageTable.txHashId }, { TxSmCodeTable.txHashId }) - } - if (tqp.smContractAddrId != null) { + if (tqp.smContractAddrId != null) join = join.innerJoin(TxSmContractTable, { TxMessageTable.txHashId }, { TxSmContractTable.txHashId }) - } val query = if (distinctQuery != null) join.slice(distinctQuery).selectAll() else join.selectAll() - if (tqp.msgTypes.isNotEmpty()) { - query.andWhere { TxMsgTypeQueryTable.typeId inList tqp.msgTypes } - } - if (tqp.txHeight != null) { + if (tqp.msgTypes.isNotEmpty()) + if (tqp.primaryTypesOnly) query.andWhere { TxMsgTypeSubtypeTable.primaryType inList tqp.msgTypes } + else query.andWhere { TxMsgTypeQueryTable.typeId inList tqp.msgTypes } + if (tqp.txHeight != null) query.andWhere { TxMessageTable.blockHeight eq tqp.txHeight } - } - if (tqp.txStatus != null) { + if (tqp.txStatus != null) query.andWhere { if (tqp.txStatus == TxStatus.FAILURE) TxCacheTable.errorCode neq 0 else TxCacheTable.errorCode.isNull() } - } - if (tqp.addressId != null && tqp.addressType != null) { + if (tqp.addressId != null && tqp.addressType != null) query.andWhere { (TxAddressJoinTable.addressId eq tqp.addressId) and (TxAddressJoinTable.addressType eq tqp.addressType) } - } else if (tqp.address != null) { + else if (tqp.address != null) query.andWhere { (TxAddressJoinTable.address eq tqp.address) } - } - if (tqp.smCodeId != null) { + if (tqp.smCodeId != null) query.andWhere { (TxSmCodeTable.smCode eq tqp.smCodeId) } - } - if (tqp.smContractAddrId != null) { + if (tqp.smContractAddrId != null) query.andWhere { (TxSmContractTable.contractId eq tqp.smContractAddrId) } - } - if (tqp.fromDate != null) { + if (tqp.fromDate != null) query.andWhere { TxMessageTable.txTimestamp greaterEq tqp.fromDate.startOfDay() } - } - if (tqp.toDate != null) { + if (tqp.toDate != null) query.andWhere { TxMessageTable.txTimestamp lessEq tqp.toDate.startOfDay().plusDays(1) } - } query } diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/TxHistoryDataViews.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/TxHistoryDataViews.kt index 29f7d83b..9d74654c 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/TxHistoryDataViews.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/TxHistoryDataViews.kt @@ -2,9 +2,9 @@ package io.provenance.explorer.domain.entities import io.provenance.explorer.config.ExplorerProperties.Companion.UTILITY_TOKEN_BASE_MULTIPLIER import io.provenance.explorer.domain.extensions.execAndMap -import io.provenance.explorer.domain.models.explorer.toFeeTypeData -import io.provenance.explorer.domain.models.explorer.toTxHistoryChartData -import io.provenance.explorer.domain.models.explorer.toTxTypeData +import io.provenance.explorer.domain.models.explorer.download.toFeeTypeData +import io.provenance.explorer.domain.models.explorer.download.toTxHistoryChartData +import io.provenance.explorer.domain.models.explorer.download.toTxTypeData import io.provenance.explorer.model.base.DateTruncGranularity import org.jetbrains.exposed.sql.ColumnType import org.jetbrains.exposed.sql.IntegerColumnType diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Validators.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Validators.kt index f3e2d512..74255a4d 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Validators.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Validators.kt @@ -13,6 +13,9 @@ import io.provenance.explorer.domain.extensions.mapper import io.provenance.explorer.domain.extensions.toDecimal import io.provenance.explorer.domain.models.explorer.CurrentValidatorState import io.provenance.explorer.domain.models.explorer.TxData +import io.provenance.explorer.domain.models.explorer.download.toValidatorMetricData +import io.provenance.explorer.model.MetricPeriod +import io.provenance.explorer.model.ValidatorMetrics import io.provenance.explorer.model.ValidatorState import io.provenance.explorer.model.ValidatorState.ACTIVE import io.provenance.explorer.model.ValidatorState.ALL @@ -25,6 +28,7 @@ import org.jetbrains.exposed.dao.id.EntityID import org.jetbrains.exposed.dao.id.IntIdTable import org.jetbrains.exposed.sql.ColumnType import org.jetbrains.exposed.sql.IntegerColumnType +import org.jetbrains.exposed.sql.ResultRow import org.jetbrains.exposed.sql.SortOrder import org.jetbrains.exposed.sql.TextColumnType import org.jetbrains.exposed.sql.VarCharColumnType @@ -35,6 +39,7 @@ import org.jetbrains.exposed.sql.insertIgnoreAndGetId import org.jetbrains.exposed.sql.jodatime.datetime import org.jetbrains.exposed.sql.leftJoin import org.jetbrains.exposed.sql.select +import org.jetbrains.exposed.sql.selectAll import org.jetbrains.exposed.sql.transactions.transaction import org.joda.time.DateTime import java.math.BigDecimal @@ -392,3 +397,82 @@ class ValidatorMarketRateRecord(id: EntityID) : IntEntity(id) { var marketRate by ValidatorMarketRateTable.marketRate var success by ValidatorMarketRateTable.success } + +object ValidatorMetricsTable : IntIdTable(name = "validator_metrics") { + val operAddrId = integer("oper_addr_id") + val operatorAddress = varchar("operator_address", 128) + val year = integer("year") + val quarter = integer("quarter") + val data = jsonb("data", OBJECT_MAPPER) +} + +class ValidatorMetricsRecord(id: EntityID) : IntEntity(id) { + companion object : IntEntityClass(ValidatorMetricsTable) { + + fun findByOperAddrForPeriod(operAddrId: Int, year: Int, quarter: Int) = transaction { + ValidatorMetricsRecord.find { (ValidatorMetricsTable.operAddrId eq operAddrId) and (ValidatorMetricsTable.year eq year) and (ValidatorMetricsTable.quarter eq quarter) } + .firstOrNull() + } + + fun findByOperAddr(operAddrId: Int) = transaction { + ValidatorMetricsRecord.find { ValidatorMetricsTable.operAddrId eq operAddrId }.toMutableList() + } + + fun findDistinctPeriods(currYear: Int, currQuarter: Int) = transaction { + ValidatorMetricsTable + .slice(ValidatorMetricsTable.year, ValidatorMetricsTable.quarter) + .selectAll() + .groupBy(ValidatorMetricsTable.year, ValidatorMetricsTable.quarter) + .map { + MetricPeriod( + it.toMetricPeriodLabel(it[ValidatorMetricsTable.year] == currYear && it[ValidatorMetricsTable.quarter] == currQuarter), + it[ValidatorMetricsTable.year], + it[ValidatorMetricsTable.quarter] + ) + }.toMutableList() + } + + fun ResultRow.toMetricPeriodLabel(isCurrent: Boolean) = + "${this[ValidatorMetricsTable.year]} Q${this[ValidatorMetricsTable.quarter]}" + (if (isCurrent) " - Current" else "") + + fun getDataForPeriod(year: Int, quarter: Int) = + transaction { + val query = """ + SELECT + year, + quarter, + data -> 'moniker'::text AS moniker, + operator_address, + (data -> 'is_active')::boolean AS is_active, + (data -> 'is_verified')::boolean AS is_verified, + (data -> 'voting_metric' -> 'count')::integer AS gov_vote, + (data -> 'voting_metric' -> 'total')::integer AS gov_proposal, + (data -> 'uptime_metrics' -> 'count')::integer AS blocks_up, + (data -> 'uptime_metrics' -> 'total')::integer AS blocks_total + FROM validator_metrics + WHERE year = ? AND quarter = ?; + """.trimIndent() + val arguments = mutableListOf(Pair(IntegerColumnType(), year), Pair(IntegerColumnType(), quarter)) + query.execAndMap(arguments) { it.toValidatorMetricData() } + } + + fun insertIgnore(operId: Int, operator: String, year: Int, quarter: Int, data: ValidatorMetrics) = + transaction { + ValidatorMetricsTable.insertIgnore { + it[this.operAddrId] = operId + it[this.operatorAddress] = operator + it[this.year] = year + it[this.quarter] = quarter + it[this.data] = data + } + } + } + + fun toMetricPeriodLabel(isCurrent: Boolean) = "$year Q$quarter" + (if (isCurrent) " - Current" else "") + + var operAddrId by ValidatorMetricsTable.operAddrId + var operatorAddress by ValidatorMetricsTable.operatorAddress + var year by ValidatorMetricsTable.year + var quarter by ValidatorMetricsTable.quarter + var data by ValidatorMetricsTable.data +} diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/extensions/CoinExtensions.kt b/service/src/main/kotlin/io/provenance/explorer/domain/extensions/CoinExtensions.kt index 5e2ac9f0..c31e6137 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/extensions/CoinExtensions.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/extensions/CoinExtensions.kt @@ -35,11 +35,16 @@ fun String.toPercentageOld() = fun String.toPercentage() = BigDecimal(this).multiply(BigDecimal(100)).stringfyWithScale(4) + "%" +// Used with protos that include `(gogoproto.customtype) = "github.com/cosmos/cosmos-sdk/types.Dec",` in descripion +// example: v1beta1.gov WeightedVote weight value fun String.toDecimal() = BigDecimal(this.toBigInteger(), UTILITY_TOKEN_BASE_DECIMAL_PLACES * 2).stripTrailingZeros() +// Used with protos that exclude `(gogoproto.customtype) = "github.com/cosmos/cosmos-sdk/types.Dec",` from description +// example: v1.gov WeightedVote weight value +fun String.toDecimalNew() = BigDecimal(this).stripTrailingZeros() + fun Double.toPercentage() = "${this * 100}%" -// fun BigDecimal.toPercentage() = "${this * 100}%" fun List.avg() = this.sum() / this.size @@ -49,6 +54,8 @@ fun BigDecimal.percentChange(orig: BigDecimal) = fun Int.padToDecString() = BigDecimal(this).multiply(BigDecimal("1e${(UTILITY_TOKEN_BASE_DECIMAL_PLACES - 1) * 2}")).toPlainString() +fun Int.padToDecStringNew() = BigDecimal(this).divide(BigDecimal(100), 4, RoundingMode.HALF_EVEN).toPlainString() + fun List.isZero(): Boolean { this.forEach { if (it.amount.toBigDecimal() != BigDecimal.ZERO) { @@ -75,6 +82,12 @@ fun String.toPercentage(num: BigDecimal, den: BigDecimal, scale: Int) = .multiply(num) .stringfyWithScale(scale) + "%" +fun Int.toPercentage(num: Int, den: Int, scale: Int) = + this.toBigDecimal() + .divide(den.toBigDecimal(), 100, RoundingMode.HALF_EVEN) + .multiply(num.toBigDecimal()) + .stringfyWithScale(scale) + "%" + // Calcs the difference between this (oldList) of denoms and the newList of denoms fun List.diff(newList: List) = if (this.isEmpty()) { diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/extensions/DbExtensions.kt b/service/src/main/kotlin/io/provenance/explorer/domain/extensions/DbExtensions.kt index 84961226..25a5cb46 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/extensions/DbExtensions.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/extensions/DbExtensions.kt @@ -12,6 +12,8 @@ import java.sql.Timestamp fun Timestamp.toDateTime(newZone: DateTimeZone, pattern: String): String = DateTime(this.time).withZone(newZone).toString(pattern) +fun Timestamp.toDateTime() = DateTime(this.time) + fun String.exec(args: Iterable>): ResultSet = with(TransactionManager.current().connection.prepareStatement(this, false)) { this.fillParameters(args) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/extensions/Extenstions.kt b/service/src/main/kotlin/io/provenance/explorer/domain/extensions/Extenstions.kt index 1872ef13..647f3f25 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/extensions/Extenstions.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/extensions/Extenstions.kt @@ -19,6 +19,7 @@ import io.provenance.explorer.config.ExplorerProperties.Companion.PROV_ACC_PREFI import io.provenance.explorer.config.ExplorerProperties.Companion.PROV_VAL_CONS_PREFIX import io.provenance.explorer.config.ExplorerProperties.Companion.PROV_VAL_OPER_PREFIX import io.provenance.explorer.domain.entities.MissedBlocksRecord +import io.provenance.explorer.domain.exceptions.InvalidArgumentException import io.provenance.explorer.domain.models.explorer.Addresses import io.provenance.explorer.model.base.Bech32 import io.provenance.explorer.model.base.toBech32Data @@ -73,6 +74,11 @@ fun String.validatorMissedBlocks(blockWindow: BigInteger, currentHeight: BigInte .sumOf { it.blocks.count() } ).let { mbCount -> Pair(mbCount, blockWindow) } +fun String.validatorMissedBlocksSpecific(fromHeight: Int, toHeight: Int) = + MissedBlocksRecord + .findValidatorsWithMissedBlocksForPeriod(fromHeight, toHeight, this) + .sumOf { it.blocks.count() } + fun Long.isPastDue(currentMillis: Long) = DateTime.now().millis - this > currentMillis // translates page (this) to offset @@ -138,6 +144,14 @@ fun DateTime.startOfDay() = this.withZone(DateTimeZone.UTC).withTimeAtStartOfDay fun String.toDateTime() = DateTime.parse(this) fun String.toDateTimeWithFormat(formatter: org.joda.time.format.DateTimeFormatter) = DateTime.parse(this, formatter) fun OffsetDateTime.toDateTime() = DateTime(this.toInstant().toEpochMilli(), DateTimeZone.UTC) +fun Int.monthToQuarter() = + when { + (1..3).contains(this) -> 1 + (4..6).contains(this) -> 2 + (7..9).contains(this) -> 3 + (10..12).contains(this) -> 4 + else -> throw InvalidArgumentException("Not a valid month: $this") + } fun ServiceOuterClass.GetTxResponse.success() = this.txResponse.code == 0 fun BlockOuterClass.Block.height() = this.header.height.toInt() diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/ExplorerModels.kt b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/ExplorerModels.kt index 29a4a713..95346ca2 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/ExplorerModels.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/ExplorerModels.kt @@ -15,3 +15,13 @@ data class GithubReleaseData( val createdAt: String, val releaseUrl: String ) + +data class BlockTimeSpread( + val year: Int, + val quarter: Int, + val minHeight: Int, + val maxHeight: Int, + val minTimestamp: DateTime, + val maxTimestamp: DateTime, + val totalBlocks: Int +) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/TokenModels.kt b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/TokenModels.kt index b44ec2b1..4c6dc211 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/TokenModels.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/TokenModels.kt @@ -1,6 +1,7 @@ package io.provenance.explorer.domain.models.explorer import io.provenance.explorer.domain.entities.TokenHistoricalDailyRecord +import io.provenance.explorer.domain.exceptions.requireToMessage import io.provenance.explorer.domain.extensions.CsvData import io.provenance.explorer.domain.extensions.startOfDay import io.provenance.explorer.model.base.CountStrTotal @@ -29,23 +30,29 @@ data class DlobHistorical( val type: String ) -fun getFileListToken(filters: TokenHistoricalDataRequest): MutableList = - mutableListOf( - CsvData( - "TokenHistoricalData", - tokenHistoricalCsvBaseHeaders(), - TokenHistoricalDailyRecord.findForDates(filters.fromDate?.startOfDay(), filters.toDate?.startOfDay()) - .map { it.toCsv() } - ) - ) - -fun tokenHistoricalCsvBaseHeaders(): MutableList = - mutableListOf("Date", "Open", "High", "Low", "Close", "Volume - USD") - data class TokenHistoricalDataRequest( val fromDate: DateTime? = null, val toDate: DateTime? = null ) { + fun getFileList(): MutableList = + mutableListOf( + CsvData( + "TokenHistoricalData", + tokenHistoricalCsvBaseHeaders, + TokenHistoricalDailyRecord.findForDates(fromDate?.startOfDay(), toDate?.startOfDay()).map { it.toCsv() } + ) + ) + + private val tokenHistoricalCsvBaseHeaders: MutableList = + mutableListOf("Date", "Open", "High", "Low", "Close", "Volume - USD") + + fun datesValidation() = + if (fromDate != null && toDate != null) { + requireToMessage(fromDate.isBefore(toDate)) { "'fromDate' ($fromDate) must be before 'toDate' ($toDate)" } + } else { + null + } + private val dateFormat = DateTimeFormat.forPattern("yyy-MM-dd") fun getFileNameBase(): String { diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/TransactionModels.kt b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/TransactionModels.kt index de3674cf..c4e607ab 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/TransactionModels.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/TransactionModels.kt @@ -42,6 +42,7 @@ data class TxQueryParams( val address: String? = null, val markerId: Int? = null, val denom: String? = null, + val primaryTypesOnly: Boolean = false, val msgTypes: List = emptyList(), val txHeight: Int? = null, val txStatus: TxStatus? = null, diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/TxHistoryDataViewModels.kt b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/download/TxHistoryDataViewModels.kt similarity index 68% rename from service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/TxHistoryDataViewModels.kt rename to service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/download/TxHistoryDataViewModels.kt index acd24be4..e0d3c900 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/TxHistoryDataViewModels.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/download/TxHistoryDataViewModels.kt @@ -1,49 +1,18 @@ -package io.provenance.explorer.domain.models.explorer +package io.provenance.explorer.domain.models.explorer.download import io.provenance.explorer.domain.entities.TxHistoryDataViews import io.provenance.explorer.domain.exceptions.requireToMessage import io.provenance.explorer.domain.extensions.CsvData -import io.provenance.explorer.model.FeeTypeData -import io.provenance.explorer.model.TxHistoryChartData -import io.provenance.explorer.model.TxTypeData import io.provenance.explorer.model.base.DateTruncGranularity +import io.provenance.explorer.model.download.FeeTypeData +import io.provenance.explorer.model.download.TxHistoryChartData +import io.provenance.explorer.model.download.TxTypeData import org.joda.time.DateTime import org.joda.time.format.DateTimeFormat import java.io.ByteArrayOutputStream import java.io.PrintWriter import java.sql.ResultSet -fun getFileList(filters: TxHistoryDataRequest, feepayer: String?): MutableList { - val hasFeepayer = feepayer != null - val fileList = mutableListOf( - CsvData( - "TxHistoryChartData", - txHistoryDataCsvBaseHeaders(filters.advancedMetrics, hasFeepayer), - TxHistoryDataViews.getTxHistoryChartData(filters.granularity, filters.fromDate, filters.toDate, feepayer) - .map { it.toCsv(filters.advancedMetrics, hasFeepayer, filters.granularity) } - ) - ) - if (filters.advancedMetrics) { - fileList.add( - CsvData( - "TxTypeData", - txTypeDataCsvBaseHeaders(hasFeepayer), - TxHistoryDataViews.getTxTypeData(filters.granularity, filters.fromDate, filters.toDate, feepayer) - .map { it.toCsv(hasFeepayer, filters.granularity) } - ) - ) - fileList.add( - CsvData( - "FeeTypeData", - feeTypeDataCsvBaseHeaders(hasFeepayer), - TxHistoryDataViews.getFeeTypeData(filters.granularity, filters.fromDate, filters.toDate, feepayer) - .map { it.toCsv(hasFeepayer, filters.granularity) } - ) - ) - } - return fileList -} - //region TxHistoryChart fun ResultSet.toTxHistoryChartData(byFeepayer: Boolean) = TxHistoryChartData( @@ -114,25 +83,56 @@ fun feeTypeDataCsvBaseHeaders(hasFeepayer: Boolean): MutableList { //region TxHistory API Request Bodies -fun granularityValidation(granularity: DateTruncGranularity) = - requireToMessage( - listOf(DateTruncGranularity.MONTH, DateTruncGranularity.DAY, DateTruncGranularity.HOUR).contains(granularity) - ) { "The specified granularity is not supported: $granularity" } - -fun datesValidation(fromDate: DateTime?, toDate: DateTime?) = - if (fromDate != null && toDate != null) { - requireToMessage(fromDate.isBefore(toDate)) { "'fromDate' ($fromDate) must be before 'toDate' ($toDate)" } - } else { - null - } - data class TxHistoryDataRequest( val fromDate: DateTime? = null, val toDate: DateTime? = null, val granularity: DateTruncGranularity = DateTruncGranularity.DAY, val advancedMetrics: Boolean = false ) { - private val dateFormat = DateTimeFormat.forPattern("yyy-MM-dd") + fun granularityValidation() = + requireToMessage( + listOf(DateTruncGranularity.MONTH, DateTruncGranularity.DAY, DateTruncGranularity.HOUR).contains(granularity) + ) { "The specified granularity is not supported: $granularity" } + + fun datesValidation() = + if (fromDate != null && toDate != null) { + requireToMessage(fromDate.isBefore(toDate)) { "'fromDate' ($fromDate) must be before 'toDate' ($toDate)" } + } else { + null + } + + fun getFileList(feepayer: String?): MutableList { + val hasFeepayer = feepayer != null + val fileList = mutableListOf( + CsvData( + "TxHistoryChartData", + txHistoryDataCsvBaseHeaders(advancedMetrics, hasFeepayer), + TxHistoryDataViews.getTxHistoryChartData(granularity, fromDate, toDate, feepayer) + .map { it.toCsv(advancedMetrics, hasFeepayer, granularity) } + ) + ) + if (advancedMetrics) { + fileList.add( + CsvData( + "TxTypeData", + txTypeDataCsvBaseHeaders(hasFeepayer), + TxHistoryDataViews.getTxTypeData(granularity, fromDate, toDate, feepayer) + .map { it.toCsv(hasFeepayer, granularity) } + ) + ) + fileList.add( + CsvData( + "FeeTypeData", + feeTypeDataCsvBaseHeaders(hasFeepayer), + TxHistoryDataViews.getFeeTypeData(granularity, fromDate, toDate, feepayer) + .map { it.toCsv(hasFeepayer, granularity) } + ) + ) + } + return fileList + } + + private val dateFormat = DateTimeFormat.forPattern("yyyy-MM-dd") fun getFileNameBase(feepayer: String?): String { val to = if (toDate != null) dateFormat.print(toDate) else "CURRENT" diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/download/ValidatorMetricModels.kt b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/download/ValidatorMetricModels.kt new file mode 100644 index 00000000..a0315f11 --- /dev/null +++ b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/download/ValidatorMetricModels.kt @@ -0,0 +1,60 @@ +package io.provenance.explorer.domain.models.explorer.download + +import io.provenance.explorer.domain.entities.ValidatorMetricsRecord +import io.provenance.explorer.domain.extensions.CsvData +import io.provenance.explorer.domain.extensions.toPercentage +import io.provenance.explorer.model.download.ValidatorMetricData +import org.jetbrains.exposed.sql.transactions.transaction +import java.sql.ResultSet + +//region ValidatorMetrics + +fun ResultSet.toValidatorMetricData() = ValidatorMetricData( + this.getInt("year"), + this.getInt("quarter"), + this.getString("moniker"), + this.getString("operator_address"), + this.getBoolean("is_active"), + this.getBoolean("is_verified"), + this.getInt("gov_vote"), + this.getInt("gov_proposal"), + this.getInt("gov_vote").toPercentage(100, this.getInt("gov_proposal"), 4), + this.getInt("blocks_up"), + this.getInt("blocks_total"), + this.getInt("blocks_up").toPercentage(100, this.getInt("blocks_total"), 4) +) + +//endregion + +//region Validator Metric Body + +data class ValidatorMetricsRequest(val year: Int, val quarter: Int) { + + fun getFilenameBase() = "$year Q$quarter Validator Metrics" + + fun getFile() = transaction { + CsvData( + "Base", + valMetricsDataCsvHeaders, + ValidatorMetricsRecord.getDataForPeriod(year, quarter).map { it.toCsv() } + ) + } + + private val valMetricsDataCsvHeaders: MutableList = + mutableListOf( + "Year", + "Quarter", + "Validator Moniker", + "Validator Address", + "Is Active", + "Is KYC/AML Verified", + "Gov Vote Count", + "Gov Proposal Count", + "Gov Participation %", + "Uptime Block Count", + "Total Blocks for Period", + "Quarterly Uptime %" + ) +} + +//endregion diff --git a/service/src/main/kotlin/io/provenance/explorer/grpc/extensions/MsgConverter.kt b/service/src/main/kotlin/io/provenance/explorer/grpc/extensions/MsgConverter.kt index 940105b1..8ec612ce 100644 --- a/service/src/main/kotlin/io/provenance/explorer/grpc/extensions/MsgConverter.kt +++ b/service/src/main/kotlin/io/provenance/explorer/grpc/extensions/MsgConverter.kt @@ -790,16 +790,17 @@ fun Any.toSoftwareUpgradeProposal() = this.unpack(SoftwareUpgradeProposal::class enum class GovMsgType { PROPOSAL, VOTE, WEIGHTED, DEPOSIT } -fun Any.getAssociatedGovMsgs() = +fun Any.getAssociatedGovMsgs(): List>? = when { typeUrl.endsWith("gov.v1beta1.MsgSubmitProposal") - || typeUrl.endsWith("gov.v1.MsgSubmitProposal") -> GovMsgType.PROPOSAL to this + || typeUrl.endsWith("gov.v1.MsgSubmitProposal") -> listOf(GovMsgType.PROPOSAL to this) typeUrl.endsWith("gov.v1beta1.MsgVote") - || typeUrl.endsWith("gov.v1.MsgVote") -> GovMsgType.VOTE to this + || typeUrl.endsWith("gov.v1.MsgVote") -> listOf(GovMsgType.VOTE to this) typeUrl.endsWith("gov.v1beta1.MsgVoteWeighted") - || typeUrl.endsWith("gov.v1.MsgVoteWeighted") -> GovMsgType.WEIGHTED to this + || typeUrl.endsWith("gov.v1.MsgVoteWeighted") -> listOf(GovMsgType.WEIGHTED to this) typeUrl.endsWith("gov.v1beta1.MsgDeposit") - || typeUrl.endsWith("gov.v1.MsgDeposit") -> GovMsgType.DEPOSIT to this + || typeUrl.endsWith("gov.v1.MsgDeposit") -> listOf(GovMsgType.DEPOSIT to this) + typeUrl.endsWith("authz.v1beta1.MsgExec") -> this.toMsgExec().msgsList.mapNotNull { it.getAssociatedGovMsgs() }.flatten() else -> null.also { logger().debug("This typeUrl is not a governance-based msg: $typeUrl") } } @@ -1020,7 +1021,6 @@ enum class MsgToDefinedEvent(val msg: String, val definedEvent: String, val uniq ), NAME_BIND("/provenance.name.v1.MsgBindNameRequest", "provenance.name.v1.EventNameBound", "address"), PROPOSAL_SUBMIT_V1BETA1("/cosmos.gov.v1beta1.MsgSubmitProposal", "submit_proposal", "proposal_id"), - PROPOSAL_SUBMIT_V1("/cosmos.gov.v1.MsgSubmitProposal", "submit_proposal", "proposal_id"), MARKER_ADD("/provenance.marker.v1.MsgAddMarkerRequest", "provenance.marker.v1.EventMarkerAdd", "denom") } diff --git a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/TransactionGrpcClient.kt b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/TransactionGrpcClient.kt index 8228a0d6..39d61225 100644 --- a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/TransactionGrpcClient.kt +++ b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/TransactionGrpcClient.kt @@ -54,6 +54,7 @@ class TransactionGrpcClient(channelUri: URI) { getTxsEventRequest { this.events.add("tx.height=$height") this.limit = limit.toLong() + this.page = page.toLong() } ).let { if (it.txResponsesList.isEmpty()) { diff --git a/service/src/main/kotlin/io/provenance/explorer/service/AccountService.kt b/service/src/main/kotlin/io/provenance/explorer/service/AccountService.kt index 3b5c617c..23921c20 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/AccountService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/AccountService.kt @@ -39,10 +39,7 @@ import io.provenance.explorer.domain.extensions.toOffset import io.provenance.explorer.domain.extensions.toProtoCoin import io.provenance.explorer.domain.extensions.typeToLabel import io.provenance.explorer.domain.models.explorer.AddrData -import io.provenance.explorer.domain.models.explorer.TxHistoryDataRequest -import io.provenance.explorer.domain.models.explorer.datesValidation -import io.provenance.explorer.domain.models.explorer.getFileList -import io.provenance.explorer.domain.models.explorer.granularityValidation +import io.provenance.explorer.domain.models.explorer.download.TxHistoryDataRequest import io.provenance.explorer.domain.models.explorer.toCoinStr import io.provenance.explorer.domain.models.explorer.toCoinStrWithPrice import io.provenance.explorer.grpc.extensions.getModuleAccName @@ -64,12 +61,12 @@ import io.provenance.explorer.model.Delegation import io.provenance.explorer.model.DenomBalanceBreakdown import io.provenance.explorer.model.Reward import io.provenance.explorer.model.TokenCounts -import io.provenance.explorer.model.TxHistoryChartData import io.provenance.explorer.model.UnpaginatedDelegation import io.provenance.explorer.model.base.CoinStr import io.provenance.explorer.model.base.PagedResults import io.provenance.explorer.model.base.PeriodInSeconds import io.provenance.explorer.model.base.USD_UPPER +import io.provenance.explorer.model.download.TxHistoryChartData import kotlinx.coroutines.async import kotlinx.coroutines.runBlocking import org.jetbrains.exposed.sql.SortOrder @@ -373,8 +370,8 @@ class AccountService( fun getAccountTxHistoryChartData(feepayer: String, filters: TxHistoryDataRequest): List { validate( - granularityValidation(filters.granularity), - datesValidation(filters.fromDate, filters.toDate) + filters.granularityValidation(), + filters.datesValidation() ) return TxHistoryDataViews.getTxHistoryChartData(filters.granularity, filters.fromDate, filters.toDate, feepayer) } @@ -385,12 +382,12 @@ class AccountService( outputStream: ServletOutputStream ): ZipOutputStream { validate( - granularityValidation(filters.granularity), - datesValidation(filters.fromDate, filters.toDate), + filters.granularityValidation(), + filters.datesValidation(), validateAddress(feepayer) ) val baseFileName = filters.getFileNameBase(feepayer) - val fileList = getFileList(filters, feepayer) + val fileList = filters.getFileList(feepayer) val zos = ZipOutputStream(outputStream) fileList.forEach { file -> diff --git a/service/src/main/kotlin/io/provenance/explorer/service/GovService.kt b/service/src/main/kotlin/io/provenance/explorer/service/GovService.kt index 8ecab97e..48d282f1 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/GovService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/GovService.kt @@ -50,7 +50,7 @@ import io.provenance.explorer.domain.extensions.getType import io.provenance.explorer.domain.extensions.mapToProtoCoin import io.provenance.explorer.domain.extensions.mhashToNhash import io.provenance.explorer.domain.extensions.pack -import io.provenance.explorer.domain.extensions.padToDecString +import io.provenance.explorer.domain.extensions.padToDecStringNew import io.provenance.explorer.domain.extensions.pageCountOfResults import io.provenance.explorer.domain.extensions.to256Hash import io.provenance.explorer.domain.extensions.toBase64 @@ -555,7 +555,7 @@ class GovService( request.votes.filter { it.weight > 0 }.map { weightedVoteOption { option = it.option - weight = it.weight.padToDecString() + weight = it.weight.padToDecStringNew() } } ) @@ -691,6 +691,12 @@ class GovService( }.pack() } } + + fun getVotesPerProposalsMetrics(addrId: Int, startDate: DateTime, endDate: DateTime) = transaction { + val props = GovProposalRecord.getCompletedProposalsForPeriod(startDate, endDate).map { it.proposalId } + val votes = GovVoteRecord.getAddressVotesForProposalList(addrId, props) + votes.size to props.size + } } fun Any.getGovMsgDetail(txHash: String) = @@ -790,14 +796,14 @@ fun Any.toWeightedVoteList() = this.typeUrl.endsWith("gov.v1beta1.MsgVote") -> listOf( weightedVoteOption { - this.weight = "1000000000000000000" + this.weight = "1" this.option = VoteOption.valueOf(this@toWeightedVoteList.toMsgVoteOld().option.name) } ) this.typeUrl.endsWith("gov.v1.MsgVote") -> listOf( weightedVoteOption { - this.weight = "1000000000000000000" + this.weight = "1" this.option = this@toWeightedVoteList.toMsgVote().option } ) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/MetricsService.kt b/service/src/main/kotlin/io/provenance/explorer/service/MetricsService.kt new file mode 100644 index 00000000..a5eb2fed --- /dev/null +++ b/service/src/main/kotlin/io/provenance/explorer/service/MetricsService.kt @@ -0,0 +1,88 @@ +package io.provenance.explorer.service + +import io.provenance.explorer.config.ResourceNotFoundException +import io.provenance.explorer.domain.entities.AccountRecord +import io.provenance.explorer.domain.entities.BlockTxCountsCacheRecord +import io.provenance.explorer.domain.entities.ValidatorMetricsRecord +import io.provenance.explorer.domain.extensions.monthToQuarter +import io.provenance.explorer.domain.extensions.validatorMissedBlocksSpecific +import io.provenance.explorer.domain.models.explorer.BlockTimeSpread +import io.provenance.explorer.domain.models.explorer.CurrentValidatorState +import io.provenance.explorer.domain.models.explorer.download.ValidatorMetricsRequest +import io.provenance.explorer.model.MetricPeriod +import io.provenance.explorer.model.ValidatorMetrics +import io.provenance.explorer.model.ValidatorState +import io.provenance.explorer.model.base.CountTotal +import org.jetbrains.exposed.sql.transactions.transaction +import org.joda.time.DateTime +import org.springframework.stereotype.Service +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream +import javax.servlet.ServletOutputStream + +@Service +class MetricsService( + private val valService: ValidatorService, + private val govService: GovService +) { + + fun getQuarters(address: String) = transaction { + valService.getValidatorOperatorAddress(address)?.let { vali -> + val (year, quarter) = DateTime.now().let { it.year to it.monthOfYear.monthToQuarter() } + ValidatorMetricsRecord.findByOperAddr(vali.operatorAddrId) + .map { + val label = it.toMetricPeriodLabel(it.year == year && it.quarter == quarter) + MetricPeriod(label, it.year, it.quarter) + } + } ?: throw ResourceNotFoundException("Invalid validator address: '$address'") + } + + fun getAllQuarters() = transaction { + val (year, quarter) = DateTime.now().let { it.year to it.monthOfYear.monthToQuarter() } + ValidatorMetricsRecord.findDistinctPeriods(year, quarter) + } + + fun getValidatorMetrics(address: String, year: Int, quarter: Int) = transaction { + valService.getValidatorOperatorAddress(address)?.let { + ValidatorMetricsRecord.findByOperAddrForPeriod(it.operatorAddrId, year, quarter)?.data + ?: ( + BlockTxCountsCacheRecord.getBlockTimeSpread(year, quarter) + ?.let { spread -> processMetricsForValObjectAndSpread(it, spread) } + ?: throw ResourceNotFoundException("No data found for quarter $year Q$quarter") + ) + } ?: throw ResourceNotFoundException("Invalid validator address: '$address'") + } + + fun processMetricsForValObjectAndSpread(vali: CurrentValidatorState, spread: BlockTimeSpread) = transaction { + val account = AccountRecord.findByAddress(vali.accountAddr) + ?: throw ResourceNotFoundException("Invalid account address: '${vali.accountAddr}'") + ValidatorMetrics( + vali.operatorAddress, + vali.moniker, + spread.year, + spread.quarter, + vali.currentState == ValidatorState.ACTIVE, + valService.isVerified(vali.accountAddr), + govService.getVotesPerProposalsMetrics( + account.id.value, + spread.minTimestamp, + spread.maxTimestamp + ).toCountTotal(), + vali.consensusAddr.validatorMissedBlocksSpecific(spread.minHeight, spread.maxHeight) + .uptimeToCountTotal(spread.totalBlocks) + ) + } + + fun downloadQuarterMetrics(filters: ValidatorMetricsRequest, resp: ServletOutputStream): ZipOutputStream { + val file = filters.getFile() + val zos = ZipOutputStream(resp) + zos.putNextEntry(ZipEntry("${filters.getFilenameBase()} - ${file.fileName}.csv")) + zos.write(file.writeCsvEntry()) + zos.closeEntry() + zos.close() + return zos + } +} + +fun Pair.toCountTotal() = CountTotal(this.first.toBigInteger(), this.second.toBigInteger()) +fun Int.uptimeToCountTotal(total: Int) = CountTotal((total - this).toBigInteger(), total.toBigInteger()) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/NameService.kt b/service/src/main/kotlin/io/provenance/explorer/service/NameService.kt index af41b5e7..e7fc933b 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/NameService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/NameService.kt @@ -55,6 +55,12 @@ class NameService( val result = it.pageOfResults(page, count) PagedResults(it.size.toLong().pageCountOfResults(count), result, it.size.toLong()) } + + fun getVerifiedKycAttributes() = transaction { + NameRecord.getAllNamesForParent("kyc.passport.pb") + .map { it.fullName } + .toSet() + } } // Splits from `figuretest2.kyc.pb` -> Pair(`kyc.pb`, `kyc.pb`) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt index 5db46e06..fe597d5a 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt @@ -32,8 +32,6 @@ import io.provenance.explorer.domain.extensions.toOffset import io.provenance.explorer.domain.extensions.toPercentage import io.provenance.explorer.domain.models.explorer.DlobHistBase import io.provenance.explorer.domain.models.explorer.TokenHistoricalDataRequest -import io.provenance.explorer.domain.models.explorer.datesValidation -import io.provenance.explorer.domain.models.explorer.getFileListToken import io.provenance.explorer.grpc.v1.AccountGrpcClient import io.provenance.explorer.model.AssetHolder import io.provenance.explorer.model.CmcLatestDataAbbrev @@ -244,9 +242,9 @@ class TokenService(private val accountClient: AccountGrpcClient) { } fun getHashPricingDataDownload(filters: TokenHistoricalDataRequest, resp: ServletOutputStream): ZipOutputStream { - validate(datesValidation(filters.fromDate, filters.toDate)) + validate(filters.datesValidation()) val baseFileName = filters.getFileNameBase() - val fileList = getFileListToken(filters) + val fileList = filters.getFileList() val zos = ZipOutputStream(resp) fileList.forEach { file -> diff --git a/service/src/main/kotlin/io/provenance/explorer/service/TransactionService.kt b/service/src/main/kotlin/io/provenance/explorer/service/TransactionService.kt index 7b751e5b..4a8a8e65 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/TransactionService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/TransactionService.kt @@ -32,12 +32,9 @@ import io.provenance.explorer.domain.extensions.toFees import io.provenance.explorer.domain.extensions.toObjectNode import io.provenance.explorer.domain.extensions.toOffset import io.provenance.explorer.domain.extensions.txEventsToObjectNodePrint -import io.provenance.explorer.domain.models.explorer.TxHistoryDataRequest import io.provenance.explorer.domain.models.explorer.TxQueryParams -import io.provenance.explorer.domain.models.explorer.datesValidation -import io.provenance.explorer.domain.models.explorer.getFileList +import io.provenance.explorer.domain.models.explorer.download.TxHistoryDataRequest import io.provenance.explorer.domain.models.explorer.getValuesPlusAddtnl -import io.provenance.explorer.domain.models.explorer.granularityValidation import io.provenance.explorer.grpc.extensions.getModuleAccName import io.provenance.explorer.model.Gas import io.provenance.explorer.model.MsgInfo @@ -45,7 +42,6 @@ import io.provenance.explorer.model.MsgTypeSet import io.provenance.explorer.model.TxDetails import io.provenance.explorer.model.TxGov import io.provenance.explorer.model.TxHeatmapRes -import io.provenance.explorer.model.TxHistoryChartData import io.provenance.explorer.model.TxMessage import io.provenance.explorer.model.TxSmartContract import io.provenance.explorer.model.TxStatus @@ -57,6 +53,7 @@ import io.provenance.explorer.model.base.getParentForType import io.provenance.explorer.model.base.isMAddress import io.provenance.explorer.model.base.toMAddress import io.provenance.explorer.model.base.toMAddressScope +import io.provenance.explorer.model.download.TxHistoryChartData import io.provenance.explorer.service.async.AsyncCachingV2 import io.provenance.explorer.service.async.getAddressType import org.jetbrains.exposed.dao.id.EntityID @@ -291,8 +288,9 @@ class TransactionService( val params = TxQueryParams( - addressId = addr?.second, addressType = addr?.first, address = address, msgTypes = msgTypeIds, - txStatus = txStatus, count = count, offset = page.toOffset(count), fromDate = fromDate, toDate = toDate + addressId = addr?.second, addressType = addr?.first, address = address, primaryTypesOnly = true, + msgTypes = msgTypeIds, txStatus = txStatus, count = count, offset = page.toOffset(count), + fromDate = fromDate, toDate = toDate ) val total = TxMessageRecord.findByQueryParamsForCount(params) @@ -361,19 +359,19 @@ class TransactionService( fun getTxHistoryChartData(filters: TxHistoryDataRequest): List { validate( - granularityValidation(filters.granularity), - datesValidation(filters.fromDate, filters.toDate) + filters.granularityValidation(), + filters.datesValidation() ) return TxHistoryDataViews.getTxHistoryChartData(filters.granularity, filters.fromDate, filters.toDate) } fun getTxHistoryChartDataDownload(filters: TxHistoryDataRequest, resp: ServletOutputStream): ZipOutputStream { validate( - granularityValidation(filters.granularity), - datesValidation(filters.fromDate, filters.toDate) + filters.granularityValidation(), + filters.datesValidation() ) val baseFileName = filters.getFileNameBase(null) - val fileList = getFileList(filters, null) + val fileList = filters.getFileList(null) val zos = ZipOutputStream(resp) fileList.forEach { file -> diff --git a/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt b/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt index e732019d..cc284f18 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt @@ -44,6 +44,7 @@ import io.provenance.explorer.domain.models.explorer.BlockProposer import io.provenance.explorer.domain.models.explorer.CurrentValidatorState import io.provenance.explorer.domain.models.explorer.hourlyBlockCount import io.provenance.explorer.domain.models.explorer.zeroOutValidatorObj +import io.provenance.explorer.grpc.v1.AttributeGrpcClient import io.provenance.explorer.grpc.v1.ValidatorGrpcClient import io.provenance.explorer.model.BlockLatencyData import io.provenance.explorer.model.CommissionList @@ -70,6 +71,7 @@ import io.provenance.explorer.model.base.CountTotal import io.provenance.explorer.model.base.PagedResults import io.provenance.explorer.model.base.Timeframe import io.provenance.explorer.model.base.stringfy +import kotlinx.coroutines.async import kotlinx.coroutines.runBlocking import org.jetbrains.exposed.sql.transactions.transaction import org.joda.time.DateTime @@ -82,7 +84,9 @@ import java.math.BigInteger class ValidatorService( private val blockService: BlockService, private val grpcClient: ValidatorGrpcClient, - private val cacheService: CacheService + private val cacheService: CacheService, + private val attrClient: AttributeGrpcClient, + private val nameService: NameService ) { protected val logger = logger(ValidatorService::class) @@ -91,6 +95,13 @@ class ValidatorService( fun getSlashingParams() = grpcClient.getSlashingParams().params + fun isVerified(address: String) = + runBlocking { + val atts = async { attrClient.getAllAttributesForAddress(address) }.await().map { it.name } + val kycAtts = nameService.getVerifiedKycAttributes() + atts.intersect(kycAtts).isNotEmpty() + } + // Assumes that the returned validators are active at that height fun getValidatorsByHeight(blockHeight: Int) = transaction { ValidatorsCacheRecord.findById(blockHeight)?.also { @@ -165,7 +176,8 @@ class ValidatorService( stakingValidator.currentState.toString().lowercase(), if (stakingValidator.currentState != ACTIVE) stakingValidator.json.unbondingHeight else null, if (stakingValidator.jailed) signingInfo?.jailedUntil?.toDateTime() else null, - stakingValidator.removed + stakingValidator.removed, + isVerified(addr.accountAddr) ) } ?: throw ResourceNotFoundException("Invalid validator address: '$address'") diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncCachingV2.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncCachingV2.kt index 785c42f4..98c07404 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncCachingV2.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncCachingV2.kt @@ -491,81 +491,94 @@ class AsyncCachingV2( private fun saveGovData(tx: ServiceOuterClass.GetTxResponse, txInfo: TxData, txUpdate: TxUpdate) = transaction { if (tx.txResponse.code == 0) { tx.tx.body.messagesList.mapNotNull { it.getAssociatedGovMsgs() } - .forEachIndexed { idx, pair -> - when (pair.first) { - GovMsgType.PROPOSAL -> - // Have to find the proposalId in the log events - tx.txResponse.logsList[idx] - .eventsList.first { it.type == "submit_proposal" } - .attributesList.first { it.key == "proposal_id" } - .value.toLong() - .let { id -> - val proposer = when { - pair.second.typeUrl.endsWith("gov.v1beta1.MsgSubmitProposal") -> pair.second.toMsgSubmitProposalOld().proposer - pair.second.typeUrl.endsWith("gov.v1.MsgSubmitProposal") -> pair.second.toMsgSubmitProposal().proposer - else -> throw InvalidArgumentException("Invalid gov proposal msg type: ${pair.second.typeUrl}") - } - txUpdate.apply { - govService.buildProposal(id, txInfo, proposer, true)?.let { this.proposals.add(it) } - govService.buildDeposit(id, txInfo, null, pair.second)?.let { this.deposits.addAll(it) } - govService.buildProposalMonitor(pair.second, id, txInfo).let { mon -> - if (mon.isNotEmpty()) this.proposalMonitors.addAll(mon) + .forEachIndexed { logsIdx, list -> + list.forEachIndexed { listIdx, pair -> + when (pair.first) { + GovMsgType.PROPOSAL -> + // Have to find the proposalId in the log events + tx.txResponse.logsList[logsIdx] + .eventsList.first { it.type == "submit_proposal" } + .attributesList.filter { it.key == "proposal_id" }.withIndex() + .first { it.index == listIdx } + .value.value.toLong() + .let { id -> + val proposer = when { + pair.second.typeUrl.endsWith("gov.v1beta1.MsgSubmitProposal") -> pair.second.toMsgSubmitProposalOld().proposer + pair.second.typeUrl.endsWith("gov.v1.MsgSubmitProposal") -> pair.second.toMsgSubmitProposal().proposer + else -> throw InvalidArgumentException("Invalid gov proposal msg type: ${pair.second.typeUrl}") + } + txUpdate.apply { + govService.buildProposal(id, txInfo, proposer, true) + ?.let { this.proposals.add(it) } + govService.buildDeposit(id, txInfo, null, pair.second) + ?.let { this.deposits.addAll(it) } + govService.buildProposalMonitor(pair.second, id, txInfo).let { mon -> + if (mon.isNotEmpty()) this.proposalMonitors.addAll(mon) + } } } + GovMsgType.DEPOSIT -> { + val (proposalId, depositor) = when { + pair.second.typeUrl.endsWith("gov.v1beta1.MsgDeposit") -> + pair.second.toMsgDepositOld().let { it.proposalId to it.depositor } + pair.second.typeUrl.endsWith("gov.v1.MsgDeposit") -> + pair.second.toMsgDeposit().let { it.proposalId to it.depositor } + else -> throw InvalidArgumentException("Invalid gov deposit msg type: ${pair.second.typeUrl}") + } + txUpdate.apply { + govService.buildProposal(proposalId, txInfo, depositor, isSubmit = false) + ?.let { this.proposals.add(it) } + govService.buildDeposit(proposalId, txInfo, pair.second, null) + ?.let { this.deposits.addAll(it) } } - GovMsgType.DEPOSIT -> { - val (proposalId, depositor) = when { - pair.second.typeUrl.endsWith("gov.v1beta1.MsgDeposit") -> - pair.second.toMsgDepositOld().let { it.proposalId to it.depositor } - pair.second.typeUrl.endsWith("gov.v1.MsgDeposit") -> - pair.second.toMsgDeposit().let { it.proposalId to it.depositor } - else -> throw InvalidArgumentException("Invalid gov deposit msg type: ${pair.second.typeUrl}") - } - txUpdate.apply { - govService.buildProposal(proposalId, txInfo, depositor, isSubmit = false)?.let { this.proposals.add(it) } - govService.buildDeposit(proposalId, txInfo, pair.second, null)?.let { this.deposits.addAll(it) } - } - } - GovMsgType.VOTE -> { - val (proposalId, voter, justification) = when { - pair.second.typeUrl.endsWith("gov.v1beta1.MsgVote") -> - pair.second.toMsgVoteOld().let { Triple(it.proposalId, it.voter, null) } - pair.second.typeUrl.endsWith("gov.v1.MsgVote") -> - pair.second.toMsgVote().let { Triple(it.proposalId, it.voter, it.metadata.toVoteMetadata()) } - else -> throw InvalidArgumentException("Invalid gov vote msg type: ${pair.second.typeUrl}") } - txUpdate.apply { - govService.buildProposal(proposalId, txInfo, voter, isSubmit = false)?.let { this.proposals.add(it) } - this.votes.addAll( - govService.buildVote( - txInfo, - pair.second.toWeightedVoteList(), - voter, - proposalId, - justification + + GovMsgType.VOTE -> { + val (proposalId, voter, justification) = when { + pair.second.typeUrl.endsWith("gov.v1beta1.MsgVote") -> + pair.second.toMsgVoteOld().let { Triple(it.proposalId, it.voter, null) } + pair.second.typeUrl.endsWith("gov.v1.MsgVote") -> + pair.second.toMsgVote() + .let { Triple(it.proposalId, it.voter, it.metadata.toVoteMetadata()) } + else -> throw InvalidArgumentException("Invalid gov vote msg type: ${pair.second.typeUrl}") + } + txUpdate.apply { + govService.buildProposal(proposalId, txInfo, voter, isSubmit = false) + ?.let { this.proposals.add(it) } + this.votes.addAll( + govService.buildVote( + txInfo, + pair.second.toWeightedVoteList(), + voter, + proposalId, + justification + ) ) - ) - } - } - GovMsgType.WEIGHTED -> { - val (proposalId, voter, justification) = when { - pair.second.typeUrl.endsWith("gov.v1beta1.MsgVoteWeighted") -> - pair.second.toMsgVoteWeightedOld().let { Triple(it.proposalId, it.voter, null) } - pair.second.typeUrl.endsWith("gov.v1.MsgVoteWeighted") -> - pair.second.toMsgVoteWeighted().let { Triple(it.proposalId, it.voter, it.metadata.toVoteMetadata()) } - else -> throw InvalidArgumentException("Invalid gov vote weighted msg type: ${pair.second.typeUrl}") + } } - txUpdate.apply { - govService.buildProposal(proposalId, txInfo, voter, isSubmit = false)?.let { this.proposals.add(it) } - this.votes.addAll( - govService.buildVote( - txInfo, - pair.second.toWeightedVoteList(), - voter, - proposalId, - justification + + GovMsgType.WEIGHTED -> { + val (proposalId, voter, justification) = when { + pair.second.typeUrl.endsWith("gov.v1beta1.MsgVoteWeighted") -> + pair.second.toMsgVoteWeightedOld().let { Triple(it.proposalId, it.voter, null) } + pair.second.typeUrl.endsWith("gov.v1.MsgVoteWeighted") -> + pair.second.toMsgVoteWeighted() + .let { Triple(it.proposalId, it.voter, it.metadata.toVoteMetadata()) } + else -> throw InvalidArgumentException("Invalid gov vote weighted msg type: ${pair.second.typeUrl}") + } + txUpdate.apply { + govService.buildProposal(proposalId, txInfo, voter, isSubmit = false) + ?.let { this.proposals.add(it) } + this.votes.addAll( + govService.buildVote( + txInfo, + pair.second.toWeightedVoteList(), + voter, + proposalId, + justification + ) ) - ) + } } } } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt index 27d97595..5c68cd75 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt @@ -32,8 +32,11 @@ import io.provenance.explorer.domain.entities.TxMsgTypeSubtypeTable import io.provenance.explorer.domain.entities.TxSingleMessageCacheRecord import io.provenance.explorer.domain.entities.ValidatorMarketRateRecord import io.provenance.explorer.domain.entities.ValidatorMarketRateStatsRecord +import io.provenance.explorer.domain.entities.ValidatorMetricsRecord +import io.provenance.explorer.domain.entities.ValidatorStateRecord import io.provenance.explorer.domain.extensions.getType import io.provenance.explorer.domain.extensions.height +import io.provenance.explorer.domain.extensions.monthToQuarter import io.provenance.explorer.domain.extensions.percentChange import io.provenance.explorer.domain.extensions.startOfDay import io.provenance.explorer.domain.extensions.toDateTime @@ -51,8 +54,10 @@ import io.provenance.explorer.service.BlockService import io.provenance.explorer.service.CacheService import io.provenance.explorer.service.ExplorerService import io.provenance.explorer.service.GovService +import io.provenance.explorer.service.MetricsService import io.provenance.explorer.service.PricingService import io.provenance.explorer.service.TokenService +import io.provenance.explorer.service.ValidatorService import io.provenance.explorer.service.getBlock import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -91,7 +96,9 @@ class AsyncService( private val cacheService: CacheService, private val tokenService: TokenService, private val pricingService: PricingService, - private val accountService: AccountService + private val accountService: AccountService, + private val valService: ValidatorService, + private val metricsService: MetricsService ) { protected val logger = logger(AsyncService::class) @@ -447,4 +454,27 @@ class AsyncService( ProcessQueueRecord.delete(ProcessQueueType.ACCOUNT, msg) } } + + @Scheduled(cron = "0 0 0 * * *") // Every beginning of every day + fun calculateValidatorMetrics() { + val (year, quarter) = DateTime.now().minusMinutes(5).let { it.year to it.monthOfYear.monthToQuarter() } + logger.info("Refreshing block spread view") + BlockTxCountsCacheRecord.updateSpreadView() + logger.info("Saving validator metrics") + val spread = BlockTxCountsCacheRecord.getBlockTimeSpread(year, quarter) ?: return + ValidatorStateRecord.findAll(valService.getActiveSet()).forEach { vali -> + try { + val metric = metricsService.processMetricsForValObjectAndSpread(vali, spread) + ValidatorMetricsRecord.insertIgnore( + vali.operatorAddrId, + vali.operatorAddress, + spread.year, + spread.quarter, + metric + ) + } catch (e: Exception) { + logger.error("Error processing metrics for validator: ${vali.operatorAddress}", e.message) + } + } + } } diff --git a/service/src/main/kotlin/io/provenance/explorer/web/v3/AccountControllerV3.kt b/service/src/main/kotlin/io/provenance/explorer/web/v3/AccountControllerV3.kt index a8e563a6..35b5546b 100644 --- a/service/src/main/kotlin/io/provenance/explorer/web/v3/AccountControllerV3.kt +++ b/service/src/main/kotlin/io/provenance/explorer/web/v3/AccountControllerV3.kt @@ -5,7 +5,7 @@ import io.provenance.explorer.config.interceptor.JwtInterceptor import io.provenance.explorer.domain.annotation.HiddenApi import io.provenance.explorer.domain.extensions.toTxBody import io.provenance.explorer.domain.extensions.toTxMessageBody -import io.provenance.explorer.domain.models.explorer.TxHistoryDataRequest +import io.provenance.explorer.domain.models.explorer.download.TxHistoryDataRequest import io.provenance.explorer.model.BankSendRequest import io.provenance.explorer.model.TxMessageBody import io.provenance.explorer.model.base.DateTruncGranularity diff --git a/service/src/main/kotlin/io/provenance/explorer/web/v3/TransactionControllerV3.kt b/service/src/main/kotlin/io/provenance/explorer/web/v3/TransactionControllerV3.kt index 9e6050e0..aa6b97fe 100644 --- a/service/src/main/kotlin/io/provenance/explorer/web/v3/TransactionControllerV3.kt +++ b/service/src/main/kotlin/io/provenance/explorer/web/v3/TransactionControllerV3.kt @@ -1,6 +1,6 @@ package io.provenance.explorer.web.v3 -import io.provenance.explorer.domain.models.explorer.TxHistoryDataRequest +import io.provenance.explorer.domain.models.explorer.download.TxHistoryDataRequest import io.provenance.explorer.model.base.DateTruncGranularity import io.provenance.explorer.model.base.Timeframe import io.provenance.explorer.service.TransactionService diff --git a/service/src/main/kotlin/io/provenance/explorer/web/v3/ValidatorControllerV3.kt b/service/src/main/kotlin/io/provenance/explorer/web/v3/ValidatorControllerV3.kt index d242c5ba..ba1d59c1 100644 --- a/service/src/main/kotlin/io/provenance/explorer/web/v3/ValidatorControllerV3.kt +++ b/service/src/main/kotlin/io/provenance/explorer/web/v3/ValidatorControllerV3.kt @@ -1,6 +1,8 @@ package io.provenance.explorer.web.v3 +import io.provenance.explorer.domain.models.explorer.download.ValidatorMetricsRequest import io.provenance.explorer.model.ValidatorState +import io.provenance.explorer.service.MetricsService import io.provenance.explorer.service.ValidatorService import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation @@ -8,9 +10,11 @@ import io.swagger.annotations.ApiParam import org.springframework.http.MediaType import org.springframework.validation.annotation.Validated import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController +import javax.servlet.http.HttpServletResponse import javax.validation.constraints.Max import javax.validation.constraints.Min @@ -23,7 +27,10 @@ import javax.validation.constraints.Min consumes = MediaType.APPLICATION_JSON_VALUE, tags = ["Validators"] ) -class ValidatorControllerV3(private val validatorService: ValidatorService) { +class ValidatorControllerV3( + private val validatorService: ValidatorService, + private val metricsService: MetricsService +) { @ApiOperation("Returns recent validators") @GetMapping("/recent") @@ -41,4 +48,49 @@ class ValidatorControllerV3(private val validatorService: ValidatorService) { @RequestParam(defaultValue = "ACTIVE") status: ValidatorState ) = validatorService.getRecentValidators(count, page, status) + + @ApiOperation("Returns a validator's metrics for the given quarter that correlate with the Validator Delegation Program") + @GetMapping("/{address}/metrics") + fun metrics( + @ApiParam(value = "The Validator's operator, owning account, or consensus address") @PathVariable address: String, + @ApiParam(value = "The year for the metrics") + @RequestParam + year: Int, + @ApiParam(value = "The quarter for the metrics") + @RequestParam + @Min(1) + @Max(4) + quarter: Int + ) = metricsService.getValidatorMetrics(address, year, quarter) + + @ApiOperation("Returns a validator's known metric periods that correlate with the Validator Delegation Program") + @GetMapping("/{address}/metrics/periods") + fun metricPeriods( + @ApiParam(value = "The Validator's operator, owning account, or consensus address") @PathVariable address: String + ) = metricsService.getQuarters(address) + + @ApiOperation("Returns all known metric periods that correlate with the Validator Delegation Program") + @GetMapping("/metrics/periods") + fun allMetricPeriods() = metricsService.getAllQuarters() + + @ApiOperation( + "Downloads validators' metrics for the given quarter that correlate with the Validator Delegation Program" + ) + @GetMapping("/metrics/download") + fun metricsDownload( + @ApiParam(value = "The year for the metrics") + @RequestParam + year: Int, + @ApiParam(value = "The quarter for the metrics") + @RequestParam + @Min(1) + @Max(4) + quarter: Int, + response: HttpServletResponse + ) { + val filters = ValidatorMetricsRequest(year, quarter) + response.status = HttpServletResponse.SC_OK + response.addHeader("Content-Disposition", "attachment; filename=\"${filters.getFilenameBase()}.zip\"") + metricsService.downloadQuarterMetrics(filters, response.outputStream) + } }