From b07225e6593cf2e4e3393e425d321171fd5a9d39 Mon Sep 17 00:00:00 2001 From: Carolyn Russell Date: Mon, 2 May 2022 16:46:08 -0600 Subject: [PATCH] Add Naming tree, `isVerified` tag for Validators * Added naming tree API * Added `isVerified` to validator data responses * Added new ENV for `explorer.verified-addresses` as a comma-deliniated list of trusted addresses closes: #134 closes: #343 --- CHANGELOG.md | 11 ++ .../db/migration/V1_64__Add_naming_tree.sql | 123 ++++++++++++++++ .../V1_65__Add_verified_to_validators.sql | 134 ++++++++++++++++++ docker/docker-compose.yml | 1 + .../explorer/config/ExplorerProperties.kt | 59 +++----- .../explorer/domain/entities/Name.kt | 98 +++++++++++++ .../explorer/domain/entities/Validators.kt | 72 ++++++---- .../domain/extensions/DbExtensions.kt | 3 +- .../domain/models/explorer/NameModels.kt | 30 ++++ .../domain/models/explorer/ValidatorModels.kt | 12 +- .../explorer/grpc/extensions/MsgConverter.kt | 17 +++ .../explorer/grpc/v1/AttributeGrpcClient.kt | 8 ++ .../explorer/service/NameService.kt | 59 ++++++++ .../explorer/service/ValidatorService.kt | 32 +++-- .../explorer/service/async/AsyncCachingV2.kt | 85 +++++++++-- .../explorer/service/async/AsyncService.kt | 2 +- .../service/utility/MigrationService.kt | 6 +- .../explorer/web/v2/NameController.kt | 27 ++++ .../web/v2/utility/MigrationController.kt | 5 +- .../application-container.properties | 1 + .../application-development.properties | 2 + 21 files changed, 689 insertions(+), 98 deletions(-) create mode 100644 database/src/main/resources/db/migration/V1_64__Add_naming_tree.sql create mode 100644 database/src/main/resources/db/migration/V1_65__Add_verified_to_validators.sql create mode 100644 service/src/main/kotlin/io/provenance/explorer/domain/entities/Name.kt create mode 100644 service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/NameModels.kt create mode 100644 service/src/main/kotlin/io/provenance/explorer/service/NameService.kt create mode 100644 service/src/main/kotlin/io/provenance/explorer/web/v2/NameController.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index ac1db297..725f8ab2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,11 @@ Ref: https://keepachangelog.com/en/1.0.0/ * Add new IBC APIs #336 * `/api/v2/txs/ibc/chain/{ibcChain}` - txs per IBC chain id, query params supporting narrowing by channel * `/api/v2/ibc/channels/src_port/{srcPort}/src_channel/{srcChannel}/relayers` - relayers by channel +* Add Name APIs #134 + * `/api/v2/names/tree` - tree map of names on chain, including restriction and owner +* Updated Validator data responses to include `isVerified` attribute #343 + * Defined by where the validator owner has a KYC attribute created by a trusted address + * Added new ENV for `explorer.verified-addresses` - is a list of trusted addresses ### Improvements * Update vote ingestion to include Weighted Votes #323 @@ -86,6 +91,12 @@ Ref: https://keepachangelog.com/en/1.0.0/ * Updated `add_tx_debug()`, `add_tx()` procedures with new ingestions * Migration 1.63 - Add function for `uuid_or_null()` #339 * Added function to determine if a string is a UUID, and if not, return null +* Migration 1.64 - Add naming tree #134 + * Created `name` table, and inserted records +* Migration 1.65 - Add `verified` to validators #343 + * Added `verified` to `validator_state` table + * Updated materialized view `current_validator_state` to include new `verified` column + * Updated the `get_validator_list()`, `get_all_validator_state()` functions to include new `verified` column ## [v4.1.0](https://github.com/provenance-io/explorer-service/releases/tag/v4.1.0) - 2022-03-24 ### Release Name: Abu Bakr II diff --git a/database/src/main/resources/db/migration/V1_64__Add_naming_tree.sql b/database/src/main/resources/db/migration/V1_64__Add_naming_tree.sql new file mode 100644 index 00000000..4051a36c --- /dev/null +++ b/database/src/main/resources/db/migration/V1_64__Add_naming_tree.sql @@ -0,0 +1,123 @@ +SELECT 'Adding name table' AS comment; + +CREATE TABLE IF NOT EXISTS name +( + id SERIAL PRIMARY KEY, + parent VARCHAR(550), + child VARCHAR(40), + full_name VARCHAR(600), + owner VARCHAR(128), + restricted BOOLEAN DEFAULT FALSE, + height_added INT +); + +DROP INDEX IF EXISTS name_unique_idx; +CREATE UNIQUE INDEX IF NOT EXISTS name_unique_idx ON name (full_name, owner); + +SELECT 'Inserting bind_names' AS comment; +WITH base AS ( + SELECT tc.height, + tc.id, + CASE WHEN attr.key = 'name' THEN attr.value::text END AS full_name, + CASE WHEN attr.key = 'address' THEN attr.value::text END AS owner + FROM tx_cache tc + JOIN tx_message tm ON tm.tx_hash_id = tc.id + JOIN tx_message_type tmt on tm.tx_message_type_id = tmt.id, + jsonb_array_elements(tc.tx_v2 -> 'tx_response' -> 'logs') with ordinality logs("events", idx), + jsonb_to_recordset(logs.events -> 'events') event("type" text, "attributes" jsonb), + jsonb_to_recordset(event.attributes) attr("key" text, "value" text) + WHERE tc.error_code IS NULL + AND tmt.proto_type IN ('/provenance.name.v1.MsgBindNameRequest', + '/cosmwasm.wasm.v1beta1.MsgInstantiateContract', + '/cosmwasm.wasm.v1.MsgInstantiateContract', + '/cosmwasm.wasm.v1.MsgExecuteContract', + '/cosmwasm.wasm.v1beta1.MsgExecuteContract') + AND logs.idx - 1 = tm.msg_idx + AND event.type IN ('provenance.name.v1.EventNameBound') + ORDER BY tc.height +), + name_agg AS ( + SELECT base.id, + array_agg(base.full_name) AS name_agg + FROM base + WHERE base.full_name IS NOT NULL + GROUP BY base.id + ), + owner_agg AS ( + SELECT base.id, + array_agg(base.owner) AS owner_agg + FROM base + WHERE base.owner IS NOT NULL + GROUP BY base.id + ), + smooshed AS ( + SELECT base.height, + base.id AS tx_id, + REPLACE(unnest(na.name_agg), '"', '') AS full_name, + REPLACE(unnest(oa.owner_agg), '"', '') AS owner + FROM base + JOIN name_agg na ON base.id = na.id + JOIN owner_agg oa ON base.id = oa.id + ), + regexed AS ( + SELECT sm.height, + sm.owner, + sm.full_name, + regexp_matches(sm.full_name, '([a-zA-Z0-9-]*)(\.[a-zA-Z0-9.-]*)') AS reg + FROM smooshed sm + ) +INSERT +INTO name (parent, child, full_name, owner, height_added) +SELECT substr(reg[2], 2) AS parent, + reg[1] AS child, + r.full_name, + r.owner, + MAX(r.height) +FROM regexed r +GROUP BY parent, child, full_name, owner +ON CONFLICT (full_name, owner) DO UPDATE + SET height_added = CASE + WHEN excluded.height_added > name.height_added THEN excluded.height_added + ELSE name.height_added END; + + +SELECT 'Removing unbind_names' AS comment; +DELETE +FROM name +WHERE id IN ( + WITH base AS (SELECT tc.height, + tc.id, + CASE WHEN attr.key = 'name' THEN attr.value::text END AS full_name, + CASE WHEN attr.key = 'address' THEN attr.value::text END AS owner + FROM tx_cache tc + JOIN tx_message tm ON tm.tx_hash_id = tc.id + JOIN tx_message_type tmt on tm.tx_message_type_id = tmt.id, + jsonb_array_elements(tc.tx_v2 -> 'tx_response' -> 'logs') with ordinality logs("events", idx), + jsonb_to_recordset(logs.events -> 'events') event("type" text, "attributes" jsonb), + jsonb_to_recordset(event.attributes) attr("key" text, "value" text) + WHERE tc.error_code IS NULL + AND tmt.proto_type IN ('/provenance.name.v1.MsgDeleteNameRequest', + '/cosmwasm.wasm.v1beta1.MsgInstantiateContract', + '/cosmwasm.wasm.v1.MsgInstantiateContract', + '/cosmwasm.wasm.v1.MsgExecuteContract', + '/cosmwasm.wasm.v1beta1.MsgExecuteContract') + AND logs.idx - 1 = msg_idx + AND event.type IN ('provenance.name.v1.EventNameUnbound') + ORDER BY tc.height + ), + smooshed AS ( + SELECT base.height, + base.id AS tx_id, + REPLACE(MAX(full_name), '"', '') AS full_name, + REPLACE(MAX(owner), '"', '') AS owner + FROM base + GROUP BY base.height, base.id + ) + SELECT name.id + FROM smooshed sm + JOIN name ON sm.full_name = name.full_name AND sm.owner = name.owner + WHERE sm.height > name.height_added +); + +DROP INDEX IF EXISTS name_unique_idx; +CREATE UNIQUE INDEX IF NOT EXISTS name_unique_idx ON name (full_name); diff --git a/database/src/main/resources/db/migration/V1_65__Add_verified_to_validators.sql b/database/src/main/resources/db/migration/V1_65__Add_verified_to_validators.sql new file mode 100644 index 00000000..5d29011e --- /dev/null +++ b/database/src/main/resources/db/migration/V1_65__Add_verified_to_validators.sql @@ -0,0 +1,134 @@ +SELECT 'Adding validator verified to validator_state' AS comment; + + +ALTER TABLE validator_state +ADD COLUMN IF NOT EXISTS verified BOOLEAN NOT NULL DEFAULT FALSE; + +SELECT 'Modify `current_validator_state` view' AS comment; +DROP MATERIALIZED VIEW IF EXISTS current_validator_state; +CREATE MATERIALIZED VIEW IF NOT EXISTS current_validator_state AS +SELECT DISTINCT ON (vs.operator_addr_id) vs.operator_addr_id, + vs.operator_address, + vs.block_height, + vs.moniker, + vs.status, + vs.jailed, + vs.token_count, + vs.json, + svc.account_address, + svc.consensus_address, + svc.consensus_pubkey, + vs.commission_rate, + vs.verified +FROM validator_state vs + JOIN staking_validator_cache svc on vs.operator_addr_id = svc.id +ORDER BY vs.operator_addr_id, vs.block_height desc +WITH DATA; + + +SELECT 'Modify `get_validator_list` function' AS comment; +DROP FUNCTION IF EXISTS get_validator_list(integer, varchar, text, integer, integer, text[]); +CREATE OR REPLACE FUNCTION get_validator_list(active_set integer, active_status character varying, search_state text, search_limit integer, search_offset integer, consensus_set text[] DEFAULT NULL, search_verified boolean DEFAULT NULL) + returns TABLE(operator_addr_id integer, operator_address character varying, block_height integer, moniker character varying, status character varying, jailed boolean, token_count numeric, json jsonb, account_address character varying, consensus_address character varying, consensus_pubkey character varying, commission_rate numeric, verified boolean, validator_state text) + language plpgsql +as +$$ +BEGIN + + RETURN QUERY + WITH active AS ( + SELECT cvs.* + FROM current_validator_state cvs + WHERE cvs.status = active_status + AND cvs.jailed = false + ORDER BY cvs.token_count DESC + LIMIT active_set + ), + jailed AS ( + SELECT cvs.* + FROM current_validator_state cvs + WHERE cvs.jailed = true + ), + candidate AS ( + SELECT cvs.* + FROM current_validator_state cvs + LEFT JOIN active a ON cvs.operator_address = a.operator_address + WHERE cvs.jailed = false + AND a.operator_address IS NULL + ), + state AS ( + SELECT cvs.operator_address, + CASE + WHEN a.operator_address IS NOT NULL THEN 'active' + WHEN j.operator_address IS NOT NULL THEN 'jailed' + WHEN c.operator_address IS NOT NULL THEN 'candidate' + END validator_state + FROM current_validator_state cvs + LEFT JOIN active a ON cvs.operator_address = a.operator_address + LEFT JOIN jailed j ON cvs.operator_address = j.operator_address + LEFT JOIN candidate c ON cvs.operator_address = c.operator_address + ) + SELECT cvs.*, + s.validator_state + FROM current_validator_state cvs + LEFT JOIN state s ON cvs.operator_address = s.operator_address + WHERE s.validator_state = search_state + AND (consensus_set IS NULL OR cvs.consensus_address = ANY (consensus_set)) + AND (search_verified IS NULL OR cvs.verified = search_verified) + ORDER BY s.validator_state, cvs.token_count DESC + LIMIT search_limit OFFSET search_offset; +END +$$; + + +SELECT 'Modify `get_all_validator_state` function' AS comment; +DROP FUNCTION IF EXISTS get_all_validator_state(integer, varchar, text[]); +CREATE OR REPLACE FUNCTION get_all_validator_state(active_set integer, active_status character varying, consensus_set text[] DEFAULT NULL, search_verified boolean DEFAULT NULL) + returns TABLE(operator_addr_id integer, operator_address character varying, block_height integer, moniker character varying, status character varying, jailed boolean, token_count numeric, json jsonb, account_address character varying, consensus_address character varying, consensus_pubkey character varying, commission_rate numeric, verified boolean, validator_state text) + language plpgsql +as +$$ +BEGIN + + RETURN QUERY + WITH active AS ( + SELECT cvs.* + FROM current_validator_state cvs + WHERE cvs.status = active_status + AND cvs.jailed = false + ORDER BY cvs.token_count DESC + LIMIT active_set + ), + jailed AS ( + SELECT cvs.* + FROM current_validator_state cvs + WHERE cvs.jailed = true + ), + candidate AS ( + SELECT cvs.* + FROM current_validator_state cvs + LEFT JOIN active a ON cvs.operator_address = a.operator_address + WHERE cvs.jailed = false + AND a.operator_address IS NULL + ), + state AS ( + SELECT cvs.operator_address, + CASE + WHEN a.operator_address IS NOT NULL THEN 'active' + WHEN j.operator_address IS NOT NULL THEN 'jailed' + WHEN c.operator_address IS NOT NULL THEN 'candidate' + END validator_state + FROM current_validator_state cvs + LEFT JOIN active a ON cvs.operator_address = a.operator_address + LEFT JOIN jailed j ON cvs.operator_address = j.operator_address + LEFT JOIN candidate c ON cvs.operator_address = c.operator_address + ) + SELECT cvs.*, + s.validator_state + FROM current_validator_state cvs + LEFT JOIN state s ON cvs.operator_address = s.operator_address + WHERE (consensus_set IS NULL OR cvs.consensus_address = ANY (consensus_set)) + AND (search_verified IS NULL OR cvs.verified = search_verified) + ORDER BY s.validator_state, cvs.token_count DESC; +END +$$; diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 86c246db..2b43d366 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -38,6 +38,7 @@ services: - EXPLORER_SWAGGER_URL=localhost:8612 - EXPLORER_SWAGGER_PROTOCOL=http - EXPLORER_PRICING_URL=https://test.figure.tech/service-pricing-engine/service-pricing-engine + - EXPLORER_VERIFIED_ADDRESSES=test depends_on: - explorer-postgres links: diff --git a/service/src/main/kotlin/io/provenance/explorer/config/ExplorerProperties.kt b/service/src/main/kotlin/io/provenance/explorer/config/ExplorerProperties.kt index 0d59d0ac..6383fb91 100644 --- a/service/src/main/kotlin/io/provenance/explorer/config/ExplorerProperties.kt +++ b/service/src/main/kotlin/io/provenance/explorer/config/ExplorerProperties.kt @@ -2,51 +2,28 @@ package io.provenance.explorer.config import io.provenance.explorer.domain.core.Bech32 import org.springframework.boot.context.properties.ConfigurationProperties +import org.springframework.boot.context.properties.ConstructorBinding import org.springframework.validation.annotation.Validated -import javax.validation.constraints.NotNull @ConfigurationProperties(prefix = "explorer") @Validated -class ExplorerProperties { - - @NotNull - lateinit var mainnet: String - - @NotNull - lateinit var pbUrl: String - - @NotNull - lateinit var initialHistoricalDayCount: String - - @NotNull - lateinit var spotlightTtlMs: String - - @NotNull - lateinit var figmentApikey: String - - @NotNull - lateinit var figmentUrl: String - - @NotNull - lateinit var genesisVersionUrl: String - - @NotNull - lateinit var upgradeVersionRegex: String - - @NotNull - lateinit var upgradeGithubRepo: String - - @NotNull - lateinit var hiddenApis: String - - @NotNull - lateinit var swaggerUrl: String - - @NotNull - lateinit var swaggerProtocol: String - - @NotNull - lateinit var pricingUrl: String +@ConstructorBinding +class ExplorerProperties( + val mainnet: String, + val pbUrl: String, + val initialHistoricalDayCount: String, + val spotlightTtlMs: String, + val figmentApikey: String, + val figmentUrl: String, + val genesisVersionUrl: String, + val upgradeVersionRegex: String, + val upgradeGithubRepo: String, + val hiddenApis: String, + val swaggerUrl: String, + val swaggerProtocol: String, + val pricingUrl: String, + val verifiedAddresses: List +) { fun initialHistoricalDays() = initialHistoricalDayCount.toInt() diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Name.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Name.kt new file mode 100644 index 00000000..7092cb70 --- /dev/null +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Name.kt @@ -0,0 +1,98 @@ +package io.provenance.explorer.domain.entities + +import io.provenance.explorer.domain.extensions.execAndMap +import io.provenance.explorer.domain.models.explorer.Name +import io.provenance.explorer.domain.models.explorer.NameObj +import org.jetbrains.exposed.dao.IntEntity +import org.jetbrains.exposed.dao.IntEntityClass +import org.jetbrains.exposed.dao.id.EntityID +import org.jetbrains.exposed.dao.id.IntIdTable +import org.jetbrains.exposed.sql.and +import org.jetbrains.exposed.sql.deleteWhere +import org.jetbrains.exposed.sql.insertIgnore +import org.jetbrains.exposed.sql.transactions.transaction +import java.sql.ResultSet + +object NameTable : IntIdTable(name = "name") { + val parent = varchar("parent", 550).nullable() + val child = varchar("child", 32) + val fullName = varchar("full_name", 600) + val owner = varchar("owner", 128) + val restricted = bool("restricted").default(false) + val heightAdded = integer("height_added") +} + +class NameRecord(id: EntityID) : IntEntity(id) { + companion object : IntEntityClass(NameTable) { + + fun insertOrUpdate(obj: Name) = transaction { + getByFullNameAndOwner(obj.fullName, obj.owner)?.apply { + if (obj.heightAdded > this.heightAdded) { + this.restricted = obj.restricted + this.heightAdded = obj.heightAdded + } + } ?: NameTable.insertIgnore { + it[this.parent] = obj.parent + it[this.child] = obj.child + it[this.fullName] = obj.fullName + it[this.owner] = obj.owner + it[this.restricted] = obj.restricted + it[this.heightAdded] = obj.heightAdded + } + } + + fun deleteByFullNameAndOwner(fullName: String, owner: String, blockHeight: Int) = transaction { + NameTable.deleteWhere { + (NameTable.fullName eq fullName) and + (NameTable.owner eq owner) and + (NameTable.heightAdded lessEq blockHeight) + } + } + + fun getByFullNameAndOwner(fullName: String, owner: String) = transaction { + NameRecord.find { (NameTable.fullName eq fullName) and (NameTable.owner eq owner) }.firstOrNull() + } + + fun getNameSet() = transaction { + val query = """ + with data as ( + select + string_to_array(name.full_name, '.') as parts, + owner, + restricted, + full_name + from name + ) + select + array( + select parts[i] + from generate_subscripts(parts, 1) as indices(i) + order by i desc + ) as reversed, + owner, + restricted, + full_name + from data; + """.trimIndent() + query.execAndMap { it.toNameObj() } + } + + fun getNamesByOwners(owners: List) = transaction { + NameRecord.find { NameTable.owner inList owners }.toSet() + } + } + + var parent by NameTable.parent + var child by NameTable.child + var fullName by NameTable.fullName + var owner by NameTable.owner + var restricted by NameTable.restricted + var heightAdded by NameTable.heightAdded +} + +fun ResultSet.toNameObj() = NameObj( + (this.getArray("reversed").array as Array).toList(), + this.getString("owner"), + this.getBoolean("restricted"), + this.getString("full_name") +) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Validators.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Validators.kt index 5cbfaae7..b8f0226e 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Validators.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Validators.kt @@ -21,6 +21,7 @@ import org.jetbrains.exposed.dao.IntEntity import org.jetbrains.exposed.dao.IntEntityClass import org.jetbrains.exposed.dao.id.EntityID import org.jetbrains.exposed.dao.id.IntIdTable +import org.jetbrains.exposed.sql.BooleanColumnType import org.jetbrains.exposed.sql.ColumnType import org.jetbrains.exposed.sql.IntegerColumnType import org.jetbrains.exposed.sql.SortOrder @@ -114,6 +115,7 @@ object ValidatorStateTable : IntIdTable(name = "validator_state") { val tokenCount = decimal("token_count", 100, 0) val json = jsonb("json", OBJECT_MAPPER) val commissionRate = decimal("commission_rate", 19, 18) + val verified = bool("verified").default(false) } class ValidatorStateRecord(id: EntityID) : IntEntity(id) { @@ -121,19 +123,21 @@ class ValidatorStateRecord(id: EntityID) : IntEntity(id) { val logger = logger(ValidatorStateRecord::class) - fun insertIgnore(blockHeight: Int, valId: Int, operator: String, json: Staking.Validator) = transaction { - ValidatorStateTable.insertIgnore { - it[this.operatorAddrId] = valId - it[this.operatorAddress] = operator - it[this.blockHeight] = blockHeight - it[this.moniker] = json.description.moniker - it[this.status] = json.status.name - it[this.jailed] = json.jailed - it[this.tokenCount] = json.tokens.toBigDecimal() - it[this.json] = json - it[this.commissionRate] = json.commission.commissionRates.rate.toDecimal() + fun insertIgnore(blockHeight: Int, valId: Int, operator: String, json: Staking.Validator, isVerified: Boolean) = + transaction { + ValidatorStateTable.insertIgnore { + it[this.operatorAddrId] = valId + it[this.operatorAddress] = operator + it[this.blockHeight] = blockHeight + it[this.moniker] = json.description.moniker + it[this.status] = json.status.name + it[this.jailed] = json.jailed + it[this.tokenCount] = json.tokens.toBigDecimal() + it[this.json] = json + it[this.commissionRate] = json.commission.commissionRates.rate.toDecimal() + it[this.verified] = isVerified + } } - } fun getCommissionHistory(operator: String) = transaction { ValidatorStateRecord.find { ValidatorStateTable.operatorAddress eq operator } @@ -147,7 +151,7 @@ class ValidatorStateRecord(id: EntityID) : IntEntity(id) { } fun findAll(activeSet: Int) = transaction { - val query = "SELECT * FROM get_all_validator_state(?, ?, NULL)".trimIndent() + val query = "SELECT * FROM get_all_validator_state(?, ?, NULL, NULL)".trimIndent() val arguments = mutableListOf>( Pair(IntegerColumnType(), activeSet), Pair(VarCharColumnType(64), Staking.BondStatus.BOND_STATUS_BONDED.name), @@ -156,7 +160,7 @@ class ValidatorStateRecord(id: EntityID) : IntEntity(id) { } fun findByValId(activeSet: Int, id: Int) = transaction { - val query = "SELECT * FROM get_all_validator_state(?, ?, NULL) WHERE operator_addr_id = ?".trimIndent() + val query = "SELECT * FROM get_all_validator_state(?, ?, NULL, NULL) WHERE operator_addr_id = ?".trimIndent() val arguments = mutableListOf>( Pair(IntegerColumnType(), activeSet), Pair(VarCharColumnType(64), Staking.BondStatus.BOND_STATUS_BONDED.name), @@ -169,7 +173,7 @@ class ValidatorStateRecord(id: EntityID) : IntEntity(id) { if (ids.isNotEmpty()) { val idList = ids.joinToString(", ", "(", ")") val query = - "SELECT * FROM get_all_validator_state(?, ?, NULL) WHERE operator_addr_id IN $idList".trimIndent() + "SELECT * FROM get_all_validator_state(?, ?, NULL, NULL) WHERE operator_addr_id IN $idList".trimIndent() val arguments = mutableListOf>( Pair(IntegerColumnType(), activeSet), Pair(VarCharColumnType(64), Staking.BondStatus.BOND_STATUS_BONDED.name), @@ -179,7 +183,7 @@ class ValidatorStateRecord(id: EntityID) : IntEntity(id) { } fun findByAccount(activeSet: Int, address: String) = transaction { - val query = "SELECT * FROM get_all_validator_state(?, ?, NULL) WHERE account_address = ?".trimIndent() + val query = "SELECT * FROM get_all_validator_state(?, ?, NULL, NULL) WHERE account_address = ?".trimIndent() val arguments = mutableListOf>( Pair(IntegerColumnType(), activeSet), Pair(VarCharColumnType(64), Staking.BondStatus.BOND_STATUS_BONDED.name), @@ -189,7 +193,7 @@ class ValidatorStateRecord(id: EntityID) : IntEntity(id) { } fun findByOperator(activeSet: Int, address: String) = transaction { - val query = "SELECT * FROM get_all_validator_state(?, ?, NULL) WHERE operator_address = ?".trimIndent() + val query = "SELECT * FROM get_all_validator_state(?, ?, NULL, NULL) WHERE operator_address = ?".trimIndent() val arguments = mutableListOf>( Pair(IntegerColumnType(), activeSet), Pair(VarCharColumnType(64), Staking.BondStatus.BOND_STATUS_BONDED.name), @@ -199,7 +203,7 @@ class ValidatorStateRecord(id: EntityID) : IntEntity(id) { } fun findByConsensusAddress(activeSet: Int, address: String) = transaction { - val query = "SELECT * FROM get_all_validator_state(?, ?, NULL) WHERE consensus_address = ?".trimIndent() + val query = "SELECT * FROM get_all_validator_state(?, ?, NULL, NULL) WHERE consensus_address = ?".trimIndent() val arguments = mutableListOf>( Pair(IntegerColumnType(), activeSet), Pair(VarCharColumnType(64), Staking.BondStatus.BOND_STATUS_BONDED.name), @@ -209,7 +213,7 @@ class ValidatorStateRecord(id: EntityID) : IntEntity(id) { } fun findByConsensusAddressIn(activeSet: Int, addresses: List) = transaction { - val query = "SELECT * FROM get_all_validator_state(?, ?, ?) ".trimIndent() + val query = "SELECT * FROM get_all_validator_state(?, ?, ?, NULL) ".trimIndent() val arguments = mutableListOf>( Pair(IntegerColumnType(), activeSet), Pair(VarCharColumnType(64), Staking.BondStatus.BOND_STATUS_BONDED.name), @@ -223,32 +227,35 @@ class ValidatorStateRecord(id: EntityID) : IntEntity(id) { searchState: ValidatorState, consensusAddrSet: List? = null, offset: Int? = null, - limit: Int? = null + limit: Int? = null, + searchVerified: Boolean? = null ) = transaction { when (searchState) { ACTIVE, JAILED, CANDIDATE -> { val offsetDefault = offset ?: 0 val limitDefault = limit ?: 10000 - val query = "SELECT * FROM get_validator_list(?, ?, ?, ?, ?, ?) " + val query = "SELECT * FROM get_validator_list(?, ?, ?, ?, ?, ?, ?) " val arguments = mutableListOf>( Pair(IntegerColumnType(), activeSet), Pair(VarCharColumnType(64), Staking.BondStatus.BOND_STATUS_BONDED.name), Pair(TextColumnType(), searchState.name.lowercase()), Pair(IntegerColumnType(), limitDefault), Pair(IntegerColumnType(), offsetDefault), - Pair(ArrayColumnType(TextColumnType()), consensusAddrSet) + Pair(ArrayColumnType(TextColumnType()), consensusAddrSet), + Pair(BooleanColumnType(), searchVerified) ) query.execAndMap(arguments) { it.toCurrentValidatorState() } } ALL -> { val limitOffset = if (offset != null && limit != null) " OFFSET $offset LIMIT $limit" else "" val query = - "SELECT * FROM get_all_validator_state(?, ?, ?) ORDER BY token_count DESC $limitOffset".trimIndent() + "SELECT * FROM get_all_validator_state(?, ?, ?, ?) ORDER BY token_count DESC $limitOffset".trimIndent() val arguments = mutableListOf>( Pair(IntegerColumnType(), activeSet), Pair(VarCharColumnType(64), Staking.BondStatus.BOND_STATUS_BONDED.name), - Pair(ArrayColumnType(TextColumnType()), consensusAddrSet) + Pair(ArrayColumnType(TextColumnType()), consensusAddrSet), + Pair(BooleanColumnType(), searchVerified) ) query.execAndMap(arguments) { it.toCurrentValidatorState() } } @@ -258,29 +265,32 @@ class ValidatorStateRecord(id: EntityID) : IntEntity(id) { fun findByStatusCount( activeSet: Int, searchState: ValidatorState, - consensusAddrSet: List? = null + consensusAddrSet: List? = null, + searchVerified: Boolean? = null ) = transaction { when (searchState) { ACTIVE, JAILED, CANDIDATE -> { val offsetDefault = 0 val limitDefault = 10000 - val query = "SELECT count(*) AS count FROM get_validator_list(?, ?, ?, ?, ?, ?) " + val query = "SELECT count(*) AS count FROM get_validator_list(?, ?, ?, ?, ?, ?, ?) " val arguments = mutableListOf>( Pair(IntegerColumnType(), activeSet), Pair(VarCharColumnType(64), Staking.BondStatus.BOND_STATUS_BONDED.name), Pair(TextColumnType(), searchState.name.lowercase()), Pair(IntegerColumnType(), limitDefault), Pair(IntegerColumnType(), offsetDefault), - Pair(ArrayColumnType(TextColumnType()), consensusAddrSet) + Pair(ArrayColumnType(TextColumnType()), consensusAddrSet), + Pair(BooleanColumnType(), searchVerified) ) query.execAndMap(arguments) { it.toCount() }.first() } ALL -> { - val query = "SELECT count(*) AS count FROM get_all_validator_state(?, ?, ?)".trimIndent() + val query = "SELECT count(*) AS count FROM get_all_validator_state(?, ?, ?, ?)".trimIndent() val arguments = mutableListOf>( Pair(IntegerColumnType(), activeSet), Pair(VarCharColumnType(64), Staking.BondStatus.BOND_STATUS_BONDED.name), - Pair(ArrayColumnType(TextColumnType()), consensusAddrSet) + Pair(ArrayColumnType(TextColumnType()), consensusAddrSet), + Pair(BooleanColumnType(), searchVerified) ) query.execAndMap(arguments) { it.toCount() }.first() } @@ -297,6 +307,7 @@ class ValidatorStateRecord(id: EntityID) : IntEntity(id) { var tokenCount by ValidatorStateTable.tokenCount var json by ValidatorStateTable.json var commissionRate by ValidatorStateTable.commissionRate + var verified by ValidatorStateTable.verified } fun ResultSet.toCurrentValidatorState() = CurrentValidatorState( @@ -312,7 +323,8 @@ fun ResultSet.toCurrentValidatorState() = CurrentValidatorState( this.getString("consensus_address"), this.getString("consensus_pubkey"), ValidatorState.valueOf(this.getString("validator_state").uppercase()), - this.getBigDecimal("commission_rate") + this.getBigDecimal("commission_rate"), + this.getBoolean("verified") ) fun ResultSet.toCount() = this.getLong("count") diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/extensions/DbExtensions.kt b/service/src/main/kotlin/io/provenance/explorer/domain/extensions/DbExtensions.kt index 41335a0e..84961226 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/extensions/DbExtensions.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/extensions/DbExtensions.kt @@ -2,6 +2,7 @@ package io.provenance.explorer.domain.extensions import io.provenance.explorer.OBJECT_MAPPER import org.jetbrains.exposed.sql.IColumnType +import org.jetbrains.exposed.sql.statements.StatementType import org.jetbrains.exposed.sql.transactions.TransactionManager import org.joda.time.DateTime import org.joda.time.DateTimeZone @@ -19,7 +20,7 @@ fun String.exec(args: Iterable>): ResultSet = fun String.execAndMap(args: Iterable> = emptyList(), transform: (ResultSet) -> T): List { val result = arrayListOf() - TransactionManager.current().exec(this, args) { rs -> + TransactionManager.current().exec(this, args, StatementType.SELECT) { rs -> while (rs.next()) { result += transform(rs) } diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/NameModels.kt b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/NameModels.kt new file mode 100644 index 00000000..6a0e6ad1 --- /dev/null +++ b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/NameModels.kt @@ -0,0 +1,30 @@ +package io.provenance.explorer.domain.models.explorer + +data class Name( + val parent: String?, + val child: String, + val fullName: String, + val owner: String, + val restricted: Boolean, + val heightAdded: Int +) + +data class NameObj( + val nameList: List, + val owner: String, + val restricted: Boolean, + val fullName: String +) + +data class NameMap( + val segmentName: String, + val children: MutableList, + val fullName: String, + var owner: String? = null, + var restricted: Boolean = false +) + +data class NameTreeResponse( + val tree: MutableList, + val depthCount: Int +) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/ValidatorModels.kt b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/ValidatorModels.kt index 7ac60c28..936860a8 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/ValidatorModels.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/ValidatorModels.kt @@ -18,14 +18,16 @@ data class ValidatorSummary( val unbondingHeight: Long?, val imgUrl: String?, val hr24Change: String?, - val uptime: BigDecimal + val uptime: BigDecimal, + val isVerified: Boolean ) data class ValidatorSummaryAbbrev( val moniker: String, val addressId: String, val commission: String, - val imgUrl: String? + val imgUrl: String?, + val isVerified: Boolean ) data class ValidatorDetails( @@ -44,7 +46,8 @@ data class ValidatorDetails( val identity: String?, val status: String, val unbondingHeight: Long?, - val jailedUntil: DateTime? + val jailedUntil: DateTime?, + val isVerified: Boolean ) data class Delegation( @@ -102,7 +105,8 @@ data class CurrentValidatorState( val consensusAddr: String, val consensusPubKey: String, val currentState: ValidatorState, - val commissionRate: BigDecimal + val commissionRate: BigDecimal, + val verified: Boolean ) data class BlockLatencyData( diff --git a/service/src/main/kotlin/io/provenance/explorer/grpc/extensions/MsgConverter.kt b/service/src/main/kotlin/io/provenance/explorer/grpc/extensions/MsgConverter.kt index 5f8ac3e1..33e0b656 100644 --- a/service/src/main/kotlin/io/provenance/explorer/grpc/extensions/MsgConverter.kt +++ b/service/src/main/kotlin/io/provenance/explorer/grpc/extensions/MsgConverter.kt @@ -678,6 +678,23 @@ enum class SmContractEventKeys(val eventType: String, val eventKey: Map this + else -> null.also { logger().debug("This typeUrl is not yet supported in as a Name msg: $typeUrl") } + } + // ///////// DENOM EVENTS enum class DenomEvents(val event: String, val idField: String, val parse: Boolean = false) { TRANSFER("transfer", "amount", true), diff --git a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/AttributeGrpcClient.kt b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/AttributeGrpcClient.kt index d1206393..61533ea3 100644 --- a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/AttributeGrpcClient.kt +++ b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/AttributeGrpcClient.kt @@ -6,6 +6,7 @@ import io.provenance.attribute.v1.queryAttributesRequest import io.provenance.attribute.v1.queryParamsRequest import io.provenance.explorer.config.GrpcLoggingInterceptor import io.provenance.explorer.grpc.extensions.getPagination +import io.provenance.name.v1.queryResolveRequest import io.provenance.name.v1.queryReverseLookupRequest import org.springframework.stereotype.Component import java.net.URI @@ -76,6 +77,13 @@ class AttributeGrpcClient(channelUri: URI) { } ) + suspend fun getOwnerForName(name: String) = + try { + nameClient.resolve(queryResolveRequest { this.name = name }) + } catch (e: Exception) { + null + } + suspend fun getAttrParams() = attrClient.params(queryParamsRequest { }) suspend fun getNameParams() = nameClient.params(io.provenance.name.v1.queryParamsRequest { }) } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/NameService.kt b/service/src/main/kotlin/io/provenance/explorer/service/NameService.kt new file mode 100644 index 00000000..5830627c --- /dev/null +++ b/service/src/main/kotlin/io/provenance/explorer/service/NameService.kt @@ -0,0 +1,59 @@ +package io.provenance.explorer.service + +import io.provenance.explorer.config.ExplorerProperties +import io.provenance.explorer.domain.entities.NameRecord +import io.provenance.explorer.domain.models.explorer.Name +import io.provenance.explorer.domain.models.explorer.NameMap +import io.provenance.explorer.domain.models.explorer.NameObj +import io.provenance.explorer.domain.models.explorer.NameTreeResponse +import io.provenance.explorer.grpc.v1.AttributeGrpcClient +import kotlinx.coroutines.runBlocking +import org.jetbrains.exposed.sql.transactions.transaction +import org.springframework.stereotype.Service + +@Service +class NameService( + private val attrClient: AttributeGrpcClient, + private val props: ExplorerProperties +) { + + fun getNameMap() = transaction { + val nameSet = NameRecord.getNameSet() + val tree = mutableListOf() + nameSet.forEach { recurseMainTree(it, tree, 0, nameSet) } + val depthCount = nameSet.map { it.nameList }.maxOf { it.size } + NameTreeResponse(tree, depthCount) + } + + fun recurseMainTree(obj: NameObj, tree: MutableList, idx: Int, nameSet: List) { + val segment = obj.nameList.getOrNull(idx) ?: return + val parentObj = tree.firstOrNull { it.segmentName == segment } + if (parentObj == null) { + val fullName = obj.nameList.subList(0, idx + 1).reversed().joinToString(".") + val fullObj = nameSet.firstOrNull { it.fullName == fullName } ?: updateName(fullName) + val parent = NameMap(segment, mutableListOf(), fullName, fullObj?.owner, fullObj?.restricted ?: false) + tree.add(parent) + } + val childList = tree.first { it.segmentName == segment } + recurseMainTree(obj, childList.children, idx + 1, nameSet) + } + + fun updateName(name: String) = runBlocking { + attrClient.getOwnerForName(name)?.let { res -> + val (child, parent) = name.splitChildParent() + NameRecord.insertOrUpdate(Name(parent, child, name, res.address, true, 0)) + NameObj(name.split(".").reversed(), res.address, true, name) + } + } + + fun getVerifiedKycAttributes() = transaction { + NameRecord.getNamesByOwners(props.verifiedAddresses) + .filter { it.fullName.contains("kyc") } + .map { it.fullName } + .toSet() + } +} + +// Splits from `figuretest2.kyc.pb` -> Pair(`kyc.pb`, `kyc.pb`) +// Splits from `pb` -> Pair(`pb`, null) +fun String.splitChildParent() = this.split(".").let { it[0] to it.getOrNull(1) } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt b/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt index af84e6b2..a0b1c4d6 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt @@ -65,7 +65,9 @@ import io.provenance.explorer.domain.models.explorer.ValidatorSummaryAbbrev import io.provenance.explorer.domain.models.explorer.ValidatorUptimeStats import io.provenance.explorer.domain.models.explorer.hourlyBlockCount import io.provenance.explorer.grpc.extensions.toAddress +import io.provenance.explorer.grpc.v1.AttributeGrpcClient import io.provenance.explorer.grpc.v1.ValidatorGrpcClient +import kotlinx.coroutines.async import kotlinx.coroutines.runBlocking import org.jetbrains.exposed.sql.transactions.transaction import org.joda.time.DateTime @@ -79,13 +81,22 @@ class ValidatorService( private val props: ExplorerProperties, private val blockService: BlockService, private val grpcClient: ValidatorGrpcClient, - private val cacheService: CacheService + private val cacheService: CacheService, + private val attrClient: AttributeGrpcClient, + private val nameService: NameService ) { protected val logger = logger(ValidatorService::class) fun getActiveSet() = grpcClient.getStakingParams().params.maxValidators + fun isVerified(address: String) = + runBlocking { + val atts = async { attrClient.getAllAttributesForAddress(address) }.await().map { it.name } + val kycAtts = nameService.getVerifiedKycAttributes() + atts.intersect(kycAtts).isNotEmpty() + } + // Assumes that the returned validators are active at that height fun getValidatorsByHeight(blockHeight: Int) = transaction { ValidatorsCacheRecord.findById(blockHeight)?.also { @@ -101,7 +112,7 @@ class ValidatorService( transaction { ValidatorStateRecord.findByOperator(getActiveSet(), operatorAddress) } ?: saveValidator(operatorAddress) - fun saveValidator(address: String) = transaction { + fun saveValidator(address: String) = runBlocking { grpcClient.getStakingValidator(address) .let { StakingValidatorCacheRecord.insertIgnore( @@ -114,7 +125,8 @@ class ValidatorService( blockService.getLatestBlockHeightIndex(), record.id.value, it.operatorAddress, - it + it, + isVerified(record.accountAddress), ) } }.also { ValidatorStateRecord.refreshCurrentStateView() } @@ -157,7 +169,8 @@ class ValidatorService( stakingValidator.json.description.identity, stakingValidator.json.getStatusString(), if (stakingValidator.currentState != ACTIVE) stakingValidator.json.unbondingHeight else null, - if (stakingValidator.jailed) signingInfo?.jailedUntil?.toDateTime() else null + if (stakingValidator.jailed) signingInfo?.jailedUntil?.toDateTime() else null, + stakingValidator.verified ) } ?: throw ResourceNotFoundException("Invalid validator address: '$address'") @@ -218,7 +231,8 @@ class ValidatorService( blockHeight, it.id.value, validator.operatorAddress, - validator + validator, + isVerified(it.accountAddress) ) }.let { true } else false @@ -234,7 +248,7 @@ class ValidatorService( val record = ValidatorStateRecord.findByValId(getActiveSet(), v)!! val data = grpcClient.getStakingValidator(record.operatorAddress) if (record.blockHeight < height && data != record.json) - ValidatorStateRecord.insertIgnore(height, v, record.operatorAddress, data) + ValidatorStateRecord.insertIgnore(height, v, record.operatorAddress, data, isVerified(record.accountAddr)) .also { if (!updated) updated = true } } return updated @@ -247,7 +261,8 @@ class ValidatorService( currVal.json.description.moniker, currVal.operatorAddress, currVal.json.commission.commissionRates.rate.toDecCoin(), - getImgUrl(currVal.json.description.identity) + getImgUrl(currVal.json.description.identity), + currVal.verified ) } PagedResults(recs.size.toLong().pageCountOfResults(recs.size), recs, recs.size.toLong()) @@ -317,7 +332,8 @@ class ValidatorService( hr24Change = get24HrBondedChange(validator, hr24Validator), uptime = stakingVal.consensusAddr.validatorUptime( grpcClient.getSlashingParams().params.signedBlocksWindow.toBigInteger(), height.toBigInteger() - ) + ), + isVerified = stakingVal.verified ) } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncCachingV2.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncCachingV2.kt index e2e92166..de68ec09 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncCachingV2.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncCachingV2.kt @@ -18,6 +18,7 @@ import io.provenance.explorer.domain.entities.FeePayer import io.provenance.explorer.domain.entities.IbcAckType import io.provenance.explorer.domain.entities.IbcLedgerRecord import io.provenance.explorer.domain.entities.IbcRelayerRecord +import io.provenance.explorer.domain.entities.NameRecord import io.provenance.explorer.domain.entities.SigJoinType import io.provenance.explorer.domain.entities.SignatureJoinRecord import io.provenance.explorer.domain.entities.TxAddressJoinRecord @@ -49,12 +50,14 @@ import io.provenance.explorer.domain.extensions.translateAddress import io.provenance.explorer.domain.models.explorer.BlockProposer import io.provenance.explorer.domain.models.explorer.BlockUpdate import io.provenance.explorer.domain.models.explorer.MsgProtoBreakout +import io.provenance.explorer.domain.models.explorer.Name import io.provenance.explorer.domain.models.explorer.TxData import io.provenance.explorer.domain.models.explorer.TxUpdate import io.provenance.explorer.domain.models.explorer.toProcedureObject import io.provenance.explorer.grpc.extensions.AddressEvents import io.provenance.explorer.grpc.extensions.DenomEvents import io.provenance.explorer.grpc.extensions.GovMsgType +import io.provenance.explorer.grpc.extensions.NameEvents import io.provenance.explorer.grpc.extensions.SmContractEventKeys import io.provenance.explorer.grpc.extensions.SmContractValue import io.provenance.explorer.grpc.extensions.denomEventRegexParse @@ -67,6 +70,8 @@ import io.provenance.explorer.grpc.extensions.getAssociatedMetadataEvents import io.provenance.explorer.grpc.extensions.getAssociatedSmContractMsgs import io.provenance.explorer.grpc.extensions.getDenomEventByEvent import io.provenance.explorer.grpc.extensions.getIbcLedgerMsgs +import io.provenance.explorer.grpc.extensions.getNameEventTypes +import io.provenance.explorer.grpc.extensions.getNameMsgs import io.provenance.explorer.grpc.extensions.getSmContractEventByEvent import io.provenance.explorer.grpc.extensions.getTxIbcClientChannel import io.provenance.explorer.grpc.extensions.isIbcTransferMsg @@ -74,6 +79,8 @@ import io.provenance.explorer.grpc.extensions.isMetadataDeletionMsg import io.provenance.explorer.grpc.extensions.mapEventAttrValues import io.provenance.explorer.grpc.extensions.scrubQuotes import io.provenance.explorer.grpc.extensions.toMsgAcknowledgement +import io.provenance.explorer.grpc.extensions.toMsgBindNameRequest +import io.provenance.explorer.grpc.extensions.toMsgDeleteNameRequest import io.provenance.explorer.grpc.extensions.toMsgDeposit import io.provenance.explorer.grpc.extensions.toMsgRecvPacket import io.provenance.explorer.grpc.extensions.toMsgSubmitProposal @@ -93,6 +100,7 @@ import io.provenance.explorer.service.IbcService import io.provenance.explorer.service.NftService import io.provenance.explorer.service.SmartContractService import io.provenance.explorer.service.ValidatorService +import io.provenance.explorer.service.splitChildParent import net.pearx.kasechange.toSnakeCase import net.pearx.kasechange.universalWordSplitter import org.jetbrains.exposed.sql.transactions.transaction @@ -129,7 +137,7 @@ class AsyncCachingV2( fun saveBlockEtc( blockRes: Query.GetBlockByHeightResponse?, - rerunTxs: Boolean = false + rerunTxs: Pair = Pair(false, false) // rerun txs, pull from db ): Query.GetBlockByHeightResponse? { if (blockRes == null) return null logger.info("saving block ${blockRes.block.height()}") @@ -170,7 +178,7 @@ class AsyncCachingV2( fun saveTxs( blockRes: Query.GetBlockByHeightResponse, proposerRec: BlockProposer, - rerunTxs: Boolean = false + rerunTxs: Pair = Pair(false, false) // rerun txs, pull from db ): List { val toBeUpdated = addTxsToCache( @@ -204,24 +212,28 @@ class AsyncCachingV2( expectedNumTxs: Int, blockTime: Timestamp, proposerRec: BlockProposer, - rerunTxs: Boolean = false + rerunTxs: Pair = Pair(false, false) // rerun txs, pull from db ) = - if (txCountForHeight(blockHeight).toInt() == expectedNumTxs && !rerunTxs) + if (txCountForHeight(blockHeight).toInt() == expectedNumTxs && !rerunTxs.first) logger.info("Cache hit for transaction at height $blockHeight with $expectedNumTxs transactions") .let { listOf() } else { logger.info("Searching for $expectedNumTxs transactions at height $blockHeight") - tryAddTxs(blockHeight, expectedNumTxs, blockTime, proposerRec) + tryAddTxs(blockHeight, expectedNumTxs, blockTime, proposerRec, rerunTxs.second) } private fun tryAddTxs( blockHeight: Int, txCount: Int, blockTime: Timestamp, - proposerRec: BlockProposer + proposerRec: BlockProposer, + pullFromDb: Boolean = false ): List = try { - txClient.getTxsByHeight(blockHeight, txCount) - .map { addTxToCacheWithTimestamp(txClient.getTxByHash(it.txhash), blockTime, proposerRec) } + if (pullFromDb) + TxCacheRecord.findByHeight(blockHeight).map { addTxToCacheWithTimestamp(it.txV2, blockTime, proposerRec) } + else + txClient.getTxsByHeight(blockHeight, txCount) + .map { addTxToCacheWithTimestamp(txClient.getTxByHash(it.txhash), blockTime, proposerRec) } } catch (e: Exception) { logger.error("Failed to retrieve transactions at block: $blockHeight", e.message) BlockTxRetryRecord.insert(blockHeight, e) @@ -252,6 +264,7 @@ class AsyncCachingV2( saveGovData(res, txInfo, txUpdate) saveIbcChannelData(res, txInfo, txUpdate) saveSmartContractData(res, txInfo, txUpdate) + saveNameData(res, txInfo) saveSignaturesTx(res, txInfo, txUpdate) return TxUpdatedItems(addrs, markers, txUpdate) } @@ -623,6 +636,62 @@ class AsyncCachingV2( .let { txUpdate.apply { this.smContracts.addAll(it) } } } + private fun saveNameData(tx: ServiceOuterClass.GetTxResponse, txInfo: TxData) = transaction { + if (tx.txResponse.code == 0) { + val insertList = mutableListOf() + tx.tx.body.messagesList.mapNotNull { it.getNameMsgs() } + .forEach { any -> + when (any.typeUrl) { + NameEvents.NAME_BIND.msg -> + any.toMsgBindNameRequest().let { + Name( + if (it.hasParent()) it.parent.name else null, + it.record.name, + it.record.name + (if (it.hasParent()) ".${it.parent.name}" else ""), + it.record.address, + it.record.restricted, + txInfo.blockHeight + ) + }.let { insertList.add(it) } + NameEvents.NAME_DELETE.msg -> + any.toMsgDeleteNameRequest().let { + NameRecord.deleteByFullNameAndOwner( + it.record.name, + it.record.address, + txInfo.blockHeight + ) + } + } + } + + tx.txResponse.logsList + .flatMap { it.eventsList } + .filter { getNameEventTypes().contains(it.type) } + .map { e -> + when (e.type) { + NameEvents.NAME_BIND.event -> { + val groupedAtts = e.attributesList.groupBy({ it.key }) { it.value } + val addrList = groupedAtts["address"]!! + groupedAtts["name"]!!.forEachIndexed { idx, name -> + val (child, parent) = name.splitChildParent() + val obj = Name(parent, child, name, addrList[idx], false, txInfo.blockHeight) + if (insertList.firstOrNull { it.fullName == name } == null) + insertList.add(obj) + } + } + NameEvents.NAME_DELETE.event -> { + val groupedAtts = e.attributesList.groupBy({ it.key }) { it.value } + val addrList = groupedAtts["address"]!! + groupedAtts["name"]!!.forEachIndexed { idx, name -> + NameRecord.deleteByFullNameAndOwner(name, addrList[idx], txInfo.blockHeight) + } + } + } + } + insertList.forEach { NameRecord.insertOrUpdate(it) } + } + } + fun saveSignaturesTx(tx: ServiceOuterClass.GetTxResponse, txInfo: TxData, txUpdate: TxUpdate) = transaction { tx.tx.authInfo.signerInfosList.flatMap { sig -> SignatureJoinRecord.buildInsert(sig.publicKey, SigJoinType.TRANSACTION, tx.txResponse.txhash) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt index cae292f7..b5c4bee9 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt @@ -131,7 +131,7 @@ class AsyncService( } } - @Scheduled(cron = "0 0/20 * * * ?") // Every 20 minutes + @Scheduled(cron = "0 0/5 * * * ?") // Every 5 minutes fun performProposalUpdates() = transaction { logger.info("Performing proposal updates") GovProposalRecord.getNonFinalProposals().forEach { govService.updateProposal(it) } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/utility/MigrationService.kt b/service/src/main/kotlin/io/provenance/explorer/service/utility/MigrationService.kt index 4107a759..df219b0d 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/utility/MigrationService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/utility/MigrationService.kt @@ -33,7 +33,7 @@ class MigrationService( logger.info("Fetching $start to ${start + inc - 1}") BlockCacheRecord.find { BlockCacheTable.id.between(start, start + inc - 1) } .orderBy(Pair(BlockCacheTable.id, SortOrder.ASC)).forEach { - asyncCaching.saveBlockEtc(it.block, true) + asyncCaching.saveBlockEtc(it.block, Pair(true, false)) } } start += inc @@ -41,10 +41,10 @@ class MigrationService( logger.info("End height: $endHeight") } - fun insertBlocks(blocks: List) = transaction { + fun insertBlocks(blocks: List, pullFromDb: Boolean) = transaction { blocks.forEach { block -> blockService.getBlockAtHeightFromChain(block)?.let { - asyncCaching.saveBlockEtc(it, true) + asyncCaching.saveBlockEtc(it, Pair(true, pullFromDb)) } } } diff --git a/service/src/main/kotlin/io/provenance/explorer/web/v2/NameController.kt b/service/src/main/kotlin/io/provenance/explorer/web/v2/NameController.kt new file mode 100644 index 00000000..053aac56 --- /dev/null +++ b/service/src/main/kotlin/io/provenance/explorer/web/v2/NameController.kt @@ -0,0 +1,27 @@ +package io.provenance.explorer.web.v2 + +import io.provenance.explorer.service.NameService +import io.swagger.annotations.Api +import io.swagger.annotations.ApiOperation +import org.springframework.http.MediaType +import org.springframework.http.ResponseEntity +import org.springframework.validation.annotation.Validated +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@Validated +@RestController +@RequestMapping(path = ["/api/v2/names"], produces = [MediaType.APPLICATION_JSON_VALUE]) +@Api( + description = "Name endpoints", + produces = MediaType.APPLICATION_JSON_VALUE, + consumes = MediaType.APPLICATION_JSON_VALUE, + tags = ["Name"] +) +class NameController(private val nameService: NameService) { + + @ApiOperation("Returns tree of names") + @GetMapping("/tree") + fun getNameTree() = ResponseEntity.ok(nameService.getNameMap()) +} diff --git a/service/src/main/kotlin/io/provenance/explorer/web/v2/utility/MigrationController.kt b/service/src/main/kotlin/io/provenance/explorer/web/v2/utility/MigrationController.kt index 4558e7c4..ffb09f9d 100644 --- a/service/src/main/kotlin/io/provenance/explorer/web/v2/utility/MigrationController.kt +++ b/service/src/main/kotlin/io/provenance/explorer/web/v2/utility/MigrationController.kt @@ -40,9 +40,10 @@ class MigrationController(private val migrationService: MigrationService, privat fun updateBlocks(@RequestParam start: Int, @RequestParam end: Int, @RequestParam inc: Int) = ResponseEntity.ok(migrationService.updateBlocks(start, end, inc)) - @ApiOperation("Updates blocks from list") + @ApiOperation("Updates blocks from list, specifying whether to reprocess from DB or chain") @PutMapping("/update/blocks/list") - fun updateBlocksList(@RequestBody blocks: List) = ResponseEntity.ok(migrationService.insertBlocks(blocks)) + fun updateBlocksList(@RequestParam pullFromDb: Boolean, @RequestBody blocks: List) = + ResponseEntity.ok(migrationService.insertBlocks(blocks, pullFromDb)) @ApiOperation("Updates denom units") @GetMapping("/update/denom/units") diff --git a/service/src/main/resources/application-container.properties b/service/src/main/resources/application-container.properties index 9fc71b1f..5610a4e3 100644 --- a/service/src/main/resources/application-container.properties +++ b/service/src/main/resources/application-container.properties @@ -19,3 +19,4 @@ explorer.hidden-apis=${EXPLORER_HIDDEN_APIS} explorer.swagger-url=${EXPLORER_SWAGGER_URL} explorer.swagger-protocol=${EXPLORER_SWAGGER_PROTOCOL} explorer.pricing-url=${EXPLORER_PRICING_URL} +explorer.verified-addresses=${EXPLORER_VERIFIED_ADDRESSES} diff --git a/service/src/main/resources/application-development.properties b/service/src/main/resources/application-development.properties index e69fa336..fd1f4fdc 100644 --- a/service/src/main/resources/application-development.properties +++ b/service/src/main/resources/application-development.properties @@ -22,6 +22,7 @@ explorer.swagger-protocol=http #explorer.figment-url=https://pio-mainnet-1--lcd--archive.datahub.figment.io #explorer.pricing-url=https://figure.tech/service-pricing-engine/service-pricing-engine #explorer.genesis-version-url=https://github.com/provenance-io/provenance/releases/download/v1.0.1/plan-v1.0.1.json +#explorer.verified-addresses=test #### TESTNET SETTINGS @@ -30,4 +31,5 @@ explorer.pricing-url=https://test.figure.tech/service-pricing-engine/service-pri explorer.figment-url=https://pio-testnet-1--lcd.datahub.figment.io explorer.pb-url=grpcs://grpc.test.provenance.io:443 explorer.genesis-version-url=https://github.com/provenance-io/provenance/releases/download/v0.2.0/plan-v0.2.0.json +explorer.verified-addresses=tp1v586fwt55n59cmh38w8ny2fxaxejg94kkdxafq