Skip to content

Commit

Permalink
Saving applicable block heights for gov params (#363)
Browse files Browse the repository at this point in the history
* Allows us to pull param sets at height vs latest always
* Separated out Proposal votes into its own paginated API
* Added functionality to support these and future usecases
* Fixing a couple bugs for IBC Recv and Gov Tx detail parsing

closes: #341
closes: #362
  • Loading branch information
minxylynx authored Jun 7, 2022
1 parent 9f4fe9b commit 51648e7
Show file tree
Hide file tree
Showing 26 changed files with 592 additions and 214 deletions.
19 changes: 19 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,27 @@ Ref: https://keepachangelog.com/en/1.0.0/

## Unreleased

### Features
* Separated out Proposal votes into a paginated API #362
* Deprecated `/api/v2/gov/proposals/{id}/votes` in favor of `/api/v3/gov/proposals/{id}/votes`
* Moved Voting params and counts into the proposal detail / proposal listview responses

### Improvements
* Now saving applicable block heights to pull governance param sets #341
* This is used to populate past proposals/votes/deposits based on the contemporary parameters on chain
* Now have the ability to fetch data at height from the chain
* Added Exposed functionality for Arrays and ArrayAgg Postgres function

### Bug Fixes
* Fixed the Validator missed block count
* Added else case for IBC Recvs where the effected Recv is in the same tx as an uneffected Recv, which makes the block error out
* Added VotWeighted msg type to `getGovMsgDetail()` function

### Data
* Migration 1.64 - Updates for gov params at height #341
* Updated `gov_proposal` with `deposit_param_check_height`, `voting_param_check_height`, inserted data
* Updated `insert_gov_proposal()` procedure
* Created `get_last_block_before_timestamp()` function

## [v4.2.0](https://github.com/provenance-io/explorer-service/releases/tag/v4.2.0) - 2022-05-24
### Release Name: Odoric of Pordenone
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
SELECT 'Updating gov_proposal table' AS comment;

ALTER TABLE gov_proposal
ADD COLUMN IF NOT EXISTS deposit_param_check_height INT NOT NULL DEFAULT -1,
ADD COLUMN IF NOT EXISTS voting_param_check_height INT NOT NULL DEFAULT -1;

CREATE INDEX IF NOT EXISTS block_timestamp_minute_idx ON block_cache (date_trunc('minute', block_timestamp));

SELECT 'Inserting new gov_proposal data columns' AS comment;
WITH base AS (
SELECT proposal_id,
CASE
WHEN data ->> 'voting_start_time' = '0001-01-01T00:00:00Z' THEN (data ->> 'deposit_end_time')::timestamp
ELSE
(data ->> 'voting_start_time')::timestamp END AS vote_start,
data ->> 'voting_end_time' AS voting_end_str,
CASE
WHEN data ->> 'voting_end_time' = '0001-01-01T00:00:00Z' THEN null
ELSE
(data ->> 'voting_end_time')::timestamp END AS voting_end
FROM gov_proposal gp
ORDER BY proposal_id
),
deposit_blocks AS (
SELECT height,
block_timestamp
FROM block_cache bc,
base
WHERE date_trunc('minute', bc.block_timestamp) = date_trunc('minute', base.vote_start)
-- allows us to capture the blocks on the frontside or backside of the minute interval
OR date_trunc('minute', bc.block_timestamp) = date_trunc('minute', base.vote_start) + INTERVAL '1 MINUTE'
OR date_trunc('minute', bc.block_timestamp) = date_trunc('minute', base.vote_start) - INTERVAL '1 MINUTE'
),
deposit_match AS (
SELECT base.proposal_id,
base.vote_start,
max(db.height) AS height
FROM deposit_blocks db,
base
WHERE block_timestamp <= base.vote_start
GROUP BY base.proposal_id, base.vote_start
),
voting_blocks AS (
SELECT height,
block_timestamp
FROM block_cache bc,
base
WHERE date_trunc('minute', bc.block_timestamp) = date_trunc('minute', base.voting_end)
-- allows us to capture the blocks on the frontside or backside of the minute interval
OR date_trunc('minute', bc.block_timestamp) = date_trunc('minute', base.voting_end) + INTERVAL '1 MINUTE'
OR date_trunc('minute', bc.block_timestamp) = date_trunc('minute', base.voting_end) - INTERVAL '1 MINUTE'
),
voting_match AS (
SELECT base.proposal_id,
base.voting_end,
max(vb.height) AS height
FROM voting_blocks vb,
base
WHERE base.voting_end IS NOT NULL
AND block_timestamp <= base.voting_end
GROUP BY base.proposal_id, base.voting_end
)
UPDATE gov_proposal gp
SET deposit_param_check_height = q.deposit_height,
voting_param_check_height = q.voting_height
FROM (
SELECT base.proposal_id,
base.vote_start AS deposit_param_height,
COALESCE(dm.height, -1) deposit_height,
base.voting_end AS voting_param_height,
COALESCE(vm.height, -1) voting_height
FROM base
LEFT JOIN deposit_match dm ON base.proposal_id = dm.proposal_id
LEFT JOIN voting_match vm ON base.proposal_id = vm.proposal_id
) q
WHERE gp.proposal_id = q.proposal_id;

SELECT 'Updating insert_gov_proposal() procedure' AS comment;
create or replace procedure insert_gov_proposal(proposals gov_proposal[], tx_height integer, tx_id integer,
timez timestamp without time zone)
language plpgsql as
$$
DECLARE
gp gov_proposal;
BEGIN
FOREACH gp IN ARRAY proposals
LOOP
INSERT INTO gov_proposal(proposal_id, proposal_type, address_id, address, is_validator, title, description,
status, data, content, block_height, tx_hash, tx_timestamp, tx_hash_id,
deposit_param_check_height, voting_param_check_height)
VALUES (gp.proposal_id,
gp.proposal_type,
gp.address_id,
gp.address,
gp.is_validator,
gp.title,
gp.description,
gp.status,
gp.data,
gp.content,
tx_height,
gp.tx_hash,
timez,
tx_id,
gp.deposit_param_check_height,
gp.voting_param_check_height)
ON CONFLICT (proposal_id) DO UPDATE
SET status = gp.status,
data = gp.data,
tx_hash = gp.tx_hash,
tx_timestamp = timez,
block_height = tx_height,
deposit_param_check_height = gp.deposit_param_check_height,
voting_param_check_height = gp.voting_param_check_height;
END LOOP;
END;
$$;

SELECT 'Creating get_last_block_before_timestamp() function' AS comment;
CREATE OR REPLACE FUNCTION get_last_block_before_timestamp(input timestamp without time zone DEFAULT NULL::timestamp without time zone)
RETURNS integer
LANGUAGE plpgsql
AS
$$
BEGIN
IF (input IS NULL) THEN
RETURN -1;
ELSE
RETURN (
WITH voting_blocks AS (
SELECT height, block_timestamp
FROM block_cache bc
WHERE date_trunc('minute', bc.block_timestamp) = date_trunc('minute', input)
-- allows us to capture the blocks on the frontside or backside of the minute interval
OR date_trunc('minute', bc.block_timestamp) = date_trunc('minute', input) + INTERVAL '1 MINUTE'
OR date_trunc('minute', bc.block_timestamp) = date_trunc('minute', input) - INTERVAL '1 MINUTE')
SELECT COALESCE(max(vb.height), -1) AS height
FROM voting_blocks vb
WHERE input IS NOT NULL
AND block_timestamp <= input
LIMIT 1
);
END IF;
END
$$;
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ class SwaggerConfig(val props: ExplorerProperties) {
.forCodeGeneration(true)
.securitySchemes(listOf())
.select()
.apis(RequestHandlerSelectors.basePackage("io.provenance.explorer.web.v2"))
.apis(RequestHandlerSelectors.basePackage("io.provenance.explorer.web"))

if (props.hiddenApis())
docket.apis(Predicate.not(RequestHandlerSelectors.withClassAnnotation(HiddenApi::class.java)))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import org.jetbrains.exposed.sql.QueryParameter
import org.jetbrains.exposed.sql.Table
import org.jetbrains.exposed.sql.statements.jdbc.JdbcConnectionImpl
import org.jetbrains.exposed.sql.transactions.TransactionManager
import kotlin.Array

fun <T> Table.array(name: String, columnType: ColumnType): Column<Array<T>> = registerColumn(name, ArrayColumnType(columnType))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ import org.jetbrains.exposed.sql.Function
import org.jetbrains.exposed.sql.IColumnType
import org.jetbrains.exposed.sql.IntegerColumnType
import org.jetbrains.exposed.sql.QueryBuilder
import org.jetbrains.exposed.sql.TextColumnType
import org.jetbrains.exposed.sql.VarCharColumnType
import org.jetbrains.exposed.sql.append
import org.jetbrains.exposed.sql.jodatime.CustomDateTimeFunction
import org.jetbrains.exposed.sql.jodatime.DateColumnType
import org.jetbrains.exposed.sql.stringLiteral
import org.joda.time.DateTime
import java.math.BigDecimal
import kotlin.Array

// Generic Distinct function
class Distinct<T>(val expr: Expression<T>, _columnType: IColumnType) : Function<T>(_columnType) {
Expand Down Expand Up @@ -68,3 +70,13 @@ class ColumnNullsLast(private val col: Expression<*>) : Expression<String>() {
fun Expression<*>.nullsLast() = ColumnNullsLast(this)

fun EntityID<Int>?.getOrNull() = try { this?.value } catch (e: Exception) { null }

class Array(val exprs: List<Expression<String>>) : Function<Array<String>>(ArrayColumnType(TextColumnType())) {
override fun toQueryBuilder(queryBuilder: QueryBuilder) = queryBuilder { append("ARRAY [", exprs.joinToString(","), "]") }
}

fun List<Expression<*>>.joinToList() = this.joinToString(",") { "$it::text" }

class ArrayAgg(val expr: Expression<Array<String>>) : Function<Array<Array<String>>>(ArrayColumnType(ArrayColumnType(TextColumnType()))) {
override fun toQueryBuilder(queryBuilder: QueryBuilder) = queryBuilder { append("array_agg(", expr, ")") }
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@ import io.provenance.explorer.domain.core.sql.jsonb
import io.provenance.explorer.domain.core.sql.toProcedureObject
import io.provenance.explorer.domain.extensions.average
import io.provenance.explorer.domain.extensions.exec
import io.provenance.explorer.domain.extensions.execAndMap
import io.provenance.explorer.domain.extensions.map
import io.provenance.explorer.domain.extensions.startOfDay
import io.provenance.explorer.domain.models.explorer.BlockProposer
import io.provenance.explorer.domain.models.explorer.BlockUpdate
import io.provenance.explorer.domain.models.explorer.DateTruncGranularity
import io.provenance.explorer.domain.models.explorer.DateTruncGranularity.DAY
import io.provenance.explorer.domain.models.explorer.DateTruncGranularity.HOUR
import io.provenance.explorer.domain.models.explorer.DateTruncGranularity.MINUTE
import io.provenance.explorer.domain.models.explorer.MissedBlockPeriod
import io.provenance.explorer.domain.models.explorer.TxHeatmap
import io.provenance.explorer.domain.models.explorer.TxHeatmapDay
Expand Down Expand Up @@ -84,14 +86,17 @@ class BlockCacheRecord(id: EntityID<Int>) : CacheEntity<Int>(id) {
fun getDaysBetweenHeights(minHeight: Int, maxHeight: Int) = transaction {
val dateTrunc =
Distinct(
DateTrunc(DateTruncGranularity.DAY.name, BlockCacheTable.blockTimestamp),
DateTrunc(DAY.name, BlockCacheTable.blockTimestamp),
DateColumnType(true)
).count()
BlockCacheTable.slice(dateTrunc)
.select { BlockCacheTable.height.between(minHeight, maxHeight) }
.first()[dateTrunc].toInt()
}

fun test() = transaction {
}

fun getMaxBlockHeightOrNull() = transaction {
val maxHeight = Max(BlockCacheTable.height, IntegerColumnType())
BlockCacheTable.slice(maxHeight).selectAll().firstOrNull()?.let { it[maxHeight] }
Expand All @@ -107,6 +112,12 @@ class BlockCacheRecord(id: EntityID<Int>) : CacheEntity<Int>(id) {
.orderBy(BlockCacheTable.height to SortOrder.ASC)
.first()
}

fun getLastBlockBeforeTime(time: DateTime?) = transaction {
val query = "SELECT get_last_block_before_timestamp(?);"
val arguments = listOf(Pair(DateColumnType(true), time))
query.execAndMap(arguments) { it.getInt("get_last_block_before_timestamp") }.first()
}
}

var height by BlockCacheTable.height
Expand Down Expand Up @@ -333,6 +344,7 @@ class BlockCacheHourlyTxCountsRecord(id: EntityID<DateTime>) : Entity<DateTime>(
when (granularity) {
HOUR -> getHourlyCounts(fromDate, toDate)
DAY -> getDailyCounts(fromDate, toDate)
MINUTE -> emptyList()
}
}

Expand Down Expand Up @@ -382,7 +394,7 @@ class BlockCacheHourlyTxCountsRecord(id: EntityID<DateTime>) : Entity<DateTime>(
}

private fun getDailyCounts(fromDate: DateTime, toDate: DateTime) = transaction {
val dateTrunc = DateTrunc("DAY", BlockCacheHourlyTxCountsTable.blockTimestamp)
val dateTrunc = DateTrunc(DAY.name, BlockCacheHourlyTxCountsTable.blockTimestamp)
val txSum = BlockCacheHourlyTxCountsTable.txCount.sum()
BlockCacheHourlyTxCountsTable.slice(dateTrunc, txSum)
.select {
Expand Down Expand Up @@ -480,6 +492,15 @@ class BlockTxRetryRecord(id: EntityID<Int>) : IntEntity(id) {
}
}

fun insertNonBlockingRetry(height: Int, e: Exception) = transaction {
BlockTxRetryTable.insertIgnore {
it[this.height] = height
it[this.errorBlock] =
"NON BLOCKING ERROR: Logged to know what happened, but didnt stop processing.\n " +
e.stackTraceToString()
}
}

fun getRecordsToRetry() = transaction {
BlockTxRetryRecord
.find { (BlockTxRetryTable.retried eq false) and (BlockTxRetryTable.success eq false) }
Expand Down
Loading

0 comments on commit 51648e7

Please sign in to comment.