Skip to content

Commit

Permalink
Merge pull request #54 from provenance-io/crussell/more-things
Browse files Browse the repository at this point in the history
Improve query performance
  • Loading branch information
Carolyn Russell authored Apr 21, 2021
2 parents 5f9699b + 8b5c689 commit 8d89dff
Show file tree
Hide file tree
Showing 50 changed files with 2,938 additions and 952 deletions.
File renamed without changes.
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
* Added current gas fee to Validator objects
* Initial conversions from nhash to hash #44
* Changed ownership on assets #45
* Improved query performance #53

### Bug Fixes
* Translated the signatures back to usable addresses
Expand All @@ -74,6 +75,11 @@ Ref: https://keepachangelog.com/en/1.0.0/
* Added a db index on tx_cache.height
* Added additional db indices
* Fixed a bug where candidate/jailed validators were being filtered out of results
* Added missing protos #55
* Properly handling Multi Msg txs #56

### Data
* Added migrations for Tx, BlockProposer, TxMessage, and TxMessageType data points #57

## PRE-HISTORY

13 changes: 8 additions & 5 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,14 @@ subprojects {
implementation(Libraries.KotlinReflect)
implementation(Libraries.KotlinStdlib)

testCompileOnly(Libraries.JunitJupiterApi)
testRuntimeOnly(Libraries.JunitVintageEngine)
testRuntimeOnly(Libraries.JunitJupiterEngine)
testImplementation(Libraries.SpringBootStarterTest) {
exclude(module = "junit")
exclude(module = "mockito-core")
exclude(module = "assertj-core")
}
testImplementation(Libraries.JunitJupiterApi)
testImplementation(Libraries.MockitoCore)
testImplementation(Libraries.KotlinTest)
testRuntimeOnly(Libraries.JunitJupiterEngine)
testImplementation(Libraries.SpringMockk)
testImplementation(Libraries.KotestAssert)
}
}
8 changes: 4 additions & 4 deletions buildSrc/src/main/kotlin/Dependencies.kt
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ object Versions {

// Testing
const val Jupiter = "5.7.1"
const val Mockito = "3.7.7"
const val SpringMockk = "3.0.1"
const val Kotest = "4.4.3"

//external protos
const val Provenance = "v0.2.1"
Expand Down Expand Up @@ -111,11 +112,10 @@ object Libraries {
const val Swagger = "io.springfox:springfox-boot-starter:${Versions.Swagger}"

// Testing
const val JunitVintageEngine = "org.junit.vintage:junit-vintage-engine:${Versions.Jupiter}"
const val JunitJupiterApi = "org.junit.jupiter:junit-jupiter-api:${Versions.Jupiter}"
const val JunitJupiterEngine = "org.junit.jupiter:junit-jupiter-engine:${Versions.Jupiter}"
const val MockitoCore = "org.mockito:mockito-core:${Versions.Mockito}"
const val KotlinTest = "org.jetbrains.kotlin:kotlin-test"
const val SpringMockk = "com.ninja-squad:springmockk:${Versions.SpringMockk}"
const val KotestAssert = "io.kotest:kotest-assertions-core:${Versions.Kotest}"
}

// gradle configurations
Expand Down
120 changes: 120 additions & 0 deletions database/src/main/resources/db/migration/V1_9__Add_tx_hash_id.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
select 'Altering tx_cache' as comment;

-- 3 minutes
ALTER TABLE tx_cache
DROP CONSTRAINT IF EXISTS transaction_cache_pkey,
ADD COLUMN IF NOT EXISTS id SERIAL PRIMARY KEY,
DROP COLUMN IF EXISTS last_hit,
DROP COLUMN IF EXISTS hit_count;
CREATE UNIQUE INDEX IF NOT EXISTS tx_cache_hash_idx ON tx_cache(hash);

select 'Altering tx_address_join' as comment;
ALTER TABLE tx_address_join
ADD COLUMN IF NOT EXISTS tx_hash_id INT NOT NULL DEFAULT 0;
CREATE INDEX IF NOT EXISTS tx_address_join_hash_id_idx ON tx_address_join(tx_hash_id);

select 'Updating tx_address_join' as comment;
-- 4 min 29 sec
UPDATE tx_address_join
SET tx_hash_id = tx_cache.id
FROM tx_cache
WHERE tx_address_join.tx_hash = tx_cache.hash;

select 'Altering tx_marker_join' as comment;
ALTER TABLE tx_marker_join
ADD COLUMN IF NOT EXISTS tx_hash_id INT NOT NULL DEFAULT 0;
CREATE INDEX IF NOT EXISTS tx_marker_join_hash_id_idx ON tx_marker_join(tx_hash_id);

select 'Updating tx_marker_join' as comment;
-- 1 min 38 sec
UPDATE tx_marker_join
SET tx_hash_id = tx_cache.id
FROM tx_cache
WHERE tx_marker_join.tx_hash = tx_cache.hash;

select 'Altering tx_message' as comment;
ALTER TABLE tx_message
ADD COLUMN IF NOT EXISTS tx_hash_id INT NOT NULL DEFAULT 0;
CREATE INDEX IF NOT EXISTS tx_message_hash_id_idx ON tx_message(tx_hash_id);

select 'Updating tx_message' as comment;
-- 4 min 16 sec
UPDATE tx_message
SET tx_hash_id = tx_cache.id
FROM tx_cache
WHERE tx_message.tx_hash = tx_cache.hash;

select 'Altering account' as comment;
ALTER TABLE account
DROP CONSTRAINT IF EXISTS account_pkey,
ADD COLUMN IF NOT EXISTS id SERIAL PRIMARY KEY;
CREATE UNIQUE INDEX IF NOT EXISTS account_address_idx ON account(account_address);

select 'Altering marker_cache' as comment;
ALTER TABLE marker_cache
DROP CONSTRAINT IF EXISTS marker_cache_pkey,
ADD COLUMN IF NOT EXISTS id SERIAL PRIMARY KEY;
CREATE UNIQUE INDEX IF NOT EXISTS marker_cache_address_idx ON marker_cache(marker_address);

select 'Altering staking_validator_cache' as comment;
ALTER TABLE staking_validator_cache
DROP CONSTRAINT IF EXISTS staking_validator_cache_pkey,
ADD COLUMN IF NOT EXISTS id SERIAL PRIMARY KEY,
ADD COLUMN IF NOT EXISTS account_address varchar(128) NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS consensus_address varchar(128) NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS consensus_pubkey varchar(128) NOT NULL DEFAULT '',
DROP COLUMN IF EXISTS last_hit,
DROP COLUMN IF EXISTS hit_count;

select 'Updating staking_validator_cache' as comment;
UPDATE staking_validator_cache
SET account_address = validator_addresses.account_address,
consensus_address = validator_addresses.consensus_address,
consensus_pubkey = validator_addresses.consensus_pubkey
FROM validator_addresses
WHERE staking_validator_cache.operator_address = validator_addresses.operator_address;

select 'Altering staking_validator_cache Part 2' as comment;
CREATE UNIQUE INDEX IF NOT EXISTS staking_validator_account_idx ON staking_validator_cache(account_address);
CREATE UNIQUE INDEX IF NOT EXISTS staking_validator_operator_idx ON staking_validator_cache(operator_address);
CREATE UNIQUE INDEX IF NOT EXISTS staking_validator_consensus_addr_idx ON staking_validator_cache(consensus_address);
CREATE UNIQUE INDEX IF NOT EXISTS staking_validator_consensus_pubkey_idx ON staking_validator_cache(consensus_pubkey);

select 'Dropping validator_addresses' as comment;
DROP TABLE IF EXISTS validator_addresses;

select 'Altering tx_marker_join' as comment;
ALTER TABLE tx_marker_join
ADD COLUMN IF NOT EXISTS marker_id INT NOT NULL DEFAULT 0;
CREATE INDEX IF NOT EXISTS tx_marker_join_marker_id_idx ON tx_marker_join(marker_id);

select 'Updating tx_marker_join' as comment;
-- 2 min 26 sec
UPDATE tx_marker_join
SET marker_id = marker_cache.id
FROM marker_cache
WHERE tx_marker_join.denom = marker_cache.denom;

select 'Altering tx_address_join' as comment;
ALTER TABLE tx_address_join
ADD COLUMN IF NOT EXISTS address_type varchar(16) NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS address_id INT NOT NULL DEFAULT 0;
CREATE INDEX IF NOT EXISTS tx_address_join_address_type_idx ON tx_address_join(address_type);
CREATE INDEX IF NOT EXISTS tx_address_join_address_id_idx ON tx_address_join(address_id);
CREATE INDEX IF NOT EXISTS tx_address_join_address_type_id_idx ON tx_address_join(address_type, address_id);

select 'Updating tx_address_join' as comment;
-- 40 sec
UPDATE tx_address_join
SET address_type = 'ACCOUNT',
address_id = account.id
FROM account
WHERE tx_address_join.address = account.account_address;

select 'Updating tx_address_join Part 2' as comment;
-- 4 sec
UPDATE tx_address_join
SET address_type = 'OPERATOR',
address_id = staking_validator_cache.id
FROM staking_validator_cache
WHERE tx_address_join.address = staking_validator_cache.operator_address;
2 changes: 0 additions & 2 deletions service/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,6 @@ dependencies {
api(Libraries.ExposedDao)
api(Libraries.ExposedJdbc)

testImplementation(Libraries.SpringBootStarterTest)

developmentOnly(Libraries.SpringBootDevTools)
}

Expand Down
60 changes: 41 additions & 19 deletions service/src/main/kotlin/io/provenance/explorer/config/RestConfig.kt
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import cosmos.params.v1beta1.Params
import cosmos.tx.v1beta1.TxOuterClass
import cosmos.upgrade.v1beta1.Upgrade
import cosmos.vesting.v1beta1.Vesting
import cosmwasm.wasm.v1beta1.Proposal
import io.provenance.attribute.v1.MsgAddAttributeRequest
import io.provenance.attribute.v1.MsgDeleteAttributeRequest
import io.provenance.marker.v1.AddMarkerProposal
Expand All @@ -31,19 +32,23 @@ import io.provenance.marker.v1.RemoveAdministratorProposal
import io.provenance.marker.v1.SetAdministratorProposal
import io.provenance.marker.v1.SupplyDecreaseProposal
import io.provenance.marker.v1.SupplyIncreaseProposal
import io.provenance.metadata.v1.MsgAddContractSpecificationRequest
import io.provenance.metadata.v1.MsgAddRecordRequest
import io.provenance.metadata.v1.MsgAddRecordSpecificationRequest
import io.provenance.metadata.v1.MsgAddScopeRequest
import io.provenance.metadata.v1.MsgAddScopeSpecificationRequest
import io.provenance.metadata.v1.MsgAddSessionRequest
import io.provenance.metadata.v1.MsgChangeOwnershipRequest
import io.provenance.marker.v1.WithdrawEscrowProposal
import io.provenance.metadata.v1.MsgBindOSLocatorRequest
import io.provenance.metadata.v1.MsgWriteScopeRequest
import io.provenance.metadata.v1.MsgDeleteScopeRequest
import io.provenance.metadata.v1.MsgWriteSessionRequest
import io.provenance.metadata.v1.MsgWriteRecordRequest
import io.provenance.metadata.v1.MsgWriteScopeSpecificationRequest
import io.provenance.metadata.v1.MsgDeleteContractSpecificationRequest
import io.provenance.metadata.v1.MsgDeleteOSLocatorRequest
import io.provenance.metadata.v1.MsgDeleteRecordRequest
import io.provenance.metadata.v1.MsgDeleteRecordSpecificationRequest
import io.provenance.metadata.v1.MsgDeleteScopeRequest
import io.provenance.metadata.v1.MsgDeleteScopeSpecificationRequest
import io.provenance.metadata.v1.MsgMemorializeContractRequest
import io.provenance.metadata.v1.MsgModifyOSLocatorRequest
import io.provenance.metadata.v1.MsgP8eMemorializeContractRequest
import io.provenance.metadata.v1.MsgWriteContractSpecificationRequest
import io.provenance.metadata.v1.MsgWriteP8eContractSpecRequest
import io.provenance.metadata.v1.MsgWriteRecordSpecificationRequest
import io.provenance.name.v1.CreateRootNameProposal
import io.provenance.name.v1.MsgBindNameRequest
import io.provenance.name.v1.MsgDeleteNameRequest
Expand Down Expand Up @@ -161,19 +166,28 @@ fun msgDescriptors() =
MsgDeleteNameRequest.getDescriptor(),
MsgAddAttributeRequest.getDescriptor(),
MsgDeleteAttributeRequest.getDescriptor(),
MsgMemorializeContractRequest.getDescriptor(),
MsgChangeOwnershipRequest.getDescriptor(),
MsgAddScopeRequest.getDescriptor(),
MsgWriteP8eContractSpecRequest.getDescriptor(),
MsgP8eMemorializeContractRequest.getDescriptor(),
MsgWriteScopeRequest.getDescriptor(),
MsgDeleteScopeRequest.getDescriptor(),
MsgAddSessionRequest.getDescriptor(),
MsgAddRecordRequest.getDescriptor(),
MsgWriteSessionRequest.getDescriptor(),
MsgWriteRecordRequest.getDescriptor(),
MsgDeleteRecordRequest.getDescriptor(),
MsgAddScopeSpecificationRequest.getDescriptor(),
MsgWriteScopeSpecificationRequest.getDescriptor(),
MsgDeleteScopeSpecificationRequest.getDescriptor(),
MsgAddContractSpecificationRequest.getDescriptor(),
MsgWriteContractSpecificationRequest.getDescriptor(),
MsgDeleteContractSpecificationRequest.getDescriptor(),
MsgAddRecordSpecificationRequest.getDescriptor(),
MsgDeleteRecordSpecificationRequest.getDescriptor()
MsgWriteRecordSpecificationRequest.getDescriptor(),
MsgDeleteRecordSpecificationRequest.getDescriptor(),
MsgBindOSLocatorRequest.getDescriptor(),
MsgDeleteOSLocatorRequest.getDescriptor(),
MsgModifyOSLocatorRequest.getDescriptor(),
cosmwasm.wasm.v1beta1.Tx.MsgStoreCode.getDescriptor(),
cosmwasm.wasm.v1beta1.Tx.MsgInstantiateContract.getDescriptor(),
cosmwasm.wasm.v1beta1.Tx.MsgExecuteContract.getDescriptor(),
cosmwasm.wasm.v1beta1.Tx.MsgMigrateContract.getDescriptor(),
cosmwasm.wasm.v1beta1.Tx.MsgUpdateAdmin.getDescriptor(),
cosmwasm.wasm.v1beta1.Tx.MsgClearAdmin.getDescriptor()
)

fun contentDescriptors() =
Expand All @@ -189,5 +203,13 @@ fun contentDescriptors() =
SetAdministratorProposal.getDescriptor(),
RemoveAdministratorProposal.getDescriptor(),
ChangeStatusProposal.getDescriptor(),
CreateRootNameProposal.getDescriptor()
WithdrawEscrowProposal.getDescriptor(),
CreateRootNameProposal.getDescriptor(),
Proposal.StoreCodeProposal.getDescriptor(),
Proposal.InstantiateContractProposal.getDescriptor(),
Proposal.MigrateContractProposal.getDescriptor(),
Proposal.UpdateAdminProposal.getDescriptor(),
Proposal.ClearAdminProposal.getDescriptor(),
Proposal.PinCodesProposal.getDescriptor(),
Proposal.UnpinCodesProposal.getDescriptor()
)
Original file line number Diff line number Diff line change
@@ -1,24 +1,15 @@
package io.provenance.explorer.domain.core.sql

import io.provenance.explorer.domain.entities.TxCacheTable
import org.jetbrains.exposed.sql.Column
import org.jetbrains.exposed.sql.DecimalColumnType
import org.jetbrains.exposed.sql.Expression
import org.jetbrains.exposed.sql.Function
import org.jetbrains.exposed.sql.IColumnType
import org.jetbrains.exposed.sql.QueryBuilder
import org.jetbrains.exposed.sql.Table
import org.jetbrains.exposed.sql.Transaction
import org.jetbrains.exposed.sql.append
import org.jetbrains.exposed.sql.jodatime.CustomDateTimeFunction
import org.jetbrains.exposed.sql.jodatime.DateColumnType
import org.jetbrains.exposed.sql.jodatime.dateTimeLiteral
import org.jetbrains.exposed.sql.jodatime.dateTimeParam
import org.jetbrains.exposed.sql.statements.InsertStatement
import org.jetbrains.exposed.sql.stringLiteral
import org.jetbrains.exposed.sql.transactions.TransactionManager
import org.joda.time.DateTime
import org.joda.time.format.ISODateTimeFormat.dateTime
import java.math.BigDecimal

// Generic Distinct function
Expand Down
Loading

0 comments on commit 8d89dff

Please sign in to comment.