Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

optimize mint statics & optimize db batch inserts #3

Merged
merged 1 commit into from
Jan 11, 2024
Merged

optimize mint statics & optimize db batch inserts #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion devents/model.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,8 +357,19 @@ func BuildDBUpdateModel(blocksEvents []*Event) (dmf *DBModelsFattened) {
}

for action, item := range event.InscriptionStats {
if _, ok := dm.InscriptionStats[action][item.SID]; ok {
if lastItem, ok := dm.InscriptionStats[action][item.SID]; ok {
xylog.Logger.Debugf("ins stats sid[%d] exist & force update, tick[%s]", item.SID, item.Tick)

// reserve history mint stats data if exist
if lastItem.MintFirstBlock > 0 {
item.MintFirstBlock = lastItem.MintFirstBlock
}
if lastItem.MintLastBlock > 0 {
item.MintLastBlock = lastItem.MintLastBlock
}
if lastItem.MintCompletedTime != nil {
item.MintCompletedTime = lastItem.MintCompletedTime
}
}
dm.InscriptionStats[action][item.SID] = item
}
Expand Down
2 changes: 1 addition & 1 deletion explorer/scan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ func (e *Explorer) Scan() {
xylog.Logger.Infof("start scanning...")

// Prioritize using data retrieved from the database
blockNum, err := e.db.LastBlock(e.config.Chain.ChainName)
blockNum, err := e.db.QueryLastBlock(e.config.Chain.ChainName)
if err != nil {
xylog.Logger.Fatalf("load hisotry block index err:%v", err)
}
Expand Down
2 changes: 1 addition & 1 deletion jsonrpc/handlers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,7 +318,7 @@ func handleGetLastBlockNumber(s *RpcServer, cmd interface{}, closeChan <-chan st
}
xylog.Logger.Infof("get last block number cmd params:%v", req)

blockNumber, err := s.dbc.LastBlock(req.Chain)
blockNumber, err := s.dbc.QueryLastBlock(req.Chain)
if err != nil {
return ErrRPCInternal, err
}
Expand Down
138 changes: 42 additions & 96 deletions storage/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"gorm.io/gorm"
"gorm.io/gorm/logger"
"math/big"
"reflect"
"strings"
)

Expand Down Expand Up @@ -57,21 +58,49 @@ func NewDbClient(cfg *utils.DatabaseConfig) (*DBClient, error) {
return nil, nil
}

func (conn *DBClient) CreateInBatches(dbTx *gorm.DB, value interface{}, batchSize int) error {
reflectValue := reflect.Indirect(reflect.ValueOf(value))

// the reflection type judgment of the optimized value
if reflectValue.Kind() != reflect.Slice && reflectValue.Kind() != reflect.Array {
return errors.New("value should be slice or array")
}

// the reflection length judgment of the optimized value
reflectLen := reflectValue.Len()
for i := 0; i < reflectLen; i += batchSize {
ends := i + batchSize
if ends > reflectLen {
ends = reflectLen
}

subTx := dbTx.Create(reflectValue.Slice(i, ends).Interface())
if subTx.Error != nil {
return subTx.Error
}
}
return nil
}

func (conn *DBClient) SaveLastBlock(tx *gorm.DB, status *model.BlockStatus) error {
if tx == nil {
return errors.New("gorm db is not valid")
}
return tx.Where("chain = ?", status.Chain).Save(status).Error
}

func (conn *DBClient) LastBlock(chain string) (*big.Int, error) {
func (conn *DBClient) QueryLastBlock(chain string) (*big.Int, error) {
var blockNumberStr string
err := conn.SqlDB.Raw("SELECT block_number FROM block where `chain` = ? ORDER BY block_number DESC LIMIT 1", chain).Scan(&blockNumberStr).Error
err := conn.SqlDB.Table(model.BlockStatus{}.TableName()).Where("chain = ?", chain).Pluck("block_number", &blockNumberStr).Error
if err != nil {
return nil, err
}

blockNumber, _ := utils.ConvetStr(blockNumberStr)
if blockNumberStr == "" {
return big.NewInt(0), nil
}

blockNumber, _ := big.NewInt(0).SetString(blockNumberStr, 10)
return blockNumber, nil
}

Expand Down Expand Up @@ -167,32 +196,32 @@ func (conn *DBClient) BatchAddInscriptionStats(dbTx *gorm.DB, ins []*model.Inscr
return dbTx.Create(ins).Error
}

func (conn *DBClient) BatchAddTransaction(dbTx *gorm.DB, txs []*model.Transaction) error {
if len(txs) < 1 {
func (conn *DBClient) BatchAddTransaction(dbTx *gorm.DB, items []*model.Transaction) error {
if len(items) < 1 {
return nil
}
return dbTx.Create(txs).Error
return conn.CreateInBatches(dbTx, items, 1000)
}

func (conn *DBClient) BatchAddBalanceTx(dbTx *gorm.DB, txs []*model.BalanceTxn) error {
if len(txs) < 1 {
func (conn *DBClient) BatchAddBalanceTx(dbTx *gorm.DB, items []*model.BalanceTxn) error {
if len(items) < 1 {
return nil
}
return dbTx.Create(txs).Error
return conn.CreateInBatches(dbTx, items, 1000)
}

func (conn *DBClient) BatchAddAddressTx(dbTx *gorm.DB, txs []*model.AddressTxs) error {
if len(txs) < 1 {
func (conn *DBClient) BatchAddAddressTx(dbTx *gorm.DB, items []*model.AddressTxs) error {
if len(items) < 1 {
return nil
}
return dbTx.Create(txs).Error
return conn.CreateInBatches(dbTx, items, 1000)
}

func (conn *DBClient) BatchAddBalances(dbTx *gorm.DB, items []*model.Balances) error {
if len(items) < 1 {
return nil
}
return dbTx.Create(items).Error
return conn.CreateInBatches(dbTx, items, 1000)
}

func (conn *DBClient) BatchUpdateBalances(dbTx *gorm.DB, chain string, items []*model.Balances) error {
Expand Down Expand Up @@ -224,54 +253,6 @@ func (conn *DBClient) UpdateInscriptionsStatsBySID(dbTx *gorm.DB, chain string,
return dbTx.Table(model.InscriptionsStats{}.TableName()).Where("chain = ?", chain).Where("sid = ?", id).Updates(updates).Error
}

func (conn *DBClient) UpdateInscriptStatsForMint(dbTx *gorm.DB, stats *model.InscriptionsStats) error {
ins := &model.InscriptionsStats{}
tableName := ins.TableName()
updateSql := ""
var updateData []interface{}
if stats.Minted.Sign() > 0 {
updateSql += " minted= ? "
updateData = append(updateData, stats.Minted)
}

if stats.MintCompletedTime != nil && stats.MintCompletedTime.Unix() > 0 {
updateSql += ", mint_completed_time=? "
updateData = append(updateData, stats.MintCompletedTime)
}

if stats.MintFirstBlock > 0 {
updateSql += ", mint_first_block=? "
updateData = append(updateData, stats.MintFirstBlock)
}

if stats.MintLastBlock > 0 {
updateSql += ", mint_last_block=? "
updateData = append(updateData, stats.MintLastBlock)
}

if stats.Holders > 0 {
updateSql += ", holders=? "
updateData = append(updateData, stats.Holders)
}

if stats.TxCnt > 0 {
updateSql += ", tx_cnt=? "
updateData = append(updateData, stats.TxCnt)
}

updateSql = strings.Trim(updateSql, ",")
if len(updateSql) > 0 && len(updateData) > 0 {
updateSql = "UPDATE " + tableName + " SET " + updateSql + "WHERE chain=? ANd protocol=? AND tick=?"
updateData = append(updateData, stats.Chain, stats.Protocol, stats.Tick)
err := dbTx.Exec(updateSql, updateData...).Error
if err != nil {
return err
}
}

return nil
}

// FindInscriptionByTick find token by tick
func (conn *DBClient) FindInscriptionByTick(chain, protocol, tick string) (*model.Inscriptions, error) {
inscriptionBaseInfo := &model.Inscriptions{}
Expand Down Expand Up @@ -533,41 +514,6 @@ func (conn *DBClient) GetUTXOsByIdLimit(start uint64, limit int) ([]model.UTXO,
return utxos, nil
}

func (conn *DBClient) FindUtxoByAddress(tx *gorm.DB, address, tick string) (*model.UTXO, error) {
utxo := &model.UTXO{}
err := conn.SqlDB.First(utxo, "address = ? and tick = ? ", address, tick).Error
if err != nil {
return nil, err
}

return utxo, nil
}

func (conn *DBClient) FirstValidUtxoByRootHash(tx *gorm.DB, chain, txid, address string) (*model.UTXO, error) {
utxo := &model.UTXO{}
err := conn.SqlDB.First(utxo, "address = ? AND root_hash = ? AND chain = ? AND status = ?", address, txid, chain, model.UTXOStatusUnspent).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return nil, err
}

return utxo, nil
}

func (conn *DBClient) FirstUTXOByRootHash(tx *gorm.DB, chain, txid string) (*model.UTXO, error) {
utxo := &model.UTXO{}
err := conn.SqlDB.First(utxo, " root_hash = ? AND chain = ?", txid, chain).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return nil, err
}
return utxo, nil
}

func (conn *DBClient) GetUtxosByAddress(address, chain, protocol, tick string) ([]*model.UTXO, error) {
var utxos []*model.UTXO
query := conn.SqlDB.Model(&model.UTXO{}).
Expand Down