Merge pull request #24 from bradzhang717/rc_20240222

add eths inscriptions and add some new jsonrpc apis.

.gitignore

@@ -4,4 +4,10 @@
 /logs.txt
 /config.ini
 /bin/
-./indexer
+./indexer
+indexer
+build.sh 
+log
+dev_config.json
+test_config.json
+dev_config_jsonrpc.json

README.md

@@ -32,7 +32,7 @@ make build install
 ### Build indexer
 ```
 make build install-indexer
-indexer -config config.json
+indexer --config config.json or  indexer -c config.json
 ```
 
 
@@ -42,7 +42,7 @@ indexer -config config.json
 ### Build apiserver
 ```
 make build install-jsonrpc
-apiserver -config config_jsonrpc.json
+apiserver --config config_jsonrpc.json or  apiserver -c config_jsonrpc.json
 ```
 
 

client/evm/client.go

@@ -24,6 +24,7 @@ package evm
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"github.com/ethereum/go-ethereum"
@@ -57,7 +58,7 @@ func (ec *RawClient) Client() *rpc.Client {
 }
 
 func (ec *RawClient) doCallContext(retry int, result interface{}, method string, args ...interface{}) (err error) {
-	timeCtx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	timeCtx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 	defer cancel()
 
 	t1 := time.Now()
@@ -88,6 +89,8 @@ func (ec *RawClient) CallContext(ctx context.Context, result interface{}, method
 			if result == nil {
 				return rpc.ErrNoResult
 			}
+			d, _ := json.Marshal(result)
+			xylog.Logger.Debugf("CallContext result=%v", string(d))
 			return nil
 		}
 

cmd/indexer/main.go

@@ -26,15 +26,15 @@ import (
 	"context"
 	"flag"
 	"github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
 	"github.com/uxuycom/indexer/client"
-	"github.com/uxuycom/indexer/devents"
-	"github.com/uxuycom/indexer/protocol"
-	"github.com/uxuycom/indexer/xylog"
-
 	"github.com/uxuycom/indexer/config"
 	"github.com/uxuycom/indexer/dcache"
+	"github.com/uxuycom/indexer/devents"
 	"github.com/uxuycom/indexer/explorer"
+	"github.com/uxuycom/indexer/protocol"
 	"github.com/uxuycom/indexer/storage"
+	"github.com/uxuycom/indexer/xylog"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
@@ -107,6 +107,8 @@ func main() {
 }
 
 func initArgs() {
-	flag.StringVar(&flagConfig, "config", "config.json", "config file")
-	flag.Parse()
+
+	pflag.StringVarP(&flagConfig, "config", "c", "config.json", "config file")
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+	pflag.Parse()
 }

cmd/jsonrpc/main.go

@@ -23,8 +23,10 @@
 package main
 
 import (
+	"encoding/json"
 	"flag"
 	"github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
 	"github.com/uxuycom/indexer/config"
 	"github.com/uxuycom/indexer/jsonrpc"
 	"github.com/uxuycom/indexer/storage"
@@ -36,7 +38,7 @@ import (
 )
 
 var (
-	cfg        config.JsonRcpConfig
+	cfg        config.RpcConfig
 	flagConfig string
 )
 
@@ -47,7 +49,13 @@ func main() {
 
 	config.LoadJsonRpcConfig(&cfg, flagConfig)
 
-	logLevel, _ := logrus.ParseLevel(cfg.LogLevel)
+	cfgJson, _ := json.Marshal(&cfg)
+	log.Printf("start server with config = %v\n", string(cfgJson))
+
+	logLevel, err := logrus.ParseLevel(cfg.LogLevel)
+	if err != nil {
+		log.Printf("start server parse log level err  = %v\n", err)
+	}
 	xylog.InitLog(logLevel, cfg.LogPath)
 
 	//db client
@@ -57,7 +65,7 @@ func main() {
 		return
 	}
 	//init server
-	server, err := jsonrpc.NewRPCServer(dbc, cfg.CacheStore)
+	server, err := jsonrpc.NewRPCServer(dbc, &cfg)
 	if err != nil {
 		log.Fatalf("server init err[%v]", err)
 	}
@@ -83,6 +91,8 @@ func main() {
 }
 
 func initArgs() {
-	flag.StringVar(&flagConfig, "config", "config_jsonrpc.json", "config file")
-	flag.Parse()
+
+	pflag.StringVarP(&flagConfig, "config", "c", "config_jsonrpc.json", "config file")
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+	pflag.Parse()
 }

config.json

@@ -37,5 +37,9 @@
   "profile": {
     "enabled": false,
     "listen": ":6060"
+  },
+  "stat": {
+    "address_start_id": 348870000,
+    "balance_start_id": 390790000
   }
 }

config/config.go

@@ -23,41 +23,45 @@
 package config
 
 import (
-	"encoding/json"
+	"github.com/spf13/viper"
 	"github.com/uxuycom/indexer/model"
 	"log"
-	"os"
 	"path/filepath"
 )
 
 type ScanConfig struct {
-	StartBlock        uint64 `json:"start_block"`
-	BlockBatchWorkers uint64 `json:"block_batch_workers"`
-	TxBatchWorkers    uint64 `json:"tx_batch_workers"`
-	DelayedBlockNum   uint64 `json:"delayed_block_num"`
+	StartBlock        uint64 `json:"start_block" mapstructure:"start_block"`
+	BlockBatchWorkers uint64 `json:"block_batch_workers" mapstructure:"block_batch_workers"`
+	TxBatchWorkers    uint64 `json:"tx_batch_workers" mapstructure:"tx_batch_workers"`
+	DelayedBlockNum   uint64 `json:"delayed_block_num" mapstructure:"delayed_block_num"`
 }
 
 type ChainConfig struct {
-	ChainName  string           `json:"chain_name"`
+	ChainName  string           `json:"chain_name" mapstructure:"chain_name"`
 	Rpc        string           `json:"rpc"`
 	UserName   string           `json:"username"`
 	PassWord   string           `json:"password"`
-	ChainGroup model.ChainGroup `json:"chain_group"`
+	ChainGroup model.ChainGroup `json:"chain_group" mapstructure:"chain_group"`
+}
+
+type StatConfig struct {
+	AddressStartId uint64 `json:"address_start_id" mapstructure:"address_start_id"`
+	BalanceStartId uint64 `json:"balance_start_id" mapstructure:"balance_start_id"`
 }
 
 type IndexFilter struct {
 	Whitelist *struct {
 		Ticks     []string `json:"ticks"`
 		Protocols []string `json:"protocols"`
 	} `json:"whitelist"`
-	EventTopics []string `json:"event_topics"`
+	EventTopics []string `json:"event_topics" mapstructure:"event_topics"`
 }
 
 // DatabaseConfig database config
 type DatabaseConfig struct {
 	Type      string `json:"type"`
 	Dsn       string `json:"dsn"`
-	EnableLog bool   `json:"enable_log"`
+	EnableLog bool   `json:"enable_log" mapstructure:"enable_log"`
 }
 
 type ProfileConfig struct {
@@ -68,82 +72,67 @@ type ProfileConfig struct {
 type Config struct {
 	Scan     ScanConfig     `json:"scan"`
 	Chain    ChainConfig    `json:"chain"`
-	LogLevel string         `json:"log_level"`
-	LogPath  string         `json:"log_path"`
+	LogLevel string         `json:"log_level" mapstructure:"log_level"`
+	LogPath  string         `json:"log_path" mapstructure:"log_path"`
 	Filters  *IndexFilter   `json:"filters"`
 	Database DatabaseConfig `json:"database"`
 	Profile  *ProfileConfig `json:"profile"`
+	Stat     *StatConfig    `json:"stat"`
 }
 
-type JsonRcpConfig struct {
-	RpcListen     []string       `json:"rpclisten"`
-	RpcMaxClients int64          `json:"rpcmaxclients"`
-	LogLevel      string         `json:"log_level"`
-	LogPath       string         `json:"log_path"`
-	Database      DatabaseConfig `json:"database"`
-	Profile       *ProfileConfig `json:"profile"`
-	CacheStore    *CacheConfig   `json:"cache_store"`
+type RpcConfig struct {
+	LogLevel     string         `json:"log_level" mapstructure:"log_level"`
+	LogPath      string         `json:"log_path" mapstructure:"log_path"`
+	Database     DatabaseConfig `json:"database"`
+	Profile      *ProfileConfig `json:"profile"`
+	CacheStore   *CacheConfig   `json:"cache_store" mapstructure:"cache_store"`
+	DebugLevel   string         `json:"debug_level" mapstructure:"debug_level"`
+	DisableTLS   bool           `json:"notls" description:"Disable TLS for the RPC server -- NOTE: This is only allowed if the RPC server is bound to localhost"`
+	RPCCert      string         `json:"rpccert" description:"File containing the certificate file"`
+	RPCKey       string         `json:"rpckey" description:"File containing the certificate key"`
+	RPCLimitPass string         `json:"rpclimitpass" default-mask:"-" description:"Password for limited RPC connections"`
+	RPCLimitUser string         `json:"rpclimituser" description:"Username for limited RPC connections"`
+	RPCListeners []string       `json:"rpclisten" mapstructure:"rpclisten" description:"Add an interface/port to listen for RPC
+connections (default port: 6583, testnet: 16583)"`
+	RPCMaxClients        int    `json:"rpcmaxclients" description:"Max number of RPC clients for standard connections"`
+	RPCMaxConcurrentReqs int    `json:"rpcmaxconcurrentreqs" description:"Max number of concurrent RPC requests that may be processed concurrently"`
+	RPCMaxWebsockets     int    `json:"rpcmaxwebsockets" description:"Max number of RPC websocket connections"`
+	RPCQuirks            bool   `json:"rpcquirks" description:"Mirror some JSON-RPC quirks of Bitcoin Core -- NOTE: Discouraged unless interoperability issues need to be worked around"`
+	RPCPass              string `json:"rpcpass" default-mask:"-" description:"Password for RPC connections"`
+	RPCUser              string `json:"rpcuser" description:"Username for RPC connections"`
 }
 
 type CacheConfig struct {
 	Started     bool   `json:"started"`
-	MaxCapacity int64  `json:"max_capacity"`
+	MaxCapacity int64  `json:"max_capacity" mapstructure:"max_capacity"`
 	Duration    uint32 `json:"duration"`
 }
 
-func LoadConfig(cfg *Config, filePath string) {
-	// Default config.
-	configFileName := "config.json"
-	if len(os.Args) > 1 {
-		configFileName = os.Args[1]
-	}
-
-	configFileName, _ = filepath.Abs(configFileName)
-	log.Printf("Loading config: %v", configFileName)
+func LoadConfig(cfg *Config, configFile string) {
+	UnmarshalConfig(configFile, cfg)
+}
 
-	if filePath != "" {
-		configFileName = filePath
-	}
-	configFile, err := os.Open(configFileName)
-	if err != nil {
-		log.Fatal("File error: ", err.Error())
-	}
-	defer func() {
-		_ = configFile.Close()
-	}()
-	jsonParser := json.NewDecoder(configFile)
-	if err := jsonParser.Decode(&cfg); err != nil {
-		log.Fatal("Config error: ", err.Error())
-	}
+func LoadJsonRpcConfig(cfg *RpcConfig, configFile string) {
+	UnmarshalConfig(configFile, cfg)
 }
 
-func LoadJsonRpcConfig(cfg *JsonRcpConfig, filePath string) {
-	// Default config.
-	configFileName := "config_jsonrpc.json"
-	if len(os.Args) > 1 {
-		configFileName = os.Args[1]
-	}
+func UnmarshalConfig(configFile string, cfg interface{}) {
+	fileName := filepath.Base(configFile)
+	viper.SetConfigFile(fileName)
+	viper.SetConfigType("json")
 
-	configFileName, _ = filepath.Abs(configFileName)
-	log.Printf("Loading config: %v", configFileName)
+	dir := filepath.Dir(configFile)
+	viper.AddConfigPath(dir)
 
-	if filePath != "" {
-		configFileName = filePath
+	if err := viper.ReadInConfig(); err != nil {
+		log.Fatalf("Read file error, error:%v", err.Error())
 	}
-	configFile, err := os.Open(configFileName)
-	if err != nil {
-		log.Fatal("File error: ", err.Error())
-	}
-	defer func() {
-		_ = configFile.Close()
-	}()
-	jsonParser := json.NewDecoder(configFile)
-	if err := jsonParser.Decode(&cfg); err != nil {
-		log.Fatal("Config error: ", err.Error())
+	if err := viper.Unmarshal(&cfg); err != nil {
+		log.Fatalf("Unmarshal config fail! error:%v ", err)
 	}
+	viper.WatchConfig()
 }
-
-func (cfg *JsonRcpConfig) GetConfig() *JsonRcpConfig {
+func (cfg *RpcConfig) GetConfig() *RpcConfig {
 	return cfg
 }
 

config_jsonrpc.json

@@ -20,4 +20,4 @@
     "enabled": false,
     "listen": ":6060"
   }
-}
+}

db/20240221-create-chain-info-add-index.sql

@@ -0,0 +1,46 @@
+Use
+tap_indexer;
+-- chain statics by hour table ---------
+CREATE TABLE `chain_stats_hour`
+(
+    `id`                 int unsigned                           NOT NULL AUTO_INCREMENT,
+    `chain`              varchar(32) COLLATE utf8mb4_general_ci NOT NULL COMMENT 'chain name',
+    `date_hour`          int unsigned                           NOT NULL COMMENT 'date_hour',
+    `address_count`      int unsigned                           NOT NULL COMMENT 'address_count',
+    `address_last_id`    bigint unsigned                        NOT NULL COMMENT 'address_last_id',
+    `inscriptions_count` int unsigned                           NOT NULL COMMENT 'inscriptions_count',
+    `balance_sum`        DECIMAL(38, 18)                        NOT NULL COMMENT 'balance_sum',
+    `balance_last_id`    bigint unsigned                        NOT NULL COMMENT 'balance_last_id',
+    `created_at`         timestamp                              NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `updated_at`         timestamp                              NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `uqx_chain_date_hour` (`chain`, `date_hour`)
+) ENGINE = InnoDB
+  DEFAULT CHARSET = utf8mb4
+  COLLATE = utf8mb4_general_ci;
+
+-- chain info ---------
+CREATE TABLE `chain_info`
+(
+    `id`          int unsigned                             NOT NULL AUTO_INCREMENT,
+    `chain_id`    int unsigned                             NOT NULL COMMENT 'chain id',
+    `chain`       varchar(32) COLLATE utf8mb4_general_ci   NOT NULL COMMENT 'inner chain name',
+    `outer_chain` varchar(32) COLLATE utf8mb4_general_ci   NOT NULL COMMENT 'outer chain name',
+    `name`        varchar(32) COLLATE utf8mb4_general_ci   NOT NULL COMMENT 'name',
+    `logo`        varchar(1024) COLLATE utf8mb4_general_ci NOT NULL COMMENT 'logo url',
+    `network_id`  int unsigned                             NOT NULL COMMENT 'network id',
+    `ext`         varchar(4098)                            NOT NUll COMMENT 'ext',
+    `created_at`  timestamp                                NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `updated_at`  timestamp                                NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `uqx_chain_id_chain_name` (`chain_id`, `chain`, `name`)
+) ENGINE = InnoDB
+  DEFAULT CHARSET = utf8mb4
+  COLLATE = utf8mb4_general_ci;
+
+INSERT INTO chain_info (chain_id, chain, outer_chain, name, logo, network_id,ext)VALUES (1, 'eth', 'eth', 'Ethereum', '', 1, '');
+INSERT INTO chain_info (chain_id, chain, outer_chain, name, logo, network_id,ext)VALUES (43114, 'avalanche', 'avax', 'Avalanche', '', 43114, '');
+INSERT INTO chain_info (chain_id, chain, outer_chain, name, logo, network_id,ext)VALUES (42161, 'arbitrum', 'ETH', 'Arbitrum One', '', 42161, '');
+INSERT INTO chain_info (chain_id, chain, outer_chain, name, logo, network_id,ext)VALUES (56, 'bsc', 'BSC', 'BNB Smart Chain Mainnet', '', 56, '');
+INSERT INTO chain_info (chain_id, chain, outer_chain, name, logo, network_id,ext)VALUES (250, 'fantom', 'FTM', 'Fantom Opera', '', 250, '');
+INSERT INTO chain_info (chain_id, chain, outer_chain, name, logo, network_id,ext)VALUES (137, 'polygon', 'Polygon', 'Polygon Mainnet', '', 137, '');