Merge branch 'extra-rest/v2.5.x' of https://github.com/bandprotocol/c…

…hain into extra-rest/v2.4.x-add-relayer-stat

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## [v2.5.3](https://github.com/bandprotocol/chain/releases/tag/v2.5.3)
+
+- (bump) Use cometbft v0.34.29
+- (yoda) Get information of requests through endpoint instead of events
+
+## [v2.5.2](https://github.com/bandprotocol/chain/releases/tag/v2.5.2)
+
+- (bump) Use cosmos-sdk v0.45.16 / ibc-go v4.3.1 / cometbft v0.34.28
+
 ## [v2.5.1](https://github.com/bandprotocol/chain/releases/tag/v2.5.1)
 
 - (bump) Use cosmos-sdk package v0.45.15 / tendermint v0.34.27

flusher/flusher/db.py

@@ -59,8 +59,7 @@ class CustomDateTime(sa.types.TypeDecorator):
     impl = sa.DateTime
 
     def process_bind_param(self, value, dialect):
-        return datetime.fromtimestamp(value / 1e9)
-
+        return datetime.fromtimestamp(value / 1e9) if value != None else None
 
 class CustomBase64(sa.types.TypeDecorator):
     """Custom LargeBinary type that accepts base64-encoded string."""
@@ -98,7 +97,7 @@ def Column(*args, **kwargs):
     metadata,
     Column("chain_id", sa.String, primary_key=True),
     Column("topic", sa.String),
-    Column("kafka_offset", sa.Integer),
+    Column("kafka_offset", sa.BigInteger),
     Column("replay_topic", sa.String),
     Column("replay_offset", sa.Integer),
 )
@@ -161,6 +160,7 @@ def Column(*args, **kwargs):
     Column("fee", sa.String),
     Column("transaction_id", sa.Integer, sa.ForeignKey("transactions.id"), nullable=True),
     Column("accumulated_revenue", sa.BigInteger),
+    Column("last_request", CustomDateTime, nullable=True),
 )
 
 oracle_scripts = sa.Table(
@@ -175,13 +175,14 @@ def Column(*args, **kwargs):
     Column("source_code_url", sa.String),
     Column("transaction_id", sa.Integer, sa.ForeignKey("transactions.id"), nullable=True),
     Column("version", sa.Integer, nullable=True),
+    Column("last_request", CustomDateTime, nullable=True),
 )
 
 requests = sa.Table(
     "requests",
     metadata,
     Column("id", sa.Integer, primary_key=True),
-    Column("transaction_id", sa.Integer, sa.ForeignKey("transactions.id"), nullable=True),
+    Column("transaction_id", sa.Integer, sa.ForeignKey("transactions.id")),
     Column("oracle_script_id", sa.Integer, sa.ForeignKey("oracle_scripts.id")),
     Column("calldata", CustomBase64),
     Column("ask_count", sa.Integer),
@@ -193,26 +194,27 @@ def Column(*args, **kwargs):
     Column("execute_gas_used", sa.Integer, default=0),
     Column("sender", sa.String, nullable=True),
     Column("client_id", sa.String),
-    Column("request_time", sa.Integer, nullable=True),
+    Column("request_time", sa.Integer, nullable=True, index=True),
     Column("resolve_status", CustomResolveStatus),
     Column("resolve_time", sa.Integer, nullable=True),
-    Column("resolve_height", sa.Integer, sa.ForeignKey("blocks.height"), nullable=True),
+    Column("resolve_height", sa.Integer, sa.ForeignKey("blocks.height"), nullable=True, index=True),
     Column("reason", sa.String, nullable=True),
     Column("result", CustomBase64, nullable=True),
     Column("total_fees", sa.String),
     Column("is_ibc", sa.Boolean),
-    sa.Index("ix_requests_oracle_script_id", "oracle_script_id", "id"),
-    sa.Index("ix_oracle_script_id_resolve_status_request_time", "oracle_script_id", "resolve_status", "request_time"),
+    sa.Index("ix_requests_oracle_script_id_id", "oracle_script_id", "id", postgresql_include=['transaction_id', 'min_count', 'ask_count', 'resolve_status', 'request_time']),
+    sa.Index("ix_requests_oracle_script_id_resolve_status_request_time", "oracle_script_id", "resolve_status", "request_time"),
 )
 
 raw_requests = sa.Table(
     "raw_requests",
     metadata,
     Column("request_id", sa.Integer, sa.ForeignKey("requests.id"), primary_key=True),
     Column("external_id", sa.BigInteger, primary_key=True),
-    Column("data_source_id", sa.Integer, sa.ForeignKey("data_sources.id")),
+    Column("data_source_id", sa.Integer, sa.ForeignKey("data_sources.id"), index=True),
     Column("fee", sa.BigInteger),
     Column("calldata", CustomBase64),
+    sa.Index("ix_raw_requests_data_source_id_request_id", "data_source_id", "request_id"),
 )
 
 val_requests = sa.Table(
@@ -226,9 +228,10 @@ def Column(*args, **kwargs):
     "reports",
     metadata,
     Column("request_id", sa.Integer, sa.ForeignKey("requests.id"), primary_key=True),
-    Column("transaction_id", sa.Integer, sa.ForeignKey("transactions.id"), nullable=True),
+    Column("transaction_id", sa.Integer, sa.ForeignKey("transactions.id")),
     Column("validator_id", sa.Integer, sa.ForeignKey("validators.id"), primary_key=True),
-    Column("reporter_id", sa.Integer, sa.ForeignKey("accounts.id"), nullable=True),
+    Column("reporter_id", sa.Integer, sa.ForeignKey("accounts.id")),
+    sa.Index("ix_reports_validator_id_request_id", "validator_id", "request_id"),
 )
 
 raw_reports = sa.Table(
@@ -311,6 +314,7 @@ def Column(*args, **kwargs):
     metadata,
     Column("transaction_id", sa.Integer, sa.ForeignKey("transactions.id"), primary_key=True),
     Column("account_id", sa.Integer, sa.ForeignKey("accounts.id"), primary_key=True),
+    sa.Index("ix_account_transactions_account_id_transaction_id", "account_id", "transaction_id"),
 )
 
 proposals = sa.Table(
@@ -394,13 +398,29 @@ def Column(*args, **kwargs):
     Column("count", sa.Integer),
 )
 
+data_source_requests_per_days = sa.Table(
+    "data_source_requests_per_days",
+    metadata,
+    Column("date", CustomDate, primary_key=True),
+    Column("data_source_id", sa.Integer, sa.ForeignKey("data_sources.id"), primary_key=True),
+    Column("count", sa.Integer),
+)
+
 oracle_script_requests = sa.Table(
     "oracle_script_requests",
     metadata,
     Column("oracle_script_id", sa.Integer, sa.ForeignKey("oracle_scripts.id"), primary_key=True),
     Column("count", sa.Integer),
 )
 
+oracle_script_requests_per_days = sa.Table(
+    "oracle_script_requests_per_days",
+    metadata,
+    Column("date", CustomDate, primary_key=True),
+    Column("oracle_script_id", sa.Integer, sa.ForeignKey("oracle_scripts.id"), primary_key=True),
+    Column("count", sa.Integer),
+)
+
 request_count_per_days = sa.Table(
     "request_count_per_days",
     metadata,

flusher/flusher/handler.py

@@ -29,7 +29,9 @@
     related_data_source_oracle_scripts,
     historical_oracle_statuses,
     data_source_requests,
+    data_source_requests_per_days,
     oracle_script_requests,
+    oracle_script_requests_per_days,
     request_count_per_days,
     incoming_packets,
     outgoing_packets,
@@ -67,6 +69,16 @@ def get_request_count(self, date):
             select([request_count_per_days.c.count]).where(request_count_per_days.c.date == date)
         ).scalar()
 
+    def get_oracle_script_requests_count_per_day(self, date, oracle_script_id):
+        return self.conn.execute(
+            select([oracle_script_requests_per_days.c.count]).where((oracle_script_requests_per_days.c.date == date) & (oracle_script_requests_per_days.c.oracle_script_id == oracle_script_id))
+        ).scalar()
+
+    def get_data_source_requests_count_per_day(self, date, data_source_id):
+        return self.conn.execute(
+            select([data_source_requests_per_days.c.count]).where((data_source_requests_per_days.c.date == date) & (data_source_requests_per_days.c.data_source_id == data_source_id))
+        ).scalar()
+
     def get_data_source_id(self, id):
         return self.conn.execute(select([data_sources.c.id]).where(data_sources.c.id == id)).scalar()
 
@@ -156,6 +168,8 @@ def handle_new_request(self, msg):
         del msg["tx_hash"]
         if "timestamp" in msg:
             self.handle_set_request_count_per_day({"date": msg["timestamp"]})
+            self.handle_update_oracle_script_requests_count_per_day({"date": msg["timestamp"], "oracle_script_id": msg["oracle_script_id"]})
+            self.update_oracle_script_last_request(msg["oracle_script_id"], msg["timestamp"])
             del msg["timestamp"]
         self.conn.execute(requests.insert(), msg)
         self.increase_oracle_script_count(msg["oracle_script_id"])
@@ -178,6 +192,10 @@ def handle_update_related_ds_os(self, msg):
 
     def handle_new_raw_request(self, msg):
         self.increase_data_source_count(msg["data_source_id"])
+        if "timestamp" in msg:
+            self.handle_update_data_source_requests_count_per_day({"date": msg["timestamp"], "data_source_id": msg["data_source_id"]})
+            self.update_data_source_last_request(msg["data_source_id"], msg["timestamp"])
+            del msg["timestamp"]
         self.handle_update_related_ds_os(
             {
                 "oracle_script_id": self.conn.execute(
@@ -403,6 +421,30 @@ def handle_set_request_count_per_day(self, msg):
                 request_count_per_days.update(condition).values(count=request_count_per_days.c.count + 1)
             )
 
+    def handle_update_oracle_script_requests_count_per_day(self, msg):
+        if self.get_oracle_script_requests_count_per_day(msg["date"], msg["oracle_script_id"]) is None:
+            msg["count"] = 1
+            self.conn.execute(oracle_script_requests_per_days.insert(), msg)
+        else:
+            condition = True
+            for col in oracle_script_requests_per_days.primary_key.columns.values():
+                condition = (col == msg[col.name]) & condition
+            self.conn.execute(
+                oracle_script_requests_per_days.update(condition).values(count=oracle_script_requests_per_days.c.count + 1)
+            )
+
+    def handle_update_data_source_requests_count_per_day(self, msg):
+        if self.get_data_source_requests_count_per_day(msg["date"], msg["data_source_id"]) is None:
+            msg["count"] = 1
+            self.conn.execute(data_source_requests_per_days.insert(), msg)
+        else:
+            condition = True
+            for col in data_source_requests_per_days.primary_key.columns.values():
+                condition = (col == msg[col.name]) & condition
+            self.conn.execute(
+                data_source_requests_per_days.update(condition).values(count=data_source_requests_per_days.c.count + 1)
+            )
+
     def handle_new_incoming_packet(self, msg):
         self.update_last_update_channel(msg["dst_port"], msg["dst_channel"], msg["block_time"])
 
@@ -445,6 +487,20 @@ def increase_oracle_script_count(self, id):
             )
         )
 
+    def update_oracle_script_last_request(self, id, timestamp):
+        self.conn.execute(
+            oracle_scripts.update(oracle_scripts.c.id == id).values(
+                last_request=timestamp
+            )
+        )
+
+    def update_data_source_last_request(self, id, timestamp):
+        self.conn.execute(
+            data_sources.update(data_sources.c.id == id).values(
+                last_request=timestamp
+            )
+        )
+
     def handle_new_historical_bonded_token_on_validator(self, msg):
         self.conn.execute(
             insert(historical_bonded_token_on_validators)

flusher/flusher/init.py

@@ -168,6 +168,21 @@ def init(chain_id, topic, replay_topic, db):
             requests.resolve_status;
 """
     )
+    engine.execute(
+        """
+CREATE VIEW data_source_statistic_last_1_day
+AS
+  SELECT data_sources.id,
+         count(*) AS count
+  FROM   data_sources
+         join raw_requests
+           ON data_sources.id = raw_requests.data_source_id
+         join requests
+           ON raw_requests.request_id  = requests.id
+  WHERE  requests.request_time >= CAST(EXTRACT(epoch FROM NOW()) AS INT) - 86400
+  GROUP  BY data_sources.id;
+        """
+    )
     # TODO: replace select&group_by d.validator_id with d.delegator_id
     engine.execute(
         """

flusher/requirements.txt

@@ -7,6 +7,6 @@ loguru==0.5.1
 pathspec==0.8.0
 psycopg2==2.8.5
 regex==2020.6.8
-SQLAlchemy==1.3.17
+SQLAlchemy==1.4.48
 toml==0.10.1
 typed-ast==1.5.4

go.mod

@@ -5,9 +5,9 @@ go 1.19
 require (
 	github.com/bandprotocol/go-owasm v0.2.3
 	github.com/confio/ics23/go v0.9.0
-	github.com/cosmos/cosmos-sdk v0.45.15
+	github.com/cosmos/cosmos-sdk v0.45.16
 	github.com/cosmos/go-bip39 v1.0.0
-	github.com/cosmos/ibc-go/v4 v4.3.0
+	github.com/cosmos/ibc-go/v4 v4.3.1
 	github.com/ethereum/go-ethereum v1.10.22
 	github.com/gin-gonic/gin v1.8.1
 	github.com/go-sql-driver/mysql v1.6.0
@@ -28,7 +28,7 @@ require (
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/viper v1.14.0
 	github.com/stretchr/testify v1.8.1
-	github.com/tendermint/tendermint v0.34.27
+	github.com/tendermint/tendermint v0.34.28
 	github.com/tendermint/tm-db v0.6.7
 	google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa
 	google.golang.org/grpc v1.52.3
@@ -56,7 +56,7 @@ require (
 	filippo.io/edwards25519 v1.0.0-rc.1 // indirect
 	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
 	github.com/99designs/keyring v1.2.1 // indirect
-	github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d // indirect
+	github.com/ChainSafe/go-schnorrkel v1.0.0 // indirect
 	github.com/DataDog/zstd v1.5.0 // indirect
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/Workiva/go-datastructures v1.0.53 // indirect
@@ -184,6 +184,6 @@ require (
 replace (
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 	// use cometbft
-	github.com/tendermint/tendermint => github.com/cometbft/cometbft v0.34.27
+	github.com/tendermint/tendermint => github.com/cometbft/cometbft v0.34.29
 	google.golang.org/grpc => google.golang.org/grpc v1.33.2
 )

go.sum

@@ -60,8 +60,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d h1:nalkkPQcITbvhmL4+C4cKA87NW0tfm3Kl9VXRoPywFg=
-github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4=
+github.com/ChainSafe/go-schnorrkel v1.0.0 h1:3aDA67lAykLaG1y3AOjs88dMxC88PgUuHRrLeDnvGIM=
+github.com/ChainSafe/go-schnorrkel v1.0.0/go.mod h1:dpzHYVxLZcp8pjlV+O+UR8K0Hp/z7vcchBSbMBEhCw4=
 github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw=
 github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=
 github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w=
@@ -210,8 +210,8 @@ github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcju
 github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI=
 github.com/coinbase/rosetta-sdk-go v0.7.9 h1:lqllBjMnazTjIqYrOGv8h8jxjg9+hJazIGZr9ZvoCcA=
 github.com/coinbase/rosetta-sdk-go v0.7.9/go.mod h1:0/knutI7XGVqXmmH4OQD8OckFrbQ8yMsUZTG7FXCR2M=
-github.com/cometbft/cometbft v0.34.27 h1:ri6BvmwjWR0gurYjywcBqRe4bbwc3QVs9KRcCzgh/J0=
-github.com/cometbft/cometbft v0.34.27/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw=
+github.com/cometbft/cometbft v0.34.29 h1:Q4FqMevP9du2pOgryZJHpDV2eA6jg/kMYxBj9ZTY6VQ=
+github.com/cometbft/cometbft v0.34.29/go.mod h1:L9shMfbkZ8B+7JlwANEr+NZbBcn+hBpwdbeYvA5rLCw=
 github.com/cometbft/cometbft-db v0.7.0 h1:uBjbrBx4QzU0zOEnU8KxoDl18dMNgDh+zZRUE0ucsbo=
 github.com/cometbft/cometbft-db v0.7.0/go.mod h1:yiKJIm2WKrt6x8Cyxtq9YTEcIMPcEe4XPxhgX59Fzf0=
 github.com/confio/ics23/go v0.9.0 h1:cWs+wdbS2KRPZezoaaj+qBleXgUk5WOQFMP3CQFGTr4=
@@ -235,17 +235,17 @@ github.com/cosmos/cosmos-db v0.0.0-20221226095112-f3c38ecb5e32 h1:zlCp9n3uwQieEL
 github.com/cosmos/cosmos-db v0.0.0-20221226095112-f3c38ecb5e32/go.mod h1:kwMlEC4wWvB48zAShGKVqboJL6w4zCLesaNQ3YLU2BQ=
 github.com/cosmos/cosmos-proto v1.0.0-beta.1 h1:iDL5qh++NoXxG8hSy93FdYJut4XfgbShIocllGaXx/0=
 github.com/cosmos/cosmos-proto v1.0.0-beta.1/go.mod h1:8k2GNZghi5sDRFw/scPL8gMSowT1vDA+5ouxL8GjaUE=
-github.com/cosmos/cosmos-sdk v0.45.15 h1:yyLZ9PylnR1DADf9FYGxnn8t3+Y5K2mMUXNUN38MI5A=
-github.com/cosmos/cosmos-sdk v0.45.15/go.mod h1:bScuNwWAP0TZJpUf+SHXRU3xGoUPp+X9nAzfeIXts40=
+github.com/cosmos/cosmos-sdk v0.45.16 h1:5ba/Bh5/LE55IwHQuCU4fiG4eXeDKtSWzehXRpaKDcw=
+github.com/cosmos/cosmos-sdk v0.45.16/go.mod h1:bScuNwWAP0TZJpUf+SHXRU3xGoUPp+X9nAzfeIXts40=
 github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y=
 github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY=
 github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw=
 github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4Y=
 github.com/cosmos/gorocksdb v1.2.0/go.mod h1:aaKvKItm514hKfNJpUJXnnOWeBnk2GL4+Qw9NHizILw=
 github.com/cosmos/iavl v0.19.5 h1:rGA3hOrgNxgRM5wYcSCxgQBap7fW82WZgY78V9po/iY=
 github.com/cosmos/iavl v0.19.5/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw=
-github.com/cosmos/ibc-go/v4 v4.3.0 h1:yOzVsyZzsv4XPBux8gq+D0LhZn45yGWKjvT+6Vyo5no=
-github.com/cosmos/ibc-go/v4 v4.3.0/go.mod h1:CcLvIoi9NNtIbNsxs4KjBGjYhlwqtsmXy1AKARKiMzQ=
+github.com/cosmos/ibc-go/v4 v4.3.1 h1:xbg0CaCdxK3lvgGvSaI91ROOLd7s30UqEcexH6Ba4Ys=
+github.com/cosmos/ibc-go/v4 v4.3.1/go.mod h1:89E+K9CxpkS/etLEcG026jPM/RSnVMcfesvRYp/0aKI=
 github.com/cosmos/ledger-cosmos-go v0.12.2 h1:/XYaBlE2BJxtvpkHiBm97gFGSGmYGKunKyF3nNqAXZA=
 github.com/cosmos/ledger-cosmos-go v0.12.2/go.mod h1:ZcqYgnfNJ6lAXe4HPtWgarNEY+B74i+2/8MhZw4ziiI=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=