diff --git a/cmd/zetae2e/config/clients.go b/cmd/zetae2e/config/clients.go index 26d1be9ba0..51c0ff617a 100644 --- a/cmd/zetae2e/config/clients.go +++ b/cmd/zetae2e/config/clients.go @@ -59,14 +59,15 @@ func getClientsFromConfig(ctx context.Context, conf config.Config, account confi } return runner.Clients{ - Zetacore: zetaCoreClients, - BtcRPC: btcRPCClient, - Solana: solanaClient, - TON: tonClient, - Evm: evmClient, - EvmAuth: evmAuth, - Zevm: zevmClient, - ZevmAuth: zevmAuth, + Zetacore: zetaCoreClients, + BtcRPC: btcRPCClient, + Solana: solanaClient, + TON: tonClient, + Evm: evmClient, + EvmAuth: evmAuth, + Zevm: zevmClient, + ZevmAuth: zevmAuth, + ZetaclientMetrics: &runner.MetricsClient{URL: conf.RPCs.ZetaclientMetrics}, }, nil } diff --git a/cmd/zetae2e/config/local.yml b/cmd/zetae2e/config/local.yml index b86b98adbc..6575c862f5 100644 --- a/cmd/zetae2e/config/local.yml +++ b/cmd/zetae2e/config/local.yml @@ -109,6 +109,7 @@ rpcs: zetacore_rpc: "http://localhost:26657" solana: "http://localhost:8899" ton_sidecar_url: "http://localhost:8111" + zetaclient_metrics: "http://localhost:8886" contracts: zevm: system_contract: "0x91d18e54DAf4F677cB28167158d6dd21F6aB3921" diff --git a/cmd/zetae2e/config/localnet.yml b/cmd/zetae2e/config/localnet.yml index cb1a10dfcd..feddb4c8ef 100644 --- a/cmd/zetae2e/config/localnet.yml +++ b/cmd/zetae2e/config/localnet.yml @@ -107,6 +107,7 @@ rpcs: ton_sidecar_url: "http://ton:8000" zetacore_grpc: "zetacore0:9090" zetacore_rpc: "http://zetacore0:26657" + zetaclient_metrics: "http://zetaclient0:8886" contracts: # configure localnet solana gateway program id solana: diff --git a/contrib/localnet/docker-compose.yml b/contrib/localnet/docker-compose.yml index 354d5e1a4b..3764d7bd06 100644 --- a/contrib/localnet/docker-compose.yml +++ b/contrib/localnet/docker-compose.yml @@ -114,6 +114,7 @@ services: restart: always ports: - "6061:6061" # pprof + - "8886:8886" # metrics volumes: - ssh:/root/.ssh - preparams:/root/preparams diff --git a/e2e/config/config.go b/e2e/config/config.go index 5f0b67446a..e208ecb04c 100644 --- a/e2e/config/config.go +++ b/e2e/config/config.go @@ -94,13 +94,14 @@ type ObserverRelayerAccounts struct { // RPCs contains the configuration for the RPC endpoints type RPCs struct { - Zevm string `yaml:"zevm"` - EVM string `yaml:"evm"` - Bitcoin BitcoinRPC `yaml:"bitcoin"` - Solana string `yaml:"solana"` - TONSidecarURL string `yaml:"ton_sidecar_url"` - ZetaCoreGRPC string `yaml:"zetacore_grpc"` - ZetaCoreRPC string `yaml:"zetacore_rpc"` + Zevm string `yaml:"zevm"` + EVM string `yaml:"evm"` + Bitcoin BitcoinRPC `yaml:"bitcoin"` + Solana string `yaml:"solana"` + TONSidecarURL string `yaml:"ton_sidecar_url"` + ZetaCoreGRPC string `yaml:"zetacore_grpc"` + ZetaCoreRPC string `yaml:"zetacore_rpc"` + ZetaclientMetrics string `yaml:"zetaclient_metrics"` } // BitcoinRPC contains the configuration for the Bitcoin RPC endpoint diff --git a/e2e/e2etests/test_operational_flags.go b/e2e/e2etests/test_operational_flags.go index 37846e8daa..5084c38b58 100644 --- a/e2e/e2etests/test_operational_flags.go +++ b/e2e/e2etests/test_operational_flags.go @@ -1,6 +1,8 @@ package e2etests import ( + "time" + "github.com/stretchr/testify/require" "github.com/zeta-chain/node/e2e/runner" @@ -8,6 +10,10 @@ import ( observertypes "github.com/zeta-chain/node/x/observer/types" ) +const ( + startTimestampMetricName = "zetaclient_last_start_timestamp_seconds" +) + // TestOperationalFlags tests the functionality of operations flags. func TestOperationalFlags(r *runner.E2ERunner, _ []string) { _, err := r.Clients.Zetacore.Observer.OperationalFlags( @@ -38,5 +44,19 @@ func TestOperationalFlags(r *runner.E2ERunner, _ []string) { require.NoError(r, err) require.Equal(r, restartHeight, operationalFlagsRes.OperationalFlags.RestartHeight) - // TODO: wait for restart height + 2 then test that start timestamp metric has increased + originalStartTime, err := r.Clients.ZetaclientMetrics.FetchGauge(startTimestampMetricName) + require.NoError(r, err, "fetching zetaclient metric name") + + // wait for height above restart height + // wait for a few extra block to account for shutdown and startup time + require.Eventually(r, func() bool { + height, err := r.Clients.Zetacore.GetBlockHeight(r.Ctx) + require.NoError(r, err) + return height > restartHeight+3 + }, time.Minute, time.Second) + + currentStartTime, err := r.Clients.ZetaclientMetrics.FetchGauge(startTimestampMetricName) + require.NoError(r, err) + + require.Greater(r, currentStartTime, originalStartTime+1) } diff --git a/e2e/runner/clients.go b/e2e/runner/clients.go index 2a67d43ffa..0c1508949b 100644 --- a/e2e/runner/clients.go +++ b/e2e/runner/clients.go @@ -1,10 +1,15 @@ package runner import ( + "fmt" + "net/http" + "github.com/btcsuite/btcd/rpcclient" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/ethclient" "github.com/gagliardetto/solana-go/rpc" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" tonrunner "github.com/zeta-chain/node/e2e/runner/ton" zetacore_rpc "github.com/zeta-chain/node/pkg/rpc" @@ -24,4 +29,46 @@ type Clients struct { // the RPC clients for ZetaChain Zevm *ethclient.Client ZevmAuth *bind.TransactOpts + + ZetaclientMetrics *MetricsClient +} + +type MetricsClient struct { + URL string +} + +// Fetch retrieves and parses the prometheus metrics from the provided URL +func (m *MetricsClient) Fetch() (map[string]*dto.MetricFamily, error) { + // Fetch metrics + resp, err := http.Get(m.URL) + if err != nil { + return nil, fmt.Errorf("failed to fetch metrics: %w", err) + } + defer resp.Body.Close() + + // Parse metrics + parser := expfmt.TextParser{} + metricFamilies, err := parser.TextToMetricFamilies(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to parse metrics: %w", err) + } + + return metricFamilies, nil +} + +// FetchGauge fetches and individual gauge metric by it's name +func (m *MetricsClient) FetchGauge(name string) (float64, error) { + metrics, err := m.Fetch() + if err != nil { + return 0, err + } + metric, ok := metrics[name] + if !ok { + return 0, fmt.Errorf("%s metric is not found", name) + } + gauge := metric.Metric[0].Gauge + if gauge == nil { + return 0, fmt.Errorf("%s metric is not a gauge", name) + } + return *gauge.Value, nil }