complete e2e test logic

cmd/zetae2e/config/clients.go

@@ -59,14 +59,15 @@ func getClientsFromConfig(ctx context.Context, conf config.Config, account confi
 	}
 
 	return runner.Clients{
-		Zetacore: zetaCoreClients,
-		BtcRPC:   btcRPCClient,
-		Solana:   solanaClient,
-		TON:      tonClient,
-		Evm:      evmClient,
-		EvmAuth:  evmAuth,
-		Zevm:     zevmClient,
-		ZevmAuth: zevmAuth,
+		Zetacore:          zetaCoreClients,
+		BtcRPC:            btcRPCClient,
+		Solana:            solanaClient,
+		TON:               tonClient,
+		Evm:               evmClient,
+		EvmAuth:           evmAuth,
+		Zevm:              zevmClient,
+		ZevmAuth:          zevmAuth,
+		ZetaclientMetrics: &runner.MetricsClient{URL: conf.RPCs.ZetaclientMetrics},
 	}, nil
 }
 

cmd/zetae2e/config/local.yml

@@ -109,6 +109,7 @@ rpcs:
   zetacore_rpc: "http://localhost:26657"
   solana: "http://localhost:8899"
   ton_sidecar_url: "http://localhost:8111"
+  zetaclient_metrics: "http://localhost:8886"
 contracts:
   zevm:
     system_contract: "0x91d18e54DAf4F677cB28167158d6dd21F6aB3921"

cmd/zetae2e/config/localnet.yml

@@ -107,6 +107,7 @@ rpcs:
   ton_sidecar_url: "http://ton:8000"
   zetacore_grpc: "zetacore0:9090"
   zetacore_rpc: "http://zetacore0:26657"
+  zetaclient_metrics: "http://zetaclient0:8886"
 contracts:
 # configure localnet solana gateway program id
   solana:

contrib/localnet/docker-compose.yml

@@ -114,6 +114,7 @@ services:
     restart: always
     ports:
       - "6061:6061" # pprof
+      - "8886:8886" # metrics
     volumes:
       - ssh:/root/.ssh
       - preparams:/root/preparams

e2e/config/config.go

@@ -94,13 +94,14 @@ type ObserverRelayerAccounts struct {
 
 // RPCs contains the configuration for the RPC endpoints
 type RPCs struct {
-	Zevm          string     `yaml:"zevm"`
-	EVM           string     `yaml:"evm"`
-	Bitcoin       BitcoinRPC `yaml:"bitcoin"`
-	Solana        string     `yaml:"solana"`
-	TONSidecarURL string     `yaml:"ton_sidecar_url"`
-	ZetaCoreGRPC  string     `yaml:"zetacore_grpc"`
-	ZetaCoreRPC   string     `yaml:"zetacore_rpc"`
+	Zevm              string     `yaml:"zevm"`
+	EVM               string     `yaml:"evm"`
+	Bitcoin           BitcoinRPC `yaml:"bitcoin"`
+	Solana            string     `yaml:"solana"`
+	TONSidecarURL     string     `yaml:"ton_sidecar_url"`
+	ZetaCoreGRPC      string     `yaml:"zetacore_grpc"`
+	ZetaCoreRPC       string     `yaml:"zetacore_rpc"`
+	ZetaclientMetrics string     `yaml:"zetaclient_metrics"`
 }
 
 // BitcoinRPC contains the configuration for the Bitcoin RPC endpoint

e2e/e2etests/test_operational_flags.go

@@ -1,13 +1,19 @@
 package e2etests
 
 import (
+	"time"
+
 	"github.com/stretchr/testify/require"
 
 	"github.com/zeta-chain/node/e2e/runner"
 	"github.com/zeta-chain/node/e2e/utils"
 	observertypes "github.com/zeta-chain/node/x/observer/types"
 )
 
+const (
+	startTimestampMetricName = "zetaclient_last_start_timestamp_seconds"
+)
+
 // TestOperationalFlags tests the functionality of operations flags.
 func TestOperationalFlags(r *runner.E2ERunner, _ []string) {
 	_, err := r.Clients.Zetacore.Observer.OperationalFlags(
@@ -38,5 +44,19 @@ func TestOperationalFlags(r *runner.E2ERunner, _ []string) {
 	require.NoError(r, err)
 	require.Equal(r, restartHeight, operationalFlagsRes.OperationalFlags.RestartHeight)
 
-	// TODO: wait for restart height + 2 then test that start timestamp metric has increased
+	originalStartTime, err := r.Clients.ZetaclientMetrics.FetchGauge(startTimestampMetricName)
+	require.NoError(r, err, "fetching zetaclient metric name")
+
+	// wait for height above restart height
+	// wait for a few extra block to account for shutdown and startup time
+	require.Eventually(r, func() bool {
+		height, err := r.Clients.Zetacore.GetBlockHeight(r.Ctx)
+		require.NoError(r, err)
+		return height > restartHeight+3
+	}, time.Minute, time.Second)
+
+	currentStartTime, err := r.Clients.ZetaclientMetrics.FetchGauge(startTimestampMetricName)
+	require.NoError(r, err)
+
+	require.Greater(r, currentStartTime, originalStartTime+1)
 }

e2e/runner/clients.go

@@ -1,10 +1,15 @@
 package runner
 
 import (
+	"fmt"
+	"net/http"
+
 	"github.com/btcsuite/btcd/rpcclient"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/gagliardetto/solana-go/rpc"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
 
 	tonrunner "github.com/zeta-chain/node/e2e/runner/ton"
 	zetacore_rpc "github.com/zeta-chain/node/pkg/rpc"
@@ -24,4 +29,46 @@ type Clients struct {
 	// the RPC clients for ZetaChain
 	Zevm     *ethclient.Client
 	ZevmAuth *bind.TransactOpts
+
+	ZetaclientMetrics *MetricsClient
+}
+
+type MetricsClient struct {
+	URL string
+}
+
+// Fetch retrieves and parses the prometheus metrics from the provided URL
+func (m *MetricsClient) Fetch() (map[string]*dto.MetricFamily, error) {
+	// Fetch metrics
+	resp, err := http.Get(m.URL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch metrics: %w", err)
+	}
+	defer resp.Body.Close()
+
+	// Parse metrics
+	parser := expfmt.TextParser{}
+	metricFamilies, err := parser.TextToMetricFamilies(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse metrics: %w", err)
+	}
+
+	return metricFamilies, nil
+}
+
+// FetchGauge fetches and individual gauge metric by it's name
+func (m *MetricsClient) FetchGauge(name string) (float64, error) {
+	metrics, err := m.Fetch()
+	if err != nil {
+		return 0, err
+	}
+	metric, ok := metrics[name]
+	if !ok {
+		return 0, fmt.Errorf("%s metric is not found", name)
+	}
+	gauge := metric.Metric[0].Gauge
+	if gauge == nil {
+		return 0, fmt.Errorf("%s metric is not a gauge", name)
+	}
+	return *gauge.Value, nil
 }