feat: implement head block age monitoring

.gitignore

@@ -2,6 +2,7 @@
 
 /bin/*
 !/bin/.gitkeep
+/cmd/__debug_*
 dist
 
 # ide

cmd/serve.go

@@ -33,14 +33,23 @@ func CommandServe(cfg *config.Config) *cli.Command {
 	healthcheckFlags := []cli.Flag{
 		&cli.DurationFlag{
 			Category:    strings.ToUpper(categoryHealthcheck),
-			Destination: &cfg.Healthcheck.CacheTimeout,
+			Destination: &cfg.Healthcheck.BlockAgeThreshold,
 			DefaultText: "disabled",
-			EnvVars:     []string{envPrefix + strings.ToUpper(categoryHealthcheck) + "_CACHE_TIMEOUT"},
-			Name:        categoryHealthcheck + "-cache-timeout",
-			Usage:       "re-use healthcheck results for the specified `duration`",
+			EnvVars:     []string{envPrefix + strings.ToUpper(categoryHealthcheck) + "_BLOCK_AGE_THRESHOLD"},
+			Name:        categoryHealthcheck + "-block-age-threshold",
+			Usage:       "monitor the age of latest block and report unhealthy if it's over specified `duration`",
 			Value:       0,
 		},
 
+		&cli.DurationFlag{
+			Category:    strings.ToUpper(categoryHealthcheck),
+			Destination: &cfg.Healthcheck.CacheCoolOff,
+			EnvVars:     []string{envPrefix + strings.ToUpper(categoryHealthcheck) + "_CACHE_COOL_OFF"},
+			Name:        categoryHealthcheck + "-cache-cool-off",
+			Usage:       "re-use healthcheck results for the specified `duration`",
+			Value:       750 * time.Millisecond,
+		},
+
 		&cli.DurationFlag{
 			Category:    strings.ToUpper(categoryHealthcheck),
 			Destination: &cfg.Healthcheck.Timeout,
@@ -167,7 +176,9 @@ func CommandServe(cfg *config.Config) *cli.Command {
 		),
 
 		Before: func(ctx *cli.Context) error {
-			// TODO: validate inputs
+			if err := cfg.Preprocess(); err != nil {
+				return err
+			}
 			return nil
 		},
 

config/config.go

@@ -13,3 +13,25 @@ type Config struct {
 	HealthcheckOpNode     HealthcheckOpNode     `yaml:"healthcheck_op_node"`
 	HealthcheckReth       HealthcheckReth       `yaml:"healthcheck_reth"`
 }
+
+func (c *Config) Preprocess() error {
+	errs := make([]error, 0)
+
+	if c.Healthcheck.BlockAgeThreshold != 0 {
+		c.HealthcheckGeth.BlockAgeThreshold = c.Healthcheck.BlockAgeThreshold
+		c.HealthcheckLighthouse.BlockAgeThreshold = c.Healthcheck.BlockAgeThreshold
+		c.HealthcheckOpNode.BlockAgeThreshold = c.Healthcheck.BlockAgeThreshold
+		c.HealthcheckReth.BlockAgeThreshold = c.Healthcheck.BlockAgeThreshold
+	}
+
+	errs = append(errs, c.Log.Preprocess())
+	errs = append(errs, c.Server.Preprocess())
+	errs = append(errs, c.HttpStatus.Preprocess())
+	errs = append(errs, c.Healthcheck.Preprocess())
+	errs = append(errs, c.HealthcheckGeth.Preprocess())
+	errs = append(errs, c.HealthcheckLighthouse.Preprocess())
+	errs = append(errs, c.HealthcheckOpNode.Preprocess())
+	errs = append(errs, c.HealthcheckReth.Preprocess())
+
+	return flatten(errs)
+}

config/errors.go

@@ -0,0 +1,23 @@
+package config
+
+import "errors"
+
+func flatten(errs []error) error {
+	next := 0
+	for _, err := range errs {
+		if err != nil {
+			errs[next] = err
+			next++
+		}
+	}
+	errs = errs[:next]
+
+	switch len(errs) {
+	default:
+		return errors.Join(errs...)
+	case 1:
+		return errs[0]
+	case 0:
+		return nil
+	}
+}

config/healthcheck.go

@@ -3,6 +3,11 @@ package config
 import "time"
 
 type Healthcheck struct {
-	CacheTimeout time.Duration `yaml:"cache_timeout"`
-	Timeout      time.Duration `yaml:"timeout"`
+	BlockAgeThreshold time.Duration `yaml:"block_age_threshold"`
+	CacheCoolOff      time.Duration `yaml:"cache_cool_off"`
+	Timeout           time.Duration `yaml:"timeout"`
+}
+
+func (c *Healthcheck) Preprocess() error {
+	return nil
 }

config/healthcheck_geth.go

@@ -1,5 +1,23 @@
 package config
 
+import (
+	"fmt"
+	"net/url"
+	"time"
+)
+
 type HealthcheckGeth struct {
-	BaseURL string `yaml:"base_url"`
+	BaseURL           string        `yaml:"base_url"`
+	BlockAgeThreshold time.Duration `yaml:"-"`
+}
+
+func (c *HealthcheckGeth) Preprocess() error {
+	if c.BaseURL != "" {
+		if _, err := url.Parse(c.BaseURL); err != nil {
+			return fmt.Errorf("invalid geth base url: %w",
+				err,
+			)
+		}
+	}
+	return nil
 }

config/healthcheck_lighthouse.go

@@ -1,5 +1,23 @@
 package config
 
+import (
+	"fmt"
+	"net/url"
+	"time"
+)
+
 type HealthcheckLighthouse struct {
-	BaseURL string `yaml:"base_url"`
+	BaseURL           string        `yaml:"base_url"`
+	BlockAgeThreshold time.Duration `yaml:"-"`
+}
+
+func (c *HealthcheckLighthouse) Preprocess() error {
+	if c.BaseURL != "" {
+		if _, err := url.Parse(c.BaseURL); err != nil {
+			return fmt.Errorf("invalid lighthouse base url: %w",
+				err,
+			)
+		}
+	}
+	return nil
 }

config/healthcheck_op_node.go

@@ -1,6 +1,24 @@
 package config
 
+import (
+	"fmt"
+	"net/url"
+	"time"
+)
+
 type HealthcheckOpNode struct {
-	BaseURL              string `yaml:"base_url"`
-	ConfirmationDistance uint64 `yaml:"confirmation_distance"`
+	BaseURL              string        `yaml:"base_url"`
+	BlockAgeThreshold    time.Duration `yaml:"-"`
+	ConfirmationDistance uint64        `yaml:"confirmation_distance"`
+}
+
+func (c *HealthcheckOpNode) Preprocess() error {
+	if c.BaseURL != "" {
+		if _, err := url.Parse(c.BaseURL); err != nil {
+			return fmt.Errorf("invalid op-node base url: %w",
+				err,
+			)
+		}
+	}
+	return nil
 }

config/healthcheck_reth.go

@@ -1,5 +1,23 @@
 package config
 
+import (
+	"fmt"
+	"net/url"
+	"time"
+)
+
 type HealthcheckReth struct {
-	BaseURL string `yaml:"base_url"`
+	BaseURL           string        `yaml:"base_url"`
+	BlockAgeThreshold time.Duration `yaml:"-"`
+}
+
+func (c *HealthcheckReth) Preprocess() error {
+	if c.BaseURL != "" {
+		if _, err := url.Parse(c.BaseURL); err != nil {
+			return fmt.Errorf("invalid reth base url: %w",
+				err,
+			)
+		}
+	}
+	return nil
 }

config/http_status.go

@@ -5,3 +5,7 @@ type HttpStatus struct {
 	Warning int `yaml:"warning"`
 	Error   int `yaml:"error"`
 }
+
+func (c *HttpStatus) Preprocess() error {
+	return nil
+}

config/log.go

@@ -4,3 +4,7 @@ type Log struct {
 	Level string `yaml:"level"`
 	Mode  string `yaml:"mode"`
 }
+
+func (c *Log) Preprocess() error {
+	return nil
+}

config/server.go

@@ -3,3 +3,7 @@ package config
 type Server struct {
 	ListenAddress string `yaml:"listen_address"`
 }
+
+func (c *Server) Preprocess() error {
+	return nil
+}