From f6709208b09d50bb594411e30818023282d1e3f0 Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Tue, 25 Oct 2022 10:19:10 -0400 Subject: [PATCH] e2e: configurable IP addresses for e2e testnet generator (#9592) * add the infrastructure types * add infra data to testnetload * extract infrastructure generation from manifest creation * add infrastructure type and data flags * rename docker ifd constructor * implement read ifd from file * add 'provider' field to the infrastructure data file to disable ip range check * return error from infrastructure from data file function * remove ifd from Setup * implement a basic infra provider with a simple setup command * remove misbehavior remnants * use manifest instead of file in all places * include cidr block range in the infrastructure data * nolint gosec * gosec * lint --- Makefile | 2 +- test/e2e/node/config.go | 1 - test/e2e/pkg/infra/docker/docker.go | 85 +++++++++++++++++++++++++++++ test/e2e/pkg/infra/provider.go | 20 +++++++ test/e2e/pkg/infrastructure.go | 80 +++++++++++++++++++++++++++ test/e2e/pkg/testnet.go | 36 +++++------- test/e2e/runner/main.go | 56 +++++++++++++++++-- test/e2e/runner/setup.go | 75 +------------------------ test/e2e/tests/e2e_test.go | 18 +++--- 9 files changed, 264 insertions(+), 109 deletions(-) create mode 100644 test/e2e/pkg/infra/docker/docker.go create mode 100644 test/e2e/pkg/infra/provider.go create mode 100644 test/e2e/pkg/infrastructure.go diff --git a/Makefile b/Makefile index 7a4ef6f9eee..97295c6cea8 100644 --- a/Makefile +++ b/Makefile @@ -271,7 +271,7 @@ format: lint: @echo "--> Running linter" - @golangci-lint run + @go run github.com/golangci/golangci-lint/cmd/golangci-lint run .PHONY: lint DESTINATION = ./index.html.md diff --git a/test/e2e/node/config.go b/test/e2e/node/config.go index 6941d0a1c91..954d382e4a2 100644 --- a/test/e2e/node/config.go +++ b/test/e2e/node/config.go @@ -23,7 +23,6 @@ type Config struct { PrivValServer string `toml:"privval_server"` PrivValKey string `toml:"privval_key"` PrivValState string `toml:"privval_state"` - Misbehaviors map[string]string `toml:"misbehaviors"` KeyType string `toml:"key_type"` } diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go new file mode 100644 index 00000000000..04811aab5e1 --- /dev/null +++ b/test/e2e/pkg/infra/docker/docker.go @@ -0,0 +1,85 @@ +package docker + +import ( + "bytes" + "os" + "path/filepath" + "text/template" + + e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" +) + +var _ infra.Provider = &Provider{} + +// Provider implements a docker-compose backed infrastructure provider. +type Provider struct { + Testnet *e2e.Testnet +} + +// Setup generates the docker-compose file and write it to disk, erroring if +// any of these operations fail. +func (p *Provider) Setup() error { + compose, err := dockerComposeBytes(p.Testnet) + if err != nil { + return err + } + //nolint: gosec + // G306: Expect WriteFile permissions to be 0600 or less + err = os.WriteFile(filepath.Join(p.Testnet.Dir, "docker-compose.yml"), compose, 0644) + if err != nil { + return err + } + return nil +} + +// dockerComposeBytes generates a Docker Compose config file for a testnet and returns the +// file as bytes to be written out to disk. +func dockerComposeBytes(testnet *e2e.Testnet) ([]byte, error) { + // Must use version 2 Docker Compose format, to support IPv6. + tmpl, err := template.New("docker-compose").Parse(`version: '2.4' +networks: + {{ .Name }}: + labels: + e2e: true + driver: bridge +{{- if .IPv6 }} + enable_ipv6: true +{{- end }} + ipam: + driver: default + config: + - subnet: {{ .IP }} + +services: +{{- range .Nodes }} + {{ .Name }}: + labels: + e2e: true + container_name: {{ .Name }} + image: tendermint/e2e-node +{{- if eq .ABCIProtocol "builtin" }} + entrypoint: /usr/bin/entrypoint-builtin +{{- end }} + init: true + ports: + - 26656 + - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657 + - 6060 + volumes: + - ./{{ .Name }}:/tendermint + networks: + {{ $.Name }}: + ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }} + +{{end}}`) + if err != nil { + return nil, err + } + var buf bytes.Buffer + err = tmpl.Execute(&buf, testnet) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go new file mode 100644 index 00000000000..03b821de384 --- /dev/null +++ b/test/e2e/pkg/infra/provider.go @@ -0,0 +1,20 @@ +package infra + +// Provider defines an API for manipulating the infrastructure of a +// specific set of testnet infrastructure. +type Provider interface { + + // Setup generates any necessary configuration for the infrastructure + // provider during testnet setup. + Setup() error +} + +// NoopProvider implements the provider interface by performing noops for every +// interface method. This may be useful if the infrastructure is managed by a +// separate process. +type NoopProvider struct { +} + +func (NoopProvider) Setup() error { return nil } + +var _ Provider = NoopProvider{} diff --git a/test/e2e/pkg/infrastructure.go b/test/e2e/pkg/infrastructure.go new file mode 100644 index 00000000000..2fc0e4bac6e --- /dev/null +++ b/test/e2e/pkg/infrastructure.go @@ -0,0 +1,80 @@ +package e2e + +import ( + "encoding/json" + "fmt" + "net" + "os" +) + +const ( + dockerIPv4CIDR = "10.186.73.0/24" + dockerIPv6CIDR = "fd80:b10c::/48" + + globalIPv4CIDR = "0.0.0.0/0" +) + +// InfrastructureData contains the relevant information for a set of existing +// infrastructure that is to be used for running a testnet. +type InfrastructureData struct { + + // Provider is the name of infrastructure provider backing the testnet. + // For example, 'docker' if it is running locally in a docker network or + // 'digital-ocean', 'aws', 'google', etc. if it is from a cloud provider. + Provider string `json:"provider"` + + // Instances is a map of all of the machine instances on which to run + // processes for a testnet. + // The key of the map is the name of the instance, which each must correspond + // to the names of one of the testnet nodes defined in the testnet manifest. + Instances map[string]InstanceData `json:"instances"` + + // Network is the CIDR notation range of IP addresses that all of the instances' + // IP addresses are expected to be within. + Network string `json:"network"` +} + +// InstanceData contains the relevant information for a machine instance backing +// one of the nodes in the testnet. +type InstanceData struct { + IPAddress net.IP `json:"ip_address"` +} + +func NewDockerInfrastructureData(m Manifest) (InfrastructureData, error) { + netAddress := dockerIPv4CIDR + if m.IPv6 { + netAddress = dockerIPv6CIDR + } + _, ipNet, err := net.ParseCIDR(netAddress) + if err != nil { + return InfrastructureData{}, fmt.Errorf("invalid IP network address %q: %w", netAddress, err) + } + ipGen := newIPGenerator(ipNet) + ifd := InfrastructureData{ + Provider: "docker", + Instances: make(map[string]InstanceData), + Network: netAddress, + } + for name := range m.Nodes { + ifd.Instances[name] = InstanceData{ + IPAddress: ipGen.Next(), + } + } + return ifd, nil +} + +func InfrastructureDataFromFile(p string) (InfrastructureData, error) { + ifd := InfrastructureData{} + b, err := os.ReadFile(p) + if err != nil { + return InfrastructureData{}, err + } + err = json.Unmarshal(b, &ifd) + if err != nil { + return InfrastructureData{}, err + } + if ifd.Network == "" { + ifd.Network = globalIPv4CIDR + } + return ifd, nil +} diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index 2ee4a44c342..3f83c0a9aef 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -21,8 +21,6 @@ import ( const ( randomSeed int64 = 2308084734268 proxyPortFirst uint32 = 5701 - networkIPv4 = "10.186.73.0/24" - networkIPv6 = "fd80:b10c::/48" ) type ( @@ -100,32 +98,20 @@ type Node struct { // The testnet generation must be deterministic, since it is generated // separately by the runner and the test cases. For this reason, testnets use a // random seed to generate e.g. keys. -func LoadTestnet(file string) (*Testnet, error) { - manifest, err := LoadManifest(file) - if err != nil { - return nil, err - } - dir := strings.TrimSuffix(file, filepath.Ext(file)) - - // Set up resource generators. These must be deterministic. - netAddress := networkIPv4 - if manifest.IPv6 { - netAddress = networkIPv6 - } - _, ipNet, err := net.ParseCIDR(netAddress) - if err != nil { - return nil, fmt.Errorf("invalid IP network address %q: %w", netAddress, err) - } - - ipGen := newIPGenerator(ipNet) +func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Testnet, error) { + dir := strings.TrimSuffix(fname, filepath.Ext(fname)) keyGen := newKeyGenerator(randomSeed) proxyPortGen := newPortGenerator(proxyPortFirst) + _, ipNet, err := net.ParseCIDR(ifd.Network) + if err != nil { + return nil, fmt.Errorf("invalid IP network address %q: %w", ifd.Network, err) + } testnet := &Testnet{ Name: filepath.Base(dir), - File: file, + File: fname, Dir: dir, - IP: ipGen.Network(), + IP: ipNet, InitialHeight: 1, InitialState: manifest.InitialState, Validators: map[*Node]int64{}, @@ -156,12 +142,16 @@ func LoadTestnet(file string) (*Testnet, error) { for _, name := range nodeNames { nodeManifest := manifest.Nodes[name] + ind, ok := ifd.Instances[name] + if !ok { + return nil, fmt.Errorf("information for node '%s' missing from infrastucture data", name) + } node := &Node{ Name: name, Testnet: testnet, PrivvalKey: keyGen.Generate(manifest.KeyType), NodeKey: keyGen.Generate("ed25519"), - IP: ipGen.Next(), + IP: ind.IPAddress, ProxyPort: proxyPortGen.Next(), Mode: ModeValidator, Database: "goleveldb", diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go index 048d2cabcac..fdfec7bbed8 100644 --- a/test/e2e/runner/main.go +++ b/test/e2e/runner/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "math/rand" "os" @@ -11,6 +12,8 @@ import ( "github.com/tendermint/tendermint/libs/log" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" + "github.com/tendermint/tendermint/test/e2e/pkg/infra/docker" ) const randomSeed = 2308084734268 @@ -26,6 +29,7 @@ type CLI struct { root *cobra.Command testnet *e2e.Testnet preserve bool + infp infra.Provider } // NewCLI sets up the CLI. @@ -41,19 +45,57 @@ func NewCLI() *CLI { if err != nil { return err } - testnet, err := e2e.LoadTestnet(file) + m, err := e2e.LoadManifest(file) if err != nil { return err } + inft, err := cmd.Flags().GetString("infrastructure-type") + if err != nil { + return err + } + + var ifd e2e.InfrastructureData + switch inft { + case "docker": + var err error + ifd, err = e2e.NewDockerInfrastructureData(m) + if err != nil { + return err + } + case "digital-ocean": + p, err := cmd.Flags().GetString("infrastructure-data") + if err != nil { + return err + } + if p == "" { + return errors.New("'--infrastructure-data' must be set when using the 'digital-ocean' infrastructure-type") + } + ifd, err = e2e.InfrastructureDataFromFile(p) + if err != nil { + return fmt.Errorf("parsing infrastructure data: %s", err) + } + default: + return fmt.Errorf("unknown infrastructure type '%s'", inft) + } + + testnet, err := e2e.LoadTestnet(m, file, ifd) + if err != nil { + return fmt.Errorf("loading testnet: %s", err) + } + cli.testnet = testnet + cli.infp = &infra.NoopProvider{} + if inft == "docker" { + cli.infp = &docker.Provider{Testnet: testnet} + } return nil }, RunE: func(cmd *cobra.Command, args []string) error { if err := Cleanup(cli.testnet); err != nil { return err } - if err := Setup(cli.testnet); err != nil { + if err := Setup(cli.testnet, cli.infp); err != nil { return err } @@ -118,6 +160,10 @@ func NewCLI() *CLI { cli.root.PersistentFlags().StringP("file", "f", "", "Testnet TOML manifest") _ = cli.root.MarkPersistentFlagRequired("file") + cli.root.PersistentFlags().StringP("infrastructure-type", "", "docker", "Backing infrastructure used to run the testnet. Either 'digital-ocean' or 'docker'") + + cli.root.PersistentFlags().StringP("infrastructure-data", "", "", "path to the json file containing the infrastructure data. Only used if the 'infrastructure-type' is set to a value other than 'docker'") + cli.root.Flags().BoolVarP(&cli.preserve, "preserve", "p", false, "Preserves the running of the test net after tests are completed") @@ -125,7 +171,7 @@ func NewCLI() *CLI { Use: "setup", Short: "Generates the testnet directory and configuration", RunE: func(cmd *cobra.Command, args []string) error { - return Setup(cli.testnet) + return Setup(cli.testnet, cli.infp) }, }) @@ -135,7 +181,7 @@ func NewCLI() *CLI { RunE: func(cmd *cobra.Command, args []string) error { _, err := os.Stat(cli.testnet.Dir) if os.IsNotExist(err) { - err = Setup(cli.testnet) + err = Setup(cli.testnet, cli.infp) } if err != nil { return err @@ -258,7 +304,7 @@ Does not run any perbutations. if err := Cleanup(cli.testnet); err != nil { return err } - if err := Setup(cli.testnet); err != nil { + if err := Setup(cli.testnet, cli.infp); err != nil { return err } diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index 202f3f79ead..e6aec065b00 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -10,9 +10,7 @@ import ( "path/filepath" "regexp" "sort" - "strconv" "strings" - "text/template" "time" "github.com/BurntSushi/toml" @@ -23,6 +21,7 @@ import ( "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/privval" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" "github.com/tendermint/tendermint/types" ) @@ -39,7 +38,7 @@ const ( ) // Setup sets up the testnet configuration. -func Setup(testnet *e2e.Testnet) error { +func Setup(testnet *e2e.Testnet, infp infra.Provider) error { logger.Info("setup", "msg", log.NewLazySprintf("Generating testnet files in %q", testnet.Dir)) err := os.MkdirAll(testnet.Dir, os.ModePerm) @@ -47,11 +46,7 @@ func Setup(testnet *e2e.Testnet) error { return err } - compose, err := MakeDockerCompose(testnet) - if err != nil { - return err - } - err = os.WriteFile(filepath.Join(testnet.Dir, "docker-compose.yml"), compose, 0o644) //nolint:gosec + err = infp.Setup() if err != nil { return err } @@ -126,70 +121,6 @@ func Setup(testnet *e2e.Testnet) error { return nil } -// MakeDockerCompose generates a Docker Compose config for a testnet. -func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) { - // Must use version 2 Docker Compose format, to support IPv6. - tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{ - "misbehaviorsToString": func(misbehaviors map[int64]string) string { - str := "" - for height, misbehavior := range misbehaviors { - // after the first behavior set, a comma must be prepended - if str != "" { - str += "," - } - heightString := strconv.Itoa(int(height)) - str += misbehavior + "," + heightString - } - return str - }, - }).Parse(`version: '2.4' - -networks: - {{ .Name }}: - labels: - e2e: true - driver: bridge -{{- if .IPv6 }} - enable_ipv6: true -{{- end }} - ipam: - driver: default - config: - - subnet: {{ .IP }} - -services: -{{- range .Nodes }} - {{ .Name }}: - labels: - e2e: true - container_name: {{ .Name }} - image: tendermint/e2e-node -{{- if eq .ABCIProtocol "builtin" }} - entrypoint: /usr/bin/entrypoint-builtin -{{- end }} - init: true - ports: - - 26656 - - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657 - - 6060 - volumes: - - ./{{ .Name }}:/tendermint - networks: - {{ $.Name }}: - ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }} - -{{end}}`) - if err != nil { - return nil, err - } - var buf bytes.Buffer - err = tmpl.Execute(&buf, testnet) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - // MakeGenesis generates a genesis document. func MakeGenesis(testnet *e2e.Testnet) (types.GenesisDoc, error) { genesis := types.GenesisDoc{ diff --git a/test/e2e/tests/e2e_test.go b/test/e2e/tests/e2e_test.go index 763b99ea388..5df3309d418 100644 --- a/test/e2e/tests/e2e_test.go +++ b/test/e2e/tests/e2e_test.go @@ -66,23 +66,27 @@ func testNode(t *testing.T, testFunc func(*testing.T, e2e.Node)) { func loadTestnet(t *testing.T) e2e.Testnet { t.Helper() - manifest := os.Getenv("E2E_MANIFEST") - if manifest == "" { + manifestFile := os.Getenv("E2E_MANIFEST") + if manifestFile == "" { t.Skip("E2E_MANIFEST not set, not an end-to-end test run") } - if !filepath.IsAbs(manifest) { - manifest = filepath.Join("..", manifest) + if !filepath.IsAbs(manifestFile) { + manifestFile = filepath.Join("..", manifestFile) } testnetCacheMtx.Lock() defer testnetCacheMtx.Unlock() - if testnet, ok := testnetCache[manifest]; ok { + if testnet, ok := testnetCache[manifestFile]; ok { return testnet } + m, err := e2e.LoadManifest(manifestFile) + require.NoError(t, err) + ifd, err := e2e.NewDockerInfrastructureData(m) + require.NoError(t, err) - testnet, err := e2e.LoadTestnet(manifest) + testnet, err := e2e.LoadTestnet(m, manifestFile, ifd) require.NoError(t, err) - testnetCache[manifest] = *testnet + testnetCache[manifestFile] = *testnet return *testnet }