allow KinD clusters to be deleted or retained

Modifies the KinD Fixture to make it more stable and more flexible.

The default behaviour is for the KinD Fixture to delete the KinD cluster
only if the KinD cluster existed before the Fixture is started.
Similarly, if the KinD Cluster did *not* exist before the Fixture is
started, the Fixture deletes the KinD Cluster when it is stopped.

This commit adds a WithDeleteOnStop() and WithRetainOnStop() modifier to
the KinD Fixture creation allowing users to indicate if they want to
change that default behaviour.

In addition to the above enhancements, this commit makes our testing
more stable by adding some KinD cluster cleanup to our Makefile as well
as serializing execution of longer-running, resource-intensive tests.

Issue #13

Signed-off-by: Jay Pipes <[email protected]>

.github/workflows/gate-tests.yml

@@ -38,7 +38,7 @@ jobs:
      - name: run non-Kind tests
        env:
          SKIP_KIND: 1
-       run: go test -v ./...
+       run: make test
   test-all:
     strategy:
       matrix:
@@ -69,4 +69,4 @@ jobs:
        with:
          cluster_name: kind
      - name: run all tests
-       run: go test -v ./...
+       run: make test-all

Makefile

@@ -1,10 +1,44 @@
 VERSION ?= $(shell git describe --tags --always --dirty)
+CLUSTERS = $(shell kind get clusters)
 
 .PHONY: test
 
+kind-install:
+ifeq (, $(shell command -v kind))
+	go install sigs.k8s.io/[email protected]
+endif
+
 # We clear the test cache because some of the tests require an out-of-band KinD
 # cluster to run against and we want to re-run tests against that KinD cluster
 # instead of from cached unit test results.
-test:
+clear-test-cache:
+	@echo -n "clearing Go test cache ... "
 	@go clean -testcache
-	@go test -v ./...
+	@echo "ok."
+
+kind-clear-clusters:
+ifneq (, $(shell command -v kind))
+ifneq (, $(CLUSTERS))
+	@echo -n "clearing KinD clusters ... "
+	@for c in $(CLUSTERS); do kind delete cluster -q --name $$c; done
+	@echo "ok."
+endif
+endif
+
+kind-create-cluster:
+ifneq (, $(shell command -v kind))
+	@echo -n "creating 'kind' cluster ... "
+	@kind create cluster -q
+	@echo "ok."
+	@sleep 5
+endif
+
+test: clear-test-cache kind-clear-clusters kind-create-cluster test-kind-simple
+
+test-kind-simple:
+	@go test -v ./parse_test.go
+	@go test -v ./eval_test.go
+
+test-all: test kind-clear-clusters
+	@go test -v ./fixtures/kind/kind_test.go
+	@go test -v ./placement_test.go

eval_test.go

@@ -5,21 +5,19 @@
 package kube_test
 
 import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"os"
 	"path/filepath"
 	"testing"
 
 	"github.com/gdt-dev/gdt"
 	gdtcontext "github.com/gdt-dev/gdt/context"
-	kindfix "github.com/gdt-dev/kube/fixtures/kind"
 	"github.com/stretchr/testify/require"
+
+	kindfix "github.com/gdt-dev/kube/fixtures/kind"
+	"github.com/gdt-dev/kube/testutil"
 )
 
 func TestListPodsEmpty(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "list-pods-empty.yaml")
@@ -36,7 +34,7 @@ func TestListPodsEmpty(t *testing.T) {
 }
 
 func TestGetPodNotFound(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "get-pod-not-found.yaml")
@@ -53,7 +51,7 @@ func TestGetPodNotFound(t *testing.T) {
 }
 
 func TestCreateUnknownResource(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "create-unknown-resource.yaml")
@@ -70,7 +68,7 @@ func TestCreateUnknownResource(t *testing.T) {
 }
 
 func TestDeleteResourceNotFound(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "delete-resource-not-found.yaml")
@@ -87,7 +85,7 @@ func TestDeleteResourceNotFound(t *testing.T) {
 }
 
 func TestDeleteUnknownResource(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "delete-unknown-resource.yaml")
@@ -104,7 +102,7 @@ func TestDeleteUnknownResource(t *testing.T) {
 }
 
 func TestPodCreateGetDelete(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "create-get-delete-pod.yaml")
@@ -121,7 +119,7 @@ func TestPodCreateGetDelete(t *testing.T) {
 }
 
 func TestMatches(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "matches.yaml")
@@ -138,7 +136,7 @@ func TestMatches(t *testing.T) {
 }
 
 func TestConditions(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "conditions.yaml")
@@ -155,7 +153,7 @@ func TestConditions(t *testing.T) {
 }
 
 func TestJSON(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "json.yaml")
@@ -172,7 +170,7 @@ func TestJSON(t *testing.T) {
 }
 
 func TestApply(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "apply-deployment.yaml")
@@ -189,7 +187,7 @@ func TestApply(t *testing.T) {
 }
 
 func TestEnvvarSubstitution(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	t.Setenv("pod_name", "foo")
@@ -208,7 +206,7 @@ func TestEnvvarSubstitution(t *testing.T) {
 }
 
 func TestWithLabels(t *testing.T) {
-	skipIfNoKind(t)
+	testutil.SkipIfNoKind(t)
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "list-pods-with-labels.yaml")
@@ -223,41 +221,3 @@ func TestWithLabels(t *testing.T) {
 	err = s.Run(ctx, t)
 	require.Nil(err)
 }
-
-func TestPlacementSpread(t *testing.T) {
-	skipIfNoKind(t)
-	require := require.New(t)
-
-	fp := filepath.Join("testdata", "placement-spread.yaml")
-
-	s, err := gdt.From(fp)
-	require.Nil(err)
-	require.NotNil(s)
-
-	kindCfgPath := filepath.Join("testdata", "kind-config-three-workers-three-zones.yaml")
-
-	var b bytes.Buffer
-	w := bufio.NewWriter(&b)
-	ctx := gdtcontext.New(gdtcontext.WithDebug(w))
-
-	ctx = gdtcontext.RegisterFixture(
-		ctx, "kind-three-workers-three-zones",
-		kindfix.New(
-			kindfix.WithClusterName("kind-three-workers-three-zones"),
-			kindfix.WithConfigPath(kindCfgPath),
-		),
-	)
-
-	err = s.Run(ctx, t)
-	require.Nil(err)
-
-	w.Flush()
-	fmt.Println(b.String())
-}
-
-func skipIfNoKind(t *testing.T) {
-	_, found := os.LookupEnv("SKIP_KIND")
-	if found {
-		t.Skipf("skipping KinD-requiring test")
-	}
-}

fixtures/kind/kind.go

@@ -5,8 +5,11 @@
 package kind
 
 import (
+	"context"
 	"strings"
 
+	gdtcontext "github.com/gdt-dev/gdt/context"
+	"github.com/gdt-dev/gdt/debug"
 	gdttypes "github.com/gdt-dev/gdt/types"
 	"github.com/samber/lo"
 	"sigs.k8s.io/kind/pkg/cluster"
@@ -20,9 +23,29 @@ import (
 type KindFixture struct {
 	// provider is the KinD cluster provider
 	provider *cluster.Provider
-	// cfgStr contains the stringified KUBECONFIG that KinD returns in its
-	// Provider.KubeConfig() call
-	cfgStr string
+	// deleteOnStop indicates that the KinD cluster should be deleted when
+	// the fixture is stopped. Fixtures are stopped when test scenarios
+	// utilizing the fixture have executed all their test steps.
+	//
+	// By default, KinD clusters that were already running when the fixture was
+	// started are not deleted. This is to prevent the deletion of KinD
+	// clusters that were in use outside of a gdt-kube execution. To override
+	// this behaviour and always delete the KinD cluster on stop, use the
+	// WithDeleteOnStop() modifier.
+	deleteOnStop bool
+	// retainOnStop indicates that the KinD cluster should *not* be deleted
+	// when the fixture is stopped. Fixtures are stopped when test scenarios
+	// utilizing the fixture have executed all their test steps.
+	//
+	// By default, KinD clusters that were *not* already running when the fixture was
+	// started are deleted when the fixture stops. This is to clean up KinD
+	// clusters that were created and used by the gdt-kube execution. To override
+	// this behaviour and always retain the KinD cluster on stop, use the
+	// WithRetainOnStop() modifier.
+	retainOnStop bool
+	// runningBeforeStart is true when the KinD cluster was already running
+	// when the fixture was started.
+	runningBeforeStart bool
 	// ClusterName is the name of the KinD cluster. If not specified, gdt will
 	// use the default cluster name that KinD uses, which is just "kind"
 	ClusterName string
@@ -34,20 +57,35 @@ type KindFixture struct {
 	ConfigPath string
 }
 
-func (f *KindFixture) Start() {
+func (f *KindFixture) Start(ctx context.Context) {
+	ctx = gdtcontext.PushTrace(ctx, "fixtures.kind.start")
+	defer func() {
+		ctx = gdtcontext.PopTrace(ctx)
+	}()
 	if f.ClusterName == "" {
 		f.ClusterName = kindconst.DefaultClusterName
 	}
 	if f.isRunning() {
+		debug.Println(ctx, "cluster %s already running", f.ClusterName)
+		f.runningBeforeStart = true
 		return
 	}
 	opts := []cluster.CreateOption{}
 	if f.ConfigPath != "" {
+		debug.Println(
+			ctx, "using custom kind config %s for cluster %s",
+			f.ConfigPath, f.ClusterName,
+		)
 		opts = append(opts, cluster.CreateWithConfigFile(f.ConfigPath))
 	}
 	if err := f.provider.Create(f.ClusterName, opts...); err != nil {
 		panic(err)
 	}
+	debug.Println(ctx, "cluster %s successfully created", f.ClusterName)
+	if !f.retainOnStop {
+		f.deleteOnStop = true
+		debug.Println(ctx, "cluster %s will be deleted on stop", f.ClusterName)
+	}
 }
 
 func (f *KindFixture) isRunning() bool {
@@ -61,7 +99,26 @@ func (f *KindFixture) isRunning() bool {
 	return lo.Contains(clusterNames, f.ClusterName)
 }
 
-func (f *KindFixture) Stop() {}
+func (f *KindFixture) Stop(ctx context.Context) {
+	ctx = gdtcontext.PushTrace(ctx, "fixtures.kind.stop")
+	defer func() {
+		ctx = gdtcontext.PopTrace(ctx)
+	}()
+	if !f.isRunning() {
+		debug.Println(ctx, "cluster %s not running", f.ClusterName)
+		return
+	}
+	if f.runningBeforeStart && !f.deleteOnStop {
+		debug.Println(ctx, "cluster %s was running before start and deleteOnStop=false so not deleting", f.ClusterName)
+		return
+	}
+	if f.deleteOnStop {
+		if err := f.provider.Delete(f.ClusterName, ""); err != nil {
+			panic(err)
+		}
+		debug.Println(ctx, "cluster %s successfully deleted", f.ClusterName)
+	}
+}
 
 func (f *KindFixture) HasState(key string) bool {
 	lkey := strings.ToLower(key)
@@ -119,6 +176,36 @@ func WithConfigPath(path string) KindFixtureModifier {
 	}
 }
 
+// WithDeleteOnStop instructs gdt-kube to always delete the KinD cluster when
+// the fixture stops. Fixtures are stopped when test scenarios utilizing the
+// fixture have executed all their test steps.
+//
+// By default, KinD clusters that were already running when the fixture was
+// started are not deleted. This is to prevent the deletion of KinD
+// clusters that were in use outside of a gdt-kube execution. To override
+// this behaviour and always delete the KinD cluster on stop, use the
+// WithDeleteOnStop() modifier.
+func WithDeleteOnStop() KindFixtureModifier {
+	return func(f *KindFixture) {
+		f.deleteOnStop = true
+	}
+}
+
+// WithRetainOnStop instructs gdt-kube that the KinD cluster should *not* be
+// deleted when the fixture is stopped. Fixtures are stopped when test
+// scenarios utilizing the fixture have executed all their test steps.
+//
+// By default, KinD clusters that were *not* already running when the fixture
+// was started are deleted when the fixture stops. This is to clean up KinD
+// clusters that were created and used by the gdt-kube execution. To override
+// this behaviour and always retain the KinD cluster on stop, use the
+// WithRetainOnStop() modifier.
+func WithRetainOnStop() KindFixtureModifier {
+	return func(f *KindFixture) {
+		f.retainOnStop = true
+	}
+}
+
 // New returns a fixture that exposes Kubernetes configuration/context
 // information about a KinD cluster. If no such KinD cluster exists, one will
 // be created. If the KinD cluster is created, it is destroyed at the end of