From 190f3fddd19ef71b9eaa9657575be95e55f023d0 Mon Sep 17 00:00:00 2001 From: Richard Kovacs Date: Tue, 6 Feb 2024 10:41:27 +0100 Subject: [PATCH 1/2] Basic E2E test for machine controller --- .github/workflows/build_test_ci.yml | 7 +- Makefile | 17 +++- cloud/scope/cluster.go | 6 +- config/manager/manager.yaml | 2 + controller/linodemachine_controller.go | 67 +++++++-------- .../linodemachine_controller_helpers.go | 56 ++++++++++++- .../linodemachine_controller_helpers_test.go | 13 +-- e2e/Makefile | 24 ++++++ e2e/README.MD | 82 +++++++++++++++++++ e2e/basic/cluster/00-assert.yaml | 0 e2e/basic/machine/00-assert.yaml | 0 e2e/basic/vpc/00-assert.yaml | 0 e2e/kuttl-config.yaml | 7 +- .../minimal/00-assert.yaml | 15 ++++ .../minimal/00-assert.yaml | 15 ++++ .../minimal/01-assert.yaml | 17 ++++ .../minimal/01-create-cluster.yaml | 39 +++++++++ .../minimal/02-assert.yaml | 10 +++ .../minimal/02-create-linodemachine.tpl.yml | 12 +++ .../minimal/02-create-linodemachine.yaml | 6 ++ .../minimal/03-delete-linodemachine.yaml | 6 ++ .../minimal/03-error.yaml | 4 + .../minimal/04-verify-linode-instance.yaml | 5 ++ e2e/linodemachine-controller/minimal/Makefile | 1 + .../minimal/00-assert.yaml | 15 ++++ go.mod | 2 +- util/reconciler/defaults.go | 2 + 27 files changed, 375 insertions(+), 55 deletions(-) create mode 100644 e2e/Makefile create mode 100644 e2e/README.MD delete mode 100644 e2e/basic/cluster/00-assert.yaml delete mode 100644 e2e/basic/machine/00-assert.yaml delete mode 100644 e2e/basic/vpc/00-assert.yaml create mode 100644 e2e/linodecluster-controller/minimal/00-assert.yaml create mode 100644 e2e/linodemachine-controller/minimal/00-assert.yaml create mode 100644 e2e/linodemachine-controller/minimal/01-assert.yaml create mode 100644 e2e/linodemachine-controller/minimal/01-create-cluster.yaml create mode 100644 e2e/linodemachine-controller/minimal/02-assert.yaml create mode 100644 e2e/linodemachine-controller/minimal/02-create-linodemachine.tpl.yml create mode 100644 e2e/linodemachine-controller/minimal/02-create-linodemachine.yaml create mode 100644 e2e/linodemachine-controller/minimal/03-delete-linodemachine.yaml create mode 100644 e2e/linodemachine-controller/minimal/03-error.yaml create mode 100644 e2e/linodemachine-controller/minimal/04-verify-linode-instance.yaml create mode 100644 e2e/linodemachine-controller/minimal/Makefile create mode 100644 e2e/linodevpc-controller/minimal/00-assert.yaml diff --git a/.github/workflows/build_test_ci.yml b/.github/workflows/build_test_ci.yml index a2c520702..6ee2f2042 100644 --- a/.github/workflows/build_test_ci.yml +++ b/.github/workflows/build_test_ci.yml @@ -76,7 +76,7 @@ jobs: - name: Docker cache uses: ScribeMD/docker-cache@0.3.7 with: - key: docker-${{ runner.os }}-${{ hashFiles('Makefile') }}} + key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }}} - name: Lint run: make lint @@ -100,6 +100,7 @@ jobs: disable-sudo: true egress-policy: block allowed-endpoints: > + api.linode.com:443 api.github.com:443 github.com:443 gcr.io:443 @@ -130,7 +131,7 @@ jobs: - name: Docker cache uses: ScribeMD/docker-cache@0.3.7 with: - key: docker-${{ runner.os }}-${{ hashFiles('Makefile') }}} + key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }}} - name: E2E test run: make e2etest @@ -173,7 +174,7 @@ jobs: - name: Docker cache uses: ScribeMD/docker-cache@0.3.7 with: - key: docker-${{ runner.os }}-${{ hashFiles('Makefile') }} + key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }} - name: Build the Docker image run: make docker-build diff --git a/Makefile b/Makefile index 702e969b5..96a3be68d 100644 --- a/Makefile +++ b/Makefile @@ -89,11 +89,14 @@ test: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -race -timeout 60s ./... -coverprofile cover.out .PHONY: e2etest -e2etest: kind ctlptl tilt kuttl kustomize clusterctl manifests generate +e2etest: + make --no-print-directory _e2etest # Workaround to force the flag on Github Action + +_e2etest: kind ctlptl tilt kuttl kustomize clusterctl envsubst manifests generate @echo -n "LINODE_TOKEN=$(LINODE_TOKEN)" > config/default/.env.linode $(CTLPTL) apply -f .tilt/ctlptl-config.yaml - $(TILT) ci --timeout 180s -f Tiltfile - $(KUTTL) test --config e2e/kuttl-config.yaml + $(TILT) ci --timeout 240s -f Tiltfile + ROOT_DIR="$(PWD)" $(KUTTL) test --config e2e/kuttl-config.yaml ##@ Build @@ -180,6 +183,7 @@ TILT ?= $(LOCALBIN)/tilt KIND ?= $(LOCALBIN)/kind KUTTL ?= $(LOCALBIN)/kuttl ENVTEST ?= $(LOCALBIN)/setup-envtest +ENVSUBST ?= $(LOCALBIN)/envsubst HUSKY ?= $(LOCALBIN)/husky NILAWAY ?= $(LOCALBIN)/nilaway GOVULNC ?= $(LOCALBIN)/govulncheck @@ -192,6 +196,7 @@ CONTROLLER_TOOLS_VERSION ?= v0.13.0 TILT_VERSION ?= 0.33.6 KIND_VERSION ?= 0.20.0 KUTTL_VERSION ?= 0.15.0 +ENVSUBST_VERSION ?= v1.4.2 HUSKY_VERSION ?= v0.2.16 NILAWAY_VERSION ?= latest GOVULNC_VERSION ?= v1.0.1 @@ -251,6 +256,12 @@ envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest +.PHONY: envsubst +envsubst: $(ENVSUBST) ## Download envsubst locally if necessary. If wrong version is installed, it will be overwritten. +$(ENVSUBST): $(LOCALBIN) + test -s $(ENVSUBST) || \ + (cd $(LOCALBIN); curl -Lso ./envsubst https://github.com/a8m/envsubst/releases/download/$(ENVSUBST_VERSION)/envsubst-$(shell uname -s)-$(ARCH) && chmod +x envsubst) + .PHONY: husky husky: $(HUSKY) ## Download husky locally if necessary. @echo Execute install command to enable git hooks: ./bin/husky install diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index c61077530..a99557394 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -21,13 +21,13 @@ import ( "errors" "fmt" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - infrav1alpha1 "github.com/linode/cluster-api-provider-linode/api/v1alpha1" "github.com/linode/linodego" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + infrav1alpha1 "github.com/linode/cluster-api-provider-linode/api/v1alpha1" ) // ClusterScopeParams defines the input parameters used to create a new Scope. diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 8fa75b04c..ab0520063 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -77,6 +77,8 @@ spec: secretKeyRef: name: token key: LINODE_TOKEN + - name: LINODE_API_VERSION + value: v4beta name: manager securityContext: allowPrivilegeEscalation: false diff --git a/controller/linodemachine_controller.go b/controller/linodemachine_controller.go index dbd5c83d9..bb6968e48 100644 --- a/controller/linodemachine_controller.go +++ b/controller/linodemachine_controller.go @@ -18,7 +18,6 @@ package controller import ( "context" - b64 "encoding/base64" "errors" "fmt" "net/http" @@ -31,6 +30,7 @@ import ( "github.com/linode/linodego" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/tools/record" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" cerrs "sigs.k8s.io/cluster-api/errors" @@ -90,6 +90,8 @@ type LinodeMachineReconciler struct { // // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.16.0/pkg/reconcile +// +//nolint:gocyclo,cyclop // As simple as possible. func (r *LinodeMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultedLoopTimeout(r.ReconcileTimeout)) defer cancel() @@ -98,17 +100,21 @@ func (r *LinodeMachineReconciler) Reconcile(ctx context.Context, req ctrl.Reques linodeMachine := &infrav1alpha1.LinodeMachine{} if err := r.Client.Get(ctx, req.NamespacedName, linodeMachine); err != nil { - log.Error(err, "Failed to fetch Linode machine") + if err = client.IgnoreNotFound(err); err != nil { + log.Error(err, "Failed to fetch Linode machine") + } - return ctrl.Result{}, client.IgnoreNotFound(err) + return ctrl.Result{}, err } machine, err := kutil.GetOwnerMachine(ctx, r.Client, linodeMachine.ObjectMeta) switch { case err != nil: - log.Error(err, "Failed to fetch owner machine") + if err = client.IgnoreNotFound(err); err != nil { + log.Error(err, "Failed to fetch owner machine") + } - return ctrl.Result{}, client.IgnoreNotFound(err) + return ctrl.Result{}, err case machine == nil: log.Info("Machine Controller has not yet set OwnerRef, skipping reconciliation") @@ -134,14 +140,25 @@ func (r *LinodeMachineReconciler) Reconcile(ctx context.Context, req ctrl.Reques log = log.WithValues("Linode machine: ", machine.Name) cluster, err := kutil.GetClusterFromMetadata(ctx, r.Client, machine.ObjectMeta) - if err != nil { - log.Info("Failed to fetch cluster by label") + switch { + case err != nil: + if err = client.IgnoreNotFound(err); err != nil { + log.Error(err, "Failed to fetch cluster by label") + } + + return ctrl.Result{}, err + case cluster == nil: + err = errors.New("missing cluster") + + log.Error(err, "Missing cluster") - return ctrl.Result{}, client.IgnoreNotFound(err) - } else if cluster == nil { - log.Info("Failed to find cluster by label") + return ctrl.Result{}, err + case cluster.Spec.InfrastructureRef == nil: + err = errors.New("missing infrastructure reference") - return ctrl.Result{}, client.IgnoreNotFound(err) + log.Error(err, "Missing infrastructure reference") + + return ctrl.Result{}, err } linodeCluster := &infrav1alpha1.LinodeCluster{} @@ -151,9 +168,11 @@ func (r *LinodeMachineReconciler) Reconcile(ctx context.Context, req ctrl.Reques } if err = r.Client.Get(ctx, linodeClusterKey, linodeCluster); err != nil { - log.Error(err, "Failed to fetch Linode cluster") + if err = client.IgnoreNotFound(err); err != nil { + log.Error(err, "Failed to fetch Linode cluster") + } - return ctrl.Result{}, client.IgnoreNotFound(err) + return ctrl.Result{}, err } machineScope, err := scope.NewMachineScope( @@ -214,7 +233,7 @@ func (r *LinodeMachineReconciler) reconcile( } // Always close the scope when exiting this function so we can persist any LinodeMachine changes. - if patchErr := machineScope.Close(ctx); patchErr != nil && client.IgnoreNotFound(patchErr) != nil { + if patchErr := machineScope.Close(ctx); patchErr != nil && utilerrors.FilterOut(patchErr) != nil { logger.Error(patchErr, "failed to patch LinodeMachine") err = errors.Join(err, patchErr) @@ -294,29 +313,13 @@ func (r *LinodeMachineReconciler) reconcileCreate( linodeInstance = &linodeInstances[0] case 0: - createConfig := linodeMachineSpecToInstanceCreateConfig(machineScope.LinodeMachine.Spec) - if createConfig == nil { - err = errors.New("failed to convert machine spec to create instance config") - - logger.Error(err, "Panic! Struct of LinodeMachineSpec is different than InstanceCreateOptions") - - return err - } - createConfig.Label = machineScope.LinodeMachine.Spec.Label - createConfig.Tags = tags - createConfig.SwapSize = util.Pointer(0) - createConfig.PrivateIP = true - // get the bootstrap data for the Linode instance and set it for create config - bootstrapData, err := machineScope.GetBootstrapData(ctx) + createConfig, err := r.newCreateConfig(ctx, machineScope, tags, logger) if err != nil { - logger.Info("Failed to get bootstrap data", "error", err.Error()) + logger.Error(err, "Failed to create Linode machine create config") return err } - createConfig.Metadata = &linodego.InstanceMetadataOptions{ - UserData: b64.StdEncoding.EncodeToString(bootstrapData), - } if machineScope.LinodeCluster.Spec.VPCRef != nil { iface, err := r.getVPCInterfaceConfig(ctx, machineScope, createConfig.Interfaces, logger) diff --git a/controller/linodemachine_controller_helpers.go b/controller/linodemachine_controller_helpers.go index 666858cf5..ecaa44e64 100644 --- a/controller/linodemachine_controller_helpers.go +++ b/controller/linodemachine_controller_helpers.go @@ -19,13 +19,15 @@ package controller import ( "bytes" "context" + b64 "encoding/base64" "encoding/gob" "errors" "sort" "github.com/go-logr/logr" - infrav1 "github.com/linode/cluster-api-provider-linode/api/v1alpha1" + "github.com/google/uuid" "github.com/linode/cluster-api-provider-linode/cloud/scope" + "github.com/linode/cluster-api-provider-linode/util" "github.com/linode/cluster-api-provider-linode/util/reconciler" "github.com/linode/linodego" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -35,8 +37,54 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" + + infrav1alpha1 "github.com/linode/cluster-api-provider-linode/api/v1alpha1" ) +func (*LinodeMachineReconciler) newCreateConfig(ctx context.Context, machineScope *scope.MachineScope, tags []string, logger logr.Logger) (*linodego.InstanceCreateOptions, error) { + var err error + + createConfig := linodeMachineSpecToInstanceCreateConfig(machineScope.LinodeMachine.Spec) + if createConfig == nil { + err = errors.New("failed to convert machine spec to create instance config") + + logger.Error(err, "Panic! Struct of LinodeMachineSpec is different than InstanceCreateOptions") + + return nil, err + } + createConfig.SwapSize = util.Pointer(0) + createConfig.PrivateIP = true + + bootstrapData, err := machineScope.GetBootstrapData(ctx) + if err != nil { + logger.Info("Failed to get bootstrap data", "error", err.Error()) + + return nil, err + } + createConfig.Metadata = &linodego.InstanceMetadataOptions{ + UserData: b64.StdEncoding.EncodeToString(bootstrapData), + } + + if createConfig.Tags == nil { + createConfig.Tags = []string{} + } + createConfig.Tags = append(createConfig.Tags, tags...) + + if createConfig.Label == "" { + createConfig.Label = util.RenderObjectLabel(machineScope.LinodeMachine.UID) + } + + if createConfig.Image == "" { + createConfig.Image = reconciler.DefaultMachineControllerLinodeImage + } + + if createConfig.RootPass == "" { + createConfig.RootPass = uuid.NewString() + } + + return createConfig, nil +} + func (r *LinodeMachineReconciler) linodeClusterToLinodeMachines(logger logr.Logger) handler.MapFunc { logger = logger.WithName("LinodeMachineReconciler").WithName("linodeClusterToLinodeMachines") @@ -44,7 +92,7 @@ func (r *LinodeMachineReconciler) linodeClusterToLinodeMachines(logger logr.Logg ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultMappingTimeout) defer cancel() - linodeCluster, ok := o.(*infrav1.LinodeCluster) + linodeCluster, ok := o.(*infrav1alpha1.LinodeCluster) if !ok { logger.Info("Failed to cast object to Cluster") @@ -142,7 +190,7 @@ func (r *LinodeMachineReconciler) getVPCInterfaceConfig(ctx context.Context, mac logger = logger.WithValues("vpcName", name, "vpcNamespace", namespace) - linodeVPC := infrav1.LinodeVPC{ + linodeVPC := infrav1alpha1.LinodeVPC{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: name, @@ -202,7 +250,7 @@ func (r *LinodeMachineReconciler) getVPCInterfaceConfig(ctx context.Context, mac }, nil } -func linodeMachineSpecToInstanceCreateConfig(machineSpec infrav1.LinodeMachineSpec) *linodego.InstanceCreateOptions { +func linodeMachineSpecToInstanceCreateConfig(machineSpec infrav1alpha1.LinodeMachineSpec) *linodego.InstanceCreateOptions { var buf bytes.Buffer enc := gob.NewEncoder(&buf) err := enc.Encode(machineSpec) diff --git a/controller/linodemachine_controller_helpers_test.go b/controller/linodemachine_controller_helpers_test.go index 3a00d3a6b..d3ce540f4 100644 --- a/controller/linodemachine_controller_helpers_test.go +++ b/controller/linodemachine_controller_helpers_test.go @@ -5,10 +5,11 @@ import ( "encoding/gob" "testing" - infrav1 "github.com/linode/cluster-api-provider-linode/api/v1alpha1" "github.com/linode/linodego" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + infrav1alpha1 "github.com/linode/cluster-api-provider-linode/api/v1alpha1" ) func TestLinodeMachineSpecToCreateInstanceConfig(t *testing.T) { @@ -16,7 +17,7 @@ func TestLinodeMachineSpecToCreateInstanceConfig(t *testing.T) { subnetID := 1 - machineSpec := infrav1.LinodeMachineSpec{ + machineSpec := infrav1alpha1.LinodeMachineSpec{ Region: "region", Type: "type", Label: "label", @@ -28,14 +29,14 @@ func TestLinodeMachineSpecToCreateInstanceConfig(t *testing.T) { StackScriptData: map[string]string{"script": "data"}, BackupID: 1, Image: "image", - Interfaces: []infrav1.InstanceConfigInterfaceCreateOptions{ + Interfaces: []infrav1alpha1.InstanceConfigInterfaceCreateOptions{ { IPAMAddress: "address", Label: "label", Purpose: linodego.InterfacePurposePublic, Primary: true, SubnetID: &subnetID, - IPv4: &infrav1.VPCIPv4{ + IPv4: &infrav1alpha1.VPCIPv4{ VPC: "vpc", NAT1To1: "nat11", }, @@ -45,7 +46,7 @@ func TestLinodeMachineSpecToCreateInstanceConfig(t *testing.T) { BackupsEnabled: true, PrivateIP: true, Tags: []string{"tag"}, - Metadata: &infrav1.InstanceMetadataOptions{ + Metadata: &infrav1alpha1.InstanceMetadataOptions{ UserData: "userdata", }, FirewallID: 1, @@ -59,7 +60,7 @@ func TestLinodeMachineSpecToCreateInstanceConfig(t *testing.T) { err := enc.Encode(createConfig) require.NoError(t, err, "Failed to encode InstanceCreateOptions") - var actualMachineSpec infrav1.LinodeMachineSpec + var actualMachineSpec infrav1alpha1.LinodeMachineSpec dec := gob.NewDecoder(&buf) err = dec.Decode(&actualMachineSpec) require.NoError(t, err, "Failed to decode LinodeMachineSpec") diff --git a/e2e/Makefile b/e2e/Makefile new file mode 100644 index 000000000..83d77f710 --- /dev/null +++ b/e2e/Makefile @@ -0,0 +1,24 @@ +runTestCase: + @T="$$(KUBECONFIG="$$ROOT_DIR/kubeconfig" kuttl test --timeout 300 --skip-delete --namespace "$$NAMESPACE" "$$TC" 2>&1)" ;\ + echo "$$T" |\ + grep -v harness.go |\ + grep -v ^=== |\ + grep -v "running without a 'kuttl-test.yaml" |\ + grep -v "creation of user-supplied namespace" |\ + grep -v "not match file name regexp" |\ + grep -v "kutt-test config testdirs is overridden" ;\ + echo "$$T" | \ + grep '^PASS' || \ + (echo "$$T" | grep harness.go && cat "$$TC/step/00-step.yaml" && exit 1) + +renderTestCase: + @D="$$(mktemp -d)" ;\ + mkdir -p "$$D/step" ;\ + envsubst -i "$$TPL" -o "$$D/step/00-step.yaml" ;\ + echo -n "$$D" + +getKubeUid: + @kubectl get -o jsonpath='{.metadata.uid}' -n "$$NAMESPACE" "$$OBJ" + +callLinodeApiGet: + @curl -s -H "Authorization: Bearer $$LINODE_TOKEN" -H "X-Filter: $$FILTER" "https://api.linode.com/v4beta/$$URI" diff --git a/e2e/README.MD b/e2e/README.MD new file mode 100644 index 000000000..e660dc72d --- /dev/null +++ b/e2e/README.MD @@ -0,0 +1,82 @@ +# E2E Framework + +The E2E framework uses Kuttl under the hood. Kuttl is a simple and most importantly static tool. +That means there is no way to dynamically generate manifests. +The solution is KIK (Kuttl-In-Kuttl)! + +## Kuttl-In-Kuttl + +## How does it work + + - Instead of creating a static Kuttl manifest, you can create template. Template extension must be `.yml` otherwise Kuttl starts executing it. + - Template is rendered via `envsubst`, so you can use Bash variables for dynamic data injection. + - Variables always present + - `${KUBECONFIG}` + - `${NAMESPACE}` + - `${ROOT_DIR}` + + Example template +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeMachine +metadata: + ownerReferences: + - apiVersion: cluster.x-k8s.io/v1beta1 + kind: Machine + name: machine-sample + uid: ${MACHINE_UID} + name: linodemachine-sample +spec: + region: us-sea + type: g5-nanode-1 +``` + + - Helper tools are available, please create a Makefile in your test folder. In this way you can use built in helpers without including any script in the test, plus this file is a good place for test specific helpers. + + Example Makefile +```makefile +include ../../Makefile +``` + + - Template is nothing without a runner, please use helpers provided in Makefile to render and run template. + + Example template runner +```yaml +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \ + TC="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestCase +``` + + - You can run testsuits (which can contain template rendering). + + Example testsuit runner: +```yaml +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: TC="$PWD/testsuit" make runTestCase +``` + + - Test execution order is not guaranteed. If a test depends on another resource it is strongly suggested to create `assert` files to wait for an event. Almost every test depends on CAPI providers, so there is a good chance, you have to create a `00-assert.yaml` in your test folder. + + Example `00-assert.yaml`: +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: capi-controller-manager + namespace: capi-system +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-api-provider-linode-controller-manager + namespace: cluster-api-provider-linode-system +status: + availableReplicas: 1 +``` diff --git a/e2e/basic/cluster/00-assert.yaml b/e2e/basic/cluster/00-assert.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/e2e/basic/machine/00-assert.yaml b/e2e/basic/machine/00-assert.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/e2e/basic/vpc/00-assert.yaml b/e2e/basic/vpc/00-assert.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/e2e/kuttl-config.yaml b/e2e/kuttl-config.yaml index b06687408..a9abaa602 100644 --- a/e2e/kuttl-config.yaml +++ b/e2e/kuttl-config.yaml @@ -1,6 +1,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestSuite testDirs: -- e2e/basic -kindConfig: e2e/kind-config.yaml -timeout: 120 +- e2e/linodecluster-controller +- e2e/linodemachine-controller +- e2e/linodevpc-controller +timeout: 300 diff --git a/e2e/linodecluster-controller/minimal/00-assert.yaml b/e2e/linodecluster-controller/minimal/00-assert.yaml new file mode 100644 index 000000000..4107fd8f0 --- /dev/null +++ b/e2e/linodecluster-controller/minimal/00-assert.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: capi-controller-manager + namespace: capi-system +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-api-provider-linode-controller-manager + namespace: cluster-api-provider-linode-system +status: + availableReplicas: 1 diff --git a/e2e/linodemachine-controller/minimal/00-assert.yaml b/e2e/linodemachine-controller/minimal/00-assert.yaml new file mode 100644 index 000000000..4107fd8f0 --- /dev/null +++ b/e2e/linodemachine-controller/minimal/00-assert.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: capi-controller-manager + namespace: capi-system +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-api-provider-linode-controller-manager + namespace: cluster-api-provider-linode-system +status: + availableReplicas: 1 diff --git a/e2e/linodemachine-controller/minimal/01-assert.yaml b/e2e/linodemachine-controller/minimal/01-assert.yaml new file mode 100644 index 000000000..ca3986171 --- /dev/null +++ b/e2e/linodemachine-controller/minimal/01-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + annotations: + cluster.x-k8s.io/paused: "true" + name: cluster-sample +spec: + paused: true +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Machine +metadata: + annotations: + cluster.x-k8s.io/paused: "true" + name: machine-sample +spec: + clusterName: cluster-sample diff --git a/e2e/linodemachine-controller/minimal/01-create-cluster.yaml b/e2e/linodemachine-controller/minimal/01-create-cluster.yaml new file mode 100644 index 000000000..c77670b2f --- /dev/null +++ b/e2e/linodemachine-controller/minimal/01-create-cluster.yaml @@ -0,0 +1,39 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeCluster +metadata: + annotations: + cluster.x-k8s.io/paused: "true" + name: linodecluster-sample +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + annotations: + cluster.x-k8s.io/paused: "true" + name: cluster-sample +spec: + paused: true + infrastructureRef: + name: linodecluster-sample +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Machine +metadata: + annotations: + cluster.x-k8s.io/paused: "true" + name: machine-sample +spec: + clusterName: cluster-sample + bootstrap: + configRef: + apiVersion: v1 + kind: "ConfigMap" + name: "boostrap-sample" + dataSecretName: bootstrap-data-sample +--- +apiVersion: v1 +kind: Secret +metadata: + name: bootstrap-data-sample +data: + value: dG91Y2ggL29rCg== diff --git a/e2e/linodemachine-controller/minimal/02-assert.yaml b/e2e/linodemachine-controller/minimal/02-assert.yaml new file mode 100644 index 000000000..1cb149503 --- /dev/null +++ b/e2e/linodemachine-controller/minimal/02-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeMachine +metadata: + name: linodemachine-sample +spec: + region: us-sea + type: g5-nanode-1 +status: + ready: true + instanceState: running diff --git a/e2e/linodemachine-controller/minimal/02-create-linodemachine.tpl.yml b/e2e/linodemachine-controller/minimal/02-create-linodemachine.tpl.yml new file mode 100644 index 000000000..0a4b23b4e --- /dev/null +++ b/e2e/linodemachine-controller/minimal/02-create-linodemachine.tpl.yml @@ -0,0 +1,12 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeMachine +metadata: + ownerReferences: + - apiVersion: cluster.x-k8s.io/v1beta1 + kind: Machine + name: machine-sample + uid: ${MACHINE_UID} + name: linodemachine-sample +spec: + region: us-sea + type: g5-nanode-1 diff --git a/e2e/linodemachine-controller/minimal/02-create-linodemachine.yaml b/e2e/linodemachine-controller/minimal/02-create-linodemachine.yaml new file mode 100644 index 000000000..de40c5987 --- /dev/null +++ b/e2e/linodemachine-controller/minimal/02-create-linodemachine.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \ + TC="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestCase diff --git a/e2e/linodemachine-controller/minimal/03-delete-linodemachine.yaml b/e2e/linodemachine-controller/minimal/03-delete-linodemachine.yaml new file mode 100644 index 000000000..3bf5e64ad --- /dev/null +++ b/e2e/linodemachine-controller/minimal/03-delete-linodemachine.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: +- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: LinodeMachine + name: linodemachine-sample diff --git a/e2e/linodemachine-controller/minimal/03-error.yaml b/e2e/linodemachine-controller/minimal/03-error.yaml new file mode 100644 index 000000000..4ff1b308e --- /dev/null +++ b/e2e/linodemachine-controller/minimal/03-error.yaml @@ -0,0 +1,4 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeMachine +metadata: + name: linodemachine-sample diff --git a/e2e/linodemachine-controller/minimal/04-verify-linode-instance.yaml b/e2e/linodemachine-controller/minimal/04-verify-linode-instance.yaml new file mode 100644 index 000000000..4515f8566 --- /dev/null +++ b/e2e/linodemachine-controller/minimal/04-verify-linode-instance.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + URI="linode/instances" FILTER="{\"tags\":\"$(OBJ=linodemachines/linodemachine-sample make getKubeUid)\"}" make callLinodeApiGet | grep 'results": 0' diff --git a/e2e/linodemachine-controller/minimal/Makefile b/e2e/linodemachine-controller/minimal/Makefile new file mode 100644 index 000000000..3924bfdc1 --- /dev/null +++ b/e2e/linodemachine-controller/minimal/Makefile @@ -0,0 +1 @@ +include ../../Makefile diff --git a/e2e/linodevpc-controller/minimal/00-assert.yaml b/e2e/linodevpc-controller/minimal/00-assert.yaml new file mode 100644 index 000000000..4107fd8f0 --- /dev/null +++ b/e2e/linodevpc-controller/minimal/00-assert.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: capi-controller-manager + namespace: capi-system +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-api-provider-linode-controller-manager + namespace: cluster-api-provider-linode-system +status: + availableReplicas: 1 diff --git a/go.mod b/go.mod index a81240a93..aa2c033a4 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.21.5 require ( github.com/go-logr/logr v1.4.1 + github.com/google/uuid v1.3.1 github.com/linode/linodego v1.28.0 github.com/onsi/ginkgo/v2 v2.15.0 github.com/onsi/gomega v1.31.1 @@ -41,7 +42,6 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.3.1 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/util/reconciler/defaults.go b/util/reconciler/defaults.go index 5a13bf0e4..db9722be8 100644 --- a/util/reconciler/defaults.go +++ b/util/reconciler/defaults.go @@ -28,6 +28,8 @@ const ( // DefaultMachineControllerWaitForBootstrapDelay is the default requeue delay if bootstrap data is not ready. DefaultMachineControllerWaitForBootstrapDelay = 5 * time.Second + // DefaultMachineControllerLinodeImage default image. + DefaultMachineControllerLinodeImage = "linode/ubuntu22.04" // DefaultMachineControllerWaitForRunningDelay is the default requeue delay if instance is not running. DefaultMachineControllerWaitForRunningDelay = 5 * time.Second // DefaultMachineControllerWaitForRunningTimeout is the default timeout if instance is not running. From 7cbdc4ceebc0227989117323382812b782f75b7d Mon Sep 17 00:00:00 2001 From: Richard Kovacs Date: Thu, 8 Feb 2024 13:45:57 +0100 Subject: [PATCH 2/2] Fix linode machine update --- controller/linodemachine_controller.go | 37 +++++++++++++------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/controller/linodemachine_controller.go b/controller/linodemachine_controller.go index bb6968e48..04d6a0531 100644 --- a/controller/linodemachine_controller.go +++ b/controller/linodemachine_controller.go @@ -271,7 +271,7 @@ func (r *LinodeMachineReconciler) reconcile( logger = logger.WithValues("ID", *machineScope.LinodeMachine.Spec.InstanceID) - res, err = r.reconcileUpdate(ctx, logger, machineScope) + res, linodeInstance, err = r.reconcileUpdate(ctx, logger, machineScope) return } @@ -284,7 +284,7 @@ func (r *LinodeMachineReconciler) reconcile( return ctrl.Result{RequeueAfter: reconciler.DefaultMachineControllerWaitForBootstrapDelay}, nil } - err = r.reconcileCreate(ctx, logger, machineScope, clusterScope) + linodeInstance, err = r.reconcileCreate(ctx, logger, machineScope, clusterScope) return } @@ -294,7 +294,7 @@ func (r *LinodeMachineReconciler) reconcileCreate( logger logr.Logger, machineScope *scope.MachineScope, clusterScope *scope.ClusterScope, -) error { +) (*linodego.Instance, error) { logger.Info("creating machine") tags := []string{string(machineScope.LinodeCluster.UID), string(machineScope.LinodeMachine.UID)} @@ -303,7 +303,7 @@ func (r *LinodeMachineReconciler) reconcileCreate( if err != nil { logger.Error(err, "Failed to list Linode machine instances") - return err + return nil, err } var linodeInstance *linodego.Instance @@ -318,7 +318,7 @@ func (r *LinodeMachineReconciler) reconcileCreate( if err != nil { logger.Error(err, "Failed to create Linode machine create config") - return err + return nil, err } if machineScope.LinodeCluster.Spec.VPCRef != nil { @@ -326,7 +326,7 @@ func (r *LinodeMachineReconciler) reconcileCreate( if err != nil { logger.Error(err, "Failed to get VPC interface confiog") - return err + return nil, err } createConfig.Interfaces = append(createConfig.Interfaces, *iface) @@ -336,14 +336,14 @@ func (r *LinodeMachineReconciler) reconcileCreate( if err != nil { logger.Error(err, "Failed to create Linode machine instance") - return err + return nil, err } default: err = errors.New("multiple instances") logger.Error(err, "Panic! Multiple instances found. This might be a concurrency issue in the controller!!!", "tags", tags) - return err + return nil, err } if linodeInstance == nil { @@ -351,7 +351,7 @@ func (r *LinodeMachineReconciler) reconcileCreate( logger.Error(err, "Panic! Failed to create isntance") - return err + return nil, err } machineScope.LinodeMachine.Status.Ready = true @@ -369,26 +369,25 @@ func (r *LinodeMachineReconciler) reconcileCreate( if err = services.AddNodeToNB(ctx, logger, machineScope, clusterScope); err != nil { logger.Error(err, "Failed to add instance to Node Balancer backend") - return err + return linodeInstance, err } - return nil + return linodeInstance, nil } func (r *LinodeMachineReconciler) reconcileUpdate( ctx context.Context, logger logr.Logger, machineScope *scope.MachineScope, -) (res reconcile.Result, err error) { +) (res reconcile.Result, linodeInstance *linodego.Instance, err error) { logger.Info("updating machine") res = ctrl.Result{} if machineScope.LinodeMachine.Spec.InstanceID == nil { - return res, errors.New("missing instance ID") + return res, nil, errors.New("missing instance ID") } - var linodeInstance *linodego.Instance if linodeInstance, err = machineScope.LinodeClient.GetInstance(ctx, *machineScope.LinodeMachine.Spec.InstanceID); err != nil { err = util.IgnoreLinodeAPIError(err, http.StatusNotFound) if err != nil { @@ -403,27 +402,27 @@ func (r *LinodeMachineReconciler) reconcileUpdate( conditions.MarkFalse(machineScope.LinodeMachine, clusterv1.ReadyCondition, "missing", clusterv1.ConditionSeverityWarning, "instance not found") } - return res, err + return res, nil, err } if _, ok := requeueInstanceStatuses[linodeInstance.Status]; ok { if linodeInstance.Updated.Add(reconciler.DefaultMachineControllerWaitForRunningTimeout).After(time.Now()) { logger.Info("Instance has one operaton running, re-queuing reconciliation", "status", linodeInstance.Status) - return ctrl.Result{RequeueAfter: reconciler.DefaultMachineControllerWaitForRunningDelay}, nil + return ctrl.Result{RequeueAfter: reconciler.DefaultMachineControllerWaitForRunningDelay}, linodeInstance, nil } logger.Info("Instance has one operaton long running, skipping reconciliation", "status", linodeInstance.Status) conditions.MarkFalse(machineScope.LinodeMachine, clusterv1.ReadyCondition, string(linodeInstance.Status), clusterv1.ConditionSeverityInfo, "skipped due to long running operation") - return res, nil + return res, linodeInstance, nil } else if linodeInstance.Status != linodego.InstanceRunning { logger.Info("Instance has incompatible status, skipping reconciliation", "status", linodeInstance.Status) conditions.MarkFalse(machineScope.LinodeMachine, clusterv1.ReadyCondition, string(linodeInstance.Status), clusterv1.ConditionSeverityInfo, "incompatible status") - return res, nil + return res, linodeInstance, nil } machineScope.LinodeMachine.Status.Ready = true @@ -432,7 +431,7 @@ func (r *LinodeMachineReconciler) reconcileUpdate( r.Recorder.Event(machineScope.LinodeMachine, corev1.EventTypeNormal, string(clusterv1.ReadyCondition), "instance is running") - return res, nil + return res, linodeInstance, nil } func (r *LinodeMachineReconciler) reconcileDelete(