Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Machine e2e test with bring your own VPC #83

Merged
merged 1 commit into from
Feb 8, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,12 @@ test: manifests generate fmt vet envtest ## Run tests.
e2etest:
make --no-print-directory _e2etest # Workaround to force the flag on Github Action

_e2etest: kind ctlptl tilt kuttl kustomize clusterctl envsubst manifests generate
_e2etest-infra: kind ctlptl tilt kuttl kustomize clusterctl
@echo -n "LINODE_TOKEN=$(LINODE_TOKEN)" > config/default/.env.linode
$(CTLPTL) apply -f .tilt/ctlptl-config.yaml
$(TILT) ci --timeout 240s -f Tiltfile

_e2etest: manifests generate envsubst _e2etest-infra
ROOT_DIR="$(PWD)" $(KUTTL) test --config e2e/kuttl-config.yaml

##@ Build
Expand Down
5 changes: 3 additions & 2 deletions controller/linodemachine_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/linode/cluster-api-provider-linode/util/reconciler"
"github.com/linode/linodego"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/client-go/tools/record"
Expand Down Expand Up @@ -233,7 +234,7 @@ func (r *LinodeMachineReconciler) reconcile(
}

// Always close the scope when exiting this function so we can persist any LinodeMachine changes.
if patchErr := machineScope.Close(ctx); patchErr != nil && utilerrors.FilterOut(patchErr) != nil {
if patchErr := machineScope.Close(ctx); patchErr != nil && utilerrors.FilterOut(patchErr, apierrors.IsNotFound) != nil {
logger.Error(patchErr, "failed to patch LinodeMachine")

err = errors.Join(err, patchErr)
Expand Down Expand Up @@ -324,7 +325,7 @@ func (r *LinodeMachineReconciler) reconcileCreate(
if machineScope.LinodeCluster.Spec.VPCRef != nil {
iface, err := r.getVPCInterfaceConfig(ctx, machineScope, createConfig.Interfaces, logger)
if err != nil {
logger.Error(err, "Failed to get VPC interface confiog")
logger.Error(err, "Failed to get VPC interface config")

return nil, err
}
Expand Down
10 changes: 9 additions & 1 deletion controller/linodemachine_controller_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -173,9 +173,14 @@ func (r *LinodeMachineReconciler) requestsForCluster(ctx context.Context, namesp
continue
}

infraNs := item.Spec.InfrastructureRef.Namespace
if infraNs == "" {
infraNs = item.Namespace
}

result = append(result, ctrl.Request{
NamespacedName: client.ObjectKey{
Namespace: item.Namespace,
Namespace: infraNs,
Name: item.Spec.InfrastructureRef.Name,
},
})
Expand All @@ -187,6 +192,9 @@ func (r *LinodeMachineReconciler) requestsForCluster(ctx context.Context, namesp
func (r *LinodeMachineReconciler) getVPCInterfaceConfig(ctx context.Context, machineScope *scope.MachineScope, existingIfaces []linodego.InstanceConfigInterfaceCreateOptions, logger logr.Logger) (*linodego.InstanceConfigInterfaceCreateOptions, error) {
name := machineScope.LinodeCluster.Spec.VPCRef.Name
namespace := machineScope.LinodeCluster.Spec.VPCRef.Namespace
if namespace == "" {
namespace = machineScope.LinodeCluster.Namespace
}

logger = logger.WithValues("vpcName", name, "vpcNamespace", namespace)

Expand Down
3 changes: 2 additions & 1 deletion controller/linodevpc_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"time"

corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/client-go/tools/record"
Expand Down Expand Up @@ -121,7 +122,7 @@ func (r *LinodeVPCReconciler) reconcile(
}

// Always close the scope when exiting this function so we can persist any LinodeMachine changes.
if patchErr := vpcScope.Close(ctx); patchErr != nil && utilerrors.FilterOut(patchErr) != nil {
if patchErr := vpcScope.Close(ctx); patchErr != nil && utilerrors.FilterOut(patchErr, apierrors.IsNotFound) != nil {
logger.Error(patchErr, "failed to patch LinodeVPC")

err = errors.Join(err, patchErr)
Expand Down
40 changes: 34 additions & 6 deletions e2e/Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,21 @@
runTestCase:
@T="$$(KUBECONFIG="$$ROOT_DIR/kubeconfig" kuttl test --timeout 300 --skip-delete --namespace "$$NAMESPACE" "$$TC" 2>&1)" ;\
ROOT_DIR ?= $(shell git rev-parse --show-toplevel)
BIN_DIR ?= $(ROOT_DIR)/bin
TARGET_API ?= api.linode.com
TARGET_API_VERSION ?= v4beta

mhmxs marked this conversation as resolved.
Show resolved Hide resolved
runThisTest:
@echo "PLease execute make -C ($ROOT_DIR)/Makefile _e2etest-infra to spin up infrastructure"
make --no-print-directory _runThisTest

_runThisTest:
@D="$$(mktemp -d)" ;\
cp $(ROOT_DIR)/e2e/Makefile $$D ;\
mkdir $$D/suit ;\
cp -r $(PWD) $$D/suit ;\
ROOT_DIR=$(ROOT_DIR) KUBECONFIG="$(ROOT_DIR)/kubeconfig" $(BIN_DIR)/kuttl test --timeout 300 --skip-delete "$$D/suit"

runTestSuit:
@T="$$(KUBECONFIG="$$ROOT_DIR/kubeconfig" $(BIN_DIR)/kuttl test --timeout 300 --skip-delete --namespace "$$NAMESPACE" "$$TS" 2>&1)" ;\
echo "$$T" |\
grep -v harness.go |\
grep -v ^=== |\
Expand All @@ -9,16 +25,28 @@ runTestCase:
grep -v "kutt-test config testdirs is overridden" ;\
echo "$$T" | \
grep '^PASS' || \
(echo "$$T" | grep harness.go && cat "$$TC/step/00-step.yaml" && exit 1)
(echo "$$T" | grep harness.go && find $$TS -name *.yaml -exec cat {} \; && exit 1)

renderTestCase:
@D="$$(mktemp -d)" ;\
mkdir -p "$$D/step" ;\
envsubst -i "$$TPL" -o "$$D/step/00-step.yaml" ;\
mkdir -p "$$D/case" ;\
$(BIN_DIR)/envsubst -i "$$TPL" -o "$$D/case/00-case.yaml" ;\
echo -n "$$D"

renderManifest:
@echo $(shell make --no-print-directory renderTestCase)/case/00-case.yaml

getKubeUid:
@kubectl get -o jsonpath='{.metadata.uid}' -n "$$NAMESPACE" "$$OBJ"

patchKubeObj:
@kubectl patch --type=merge -p "$$PATCH" -n "$$NAMESPACE" "$$OBJ"

patchKubeObjStatus:
@kubectl patch --type=merge --subresource status -p "$$PATCH" -n "$$NAMESPACE" "$$OBJ"

callLinodeApiGet:
@curl -s -H "Authorization: Bearer $$LINODE_TOKEN" -H "X-Filter: $$FILTER" "https://api.linode.com/v4beta/$$URI"
@curl -s -H "Authorization: Bearer $$LINODE_TOKEN" -H "X-Filter: $$FILTER" -H "Content-Type: application/json" "https://$(TARGET_API)/$(TARGET_API_VERSION)/$$URI"

callLinodeApiPost:
@curl -s -H "Authorization: Bearer $$LINODE_TOKEN" -H "Content-Type: application/json" -d "$$BODY" -X POST "https://$(TARGET_API)/$(TARGET_API_VERSION)/$$URI"
11 changes: 9 additions & 2 deletions e2e/README.MD
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ kind: TestStep
commands:
- script: |-
MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \
TC="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestCase
TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit
```
- You can run testsuits (which can contain template rendering).
Expand All @@ -57,7 +57,7 @@ commands:
apiVersion: kuttl.dev/v1beta1
kind: TestStep
commands:
- script: TC="$PWD/testsuit" make runTestCase
- script: TS="$PWD/testsuit" make runTestSuit
```
- Test execution order is not guaranteed. If a test depends on another resource it is strongly suggested to create `assert` files to wait for an event. Almost every test depends on CAPI providers, so there is a good chance, you have to create a `00-assert.yaml` in your test folder.
Expand All @@ -80,3 +80,10 @@ metadata:
status:
availableReplicas: 1
```

## Executing individual test suit

```bash
make _e2etest-infra # Only once per cluster!
(cd e2e/linodemachine-controller/minimal ; make runThisTest)
```
15 changes: 15 additions & 0 deletions e2e/linodemachine-controller/byovpc/00-assert.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: capi-controller-manager
namespace: capi-system
status:
availableReplicas: 1
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: cluster-api-provider-linode-controller-manager
namespace: cluster-api-provider-linode-system
status:
availableReplicas: 1
8 changes: 8 additions & 0 deletions e2e/linodemachine-controller/byovpc/01-assert.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LinodeVPC
metadata:
annotations:
cluster.x-k8s.io/paused: "true"
name: linodevpc-sample
status:
ready: true
46 changes: 46 additions & 0 deletions e2e/linodemachine-controller/byovpc/01-create-cluster.tpl.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LinodeCluster
metadata:
annotations:
cluster.x-k8s.io/paused: "true"
name: linodecluster-sample
spec:
region: us-sea
vpcRef:
kind: LinodeVPC
name: linodevpc-sample
namespace: ${NAMESPACE}
uid: ${VPC_UID}
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
annotations:
cluster.x-k8s.io/paused: "true"
name: cluster-sample
spec:
paused: true
infrastructureRef:
name: linodecluster-sample
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: Machine
metadata:
annotations:
cluster.x-k8s.io/paused: "true"
name: machine-sample
spec:
clusterName: cluster-sample
bootstrap:
configRef:
apiVersion: v1
kind: "ConfigMap"
name: "boostrap-sample"
dataSecretName: bootstrap-data-sample
---
apiVersion: v1
kind: Secret
metadata:
name: bootstrap-data-sample
data:
value: dG91Y2ggL29rCg==
11 changes: 11 additions & 0 deletions e2e/linodemachine-controller/byovpc/01-create-cluster.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
apiVersion: kuttl.dev/v1beta1
kind: TestStep
commands:
- command: make createVPC
- script: |-
VPC_ID=$(make fetchVPCID) \
TS="$(TPL="$PWD/01-create-vpc.tpl.yml" make renderTestCase)" make runTestSuit
- command: make enableVPC
- script: |-
VPC_UID="$(OBJ=linodevpcs/linodevpc-sample make getKubeUid)" \
TS="$(TPL="$PWD/01-create-cluster.tpl.yml" make renderTestCase)" make runTestSuit
10 changes: 10 additions & 0 deletions e2e/linodemachine-controller/byovpc/01-create-vpc.tpl.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LinodeVPC
metadata:
annotations:
cluster.x-k8s.io/paused: "true"
name: linodevpc-sample
spec:
label: capli-e2e-byovpc-sample
region: us-sea
vpcID: ${VPC_ID}
10 changes: 10 additions & 0 deletions e2e/linodemachine-controller/byovpc/01-vpc-create-options.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"label": "capli-e2e-byovpc-sample",
"region": "us-sea",
"subnets": [
{
"ipv4": "10.0.0.0/24",
"label": "capli-e2e-byovpc-sample"
}
]
}
10 changes: 10 additions & 0 deletions e2e/linodemachine-controller/byovpc/02-assert.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LinodeMachine
metadata:
name: linodemachine-sample
spec:
region: us-sea
type: g5-nanode-1
status:
ready: true
instanceState: running
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LinodeMachine
metadata:
ownerReferences:
- apiVersion: cluster.x-k8s.io/v1beta1
kind: Machine
name: machine-sample
uid: ${MACHINE_UID}
name: linodemachine-sample
spec:
region: us-sea
type: g5-nanode-1
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
apiVersion: kuttl.dev/v1beta1
kind: TestStep
commands:
- script: |-
MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \
TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
apiVersion: kuttl.dev/v1beta1
kind: TestStep
delete:
- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LinodeMachine
name: linodemachine-sample
4 changes: 4 additions & 0 deletions e2e/linodemachine-controller/byovpc/03-error.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LinodeMachine
metadata:
name: linodemachine-sample
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
apiVersion: kuttl.dev/v1beta1
kind: TestStep
commands:
- script: |-
URI="linode/instances" FILTER="{\"tags\":\"$(OBJ=linodemachines/linodemachine-sample make getKubeUid)\"}" make callLinodeApiGet | grep 'results": 0'
11 changes: 11 additions & 0 deletions e2e/linodemachine-controller/byovpc/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
include ../../Makefile

createVPC:
@URI=vpcs FILTER='{"label":"capli-e2e-byovpc-sample"}' make --no-print-directory callLinodeApiGet | grep -v 'results": 0' ||\
URI=vpcs BODY='@01-vpc-create-options.json' make --no-print-directory callLinodeApiPost

enableVPC:
PATCH='{"status":{"ready":true}}' OBJ="linodevpcs/linodevpc-sample" make --no-print-directory patchKubeObjStatus

fetchVPCID:
@URI=vpcs FILTER='{"label":"capli-e2e-byovpc-sample"}' make --no-print-directory callLinodeApiGet | jq -r .data[0].id
17 changes: 0 additions & 17 deletions e2e/linodemachine-controller/minimal/01-assert.yaml

This file was deleted.

2 changes: 2 additions & 0 deletions e2e/linodemachine-controller/minimal/01-create-cluster.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ metadata:
annotations:
cluster.x-k8s.io/paused: "true"
name: linodecluster-sample
spec:
region: us-sea
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@ kind: TestStep
commands:
- script: |-
MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \
TC="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestCase
TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit