From e1fccfc033554f9830fd2b1fd351824ce4f2a6cc Mon Sep 17 00:00:00 2001 From: Richard Kovacs Date: Mon, 5 Feb 2024 17:42:41 +0100 Subject: [PATCH] E2E framework (#77) * Introduce E2E framework * Use Tilt for provisioning on CI * Allow oud.tilt.dev for e2e test GA * Allow storage.googleapis.com for build-test GA --------- Co-authored-by: Richard Kovacs --- .github/workflows/build_test_ci.yml | 61 ++++++++++++++++++- .gitignore | 1 + .../ctlptl-config.yaml | 1 + Makefile | 19 +++++- README.md | 28 +++++++++ Tiltfile | 6 +- config/manager/kustomization.yaml | 6 ++ .../infrastructure_v1alpha1_linodevpc.yaml | 12 ++++ config/samples/kustomization.yaml | 1 + e2e/basic/cluster/00-assert.yaml | 0 e2e/basic/machine/00-assert.yaml | 0 e2e/basic/vpc/00-assert.yaml | 0 e2e/kind-config.yaml | 11 ++++ e2e/kuttl-config.yaml | 6 ++ 14 files changed, 146 insertions(+), 6 deletions(-) rename ctlptl-config.yaml => .tilt/ctlptl-config.yaml (86%) create mode 100644 config/samples/infrastructure_v1alpha1_linodevpc.yaml create mode 100644 e2e/basic/cluster/00-assert.yaml create mode 100644 e2e/basic/machine/00-assert.yaml create mode 100644 e2e/basic/vpc/00-assert.yaml create mode 100644 e2e/kind-config.yaml create mode 100644 e2e/kuttl-config.yaml diff --git a/.github/workflows/build_test_ci.yml b/.github/workflows/build_test_ci.yml index 3a108b61b..6343cba26 100644 --- a/.github/workflows/build_test_ci.yml +++ b/.github/workflows/build_test_ci.yml @@ -30,6 +30,7 @@ jobs: proxy.golang.org:443 sum.golang.org:443 objects.githubusercontent.com:443 + storage.googleapis.com:443 - uses: actions/checkout@v4 @@ -89,6 +90,64 @@ jobs: - name: Nilcheck run: make nilcheck + e2e-test: + needs: [go-build-test, docker-build] + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + gcr.io:443 + proxy.golang.org:443 + sum.golang.org:443 + *.githubusercontent.com:443 + docker.io:443 + registry-1.docker.io:443 + auth.docker.io:443 + production.cloudflare.docker.com:443 + storage.googleapis.com:443 + registry.k8s.io:443 + *.pkg.dev:443 + *.amazonaws.com:443 + *.blob.core.windows.net:443 + quay.io:443 + *.quay.io:443 + api.snapcraft.io:443 + cloud.tilt.dev:443 + + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 'stable' + + - name: Docker cache + uses: ScribeMD/docker-cache@0.3.7 + with: + key: docker-${{ runner.os }}-${{ hashFiles('Makefile') }}} + + - name: E2E test + run: make e2etest + env: + GITHUB_TOKEN: ${{ secrets.github_token }} + LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }} + + - name: Copy logs + if: ${{ always() }} + run: docker cp tilt-control-plane:/var/log .logs + + - uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: logs + path: .logs/* + docker-build: runs-on: ubuntu-latest steps: @@ -114,7 +173,7 @@ jobs: - name: Docker cache uses: ScribeMD/docker-cache@0.3.7 with: - key: docker-${{ runner.os }}-${{ hashFiles('Dockerfile') }} + key: docker-${{ runner.os }}-${{ hashFiles('Makefile') }} - name: Build the Docker image run: make docker-build diff --git a/.gitignore b/.gitignore index dbc337bf9..938e4bfb2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store .idea bin/* +kind-logs-* cover.out kubeconfig diff --git a/ctlptl-config.yaml b/.tilt/ctlptl-config.yaml similarity index 86% rename from ctlptl-config.yaml rename to .tilt/ctlptl-config.yaml index bef997e59..75b0354ed 100644 --- a/ctlptl-config.yaml +++ b/.tilt/ctlptl-config.yaml @@ -2,6 +2,7 @@ apiVersion: ctlptl.dev/v1alpha1 kind: Cluster product: kind +registry: ctlptl-registry kindV1Alpha4Cluster: name: tilt nodes: diff --git a/Makefile b/Makefile index ac31bbd83..702e969b5 100644 --- a/Makefile +++ b/Makefile @@ -88,6 +88,13 @@ vulncheck: govulncheck ## Run vulnerability check against code. test: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -race -timeout 60s ./... -coverprofile cover.out +.PHONY: e2etest +e2etest: kind ctlptl tilt kuttl kustomize clusterctl manifests generate + @echo -n "LINODE_TOKEN=$(LINODE_TOKEN)" > config/default/.env.linode + $(CTLPTL) apply -f .tilt/ctlptl-config.yaml + $(TILT) ci --timeout 180s -f Tiltfile + $(KUTTL) test --config e2e/kuttl-config.yaml + ##@ Build .PHONY: build @@ -152,8 +159,8 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi .PHONY: tilt-cluster tilt-cluster: ctlptl tilt kind clusterctl @echo -n "LINODE_TOKEN=$(LINODE_TOKEN)" > config/default/.env.linode - $(CTLPTL) apply -f ctlptl-config.yaml - $(TILT) up + $(CTLPTL) apply -f .tilt/ctlptl-config.yaml + $(TILT) up --stream ##@ Build Dependencies @@ -171,6 +178,7 @@ CLUSTERCTL ?= $(LOCALBIN)/clusterctl CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen TILT ?= $(LOCALBIN)/tilt KIND ?= $(LOCALBIN)/kind +KUTTL ?= $(LOCALBIN)/kuttl ENVTEST ?= $(LOCALBIN)/setup-envtest HUSKY ?= $(LOCALBIN)/husky NILAWAY ?= $(LOCALBIN)/nilaway @@ -183,6 +191,7 @@ CLUSTERCTL_VERSION ?= v1.5.3 CONTROLLER_TOOLS_VERSION ?= v0.13.0 TILT_VERSION ?= 0.33.6 KIND_VERSION ?= 0.20.0 +KUTTL_VERSION ?= 0.15.0 HUSKY_VERSION ?= v0.2.16 NILAWAY_VERSION ?= latest GOVULNC_VERSION ?= v1.0.1 @@ -231,6 +240,12 @@ $(KIND): $(LOCALBIN) test -s $(KIND) && $(KIND) version | grep -q $(KIND_VERSION) || \ (cd $(LOCALBIN); curl -Lso ./kind https://github.com/kubernetes-sigs/kind/releases/download/v$(KIND_VERSION)/kind-$(OS)-$(ARCH_SHORT) && chmod +x kind) +.PHONY: kuttl +kuttl: $(KUTTL) ## Download kuttl locally if necessary. If wrong version is installed, it will be overwritten. +$(KUTTL): $(LOCALBIN) + test -s $(KUTTL) && $(KUTTL) version | grep -q $(KUTTL_VERSION) || \ + (cd $(LOCALBIN); curl -Lso ./kuttl https://github.com/kudobuilder/kuttl/releases/download/v$(KUTTL_VERSION)/kubectl-kuttl_$(KUTTL_VERSION)_$(OS)_$(ARCH) && chmod +x kuttl) + .PHONY: envtest envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) diff --git a/README.md b/README.md index c976f3173..480a2d5cb 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,22 @@ # Cluster API Provider Linode A [Cluster API](https://cluster-api.sigs.k8s.io/) implementation for the [Linode](https://www.linode.com/) to create kubernetes clusters. +## Local development + +### Enable git hooks + +To enable automatic code validation on code push, execute the following commands: + +```bash +PATH="$PWD/bin:$PATH" make husky && husky install +``` + +If you temporary would like to disable git hook, set `SKIP_GIT_PUSH_HOOK` value: + +```bash +SKIP_GIT_PUSH_HOOK=1 git push +``` + ### Local development with Tilt For local development execute the following `make` target: @@ -19,3 +35,15 @@ LINODE_TOKEN= make tilt-cluster ``` This command creates a Kind cluster, and deploys resources via Tilt. You can freely change the code and wait for Tilt to update provider. + +### E2E testing + +For local development execute the following `make` target: + +```bash +LINODE_TOKEN= make e2etest +``` + +This command creates a Kind cluster, and executes all the defined tests. + +> Please ensure you have increased maximum open files on your host: https://kind.sigs.k8s.io/docs/user/known-issues/#pod-errors-due-to-too-many-open-files diff --git a/Tiltfile b/Tiltfile index 4cec7014f..d67d27d0a 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,8 +1,8 @@ -docker_build("controller", ".", only=("vendor","go.mod", "go.sum", "./api", "./cloud","./cmd", "./controller", "./util")) +docker_build("controller", ".", only=("Dockerfile", "Makefile", "vendor","go.mod", "go.sum", "./api", "./cloud","./cmd", "./controller", "./util")) local_resource( - 'capi-controller-manager', - cmd='clusterctl init', + 'capi-controller-manager', + cmd='clusterctl init', ) k8s_yaml(kustomize('config/default')) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b84c..ad13e96b3 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: controller + newTag: latest diff --git a/config/samples/infrastructure_v1alpha1_linodevpc.yaml b/config/samples/infrastructure_v1alpha1_linodevpc.yaml new file mode 100644 index 000000000..34ed757c4 --- /dev/null +++ b/config/samples/infrastructure_v1alpha1_linodevpc.yaml @@ -0,0 +1,12 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeVPC +metadata: + labels: + app.kubernetes.io/name: linodevpc + app.kubernetes.io/instance: linodevpc-sample + app.kubernetes.io/part-of: cluster-api-provider-linode + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: cluster-api-provider-linode + name: linodevpc-sample +spec: + region: us-east diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index db7d2f6fa..69a2f69e0 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -4,4 +4,5 @@ resources: - infrastructure_v1alpha1_linodemachine.yaml - infrastructure_v1alpha1_linodemachinetemplate.yaml - infrastructure_v1alpha1_linodeclustertemplate.yaml +- infrastructure_v1alpha1_linodevpc.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/e2e/basic/cluster/00-assert.yaml b/e2e/basic/cluster/00-assert.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/e2e/basic/machine/00-assert.yaml b/e2e/basic/machine/00-assert.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/e2e/basic/vpc/00-assert.yaml b/e2e/basic/vpc/00-assert.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/e2e/kind-config.yaml b/e2e/kind-config.yaml new file mode 100644 index 000000000..93bbb102a --- /dev/null +++ b/e2e/kind-config.yaml @@ -0,0 +1,11 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: kuttl +nodes: +- role: control-plane + image: kindest/node:v1.28.0 + extraPortMappings: + - containerPort: 8080 + hostPort: 28080 + - containerPort: 8081 + hostPort: 28081 diff --git a/e2e/kuttl-config.yaml b/e2e/kuttl-config.yaml new file mode 100644 index 000000000..b06687408 --- /dev/null +++ b/e2e/kuttl-config.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +testDirs: +- e2e/basic +kindConfig: e2e/kind-config.yaml +timeout: 120