From 8da4cc822c9c2fde41773c9b4cf6d4cc56f149ff Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 3 May 2024 12:10:53 -0400 Subject: [PATCH] split out e2e tests into its own gha file (#301) * split out e2e tests into its own gha file --- .github/workflows/build_test_ci.yml | 126 +++++------------- .github/workflows/e2e-test.yaml | 119 +++++++++++++++++ Makefile | 5 +- docs/src/developers/testing.md | 29 ++-- .../default-CAPL-cluster/chainsaw-test.yaml | 1 + .../k3s-capl-cluster/chainsaw-test.yaml | 1 + .../rke2-capl-cluster/chainsaw-test.yaml | 1 + .../minimal-linodecluster/chainsaw-test.yaml | 1 + .../minimal-linodemachine/chainsaw-test.yaml | 1 + .../vpc-integration/chainsaw-test.yaml | 1 + .../chainsaw-test.yaml | 1 + .../minimal-linodevpc/chainsaw-test.yaml | 1 + 12 files changed, 183 insertions(+), 104 deletions(-) create mode 100644 .github/workflows/e2e-test.yaml diff --git a/.github/workflows/build_test_ci.yml b/.github/workflows/build_test_ci.yml index 51aadb088..12b9b733d 100644 --- a/.github/workflows/build_test_ci.yml +++ b/.github/workflows/build_test_ci.yml @@ -4,29 +4,49 @@ on: push: branches: - main - paths-ignore: - - '**/**.md' - - 'docs/**' pull_request: branches: - "*" - paths-ignore: - - '**/**.md' - - 'docs/**' workflow_dispatch: - + permissions: contents: read pull-requests: read actions: read concurrency: - group: build-test-ci-${{ github.ref }}-1 + group: build-test-ci-${{ github.ref }} cancel-in-progress: true jobs: + changes: + runs-on: ubuntu-latest + outputs: + # Expose matched filters as job 'src' output variable + src: ${{ steps.filter.outputs.src }} + steps: + - uses: actions/checkout@v4 + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + - uses: dorny/paths-filter@v3 + id: filter + with: + predicate-quantifier: 'every' + filters: | + src: + - '!**/**.md' + - '!docs/**' + go-build-test: runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.src == 'true' }} steps: - name: Harden Runner uses: step-security/harden-runner@v2 @@ -71,90 +91,18 @@ jobs: slug: linode/cluster-api-provider-linode e2e-test: - needs: [go-build-test, docker-build] - runs-on: ubuntu-latest - if: github.event.pull_request.draft == false - env: - GITHUB_TOKEN: ${{ secrets.github_token }} - LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }} - steps: - - name: Harden Runner - uses: step-security/harden-runner@v2 - with: - disable-sudo: true - egress-policy: block - allowed-endpoints: > - api.linode.com:443 - api.github.com:443 - github.com:443 - gcr.io:443 - ghcr.io:443 - proxy.golang.org:443 - sum.golang.org:443 - *.githubusercontent.com:443 - docker.io:443 - registry-1.docker.io:443 - auth.docker.io:443 - production.cloudflare.docker.com:443 - storage.googleapis.com:443 - registry.k8s.io:443 - *.pkg.dev:443 - *.amazonaws.com:443 - *.blob.core.windows.net:443 - quay.io:443 - *.quay.io:443 - api.snapcraft.io:443 - cloud.tilt.dev:443 - kubernetes-sigs.github.io:443 - charts.jetstack.io:443 - helm.cilium.io:443 - linode.github.io:443 - - - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - check-latest: true - - - name: Docker cache - uses: ScribeMD/docker-cache@0.5.0 - with: - key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }} - - - name: Complete E2E Test - if: github.ref == 'refs/heads/main' - run: make e2etest - env: - E2E_FLAGS: '--assert-timeout 15m0s' - INSTALL_K3S_PROVIDER: true - INSTALL_RKE2_PROVIDER: true - LINODE_REGION: us-sea - LINODE_CONTROL_PLANE_MACHINE_TYPE: g6-standard-2 - LINODE_MACHINE_TYPE: g6-standard-2 - CLUSTERCTL_CONFIG: /home/runner/work/cluster-api-provider-linode/cluster-api-provider-linode/e2e/gha-clusterctl-config.yaml - - - - name: Quick E2E Test - if: github.ref != 'refs/heads/main' - run: make e2etest - env: - E2E_FLAGS: '--selector quick' - - - name: Copy logs - if: ${{ always() }} - run: docker cp tilt-control-plane:/var/log .logs - - - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: logs - path: .logs/* - overwrite: true + needs: changes + if: ${{ github.event.pull_request.draft == false && needs.changes.outputs.src == 'true' }} + uses: ./.github/workflows/e2e-test.yaml + secrets: inherit + with: + e2e-selector: ${{ github.ref == 'refs/heads/main' && 'all' || 'quick' }} + e2e-flags: ${{ github.ref == 'refs/heads/main' && '--assert-timeout 15m0s' || '' }} docker-build: runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.src == 'true' }} steps: - name: Harden Runner uses: step-security/harden-runner@v2 diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml new file mode 100644 index 000000000..6f09c6be5 --- /dev/null +++ b/.github/workflows/e2e-test.yaml @@ -0,0 +1,119 @@ +name: Run e2e tests + +on: + workflow_call: + inputs: + e2e-selector: + default: quick + description: "Selector for which tests to run, defaults to 'quick'" + required: true + type: string + e2e-flags: + type: string + description: "Flags to pass to chainsaw when running e2e tests" + workflow_dispatch: + inputs: + e2e-selector: + default: quick + description: "Selector for which tests to run, defaults to 'quick'" + type: choice + required: true + options: + - quick + - flavors + - k3s + - rke2 + - default-cluster + - linodecluster + - linodemachine + - linodeobj + - linodevpc + - all + e2e-flags: + type: string + description: "Flags to pass to chainsaw when running e2e tests" + +permissions: + contents: read + pull-requests: read + actions: read + +concurrency: + group: e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + e2e-test: + name: ${{ format('{0}-e2e-tests', inputs.e2e-selector) }} + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.github_token }} + LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }} + steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.linode.com:443 + api.github.com:443 + github.com:443 + gcr.io:443 + ghcr.io:443 + proxy.golang.org:443 + sum.golang.org:443 + *.githubusercontent.com:443 + docker.io:443 + registry-1.docker.io:443 + auth.docker.io:443 + production.cloudflare.docker.com:443 + storage.googleapis.com:443 + registry.k8s.io:443 + *.pkg.dev:443 + *.amazonaws.com:443 + *.blob.core.windows.net:443 + quay.io:443 + *.quay.io:443 + api.snapcraft.io:443 + cloud.tilt.dev:443 + kubernetes-sigs.github.io:443 + charts.jetstack.io:443 + helm.cilium.io:443 + linode.github.io:443 + + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + check-latest: true + + - name: Docker cache + uses: ScribeMD/docker-cache@0.5.0 + with: + key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }} + + - name: Run E2E Test + env: + E2E_FLAGS: ${{ inputs.e2e-flags }} + E2E_SELECTOR: ${{ inputs.e2e-selector }} + INSTALL_K3S_PROVIDER: true + INSTALL_RKE2_PROVIDER: true + LINODE_REGION: us-sea + LINODE_CONTROL_PLANE_MACHINE_TYPE: g6-standard-2 + LINODE_MACHINE_TYPE: g6-standard-2 + CLUSTERCTL_CONFIG: /home/runner/work/cluster-api-provider-linode/cluster-api-provider-linode/e2e/gha-clusterctl-config.yaml + run: make e2etest + + - name: Copy logs + if: ${{ always() }} + run: docker cp tilt-control-plane:/var/log .logs + + - uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: logs + path: .logs/* + overwrite: true diff --git a/Makefile b/Makefile index a1985c22c..40af9dfde 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,7 @@ SHELL = /usr/bin/env bash -o pipefail CONTAINER_TOOL ?= docker MDBOOK_DEV_HOST = 0.0.0.0 MDBOOK_DEV_PORT = 3000 +E2E_SELECTOR ?= all # ENVTEST_K8S_VERSION # - refers to the version of kubebuilder assets to be downloaded by envtest binary. @@ -119,7 +120,7 @@ gosec: ## Run gosec against code. .PHONY: lint lint: ## Run lint against code. - docker run --rm -w /workdir -v $(PWD):/workdir golangci/golangci-lint:v1.57.2 golangci-lint run -c .golangci.yml + docker run --rm -w /workdir -v $(PWD):/workdir golangci/golangci-lint:v1.57.2 golangci-lint run -c .golangci.yml --fix .PHONY: nilcheck nilcheck: nilaway ## Run nil check against code. @@ -147,7 +148,7 @@ test: generate fmt vet envtest ## Run tests. .PHONY: e2etest e2etest: generate local-release local-deploy chainsaw - GIT_REF=$(GIT_REF) $(CHAINSAW) test ./e2e $(E2E_FLAGS) + GIT_REF=$(GIT_REF) $(CHAINSAW) test ./e2e --selector $(E2E_SELECTOR) $(E2E_FLAGS) local-deploy: kind ctlptl tilt kustomize clusterctl @echo -n "LINODE_TOKEN=$(LINODE_TOKEN)" > config/default/.env.linode diff --git a/docs/src/developers/testing.md b/docs/src/developers/testing.md index 24dc6307d..d920e39bb 100644 --- a/docs/src/developers/testing.md +++ b/docs/src/developers/testing.md @@ -207,27 +207,30 @@ make e2etest ``` *Note: By default `make e2etest` runs all the e2e tests defined under `/e2e` dir* -In order to run specific test, you need to pass flags to chainsaw by setting env var `E2E_FLAGS` +In order to run specific test, you need to pass flags to chainsaw by setting env var `E2E_SELECTOR` + +Additional settings can be passed to chainsaw by setting env var `E2E_FLAGS` Example: Only running e2e tests for flavors *(default, k3s, rke2)* ```bash -make e2etest E2E_FLAGS='--selector flavors --assert-timeout 10m0s' +make e2etest E2E_SELECTOR='flavors' E2E_FLAGS='--assert-timeout 10m0s' ``` *Note: We need to bump up the assert timeout to 10 mins to allow the cluster to complete building and become available* There are other selectors you can use to invoke specfic tests. Please look at the table below for all the selectors available: -| Tests | Selector | -| ------------- | ------------- | -| All Controllers | `quick` | -| All Flavors (default, k3s, rke2) | `flavors` | -| K3S Cluster | `k3s` | -| RKE2 Cluster | `rke2` | -| Default (kubeadm) Cluster | `default-cluster` | -| Linode Cluster Controller | `linodecluster` | -| Linode Machine Controller | `linodemachine` | -| Linode Obj Controller | `linodeobj` | -| Linode VPC Controller | `linodevpc` | +| Tests | Selector | +|----------------------------------|-------------------| +| All Tests | `all` | +| All Controllers | `quick` | +| All Flavors (default, k3s, rke2) | `flavors` | +| K3S Cluster | `k3s` | +| RKE2 Cluster | `rke2` | +| Default (kubeadm) Cluster | `default-cluster` | +| Linode Cluster Controller | `linodecluster` | +| Linode Machine Controller | `linodemachine` | +| Linode Obj Controller | `linodeobj` | +| Linode VPC Controller | `linodevpc` | *Note: For any flavor e2e tests, please set the required env variables* diff --git a/e2e/capl-cluster-flavors/default-CAPL-cluster/chainsaw-test.yaml b/e2e/capl-cluster-flavors/default-CAPL-cluster/chainsaw-test.yaml index 1b1f62acb..6281038d3 100755 --- a/e2e/capl-cluster-flavors/default-CAPL-cluster/chainsaw-test.yaml +++ b/e2e/capl-cluster-flavors/default-CAPL-cluster/chainsaw-test.yaml @@ -6,6 +6,7 @@ metadata: name: default-capl-cluster # Labels to allow the test to be triggered based on selector flag labels: + all: default-cluster: flavors: spec: diff --git a/e2e/capl-cluster-flavors/k3s-capl-cluster/chainsaw-test.yaml b/e2e/capl-cluster-flavors/k3s-capl-cluster/chainsaw-test.yaml index 3d36dc716..b709d7fef 100755 --- a/e2e/capl-cluster-flavors/k3s-capl-cluster/chainsaw-test.yaml +++ b/e2e/capl-cluster-flavors/k3s-capl-cluster/chainsaw-test.yaml @@ -6,6 +6,7 @@ metadata: name: k3s-capl-cluster # Labels to allow the test to be triggered based on selector flag labels: + all: k3s: flavors: spec: diff --git a/e2e/capl-cluster-flavors/rke2-capl-cluster/chainsaw-test.yaml b/e2e/capl-cluster-flavors/rke2-capl-cluster/chainsaw-test.yaml index a2bdb5a64..a9bc8ed5d 100755 --- a/e2e/capl-cluster-flavors/rke2-capl-cluster/chainsaw-test.yaml +++ b/e2e/capl-cluster-flavors/rke2-capl-cluster/chainsaw-test.yaml @@ -6,6 +6,7 @@ metadata: name: rke2-capl-cluster # Labels to allow the test to be triggered based on selector flag labels: + all: rke2: flavors: spec: diff --git a/e2e/linodecluster-controller/minimal-linodecluster/chainsaw-test.yaml b/e2e/linodecluster-controller/minimal-linodecluster/chainsaw-test.yaml index a00c8e996..1527c83a6 100755 --- a/e2e/linodecluster-controller/minimal-linodecluster/chainsaw-test.yaml +++ b/e2e/linodecluster-controller/minimal-linodecluster/chainsaw-test.yaml @@ -6,6 +6,7 @@ metadata: name: minimal-linodecluster # Label to trigger the test on every PR labels: + all: quick: linodecluster: spec: diff --git a/e2e/linodemachine-controller/minimal-linodemachine/chainsaw-test.yaml b/e2e/linodemachine-controller/minimal-linodemachine/chainsaw-test.yaml index 74a63c18b..d3acd7226 100755 --- a/e2e/linodemachine-controller/minimal-linodemachine/chainsaw-test.yaml +++ b/e2e/linodemachine-controller/minimal-linodemachine/chainsaw-test.yaml @@ -6,6 +6,7 @@ metadata: name: minimal-linodemachine # Label to trigger the test on every PR labels: + all: quick: linodemachine: spec: diff --git a/e2e/linodemachine-controller/vpc-integration/chainsaw-test.yaml b/e2e/linodemachine-controller/vpc-integration/chainsaw-test.yaml index dd61e84f8..68e9a982c 100755 --- a/e2e/linodemachine-controller/vpc-integration/chainsaw-test.yaml +++ b/e2e/linodemachine-controller/vpc-integration/chainsaw-test.yaml @@ -6,6 +6,7 @@ metadata: name: vpc-integration # Label to trigger the test on every PR labels: + all: quick: linodemachine: linodevpc: diff --git a/e2e/linodeobjectstoragebucket-controller/minimal-linodeobjectstoragebucket/chainsaw-test.yaml b/e2e/linodeobjectstoragebucket-controller/minimal-linodeobjectstoragebucket/chainsaw-test.yaml index e5508d089..b5499c1d2 100755 --- a/e2e/linodeobjectstoragebucket-controller/minimal-linodeobjectstoragebucket/chainsaw-test.yaml +++ b/e2e/linodeobjectstoragebucket-controller/minimal-linodeobjectstoragebucket/chainsaw-test.yaml @@ -5,6 +5,7 @@ metadata: name: minimal-linodeobjectstoragebucket # Label to trigger the test on every PR labels: + all: quick: linodeobj: spec: diff --git a/e2e/linodevpc-controller/minimal-linodevpc/chainsaw-test.yaml b/e2e/linodevpc-controller/minimal-linodevpc/chainsaw-test.yaml index 7fd41d240..951f23eb4 100644 --- a/e2e/linodevpc-controller/minimal-linodevpc/chainsaw-test.yaml +++ b/e2e/linodevpc-controller/minimal-linodevpc/chainsaw-test.yaml @@ -5,6 +5,7 @@ metadata: name: minimal-linodevpc # Label to trigger the test on every PR labels: + all: quick: linodevpc: spec: