From b01b7ee79a0d38dcb942ede4564eda3d589068e3 Mon Sep 17 00:00:00 2001 From: Evan Date: Thu, 30 May 2024 16:18:37 -0400 Subject: [PATCH] [docs] clean up documentation based on latest clusterctl release (#347) * clean up documentation based on latest clusterctl release and standardize CCM installs for RKE2 and K3S --------- Co-authored-by: amold1 --- .github/filters.yml | 4 +- .github/workflows/build_test_ci.yml | 2 +- docs/src/developers/development.md | 7 +--- docs/src/developers/testing.md | 38 +++++++++++-------- docs/src/topics/getting-started.md | 12 ++---- .../k3s-capl-cluster/chainsaw-test.yaml | 2 +- .../kubeadm-capl-cluster/chainsaw-test.yaml | 2 +- .../rke2-capl-cluster/chainsaw-test.yaml | 3 +- e2e/gha-clusterctl-config.yaml | 8 +--- .../flavors/k3s/default/k3sControlPlane.yaml | 29 -------------- .../flavors/k3s/default/kustomization.yaml | 32 +++++++++++++++- .../flavors/k3s/vpcless/kustomization.yaml | 30 +-------------- .../flavors/rke2/default/kustomization.yaml | 31 +++++++++------ .../rke2/default/rke2ControlPlane.yaml | 30 --------------- templates/flavors/rke2/default/secret.yaml | 17 --------- .../flavors/rke2/vpcless/kustomization.yaml | 30 +-------------- 16 files changed, 89 insertions(+), 188 deletions(-) delete mode 100644 templates/flavors/rke2/default/secret.yaml diff --git a/.github/filters.yml b/.github/filters.yml index 1d246226d..326b5dede 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -1,9 +1,9 @@ src: - '!**/**.md' - - '!docs/**' kubeadm: - templates/flavors/kubeadm/default/* + - e2e/capl-cluster-flavors/kubeadm-capl-cluster/* kubeadm_cluster-autoscaler: - templates/flavors/kubeadm/cluster-autoscaler/* kubeadm_dual-stack: @@ -23,6 +23,7 @@ kubeadm_vpcless: k3s: - templates/flavors/k3s/default/* + - e2e/capl-cluster-flavors/k3s-capl-cluster/* k3s_cluster-autoscaler: - templates/flavors/k3s/cluster-autoscaler/* k3s_dual-stack: @@ -40,6 +41,7 @@ k3s_vpcless: rke2: - templates/flavors/rke2/default/* + - e2e/capl-cluster-flavors/rke2-capl-cluster/* rke2_cluster-autoscaler: - templates/flavors/rke2/cluster-autoscaler/* rke2_etcd-backup-restore: diff --git a/.github/workflows/build_test_ci.yml b/.github/workflows/build_test_ci.yml index 2b3383f54..8c1173fd0 100644 --- a/.github/workflows/build_test_ci.yml +++ b/.github/workflows/build_test_ci.yml @@ -37,7 +37,6 @@ jobs: - uses: dorny/paths-filter@v3 id: filter with: - predicate-quantifier: 'every' filters: .github/filters.yml yamllint: @@ -100,6 +99,7 @@ jobs: e2e-test: needs: changes strategy: + fail-fast: false matrix: flavor: ${{ fromJSON(needs.changes.outputs.paths) }} exclude: diff --git a/docs/src/developers/development.md b/docs/src/developers/development.md index 0f5e21d5d..ea5a53ffb 100644 --- a/docs/src/developers/development.md +++ b/docs/src/developers/development.md @@ -164,8 +164,7 @@ infrastructure-local-linode/v0.0.0 └── metadata.yaml ``` -This can then be used with `clusterctl` by adding the following to `~/.clusterctl/cluster-api.yaml` -(assuming the repo exists in the `$HOME` directory): +This can then be used with `clusterctl` by adding the following to `~/.cluster-api/clusterctl.yaml`: ``` providers: @@ -273,10 +272,6 @@ export LINODE_REGION=us-sea export LINODE_CONTROL_PLANE_MACHINE_TYPE=g6-standard-2 export LINODE_MACHINE_TYPE=g6-standard-2 -# IMPORTANT: Set linode, k3s, and rke2 providers in this config file. -# Find an example at e2e/gha-clusterctl-config.yaml -export CLUSTERCTL_CONFIG=~/.cluster-api/clusterctl.yaml - make e2etest ``` diff --git a/docs/src/developers/testing.md b/docs/src/developers/testing.md index d920e39bb..bc2d7a786 100644 --- a/docs/src/developers/testing.md +++ b/docs/src/developers/testing.md @@ -189,6 +189,16 @@ For e2e tests CAPL uses the [Chainsaw project](https://kyverno.github.io/chainsa spin up a cluster with the CAPL controllers installed and then uses `chainsaw-test.yaml` files to drive e2e testing. All test live in the e2e folder with a directory structure of `e2e/${COMPONENT}/${TEST_NAME}` + +### Environment Setup +The e2e tests use the `local-linode` infrastructure provider, this is registered by adding the following to `~/.cluster-api/clusterctl.yaml`: +``` +providers: + - name: local-linode + url: ${HOME}/cluster-api-provider-linode/infrastructure-local-linode/v0.0.0/infrastructure-components.yaml + type: InfrastructureProvider +``` + ### Running Tests In order to run e2e tests run the following commands: ```bash @@ -199,10 +209,6 @@ export LINODE_REGION=us-sea export LINODE_CONTROL_PLANE_MACHINE_TYPE=g6-standard-2 export LINODE_MACHINE_TYPE=g6-standard-2 -# IMPORTANT: Set linode, k3s, and rke2 providers in this config file. -# Find an example at e2e/gha-clusterctl-config.yaml -export CLUSTERCTL_CONFIG=~/.cluster-api/clusterctl.yaml - make e2etest ``` *Note: By default `make e2etest` runs all the e2e tests defined under `/e2e` dir* @@ -219,18 +225,18 @@ make e2etest E2E_SELECTOR='flavors' E2E_FLAGS='--assert-timeout 10m0s' There are other selectors you can use to invoke specfic tests. Please look at the table below for all the selectors available: -| Tests | Selector | -|----------------------------------|-------------------| -| All Tests | `all` | -| All Controllers | `quick` | -| All Flavors (default, k3s, rke2) | `flavors` | -| K3S Cluster | `k3s` | -| RKE2 Cluster | `rke2` | -| Default (kubeadm) Cluster | `default-cluster` | -| Linode Cluster Controller | `linodecluster` | -| Linode Machine Controller | `linodemachine` | -| Linode Obj Controller | `linodeobj` | -| Linode VPC Controller | `linodevpc` | +| Tests | Selector | +|----------------------------------|-----------------| +| All Tests | `all` | +| All Controllers | `quick` | +| All Flavors (default, k3s, rke2) | `flavors` | +| K3S Cluster | `k3s` | +| RKE2 Cluster | `rke2` | +| Default (kubeadm) Cluster | `kubeadm` | +| Linode Cluster Controller | `linodecluster` | +| Linode Machine Controller | `linodemachine` | +| Linode Obj Controller | `linodeobj` | +| Linode VPC Controller | `linodevpc` | *Note: For any flavor e2e tests, please set the required env variables* diff --git a/docs/src/topics/getting-started.md b/docs/src/topics/getting-started.md index 3106db3a8..e1c1be9da 100644 --- a/docs/src/topics/getting-started.md +++ b/docs/src/topics/getting-started.md @@ -36,16 +36,10 @@ to provision the node. If you are using a custom image ensure the [cloud_init](h By default, clusters are provisioned within VPC. For Regions which do not have [VPC support](https://www.linode.com/docs/products/networking/vpc/#availability) yet, use the [VPCLess](./flavors/vpcless.md) flavor to have clusters provisioned. ``` -## Register linode as an infrastructure provider -1. Add `linode` as an infrastructure provider in `~/.cluster-api/clusterctl.yaml` - ```yaml - providers: - - name: linode-linode - url: https://github.com/linode/cluster-api-provider-linode/releases/latest/infrastructure-components.yaml - type: InfrastructureProvider - ``` - ## Install CAPL on your management cluster +```admonish warning +The `linode-linode` infrastructure provider requires clusterctl version 1.7.2 or higher +``` Install CAPL and enable the helm addon provider which is used by the majority of the CAPL flavors ```bash clusterctl init --infrastructure linode-linode --addon helm diff --git a/e2e/capl-cluster-flavors/k3s-capl-cluster/chainsaw-test.yaml b/e2e/capl-cluster-flavors/k3s-capl-cluster/chainsaw-test.yaml index 34948793b..076f31c0f 100755 --- a/e2e/capl-cluster-flavors/k3s-capl-cluster/chainsaw-test.yaml +++ b/e2e/capl-cluster-flavors/k3s-capl-cluster/chainsaw-test.yaml @@ -39,7 +39,7 @@ spec: --flavor k3s --kubernetes-version v1.29.1+k3s2 \ --infrastructure local-linode:v0.0.0 \ --control-plane-machine-count 1 --worker-machine-count 1 \ - --config $CLUSTERCTL_CONFIG > k3s-cluster.yaml + --config ${CLUSTERCTL_CONFIG:=${HOME}/.cluster-api/clusterctl.yaml} > k3s-cluster.yaml check: ($error == null): true - name: Apply generated cluster yaml diff --git a/e2e/capl-cluster-flavors/kubeadm-capl-cluster/chainsaw-test.yaml b/e2e/capl-cluster-flavors/kubeadm-capl-cluster/chainsaw-test.yaml index 0e22ec1a4..e0788cd52 100755 --- a/e2e/capl-cluster-flavors/kubeadm-capl-cluster/chainsaw-test.yaml +++ b/e2e/capl-cluster-flavors/kubeadm-capl-cluster/chainsaw-test.yaml @@ -39,7 +39,7 @@ spec: --kubernetes-version v1.29.1 \ --infrastructure local-linode:v0.0.0 \ --control-plane-machine-count 1 --worker-machine-count 1 \ - --config $CLUSTERCTL_CONFIG > default-cluster.yaml + --config ${CLUSTERCTL_CONFIG:=${HOME}/.cluster-api/clusterctl.yaml} > default-cluster.yaml check: ($error == null): true - name: Apply generated cluster yaml diff --git a/e2e/capl-cluster-flavors/rke2-capl-cluster/chainsaw-test.yaml b/e2e/capl-cluster-flavors/rke2-capl-cluster/chainsaw-test.yaml index 5fa77f047..c5ba14592 100755 --- a/e2e/capl-cluster-flavors/rke2-capl-cluster/chainsaw-test.yaml +++ b/e2e/capl-cluster-flavors/rke2-capl-cluster/chainsaw-test.yaml @@ -2,7 +2,6 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - creationTimestamp: null name: rke2-capl-cluster # Labels to allow the test to be triggered based on selector flag labels: @@ -39,7 +38,7 @@ spec: --flavor rke2 --kubernetes-version v1.29.1+rke2r1 \ --infrastructure local-linode:v0.0.0 \ --control-plane-machine-count 1 --worker-machine-count 1 \ - --config $CLUSTERCTL_CONFIG> rke2-cluster.yaml + --config ${CLUSTERCTL_CONFIG:=${HOME}/.cluster-api/clusterctl.yaml} > rke2-cluster.yaml check: ($error == null): true - name: Apply generated cluster yaml diff --git a/e2e/gha-clusterctl-config.yaml b/e2e/gha-clusterctl-config.yaml index 460e1b6fb..c6d205d4f 100644 --- a/e2e/gha-clusterctl-config.yaml +++ b/e2e/gha-clusterctl-config.yaml @@ -1,6 +1,6 @@ providers: - name: local-linode - url: /home/runner/work/cluster-api-provider-linode/cluster-api-provider-linode/infrastructure-local-linode/v0.0.0/infrastructure-components.yaml + url: ${GITHUB_WORKSPACE}/infrastructure-local-linode/v0.0.0/infrastructure-components.yaml type: InfrastructureProvider - name: "k3s" url: https://github.com/k3s-io/cluster-api-k3s/releases/latest/bootstrap-components.yaml @@ -8,9 +8,3 @@ providers: - name: "k3s" url: https://github.com/k3s-io/cluster-api-k3s/releases/latest/control-plane-components.yaml type: "ControlPlaneProvider" - - name: "rke2" - url: "https://github.com/rancher-sandbox/cluster-api-provider-rke2/releases/latest/bootstrap-components.yaml" - type: "BootstrapProvider" - - name: "rke2" - url: "https://github.com/rancher-sandbox/cluster-api-provider-rke2/releases/latest/control-plane-components.yaml" - type: "ControlPlaneProvider" diff --git a/templates/flavors/k3s/default/k3sControlPlane.yaml b/templates/flavors/k3s/default/k3sControlPlane.yaml index 68eb67081..d4d82d171 100644 --- a/templates/flavors/k3s/default/k3sControlPlane.yaml +++ b/templates/flavors/k3s/default/k3sControlPlane.yaml @@ -15,29 +15,6 @@ spec: content: | flannel-backend: none disable-network-policy: true - - path: /var/lib/rancher/k3s/server/manifests/ccm-linode.yaml - owner: root:root - content: |- - apiVersion: helm.cattle.io/v1 - kind: HelmChart - metadata: - namespace: kube-system - name: ccm-linode - spec: - targetNamespace: kube-system - version: ${LINODE_CCM_VERSION:=v0.4.4} - chart: ccm-linode - repo: https://linode.github.io/linode-cloud-controller-manager/ - bootstrap: true - valuesContent: |- - routeController: - vpcName: ${VPC_NAME:=${CLUSTER_NAME}} - clusterCIDR: 10.0.0.0/8 - configureCloudRoutes: true - secretRef: - name: "linode-token-region" - nodeSelector: - node-role.kubernetes.io/control-plane: "true" - path: /var/lib/rancher/k3s/server/manifests/cilium.yaml owner: root:root content: |- @@ -86,12 +63,6 @@ spec: enabled: true ui: enabled: true - - path: /var/lib/rancher/k3s/server/manifests/linode-token-region.yaml - owner: root:root - contentFrom: - secret: - key: linode-token-region.yaml - name: linode-${CLUSTER_NAME}-crs-0 serverConfig: disableComponents: - servicelb diff --git a/templates/flavors/k3s/default/kustomization.yaml b/templates/flavors/k3s/default/kustomization.yaml index c1b734a18..e20d37f78 100644 --- a/templates/flavors/k3s/default/kustomization.yaml +++ b/templates/flavors/k3s/default/kustomization.yaml @@ -4,8 +4,10 @@ resources: - ../../../infra - k3sControlPlane.yaml - k3sConfigTemplate.yaml - - secret.yaml - ../../../addons/cilium-network-policies + - ../../../addons/csi-driver-linode + - ../../../addons/ccm-linode + - ../../../addons/cluster-resource-set patches: - target: group: cluster.x-k8s.io @@ -15,6 +17,19 @@ patches: - op: replace path: /spec/controlPlaneRef/kind value: KThreesControlPlane + - target: + group: cluster.x-k8s.io + version: v1beta1 + kind: Cluster + patch: |- + apiVersion: cluster.x-k8s.io/v1beta1 + kind: Cluster + metadata: + name: ${CLUSTER_NAME} + labels: + ccm: ${CLUSTER_NAME}-linode + csi: ${CLUSTER_NAME}-linode + crs: ${CLUSTER_NAME}-crs - target: group: cluster.x-k8s.io version: v1beta1 @@ -23,3 +38,18 @@ patches: - op: replace path: /spec/template/spec/bootstrap/configRef/kind value: KThreesConfigTemplate + - target: + kind: HelmChartProxy + name: .*-linode-cloud-controller-manager + patch: |- + - op: replace + path: /spec/valuesTemplate + value: | + routeController: + vpcName: ${VPC_NAME:=${CLUSTER_NAME}} + clusterCIDR: 10.0.0.0/8 + configureCloudRoutes: true + secretRef: + name: "linode-token-region" + nodeSelector: + node-role.kubernetes.io/control-plane: "true" \ No newline at end of file diff --git a/templates/flavors/k3s/vpcless/kustomization.yaml b/templates/flavors/k3s/vpcless/kustomization.yaml index 972dde10a..3e2da3d4b 100644 --- a/templates/flavors/k3s/vpcless/kustomization.yaml +++ b/templates/flavors/k3s/vpcless/kustomization.yaml @@ -33,6 +33,8 @@ patches: name: "linode-token-region" image: pullPolicy: IfNotPresent + nodeSelector: + node-role.kubernetes.io/control-plane: "true" - target: kind: LinodeVPC patch: |- @@ -76,34 +78,6 @@ patches: - sed -i '/swap/d' /etc/fstab - swapoff -a - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname - - op: replace - path: /spec/kthreesConfigSpec/files - value: - - path: /var/lib/rancher/k3s/server/manifests/ccm-linode.yaml - owner: root:root - content: |- - apiVersion: helm.cattle.io/v1 - kind: HelmChart - metadata: - namespace: kube-system - name: ccm-linode - spec: - targetNamespace: kube-system - version: ${LINODE_CCM_VERSION:=v0.4.4} - chart: ccm-linode - repo: https://linode.github.io/linode-cloud-controller-manager/ - bootstrap: true - valuesContent: |- - secretRef: - name: "linode-token-region" - nodeSelector: - node-role.kubernetes.io/control-plane: "true" - - contentFrom: - secret: - key: linode-token-region.yaml - name: linode-${CLUSTER_NAME}-crs-0 - owner: root:root - path: /var/lib/rancher/k3s/server/manifests/linode-token-region.yaml - target: group: cluster.x-k8s.io version: v1beta1 diff --git a/templates/flavors/rke2/default/kustomization.yaml b/templates/flavors/rke2/default/kustomization.yaml index 1f8448567..5881c1427 100644 --- a/templates/flavors/rke2/default/kustomization.yaml +++ b/templates/flavors/rke2/default/kustomization.yaml @@ -4,9 +4,11 @@ resources: - ../../../infra - rke2ControlPlane.yaml - rke2ConfigTemplate.yaml - - secret.yaml - ../../../addons/cilium - ../../../addons/cilium-network-policies + - ../../../addons/csi-driver-linode + - ../../../addons/ccm-linode + - ../../../addons/cluster-resource-set patches: - target: group: cluster.x-k8s.io @@ -26,7 +28,10 @@ patches: metadata: name: ${CLUSTER_NAME} labels: - cni: ${CLUSTER_NAME}-cilium + cni: ${CLUSTER_NAME}-cilium + ccm: ${CLUSTER_NAME}-linode + csi: ${CLUSTER_NAME}-linode + crs: ${CLUSTER_NAME}-crs - target: group: cluster.x-k8s.io version: v1beta1 @@ -36,13 +41,17 @@ patches: path: /spec/template/spec/bootstrap/configRef/kind value: RKE2ConfigTemplate - target: - group: cluster.x-k8s.io - version: v1beta1 - kind: Cluster + kind: HelmChartProxy + name: .*-linode-cloud-controller-manager patch: |- - apiVersion: cluster.x-k8s.io/v1beta1 - kind: Cluster - metadata: - name: ${CLUSTER_NAME} - labels: - cni: ${CLUSTER_NAME}-cilium + - op: replace + path: /spec/valuesTemplate + value: | + routeController: + vpcName: ${VPC_NAME:=${CLUSTER_NAME}} + clusterCIDR: 10.0.0.0/8 + configureCloudRoutes: true + secretRef: + name: "linode-token-region" + nodeSelector: + node-role.kubernetes.io/control-plane: "true" \ No newline at end of file diff --git a/templates/flavors/rke2/default/rke2ControlPlane.yaml b/templates/flavors/rke2/default/rke2ControlPlane.yaml index 506735cb3..7730885f0 100644 --- a/templates/flavors/rke2/default/rke2ControlPlane.yaml +++ b/templates/flavors/rke2/default/rke2ControlPlane.yaml @@ -8,36 +8,6 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: LinodeMachineTemplate name: ${CLUSTER_NAME}-control-plane - files: - - path: /var/lib/rancher/rke2/server/manifests/ccm-linode.yaml - owner: root:root - content: |- - apiVersion: helm.cattle.io/v1 - kind: HelmChart - metadata: - namespace: kube-system - name: ccm-linode - spec: - targetNamespace: kube-system - version: ${LINODE_CCM_VERSION:=v0.4.4} - chart: ccm-linode - repo: https://linode.github.io/linode-cloud-controller-manager/ - bootstrap: true - valuesContent: |- - routeController: - vpcName: ${VPC_NAME:=${CLUSTER_NAME}} - clusterCIDR: 10.0.0.0/8 - configureCloudRoutes: true - secretRef: - name: "linode-token-region" - nodeSelector: - node-role.kubernetes.io/control-plane: "true" - - contentFrom: - secret: - key: linode-token-region.yaml - name: linode-${CLUSTER_NAME}-crs-0 - owner: root:root - path: /var/lib/rancher/rke2/server/manifests/linode-token-region.yaml registrationMethod: internal-only-ips serverConfig: cni: none diff --git a/templates/flavors/rke2/default/secret.yaml b/templates/flavors/rke2/default/secret.yaml deleted file mode 100644 index 04b685e1f..000000000 --- a/templates/flavors/rke2/default/secret.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - clusterctl.cluster.x-k8s.io/move: "true" - name: linode-${CLUSTER_NAME}-crs-0 -stringData: - linode-token-region.yaml: |- - kind: Secret - apiVersion: v1 - metadata: - name: linode-token-region - namespace: kube-system - stringData: - apiToken: ${LINODE_TOKEN} - region: ${LINODE_REGION} diff --git a/templates/flavors/rke2/vpcless/kustomization.yaml b/templates/flavors/rke2/vpcless/kustomization.yaml index 03c2df75b..7f173d8e7 100644 --- a/templates/flavors/rke2/vpcless/kustomization.yaml +++ b/templates/flavors/rke2/vpcless/kustomization.yaml @@ -33,6 +33,8 @@ patches: name: "linode-token-region" image: pullPolicy: IfNotPresent + nodeSelector: + node-role.kubernetes.io/control-plane: "true" - target: kind: LinodeVPC patch: |- @@ -83,34 +85,6 @@ patches: - sed -i '/swap/d' /etc/fstab - swapoff -a - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname - - op: replace - path: /spec/files - value: - - path: /var/lib/rancher/rke2/server/manifests/ccm-linode.yaml - owner: root:root - content: |- - apiVersion: helm.cattle.io/v1 - kind: HelmChart - metadata: - namespace: kube-system - name: ccm-linode - spec: - targetNamespace: kube-system - version: ${LINODE_CCM_VERSION:=v0.4.4} - chart: ccm-linode - repo: https://linode.github.io/linode-cloud-controller-manager/ - bootstrap: true - valuesContent: |- - secretRef: - name: "linode-token-region" - nodeSelector: - node-role.kubernetes.io/control-plane: "true" - - contentFrom: - secret: - key: linode-token-region.yaml - name: linode-${CLUSTER_NAME}-crs-0 - owner: root:root - path: /var/lib/rancher/rke2/server/manifests/linode-token-region.yaml - target: group: cluster.x-k8s.io version: v1beta1