From a2c798aa8eda23f45d5a84273d3ad3218c2d3430 Mon Sep 17 00:00:00 2001 From: Richard Case Date: Thu, 15 Feb 2024 10:48:34 +0000 Subject: [PATCH] docs: updates to README Some updates to the readme file to use docker in the getting started guide. Signed-off-by: Richard Case Signed-off-by: nasusoba add samples and developer guide Signed-off-by: nasusoba tiny fix --- README.md | 200 +++++++++++++++--- samples/azure/azure-ccm.yaml | 187 ---------------- samples/azure/azure-cn.yaml | 85 -------- samples/azure/azure-setup.sh | 35 +-- samples/azure/k3s-template.yaml | 134 ++++++------ samples/azure/resource-set.yaml | 28 --- .../v1.2.2/bootstrap-components.yaml | 1 + .../v1.2.2/control-plane-components.yaml | 1 + .../docker/cluster-template-quickstart.yaml | 104 +++++++++ .../cluster-template-topology.yaml | 32 +++ .../docker/clusterclass/clusterclass-k3s.yaml | 113 ++++++++++ 11 files changed, 514 insertions(+), 406 deletions(-) delete mode 100644 samples/azure/azure-ccm.yaml delete mode 100644 samples/azure/azure-cn.yaml delete mode 100644 samples/azure/resource-set.yaml create mode 100644 samples/docker/cluster-template-quickstart.yaml create mode 100644 samples/docker/clusterclass/cluster-template-topology.yaml create mode 100644 samples/docker/clusterclass/clusterclass-k3s.yaml diff --git a/README.md b/README.md index 59cc6e6b..41bea68d 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,189 @@ -# Cluster API k3s +# Cluster API Provider k3s -Cluster API bootstrap provider k3s (CABP3) is a component of [Cluster API](https://github.com/kubernetes-sigs/cluster-api/blob/master/README.md) that is responsible for generating a cloud-init script to turn a Machine into a Kubernetes Node; this implementation brings up [k3s](https://k3s.io/) clusters instead of full kubernetes clusters. +Cluster API Provider k3s provides the following [Cluster API](https://github.com/kubernetes-sigs/cluster-api) (CAPI) providers: -CABP3 is the bootstrap component of Cluster API for k3s and brings in the following CRDS and controllers: -- k3s bootstrap provider (KThrees, KThreesTemplate) +- **Cluster API Bootstrap Provider k3s (CABP3)** is responsible for generating the instructions (and encoding them as cloud-init) to turn a Machine into a Kubernetes Node; this implementation brings up [k3s](https://k3s.io/) clusters instead of full kubernetes clusters. +- **Cluster API ControlPlane Provider k3s (CACP3)** is responsible for managing the lifecycle of control plane machines for k3s; this implementation brings up [k3s](https://k3s.io/) clusters instead of full kubernetes clusters. -Cluster API ControlPlane provider k3s (CACP3) is a component of [Cluster API](https://github.com/kubernetes-sigs/cluster-api/blob/master/README.md) that is responsible for managing the lifecycle of control plane machines for k3s; this implementation brings up [k3s](https://k3s.io/) clusters instead of full kubernetes clusters. +## Getting Started -CACP3 is the controlplane component of Cluster API for k3s and brings in the following CRDS and controllers: -- k3s controlplane provider (KThreesControlPlane) +**Warning**: Project and documentation are in an early stage, there is an assumption that a user of this provider is already familiar with Cluster API. Please consider contributing. -Together these two components make up Cluster API k3s... +### Prerequisites -## Testing it out. +Check out the general [Cluster API Quickstart](https://cluster-api.sigs.k8s.io/user/quick-start.html) page to see the prerequisites for Cluster API. -**Warning**: Project and documentation are in an early stage, there is an assumption that an user of this provider is already familiar with ClusterAPI. +Three main pieces are: -### Prerequisites +1. Management cluster. In the `samples/azure/azure-setup.sh` script, [k3d](https://k3d.io/) is used, but feel free to use [kind](https://kind.sigs.k8s.io/) as well . +2. clusterctl. Please check out [Cluster API Quickstart](https://cluster-api.sigs.k8s.io/user/quick-start.html) for instructions. +3. Infrastructure specific prerequisites: + - For more Azure information go to [CAPZ Getting Started](https://capz.sigs.k8s.io/topics/getting-started.html) + - For more AWS information go to [CAPA Getting Started](https://cluster-api-aws.sigs.k8s.io/) + - For more Nutanix information go to [CAPX Getting Started](https://opendocs.nutanix.com/capx/latest/getting_started/) + - For more OpenStack information go to [CAPO Getting Started](https://cluster-api.sigs.k8s.io/user/quick-start.html) + - For more Vsphere information go to [CAPV Getting Started](https://cluster-api.sigs.k8s.io/user/quick-start.html) -Check out the [ClusterAPI Quickstart](https://cluster-api.sigs.k8s.io/user/quick-start.html) page to see the prerequisites for ClusterAPI. +In this getting started guide we'll be using Docker as the infrastructure provider (CAPD). -Three main pieces are +### Create a management cluster -1. Bootstrap cluster. In the `samples/azure/azure-setup.sh` script, I use [k3d](https://k3d.io/), but feel free to use [kind](https://kind.sigs.k8s.io/) as well. -2. clusterctl. Please check out [ClusterAPI Quickstart](https://cluster-api.sigs.k8s.io/user/quick-start.html) for instructions. -3. Infrastructure Specific Prerequisites: +1. Ensure kind is installed ([instructions](https://kind.sigs.k8s.io/docs/user/quick-start/#installation)) +2. Create a kind configuration to expose the local docker socket: - * For more Azure information go to [CAPZ Getting Started](https://capz.sigs.k8s.io/topics/getting-started.html) - * For more AWS information go to [CAPA Getting Started](https://cluster-api-aws.sigs.k8s.io/) - * For more Nutanix information go to [CAPX Getting Started](https://opendocs.nutanix.com/capx/latest/getting_started/) - * For more OpenStack information go to [CAPO Getting Started](https://cluster-api.sigs.k8s.io/user/quick-start.html) - * For more Vsphere information go to [CAPV Getting Started](https://cluster-api.sigs.k8s.io/user/quick-start.html) +```bash +cat > kind-cluster-with-extramounts.yaml < This configuration tells clusterctl where to look for the provider manifests. You could run `clusterctl config -h` to check default clusterctl configuration file path. + +2. Install the providers: + +```bash +clusterctl init --bootstrap k3s --control-plane k3s --infrastructure docker +``` + +3. Wait for the pods to start + +### Create a workload cluster + +There are a number of different cluster templates in the [samples](./samples/) directory. + +> Note: there is an issue with CAPD, it would be better you could do this setup beforehand. [Cluster API with Docker - "too many open files".](https://cluster-api.sigs.k8s.io/user/troubleshooting.html?highlight=too%20many#cluster-api-with-docker----too-many-open-files) + +1. Run the following command to generate your cluster definition: + +```bash +export KIND_IMAGE_VERSION=v1.30.0 +clusterctl generate cluster --from samples/docker/cluster-template-quickstart.yaml test1 --kubernetes-version v1.30.2+k3s2 --worker-machine-count 2 --control-plane-machine-count 1 > cluster.yaml +``` + +> NOTE: the kubernetes version specified with the k3s suffix `+k3s2`. + +2. Check the contents of the generated cluster definition in **cluster.yaml** +3. Ensure the definition is valid by doing a dry run: + +```bash +kubectl apply -f cluster.yaml --dry-run=server +``` + +4. When you are happy apply the definition: + +```bash +kubectl apply -f cluster.yaml +``` + +### Check the workload cluster + +- Check the state of the CAPI machines: + +```bash +kubectl get machine +``` + +- Get the kubeconfig for the cluster: -* Support for External Databases -* Fix Token Logic -* Clean up Control Plane Provider Code -* Post an issue! +```bash +clusterctl get kubeconfig test1 > workload-kubeconfig.yaml +``` + +> Note: if you are using Docker Desktop, you need to fix the kubeconfig by running: + +```bash +# Point the kubeconfig to the exposed port of the load balancer, rather than the inaccessible container IP. +sed -i -e "s/server:.*/server: https:\/\/$(docker port test1-lb 6443/tcp | sed "s/0.0.0.0/127.0.0.1/")/g" ./workload-kubeconfig.yaml +``` + +- Connect to the child cluster + +```bash +kubectl --kubeconfig workload-kubeconfig.yaml get pods -A +``` + +### Deleting the workload cluster + +When deleting a cluster created via CAPI you must delete the top level **Cluster** resource. DO NOT delete using the original file. + +For the quick start: + +```bash +kubectl delete cluster test1 +``` + +### Additional Samples + +Cluster API k3s has been tested on AWS, Azure, AzureStackHCI, Nutanix, OpenStack, Docker and Vsphere environments. + +- To try out the Azure flow, fork the repo and look at `samples/azure/azure-setup.sh`. +- To try out the AWS flow, fork the repo and look at `samples/aws/aws-setup.sh`. +- To try out the Nutanix flow, fork the repo and look at `samples/nutanix/nutanix-setup.sh`. +- To try out the OpenStack flow, fork the repo and look at `samples/openstack/setup.sh`. +- To try out the Vsphere flow, fork the repo and look at `samples/vsphere-capv/setup.sh`. + +## Developer Setup + +You could also build and install CABP3 and CACP3 from src: + +```sh +# Build image with `dev` tag +make BOOTSTRAP_IMG_TAG=dev docker-build-bootstrap +make CONTROLPLANE_IMG_TAG=dev docker-build-controlplane + +# Push image to your registry +export REGISTRY="localhost:5001" # Set this to your local/remote registry +docker tag ghcr.io/k3s-io/cluster-api-k3s/controlplane-controller:dev ${REGISTRY}/controlplane-controller:dev +docker tag ghcr.io/k3s-io/cluster-api-k3s/bootstrap-controller:dev ${REGISTRY}/bootstrap-controller:dev +docker push ${REGISTRY}/controlplane-controller:dev +docker push ${REGISTRY}/bootstrap-controller:dev + +# Install CAPI k3s to management cluster +make install-controlplane # install CRDs +make install-bootstrap +make CONTROLPLANE_IMG=${REGISTRY}/controlplane-controller CONTROLPLANE_IMG_TAG=dev deploy-controlplane # deploy the component +make BOOTSTRAP_IMG=${REGISTRY}/bootstrap-controller BOOTSTRAP_IMG_TAG=dev deploy-bootstrap +``` + +For easy development, please refer to [tilt-setup.md](docs/tilt-setup.md). + +## Roadmap +- Support for External Databases +- Fix Token Logic +- Clean up Control Plane Provider Code +- Post an issue! diff --git a/samples/azure/azure-ccm.yaml b/samples/azure/azure-ccm.yaml deleted file mode 100644 index 07017278..00000000 --- a/samples/azure/azure-ccm.yaml +++ /dev/null @@ -1,187 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cloud-controller-manager - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: system:cloud-controller-manager - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - labels: - k8s-app: cloud-controller-manager -rules: - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - "*" - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - "" - resources: - - services - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - services/status - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - update - - watch - - apiGroups: - - "" - resources: - - endpoints - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - create - - update ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:cloud-controller-manager -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:cloud-controller-manager -subjects: - - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system - - kind: User - name: cloud-controller-manager ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: system:cloud-controller-manager:extension-apiserver-authentication-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: - - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system - - apiGroup: "" - kind: User - name: cloud-controller-manager ---- -apiVersion: v1 -kind: Pod -metadata: - name: cloud-controller-manager - namespace: kube-system - labels: - tier: control-plane - component: cloud-controller-manager -spec: - priorityClassName: system-node-critical - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/master: "true" - tolerations: - - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - effect: NoSchedule - - key: node-role.kubernetes.io/master - effect: NoSchedule - serviceAccountName: cloud-controller-manager - containers: - - name: cloud-controller-manager - image: mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v0.5.0 - imagePullPolicy: IfNotPresent - command: ["cloud-controller-manager"] - args: - - --allocate-node-cidrs=false - - --cloud-config=/etc/kubernetes/azure.json - - --cloud-provider=azure - - --cluster-cidr=192.168.0.0/16 - - --configure-cloud-routes=true - - --controllers=* - - --leader-elect=true - - --route-reconciliation-period=10s - - --v=2 - - --profiling=false - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: "4" - memory: 2Gi - volumeMounts: - - name: etc-kubernetes - mountPath: /etc/kubernetes - - name: etc-ssl - mountPath: /etc/ssl - readOnly: true - volumes: - - name: etc-kubernetes - hostPath: - path: /etc/kubernetes - - name: etc-ssl - hostPath: - path: /etc/ssl \ No newline at end of file diff --git a/samples/azure/azure-cn.yaml b/samples/azure/azure-cn.yaml deleted file mode 100644 index 97b6afaf..00000000 --- a/samples/azure/azure-cn.yaml +++ /dev/null @@ -1,85 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - k8s-app: cloud-node-manager - name: cloud-node-manager - namespace: kube-system ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cloud-node-manager - labels: - k8s-app: cloud-node-manager -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["watch", "list", "get", "update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cloud-node-manager - labels: - k8s-app: cloud-node-manager -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cloud-node-manager -subjects: - - kind: ServiceAccount - name: cloud-node-manager - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: cloud-node-manager - namespace: kube-system - labels: - component: cloud-node-manager -spec: - selector: - matchLabels: - k8s-app: cloud-node-manager - template: - metadata: - labels: - k8s-app: cloud-node-manager - annotations: - cluster-autoscaler.kubernetes.io/daemonset-pod: "true" - spec: - priorityClassName: system-node-critical - serviceAccountName: cloud-node-manager - hostNetwork: true # required to fetch correct hostname - nodeSelector: - kubernetes.io/os: linux - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - effect: NoSchedule - - operator: "Exists" - effect: NoExecute - - operator: "Exists" - effect: NoSchedule - containers: - - name: cloud-node-manager - image: mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v0.5.0 - imagePullPolicy: IfNotPresent - command: - - cloud-node-manager - - --node-name=$(NODE_NAME) - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - resources: - requests: - cpu: 50m - memory: 50Mi - limits: - cpu: 2000m - memory: 512Mi \ No newline at end of file diff --git a/samples/azure/azure-setup.sh b/samples/azure/azure-setup.sh index 3e122b88..7241097d 100755 --- a/samples/azure/azure-setup.sh +++ b/samples/azure/azure-setup.sh @@ -1,3 +1,5 @@ +read -p "Please follow Getting Started to setup the clusterctl configuration first before running this script!" + if [ -z "${AZURE_SUBSCRIPTION_ID}" ]; then echo "please set AZURE_SUBSCRIPTION_ID" exit 0 @@ -8,13 +10,11 @@ if [ -z "${AZURE_TENANT_ID}" ]; then exit 0 fi - if [ -z "${AZURE_CLIENT_ID}" ]; then echo "please set AZURE_CLIENT_ID" exit 0 fi - if [ -z "${AZURE_CLIENT_SECRET}" ]; then echo "please set AZURE_CLIENT_SECRET" exit 0 @@ -34,13 +34,6 @@ export AZURE_TENANT_ID_B64="$(echo -n "$AZURE_TENANT_ID" | base64 | tr -d '\n')" export AZURE_CLIENT_ID_B64="$(echo -n "$AZURE_CLIENT_ID" | base64 | tr -d '\n')" export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZURE_CLIENT_SECRET" | base64 | tr -d '\n')" -export EXP_CLUSTER_RESOURCE_SET=true - - -export PWD="$(pwd)" -mkdir -p ~/.cluster-api -cat samples/clusterctl.yaml | envsubst > ~/.cluster-api/clusterctl.yaml - clusterctl init --infrastructure azure --bootstrap k3s --control-plane k3s kubectl wait --for=condition=Available --timeout=5m -n capi-system deployment/capi-controller-manager @@ -48,12 +41,26 @@ kubectl wait --for=condition=Available --timeout=5m -n capi-k3s-control-plane-sy kubectl wait --for=condition=Available --timeout=5m -n capz-system deployment/capz-controller-manager kubectl wait --for=condition=Available --timeout=5m -n capi-k3s-bootstrap-system deployment/capi-k3s-bootstrap-controller-manager +# Settings needed for AzureClusterIdentity used by the AzureCluster +export AZURE_CLUSTER_IDENTITY_SECRET_NAME="cluster-identity-secret" +export CLUSTER_IDENTITY_NAME="cluster-identity" +export AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE="default" + +# Create a secret to include the password of the Service Principal identity created in Azure +# This secret will be referenced by the AzureClusterIdentity used by the AzureCluster +kubectl create secret generic "${AZURE_CLUSTER_IDENTITY_SECRET_NAME}" --from-literal=clientSecret="${AZURE_CLIENT_SECRET}" --namespace "${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}" + +clusterctl generate cluster --from samples/azure/k3s-template.yaml $CLUSTER_NAME --kubernetes-version v1.30.2+k3s2 --worker-machine-count 2 --control-plane-machine-count 1 > samples/azure/k3s-cluster.yaml -cat samples/azure/k3s-template.yaml | envsubst > samples/azure/k3s-cluster.yaml -kubectl create configmap azure-ccm-addon --from-file=samples/azure/azure-ccm.yaml -kubectl create configmap azure-cn-addon --from-file=samples/azure/azure-cn.yaml kubectl apply -f samples/azure/k3s-cluster.yaml -kubectl apply -f samples/azure/resource-set.yaml +read -p "Please wait for the cluster to be up and running, press any key to continue" + +clusterctl get kubeconfig $CLUSTER_NAME > k3s.yaml + +echo "Manually apply cloud provider to workload cluster, you could visit https://cluster-api.sigs.k8s.io/user/quick-start for the latest info" + +# Note that the nodeSelector should be modified to fit k3s. Run this command for installing cloud provider: +helm install --kubeconfig=./k3s.yaml --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=$CLUSTER_NAME --set cloudControllerManager.clusterCIDR="10.42.0.0/16" --set-string cloudControllerManager.nodeSelector."node-role\.kubernetes\.io/control-plane"=true -echo "once the cluster is up run clusterctl get kubeconfig $CLUSTER_NAME > k3s.yaml or kubectl scale kthreescontrolplane $CLUSTER_NAME-control-plane --replicas 3 for HA" \ No newline at end of file +echo "You could run kubectl scale kthreescontrolplane $CLUSTER_NAME-control-plane --replicas 3 for HA" \ No newline at end of file diff --git a/samples/azure/k3s-template.yaml b/samples/azure/k3s-template.yaml index ef8c4230..120baa09 100644 --- a/samples/azure/k3s-template.yaml +++ b/samples/azure/k3s-template.yaml @@ -3,18 +3,18 @@ kind: Cluster metadata: name: ${CLUSTER_NAME} namespace: default - labels: - ccm: azure spec: clusterNetwork: pods: cidrBlocks: - - 192.168.0.0/16 + - 10.42.0.0/16 + services: + cidrBlocks: + - 10.43.0.0/16 controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: KThreesControlPlane name: ${CLUSTER_NAME}-control-plane - namespace: default infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureCluster @@ -26,24 +26,34 @@ metadata: name: ${CLUSTER_NAME} namespace: default spec: - location: eastus + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} networkSpec: - vnet: - name: ${CLUSTER_NAME}-vnet - resourceGroup: ${CLUSTER_NAME} + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} --- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: KThreesControlPlane metadata: name: ${CLUSTER_NAME}-control-plane namespace: default spec: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane kthreesConfigSpec: + serverConfig: + kubeControllerManagerArgs: + # allocate-node-cidrs needs to be true to use flannel as CNI + - allocate-node-cidrs=true + - cluster-name=${CLUSTER_NAME} + disableCloudController: true + cloudProviderName: external files: - contentFrom: secret: @@ -52,8 +62,17 @@ spec: owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" - replicas: 1 - version: v1.21.5+k3s2 + agentConfig: + nodeName: '{{ ds.meta_data["local_hostname"] }}' + postK3sCommands: [] + preK3sCommands: [] + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION:=v1.30.2+k3s2} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureMachineTemplate @@ -65,17 +84,33 @@ spec: spec: osDisk: diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS osType: Linux - sshPublicKey: "" - vmSize: Standard_D2s_v3 - image: - marketplace: - publisher: Canonical - sku: 18.04-LTS - offer: UbuntuServer - version: latest + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE:="Standard_D2s_v3"} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KThreesConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION:=v1.30.2+k3s2} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureMachineTemplate @@ -87,19 +122,11 @@ spec: spec: osDisk: diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS osType: Linux - sshPublicKey: "" - vmSize: Standard_D2s_v3 - image: - marketplace: - publisher: Canonical - sku: 18.04-LTS - offer: UbuntuServer - version: latest + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE:="Standard_D2s_v3"} --- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: KThreesConfigTemplate metadata: name: ${CLUSTER_NAME}-md-0 @@ -115,29 +142,18 @@ spec: owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" + agentConfig: + nodeName: '{{ ds.meta_data["local_hostname"] }}' --- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity metadata: + name: "${CLUSTER_IDENTITY_NAME}" labels: - cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} - name: ${CLUSTER_NAME}-md-0 - namespace: default + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" spec: - clusterName: ${CLUSTER_NAME} - replicas: 2 - selector: - matchLabels: null - template: - spec: - version: v1.21.5+k3s2 - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KThreesConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 + type: ServicePrincipal + allowedNamespaces: {} + tenantID: "${AZURE_TENANT_ID}" + clientID: "${AZURE_CLIENT_ID}" + clientSecret: {"name":"${AZURE_CLUSTER_IDENTITY_SECRET_NAME}","namespace":"${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}"} \ No newline at end of file diff --git a/samples/azure/resource-set.yaml b/samples/azure/resource-set.yaml deleted file mode 100644 index 683b69ac..00000000 --- a/samples/azure/resource-set.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: crs-azure-cn - namespace: default -spec: - strategy: "ApplyOnce" - clusterSelector: - matchLabels: - ccm: azure - resources: - - name: azure-cn-addon - kind: ConfigMap ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: crs-azure-ccm - namespace: default -spec: - strategy: "ApplyOnce" - clusterSelector: - matchLabels: - ccm: azure - resources: - - name: azure-ccm-addon - kind: ConfigMap \ No newline at end of file diff --git a/samples/deployment/bootstrap-k3s/v1.2.2/bootstrap-components.yaml b/samples/deployment/bootstrap-k3s/v1.2.2/bootstrap-components.yaml index 192f2e0e..ab1b9c95 100644 --- a/samples/deployment/bootstrap-k3s/v1.2.2/bootstrap-components.yaml +++ b/samples/deployment/bootstrap-k3s/v1.2.2/bootstrap-components.yaml @@ -1,3 +1,4 @@ +# WARNING: This file is outdated, it is kept only for tryout. Please refer to the steps in Getting Started Guide to deploy the latest version of the bootstrap provider. apiVersion: v1 kind: Namespace metadata: diff --git a/samples/deployment/control-plane-k3s/v1.2.2/control-plane-components.yaml b/samples/deployment/control-plane-k3s/v1.2.2/control-plane-components.yaml index 0ea3198c..f18b9648 100644 --- a/samples/deployment/control-plane-k3s/v1.2.2/control-plane-components.yaml +++ b/samples/deployment/control-plane-k3s/v1.2.2/control-plane-components.yaml @@ -1,3 +1,4 @@ +# WARNING: This file is outdated, it is kept only for tryout. please refer to the steps in Getting Started Guide to deploy the latest version of the controlplane provider. apiVersion: v1 kind: Namespace metadata: diff --git a/samples/docker/cluster-template-quickstart.yaml b/samples/docker/cluster-template-quickstart.yaml new file mode 100644 index 00000000..d317ae15 --- /dev/null +++ b/samples/docker/cluster-template-quickstart.yaml @@ -0,0 +1,104 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.45.0.0/16 + serviceDomain: cluster.local + services: + cidrBlocks: + - 10.46.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KThreesControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerCluster + name: ${CLUSTER_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + template: + metadata: + labels: + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KThreesConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KThreesControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} +spec: + kthreesConfigSpec: + serverConfig: + tlsSan: + - 0.0.0.0 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerCluster +metadata: + name: ${CLUSTER_NAME} +spec: {} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} +spec: + template: + spec: + customImage: kindest/node:${KIND_IMAGE_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} +spec: + template: + spec: + customImage: kindest/node:${KIND_IMAGE_VERSION} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KThreesConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} +spec: + template: + spec: null \ No newline at end of file diff --git a/samples/docker/clusterclass/cluster-template-topology.yaml b/samples/docker/clusterclass/cluster-template-topology.yaml new file mode 100644 index 00000000..67b46ecf --- /dev/null +++ b/samples/docker/clusterclass/cluster-template-topology.yaml @@ -0,0 +1,32 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.45.0.0/16 + serviceDomain: cluster.local + services: + cidrBlocks: + - 10.46.0.0/16 + topology: + class: k3s + controlPlane: + nodeDeletionTimeout: 30s + nodeVolumeDetachTimeout: 5m + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + variables: + - name: kindImageVersion + value: ${KIND_IMAGE_VERSION} + version: ${KUBERNETES_VERSION} + workers: + machineDeployments: + - class: k3s-default-worker + minReadySeconds: 5 + name: ${CLUSTER_NAME}-md-0 + nodeDeletionTimeout: 30s + nodeVolumeDetachTimeout: 5m + replicas: ${WORKER_MACHINE_COUNT} diff --git a/samples/docker/clusterclass/clusterclass-k3s.yaml b/samples/docker/clusterclass/clusterclass-k3s.yaml new file mode 100644 index 00000000..e707d875 --- /dev/null +++ b/samples/docker/clusterclass/clusterclass-k3s.yaml @@ -0,0 +1,113 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: k3s +spec: + controlPlane: + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KThreesControlPlaneTemplate + name: k3s-control-plane + machineInfrastructure: + ref: + kind: DockerMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: k3s-control-plane + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerClusterTemplate + name: k3s-cluster + workers: + machineDeployments: + - class: k3s-default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KThreesConfigTemplate + name: k3s-default-worker-bootstrap + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: k3s-default-worker-machinetemplate + variables: + - name: kindImageVersion + required: true + schema: + openAPIV3Schema: + type: string + default: "v1.28.0" + patches: + - name: customImage + description: "Sets the container image that is used for running dockerMachines for the controlPlane and default-worker machineDeployments." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + matchResources: + machineDeploymentClass: + names: + - k3s-default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + valueFrom: + template: | + kindest/node:{{ .kindImageVersion }} + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + valueFrom: + template: | + kindest/node:{{ .kindImageVersion }} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerClusterTemplate +metadata: + name: k3s-cluster +spec: + template: + spec: {} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KThreesControlPlaneTemplate +metadata: + name: k3s-control-plane +spec: + template: + spec: + kthreesConfigSpec: + serverConfig: + tlsSan: + - 0.0.0.0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: k3s-control-plane +spec: + template: + spec: {} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KThreesConfigTemplate +metadata: + name: k3s-default-worker-bootstrap +spec: + template: + spec: {} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: k3s-default-worker-machinetemplate +spec: + template: + spec: {} \ No newline at end of file