support airgap install

Signed-off-by: nasusoba <[email protected]>
k3s-io · Apr 17, 2024 · 3bf8b06 · 3bf8b06
1 parent 09c603f
commit 3bf8b06
Show file tree

Hide file tree

Showing 16 changed files with 233 additions and 20 deletions.
diff --git a/bootstrap/api/v1beta1/kthreesconfig_types.go b/bootstrap/api/v1beta1/kthreesconfig_types.go
@@ -140,6 +140,13 @@ type KThreesAgentConfig struct {
 	// NodeName Name of the Node
 	// +optional
 	NodeName string `json:"nodeName,omitempty"`
+
+	// AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
+	// basically supposing that online container registries and k3s install scripts are not reachable.
+	// User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
+	// on all nodes in the air-gap environment.
+	// +optional
+	AirGapped bool `json:"airGapped,omitempty"`
 }
 
 // KThreesConfigStatus defines the observed state of KThreesConfig.

diff --git a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml
@@ -42,6 +42,13 @@ spec:
               agentConfig:
                 description: AgentConfig specifies configuration for the agent nodes
                 properties:
+                  airGapped:
+                    description: |-
+                      AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
+                      basically supposing that online container registries and k3s install scripts are not reachable.
+                      User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
+                      on all nodes in the air-gap environment.
+                    type: boolean
                   kubeProxyArgs:
                     description: KubeProxyArgs Customized flag for kube-proxy process
                     items:

diff --git a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml
@@ -50,6 +50,13 @@ spec:
                         description: AgentConfig specifies configuration for the agent
                           nodes
                         properties:
+                          airGapped:
+                            description: |-
+                              AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
+                              basically supposing that online container registries and k3s install scripts are not reachable.
+                              User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
+                              on all nodes in the air-gap environment.
+                            type: boolean
                           kubeProxyArgs:
                             description: KubeProxyArgs Customized flag for kube-proxy
                               process

diff --git a/bootstrap/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml b/bootstrap/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml
@@ -130,6 +130,13 @@ spec:
                     description: AgentConfig specifies configuration for the agent
                       nodes
                     properties:
+                      airGapped:
+                        description: |-
+                          AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
+                          basically supposing that online container registries and k3s install scripts are not reachable.
+                          User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
+                          on all nodes in the air-gap environment.
+                        type: boolean
                       kubeProxyArgs:
                         description: KubeProxyArgs Customized flag for kube-proxy
                           process

diff --git a/bootstrap/controllers/kthreesconfig_controller.go b/bootstrap/controllers/kthreesconfig_controller.go
@@ -269,6 +269,7 @@ func (r *KThreesConfigReconciler) joinControlplane(ctx context.Context, scope *S
 			AdditionalFiles: files,
 			ConfigFile:      workerConfigFile,
 			K3sVersion:      scope.Config.Spec.Version,
+			AirGapped:       scope.Config.Spec.AgentConfig.AirGapped,
 		},
 	}
 
@@ -328,6 +329,7 @@ func (r *KThreesConfigReconciler) joinWorker(ctx context.Context, scope *Scope)
 			AdditionalFiles: files,
 			ConfigFile:      workerConfigFile,
 			K3sVersion:      scope.Config.Spec.Version,
+			AirGapped:       scope.Config.Spec.AgentConfig.AirGapped,
 		},
 	}
 
@@ -483,6 +485,7 @@ func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Contex
 			AdditionalFiles: files,
 			ConfigFile:      initConfigFile,
 			K3sVersion:      scope.Config.Spec.Version,
+			AirGapped:       scope.Config.Spec.AgentConfig.AirGapped,
 		},
 		Certificates: certificates,
 	}

diff --git a/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml b/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml
@@ -42,6 +42,13 @@ spec:
               agentConfig:
                 description: AgentConfig specifies configuration for the agent nodes
                 properties:
+                  airGapped:
+                    description: |-
+                      AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
+                      basically supposing that online container registries and k3s install scripts are not reachable.
+                      User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
+                      on all nodes in the air-gap environment.
+                    type: boolean
                   kubeProxyArgs:
                     description: KubeProxyArgs Customized flag for kube-proxy process
                     items:

diff --git a/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml b/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml
@@ -50,6 +50,13 @@ spec:
                         description: AgentConfig specifies configuration for the agent
                           nodes
                         properties:
+                          airGapped:
+                            description: |-
+                              AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
+                              basically supposing that online container registries and k3s install scripts are not reachable.
+                              User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
+                              on all nodes in the air-gap environment.
+                            type: boolean
                           kubeProxyArgs:
                             description: KubeProxyArgs Customized flag for kube-proxy
                               process

diff --git a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml
@@ -130,6 +130,13 @@ spec:
                     description: AgentConfig specifies configuration for the agent
                       nodes
                     properties:
+                      airGapped:
+                        description: |-
+                          AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
+                          basically supposing that online container registries and k3s install scripts are not reachable.
+                          User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
+                          on all nodes in the air-gap environment.
+                        type: boolean
                       kubeProxyArgs:
                         description: KubeProxyArgs Customized flag for kube-proxy
                           process

diff --git a/pkg/cloudinit/cloudinit.go b/pkg/cloudinit/cloudinit.go
@@ -69,17 +69,20 @@ write_files:{{ range . }}
 {{- end -}}
 {{- end -}}
 `
+	sentinelFileCommand = "mkdir -p /run/cluster-api && echo success > /run/cluster-api/bootstrap-success.complete"
 )
 
 // BaseUserData is shared across all the various types of files written to disk.
 type BaseUserData struct {
-	Header          string
-	PreK3sCommands  []string
-	PostK3sCommands []string
-	AdditionalFiles []bootstrapv1.File
-	WriteFiles      []bootstrapv1.File
-	ConfigFile      bootstrapv1.File
-	K3sVersion      string
+	Header              string
+	PreK3sCommands      []string
+	PostK3sCommands     []string
+	AdditionalFiles     []bootstrapv1.File
+	WriteFiles          []bootstrapv1.File
+	ConfigFile          bootstrapv1.File
+	K3sVersion          string
+	AirGapped           bool
+	SentinelFileCommand string
 }
 
 func generate(kind string, tpl string, data interface{}) ([]byte, error) {

diff --git a/pkg/cloudinit/controlplane_init.go b/pkg/cloudinit/controlplane_init.go
@@ -27,7 +27,8 @@ const (
 {{template "files" .WriteFiles}}
 runcmd:
 {{- template "commands" .PreK3sCommands }}
-  - 'curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s sh -s - server && mkdir -p /run/cluster-api && echo success > /run/cluster-api/bootstrap-success.complete'
+  - {{ if .AirGapped }} "INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC='server' sh /opt/install.sh" {{ else }}'curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s sh -s - server' {{ end }}
+  - '{{ .SentinelFileCommand }}'
 {{- template "commands" .PostK3sCommands }}
 `
 )
@@ -44,6 +45,7 @@ func NewInitControlPlane(input *ControlPlaneInput) ([]byte, error) {
 	input.WriteFiles = input.Certificates.AsFiles()
 	input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...)
 	input.WriteFiles = append(input.WriteFiles, input.ConfigFile)
+	input.SentinelFileCommand = sentinelFileCommand
 
 	controlPlaneCloudJoinWithVersion := fmt.Sprintf(controlPlaneCloudInit, input.K3sVersion)
 	userData, err := generate("InitControlplane", controlPlaneCloudJoinWithVersion, input)

diff --git a/pkg/cloudinit/controlplane_init_test.go b/pkg/cloudinit/controlplane_init_test.go
@@ -51,3 +51,33 @@ func TestControlPlaneInit(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred())
 	t.Log(string(out))
 }
+
+func TestControlPlaneInitAirGapped(t *testing.T) {
+	g := NewWithT(t)
+
+	cpinput := &ControlPlaneInput{
+		BaseUserData: BaseUserData{
+			PreK3sCommands:  nil,
+			PostK3sCommands: nil,
+			AdditionalFiles: []infrav1.File{
+				{
+					Path:     "/tmp/my-path",
+					Encoding: infrav1.Base64,
+					Content:  "aGk=",
+				},
+				{
+					Path:    "/tmp/my-other-path",
+					Content: "hi",
+				},
+			},
+			AirGapped: true,
+		},
+		Certificates: secret.Certificates{},
+	}
+
+	out, err := NewInitControlPlane(cpinput)
+	g.Expect(err).NotTo(HaveOccurred())
+	result := string(out)
+	g.Expect(result).To(ContainSubstring("sh /opt/install.sh"))
+	g.Expect(result).NotTo(ContainSubstring("get.k3s.io"))
+}
diff --git a/pkg/cloudinit/controlplane_join.go b/pkg/cloudinit/controlplane_join.go
@@ -18,23 +18,14 @@ package cloudinit
 
 import "fmt"
 
-const (
-	controlPlaneCloudJoin = `{{.Header}}
-{{template "files" .WriteFiles}}
-runcmd:
-{{- template "commands" .PreK3sCommands }}
-  - 'curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s sh -s - server && mkdir -p /run/cluster-api && echo success > /run/cluster-api/bootstrap-success.complete'
-{{- template "commands" .PostK3sCommands }}
-`
-)
-
 // NewInitControlPlane returns the user data string to be used on a controlplane instance.
 func NewJoinControlPlane(input *ControlPlaneInput) ([]byte, error) {
 	input.Header = cloudConfigHeader
 	input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...)
 	input.WriteFiles = append(input.WriteFiles, input.ConfigFile)
+	input.SentinelFileCommand = sentinelFileCommand
 
-	controlPlaneCloudJoinWithVersion := fmt.Sprintf(controlPlaneCloudJoin, input.K3sVersion)
+	controlPlaneCloudJoinWithVersion := fmt.Sprintf(controlPlaneCloudInit, input.K3sVersion)
 	userData, err := generate("JoinControlplane", controlPlaneCloudJoinWithVersion, input)
 	if err != nil {
 		return nil, err

diff --git a/pkg/cloudinit/worker_join.go b/pkg/cloudinit/worker_join.go
@@ -23,7 +23,8 @@ const (
 {{template "files" .WriteFiles}}
 runcmd:
 {{- template "commands" .PreK3sCommands }}
-  - 'curl -sfL https://get.k3s.io |  INSTALL_K3S_VERSION=%s sh -s - agent && mkdir -p /run/cluster-api && echo success > /run/cluster-api/bootstrap-success.complete'
+  - {{ if .AirGapped }} "INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC='agent' sh /opt/install.sh" {{ else }} 'curl -sfL https://get.k3s.io |  INSTALL_K3S_VERSION=%s sh -s - agent' {{ end }}
+  - '{{ .SentinelFileCommand }}'
 {{- template "commands" .PostK3sCommands }}
 `
 )
@@ -38,6 +39,7 @@ func NewWorker(input *WorkerInput) ([]byte, error) {
 	input.Header = cloudConfigHeader
 	input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...)
 	input.WriteFiles = append(input.WriteFiles, input.ConfigFile)
+	input.SentinelFileCommand = sentinelFileCommand
 
 	workerCloudInitWithVersion := fmt.Sprintf(workerCloudInit, input.K3sVersion)
 	userData, err := generate("Worker", workerCloudInitWithVersion, input)

diff --git a/samples/docker/air-gapped/Dockerfile b/samples/docker/air-gapped/Dockerfile
@@ -0,0 +1,15 @@
+FROM kindest/node:v1.28.0
+ARG K3S_VERSION=v1.28.6+k3s2
+
+# Load docker images
+# Note that the flow follows the manually deploy image steps, but private registry method should also be supported
+RUN mkdir -p /var/lib/rancher/k3s/agent/images/
+RUN curl -L -o /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar.zst "https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}/k3s-airgap-images-amd64.tar.zst"
+
+# Download install script to /opt/install.sh
+RUN curl -L -o /opt/install.sh  https://get.k3s.io
+RUN chmod +x /opt/install.sh
+
+# Download k3s binary
+RUN curl -L -o /usr/local/bin/k3s "https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}/k3s"
+RUN chmod +x /usr/local/bin/k3s
diff --git a/samples/docker/air-gapped/README.md b/samples/docker/air-gapped/README.md
@@ -0,0 +1,23 @@
+# Air-gapped K3s Cluster
+
+K3s is supporting air-gapped installations. This sample demonstrates how to create a K3s cluster in an air-gapped environment with cluster API k3s and Docker.
+
+It will first build a kind node docker image with the K3s binary, the required images and scripts, following [k3s Air-Gap Install](https://docs.k3s.io/installation/airgap). Then it will create a K3s cluster with this kind node image.
+
+```shell
+export AIRGAPPED_KIND_IMAGE=kindnode:airgapped
+export CLUSTER_NAME=k3s-airgapped
+export CONTROL_PLANE_MACHINE_COUNT=1
+export KUBERNETES_VERSION=v1.28.6+k3s2
+export WORKER_MACHINE_COUNT=3
+
+# Build the kind node image
+docker build -t $AIRGAPPED_KIND_IMAGE . --build-arg="K3S_VERSION=$KUBERNETES_VERSION"
+
+# Generate the cluster yaml
+# Note that `airGapped` is set to true in `agentConfig`
+clusterctl generate yaml --from ./k3s-template.yaml > k3s-airgapped.yaml
+
+# Create the cluster to the management cluster
+kubectl apply -f k3s-airgapped.yaml
+```
diff --git a/samples/docker/air-gapped/k3s-template.yaml b/samples/docker/air-gapped/k3s-template.yaml
@@ -0,0 +1,95 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster 
+metadata:
+  name: ${CLUSTER_NAME} 
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+      - 10.45.0.0/16
+    services:
+      cidrBlocks:
+      - 10.46.0.0/16
+    serviceDomain: cluster.local
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    kind: KThreesControlPlane
+    name: ${CLUSTER_NAME}-control-plane
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerCluster
+    name: ${CLUSTER_NAME}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec: {}
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+kind: KThreesControlPlane
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+  namespace: default
+spec:
+  infrastructureTemplate:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerMachineTemplate
+    name: ${CLUSTER_NAME}-control-plane
+  kthreesConfigSpec:
+    agentConfig:
+      airGapped: true
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  version: ${KUBERNETES_VERSION}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  template:
+    spec:
+      customImage: ${AIRGAPPED_KIND_IMAGE}
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: worker-md-0
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels:
+      cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+  template:
+    spec:
+      version: ${KUBERNETES_VERSION}
+      clusterName: ${CLUSTER_NAME}
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KThreesConfigTemplate
+          name: ${CLUSTER_NAME}-md-0
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: DockerMachineTemplate
+        name: ${CLUSTER_NAME}-md-0
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      customImage: ${AIRGAPPED_KIND_IMAGE}
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KThreesConfigTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      agentConfig:
+        airGapped: true