Improve and conslidate cinder node build inputs

This moves all of the input values used to build the cinder image into the internal/cinder/config/constants package. This ensures that updating versions that go into the cinder image will only need to take place there.

This also changes the way the Kubernetes version is set. Rather than being hardcoded in a const, it is determined when creating a new node by reading the /cinder/kubernetes_version file.

The Kind base image has also been updated and a fix has been applied to account for changes in the image. The newer versions of the image include the kubelet systemd files which means that the /etc/default/kubelet file was already created.

.goreleaser.yml

@@ -43,6 +43,7 @@ builds:
      - -X "github.com/criticalstack/crit/internal/buildinfo.Date={{.Date}}"
      - -X "github.com/criticalstack/crit/internal/buildinfo.GitSHA={{.ShortCommit}}"
      - -X "github.com/criticalstack/crit/internal/buildinfo.Version={{.Tag}}"
+     - -X "github.com/criticalstack/crit/internal/cinder/config/constants.DefaultNodeImage=criticalstack/cinder:{{.Tag}}"
 archives:
 - replacements:
     darwin: Darwin

build/images/node/Makefile

@@ -15,20 +15,26 @@ $(CINDER): # Build cinder
 	$(MAKE) -s -C ../../../ cinder BIN_DIR=$(PWD)/$(BIN_DIR)
 
 
+# Versions are all pulled from this file, any version updates should by made in this file
+CONST_FILE := ../../../internal/cinder/config/constants/constants.go
+define readvar
+$(or $(shell sed -nE 's/^.*$(1).*"(.*)"$$/\1/p' $(CONST_FILE)),$(error "Cannot find variable $(1)"))
+endef
+
 TAG   ?= latest
 IMAGE ?= docker.io/criticalstack/cinder:$(TAG)
 
 BUILD_DIR             := $(PWD)/build
 BUILD_CONTAINER_NAME  := cinder-build
-BASE_IMAGE            := docker.io/kindest/base:v20200709-8b878ace
-KUBERNETES_VERSION    := 1.18.10
+BASE_IMAGE            :=  $(call readvar,KindBaseImage)
+KUBERNETES_VERSION    := $(call readvar,KubernetesVersion)
 
 # charts
 CHARTS_DIR                          := $(PWD)/charts
-CILIUM_VERSION                      := 1.8.5
-LOCAL_PATH_PROVISIONER_VERSION      := 0.0.12
-MACHINE_API_VERSION                 := 1.0.6
-MACHINE_API_PROVIDER_DOCKER_VERSION := 1.0.7
+CILIUM_VERSION                      := $(call readvar,CiliumVersion)
+LOCAL_PATH_PROVISIONER_VERSION      := $(call readvar,LocalPathProvisionerVersion)
+MACHINE_API_VERSION                 := $(call readvar,MachineAPIVersion)
+MACHINE_API_PROVIDER_DOCKER_VERSION := $(call readvar,MachineAPIProviderDockerVersion)
 
 charts/*.tgz:
 	mkdir -p $(CHARTS_DIR)

build/images/node/scripts/build.sh

@@ -10,7 +10,11 @@ apt-add-repository "deb http://apt.kubernetes.io/ kubernetes-xenial main"
 DEBIAN_FRONTEND=noninteractive && clean-install kubectl=${KUBERNETES_VERSION}-00
 curl -L https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubelet -o /usr/bin/kubelet
 chmod +x /usr/bin/kubelet
-echo "KUBELET_EXTRA_ARGS=--fail-swap-on=false" >> /etc/default/kubelet
+
+# Kind has moved the kubelet systemd files into the base image, so this is no
+# longer necessary
+#echo "KUBELET_EXTRA_ARGS=--fail-swap-on=false" >> /etc/default/kubelet
+
 curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
 mkdir -p /cinder/manifests
 curl -L https://github.com/criticalstack/machine-api/releases/download/v${MACHINE_API_VERSION}/machine-api.yaml -o /cinder/manifests/machine-api.yaml
@@ -29,3 +33,5 @@ mkdir -p /etc/cni/net.d
 mkdir -p /var/lib/crit
 mkdir -p /var/log/kubernetes
 chmod 700 /var/lib/etcd
+
+echo ${KUBERNETES_VERSION} > /cinder/kubernetes_version

cmd/cinder/app/create/cluster/cluster.go

@@ -17,8 +17,6 @@ import (
 	"github.com/criticalstack/crit/internal/cinder/config/constants"
 	"github.com/criticalstack/crit/internal/cinder/feature"
 	"github.com/criticalstack/crit/internal/cinder/utils"
-	critconfig "github.com/criticalstack/crit/internal/config"
-	yamlutil "github.com/criticalstack/crit/pkg/kubernetes/yaml"
 )
 
 var opts struct {
@@ -121,23 +119,7 @@ func NewCommand() *cobra.Command {
 				if err := feature.MutableGates.SetFromMap(cfg.FeatureGates); err != nil {
 					return err
 				}
-				if cfg.ControlPlaneConfiguration == nil {
-					cfg.ControlPlaneConfiguration = &critconfig.ControlPlaneConfiguration{}
-				}
-				cfg.ControlPlaneConfiguration.ClusterName = opts.Name
-				cluster.SetControlPlaneConfigurationDefaults(cfg.ControlPlaneConfiguration)
 				cfg.Files = append(cfg.Files, files...)
-				data, err := yamlutil.MarshalToYaml(cfg.ControlPlaneConfiguration, critconfig.SchemeGroupVersion)
-				if err != nil {
-					return err
-				}
-				cfg.Files = append(cfg.Files, config.File{
-					Path:        "/var/lib/crit/config.yaml",
-					Owner:       "root:root",
-					Permissions: "0644",
-					Encoding:    config.Base64,
-					Content:     base64.StdEncoding.EncodeToString(data),
-				})
 				if feature.Gates.Enabled(feature.LocalRegistry) {
 					cfg.RegistryMirrors[fmt.Sprintf("%s:%d", cfg.LocalRegistryName, cfg.LocalRegistryPort)] = fmt.Sprintf("http://%s:%d", cfg.LocalRegistryName, cfg.LocalRegistryPort)
 				}

cmd/cinder/app/create/node/node.go

@@ -16,7 +16,6 @@ import (
 	"github.com/criticalstack/crit/internal/cinder/utils"
 	critconfig "github.com/criticalstack/crit/internal/config"
 	"github.com/criticalstack/crit/pkg/kubernetes/pki"
-	yamlutil "github.com/criticalstack/crit/pkg/kubernetes/yaml"
 )
 
 var opts struct {
@@ -82,18 +81,6 @@ func NewCommand() *cobra.Command {
 					Encoding:    config.Base64,
 					Content:     base64.StdEncoding.EncodeToString(data),
 				})
-				data, err = yamlutil.MarshalToYaml(cfg.WorkerConfiguration, critconfig.SchemeGroupVersion)
-				if err != nil {
-					return err
-				}
-				cfg.Files = append(cfg.Files, config.File{
-					Path:        "/var/lib/crit/config.yaml",
-					Owner:       "root:root",
-					Permissions: "0644",
-					Encoding:    config.Base64,
-					Content:     base64.StdEncoding.EncodeToString(data),
-				})
-
 				if feature.Gates.Enabled(feature.LocalRegistry) {
 					cfg.RegistryMirrors[fmt.Sprintf("%s:%d", cfg.LocalRegistryName, cfg.LocalRegistryPort)] = fmt.Sprintf("http://%s:%d", cfg.LocalRegistryName, cfg.LocalRegistryPort)
 				}

internal/cinder/cluster/controlplane.go

@@ -2,7 +2,9 @@ package cluster
 
 import (
 	"context"
+	"encoding/base64"
 	"os"
+	"strings"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -14,6 +16,7 @@ import (
 	"github.com/criticalstack/crit/internal/cinder/config/constants"
 	"github.com/criticalstack/crit/internal/cinder/feature"
 	critconfig "github.com/criticalstack/crit/internal/config"
+	yamlutil "github.com/criticalstack/crit/pkg/kubernetes/yaml"
 )
 
 var (
@@ -50,6 +53,27 @@ func CreateControlPlaneNode(ctx context.Context, cfg *ControlPlaneConfig) (*Node
 		node.Stdout = os.Stdout
 		node.Stderr = os.Stderr
 	}
+	if cfg.ClusterConfiguration.ControlPlaneConfiguration == nil {
+		cfg.ClusterConfiguration.ControlPlaneConfiguration = &critconfig.ControlPlaneConfiguration{}
+	}
+	cfg.ClusterConfiguration.ControlPlaneConfiguration.ClusterName = cfg.ClusterName
+	SetControlPlaneConfigurationDefaults(cfg.ClusterConfiguration.ControlPlaneConfiguration)
+	data, err := node.ReadFile("/cinder/kubernetes_version")
+	if err != nil {
+		return nil, err
+	}
+	cfg.ClusterConfiguration.ControlPlaneConfiguration.NodeConfiguration.KubernetesVersion = strings.TrimSpace(string(data))
+	data, err = yamlutil.MarshalToYaml(cfg.ClusterConfiguration.ControlPlaneConfiguration, critconfig.SchemeGroupVersion)
+	if err != nil {
+		return nil, err
+	}
+	cfg.ClusterConfiguration.Files = append(cfg.ClusterConfiguration.Files, config.File{
+		Path:        "/var/lib/crit/config.yaml",
+		Owner:       "root:root",
+		Permissions: "0644",
+		Encoding:    config.Base64,
+		Content:     base64.StdEncoding.EncodeToString(data),
+	})
 	if feature.Gates.Enabled(feature.FixCgroupMounts) {
 		if err := node.Command("bash", "/cinder/scripts/fix-cgroup-mounts.sh").Run(); err != nil {
 			return node, err
@@ -70,6 +94,7 @@ func CreateControlPlaneNode(ctx context.Context, cfg *ControlPlaneConfig) (*Node
 }
 
 func SetControlPlaneConfigurationDefaults(cfg *critconfig.ControlPlaneConfiguration) {
+	// this will be overriden by the kubernetes version file in the image
 	cfg.NodeConfiguration.KubernetesVersion = constants.KubernetesVersion
 	if cfg.KubeAPIServerConfiguration.ExtraArgs == nil {
 		cfg.KubeAPIServerConfiguration.ExtraArgs = make(map[string]string)

internal/cinder/cluster/registry.go

@@ -22,7 +22,7 @@ func CreateRegistry(name string, port int) error {
 		"--name", name,
 		"-p", fmt.Sprintf("%d:5000", port),
 		"--network", constants.DefaultNetwork,
-		fmt.Sprintf("docker.io/library/registry:%s", constants.DefaultRegistryVersion),
+		fmt.Sprintf("docker.io/library/registry:%s", constants.RegistryVersion),
 	}
 	providerName := "docker"
 	if v, ok := os.LookupEnv("KIND_EXPERIMENTAL_PROVIDER"); ok {

internal/cinder/cluster/worker.go

@@ -2,7 +2,9 @@ package cluster
 
 import (
 	"context"
+	"encoding/base64"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -14,6 +16,7 @@ import (
 	"github.com/criticalstack/crit/internal/cinder/config"
 	"github.com/criticalstack/crit/internal/cinder/config/constants"
 	critconfig "github.com/criticalstack/crit/internal/config"
+	yamlutil "github.com/criticalstack/crit/pkg/kubernetes/yaml"
 )
 
 type WorkerConfig struct {
@@ -40,6 +43,22 @@ func CreateWorkerNode(ctx context.Context, cfg *WorkerConfig) (*Node, error) {
 		node.Stdout = os.Stdout
 		node.Stderr = os.Stderr
 	}
+	data, err := node.ReadFile("/cinder/kubernetes_version")
+	if err != nil {
+		return nil, err
+	}
+	cfg.ClusterConfiguration.WorkerConfiguration.NodeConfiguration.KubernetesVersion = strings.TrimSpace(string(data))
+	data, err = yamlutil.MarshalToYaml(cfg.ClusterConfiguration.WorkerConfiguration, critconfig.SchemeGroupVersion)
+	if err != nil {
+		return nil, err
+	}
+	cfg.ClusterConfiguration.Files = append(cfg.ClusterConfiguration.Files, config.File{
+		Path:        "/var/lib/crit/config.yaml",
+		Owner:       "root:root",
+		Permissions: "0644",
+		Encoding:    config.Base64,
+		Content:     base64.StdEncoding.EncodeToString(data),
+	})
 	if err := node.Command("bash", "/cinder/scripts/fix-cgroup-mounts.sh").Run(); err != nil {
 		return nil, err
 	}

internal/cinder/config/constants/constants.go

@@ -6,21 +6,29 @@ import (
 	"github.com/criticalstack/crit/pkg/config/constants"
 )
 
+var (
+	DefaultNodeImage = "criticalstack/cinder:latest"
+)
+
 const (
-	DefaultNodeImage  = "criticalstack/cinder:v1.0.6"
-	DefaultNetwork    = "cinder"
+	// the Kubernetes version to beused when building the cinder image
 	KubernetesVersion = "1.18.10"
 
-	DefaultMachineAPIVersion               = "1.0.6"
-	DefaultMachineAPIProviderDockerVersion = "1.0.7"
-	DefaultKubeRBACProxyVersion            = "0.5.0"
-	DefaultCiliumVersion                   = "1.8.5"
-	DefaultCiliumStartupScriptVersion      = "af2a99046eca96c0138551393b21a5c044c7fe79"
-	DefaultLocalPathProvisionerVersion     = "0.0.12"
-	DefaultRegistryVersion                 = "2.7.1"
+	// Kind base image to use as the base for the cinder image
+	KindBaseImage = "docker.io/kindest/base:v20200928-02f74589"
 
+	DefaultNetwork           = "cinder"
 	DefaultLocalRegistryName = "cinderegg"
 	DefaultLocalRegistryPort = 5000
+
+	// container image versions
+	CiliumStartupScriptVersion      = "af2a99046eca96c0138551393b21a5c044c7fe79"
+	CiliumVersion                   = "1.8.5"
+	KubeRBACProxyVersion            = "0.5.0"
+	LocalPathProvisionerVersion     = "0.0.12"
+	MachineAPIProviderDockerVersion = "1.0.7"
+	MachineAPIVersion               = "1.0.6"
+	RegistryVersion                 = "2.7.1"
 )
 
 func GetImages() map[string]string {
@@ -33,14 +41,14 @@ func GetImages() map[string]string {
 		"coredns":                     fmt.Sprintf("%s:%s", constants.CoreDNSImage, constants.DefaultCoreDNSVersion),
 		"bootstrap-server":            fmt.Sprintf("%s:v%s", constants.CritBootstrapServerImage, constants.DefaultBootstrapServerVersion),
 		"healthcheck-proxy":           fmt.Sprintf("%s:v%s", constants.CritHealthCheckProxyImage, constants.DefaultHealthcheckProxyVersion),
-		"machine-api":                 fmt.Sprintf("docker.io/criticalstack/machine-api:v%s", DefaultMachineAPIVersion),
-		"machine-api-provider-docker": fmt.Sprintf("docker.io/criticalstack/machine-api-provider-docker:v%s", DefaultMachineAPIProviderDockerVersion),
-		"kube-rbac-proxy":             fmt.Sprintf("gcr.io/kubebuilder/kube-rbac-proxy:v%s", DefaultKubeRBACProxyVersion),
-		"cilium":                      fmt.Sprintf("docker.io/cilium/cilium:v%s", DefaultCiliumVersion),
-		"cilium-operator-generic":     fmt.Sprintf("docker.io/cilium/operator-generic:v%s", DefaultCiliumVersion),
-		"cilium-startup-script":       fmt.Sprintf("docker.io/cilium/startup-script:%s", DefaultCiliumStartupScriptVersion),
-		"local-path-provisioner":      fmt.Sprintf("docker.io/rancher/local-path-provisioner:v%s", DefaultLocalPathProvisionerVersion),
-		"registry":                    fmt.Sprintf("docker.io/library/registry:%s", DefaultRegistryVersion),
+		"machine-api":                 fmt.Sprintf("docker.io/criticalstack/machine-api:v%s", MachineAPIVersion),
+		"machine-api-provider-docker": fmt.Sprintf("docker.io/criticalstack/machine-api-provider-docker:v%s", MachineAPIProviderDockerVersion),
+		"kube-rbac-proxy":             fmt.Sprintf("gcr.io/kubebuilder/kube-rbac-proxy:v%s", KubeRBACProxyVersion),
+		"cilium":                      fmt.Sprintf("docker.io/cilium/cilium:v%s", CiliumVersion),
+		"cilium-operator-generic":     fmt.Sprintf("docker.io/cilium/operator-generic:v%s", CiliumVersion),
+		"cilium-startup-script":       fmt.Sprintf("docker.io/cilium/startup-script:%s", CiliumStartupScriptVersion),
+		"local-path-provisioner":      fmt.Sprintf("docker.io/rancher/local-path-provisioner:v%s", LocalPathProvisionerVersion),
+		"registry":                    fmt.Sprintf("docker.io/library/registry:%s", RegistryVersion),
 	}
 	return images
 }