From e502f3286b0b8d9eeee90cb89e4a047f7ed3e289 Mon Sep 17 00:00:00 2001 From: chris Date: Fri, 30 Oct 2020 12:59:19 -0400 Subject: [PATCH] Improve and conslidate cinder node build inputs This moves all of the input values used to build the cinder image into the internal/cinder/config/constants package. This ensures that updating versions that go into the cinder image will only need to take place there. This also changes the way the Kubernetes version is set. Rather than being hardcoded in a const, it is determined when creating a new node by reading the /cinder/kubernetes_version file. The Kind base image has also been updated and a fix has been applied to account for changes in the image. The newer versions of the image include the kubelet systemd files which means that the /etc/default/kubelet file was already created. --- .goreleaser.yml | 1 + build/images/node/Makefile | 18 +++++--- build/images/node/scripts/build.sh | 8 +++- cmd/cinder/app/create/cluster/cluster.go | 18 -------- cmd/cinder/app/create/node/node.go | 13 ------ internal/cinder/cluster/controlplane.go | 25 +++++++++++ internal/cinder/cluster/registry.go | 2 +- internal/cinder/cluster/worker.go | 19 +++++++++ internal/cinder/config/constants/constants.go | 42 +++++++++++-------- 9 files changed, 90 insertions(+), 56 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index d6e06f7..7664a92 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -43,6 +43,7 @@ builds: - -X "github.com/criticalstack/crit/internal/buildinfo.Date={{.Date}}" - -X "github.com/criticalstack/crit/internal/buildinfo.GitSHA={{.ShortCommit}}" - -X "github.com/criticalstack/crit/internal/buildinfo.Version={{.Tag}}" + - -X "github.com/criticalstack/crit/internal/cinder/config/constants.DefaultNodeImage=criticalstack/cinder:{{.Tag}}" archives: - replacements: darwin: Darwin diff --git a/build/images/node/Makefile b/build/images/node/Makefile index 761ff92..34987e4 100644 --- a/build/images/node/Makefile +++ b/build/images/node/Makefile @@ -15,20 +15,26 @@ $(CINDER): # Build cinder $(MAKE) -s -C ../../../ cinder BIN_DIR=$(PWD)/$(BIN_DIR) +# Versions are all pulled from this file, any version updates should by made in this file +CONST_FILE := ../../../internal/cinder/config/constants/constants.go +define readvar +$(or $(shell sed -nE 's/^.*$(1).*"(.*)"$$/\1/p' $(CONST_FILE)),$(error "Cannot find variable $(1)")) +endef + TAG ?= latest IMAGE ?= docker.io/criticalstack/cinder:$(TAG) BUILD_DIR := $(PWD)/build BUILD_CONTAINER_NAME := cinder-build -BASE_IMAGE := docker.io/kindest/base:v20200709-8b878ace -KUBERNETES_VERSION := 1.18.10 +BASE_IMAGE := $(call readvar,KindBaseImage) +KUBERNETES_VERSION := $(call readvar,KubernetesVersion) # charts CHARTS_DIR := $(PWD)/charts -CILIUM_VERSION := 1.8.5 -LOCAL_PATH_PROVISIONER_VERSION := 0.0.12 -MACHINE_API_VERSION := 1.0.6 -MACHINE_API_PROVIDER_DOCKER_VERSION := 1.0.7 +CILIUM_VERSION := $(call readvar,CiliumVersion) +LOCAL_PATH_PROVISIONER_VERSION := $(call readvar,LocalPathProvisionerVersion) +MACHINE_API_VERSION := $(call readvar,MachineAPIVersion) +MACHINE_API_PROVIDER_DOCKER_VERSION := $(call readvar,MachineAPIProviderDockerVersion) charts/*.tgz: mkdir -p $(CHARTS_DIR) diff --git a/build/images/node/scripts/build.sh b/build/images/node/scripts/build.sh index e85a043..4443e86 100755 --- a/build/images/node/scripts/build.sh +++ b/build/images/node/scripts/build.sh @@ -10,7 +10,11 @@ apt-add-repository "deb http://apt.kubernetes.io/ kubernetes-xenial main" DEBIAN_FRONTEND=noninteractive && clean-install kubectl=${KUBERNETES_VERSION}-00 curl -L https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubelet -o /usr/bin/kubelet chmod +x /usr/bin/kubelet -echo "KUBELET_EXTRA_ARGS=--fail-swap-on=false" >> /etc/default/kubelet + +# Kind has moved the kubelet systemd files into the base image, so this is no +# longer necessary +#echo "KUBELET_EXTRA_ARGS=--fail-swap-on=false" >> /etc/default/kubelet + curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash mkdir -p /cinder/manifests curl -L https://github.com/criticalstack/machine-api/releases/download/v${MACHINE_API_VERSION}/machine-api.yaml -o /cinder/manifests/machine-api.yaml @@ -29,3 +33,5 @@ mkdir -p /etc/cni/net.d mkdir -p /var/lib/crit mkdir -p /var/log/kubernetes chmod 700 /var/lib/etcd + +echo ${KUBERNETES_VERSION} > /cinder/kubernetes_version diff --git a/cmd/cinder/app/create/cluster/cluster.go b/cmd/cinder/app/create/cluster/cluster.go index e463a41..9fde418 100644 --- a/cmd/cinder/app/create/cluster/cluster.go +++ b/cmd/cinder/app/create/cluster/cluster.go @@ -17,8 +17,6 @@ import ( "github.com/criticalstack/crit/internal/cinder/config/constants" "github.com/criticalstack/crit/internal/cinder/feature" "github.com/criticalstack/crit/internal/cinder/utils" - critconfig "github.com/criticalstack/crit/internal/config" - yamlutil "github.com/criticalstack/crit/pkg/kubernetes/yaml" ) var opts struct { @@ -121,23 +119,7 @@ func NewCommand() *cobra.Command { if err := feature.MutableGates.SetFromMap(cfg.FeatureGates); err != nil { return err } - if cfg.ControlPlaneConfiguration == nil { - cfg.ControlPlaneConfiguration = &critconfig.ControlPlaneConfiguration{} - } - cfg.ControlPlaneConfiguration.ClusterName = opts.Name - cluster.SetControlPlaneConfigurationDefaults(cfg.ControlPlaneConfiguration) cfg.Files = append(cfg.Files, files...) - data, err := yamlutil.MarshalToYaml(cfg.ControlPlaneConfiguration, critconfig.SchemeGroupVersion) - if err != nil { - return err - } - cfg.Files = append(cfg.Files, config.File{ - Path: "/var/lib/crit/config.yaml", - Owner: "root:root", - Permissions: "0644", - Encoding: config.Base64, - Content: base64.StdEncoding.EncodeToString(data), - }) if feature.Gates.Enabled(feature.LocalRegistry) { cfg.RegistryMirrors[fmt.Sprintf("%s:%d", cfg.LocalRegistryName, cfg.LocalRegistryPort)] = fmt.Sprintf("http://%s:%d", cfg.LocalRegistryName, cfg.LocalRegistryPort) } diff --git a/cmd/cinder/app/create/node/node.go b/cmd/cinder/app/create/node/node.go index 09a0689..6028918 100644 --- a/cmd/cinder/app/create/node/node.go +++ b/cmd/cinder/app/create/node/node.go @@ -16,7 +16,6 @@ import ( "github.com/criticalstack/crit/internal/cinder/utils" critconfig "github.com/criticalstack/crit/internal/config" "github.com/criticalstack/crit/pkg/kubernetes/pki" - yamlutil "github.com/criticalstack/crit/pkg/kubernetes/yaml" ) var opts struct { @@ -82,18 +81,6 @@ func NewCommand() *cobra.Command { Encoding: config.Base64, Content: base64.StdEncoding.EncodeToString(data), }) - data, err = yamlutil.MarshalToYaml(cfg.WorkerConfiguration, critconfig.SchemeGroupVersion) - if err != nil { - return err - } - cfg.Files = append(cfg.Files, config.File{ - Path: "/var/lib/crit/config.yaml", - Owner: "root:root", - Permissions: "0644", - Encoding: config.Base64, - Content: base64.StdEncoding.EncodeToString(data), - }) - if feature.Gates.Enabled(feature.LocalRegistry) { cfg.RegistryMirrors[fmt.Sprintf("%s:%d", cfg.LocalRegistryName, cfg.LocalRegistryPort)] = fmt.Sprintf("http://%s:%d", cfg.LocalRegistryName, cfg.LocalRegistryPort) } diff --git a/internal/cinder/cluster/controlplane.go b/internal/cinder/cluster/controlplane.go index 4b18d16..3e78de8 100644 --- a/internal/cinder/cluster/controlplane.go +++ b/internal/cinder/cluster/controlplane.go @@ -2,7 +2,9 @@ package cluster import ( "context" + "encoding/base64" "os" + "strings" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -14,6 +16,7 @@ import ( "github.com/criticalstack/crit/internal/cinder/config/constants" "github.com/criticalstack/crit/internal/cinder/feature" critconfig "github.com/criticalstack/crit/internal/config" + yamlutil "github.com/criticalstack/crit/pkg/kubernetes/yaml" ) var ( @@ -50,6 +53,27 @@ func CreateControlPlaneNode(ctx context.Context, cfg *ControlPlaneConfig) (*Node node.Stdout = os.Stdout node.Stderr = os.Stderr } + if cfg.ClusterConfiguration.ControlPlaneConfiguration == nil { + cfg.ClusterConfiguration.ControlPlaneConfiguration = &critconfig.ControlPlaneConfiguration{} + } + cfg.ClusterConfiguration.ControlPlaneConfiguration.ClusterName = cfg.ClusterName + SetControlPlaneConfigurationDefaults(cfg.ClusterConfiguration.ControlPlaneConfiguration) + data, err := node.ReadFile("/cinder/kubernetes_version") + if err != nil { + return nil, err + } + cfg.ClusterConfiguration.ControlPlaneConfiguration.NodeConfiguration.KubernetesVersion = strings.TrimSpace(string(data)) + data, err = yamlutil.MarshalToYaml(cfg.ClusterConfiguration.ControlPlaneConfiguration, critconfig.SchemeGroupVersion) + if err != nil { + return nil, err + } + cfg.ClusterConfiguration.Files = append(cfg.ClusterConfiguration.Files, config.File{ + Path: "/var/lib/crit/config.yaml", + Owner: "root:root", + Permissions: "0644", + Encoding: config.Base64, + Content: base64.StdEncoding.EncodeToString(data), + }) if feature.Gates.Enabled(feature.FixCgroupMounts) { if err := node.Command("bash", "/cinder/scripts/fix-cgroup-mounts.sh").Run(); err != nil { return node, err @@ -70,6 +94,7 @@ func CreateControlPlaneNode(ctx context.Context, cfg *ControlPlaneConfig) (*Node } func SetControlPlaneConfigurationDefaults(cfg *critconfig.ControlPlaneConfiguration) { + // this will be overriden by the kubernetes version file in the image cfg.NodeConfiguration.KubernetesVersion = constants.KubernetesVersion if cfg.KubeAPIServerConfiguration.ExtraArgs == nil { cfg.KubeAPIServerConfiguration.ExtraArgs = make(map[string]string) diff --git a/internal/cinder/cluster/registry.go b/internal/cinder/cluster/registry.go index 8a5cbee..a51259f 100644 --- a/internal/cinder/cluster/registry.go +++ b/internal/cinder/cluster/registry.go @@ -22,7 +22,7 @@ func CreateRegistry(name string, port int) error { "--name", name, "-p", fmt.Sprintf("%d:5000", port), "--network", constants.DefaultNetwork, - fmt.Sprintf("docker.io/library/registry:%s", constants.DefaultRegistryVersion), + fmt.Sprintf("docker.io/library/registry:%s", constants.RegistryVersion), } providerName := "docker" if v, ok := os.LookupEnv("KIND_EXPERIMENTAL_PROVIDER"); ok { diff --git a/internal/cinder/cluster/worker.go b/internal/cinder/cluster/worker.go index f7720c0..14cf104 100644 --- a/internal/cinder/cluster/worker.go +++ b/internal/cinder/cluster/worker.go @@ -2,7 +2,9 @@ package cluster import ( "context" + "encoding/base64" "os" + "strings" "time" "github.com/pkg/errors" @@ -14,6 +16,7 @@ import ( "github.com/criticalstack/crit/internal/cinder/config" "github.com/criticalstack/crit/internal/cinder/config/constants" critconfig "github.com/criticalstack/crit/internal/config" + yamlutil "github.com/criticalstack/crit/pkg/kubernetes/yaml" ) type WorkerConfig struct { @@ -40,6 +43,22 @@ func CreateWorkerNode(ctx context.Context, cfg *WorkerConfig) (*Node, error) { node.Stdout = os.Stdout node.Stderr = os.Stderr } + data, err := node.ReadFile("/cinder/kubernetes_version") + if err != nil { + return nil, err + } + cfg.ClusterConfiguration.WorkerConfiguration.NodeConfiguration.KubernetesVersion = strings.TrimSpace(string(data)) + data, err = yamlutil.MarshalToYaml(cfg.ClusterConfiguration.WorkerConfiguration, critconfig.SchemeGroupVersion) + if err != nil { + return nil, err + } + cfg.ClusterConfiguration.Files = append(cfg.ClusterConfiguration.Files, config.File{ + Path: "/var/lib/crit/config.yaml", + Owner: "root:root", + Permissions: "0644", + Encoding: config.Base64, + Content: base64.StdEncoding.EncodeToString(data), + }) if err := node.Command("bash", "/cinder/scripts/fix-cgroup-mounts.sh").Run(); err != nil { return nil, err } diff --git a/internal/cinder/config/constants/constants.go b/internal/cinder/config/constants/constants.go index d8de7de..438d196 100644 --- a/internal/cinder/config/constants/constants.go +++ b/internal/cinder/config/constants/constants.go @@ -6,21 +6,29 @@ import ( "github.com/criticalstack/crit/pkg/config/constants" ) +var ( + DefaultNodeImage = "criticalstack/cinder:latest" +) + const ( - DefaultNodeImage = "criticalstack/cinder:v1.0.6" - DefaultNetwork = "cinder" + // the Kubernetes version to beused when building the cinder image KubernetesVersion = "1.18.10" - DefaultMachineAPIVersion = "1.0.6" - DefaultMachineAPIProviderDockerVersion = "1.0.7" - DefaultKubeRBACProxyVersion = "0.5.0" - DefaultCiliumVersion = "1.8.5" - DefaultCiliumStartupScriptVersion = "af2a99046eca96c0138551393b21a5c044c7fe79" - DefaultLocalPathProvisionerVersion = "0.0.12" - DefaultRegistryVersion = "2.7.1" + // Kind base image to use as the base for the cinder image + KindBaseImage = "docker.io/kindest/base:v20200928-02f74589" + DefaultNetwork = "cinder" DefaultLocalRegistryName = "cinderegg" DefaultLocalRegistryPort = 5000 + + // container image versions + CiliumStartupScriptVersion = "af2a99046eca96c0138551393b21a5c044c7fe79" + CiliumVersion = "1.8.5" + KubeRBACProxyVersion = "0.5.0" + LocalPathProvisionerVersion = "0.0.12" + MachineAPIProviderDockerVersion = "1.0.7" + MachineAPIVersion = "1.0.6" + RegistryVersion = "2.7.1" ) func GetImages() map[string]string { @@ -33,14 +41,14 @@ func GetImages() map[string]string { "coredns": fmt.Sprintf("%s:%s", constants.CoreDNSImage, constants.DefaultCoreDNSVersion), "bootstrap-server": fmt.Sprintf("%s:v%s", constants.CritBootstrapServerImage, constants.DefaultBootstrapServerVersion), "healthcheck-proxy": fmt.Sprintf("%s:v%s", constants.CritHealthCheckProxyImage, constants.DefaultHealthcheckProxyVersion), - "machine-api": fmt.Sprintf("docker.io/criticalstack/machine-api:v%s", DefaultMachineAPIVersion), - "machine-api-provider-docker": fmt.Sprintf("docker.io/criticalstack/machine-api-provider-docker:v%s", DefaultMachineAPIProviderDockerVersion), - "kube-rbac-proxy": fmt.Sprintf("gcr.io/kubebuilder/kube-rbac-proxy:v%s", DefaultKubeRBACProxyVersion), - "cilium": fmt.Sprintf("docker.io/cilium/cilium:v%s", DefaultCiliumVersion), - "cilium-operator-generic": fmt.Sprintf("docker.io/cilium/operator-generic:v%s", DefaultCiliumVersion), - "cilium-startup-script": fmt.Sprintf("docker.io/cilium/startup-script:%s", DefaultCiliumStartupScriptVersion), - "local-path-provisioner": fmt.Sprintf("docker.io/rancher/local-path-provisioner:v%s", DefaultLocalPathProvisionerVersion), - "registry": fmt.Sprintf("docker.io/library/registry:%s", DefaultRegistryVersion), + "machine-api": fmt.Sprintf("docker.io/criticalstack/machine-api:v%s", MachineAPIVersion), + "machine-api-provider-docker": fmt.Sprintf("docker.io/criticalstack/machine-api-provider-docker:v%s", MachineAPIProviderDockerVersion), + "kube-rbac-proxy": fmt.Sprintf("gcr.io/kubebuilder/kube-rbac-proxy:v%s", KubeRBACProxyVersion), + "cilium": fmt.Sprintf("docker.io/cilium/cilium:v%s", CiliumVersion), + "cilium-operator-generic": fmt.Sprintf("docker.io/cilium/operator-generic:v%s", CiliumVersion), + "cilium-startup-script": fmt.Sprintf("docker.io/cilium/startup-script:%s", CiliumStartupScriptVersion), + "local-path-provisioner": fmt.Sprintf("docker.io/rancher/local-path-provisioner:v%s", LocalPathProvisionerVersion), + "registry": fmt.Sprintf("docker.io/library/registry:%s", RegistryVersion), } return images }