From 2ba7bfe1ab62818cc6bfb41cd0ea212d852b2b49 Mon Sep 17 00:00:00 2001 From: Vito Botta Date: Mon, 15 Apr 2024 00:17:42 +0000 Subject: [PATCH] Refactor and fix System Upgrade Controller installation --- README.md | 3 +- cluster_config.yaml.example | 3 +- src/configuration/main.cr | 3 +- src/kubernetes/installer.cr | 23 +---- src/kubernetes/resources/deployment.cr | 10 ++ src/kubernetes/resources/deployment/spec.cr | 12 +++ .../resources/deployment/spec/template.cr | 14 +++ .../deployment/spec/template/spec.cr | 16 ++++ .../manifest_sections/section_with_spec.cr | 12 +++ src/kubernetes/resources/pod.cr | 10 ++ src/kubernetes/resources/pod/spec.cr | 12 +++ .../resources/pod/spec/toleration.cr | 15 +++ src/kubernetes/resources/resource.cr | 8 ++ .../software/system_upgrade_controller.cr | 93 +++++++++++++++++++ src/util/prefixed_io.cr | 8 +- src/util/shell.cr | 8 +- 16 files changed, 222 insertions(+), 28 deletions(-) create mode 100644 src/kubernetes/resources/deployment.cr create mode 100644 src/kubernetes/resources/deployment/spec.cr create mode 100644 src/kubernetes/resources/deployment/spec/template.cr create mode 100644 src/kubernetes/resources/deployment/spec/template/spec.cr create mode 100644 src/kubernetes/resources/manifest_sections/section_with_spec.cr create mode 100644 src/kubernetes/resources/pod.cr create mode 100644 src/kubernetes/resources/pod/spec.cr create mode 100644 src/kubernetes/resources/pod/spec/toleration.cr create mode 100644 src/kubernetes/resources/resource.cr create mode 100644 src/kubernetes/software/system_upgrade_controller.cr diff --git a/README.md b/README.md index 9b0802c..d90b812 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,8 @@ schedule_workloads_on_masters: false # snapshot_os: microos # optional: specified the os type when using a custom snapshot # cloud_controller_manager_manifest_url: "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.19.0/ccm-networks.yaml" # csi_driver_manifest_url: "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.6.0/deploy/kubernetes/hcloud-csi.yml" -# system_upgrade_controller_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.4.0/system-upgrade-controller.yaml" +# system_upgrade_controller_deployment_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/system-upgrade-controller.yaml" +# system_upgrade_controller_crd_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml" datastore: mode: etcd # etcd (default) or external external_datastore_endpoint: postgres://.... diff --git a/cluster_config.yaml.example b/cluster_config.yaml.example index d550465..d3ca337 100644 --- a/cluster_config.yaml.example +++ b/cluster_config.yaml.example @@ -24,7 +24,8 @@ disable_flannel: false # set to true if you want to install a different CNI # snapshot_os: microos # optional: specified the os type when using a custom snapshot # cloud_controller_manager_manifest_url: "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.19.0/ccm-networks.yaml" # csi_driver_manifest_url: "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.6.0/deploy/kubernetes/hcloud-csi.yml" -# system_upgrade_controller_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.4.0/system-upgrade-controller.yaml" +# system_upgrade_controller_deployment_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/system-upgrade-controller.yaml" +# system_upgrade_controller_crd_manifest_urlL "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml" datastore: mode: etcd # etcd (default) or external external_datastore_endpoint: postgres://.... diff --git a/src/configuration/main.cr b/src/configuration/main.cr index 9f73103..d6bf7d8 100644 --- a/src/configuration/main.cr +++ b/src/configuration/main.cr @@ -41,7 +41,8 @@ class Configuration::Main getter cluster_dns : String = "10.43.0.10" getter cloud_controller_manager_manifest_url : String = "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.19.0/ccm-networks.yaml" getter csi_driver_manifest_url : String = "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.6.0/deploy/kubernetes/hcloud-csi.yml" - getter system_upgrade_controller_manifest_url : String = "https://github.com/rancher/system-upgrade-controller/releases/download/v0.4.0/system-upgrade-controller.yaml" + getter system_upgrade_controller_deployment_manifest_url : String = "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/system-upgrade-controller.yaml" + getter system_upgrade_controller_crd_manifest_url : String = "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml" getter disable_flannel : Bool = false getter ssh_port : Int32 = 22 getter datastore : Configuration::Datastore = Configuration::Datastore.new diff --git a/src/kubernetes/installer.cr b/src/kubernetes/installer.cr index 59e58fe..1613d77 100644 --- a/src/kubernetes/installer.cr +++ b/src/kubernetes/installer.cr @@ -1,12 +1,14 @@ require "crinja" require "base64" +require "file_utils" + require "../util" require "../util/ssh" require "../util/shell" require "../hetzner/server" require "../hetzner/load_balancer" require "../configuration/loader" -require "file_utils" +require "./software/system_upgrade_controller" class Kubernetes::Installer MASTER_INSTALL_SCRIPT = {{ read_file("#{__DIR__}/../../templates/master_install_script.sh") }} @@ -46,7 +48,7 @@ class Kubernetes::Installer create_hetzner_cloud_secret deploy_cloud_controller_manager deploy_csi_driver - deploy_system_upgrade_controller + Kubernetes::Software::SystemUpgradeController.new(configuration, settings).install deploy_cluster_autoscaler unless autoscaling_worker_node_pools.size.zero? end @@ -296,23 +298,6 @@ class Kubernetes::Installer puts "...CSI Driver deployed" end - private def deploy_system_upgrade_controller - puts "\nDeploying k3s System Upgrade Controller..." - - # Run second manifest twice to fix problem with namespace creation - command = "kubectl apply -f #{settings.system_upgrade_controller_manifest_url}" - - result = Util::Shell.run(command, configuration.kubeconfig_path, settings.hetzner_token) - - unless result.success? - puts "Failed to deploy k3s System Upgrade Controller:" - puts result.output - exit 1 - end - - puts "...k3s System Upgrade Controller deployed." - end - private def deploy_cluster_autoscaler puts "\nDeploying Cluster Autoscaler..." diff --git a/src/kubernetes/resources/deployment.cr b/src/kubernetes/resources/deployment.cr new file mode 100644 index 0000000..730a04b --- /dev/null +++ b/src/kubernetes/resources/deployment.cr @@ -0,0 +1,10 @@ +require "./deployment/spec" + +module Kubernetes::Resources + class Deployment + include YAML::Serializable + include YAML::Serializable::Unmapped + + property spec : Kubernetes::Resources::Deployment::Spec + end +end diff --git a/src/kubernetes/resources/deployment/spec.cr b/src/kubernetes/resources/deployment/spec.cr new file mode 100644 index 0000000..6ae160e --- /dev/null +++ b/src/kubernetes/resources/deployment/spec.cr @@ -0,0 +1,12 @@ +require "./spec/template" + +module Kubernetes::Resources + class Deployment + class Spec + include YAML::Serializable + include YAML::Serializable::Unmapped + + property template : Kubernetes::Resources::Deployment::Spec::Template + end + end +end diff --git a/src/kubernetes/resources/deployment/spec/template.cr b/src/kubernetes/resources/deployment/spec/template.cr new file mode 100644 index 0000000..3876feb --- /dev/null +++ b/src/kubernetes/resources/deployment/spec/template.cr @@ -0,0 +1,14 @@ +require "../../pod/spec" + +module Kubernetes::Resources + class Deployment + class Spec + class Template + include YAML::Serializable + include YAML::Serializable::Unmapped + + property spec : Kubernetes::Resources::Pod::Spec + end + end + end +end diff --git a/src/kubernetes/resources/deployment/spec/template/spec.cr b/src/kubernetes/resources/deployment/spec/template/spec.cr new file mode 100644 index 0000000..ab2ad26 --- /dev/null +++ b/src/kubernetes/resources/deployment/spec/template/spec.cr @@ -0,0 +1,16 @@ +require "../../../pod/spec/toleration" + +module Kubernetes::Resources + class Deployment + class Spec + class Template + class Spec + include YAML::Serializable + include YAML::Serializable::Unmapped + + property tolerations : Array(Kubernetes::Resources::Pod::Spec::Toleration)? + end + end + end + end +end diff --git a/src/kubernetes/resources/manifest_sections/section_with_spec.cr b/src/kubernetes/resources/manifest_sections/section_with_spec.cr new file mode 100644 index 0000000..f30cb98 --- /dev/null +++ b/src/kubernetes/resources/manifest_sections/section_with_spec.cr @@ -0,0 +1,12 @@ +module Kubernetes::Resources + class Deployment + class Spec + class Template + include YAML::Serializable + include YAML::Serializable::Unmapped + + property spec : Kubernetes::Resources::Deployment::Spec::Template::Spec + end + end + end +end diff --git a/src/kubernetes/resources/pod.cr b/src/kubernetes/resources/pod.cr new file mode 100644 index 0000000..b9bc7ed --- /dev/null +++ b/src/kubernetes/resources/pod.cr @@ -0,0 +1,10 @@ +require "./pod/spec" + +module Kubernetes::Resources + class Pod + include YAML::Serializable + include YAML::Serializable::Unmapped + + property spec : Kubernetes::Resources::Pod::Spec + end +end diff --git a/src/kubernetes/resources/pod/spec.cr b/src/kubernetes/resources/pod/spec.cr new file mode 100644 index 0000000..2ec5526 --- /dev/null +++ b/src/kubernetes/resources/pod/spec.cr @@ -0,0 +1,12 @@ +require "./spec/toleration" + +module Kubernetes::Resources + class Pod + class Spec + include YAML::Serializable + include YAML::Serializable::Unmapped + + property tolerations : Array(Kubernetes::Resources::Pod::Spec::Toleration)? + end + end +end diff --git a/src/kubernetes/resources/pod/spec/toleration.cr b/src/kubernetes/resources/pod/spec/toleration.cr new file mode 100644 index 0000000..c092d4e --- /dev/null +++ b/src/kubernetes/resources/pod/spec/toleration.cr @@ -0,0 +1,15 @@ +class Kubernetes::Resources::Pod + class Spec + class Toleration + include YAML::Serializable + include YAML::Serializable::Unmapped + + property effect : String? + property key : String? + property value : String? + + def initialize(@effect, @key, @value) + end + end + end +end diff --git a/src/kubernetes/resources/resource.cr b/src/kubernetes/resources/resource.cr new file mode 100644 index 0000000..1d38547 --- /dev/null +++ b/src/kubernetes/resources/resource.cr @@ -0,0 +1,8 @@ +module Kubernetes::Resources + class Resource + include YAML::Serializable + include YAML::Serializable::Unmapped + + property kind : String + end +end diff --git a/src/kubernetes/software/system_upgrade_controller.cr b/src/kubernetes/software/system_upgrade_controller.cr new file mode 100644 index 0000000..3092126 --- /dev/null +++ b/src/kubernetes/software/system_upgrade_controller.cr @@ -0,0 +1,93 @@ +require "../resources/resource" +require "../resources/deployment" +require "../resources/pod/spec/toleration" +require "../../configuration/loader" +require "../../configuration/main" + +class Kubernetes::Software::SystemUpgradeController + getter configuration : Configuration::Loader + getter settings : Configuration::Main { configuration.settings } + + def initialize(@configuration, @settings) + + end + + def install + puts "\n[System Upgrade Controller] Deploying k3s System Upgrade Controller..." + + create_namespace + create_crd + create_resources + + puts "[System Upgrade Controller] ...k3s System Upgrade Controller deployed." + end + + private def create_namespace + run_command "kubectl create ns system-upgrade --dry-run=client -o yaml | kubectl apply -f -" + end + + private def create_crd + run_command "kubectl apply -f #{settings.system_upgrade_controller_crd_manifest_url}" + end + + private def create_resources + manifest = fetch_resources_manifest + resources = YAML.parse_all(manifest) + patched_resources = patch_resources(resources) + patched_manifest = patched_resources.map(&.to_yaml).join + updated_manifest_path = "/tmp/manifest.yaml" + + File.write(updated_manifest_path, patched_manifest) + + run_command "kubectl apply -f #{updated_manifest_path}" + + File.delete(updated_manifest_path) + end + + private def fetch_resources_manifest + response = Crest.get(settings.system_upgrade_controller_deployment_manifest_url) + + unless response.success? + puts "[System Upgrade Controller] Failed to download System Upgrade Controller manifest from #{settings.system_upgrade_controller_deployment_manifest_url}" + puts "[System Upgrade Controller] Server responded with status #{response.status_code}" + exit 1 + end + + response.body.to_s + end + + private def deployment_with_added_toleration(resource) + deployment = Kubernetes::Resources::Deployment.from_yaml(resource.to_yaml) + toleration = Kubernetes::Resources::Pod::Spec::Toleration.new(effect: "NoExecute", key: "CriticalAddonsOnly", value: "true") + + if tolerations = deployment.spec.template.spec.tolerations + tolerations << toleration + else + deployment.spec.template.spec.tolerations = [toleration] + end + + deployment + end + + private def run_command(command) + result = Util::Shell.run(command, configuration.kubeconfig_path, settings.hetzner_token, prefix: "System Upgrade Controller") + + unless result.success? + puts "[System Upgrade Controller] Failed to deploy k3s System Upgrade Controller:" + puts result.output + exit 1 + end + end + + private def patch_resources(resources) + resources.map do |resource| + resource = Kubernetes::Resources::Resource.from_yaml(resource.to_yaml) + + if resource.kind == "Deployment" + deployment_with_added_toleration(resource) + else + resource + end + end + end +end diff --git a/src/util/prefixed_io.cr b/src/util/prefixed_io.cr index 10dfb6d..78142fa 100644 --- a/src/util/prefixed_io.cr +++ b/src/util/prefixed_io.cr @@ -6,10 +6,10 @@ class PrefixedIO < IO end def write(slice : Bytes) : Nil - @io.print @prefix - slice.each do |byte| - @io.write_byte byte - @io.print @prefix if 10 == byte + content = String.new(slice) + lines = content.lines + lines.each do |line| + @io << @prefix << "#{line}\n" end end end diff --git a/src/util/shell.cr b/src/util/shell.cr index 09b514a..2b26d9b 100644 --- a/src/util/shell.cr +++ b/src/util/shell.cr @@ -12,7 +12,7 @@ module Util end end - def self.run(command : String, kubeconfig_path : String, hetzner_token : String) : Result + def self.run(command : String, kubeconfig_path : String, hetzner_token : String, prefix : String = "") : Result cmd_file_path = "/tmp/cli.cmd" File.write(cmd_file_path, <<-CONTENT @@ -26,7 +26,11 @@ module Util stdout = IO::Memory.new stderr = IO::Memory.new - all_io_out = IO::MultiWriter.new(STDOUT, stdout) + all_io_out = if prefix.blank? + IO::MultiWriter.new(STDOUT, stdout) + else + IO::MultiWriter.new(PrefixedIO.new("[#{prefix}] ", STDOUT), stdout) + end all_io_err = IO::MultiWriter.new(STDERR, stderr) env = {