Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor and fix System Upgrade Controller installation #343

Merged
merged 1 commit into from
Apr 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,8 @@ schedule_workloads_on_masters: false
# snapshot_os: microos # optional: specified the os type when using a custom snapshot
# cloud_controller_manager_manifest_url: "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.19.0/ccm-networks.yaml"
# csi_driver_manifest_url: "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.6.0/deploy/kubernetes/hcloud-csi.yml"
# system_upgrade_controller_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.4.0/system-upgrade-controller.yaml"
# system_upgrade_controller_deployment_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/system-upgrade-controller.yaml"
# system_upgrade_controller_crd_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml"
datastore:
mode: etcd # etcd (default) or external
external_datastore_endpoint: postgres://....
Expand Down
3 changes: 2 additions & 1 deletion cluster_config.yaml.example
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ disable_flannel: false # set to true if you want to install a different CNI
# snapshot_os: microos # optional: specified the os type when using a custom snapshot
# cloud_controller_manager_manifest_url: "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.19.0/ccm-networks.yaml"
# csi_driver_manifest_url: "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.6.0/deploy/kubernetes/hcloud-csi.yml"
# system_upgrade_controller_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.4.0/system-upgrade-controller.yaml"
# system_upgrade_controller_deployment_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/system-upgrade-controller.yaml"
# system_upgrade_controller_crd_manifest_urlL "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml"
datastore:
mode: etcd # etcd (default) or external
external_datastore_endpoint: postgres://....
Expand Down
3 changes: 2 additions & 1 deletion src/configuration/main.cr
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ class Configuration::Main
getter cluster_dns : String = "10.43.0.10"
getter cloud_controller_manager_manifest_url : String = "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.19.0/ccm-networks.yaml"
getter csi_driver_manifest_url : String = "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.6.0/deploy/kubernetes/hcloud-csi.yml"
getter system_upgrade_controller_manifest_url : String = "https://github.com/rancher/system-upgrade-controller/releases/download/v0.4.0/system-upgrade-controller.yaml"
getter system_upgrade_controller_deployment_manifest_url : String = "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/system-upgrade-controller.yaml"
getter system_upgrade_controller_crd_manifest_url : String = "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml"
getter disable_flannel : Bool = false
getter ssh_port : Int32 = 22
getter datastore : Configuration::Datastore = Configuration::Datastore.new
Expand Down
23 changes: 4 additions & 19 deletions src/kubernetes/installer.cr
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
require "crinja"
require "base64"
require "file_utils"

require "../util"
require "../util/ssh"
require "../util/shell"
require "../hetzner/server"
require "../hetzner/load_balancer"
require "../configuration/loader"
require "file_utils"
require "./software/system_upgrade_controller"

class Kubernetes::Installer
MASTER_INSTALL_SCRIPT = {{ read_file("#{__DIR__}/../../templates/master_install_script.sh") }}
Expand Down Expand Up @@ -46,7 +48,7 @@ class Kubernetes::Installer
create_hetzner_cloud_secret
deploy_cloud_controller_manager
deploy_csi_driver
deploy_system_upgrade_controller
Kubernetes::Software::SystemUpgradeController.new(configuration, settings).install
deploy_cluster_autoscaler unless autoscaling_worker_node_pools.size.zero?
end

Expand Down Expand Up @@ -296,23 +298,6 @@ class Kubernetes::Installer
puts "...CSI Driver deployed"
end

private def deploy_system_upgrade_controller
puts "\nDeploying k3s System Upgrade Controller..."

# Run second manifest twice to fix problem with namespace creation
command = "kubectl apply -f #{settings.system_upgrade_controller_manifest_url}"

result = Util::Shell.run(command, configuration.kubeconfig_path, settings.hetzner_token)

unless result.success?
puts "Failed to deploy k3s System Upgrade Controller:"
puts result.output
exit 1
end

puts "...k3s System Upgrade Controller deployed."
end

private def deploy_cluster_autoscaler
puts "\nDeploying Cluster Autoscaler..."

Expand Down
10 changes: 10 additions & 0 deletions src/kubernetes/resources/deployment.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
require "./deployment/spec"

module Kubernetes::Resources
class Deployment
include YAML::Serializable
include YAML::Serializable::Unmapped

property spec : Kubernetes::Resources::Deployment::Spec
end
end
12 changes: 12 additions & 0 deletions src/kubernetes/resources/deployment/spec.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
require "./spec/template"

module Kubernetes::Resources
class Deployment
class Spec
include YAML::Serializable
include YAML::Serializable::Unmapped

property template : Kubernetes::Resources::Deployment::Spec::Template
end
end
end
14 changes: 14 additions & 0 deletions src/kubernetes/resources/deployment/spec/template.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
require "../../pod/spec"

module Kubernetes::Resources
class Deployment
class Spec
class Template
include YAML::Serializable
include YAML::Serializable::Unmapped

property spec : Kubernetes::Resources::Pod::Spec
end
end
end
end
16 changes: 16 additions & 0 deletions src/kubernetes/resources/deployment/spec/template/spec.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
require "../../../pod/spec/toleration"

module Kubernetes::Resources
class Deployment
class Spec
class Template
class Spec
include YAML::Serializable
include YAML::Serializable::Unmapped

property tolerations : Array(Kubernetes::Resources::Pod::Spec::Toleration)?
end
end
end
end
end
12 changes: 12 additions & 0 deletions src/kubernetes/resources/manifest_sections/section_with_spec.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
module Kubernetes::Resources
class Deployment
class Spec
class Template
include YAML::Serializable
include YAML::Serializable::Unmapped

property spec : Kubernetes::Resources::Deployment::Spec::Template::Spec
end
end
end
end
10 changes: 10 additions & 0 deletions src/kubernetes/resources/pod.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
require "./pod/spec"

module Kubernetes::Resources
class Pod
include YAML::Serializable
include YAML::Serializable::Unmapped

property spec : Kubernetes::Resources::Pod::Spec
end
end
12 changes: 12 additions & 0 deletions src/kubernetes/resources/pod/spec.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
require "./spec/toleration"

module Kubernetes::Resources
class Pod
class Spec
include YAML::Serializable
include YAML::Serializable::Unmapped

property tolerations : Array(Kubernetes::Resources::Pod::Spec::Toleration)?
end
end
end
15 changes: 15 additions & 0 deletions src/kubernetes/resources/pod/spec/toleration.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
class Kubernetes::Resources::Pod
class Spec
class Toleration
include YAML::Serializable
include YAML::Serializable::Unmapped

property effect : String?
property key : String?
property value : String?

def initialize(@effect, @key, @value)
end
end
end
end
8 changes: 8 additions & 0 deletions src/kubernetes/resources/resource.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
module Kubernetes::Resources
class Resource
include YAML::Serializable
include YAML::Serializable::Unmapped

property kind : String
end
end
93 changes: 93 additions & 0 deletions src/kubernetes/software/system_upgrade_controller.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
require "../resources/resource"
require "../resources/deployment"
require "../resources/pod/spec/toleration"
require "../../configuration/loader"
require "../../configuration/main"

class Kubernetes::Software::SystemUpgradeController
getter configuration : Configuration::Loader
getter settings : Configuration::Main { configuration.settings }

def initialize(@configuration, @settings)

end

def install
puts "\n[System Upgrade Controller] Deploying k3s System Upgrade Controller..."

create_namespace
create_crd
create_resources

puts "[System Upgrade Controller] ...k3s System Upgrade Controller deployed."
end

private def create_namespace
run_command "kubectl create ns system-upgrade --dry-run=client -o yaml | kubectl apply -f -"
end

private def create_crd
run_command "kubectl apply -f #{settings.system_upgrade_controller_crd_manifest_url}"
end

private def create_resources
manifest = fetch_resources_manifest
resources = YAML.parse_all(manifest)
patched_resources = patch_resources(resources)
patched_manifest = patched_resources.map(&.to_yaml).join
updated_manifest_path = "/tmp/manifest.yaml"

File.write(updated_manifest_path, patched_manifest)

run_command "kubectl apply -f #{updated_manifest_path}"

File.delete(updated_manifest_path)
end

private def fetch_resources_manifest
response = Crest.get(settings.system_upgrade_controller_deployment_manifest_url)

unless response.success?
puts "[System Upgrade Controller] Failed to download System Upgrade Controller manifest from #{settings.system_upgrade_controller_deployment_manifest_url}"
puts "[System Upgrade Controller] Server responded with status #{response.status_code}"
exit 1
end

response.body.to_s
end

private def deployment_with_added_toleration(resource)
deployment = Kubernetes::Resources::Deployment.from_yaml(resource.to_yaml)
toleration = Kubernetes::Resources::Pod::Spec::Toleration.new(effect: "NoExecute", key: "CriticalAddonsOnly", value: "true")

if tolerations = deployment.spec.template.spec.tolerations
tolerations << toleration
else
deployment.spec.template.spec.tolerations = [toleration]
end

deployment
end

private def run_command(command)
result = Util::Shell.run(command, configuration.kubeconfig_path, settings.hetzner_token, prefix: "System Upgrade Controller")

unless result.success?
puts "[System Upgrade Controller] Failed to deploy k3s System Upgrade Controller:"
puts result.output
exit 1
end
end

private def patch_resources(resources)
resources.map do |resource|
resource = Kubernetes::Resources::Resource.from_yaml(resource.to_yaml)

if resource.kind == "Deployment"
deployment_with_added_toleration(resource)
else
resource
end
end
end
end
8 changes: 4 additions & 4 deletions src/util/prefixed_io.cr
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ class PrefixedIO < IO
end

def write(slice : Bytes) : Nil
@io.print @prefix
slice.each do |byte|
@io.write_byte byte
@io.print @prefix if 10 == byte
content = String.new(slice)
lines = content.lines
lines.each do |line|
@io << @prefix << "#{line}\n"
end
end
end
8 changes: 6 additions & 2 deletions src/util/shell.cr
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ module Util
end
end

def self.run(command : String, kubeconfig_path : String, hetzner_token : String) : Result
def self.run(command : String, kubeconfig_path : String, hetzner_token : String, prefix : String = "") : Result
cmd_file_path = "/tmp/cli.cmd"

File.write(cmd_file_path, <<-CONTENT
Expand All @@ -26,7 +26,11 @@ module Util
stdout = IO::Memory.new
stderr = IO::Memory.new

all_io_out = IO::MultiWriter.new(STDOUT, stdout)
all_io_out = if prefix.blank?
IO::MultiWriter.new(STDOUT, stdout)
else
IO::MultiWriter.new(PrefixedIO.new("[#{prefix}] ", STDOUT), stdout)
end
all_io_err = IO::MultiWriter.new(STDERR, stderr)

env = {
Expand Down
Loading