From 539d6c868bdeb214f3d52108522085ec804d97e4 Mon Sep 17 00:00:00 2001
From: Kenneth Bennett <kennybll@outlook.com>
Date: Mon, 21 Oct 2024 12:52:09 -0400
Subject: [PATCH 01/13] add support for etcd backups with s3 storage

---
 README.md                               |  1 +
 docs/Creating_a_cluster.md              | 12 ++++++++++++
 docs/S3 backups.md                      |  5 +++++
 src/configuration/datastore.cr          |  3 +++
 src/configuration/s3.cr                 | 20 ++++++++++++++++++++
 src/configuration/settings/datastore.cr |  4 ++++
 src/configuration/settings/s3.cr        | 16 ++++++++++++++++
 src/kubernetes/installer.cr             | 19 +++++++++++++++++++
 src/kubernetes/util.cr                  |  4 ++++
 templates/master_install_script.sh      |  2 +-
 10 files changed, 85 insertions(+), 1 deletion(-)
 create mode 100644 docs/S3 backups.md
 create mode 100644 src/configuration/s3.cr
 create mode 100644 src/configuration/settings/s3.cr

diff --git a/README.md b/README.md
index 08f83602..72593b08 100644
--- a/README.md
+++ b/README.md
@@ -57,6 +57,7 @@ See my public profile with links for connecting with me [here](https://vitobotta
 - [Load balancers](docs/Load_balancers.md)
 - [Storage](docs/Storage.md)
 - [Troubleshooting](docs/Troubleshooting.md)
+- [S3 Backups](docs/S3%20backups.md)
 - [Contributing and support](docs/Contributing_and_support.md)
 
 ___
diff --git a/docs/Creating_a_cluster.md b/docs/Creating_a_cluster.md
index cb6c716c..08089694 100644
--- a/docs/Creating_a_cluster.md
+++ b/docs/Creating_a_cluster.md
@@ -47,6 +47,18 @@ networking:
 datastore:
   mode: etcd # etcd (default) or external
   external_datastore_endpoint: postgres://....
+  # s3: # can only be enabled for etcd mode
+  #   enabled: false
+  #   endpoint: "s3.amazonaws.com" # optional defaults to s3.amazonaws.com
+  #   endpoint_ca: "" # optional
+  #   skip_ssl_verify: false # optional defaults to false
+  #   access_key: ""
+  #   secret_key: ""
+  #   bucket: ""
+  #   region: "us-east-1" # optional defaults to us-east-1
+  #   folder: "" # optional
+  #   insecure: false # optional defaults to false
+  #   timeout: "5m0s" # optional defaults to 5m0s
 
 schedule_workloads_on_masters: false
 
diff --git a/docs/S3 backups.md b/docs/S3 backups.md
new file mode 100644
index 00000000..074d0d62
--- /dev/null
+++ b/docs/S3 backups.md	
@@ -0,0 +1,5 @@
+### S3 Backups
+
+S3 backups can be enabled for the embedded ectd mode only. You can see the explainations for each option in the [K3s docs](https://docs.k3s.io/cli/etcd-snapshot).
+
+> "In addition to backing up the datastore itself, you must also back up the server token file at /var/lib/rancher/k3s/server/token. You must restore this file, or pass its value into the --token option, when restoring from backup. If you do not use the same token value when restoring, the snapshot will be unusable, as the token is used to encrypt confidential data within the datastore itself." [K3S Backup/Restore Docs](https://docs.k3s.io/datastore/backup-restore)
diff --git a/src/configuration/datastore.cr b/src/configuration/datastore.cr
index 4d28fe20..4a5f6edd 100644
--- a/src/configuration/datastore.cr
+++ b/src/configuration/datastore.cr
@@ -1,4 +1,5 @@
 require "yaml"
+require "./s3"
 
 class Configuration::Datastore
   include YAML::Serializable
@@ -6,6 +7,8 @@ class Configuration::Datastore
   getter mode : String = "etcd"
   getter external_datastore_endpoint : String = ""
 
+  getter s3 : Configuration::S3 = Configuration::S3.new
+
   def initialize(@mode : String = "etcd", @external_datastore_endpoint : String = "")
   end
 end
diff --git a/src/configuration/s3.cr b/src/configuration/s3.cr
new file mode 100644
index 00000000..f3e357ec
--- /dev/null
+++ b/src/configuration/s3.cr
@@ -0,0 +1,20 @@
+require "yaml"
+
+class Configuration::S3
+  include YAML::Serializable
+
+  getter enabled : Bool = false
+  getter endpoint : String?
+  getter endpoint_ca : String?
+  getter skip_ssl_verify : Bool = false
+  getter access_key : String = ""
+  getter secret_key : String = ""
+  getter bucket : String = ""
+  getter region : String?
+  getter folder : String?
+  getter insecure : Bool = false
+  getter timeout : String?
+
+  def initialize()
+  end
+end
diff --git a/src/configuration/settings/datastore.cr b/src/configuration/settings/datastore.cr
index e199fba7..17843f16 100644
--- a/src/configuration/settings/datastore.cr
+++ b/src/configuration/settings/datastore.cr
@@ -1,3 +1,5 @@
+require "./s3"
+
 class Configuration::Settings::Datastore
   getter errors : Array(String)
   getter datastore : Configuration::Datastore
@@ -8,8 +10,10 @@ class Configuration::Settings::Datastore
   def validate
     case datastore.mode
     when "etcd"
+      Settings::S3.new(errors, datastore.s3).validate
     when "external"
       errors << "external_datastore_endpoint is required for external datastore" if datastore.external_datastore_endpoint.strip.empty?
+      errors << "s3 backups is only availabe on 'etcd' datastore" if datastore.s3.enabled
     else
       errors << "datastore mode is invalid - allowed values are 'etcd' and 'external'"
     end
diff --git a/src/configuration/settings/s3.cr b/src/configuration/settings/s3.cr
new file mode 100644
index 00000000..c3954f50
--- /dev/null
+++ b/src/configuration/settings/s3.cr
@@ -0,0 +1,16 @@
+class Configuration::Settings::S3
+  getter errors : Array(String)
+  getter s3 : Configuration::S3
+
+  def initialize(@errors, @s3)
+  end
+
+  def validate
+    case s3.enabled
+    when true
+      errors << "access_key is required for S3 backups" if s3.access_key.strip.empty?
+      errors << "secret_key is required for S3 backups" if s3.secret_key.strip.empty?
+      errors << "bucket is required for S3 backups" if s3.bucket.strip.empty?
+    end
+  end
+end
diff --git a/src/kubernetes/installer.cr b/src/kubernetes/installer.cr
index 53241ad4..4275e8b4 100644
--- a/src/kubernetes/installer.cr
+++ b/src/kubernetes/installer.cr
@@ -183,6 +183,7 @@ class Kubernetes::Installer
       cluster_dns: settings.networking.cluster_dns,
       datastore_endpoint: datastore_endpoint,
       etcd_arguments: etcd_arguments,
+      s3_arguments: generate_s3_arguments(),
       embedded_registry_mirror_enabled: settings.embedded_registry_mirror.enabled.to_s,
     })
   end
@@ -388,4 +389,22 @@ class Kubernetes::Installer
       first_master.private_ip_address
     end
   end
+
+  private def generate_s3_arguments
+    opts = [] of String
+
+    opts << "--etcd-s3" if settings.datastore.s3.enabled
+    opts << "--etcd-s3-endpoint=#{settings.datastore.s3.endpoint}" if present?(settings.datastore.s3.endpoint)
+    opts << "--etcd-s3-endpoint-ca=#{settings.datastore.s3.endpoint_ca}" if present?(settings.datastore.s3.endpoint_ca)
+    opts << "--etcd-s3-skip-ssl-verify" if settings.datastore.s3.skip_ssl_verify
+    opts << "--etcd-s3-access-key=#{settings.datastore.s3.access_key}" if present?(settings.datastore.s3.access_key)
+    opts << "--etcd-s3-secret-key=#{settings.datastore.s3.secret_key}" if present?(settings.datastore.s3.secret_key)
+    opts << "--etcd-s3-bucket=#{settings.datastore.s3.bucket}" if present?(settings.datastore.s3.bucket)
+    opts << "--etcd-s3-region=#{settings.datastore.s3.region}" if present?(settings.datastore.s3.region)
+    opts << "--etcd-s3-folder=#{settings.datastore.s3.folder}" if present?(settings.datastore.s3.folder)
+    opts << "--etcd-s3-insecure" if settings.datastore.s3.insecure
+    opts << "--etcd-s3-timeout=#{settings.datastore.s3.timeout}" if present?(settings.datastore.s3.timeout)
+
+    opts.uniq.sort.join(" ")
+  end
 end
diff --git a/src/kubernetes/util.cr b/src/kubernetes/util.cr
index 4ac020b9..bc84efd0 100644
--- a/src/kubernetes/util.cr
+++ b/src/kubernetes/util.cr
@@ -83,4 +83,8 @@ module Kubernetes::Util
 
     port_open?(ip_address, port, timeout = 1.0)
   end
+
+  private def present?(value : String?)
+    value.try(&.strip).try(&.empty?) == false
+  end
 end
diff --git a/templates/master_install_script.sh b/templates/master_install_script.sh
index 76a592d8..47662a42 100644
--- a/templates/master_install_script.sh
+++ b/templates/master_install_script.sh
@@ -65,7 +65,7 @@ curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="{{ k3s_version }}" K3S_TOKEN
 --kube-controller-manager-arg="bind-address=0.0.0.0" \
 --kube-proxy-arg="metrics-bind-address=0.0.0.0" \
 --kube-scheduler-arg="bind-address=0.0.0.0" \
-{{ taint }} {{ extra_args }} {{ etcd_arguments }} $FLANNEL_SETTINGS $EMBEDDED_REGISTRY_MIRROR \
+{{ taint }} {{ extra_args }} {{ etcd_arguments }} {{ s3_arguments }} $FLANNEL_SETTINGS $EMBEDDED_REGISTRY_MIRROR \
 --advertise-address=$PRIVATE_IP \
 --node-ip=$PRIVATE_IP \
 --node-external-ip=$PUBLIC_IP \

From 1f382fc436725cca2cd234cbdcb635eca882d7d5 Mon Sep 17 00:00:00 2001
From: Kenneth Bennett <kennybll@outlook.com>
Date: Sat, 26 Oct 2024 19:29:36 -0400
Subject: [PATCH 02/13] refactor backups, include regular back up options

also make clear the s3 options are only available for the etcd option
---
 README.md                                  |  1 +
 docs/Creating_a_cluster.md                 | 29 +++++++++------
 docs/{S3 backups.md => etcd S3 backups.md} |  4 +-
 src/configuration/datastore.cr             | 43 +++++++++++++++++++++-
 src/configuration/s3.cr                    | 20 ----------
 src/configuration/settings/datastore.cr    | 11 ++++--
 src/configuration/settings/s3.cr           | 16 --------
 src/kubernetes/installer.cr                | 37 ++++++++++++-------
 templates/master_install_script.sh         |  2 +-
 9 files changed, 93 insertions(+), 70 deletions(-)
 rename docs/{S3 backups.md => etcd S3 backups.md} (87%)
 delete mode 100644 src/configuration/s3.cr
 delete mode 100644 src/configuration/settings/s3.cr

diff --git a/README.md b/README.md
index 72593b08..a718c570 100644
--- a/README.md
+++ b/README.md
@@ -58,6 +58,7 @@ See my public profile with links for connecting with me [here](https://vitobotta
 - [Storage](docs/Storage.md)
 - [Troubleshooting](docs/Troubleshooting.md)
 - [S3 Backups](docs/S3%20backups.md)
+- [etcd S3 Backups](docs/etcd%20S3%20backups.md)
 - [Contributing and support](docs/Contributing_and_support.md)
 
 ___
diff --git a/docs/Creating_a_cluster.md b/docs/Creating_a_cluster.md
index 08089694..e21621bd 100644
--- a/docs/Creating_a_cluster.md
+++ b/docs/Creating_a_cluster.md
@@ -47,18 +47,23 @@ networking:
 datastore:
   mode: etcd # etcd (default) or external
   external_datastore_endpoint: postgres://....
-  # s3: # can only be enabled for etcd mode
-  #   enabled: false
-  #   endpoint: "s3.amazonaws.com" # optional defaults to s3.amazonaws.com
-  #   endpoint_ca: "" # optional
-  #   skip_ssl_verify: false # optional defaults to false
-  #   access_key: ""
-  #   secret_key: ""
-  #   bucket: ""
-  #   region: "us-east-1" # optional defaults to us-east-1
-  #   folder: "" # optional
-  #   insecure: false # optional defaults to false
-  #   timeout: "5m0s" # optional defaults to 5m0s
+  # etcd: # optional
+  #   backups: # optional
+  #     enabled: true # optional
+  #     retention: 5 # optional
+  #     dir: ${data-dir}/db/snapshots # optional
+  #     s3: # optional - can only be enabled for etcd mode
+  #       enabled: true
+  #       endpoint: "s3.amazonaws.com" # optional
+  #       endpoint_ca: "" # optional
+  #       skip_ssl_verify: false # optional
+  #       access_key: ""
+  #       secret_key: ""
+  #       bucket: ""
+  #       region: "us-east-1" # optional
+  #       folder: "" # optional
+  #       insecure: false # optional
+  #       timeout: "5m0s" # optional
 
 schedule_workloads_on_masters: false
 
diff --git a/docs/S3 backups.md b/docs/etcd S3 backups.md
similarity index 87%
rename from docs/S3 backups.md
rename to docs/etcd S3 backups.md
index 074d0d62..a5d7ff79 100644
--- a/docs/S3 backups.md	
+++ b/docs/etcd S3 backups.md	
@@ -1,5 +1,5 @@
-### S3 Backups
+### etcd S3 Backups
 
-S3 backups can be enabled for the embedded ectd mode only. You can see the explainations for each option in the [K3s docs](https://docs.k3s.io/cli/etcd-snapshot).
+S3 backups can be enabled for the embedded etcd mode only. You can see the explainations for each option in the [K3s docs](https://docs.k3s.io/cli/etcd-snapshot).
 
 > "In addition to backing up the datastore itself, you must also back up the server token file at /var/lib/rancher/k3s/server/token. You must restore this file, or pass its value into the --token option, when restoring from backup. If you do not use the same token value when restoring, the snapshot will be unusable, as the token is used to encrypt confidential data within the datastore itself." [K3S Backup/Restore Docs](https://docs.k3s.io/datastore/backup-restore)
diff --git a/src/configuration/datastore.cr b/src/configuration/datastore.cr
index 4a5f6edd..5abf2389 100644
--- a/src/configuration/datastore.cr
+++ b/src/configuration/datastore.cr
@@ -1,5 +1,4 @@
 require "yaml"
-require "./s3"
 
 class Configuration::Datastore
   include YAML::Serializable
@@ -7,8 +6,48 @@ class Configuration::Datastore
   getter mode : String = "etcd"
   getter external_datastore_endpoint : String = ""
 
-  getter s3 : Configuration::S3 = Configuration::S3.new
+  getter etcd : Etcd = Etcd.new
 
   def initialize(@mode : String = "etcd", @external_datastore_endpoint : String = "")
   end
+
+  class Etcd
+    include YAML::Serializable
+
+    getter backups : Backups = Backups.new
+
+    def initialize(@backups : Backups = Backups.new)
+    end
+
+    class Backups
+      include YAML::Serializable
+
+      getter enabled : Bool = true
+      getter retention : Int32?
+      getter dir : String?
+      getter s3 : S3 = S3.new
+
+      def initialize(@enabled : Bool = true, @s3 : S3 = S3.new)
+      end
+
+      class S3
+        include YAML::Serializable
+
+        getter enabled : Bool = false
+        getter endpoint : String?
+        getter endpoint_ca : String?
+        getter skip_ssl_verify : Bool = false
+        getter access_key : String = ""
+        getter secret_key : String = ""
+        getter bucket : String = ""
+        getter region : String?
+        getter folder : String?
+        getter insecure : Bool = false
+        getter timeout : String?
+
+        def initialize(@enabled : Bool = false, @skip_ssl_verify : Bool = false, @access_key : String = "", @secret_key : String = "", @bucket : String = "", @insecure : Bool = false)
+        end
+      end
+    end
+  end
 end
diff --git a/src/configuration/s3.cr b/src/configuration/s3.cr
deleted file mode 100644
index f3e357ec..00000000
--- a/src/configuration/s3.cr
+++ /dev/null
@@ -1,20 +0,0 @@
-require "yaml"
-
-class Configuration::S3
-  include YAML::Serializable
-
-  getter enabled : Bool = false
-  getter endpoint : String?
-  getter endpoint_ca : String?
-  getter skip_ssl_verify : Bool = false
-  getter access_key : String = ""
-  getter secret_key : String = ""
-  getter bucket : String = ""
-  getter region : String?
-  getter folder : String?
-  getter insecure : Bool = false
-  getter timeout : String?
-
-  def initialize()
-  end
-end
diff --git a/src/configuration/settings/datastore.cr b/src/configuration/settings/datastore.cr
index 17843f16..8ae63077 100644
--- a/src/configuration/settings/datastore.cr
+++ b/src/configuration/settings/datastore.cr
@@ -1,5 +1,3 @@
-require "./s3"
-
 class Configuration::Settings::Datastore
   getter errors : Array(String)
   getter datastore : Configuration::Datastore
@@ -10,10 +8,15 @@ class Configuration::Settings::Datastore
   def validate
     case datastore.mode
     when "etcd"
-      Settings::S3.new(errors, datastore.s3).validate
+      return unless datastore.etcd.backups.s3.enabled
+
+      s3 = datastore.etcd.backups.s3
+      errors << "access_key is required for S3 backups" if s3.access_key.strip.empty?
+      errors << "secret_key is required for S3 backups" if s3.secret_key.strip.empty?
+      errors << "bucket is required for S3 backups" if s3.bucket.strip.empty?
     when "external"
       errors << "external_datastore_endpoint is required for external datastore" if datastore.external_datastore_endpoint.strip.empty?
-      errors << "s3 backups is only availabe on 'etcd' datastore" if datastore.s3.enabled
+      errors << "etcd options cannot be set for external datastore" if datastore.etcd
     else
       errors << "datastore mode is invalid - allowed values are 'etcd' and 'external'"
     end
diff --git a/src/configuration/settings/s3.cr b/src/configuration/settings/s3.cr
deleted file mode 100644
index c3954f50..00000000
--- a/src/configuration/settings/s3.cr
+++ /dev/null
@@ -1,16 +0,0 @@
-class Configuration::Settings::S3
-  getter errors : Array(String)
-  getter s3 : Configuration::S3
-
-  def initialize(@errors, @s3)
-  end
-
-  def validate
-    case s3.enabled
-    when true
-      errors << "access_key is required for S3 backups" if s3.access_key.strip.empty?
-      errors << "secret_key is required for S3 backups" if s3.secret_key.strip.empty?
-      errors << "bucket is required for S3 backups" if s3.bucket.strip.empty?
-    end
-  end
-end
diff --git a/src/kubernetes/installer.cr b/src/kubernetes/installer.cr
index 4275e8b4..c7fbb178 100644
--- a/src/kubernetes/installer.cr
+++ b/src/kubernetes/installer.cr
@@ -183,7 +183,7 @@ class Kubernetes::Installer
       cluster_dns: settings.networking.cluster_dns,
       datastore_endpoint: datastore_endpoint,
       etcd_arguments: etcd_arguments,
-      s3_arguments: generate_s3_arguments(),
+      etcd_backup_settings: etcd_backup_settings,
       embedded_registry_mirror_enabled: settings.embedded_registry_mirror.enabled.to_s,
     })
   end
@@ -390,20 +390,31 @@ class Kubernetes::Installer
     end
   end
 
-  private def generate_s3_arguments
+  private def etcd_backup_settings
+    if !settings.datastore.etcd.backups.enabled
+      return "--etcd-disable-snapshots"
+    end
+
     opts = [] of String
 
-    opts << "--etcd-s3" if settings.datastore.s3.enabled
-    opts << "--etcd-s3-endpoint=#{settings.datastore.s3.endpoint}" if present?(settings.datastore.s3.endpoint)
-    opts << "--etcd-s3-endpoint-ca=#{settings.datastore.s3.endpoint_ca}" if present?(settings.datastore.s3.endpoint_ca)
-    opts << "--etcd-s3-skip-ssl-verify" if settings.datastore.s3.skip_ssl_verify
-    opts << "--etcd-s3-access-key=#{settings.datastore.s3.access_key}" if present?(settings.datastore.s3.access_key)
-    opts << "--etcd-s3-secret-key=#{settings.datastore.s3.secret_key}" if present?(settings.datastore.s3.secret_key)
-    opts << "--etcd-s3-bucket=#{settings.datastore.s3.bucket}" if present?(settings.datastore.s3.bucket)
-    opts << "--etcd-s3-region=#{settings.datastore.s3.region}" if present?(settings.datastore.s3.region)
-    opts << "--etcd-s3-folder=#{settings.datastore.s3.folder}" if present?(settings.datastore.s3.folder)
-    opts << "--etcd-s3-insecure" if settings.datastore.s3.insecure
-    opts << "--etcd-s3-timeout=#{settings.datastore.s3.timeout}" if present?(settings.datastore.s3.timeout)
+    backups = settings.datastore.etcd.backups
+    opts << "--etcd-snapshot-retention=#{backups.retention}" if backups.retention != nil
+    opts << "--etcd-snapshot-dir=#{backups.dir}" if present?(backups.dir)
+
+    s3 = backups.s3
+    if s3.enabled
+      opts << "--etcd-s3"
+      opts << "--etcd-s3-endpoint=#{s3.endpoint}" if present?(s3.endpoint)
+      opts << "--etcd-s3-endpoint-ca=#{s3.endpoint_ca}" if present?(s3.endpoint_ca)
+      opts << "--etcd-s3-skip-ssl-verify" if s3.skip_ssl_verify
+      opts << "--etcd-s3-access-key=#{s3.access_key}" if present?(s3.access_key)
+      opts << "--etcd-s3-secret-key=#{s3.secret_key}" if present?(s3.secret_key)
+      opts << "--etcd-s3-bucket=#{s3.bucket}" if present?(s3.bucket)
+      opts << "--etcd-s3-region=#{s3.region}" if present?(s3.region)
+      opts << "--etcd-s3-folder=#{s3.folder}" if present?(s3.folder)
+      opts << "--etcd-s3-insecure" if s3.insecure
+      opts << "--etcd-s3-timeout=#{s3.timeout}" if present?(s3.timeout)
+    end
 
     opts.uniq.sort.join(" ")
   end
diff --git a/templates/master_install_script.sh b/templates/master_install_script.sh
index 47662a42..ea87033b 100644
--- a/templates/master_install_script.sh
+++ b/templates/master_install_script.sh
@@ -65,7 +65,7 @@ curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="{{ k3s_version }}" K3S_TOKEN
 --kube-controller-manager-arg="bind-address=0.0.0.0" \
 --kube-proxy-arg="metrics-bind-address=0.0.0.0" \
 --kube-scheduler-arg="bind-address=0.0.0.0" \
-{{ taint }} {{ extra_args }} {{ etcd_arguments }} {{ s3_arguments }} $FLANNEL_SETTINGS $EMBEDDED_REGISTRY_MIRROR \
+{{ taint }} {{ extra_args }} {{ etcd_arguments }} {{ etcd_backup_settings }} $FLANNEL_SETTINGS $EMBEDDED_REGISTRY_MIRROR \
 --advertise-address=$PRIVATE_IP \
 --node-ip=$PRIVATE_IP \
 --node-external-ip=$PUBLIC_IP \

From be631bb19327e6b7a9910b41bd73164ab1291bd1 Mon Sep 17 00:00:00 2001
From: Kenneth Bennett <kennybll@outlook.com>
Date: Sun, 27 Oct 2024 14:20:18 -0400
Subject: [PATCH 03/13] utilize the present? helper for Int32s as well

---
 src/kubernetes/installer.cr |  4 ++--
 src/kubernetes/util.cr      | 11 +++++++++--
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/kubernetes/installer.cr b/src/kubernetes/installer.cr
index c7fbb178..88e6b6d1 100644
--- a/src/kubernetes/installer.cr
+++ b/src/kubernetes/installer.cr
@@ -391,14 +391,14 @@ class Kubernetes::Installer
   end
 
   private def etcd_backup_settings
-    if !settings.datastore.etcd.backups.enabled
+    unless settings.datastore.etcd.backups.enabled
       return "--etcd-disable-snapshots"
     end
 
     opts = [] of String
 
     backups = settings.datastore.etcd.backups
-    opts << "--etcd-snapshot-retention=#{backups.retention}" if backups.retention != nil
+    opts << "--etcd-snapshot-retention=#{backups.retention}" if present?(backups.retention)
     opts << "--etcd-snapshot-dir=#{backups.dir}" if present?(backups.dir)
 
     s3 = backups.s3
diff --git a/src/kubernetes/util.cr b/src/kubernetes/util.cr
index bc84efd0..d1f53d3b 100644
--- a/src/kubernetes/util.cr
+++ b/src/kubernetes/util.cr
@@ -84,7 +84,14 @@ module Kubernetes::Util
     port_open?(ip_address, port, timeout = 1.0)
   end
 
-  private def present?(value : String?)
-    value.try(&.strip).try(&.empty?) == false
+  private def present?(value : Object?)
+    case value
+    when String
+      !value.strip.empty?
+    when Int32
+      value != 0
+    else
+      !value.nil?
+    end
   end
 end

From 9b61714de5ac70ea3d281422b876a2244664ff2f Mon Sep 17 00:00:00 2001
From: Kenneth Bennett <kennybll@outlook.com>
Date: Sun, 27 Oct 2024 14:21:22 -0400
Subject: [PATCH 04/13] breakdown datastore classes into their own files

---
 src/configuration/datastore.cr                | 51 +++----------------
 src/configuration/datastore/etcd.cr           | 11 ++++
 src/configuration/datastore/etcd_backups.cr   | 14 +++++
 src/configuration/datastore/etcd_s3.cr        | 20 ++++++++
 src/configuration/main.cr                     |  2 +-
 src/configuration/settings/datastore.cr       |  2 +-
 src/configuration/settings/node_pool.cr       |  2 +-
 .../settings/node_pool/instance_count.cr      |  2 +-
 8 files changed, 57 insertions(+), 47 deletions(-)
 create mode 100644 src/configuration/datastore/etcd.cr
 create mode 100644 src/configuration/datastore/etcd_backups.cr
 create mode 100644 src/configuration/datastore/etcd_s3.cr

diff --git a/src/configuration/datastore.cr b/src/configuration/datastore.cr
index 5abf2389..64b37f23 100644
--- a/src/configuration/datastore.cr
+++ b/src/configuration/datastore.cr
@@ -1,52 +1,17 @@
 require "yaml"
+require "./datastore/etcd"
 
-class Configuration::Datastore
-  include YAML::Serializable
-
-  getter mode : String = "etcd"
-  getter external_datastore_endpoint : String = ""
-
-  getter etcd : Etcd = Etcd.new
-
-  def initialize(@mode : String = "etcd", @external_datastore_endpoint : String = "")
-  end
-
-  class Etcd
-    include YAML::Serializable
-
-    getter backups : Backups = Backups.new
-
-    def initialize(@backups : Backups = Backups.new)
-    end
-
-    class Backups
+module Configuration
+  module Datastore
+    class Config
       include YAML::Serializable
 
-      getter enabled : Bool = true
-      getter retention : Int32?
-      getter dir : String?
-      getter s3 : S3 = S3.new
-
-      def initialize(@enabled : Bool = true, @s3 : S3 = S3.new)
-      end
-
-      class S3
-        include YAML::Serializable
+      getter mode : String = "etcd"
+      getter external_datastore_endpoint : String = ""
 
-        getter enabled : Bool = false
-        getter endpoint : String?
-        getter endpoint_ca : String?
-        getter skip_ssl_verify : Bool = false
-        getter access_key : String = ""
-        getter secret_key : String = ""
-        getter bucket : String = ""
-        getter region : String?
-        getter folder : String?
-        getter insecure : Bool = false
-        getter timeout : String?
+      getter etcd : ::Configuration::Datastore::Etcd = ::Configuration::Datastore::Etcd.new
 
-        def initialize(@enabled : Bool = false, @skip_ssl_verify : Bool = false, @access_key : String = "", @secret_key : String = "", @bucket : String = "", @insecure : Bool = false)
-        end
+      def initialize(@mode : String = "etcd", @external_datastore_endpoint : String = "")
       end
     end
   end
diff --git a/src/configuration/datastore/etcd.cr b/src/configuration/datastore/etcd.cr
new file mode 100644
index 00000000..646313fe
--- /dev/null
+++ b/src/configuration/datastore/etcd.cr
@@ -0,0 +1,11 @@
+require "yaml"
+require "./etcd_backups"
+
+class Configuration::Datastore::Etcd
+  include YAML::Serializable
+
+  getter backups : Configuration::Datastore::Backups = Configuration::Datastore::Backups.new
+
+  def initialize(@backups : Configuration::Datastore::Backups = Configuration::Datastore::Backups.new)
+  end
+end
diff --git a/src/configuration/datastore/etcd_backups.cr b/src/configuration/datastore/etcd_backups.cr
new file mode 100644
index 00000000..28b0e091
--- /dev/null
+++ b/src/configuration/datastore/etcd_backups.cr
@@ -0,0 +1,14 @@
+require "yaml"
+require "./etcd_s3"
+
+class Configuration::Datastore::Backups
+  include YAML::Serializable
+
+  getter enabled : Bool = true
+  getter retention : Int32?
+  getter dir : String?
+  getter s3 : Configuration::Datastore::S3 = Configuration::Datastore::S3.new
+
+  def initialize(@enabled : Bool = true, @s3 : Configuration::Datastore::S3 = Configuration::Datastore::S3.new)
+  end
+end
diff --git a/src/configuration/datastore/etcd_s3.cr b/src/configuration/datastore/etcd_s3.cr
new file mode 100644
index 00000000..4542f91e
--- /dev/null
+++ b/src/configuration/datastore/etcd_s3.cr
@@ -0,0 +1,20 @@
+require "yaml"
+
+class Configuration::Datastore::S3
+  include YAML::Serializable
+
+  getter enabled : Bool = false
+  getter endpoint : String?
+  getter endpoint_ca : String?
+  getter skip_ssl_verify : Bool = false
+  getter access_key : String = ""
+  getter secret_key : String = ""
+  getter bucket : String = ""
+  getter region : String?
+  getter folder : String?
+  getter insecure : Bool = false
+  getter timeout : String?
+
+  def initialize(@enabled : Bool = false, @skip_ssl_verify : Bool = false, @access_key : String = "", @secret_key : String = "", @bucket : String = "", @insecure : Bool = false)
+  end
+end
diff --git a/src/configuration/main.cr b/src/configuration/main.cr
index 358e9e66..4233ff0c 100644
--- a/src/configuration/main.cr
+++ b/src/configuration/main.cr
@@ -28,7 +28,7 @@ class Configuration::Main
   getter autoscaling_image : String?
   getter snapshot_os : String = "default"
   getter networking : Configuration::Networking = Configuration::Networking.new
-  getter datastore : Configuration::Datastore = Configuration::Datastore.new
+  getter datastore : Configuration::Datastore::Config = Configuration::Datastore::Config.new
   getter manifests : Configuration::Manifests = Configuration::Manifests.new
   getter embedded_registry_mirror : Configuration::EmbeddedRegistryMirror = Configuration::EmbeddedRegistryMirror.new
   getter include_instance_type_in_instance_name : Bool = false
diff --git a/src/configuration/settings/datastore.cr b/src/configuration/settings/datastore.cr
index 8ae63077..42565262 100644
--- a/src/configuration/settings/datastore.cr
+++ b/src/configuration/settings/datastore.cr
@@ -1,6 +1,6 @@
 class Configuration::Settings::Datastore
   getter errors : Array(String)
-  getter datastore : Configuration::Datastore
+  getter datastore : Configuration::Datastore::Config
 
   def initialize(@errors, @datastore)
   end
diff --git a/src/configuration/settings/node_pool.cr b/src/configuration/settings/node_pool.cr
index 5e20be78..b71fdbfd 100644
--- a/src/configuration/settings/node_pool.cr
+++ b/src/configuration/settings/node_pool.cr
@@ -14,7 +14,7 @@ class Configuration::Settings::NodePool
   getter pool_name : String { masters? ? "masters" : pool.try(&.name) || "<unnamed-pool>" }
   getter pool_description : String { workers? ? "Worker mode pool '#{pool_name}'" : "Masters pool" }
 
-  getter datastore : Configuration::Datastore
+  getter datastore : Configuration::Datastore::Config
 
   def initialize(@errors, @pool, @pool_type, @masters_location, @instance_types, @locations, @datastore)
   end
diff --git a/src/configuration/settings/node_pool/instance_count.cr b/src/configuration/settings/node_pool/instance_count.cr
index 256482e5..4156f1b1 100644
--- a/src/configuration/settings/node_pool/instance_count.cr
+++ b/src/configuration/settings/node_pool/instance_count.cr
@@ -5,7 +5,7 @@ class Configuration::Settings::NodePool::InstanceCount
   getter errors : Array(String)
   getter pool : Configuration::NodePool
   getter pool_type : Symbol
-  getter datastore : Configuration::Datastore
+  getter datastore : Configuration::Datastore::Config
 
   def initialize(@errors, @pool, @pool_type, @datastore)
   end

From 9e7c48c36be430c7a6dd8d53dc13e33a57f07791 Mon Sep 17 00:00:00 2001
From: Kenneth Bennett <kennybll@outlook.com>
Date: Sun, 27 Oct 2024 14:22:05 -0400
Subject: [PATCH 05/13] add support to take server token file to disk

---
 .gitignore                  |  1 +
 docs/Creating_a_cluster.md  |  1 +
 docs/etcd S3 backups.md     |  6 ++++++
 src/configuration/main.cr   |  1 +
 src/kubernetes/installer.cr | 15 +++++++++++++++
 5 files changed, 24 insertions(+)

diff --git a/.gitignore b/.gitignore
index d379665e..faed0c78 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@
 .rspec_status
 /kubeconfig
 /cluster_config.yaml
+/token
 dist/hetzner-k3s.jar
 dist/hetzner-k3s
 
diff --git a/docs/Creating_a_cluster.md b/docs/Creating_a_cluster.md
index fd8123b7..ebd77617 100644
--- a/docs/Creating_a_cluster.md
+++ b/docs/Creating_a_cluster.md
@@ -7,6 +7,7 @@ The tool requires a simple configuration file in order to create/upgrade/delete
 hetzner_token: <your token>
 cluster_name: test
 kubeconfig_path: "./kubeconfig"
+# token_path: "./token" # optional, saves the server token file
 k3s_version: v1.30.3+k3s1
 
 networking:
diff --git a/docs/etcd S3 backups.md b/docs/etcd S3 backups.md
index a5d7ff79..77993cca 100644
--- a/docs/etcd S3 backups.md	
+++ b/docs/etcd S3 backups.md	
@@ -3,3 +3,9 @@
 S3 backups can be enabled for the embedded etcd mode only. You can see the explainations for each option in the [K3s docs](https://docs.k3s.io/cli/etcd-snapshot).
 
 > "In addition to backing up the datastore itself, you must also back up the server token file at /var/lib/rancher/k3s/server/token. You must restore this file, or pass its value into the --token option, when restoring from backup. If you do not use the same token value when restoring, the snapshot will be unusable, as the token is used to encrypt confidential data within the datastore itself." [K3S Backup/Restore Docs](https://docs.k3s.io/datastore/backup-restore)
+
+You can save the server token file to desk by setting the `token_path` value in your cluster_config.yaml:
+
+```yaml
+token_path: "./token"
+```
diff --git a/src/configuration/main.cr b/src/configuration/main.cr
index 4233ff0c..301a2c54 100644
--- a/src/configuration/main.cr
+++ b/src/configuration/main.cr
@@ -11,6 +11,7 @@ class Configuration::Main
   getter hetzner_token : String = ENV.fetch("HCLOUD_TOKEN", "")
   getter cluster_name : String
   getter kubeconfig_path : String
+  getter token_path : String?
   getter k3s_version : String
   getter api_server_hostname : String?
   getter schedule_workloads_on_masters : Bool = false
diff --git a/src/kubernetes/installer.cr b/src/kubernetes/installer.cr
index 88e6b6d1..701fd497 100644
--- a/src/kubernetes/installer.cr
+++ b/src/kubernetes/installer.cr
@@ -116,6 +116,8 @@ class Kubernetes::Installer
 
     save_kubeconfig(master_count)
 
+    save_server_token()
+
     sleep 5
 
     command = "kubectl cluster-info 2> /dev/null"
@@ -317,6 +319,19 @@ class Kubernetes::Installer
     log_line "...kubeconfig file generated as #{kubeconfig_path}.", "Control plane"
   end
 
+  private def save_server_token()
+    return unless present?(settings.token_path)
+    token_path = settings.token_path.not_nil!
+
+    log_line "Generating the token file to #{token_path}...", "Control plane"
+
+    token = ssh.run(first_master, settings.networking.ssh.port, "cat /var/lib/rancher/k3s/server/token", settings.networking.ssh.use_agent, print_output: false)
+
+    File.write(token_path, token)
+
+    log_line "...token file generated as #{token_path}.", "Control plane"
+  end
+
   private def add_labels_and_taints_to_masters
     add_labels_or_taints(:label, masters, settings.masters_pool.labels, "masters_pool")
     add_labels_or_taints(:taint, masters, settings.masters_pool.taints, "masters_pool")

From ad4429a4cdec79cc975bc2aeb8167ce6f0a928e4 Mon Sep 17 00:00:00 2001
From: Kenneth Bennett <kennybll@outlook.com>
Date: Sun, 27 Oct 2024 14:22:21 -0400
Subject: [PATCH 06/13] remove old link to docs

---
 README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.md b/README.md
index a718c570..2bc5d872 100644
--- a/README.md
+++ b/README.md
@@ -57,7 +57,6 @@ See my public profile with links for connecting with me [here](https://vitobotta
 - [Load balancers](docs/Load_balancers.md)
 - [Storage](docs/Storage.md)
 - [Troubleshooting](docs/Troubleshooting.md)
-- [S3 Backups](docs/S3%20backups.md)
 - [etcd S3 Backups](docs/etcd%20S3%20backups.md)
 - [Contributing and support](docs/Contributing_and_support.md)
 

From bb016e301baa93a2482d54d7a76500fe7dbf33eb Mon Sep 17 00:00:00 2001
From: Kenneth Bennett <kennybll@outlook.com>
Date: Sun, 27 Oct 2024 14:26:34 -0400
Subject: [PATCH 07/13] fix typo in etcd s3 docs

---
 docs/etcd S3 backups.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/etcd S3 backups.md b/docs/etcd S3 backups.md
index 77993cca..c7b09239 100644
--- a/docs/etcd S3 backups.md	
+++ b/docs/etcd S3 backups.md	
@@ -4,7 +4,7 @@ S3 backups can be enabled for the embedded etcd mode only. You can see the expla
 
 > "In addition to backing up the datastore itself, you must also back up the server token file at /var/lib/rancher/k3s/server/token. You must restore this file, or pass its value into the --token option, when restoring from backup. If you do not use the same token value when restoring, the snapshot will be unusable, as the token is used to encrypt confidential data within the datastore itself." [K3S Backup/Restore Docs](https://docs.k3s.io/datastore/backup-restore)
 
-You can save the server token file to desk by setting the `token_path` value in your cluster_config.yaml:
+You can save the server token file to disk by setting the `token_path` value in your cluster_config.yaml:
 
 ```yaml
 token_path: "./token"

From e4b0d1dd6686a71e4fa199fc88f52df46a20601b Mon Sep 17 00:00:00 2001
From: Kenneth Bennett <kennybll@outlook.com>
Date: Sun, 27 Oct 2024 17:37:47 -0400
Subject: [PATCH 08/13] add snapshot compress option

---
 docs/Creating_a_cluster.md                  | 1 +
 src/configuration/datastore/etcd_backups.cr | 3 ++-
 src/kubernetes/installer.cr                 | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/Creating_a_cluster.md b/docs/Creating_a_cluster.md
index ebd77617..6a058d5e 100644
--- a/docs/Creating_a_cluster.md
+++ b/docs/Creating_a_cluster.md
@@ -53,6 +53,7 @@ datastore:
   #     enabled: true # optional
   #     retention: 5 # optional
   #     dir: ${data-dir}/db/snapshots # optional
+  #     compress: false # optional
   #     s3: # optional - can only be enabled for etcd mode
   #       enabled: true
   #       endpoint: "s3.amazonaws.com" # optional
diff --git a/src/configuration/datastore/etcd_backups.cr b/src/configuration/datastore/etcd_backups.cr
index 28b0e091..a5d2b1b4 100644
--- a/src/configuration/datastore/etcd_backups.cr
+++ b/src/configuration/datastore/etcd_backups.cr
@@ -7,8 +7,9 @@ class Configuration::Datastore::Backups
   getter enabled : Bool = true
   getter retention : Int32?
   getter dir : String?
+  getter compress : Bool = false
   getter s3 : Configuration::Datastore::S3 = Configuration::Datastore::S3.new
 
-  def initialize(@enabled : Bool = true, @s3 : Configuration::Datastore::S3 = Configuration::Datastore::S3.new)
+  def initialize(@enabled : Bool = true, @compress : Bool = false, @s3 : Configuration::Datastore::S3 = Configuration::Datastore::S3.new)
   end
 end
diff --git a/src/kubernetes/installer.cr b/src/kubernetes/installer.cr
index 701fd497..090689c5 100644
--- a/src/kubernetes/installer.cr
+++ b/src/kubernetes/installer.cr
@@ -415,6 +415,7 @@ class Kubernetes::Installer
     backups = settings.datastore.etcd.backups
     opts << "--etcd-snapshot-retention=#{backups.retention}" if present?(backups.retention)
     opts << "--etcd-snapshot-dir=#{backups.dir}" if present?(backups.dir)
+    opts << "--etcd-snapshot-compress" if backups.compress
 
     s3 = backups.s3
     if s3.enabled

From bcc9eb3137c97a72cd85d70b5651b05a38e7f661 Mon Sep 17 00:00:00 2001
From: Kenneth Bennett <kennybll@outlook.com>
Date: Sun, 27 Oct 2024 17:38:18 -0400
Subject: [PATCH 09/13] add note about retention value for HA setups

---
 docs/etcd S3 backups.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/etcd S3 backups.md b/docs/etcd S3 backups.md
index c7b09239..8c1c33a9 100644
--- a/docs/etcd S3 backups.md	
+++ b/docs/etcd S3 backups.md	
@@ -2,6 +2,8 @@
 
 S3 backups can be enabled for the embedded etcd mode only. You can see the explainations for each option in the [K3s docs](https://docs.k3s.io/cli/etcd-snapshot).
 
+A backup is created for each server instance. If you run your cluster in high availability mode you may want to update the retention value to be how many master instances you have multiplied by how many backups you want for each master. For example if you have 3 masters and you want 3 backups per master you would set retention to 9. The default is 5, so when running 3 master instances, one instance will only have 1 backup.
+
 > "In addition to backing up the datastore itself, you must also back up the server token file at /var/lib/rancher/k3s/server/token. You must restore this file, or pass its value into the --token option, when restoring from backup. If you do not use the same token value when restoring, the snapshot will be unusable, as the token is used to encrypt confidential data within the datastore itself." [K3S Backup/Restore Docs](https://docs.k3s.io/datastore/backup-restore)
 
 You can save the server token file to disk by setting the `token_path` value in your cluster_config.yaml:

From d259c29583b40cb2770e8537e57d85da35926eff Mon Sep 17 00:00:00 2001
From: Kenneth Bennett <kennybll@outlook.com>
Date: Sun, 27 Oct 2024 17:39:49 -0400
Subject: [PATCH 10/13] add schedule-cron option

I was only able to get it to work using a config file

config files when changed do not automatically restart the k3s service, it has to be done manually
---
 docs/Creating_a_cluster.md                  |  1 +
 src/configuration/datastore/etcd_backups.cr |  1 +
 src/kubernetes/installer.cr                 |  4 ++++
 templates/master_install_script.sh          | 13 ++++++++++++-
 4 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/docs/Creating_a_cluster.md b/docs/Creating_a_cluster.md
index 6a058d5e..e3c2f9dd 100644
--- a/docs/Creating_a_cluster.md
+++ b/docs/Creating_a_cluster.md
@@ -52,6 +52,7 @@ datastore:
   #   backups: # optional
   #     enabled: true # optional
   #     retention: 5 # optional
+  #     schedule_cron: "0 */12 * * *"  # optional
   #     dir: ${data-dir}/db/snapshots # optional
   #     compress: false # optional
   #     s3: # optional - can only be enabled for etcd mode
diff --git a/src/configuration/datastore/etcd_backups.cr b/src/configuration/datastore/etcd_backups.cr
index a5d2b1b4..4dc5046c 100644
--- a/src/configuration/datastore/etcd_backups.cr
+++ b/src/configuration/datastore/etcd_backups.cr
@@ -5,6 +5,7 @@ class Configuration::Datastore::Backups
   include YAML::Serializable
 
   getter enabled : Bool = true
+  getter schedule_cron : String?
   getter retention : Int32?
   getter dir : String?
   getter compress : Bool = false
diff --git a/src/kubernetes/installer.cr b/src/kubernetes/installer.cr
index 090689c5..1fb43ed9 100644
--- a/src/kubernetes/installer.cr
+++ b/src/kubernetes/installer.cr
@@ -155,6 +155,7 @@ class Kubernetes::Installer
     server = ""
     datastore_endpoint = ""
     etcd_arguments = ""
+    etcd_schedule_cron = ""
 
     if settings.datastore.mode == "etcd"
       server = master == first_master ? " --cluster-init " : " --server https://#{api_server_ip_address}:6443 "
@@ -166,6 +167,8 @@ class Kubernetes::Installer
     extra_args = "#{kube_api_server_args_list} #{kube_scheduler_args_list} #{kube_controller_manager_args_list} #{kube_cloud_controller_manager_args_list} #{kubelet_args_list} #{kube_proxy_args_list}"
     taint = settings.schedule_workloads_on_masters ? " " : " --node-taint CriticalAddonsOnly=true:NoExecute "
 
+    etcd_schedule_cron = settings.datastore.etcd.backups.schedule_cron if present?(settings.datastore.etcd.backups.schedule_cron)
+
     Crinja.render(MASTER_INSTALL_SCRIPT, {
       cluster_name: settings.cluster_name,
       k3s_version: settings.k3s_version,
@@ -186,6 +189,7 @@ class Kubernetes::Installer
       datastore_endpoint: datastore_endpoint,
       etcd_arguments: etcd_arguments,
       etcd_backup_settings: etcd_backup_settings,
+      etcd_schedule_cron: etcd_schedule_cron,
       embedded_registry_mirror_enabled: settings.embedded_registry_mirror.enabled.to_s,
     })
   end
diff --git a/templates/master_install_script.sh b/templates/master_install_script.sh
index ea87033b..31b7ef07 100644
--- a/templates/master_install_script.sh
+++ b/templates/master_install_script.sh
@@ -45,7 +45,15 @@ else
   EMBEDDED_REGISTRY_MIRROR=" "
 fi
 
-mkdir -p /etc/rancher/k3s
+mkdir -p /etc/rancher/k3s/config.yaml.d
+
+if [ "{{ etcd_schedule_cron }}" != "" ]; then
+    cat > /etc/rancher/k3s/config.yaml.d/hetzner-k3s.yaml <<EOF
+etcd-snapshot-schedule-cron: "{{ etcd_schedule_cron }}"
+EOF
+else
+    rm -f /etc/rancher/k3s/config.yaml.d/hetzner-k3s.yaml 2>/dev/null
+fi
 
 cat > /etc/rancher/k3s/registries.yaml <<EOF
 mirrors:
@@ -71,4 +79,7 @@ curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="{{ k3s_version }}" K3S_TOKEN
 --node-external-ip=$PUBLIC_IP \
 {{ server }} {{ tls_sans }}" sh -
 
+echo "Restarting k3s service" 2>&1 | tee -a /var/log/hetzner-k3s.log
+systemctl restart k3s
+
 echo true > /etc/initialized

From 49d98541817209a07540a323d858318ae99d388b Mon Sep 17 00:00:00 2001
From: Kenneth Bennett <kennybll@outlook.com>
Date: Sun, 27 Oct 2024 18:13:19 -0400
Subject: [PATCH 11/13] remove etcd error when using external datastore
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

since the etcd class isn’t optional in the Datastore Config is will always be set technically, which is fine if they are using an external datastore since the options won’t matter
---
 src/configuration/settings/datastore.cr | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/configuration/settings/datastore.cr b/src/configuration/settings/datastore.cr
index 42565262..c1e76bb3 100644
--- a/src/configuration/settings/datastore.cr
+++ b/src/configuration/settings/datastore.cr
@@ -16,7 +16,6 @@ class Configuration::Settings::Datastore
       errors << "bucket is required for S3 backups" if s3.bucket.strip.empty?
     when "external"
       errors << "external_datastore_endpoint is required for external datastore" if datastore.external_datastore_endpoint.strip.empty?
-      errors << "etcd options cannot be set for external datastore" if datastore.etcd
     else
       errors << "datastore mode is invalid - allowed values are 'etcd' and 'external'"
     end

From 0c2034ab181bba5de1da179412acdbf2f382599a Mon Sep 17 00:00:00 2001
From: Kenneth Bennett <kennybll@outlook.com>
Date: Sun, 27 Oct 2024 18:19:46 -0400
Subject: [PATCH 12/13] =?UTF-8?q?don=E2=80=99t=20provide=20etcd=20flags=20?=
 =?UTF-8?q?if=20mode=20is=20not=20etcd?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/kubernetes/installer.cr | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/kubernetes/installer.cr b/src/kubernetes/installer.cr
index 1fb43ed9..9b7ae1ec 100644
--- a/src/kubernetes/installer.cr
+++ b/src/kubernetes/installer.cr
@@ -410,6 +410,8 @@ class Kubernetes::Installer
   end
 
   private def etcd_backup_settings
+    return "" unless settings.datastore.mode == "etcd"
+
     unless settings.datastore.etcd.backups.enabled
       return "--etcd-disable-snapshots"
     end

From 6ba97e0856b21ba2e91512a41cf7e6c0bec1e857 Mon Sep 17 00:00:00 2001
From: Kenneth Bennett <kennybll@outlook.com>
Date: Sun, 27 Oct 2024 18:46:34 -0400
Subject: [PATCH 13/13] restart both systemd and openrc services

---
 templates/master_install_script.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/templates/master_install_script.sh b/templates/master_install_script.sh
index 31b7ef07..51cab6d9 100644
--- a/templates/master_install_script.sh
+++ b/templates/master_install_script.sh
@@ -80,6 +80,12 @@ curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="{{ k3s_version }}" K3S_TOKEN
 {{ server }} {{ tls_sans }}" sh -
 
 echo "Restarting k3s service" 2>&1 | tee -a /var/log/hetzner-k3s.log
-systemctl restart k3s
+if [ -x /sbin/openrc-run ]; then
+    rc-service k3s restart
+fi
+if [ -d /run/systemd ]; then
+    systemctl restart k3s
+fi
+
 
 echo true > /etc/initialized