add support for etcd backups with s3 storage

README.md

@@ -57,6 +57,8 @@ See my public profile with links for connecting with me [here](https://vitobotta
 - [Load balancers](docs/Load_balancers.md)
 - [Storage](docs/Storage.md)
 - [Troubleshooting](docs/Troubleshooting.md)
+- [S3 Backups](docs/S3%20backups.md)
+- [etcd S3 Backups](docs/etcd%20S3%20backups.md)
 - [Contributing and support](docs/Contributing_and_support.md)
 
 ___

docs/Creating_a_cluster.md

@@ -47,6 +47,23 @@ networking:
 datastore:
   mode: etcd # etcd (default) or external
   external_datastore_endpoint: postgres://....
+  # etcd: # optional
+  #   backups: # optional
+  #     enabled: true # optional
+  #     retention: 5 # optional
+  #     dir: ${data-dir}/db/snapshots # optional
+  #     s3: # optional - can only be enabled for etcd mode
+  #       enabled: true
+  #       endpoint: "s3.amazonaws.com" # optional
+  #       endpoint_ca: "" # optional
+  #       skip_ssl_verify: false # optional
+  #       access_key: ""
+  #       secret_key: ""
+  #       bucket: ""
+  #       region: "us-east-1" # optional
+  #       folder: "" # optional
+  #       insecure: false # optional
+  #       timeout: "5m0s" # optional
 
 schedule_workloads_on_masters: false
 

docs/etcd S3 backups.md

@@ -0,0 +1,5 @@
+### etcd S3 Backups
+
+S3 backups can be enabled for the embedded etcd mode only. You can see the explainations for each option in the [K3s docs](https://docs.k3s.io/cli/etcd-snapshot).
+
+> "In addition to backing up the datastore itself, you must also back up the server token file at /var/lib/rancher/k3s/server/token. You must restore this file, or pass its value into the --token option, when restoring from backup. If you do not use the same token value when restoring, the snapshot will be unusable, as the token is used to encrypt confidential data within the datastore itself." [K3S Backup/Restore Docs](https://docs.k3s.io/datastore/backup-restore)

src/configuration/datastore.cr

@@ -6,6 +6,48 @@ class Configuration::Datastore
   getter mode : String = "etcd"
   getter external_datastore_endpoint : String = ""
 
+  getter etcd : Etcd = Etcd.new
+
   def initialize(@mode : String = "etcd", @external_datastore_endpoint : String = "")
   end
+
+  class Etcd
+    include YAML::Serializable
+
+    getter backups : Backups = Backups.new
+
+    def initialize(@backups : Backups = Backups.new)
+    end
+
+    class Backups
+      include YAML::Serializable
+
+      getter enabled : Bool = true
+      getter retention : Int32?
+      getter dir : String?
+      getter s3 : S3 = S3.new
+
+      def initialize(@enabled : Bool = true, @s3 : S3 = S3.new)
+      end
+
+      class S3
+        include YAML::Serializable
+
+        getter enabled : Bool = false
+        getter endpoint : String?
+        getter endpoint_ca : String?
+        getter skip_ssl_verify : Bool = false
+        getter access_key : String = ""
+        getter secret_key : String = ""
+        getter bucket : String = ""
+        getter region : String?
+        getter folder : String?
+        getter insecure : Bool = false
+        getter timeout : String?
+
+        def initialize(@enabled : Bool = false, @skip_ssl_verify : Bool = false, @access_key : String = "", @secret_key : String = "", @bucket : String = "", @insecure : Bool = false)
+        end
+      end
+    end
+  end
 end

src/configuration/settings/datastore.cr

@@ -8,8 +8,15 @@ class Configuration::Settings::Datastore
   def validate
     case datastore.mode
     when "etcd"
+      return unless datastore.etcd.backups.s3.enabled
+
+      s3 = datastore.etcd.backups.s3
+      errors << "access_key is required for S3 backups" if s3.access_key.strip.empty?
+      errors << "secret_key is required for S3 backups" if s3.secret_key.strip.empty?
+      errors << "bucket is required for S3 backups" if s3.bucket.strip.empty?
     when "external"
       errors << "external_datastore_endpoint is required for external datastore" if datastore.external_datastore_endpoint.strip.empty?
+      errors << "etcd options cannot be set for external datastore" if datastore.etcd
     else
       errors << "datastore mode is invalid - allowed values are 'etcd' and 'external'"
     end

src/kubernetes/installer.cr

@@ -183,6 +183,7 @@ class Kubernetes::Installer
       cluster_dns: settings.networking.cluster_dns,
       datastore_endpoint: datastore_endpoint,
       etcd_arguments: etcd_arguments,
+      etcd_backup_settings: etcd_backup_settings,
       embedded_registry_mirror_enabled: settings.embedded_registry_mirror.enabled.to_s,
     })
   end
@@ -388,4 +389,33 @@ class Kubernetes::Installer
       first_master.private_ip_address
     end
   end
+
+  private def etcd_backup_settings
+    if !settings.datastore.etcd.backups.enabled
+      return "--etcd-disable-snapshots"
+    end
+
+    opts = [] of String
+
+    backups = settings.datastore.etcd.backups
+    opts << "--etcd-snapshot-retention=#{backups.retention}" if backups.retention != nil
+    opts << "--etcd-snapshot-dir=#{backups.dir}" if present?(backups.dir)
+
+    s3 = backups.s3
+    if s3.enabled
+      opts << "--etcd-s3"
+      opts << "--etcd-s3-endpoint=#{s3.endpoint}" if present?(s3.endpoint)
+      opts << "--etcd-s3-endpoint-ca=#{s3.endpoint_ca}" if present?(s3.endpoint_ca)
+      opts << "--etcd-s3-skip-ssl-verify" if s3.skip_ssl_verify
+      opts << "--etcd-s3-access-key=#{s3.access_key}" if present?(s3.access_key)
+      opts << "--etcd-s3-secret-key=#{s3.secret_key}" if present?(s3.secret_key)
+      opts << "--etcd-s3-bucket=#{s3.bucket}" if present?(s3.bucket)
+      opts << "--etcd-s3-region=#{s3.region}" if present?(s3.region)
+      opts << "--etcd-s3-folder=#{s3.folder}" if present?(s3.folder)
+      opts << "--etcd-s3-insecure" if s3.insecure
+      opts << "--etcd-s3-timeout=#{s3.timeout}" if present?(s3.timeout)
+    end
+
+    opts.uniq.sort.join(" ")
+  end
 end

src/kubernetes/util.cr

@@ -83,4 +83,8 @@ module Kubernetes::Util
 
     port_open?(ip_address, port, timeout = 1.0)
   end
+
+  private def present?(value : String?)
+    value.try(&.strip).try(&.empty?) == false
+  end
 end

templates/master_install_script.sh

@@ -65,7 +65,7 @@ curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="{{ k3s_version }}" K3S_TOKEN
 --kube-controller-manager-arg="bind-address=0.0.0.0" \
 --kube-proxy-arg="metrics-bind-address=0.0.0.0" \
 --kube-scheduler-arg="bind-address=0.0.0.0" \
-{{ taint }} {{ extra_args }} {{ etcd_arguments }} $FLANNEL_SETTINGS $EMBEDDED_REGISTRY_MIRROR \
+{{ taint }} {{ extra_args }} {{ etcd_arguments }} {{ etcd_backup_settings }} $FLANNEL_SETTINGS $EMBEDDED_REGISTRY_MIRROR \
 --advertise-address=$PRIVATE_IP \
 --node-ip=$PRIVATE_IP \
 --node-external-ip=$PUBLIC_IP \