Improve docs

vitobotta · Jul 23, 2024 · ba7f5e7 · ba7f5e7
1 parent bf52992
commit ba7f5e7
Show file tree

Hide file tree

Showing 5 changed files with 64 additions and 122 deletions.
diff --git a/README.md b/README.md
@@ -50,6 +50,7 @@ See my public profile with links for connecting with me [here](https://vitobotta
 
 - [Installation](docs/Installation.md)
 - [Creating a cluster](docs/Creating_a_cluster.md)
+- [Recommendations](docs/Recommendations.md)
 - [Maintenance](docs/Maintenance.md)
 - [Deleting a cluster](docs/Deleting_a_cluster.md)
 - [Load balancers](docs/Load_balancers.md)

diff --git a/cluster_config.yaml.example b/cluster_config.yaml.example
diff --git a/docs/Creating_a_cluster.md b/docs/Creating_a_cluster.md
@@ -8,44 +8,56 @@ hetzner_token: <your token>
 cluster_name: test
 kubeconfig_path: "./kubeconfig"
 k3s_version: v1.26.4+k3s1
-public_ssh_key_path: "~/.ssh/id_rsa.pub"
-private_ssh_key_path: "~/.ssh/id_rsa"
-use_ssh_agent: false # set to true if your key has a passphrase or if SSH connections don't work or seem to hang without agent. See https://github.com/vitobotta/hetzner-k3s#limitations
-# ssh_port: 22
-ssh_allowed_networks:
-  - 0.0.0.0/0 # ensure your current IP is included in the range
-api_allowed_networks:
-  - 0.0.0.0/0 # ensure your current IP is included in the range
-private_network_subnet: 10.0.0.0/16 # ensure this doesn't overlap with other networks in the same project
-disable_flannel: false # set to true if you want to install a different CNI
-schedule_workloads_on_masters: false
-# cluster_cidr: 10.244.0.0/16 # optional: a custom IPv4/IPv6 network CIDR to use for pod IPs
-# service_cidr: 10.43.0.0/16 # optional: a custom IPv4/IPv6 network CIDR to use for service IPs. Warning, if you change this, you should also change cluster_dns!
-# cluster_dns: 10.43.0.10 # optional: IPv4 Cluster IP for coredns service. Needs to be an address from the service_cidr range
-# enable_public_net_ipv4: false # default is true
-# enable_public_net_ipv6: false # default is true
-# image: rocky-9 # optional: default is ubuntu-22.04
-# autoscaling_image: 103908130 # optional, defaults to the `image` setting
-# snapshot_os: microos # optional: specified the os type when using a custom snapshot
 
-manifests:
-  cloud_controller_manager_manifest_url: "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.19.0/ccm-networks.yaml"
-  csi_driver_manifest_url: "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.6.0/deploy/kubernetes/hcloud-csi.yml"
-  system_upgrade_controller_deployment_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/system-upgrade-controller.yaml"
-  system_upgrade_controller_crd_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml"
-  cluster_autoscaler_manifest_url: "https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml"
+networking:
+  ssh:
+    port: 22
+    use_agent: false # set to true if your key has a passphrase
+    public_key_path: "~/.ssh/id_ed25519.pub"
+    private_key_path: "~/.ssh/id_ed25519"
+  allowed_networks:
+    ssh:
+      - 0.0.0.0/0
+    api:
+      - 0.0.0.0/0
+  public_network:
+    ipv4: true
+    ipv6: true
+  private_network:
+    enabled : true
+    subnet: 10.0.0.0/16
+    existing_network_name: ""
+  cni:
+    enabled: true
+    encryption: false
+    mode: flannel
+  # cluster_cidr: 10.244.0.0/16 # optional: a custom IPv4/IPv6 network CIDR to use for pod IPs
+  # service_cidr: 10.43.0.0/16 # optional: a custom IPv4/IPv6 network CIDR to use for service IPs. Warning, if you change this, you should also change cluster_dns!
+  # cluster_dns: 10.43.0.10 # optional: IPv4 Cluster IP for coredns service. Needs to be an address from the service_cidr range
+
+
+# manifests:
+#   cloud_controller_manager_manifest_url: "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.20.0/ccm-networks.yaml"
+#   csi_driver_manifest_url: "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.8.0/deploy/kubernetes/hcloud-csi.yml"
+#   system_upgrade_controller_deployment_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/system-upgrade-controller.yaml"
+#   system_upgrade_controller_crd_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml"
+#   cluster_autoscaler_manifest_url: "https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml"
 
 datastore:
   mode: etcd # etcd (default) or external
   external_datastore_endpoint: postgres://....
 
+schedule_workloads_on_masters: false
+
+# image: rocky-9 # optional: default is ubuntu-22.04
+# autoscaling_image: 103908130 # optional, defaults to the `image` setting
+# snapshot_os: microos # optional: specified the os type when using a custom snapshot
+
 masters_pool:
   instance_type: cpx21
   instance_count: 3
   location: nbg1
 
-schedule_workloads_on_masters: false
-
 worker_node_pools:
 - name: small-static
   instance_type: cpx21
@@ -172,3 +184,5 @@ The networks you provide should provide enough space for the expected amount of
 ### Idempotency
 
 The `create` command can be run any number of times with the same configuration without causing any issue, since the process is idempotent. This means that if for some reason the create process gets stuck or throws errors (for example if the Hetzner API is unavailable or there are timeouts etc), you can just stop the current command, and re-run it with the same configuration to continue from where it left.
+
+Note that the kubeconfig will be overwritten when you re-run the `create` command.
diff --git a/docs/Maintenance.md b/docs/Maintenance.md
@@ -26,6 +26,10 @@ In a future release I will add some automation for the cleanup.
 
 It's easy to convert a non-HA with a single master cluster to HA with multiple masters. Just change the masters instance count and re-run the `create` command. This will create a load balancer for the API server and update the kubeconfig so that all the API requests go through the load balancer.
 
+## Replacing the seed master
+
+When creating a new cluster, the seed master (or first master) in a HA configuration is `master1`. The seed master will change if you delete `master1` due to some issues with the node so it gets recreated. Whenever the seed master changes, k3s must be restarted on the existing masters.
+
 ___
 ## Upgrading to a new version of k3s
 

diff --git a/docs/Recommendations.md b/docs/Recommendations.md
@@ -0,0 +1,18 @@
+# Recommendations
+
+## Larger clusters
+
+The default configuration settings are pretty good for most small-medium clusters, so you can leave most settings unchanged if you want to go with a configuration that has been tested extensively.
+
+However keep in mind that this default configuration - that uses a Hetzner private network with the default Flannel CNI built into k3s - may not be optimal for larger clusters:
+
+1. Private networks in Hetzner cloud supports max 100 nodes, so I recommend you disable the private network in the configuration if you expect your cluster to grow beyond 100 nodes.
+2. Flannel is fine for small to medium clusters, but performance starts to degrade with clusters made of several hundreds or thousands of nodes, so I recommend to switch to Cilium as CNI as its performance is excellent and scales well with very large clusters.
+
+Notes:
+- if you disable the private network due to the limitation mentioned above, encryption will be enforced at CNI level to secure the traffic between nodes over the public network.
+- if you want to use something other than Cilium or Flannel (e.g. Calico), then you can disable the automatic setup of the CNI so you can install a CNI of your choice. We may add built in support for more CNIs in future releases.
+
+## Registry mirror
+
+v2.0.0 introduces a setting to optionally enable the `embedded registry mirror` in k3s (see [this page](https://docs.k3s.io/installation/registry-mirror) for more information. This is basically an installation of [Spegel](https://github.com/spegel-org/spegel) which enables peer-to-peer distribution of container images between the nodes of a cluster. This can help avoid problems with nodes not being able to pull images because their IPs have been banned by registry (due to malicious use of the same IPs in the past or similar reason), because a node will try pulling an image from other nodes via the embedded registry mirror, before pulling the image from the upstream registry. This also speeds up pods creation because less time is spent downloading images from the upstream registries when deployments have many replicas spread across many nodes.