From ba7f5e72438498e255f51d8c8771afa430029bab Mon Sep 17 00:00:00 2001 From: Vito Botta Date: Tue, 23 Jul 2024 16:06:36 +0300 Subject: [PATCH] Improve docs --- README.md | 1 + cluster_config.yaml.example | 95 ------------------------------------- docs/Creating_a_cluster.md | 68 +++++++++++++++----------- docs/Maintenance.md | 4 ++ docs/Recommendations.md | 18 +++++++ 5 files changed, 64 insertions(+), 122 deletions(-) delete mode 100644 cluster_config.yaml.example create mode 100644 docs/Recommendations.md diff --git a/README.md b/README.md index d78bcc9..3227a00 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ See my public profile with links for connecting with me [here](https://vitobotta - [Installation](docs/Installation.md) - [Creating a cluster](docs/Creating_a_cluster.md) +- [Recommendations](docs/Recommendations.md) - [Maintenance](docs/Maintenance.md) - [Deleting a cluster](docs/Deleting_a_cluster.md) - [Load balancers](docs/Load_balancers.md) diff --git a/cluster_config.yaml.example b/cluster_config.yaml.example deleted file mode 100644 index cd5c594..0000000 --- a/cluster_config.yaml.example +++ /dev/null @@ -1,95 +0,0 @@ ---- -hetzner_token: -cluster_name: test -kubeconfig_path: "./kubeconfig" -k3s_version: v1.26.4+k3s1 -public_ssh_key_path: "~/.ssh/id_rsa.pub" -private_ssh_key_path: "~/.ssh/id_rsa" -use_ssh_agent: false -# ssh_port: 22 -ssh_allowed_networks: - - 0.0.0.0/0 -api_allowed_networks: - - 0.0.0.0/0 -schedule_workloads_on_masters: false -private_network_subnet: 10.0.0.0/16 -# cluster_cidr: 10.244.0.0/16 # optional: a custom IPv4/IPv6 network CIDR to use for pod IPs -# service_cidr: 10.43.0.0/16 # optional: a custom IPv4/IPv6 network CIDR to use for service IPs. Warning, if you change this, you should also change cluster_dns! -# cluster_dns: 10.43.0.10 # optional: IPv4 Cluster IP for coredns service. Needs to be an address from the service_cidr range -disable_flannel: false # set to true if you want to install a different CNI -# enable_public_net_ipv4: false # default is true -# enable_public_net_ipv6: false # default is true -# image: rocky-9 # optional: default is ubuntu-22.04 -# autoscaling_image: 103908130 # optional, defaults to the `image` setting -# snapshot_os: microos # optional: specified the os type when using a custom snapshot - -manifests: - cloud_controller_manager_manifest_url: "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.19.0/ccm-networks.yaml" - csi_driver_manifest_url: "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.6.0/deploy/kubernetes/hcloud-csi.yml" - system_upgrade_controller_deployment_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/system-upgrade-controller.yaml" - system_upgrade_controller_crd_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml" - cluster_autoscaler_manifest_url: "https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml" - -datastore: - mode: etcd # etcd (default) or external - external_datastore_endpoint: postgres://.... - -masters_pool: - instance_type: cpx21 - instance_count: 3 - location: nbg1 - -schedule_workloads_on_masters: false - -worker_node_pools: -- name: small-static - instance_type: cpx21 - instance_count: 4 - location: hel1 - # image: debian-11 - # labels: - # - key: purpose - # value: blah - # taints: - # - key: something - # value: value1:NoSchedule -- name: medium-autoscaled - instance_type: cpx31 - instance_count: 2 - location: fsn1 - autoscaling: - enabled: true - min_instances: 0 - max_instances: 3 - -embedded_registry_mirror: - enabled: true - - -# additional_packages: -# - somepackage - -# post_create_commands: -# - apt update -# - apt upgrade -y -# - apt autoremove -y - -# kube_api_server_args: -# - arg1 -# - ... -# kube_scheduler_args: -# - arg1 -# - ... -# kube_controller_manager_args: -# - arg1 -# - ... -# kube_cloud_controller_manager_args: -# - arg1 -# - ... -# kubelet_args: -# - arg1 -# - ... -# kube_proxy_args: -# - arg1 -# - ... -# api_server_hostname: k8s.example.com # optional: DNS for the k8s API LoadBalancer. After the script has run, create a DNS record with the address of the API LoadBalancer. diff --git a/docs/Creating_a_cluster.md b/docs/Creating_a_cluster.md index 4919fb7..66f9e49 100644 --- a/docs/Creating_a_cluster.md +++ b/docs/Creating_a_cluster.md @@ -8,44 +8,56 @@ hetzner_token: cluster_name: test kubeconfig_path: "./kubeconfig" k3s_version: v1.26.4+k3s1 -public_ssh_key_path: "~/.ssh/id_rsa.pub" -private_ssh_key_path: "~/.ssh/id_rsa" -use_ssh_agent: false # set to true if your key has a passphrase or if SSH connections don't work or seem to hang without agent. See https://github.com/vitobotta/hetzner-k3s#limitations -# ssh_port: 22 -ssh_allowed_networks: - - 0.0.0.0/0 # ensure your current IP is included in the range -api_allowed_networks: - - 0.0.0.0/0 # ensure your current IP is included in the range -private_network_subnet: 10.0.0.0/16 # ensure this doesn't overlap with other networks in the same project -disable_flannel: false # set to true if you want to install a different CNI -schedule_workloads_on_masters: false -# cluster_cidr: 10.244.0.0/16 # optional: a custom IPv4/IPv6 network CIDR to use for pod IPs -# service_cidr: 10.43.0.0/16 # optional: a custom IPv4/IPv6 network CIDR to use for service IPs. Warning, if you change this, you should also change cluster_dns! -# cluster_dns: 10.43.0.10 # optional: IPv4 Cluster IP for coredns service. Needs to be an address from the service_cidr range -# enable_public_net_ipv4: false # default is true -# enable_public_net_ipv6: false # default is true -# image: rocky-9 # optional: default is ubuntu-22.04 -# autoscaling_image: 103908130 # optional, defaults to the `image` setting -# snapshot_os: microos # optional: specified the os type when using a custom snapshot -manifests: - cloud_controller_manager_manifest_url: "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.19.0/ccm-networks.yaml" - csi_driver_manifest_url: "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.6.0/deploy/kubernetes/hcloud-csi.yml" - system_upgrade_controller_deployment_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/system-upgrade-controller.yaml" - system_upgrade_controller_crd_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml" - cluster_autoscaler_manifest_url: "https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml" +networking: + ssh: + port: 22 + use_agent: false # set to true if your key has a passphrase + public_key_path: "~/.ssh/id_ed25519.pub" + private_key_path: "~/.ssh/id_ed25519" + allowed_networks: + ssh: + - 0.0.0.0/0 + api: + - 0.0.0.0/0 + public_network: + ipv4: true + ipv6: true + private_network: + enabled : true + subnet: 10.0.0.0/16 + existing_network_name: "" + cni: + enabled: true + encryption: false + mode: flannel + # cluster_cidr: 10.244.0.0/16 # optional: a custom IPv4/IPv6 network CIDR to use for pod IPs + # service_cidr: 10.43.0.0/16 # optional: a custom IPv4/IPv6 network CIDR to use for service IPs. Warning, if you change this, you should also change cluster_dns! + # cluster_dns: 10.43.0.10 # optional: IPv4 Cluster IP for coredns service. Needs to be an address from the service_cidr range + + +# manifests: +# cloud_controller_manager_manifest_url: "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.20.0/ccm-networks.yaml" +# csi_driver_manifest_url: "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.8.0/deploy/kubernetes/hcloud-csi.yml" +# system_upgrade_controller_deployment_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/system-upgrade-controller.yaml" +# system_upgrade_controller_crd_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml" +# cluster_autoscaler_manifest_url: "https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml" datastore: mode: etcd # etcd (default) or external external_datastore_endpoint: postgres://.... +schedule_workloads_on_masters: false + +# image: rocky-9 # optional: default is ubuntu-22.04 +# autoscaling_image: 103908130 # optional, defaults to the `image` setting +# snapshot_os: microos # optional: specified the os type when using a custom snapshot + masters_pool: instance_type: cpx21 instance_count: 3 location: nbg1 -schedule_workloads_on_masters: false - worker_node_pools: - name: small-static instance_type: cpx21 @@ -172,3 +184,5 @@ The networks you provide should provide enough space for the expected amount of ### Idempotency The `create` command can be run any number of times with the same configuration without causing any issue, since the process is idempotent. This means that if for some reason the create process gets stuck or throws errors (for example if the Hetzner API is unavailable or there are timeouts etc), you can just stop the current command, and re-run it with the same configuration to continue from where it left. + +Note that the kubeconfig will be overwritten when you re-run the `create` command. diff --git a/docs/Maintenance.md b/docs/Maintenance.md index f3fb8ce..8b45dc7 100644 --- a/docs/Maintenance.md +++ b/docs/Maintenance.md @@ -26,6 +26,10 @@ In a future release I will add some automation for the cleanup. It's easy to convert a non-HA with a single master cluster to HA with multiple masters. Just change the masters instance count and re-run the `create` command. This will create a load balancer for the API server and update the kubeconfig so that all the API requests go through the load balancer. +## Replacing the seed master + +When creating a new cluster, the seed master (or first master) in a HA configuration is `master1`. The seed master will change if you delete `master1` due to some issues with the node so it gets recreated. Whenever the seed master changes, k3s must be restarted on the existing masters. + ___ ## Upgrading to a new version of k3s diff --git a/docs/Recommendations.md b/docs/Recommendations.md new file mode 100644 index 0000000..bd4d1bf --- /dev/null +++ b/docs/Recommendations.md @@ -0,0 +1,18 @@ +# Recommendations + +## Larger clusters + +The default configuration settings are pretty good for most small-medium clusters, so you can leave most settings unchanged if you want to go with a configuration that has been tested extensively. + +However keep in mind that this default configuration - that uses a Hetzner private network with the default Flannel CNI built into k3s - may not be optimal for larger clusters: + +1. Private networks in Hetzner cloud supports max 100 nodes, so I recommend you disable the private network in the configuration if you expect your cluster to grow beyond 100 nodes. +2. Flannel is fine for small to medium clusters, but performance starts to degrade with clusters made of several hundreds or thousands of nodes, so I recommend to switch to Cilium as CNI as its performance is excellent and scales well with very large clusters. + +Notes: +- if you disable the private network due to the limitation mentioned above, encryption will be enforced at CNI level to secure the traffic between nodes over the public network. +- if you want to use something other than Cilium or Flannel (e.g. Calico), then you can disable the automatic setup of the CNI so you can install a CNI of your choice. We may add built in support for more CNIs in future releases. + +## Registry mirror + +v2.0.0 introduces a setting to optionally enable the `embedded registry mirror` in k3s (see [this page](https://docs.k3s.io/installation/registry-mirror) for more information. This is basically an installation of [Spegel](https://github.com/spegel-org/spegel) which enables peer-to-peer distribution of container images between the nodes of a cluster. This can help avoid problems with nodes not being able to pull images because their IPs have been banned by registry (due to malicious use of the same IPs in the past or similar reason), because a node will try pulling an image from other nodes via the embedded registry mirror, before pulling the image from the upstream registry. This also speeds up pods creation because less time is spent downloading images from the upstream registries when deployments have many replicas spread across many nodes.