From 0c6e1d6360cebd0814e1656c5899dd8c91094573 Mon Sep 17 00:00:00 2001 From: dbw7 Date: Fri, 22 Nov 2024 01:55:20 -0500 Subject: [PATCH] feedback updates + almost done --- RELEASE_NOTES.md | 3 +- pkg/combustion/kubernetes.go | 53 +- pkg/combustion/kubernetes_test.go | 1175 ++++++++++++++++- .../templates/k3s-multi-node-installer.sh.tpl | 1 - .../k3s-single-node-installer.sh.tpl | 1 - pkg/combustion/templates/k8s-vip.yaml.tpl | 8 +- .../rke2-multi-node-installer.sh.tpl | 1 - .../rke2-single-node-installer.sh.tpl | 1 - pkg/combustion/templates/set-node-ip.sh.tpl | 16 +- pkg/image/validation/kubernetes.go | 161 ++- pkg/image/validation/kubernetes_test.go | 166 ++- pkg/kubernetes/cluster.go | 18 +- pkg/kubernetes/cluster_test.go | 234 +++- .../testdata/{ => default}/agent.yaml | 0 .../testdata/{ => default}/server.yaml | 0 .../testdata/dualstack-prio-ipv4/agent.yaml | 3 + .../testdata/dualstack-prio-ipv4/server.yaml | 5 + .../testdata/dualstack-prio-ipv6/agent.yaml | 3 + .../testdata/dualstack-prio-ipv6/server.yaml | 5 + 19 files changed, 1688 insertions(+), 166 deletions(-) rename pkg/kubernetes/testdata/{ => default}/agent.yaml (100%) rename pkg/kubernetes/testdata/{ => default}/server.yaml (100%) create mode 100644 pkg/kubernetes/testdata/dualstack-prio-ipv4/agent.yaml create mode 100644 pkg/kubernetes/testdata/dualstack-prio-ipv4/server.yaml create mode 100644 pkg/kubernetes/testdata/dualstack-prio-ipv6/agent.yaml create mode 100644 pkg/kubernetes/testdata/dualstack-prio-ipv6/server.yaml diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c87487f3..4be9b13b 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -4,7 +4,8 @@ ## General -* Added support for IPv6 addresses in the Kubernetes 'apiVIP' field +* Added 'apiVIP6' in Kubernetes field for IPv6 addresses +* Implemented IPv6 Single and Dual-Stack handling for Kubernetes ## API diff --git a/pkg/combustion/kubernetes.go b/pkg/combustion/kubernetes.go index 231bbb37..dbfa9c0d 100644 --- a/pkg/combustion/kubernetes.go +++ b/pkg/combustion/kubernetes.go @@ -3,7 +3,6 @@ package combustion import ( _ "embed" "fmt" - "net/netip" "os" "path/filepath" "strings" @@ -169,12 +168,12 @@ func (c *Combustion) configureK3S(ctx *image.Context, cluster *kubernetes.Cluste singleNode := len(ctx.ImageDefinition.Kubernetes.Nodes) < 2 if singleNode { - if ctx.ImageDefinition.Kubernetes.Network.APIVIP4 == "" || ctx.ImageDefinition.Kubernetes.Network.APIVIP6 == "" { - zap.S().Info("Virtual IP address for k3s cluster is not provided and will not be configured") + if ctx.ImageDefinition.Kubernetes.Network.APIVIP4 == "" && ctx.ImageDefinition.Kubernetes.Network.APIVIP6 == "" { + zap.S().Info("Virtual IP address(es) for k3s cluster is not provided and will not be configured") } else { - log.Audit("WARNING: A Virtual IP address for the k3s cluster has been provided. " + + log.Audit("WARNING: Virtual IP address(es) for the k3s cluster provided. " + "An external IP address for the Ingress Controller (Traefik) must be manually configured.") - zap.S().Warn("Virtual IP address for k3s cluster is requested and will invalidate Traefik configuration") + zap.S().Warn("Virtual IP address(es) for k3s cluster requested and will invalidate Traefik configuration") } templateValues["configFile"] = k8sServerConfigFile @@ -183,7 +182,7 @@ func (c *Combustion) configureK3S(ctx *image.Context, cluster *kubernetes.Cluste } log.Audit("WARNING: An external IP address for the Ingress Controller (Traefik) must be manually configured in multi-node clusters.") - zap.S().Warn("Virtual IP address for k3s cluster is necessary for multi node clusters and will invalidate Traefik configuration") + zap.S().Warn("Virtual IP address(es) for k3s cluster necessary for multi node clusters and will invalidate Traefik configuration") templateValues["nodes"] = ctx.ImageDefinition.Kubernetes.Nodes templateValues["initialiser"] = cluster.InitialiserName @@ -266,8 +265,8 @@ func (c *Combustion) configureRKE2(ctx *image.Context, cluster *kubernetes.Clust singleNode := len(ctx.ImageDefinition.Kubernetes.Nodes) < 2 if singleNode { - if ctx.ImageDefinition.Kubernetes.Network.APIVIP4 == "" || ctx.ImageDefinition.Kubernetes.Network.APIVIP6 == "" { - zap.S().Info("Virtual IP address for RKE2 cluster is not provided and will not be configured") + if ctx.ImageDefinition.Kubernetes.Network.APIVIP4 == "" && ctx.ImageDefinition.Kubernetes.Network.APIVIP6 == "" { + zap.S().Info("Virtual IP address(es) for RKE2 cluster is not provided and will not be configured") } templateValues["configFile"] = k8sServerConfigFile @@ -329,36 +328,21 @@ func (c *Combustion) downloadRKE2Artefacts(ctx *image.Context, cluster *kubernet } func kubernetesVIPManifest(k *image.Kubernetes) (string, error) { - var err error - - var ip4 netip.Addr - if k.Network.APIVIP4 != "" { - ip4, err = netip.ParseAddr(k.Network.APIVIP4) - if err != nil { - return "", fmt.Errorf("parsing kubernetes apiVIP address: %w", err) - } - } - - var ip6 netip.Addr - if k.Network.APIVIP6 != "" { - ip6, err = netip.ParseAddr(k.Network.APIVIP6) - if err != nil { - return "", fmt.Errorf("parsing kubernetes apiVIP address: %w", err) - } + onlyIPv6 := false + if k.Network.APIVIP4 == "" && k.Network.APIVIP6 != "" { + onlyIPv6 = true } manifest := struct { APIAddress4 string APIAddress6 string - IsIPV4 bool - IsIPV6 bool RKE2 bool + OnlyIPv6 bool }{ APIAddress4: k.Network.APIVIP4, APIAddress6: k.Network.APIVIP6, - IsIPV4: ip4.Is4(), - IsIPV6: ip6.Is6(), RKE2: strings.Contains(k.Version, image.KubernetesDistroRKE2), + OnlyIPv6: onlyIPv6, } return template.Parse("k8s-vip", k8sVIPManifest, &manifest) @@ -383,7 +367,7 @@ func createNodeIPScript(ctx *image.Context, serverConfig map[string]any) error { }{ IPv4Enabled: isIPv4Enabled, IPv6Enabled: true, - PrioritizeIPv6: isIPv6Priority(serverConfig), + PrioritizeIPv6: kubernetes.IsIPv6Priority(serverConfig), RKE2: strings.Contains(ctx.ImageDefinition.Kubernetes.Version, image.KubernetesDistroRKE2), } @@ -523,14 +507,3 @@ func HelmCertsPath(ctx *image.Context) string { func kubernetesArtefactsPath(ctx *image.Context) string { return filepath.Join(ctx.ArtefactsDir, k8sDir) } - -func isIPv6Priority(serverConfig map[string]any) bool { - if clusterCIDR, ok := serverConfig["cluster-cidr"].(string); ok { - cidrs := strings.Split(clusterCIDR, ",") - if len(cidrs) > 0 { - return strings.Contains(cidrs[0], ":") - } - } - - return false -} diff --git a/pkg/combustion/kubernetes_test.go b/pkg/combustion/kubernetes_test.go index 50de6998..47e1f44d 100644 --- a/pkg/combustion/kubernetes_test.go +++ b/pkg/combustion/kubernetes_test.go @@ -213,7 +213,7 @@ func TestConfigureKubernetes_ArtefactDownloaderErrorRKE2(t *testing.T) { assert.Nil(t, scripts) } -func TestConfigureKubernetes_SuccessfulSingleNodeK3sCluster(t *testing.T) { +func TestConfigureKubernetes_SuccessfulSingleNodeK3sClusterIPv4(t *testing.T) { ctx, teardown := setupContext(t) defer teardown() @@ -264,6 +264,7 @@ func TestConfigureKubernetes_SuccessfulSingleNodeK3sCluster(t *testing.T) { assert.Contains(t, contents, "chmod +x $INSTALL_K3S_BIN_DIR/k3s") assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/install/cool-k3s-binary $INSTALL_K3S_BIN_DIR/k3s") assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-kubernetes.sh") + assert.NotContains(t, contents, "sh set-node-ip.sh") // Config file assertions configPath := filepath.Join(ctx.ArtefactsDir, "kubernetes", "server.yaml") @@ -285,12 +286,904 @@ func TestConfigureKubernetes_SuccessfulSingleNodeK3sCluster(t *testing.T) { assert.Equal(t, []any{"servicelb"}, configContents["disable"]) } -func TestConfigureKubernetes_SuccessfulMultiNodeK3sCluster(t *testing.T) { +func TestConfigureKubernetes_SuccessfulSingleNodeK3sClusterIPv6(t *testing.T) { ctx, teardown := setupContext(t) defer teardown() ctx.ImageDefinition.Kubernetes = image.Kubernetes{ Version: "v1.30.3+k3s1", + Network: image.Network{ + APIVIP6: "fd12:3456:789a::21", + APIHost: "api.cluster01.hosted.on.edge.suse.com", + }, + } + + c := Combustion{ + KubernetesScriptDownloader: mockKubernetesScriptDownloader{ + downloadScript: func(distribution, destPath string) (string, error) { + return kubernetesScriptInstaller, nil + }, + }, + KubernetesArtefactDownloader: mockKubernetesArtefactDownloader{ + downloadK3sArtefacts: func(arch image.Arch, version string, installPath, imagesPath string) error { + binary := filepath.Join(installPath, "cool-k3s-binary") + return os.WriteFile(binary, nil, os.ModePerm) + }, + }, + } + + scripts, err := c.configureKubernetes(ctx) + require.NoError(t, err) + require.Len(t, scripts, 1) + + // Script file assertions + scriptPath := filepath.Join(ctx.CombustionDir, scripts[0]) + + info, err := os.Stat(scriptPath) + require.NoError(t, err) + + assert.Equal(t, fileio.ExecutablePerms, info.Mode()) + + b, err := os.ReadFile(scriptPath) + require.NoError(t, err) + + contents := string(b) + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/images/* /var/lib/rancher/k3s/agent/images/") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/server.yaml /etc/rancher/k3s/config.yaml") + assert.Contains(t, contents, "echo \"fd12:3456:789a::21 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "export INSTALL_K3S_SKIP_DOWNLOAD=true") + assert.Contains(t, contents, "export INSTALL_K3S_SKIP_START=true") + assert.Contains(t, contents, "export INSTALL_K3S_BIN_DIR=/opt/bin") + assert.Contains(t, contents, "chmod +x $INSTALL_K3S_BIN_DIR/k3s") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/install/cool-k3s-binary $INSTALL_K3S_BIN_DIR/k3s") + assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-kubernetes.sh") + assert.Contains(t, contents, "sh set-node-ip.sh") + + // Config file assertions + configPath := filepath.Join(ctx.ArtefactsDir, "kubernetes", "server.yaml") + + info, err = os.Stat(configPath) + require.NoError(t, err) + + assert.Equal(t, fileio.NonExecutablePerms, info.Mode()) + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + var configContents map[string]any + require.NoError(t, yaml.Unmarshal(b, &configContents)) + + assert.Nil(t, configContents["cni"]) + assert.Nil(t, configContents["server"]) + assert.Equal(t, []any{"fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + assert.Equal(t, []any{"servicelb"}, configContents["disable"]) +} + +func TestConfigureKubernetes_SuccessfulSingleNodeK3sClusterDualstack(t *testing.T) { + ctx, teardown := setupContext(t) + defer teardown() + + ctx.ImageDefinition.Kubernetes = image.Kubernetes{ + Version: "v1.30.3+k3s1", + Network: image.Network{ + APIVIP4: "192.168.122.100", + APIVIP6: "fd12:3456:789a::21", + APIHost: "api.cluster01.hosted.on.edge.suse.com", + }, + } + + c := Combustion{ + KubernetesScriptDownloader: mockKubernetesScriptDownloader{ + downloadScript: func(distribution, destPath string) (string, error) { + return kubernetesScriptInstaller, nil + }, + }, + KubernetesArtefactDownloader: mockKubernetesArtefactDownloader{ + downloadK3sArtefacts: func(arch image.Arch, version string, installPath, imagesPath string) error { + binary := filepath.Join(installPath, "cool-k3s-binary") + return os.WriteFile(binary, nil, os.ModePerm) + }, + }, + } + + scripts, err := c.configureKubernetes(ctx) + require.NoError(t, err) + require.Len(t, scripts, 1) + + // Script file assertions + scriptPath := filepath.Join(ctx.CombustionDir, scripts[0]) + + info, err := os.Stat(scriptPath) + require.NoError(t, err) + + assert.Equal(t, fileio.ExecutablePerms, info.Mode()) + + b, err := os.ReadFile(scriptPath) + require.NoError(t, err) + + contents := string(b) + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/images/* /var/lib/rancher/k3s/agent/images/") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/server.yaml /etc/rancher/k3s/config.yaml") + assert.Contains(t, contents, "echo \"fd12:3456:789a::21 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "export INSTALL_K3S_SKIP_DOWNLOAD=true") + assert.Contains(t, contents, "export INSTALL_K3S_SKIP_START=true") + assert.Contains(t, contents, "export INSTALL_K3S_BIN_DIR=/opt/bin") + assert.Contains(t, contents, "chmod +x $INSTALL_K3S_BIN_DIR/k3s") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/install/cool-k3s-binary $INSTALL_K3S_BIN_DIR/k3s") + assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-kubernetes.sh") + assert.Contains(t, contents, "sh set-node-ip.sh") + + // Config file assertions + configPath := filepath.Join(ctx.ArtefactsDir, "kubernetes", "server.yaml") + + info, err = os.Stat(configPath) + require.NoError(t, err) + + assert.Equal(t, fileio.NonExecutablePerms, info.Mode()) + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + var configContents map[string]any + require.NoError(t, yaml.Unmarshal(b, &configContents)) + + assert.Nil(t, configContents["cni"]) + assert.Nil(t, configContents["server"]) + assert.ElementsMatch(t, []any{"192.168.122.100", "fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + assert.Equal(t, []any{"servicelb"}, configContents["disable"]) +} + +func TestConfigureKubernetes_SuccessfulMultiNodeK3sClusterIPv4(t *testing.T) { + ctx, teardown := setupContext(t) + defer teardown() + + ctx.ImageDefinition.Kubernetes = image.Kubernetes{ + Version: "v1.30.3+k3s1", + Network: image.Network{ + APIHost: "api.cluster01.hosted.on.edge.suse.com", + APIVIP4: "192.168.122.100", + }, + Nodes: []image.Node{ + { + Hostname: "node1.suse.com", + Type: "server", + }, + { + Hostname: "node2.suse.com", + Type: "agent", + }, + }, + } + + c := Combustion{ + KubernetesScriptDownloader: mockKubernetesScriptDownloader{ + downloadScript: func(distribution, destPath string) (string, error) { + return kubernetesScriptInstaller, nil + }, + }, + KubernetesArtefactDownloader: mockKubernetesArtefactDownloader{ + downloadK3sArtefacts: func(arch image.Arch, version, installPath, imagesPath string) error { + binary := filepath.Join(installPath, "cool-k3s-binary") + return os.WriteFile(binary, nil, os.ModePerm) + }, + }, + } + + serverConfig := map[string]any{ + "token": "123", + "tls-san": []string{ + "k8s-host.com", + }, + } + + b, err := yaml.Marshal(serverConfig) + require.NoError(t, err) + + configDir := filepath.Join(ctx.ImageConfigDir, k8sDir, k8sConfigDir) + require.NoError(t, os.MkdirAll(configDir, os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(configDir, "server.yaml"), b, os.ModePerm)) + + scripts, err := c.configureKubernetes(ctx) + require.NoError(t, err) + require.Len(t, scripts, 1) + + // Script file assertions + scriptPath := filepath.Join(ctx.CombustionDir, scripts[0]) + + info, err := os.Stat(scriptPath) + require.NoError(t, err) + + assert.Equal(t, fileio.ExecutablePerms, info.Mode()) + + b, err = os.ReadFile(scriptPath) + require.NoError(t, err) + + contents := string(b) + assert.Contains(t, contents, "hosts[node1.suse.com]=server") + assert.Contains(t, contents, "hosts[node2.suse.com]=agent") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/images/* /var/lib/rancher/k3s/agent/images/") + assert.Contains(t, contents, "cp $CONFIGFILE /etc/rancher/k3s/config.yaml") + assert.Contains(t, contents, "if [ \"$HOSTNAME\" = node1.suse.com ]; then") + assert.Contains(t, contents, "echo \"192.168.122.100 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "export INSTALL_K3S_EXEC=$NODETYPE") + assert.Contains(t, contents, "export INSTALL_K3S_SKIP_DOWNLOAD=true") + assert.Contains(t, contents, "export INSTALL_K3S_SKIP_START=true") + assert.Contains(t, contents, "export INSTALL_K3S_BIN_DIR=/opt/bin") + assert.Contains(t, contents, "chmod +x $INSTALL_K3S_BIN_DIR/k3s") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/install/cool-k3s-binary $INSTALL_K3S_BIN_DIR/k3s") + assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-kubernetes.sh") + assert.NotContains(t, contents, "sh set-node-ip.sh") + + // Server config file assertions + configPath := filepath.Join(ctx.ArtefactsDir, "kubernetes", "server.yaml") + + info, err = os.Stat(configPath) + require.NoError(t, err) + + assert.Equal(t, fileio.NonExecutablePerms, info.Mode()) + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + var configContents map[string]any + require.NoError(t, yaml.Unmarshal(b, &configContents)) + + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, "https://192.168.122.100:6443", configContents["server"]) + assert.Equal(t, []any{"k8s-host.com", "192.168.122.100", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + assert.Equal(t, []any{"servicelb"}, configContents["disable"]) + assert.Nil(t, configContents["cluster-init"]) + + // Initialising server config file assertions + configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "init_server.yaml") + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + configContents = map[string]any{} // clear the map + require.NoError(t, yaml.Unmarshal(b, configContents)) + + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, nil, configContents["server"]) + assert.Equal(t, []any{"k8s-host.com", "192.168.122.100", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + assert.Equal(t, []any{"servicelb"}, configContents["disable"]) + assert.Equal(t, true, configContents["cluster-init"]) + + // Agent config file assertions + configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "agent.yaml") + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + configContents = map[string]any{} // clear the map + require.NoError(t, yaml.Unmarshal(b, configContents)) + + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, "https://192.168.122.100:6443", configContents["server"]) + assert.Nil(t, configContents["tls-san"]) + assert.Nil(t, configContents["disable"]) + assert.Nil(t, configContents["cluster-init"]) +} + +func TestConfigureKubernetes_SuccessfulMultiNodeK3sClusterIPv6(t *testing.T) { + ctx, teardown := setupContext(t) + defer teardown() + + ctx.ImageDefinition.Kubernetes = image.Kubernetes{ + Version: "v1.30.3+k3s1", + Network: image.Network{ + APIHost: "api.cluster01.hosted.on.edge.suse.com", + APIVIP6: "fd12:3456:789a::21", + }, + Nodes: []image.Node{ + { + Hostname: "node1.suse.com", + Type: "server", + }, + { + Hostname: "node2.suse.com", + Type: "agent", + }, + }, + } + + c := Combustion{ + KubernetesScriptDownloader: mockKubernetesScriptDownloader{ + downloadScript: func(distribution, destPath string) (string, error) { + return kubernetesScriptInstaller, nil + }, + }, + KubernetesArtefactDownloader: mockKubernetesArtefactDownloader{ + downloadK3sArtefacts: func(arch image.Arch, version, installPath, imagesPath string) error { + binary := filepath.Join(installPath, "cool-k3s-binary") + return os.WriteFile(binary, nil, os.ModePerm) + }, + }, + } + + serverConfig := map[string]any{ + "token": "123", + "tls-san": []string{ + "k8s-host.com", + }, + } + + b, err := yaml.Marshal(serverConfig) + require.NoError(t, err) + + configDir := filepath.Join(ctx.ImageConfigDir, k8sDir, k8sConfigDir) + require.NoError(t, os.MkdirAll(configDir, os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(configDir, "server.yaml"), b, os.ModePerm)) + + scripts, err := c.configureKubernetes(ctx) + require.NoError(t, err) + require.Len(t, scripts, 1) + + // Script file assertions + scriptPath := filepath.Join(ctx.CombustionDir, scripts[0]) + + info, err := os.Stat(scriptPath) + require.NoError(t, err) + + assert.Equal(t, fileio.ExecutablePerms, info.Mode()) + + b, err = os.ReadFile(scriptPath) + require.NoError(t, err) + + contents := string(b) + assert.Contains(t, contents, "hosts[node1.suse.com]=server") + assert.Contains(t, contents, "hosts[node2.suse.com]=agent") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/images/* /var/lib/rancher/k3s/agent/images/") + assert.Contains(t, contents, "cp $CONFIGFILE /etc/rancher/k3s/config.yaml") + assert.Contains(t, contents, "if [ \"$HOSTNAME\" = node1.suse.com ]; then") + assert.Contains(t, contents, "echo \"fd12:3456:789a::21 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "export INSTALL_K3S_EXEC=$NODETYPE") + assert.Contains(t, contents, "export INSTALL_K3S_SKIP_DOWNLOAD=true") + assert.Contains(t, contents, "export INSTALL_K3S_SKIP_START=true") + assert.Contains(t, contents, "export INSTALL_K3S_BIN_DIR=/opt/bin") + assert.Contains(t, contents, "chmod +x $INSTALL_K3S_BIN_DIR/k3s") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/install/cool-k3s-binary $INSTALL_K3S_BIN_DIR/k3s") + assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-kubernetes.sh") + assert.Contains(t, contents, "sh set-node-ip.sh") + + // Server config file assertions + configPath := filepath.Join(ctx.ArtefactsDir, "kubernetes", "server.yaml") + + info, err = os.Stat(configPath) + require.NoError(t, err) + + assert.Equal(t, fileio.NonExecutablePerms, info.Mode()) + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + var configContents map[string]any + require.NoError(t, yaml.Unmarshal(b, &configContents)) + + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, "https://[fd12:3456:789a::21]:6443", configContents["server"]) + assert.Equal(t, []any{"k8s-host.com", "fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + assert.Equal(t, []any{"servicelb"}, configContents["disable"]) + assert.Nil(t, configContents["cluster-init"]) + + // Initialising server config file assertions + configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "init_server.yaml") + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + configContents = map[string]any{} // clear the map + require.NoError(t, yaml.Unmarshal(b, configContents)) + + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, nil, configContents["server"]) + assert.Equal(t, []any{"k8s-host.com", "fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + assert.Equal(t, []any{"servicelb"}, configContents["disable"]) + assert.Equal(t, true, configContents["cluster-init"]) + + // Agent config file assertions + configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "agent.yaml") + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + configContents = map[string]any{} // clear the map + require.NoError(t, yaml.Unmarshal(b, configContents)) + + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, "https://[fd12:3456:789a::21]:6443", configContents["server"]) + assert.Nil(t, configContents["tls-san"]) + assert.Nil(t, configContents["disable"]) + assert.Nil(t, configContents["cluster-init"]) +} + +func TestConfigureKubernetes_SuccessfulMultiNodeK3sClusterDualstackPrioIPv4(t *testing.T) { + ctx, teardown := setupContext(t) + defer teardown() + + ctx.ImageDefinition.Kubernetes = image.Kubernetes{ + Version: "v1.30.3+k3s1", + Network: image.Network{ + APIHost: "api.cluster01.hosted.on.edge.suse.com", + APIVIP4: "192.168.122.100", + APIVIP6: "fd12:3456:789a::21", + }, + Nodes: []image.Node{ + { + Hostname: "node1.suse.com", + Type: "server", + }, + { + Hostname: "node2.suse.com", + Type: "agent", + }, + }, + } + + c := Combustion{ + KubernetesScriptDownloader: mockKubernetesScriptDownloader{ + downloadScript: func(distribution, destPath string) (string, error) { + return kubernetesScriptInstaller, nil + }, + }, + KubernetesArtefactDownloader: mockKubernetesArtefactDownloader{ + downloadK3sArtefacts: func(arch image.Arch, version, installPath, imagesPath string) error { + binary := filepath.Join(installPath, "cool-k3s-binary") + return os.WriteFile(binary, nil, os.ModePerm) + }, + }, + } + + serverConfig := map[string]any{ + "token": "123", + "tls-san": []string{ + "k8s-host.com", + }, + "cluster-cidr": "10.42.0.0/16,fd12:3456:789b::/48", + "service-cidr": "10.43.0.0/16,fd12:3456:789c::/112", + } + + b, err := yaml.Marshal(serverConfig) + require.NoError(t, err) + + configDir := filepath.Join(ctx.ImageConfigDir, k8sDir, k8sConfigDir) + require.NoError(t, os.MkdirAll(configDir, os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(configDir, "server.yaml"), b, os.ModePerm)) + + scripts, err := c.configureKubernetes(ctx) + require.NoError(t, err) + require.Len(t, scripts, 1) + + // Script file assertions + scriptPath := filepath.Join(ctx.CombustionDir, scripts[0]) + + info, err := os.Stat(scriptPath) + require.NoError(t, err) + + assert.Equal(t, fileio.ExecutablePerms, info.Mode()) + + b, err = os.ReadFile(scriptPath) + require.NoError(t, err) + + contents := string(b) + assert.Contains(t, contents, "hosts[node1.suse.com]=server") + assert.Contains(t, contents, "hosts[node2.suse.com]=agent") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/images/* /var/lib/rancher/k3s/agent/images/") + assert.Contains(t, contents, "cp $CONFIGFILE /etc/rancher/k3s/config.yaml") + assert.Contains(t, contents, "if [ \"$HOSTNAME\" = node1.suse.com ]; then") + assert.Contains(t, contents, "echo \"192.168.122.100 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "echo \"fd12:3456:789a::21 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "export INSTALL_K3S_EXEC=$NODETYPE") + assert.Contains(t, contents, "export INSTALL_K3S_SKIP_DOWNLOAD=true") + assert.Contains(t, contents, "export INSTALL_K3S_SKIP_START=true") + assert.Contains(t, contents, "export INSTALL_K3S_BIN_DIR=/opt/bin") + assert.Contains(t, contents, "chmod +x $INSTALL_K3S_BIN_DIR/k3s") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/install/cool-k3s-binary $INSTALL_K3S_BIN_DIR/k3s") + assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-kubernetes.sh") + assert.Contains(t, contents, "sh set-node-ip.sh") + + // Server config file assertions + configPath := filepath.Join(ctx.ArtefactsDir, "kubernetes", "server.yaml") + + info, err = os.Stat(configPath) + require.NoError(t, err) + + assert.Equal(t, fileio.NonExecutablePerms, info.Mode()) + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + var configContents map[string]any + require.NoError(t, yaml.Unmarshal(b, &configContents)) + + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, "https://192.168.122.100:6443", configContents["server"]) + assert.ElementsMatch(t, []any{"k8s-host.com", "192.168.122.100", "fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + assert.Equal(t, []any{"servicelb"}, configContents["disable"]) + assert.Nil(t, configContents["cluster-init"]) + + // Initialising server config file assertions + configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "init_server.yaml") + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + configContents = map[string]any{} // clear the map + require.NoError(t, yaml.Unmarshal(b, configContents)) + + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, nil, configContents["server"]) + assert.ElementsMatch(t, []any{"k8s-host.com", "192.168.122.100", "fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + assert.Equal(t, []any{"servicelb"}, configContents["disable"]) + assert.Equal(t, true, configContents["cluster-init"]) + + // Agent config file assertions + configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "agent.yaml") + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + configContents = map[string]any{} // clear the map + require.NoError(t, yaml.Unmarshal(b, configContents)) + + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, "https://192.168.122.100:6443", configContents["server"]) + assert.Nil(t, configContents["tls-san"]) + assert.Nil(t, configContents["disable"]) + assert.Nil(t, configContents["cluster-init"]) +} + +func TestConfigureKubernetes_SuccessfulMultiNodeK3sClusterDualstackPrioIPv6(t *testing.T) { + ctx, teardown := setupContext(t) + defer teardown() + + ctx.ImageDefinition.Kubernetes = image.Kubernetes{ + Version: "v1.30.3+k3s1", + Network: image.Network{ + APIHost: "api.cluster01.hosted.on.edge.suse.com", + APIVIP4: "192.168.122.100", + APIVIP6: "fd12:3456:789a::21", + }, + Nodes: []image.Node{ + { + Hostname: "node1.suse.com", + Type: "server", + }, + { + Hostname: "node2.suse.com", + Type: "agent", + }, + }, + } + + c := Combustion{ + KubernetesScriptDownloader: mockKubernetesScriptDownloader{ + downloadScript: func(distribution, destPath string) (string, error) { + return kubernetesScriptInstaller, nil + }, + }, + KubernetesArtefactDownloader: mockKubernetesArtefactDownloader{ + downloadK3sArtefacts: func(arch image.Arch, version, installPath, imagesPath string) error { + binary := filepath.Join(installPath, "cool-k3s-binary") + return os.WriteFile(binary, nil, os.ModePerm) + }, + }, + } + + serverConfig := map[string]any{ + "token": "123", + "tls-san": []string{ + "k8s-host.com", + }, + "cluster-cidr": "fd12:3456:789b::/48,10.42.0.0/16", + "service-cidr": "fd12:3456:789c::/112,10.43.0.0/16", + } + + b, err := yaml.Marshal(serverConfig) + require.NoError(t, err) + + configDir := filepath.Join(ctx.ImageConfigDir, k8sDir, k8sConfigDir) + require.NoError(t, os.MkdirAll(configDir, os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(configDir, "server.yaml"), b, os.ModePerm)) + + scripts, err := c.configureKubernetes(ctx) + require.NoError(t, err) + require.Len(t, scripts, 1) + + // Script file assertions + scriptPath := filepath.Join(ctx.CombustionDir, scripts[0]) + + info, err := os.Stat(scriptPath) + require.NoError(t, err) + + assert.Equal(t, fileio.ExecutablePerms, info.Mode()) + + b, err = os.ReadFile(scriptPath) + require.NoError(t, err) + + contents := string(b) + assert.Contains(t, contents, "hosts[node1.suse.com]=server") + assert.Contains(t, contents, "hosts[node2.suse.com]=agent") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/images/* /var/lib/rancher/k3s/agent/images/") + assert.Contains(t, contents, "cp $CONFIGFILE /etc/rancher/k3s/config.yaml") + assert.Contains(t, contents, "if [ \"$HOSTNAME\" = node1.suse.com ]; then") + assert.Contains(t, contents, "echo \"192.168.122.100 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "echo \"fd12:3456:789a::21 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "export INSTALL_K3S_EXEC=$NODETYPE") + assert.Contains(t, contents, "export INSTALL_K3S_SKIP_DOWNLOAD=true") + assert.Contains(t, contents, "export INSTALL_K3S_SKIP_START=true") + assert.Contains(t, contents, "export INSTALL_K3S_BIN_DIR=/opt/bin") + assert.Contains(t, contents, "chmod +x $INSTALL_K3S_BIN_DIR/k3s") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/install/cool-k3s-binary $INSTALL_K3S_BIN_DIR/k3s") + assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-kubernetes.sh") + assert.Contains(t, contents, "sh set-node-ip.sh") + + // Server config file assertions + configPath := filepath.Join(ctx.ArtefactsDir, "kubernetes", "server.yaml") + + info, err = os.Stat(configPath) + require.NoError(t, err) + + assert.Equal(t, fileio.NonExecutablePerms, info.Mode()) + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + var configContents map[string]any + require.NoError(t, yaml.Unmarshal(b, &configContents)) + + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, "https://[fd12:3456:789a::21]:6443", configContents["server"]) + assert.ElementsMatch(t, []any{"k8s-host.com", "192.168.122.100", "fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + assert.Equal(t, []any{"servicelb"}, configContents["disable"]) + assert.Nil(t, configContents["cluster-init"]) + + // Initialising server config file assertions + configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "init_server.yaml") + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + configContents = map[string]any{} // clear the map + require.NoError(t, yaml.Unmarshal(b, configContents)) + + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, nil, configContents["server"]) + assert.ElementsMatch(t, []any{"k8s-host.com", "192.168.122.100", "fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + assert.Equal(t, []any{"servicelb"}, configContents["disable"]) + assert.Equal(t, true, configContents["cluster-init"]) + + // Agent config file assertions + configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "agent.yaml") + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + configContents = map[string]any{} // clear the map + require.NoError(t, yaml.Unmarshal(b, configContents)) + + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, "https://[fd12:3456:789a::21]:6443", configContents["server"]) + assert.Nil(t, configContents["tls-san"]) + assert.Nil(t, configContents["disable"]) + assert.Nil(t, configContents["cluster-init"]) +} + +func TestConfigureKubernetes_SuccessfulSingleNodeRKE2ClusterIPv4(t *testing.T) { + ctx, teardown := setupContext(t) + defer teardown() + + ctx.ImageDefinition.Kubernetes = image.Kubernetes{ + Version: "v1.30.3+rke2r1", + Network: image.Network{ + APIVIP4: "192.168.122.100", + APIHost: "api.cluster01.hosted.on.edge.suse.com", + }, + } + + c := Combustion{ + KubernetesScriptDownloader: mockKubernetesScriptDownloader{ + downloadScript: func(distribution, destPath string) (string, error) { + return kubernetesScriptInstaller, nil + }, + }, + KubernetesArtefactDownloader: mockKubernetesArtefactDownloader{ + downloadRKE2Artefacts: func(arch image.Arch, version, cni string, multusEnabled bool, installPath, imagesPath string) error { + return nil + }, + }, + } + + scripts, err := c.configureKubernetes(ctx) + require.NoError(t, err) + require.Len(t, scripts, 1) + + // Script file assertions + scriptPath := filepath.Join(ctx.CombustionDir, scripts[0]) + + info, err := os.Stat(scriptPath) + require.NoError(t, err) + + assert.Equal(t, fileio.ExecutablePerms, info.Mode()) + + b, err := os.ReadFile(scriptPath) + require.NoError(t, err) + + contents := string(b) + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/images/* /var/lib/rancher/rke2/agent/images/") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/server.yaml /etc/rancher/rke2/config.yaml") + assert.Contains(t, contents, "echo \"192.168.122.100 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "export INSTALL_RKE2_ARTIFACT_PATH=$ARTEFACTS_DIR/kubernetes/install") + assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-kubernetes.sh") + assert.Contains(t, contents, "systemctl enable rke2-server.service") + assert.NotContains(t, contents, "sh set-node-ip.sh") + + // Config file assertions + configPath := filepath.Join(ctx.ArtefactsDir, "kubernetes", "server.yaml") + + info, err = os.Stat(configPath) + require.NoError(t, err) + + assert.Equal(t, fileio.NonExecutablePerms, info.Mode()) + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + var configContents map[string]any + require.NoError(t, yaml.Unmarshal(b, &configContents)) + + require.Contains(t, configContents, "cni") + assert.Equal(t, "cilium", configContents["cni"], "default CNI is not set") + assert.Equal(t, nil, configContents["server"]) + assert.Equal(t, []any{"192.168.122.100", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) +} + +func TestConfigureKubernetes_SuccessfulSingleNodeRKE2ClusterIPv6(t *testing.T) { + ctx, teardown := setupContext(t) + defer teardown() + + ctx.ImageDefinition.Kubernetes = image.Kubernetes{ + Version: "v1.30.3+rke2r1", + Network: image.Network{ + APIVIP6: "fd12:3456:789a::21", + APIHost: "api.cluster01.hosted.on.edge.suse.com", + }, + } + + c := Combustion{ + KubernetesScriptDownloader: mockKubernetesScriptDownloader{ + downloadScript: func(distribution, destPath string) (string, error) { + return kubernetesScriptInstaller, nil + }, + }, + KubernetesArtefactDownloader: mockKubernetesArtefactDownloader{ + downloadRKE2Artefacts: func(arch image.Arch, version, cni string, multusEnabled bool, installPath, imagesPath string) error { + return nil + }, + }, + } + + scripts, err := c.configureKubernetes(ctx) + require.NoError(t, err) + require.Len(t, scripts, 1) + + // Script file assertions + scriptPath := filepath.Join(ctx.CombustionDir, scripts[0]) + + info, err := os.Stat(scriptPath) + require.NoError(t, err) + + assert.Equal(t, fileio.ExecutablePerms, info.Mode()) + + b, err := os.ReadFile(scriptPath) + require.NoError(t, err) + + contents := string(b) + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/images/* /var/lib/rancher/rke2/agent/images/") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/server.yaml /etc/rancher/rke2/config.yaml") + assert.Contains(t, contents, "echo \"fd12:3456:789a::21 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "export INSTALL_RKE2_ARTIFACT_PATH=$ARTEFACTS_DIR/kubernetes/install") + assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-kubernetes.sh") + assert.Contains(t, contents, "systemctl enable rke2-server.service") + assert.Contains(t, contents, "sh set-node-ip.sh") + + // Config file assertions + configPath := filepath.Join(ctx.ArtefactsDir, "kubernetes", "server.yaml") + + info, err = os.Stat(configPath) + require.NoError(t, err) + + assert.Equal(t, fileio.NonExecutablePerms, info.Mode()) + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + var configContents map[string]any + require.NoError(t, yaml.Unmarshal(b, &configContents)) + + require.Contains(t, configContents, "cni") + assert.Equal(t, "cilium", configContents["cni"], "default CNI is not set") + assert.Equal(t, nil, configContents["server"]) + assert.Equal(t, []any{"fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) +} + +func TestConfigureKubernetes_SuccessfulSingleNodeRKE2ClusterDualstack(t *testing.T) { + ctx, teardown := setupContext(t) + defer teardown() + + ctx.ImageDefinition.Kubernetes = image.Kubernetes{ + Version: "v1.30.3+rke2r1", + Network: image.Network{ + APIVIP4: "192.168.122.100", + APIVIP6: "fd12:3456:789a::21", + APIHost: "api.cluster01.hosted.on.edge.suse.com", + }, + } + + c := Combustion{ + KubernetesScriptDownloader: mockKubernetesScriptDownloader{ + downloadScript: func(distribution, destPath string) (string, error) { + return kubernetesScriptInstaller, nil + }, + }, + KubernetesArtefactDownloader: mockKubernetesArtefactDownloader{ + downloadRKE2Artefacts: func(arch image.Arch, version, cni string, multusEnabled bool, installPath, imagesPath string) error { + return nil + }, + }, + } + + scripts, err := c.configureKubernetes(ctx) + require.NoError(t, err) + require.Len(t, scripts, 1) + + // Script file assertions + scriptPath := filepath.Join(ctx.CombustionDir, scripts[0]) + + info, err := os.Stat(scriptPath) + require.NoError(t, err) + + assert.Equal(t, fileio.ExecutablePerms, info.Mode()) + + b, err := os.ReadFile(scriptPath) + require.NoError(t, err) + + contents := string(b) + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/images/* /var/lib/rancher/rke2/agent/images/") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/server.yaml /etc/rancher/rke2/config.yaml") + assert.Contains(t, contents, "echo \"192.168.122.100 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "echo \"fd12:3456:789a::21 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "export INSTALL_RKE2_ARTIFACT_PATH=$ARTEFACTS_DIR/kubernetes/install") + assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-kubernetes.sh") + assert.Contains(t, contents, "systemctl enable rke2-server.service") + assert.Contains(t, contents, "sh set-node-ip.sh") + + // Config file assertions + configPath := filepath.Join(ctx.ArtefactsDir, "kubernetes", "server.yaml") + + info, err = os.Stat(configPath) + require.NoError(t, err) + + assert.Equal(t, fileio.NonExecutablePerms, info.Mode()) + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + var configContents map[string]any + require.NoError(t, yaml.Unmarshal(b, &configContents)) + + require.Contains(t, configContents, "cni") + assert.Equal(t, "cilium", configContents["cni"], "default CNI is not set") + assert.Equal(t, nil, configContents["server"]) + assert.ElementsMatch(t, []any{"192.168.122.100", "fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) +} + +func TestConfigureKubernetes_SuccessfulMultiNodeRKE2ClusterIPv4(t *testing.T) { + ctx, teardown := setupContext(t) + defer teardown() + + ctx.ImageDefinition.Kubernetes = image.Kubernetes{ + Version: "v1.30.3+rke2r1", Network: image.Network{ APIHost: "api.cluster01.hosted.on.edge.suse.com", APIVIP4: "192.168.122.100", @@ -314,17 +1207,17 @@ func TestConfigureKubernetes_SuccessfulMultiNodeK3sCluster(t *testing.T) { }, }, KubernetesArtefactDownloader: mockKubernetesArtefactDownloader{ - downloadK3sArtefacts: func(arch image.Arch, version, installPath, imagesPath string) error { - binary := filepath.Join(installPath, "cool-k3s-binary") - return os.WriteFile(binary, nil, os.ModePerm) + downloadRKE2Artefacts: func(arch image.Arch, version, cni string, multusEnabled bool, installPath, imagesPath string) error { + return nil }, }, } serverConfig := map[string]any{ "token": "123", + "cni": "canal", "tls-san": []string{ - "192-168-122-100.sslip.io", + "k8s-host.com", }, } @@ -353,17 +1246,14 @@ func TestConfigureKubernetes_SuccessfulMultiNodeK3sCluster(t *testing.T) { contents := string(b) assert.Contains(t, contents, "hosts[node1.suse.com]=server") assert.Contains(t, contents, "hosts[node2.suse.com]=agent") - assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/images/* /var/lib/rancher/k3s/agent/images/") - assert.Contains(t, contents, "cp $CONFIGFILE /etc/rancher/k3s/config.yaml") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/images/* /var/lib/rancher/rke2/agent/images/") + assert.Contains(t, contents, "cp $CONFIGFILE /etc/rancher/rke2/config.yaml") assert.Contains(t, contents, "if [ \"$HOSTNAME\" = node1.suse.com ]; then") assert.Contains(t, contents, "echo \"192.168.122.100 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") - assert.Contains(t, contents, "export INSTALL_K3S_EXEC=$NODETYPE") - assert.Contains(t, contents, "export INSTALL_K3S_SKIP_DOWNLOAD=true") - assert.Contains(t, contents, "export INSTALL_K3S_SKIP_START=true") - assert.Contains(t, contents, "export INSTALL_K3S_BIN_DIR=/opt/bin") - assert.Contains(t, contents, "chmod +x $INSTALL_K3S_BIN_DIR/k3s") - assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/install/cool-k3s-binary $INSTALL_K3S_BIN_DIR/k3s") + assert.Contains(t, contents, "export INSTALL_RKE2_ARTIFACT_PATH=$ARTEFACTS_DIR/kubernetes/install") assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-kubernetes.sh") + assert.Contains(t, contents, "systemctl enable rke2-$NODETYPE.service") + assert.NotContains(t, contents, "sh set-node-ip.sh") // Server config file assertions configPath := filepath.Join(ctx.ArtefactsDir, "kubernetes", "server.yaml") @@ -379,11 +1269,10 @@ func TestConfigureKubernetes_SuccessfulMultiNodeK3sCluster(t *testing.T) { var configContents map[string]any require.NoError(t, yaml.Unmarshal(b, &configContents)) + assert.Equal(t, "canal", configContents["cni"]) assert.Equal(t, "123", configContents["token"]) - assert.Equal(t, "https://192.168.122.100:6443", configContents["server"]) - assert.Equal(t, []any{"192-168-122-100.sslip.io", "192.168.122.100", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) - assert.Equal(t, []any{"servicelb"}, configContents["disable"]) - assert.Nil(t, configContents["cluster-init"]) + assert.Equal(t, "https://192.168.122.100:9345", configContents["server"]) + assert.Equal(t, []any{"k8s-host.com", "192.168.122.100", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) // Initialising server config file assertions configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "init_server.yaml") @@ -394,11 +1283,10 @@ func TestConfigureKubernetes_SuccessfulMultiNodeK3sCluster(t *testing.T) { configContents = map[string]any{} // clear the map require.NoError(t, yaml.Unmarshal(b, configContents)) + assert.Equal(t, "canal", configContents["cni"]) assert.Equal(t, "123", configContents["token"]) assert.Equal(t, nil, configContents["server"]) - assert.Equal(t, []any{"192-168-122-100.sslip.io", "192.168.122.100", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) - assert.Equal(t, []any{"servicelb"}, configContents["disable"]) - assert.Equal(t, true, configContents["cluster-init"]) + assert.Equal(t, []any{"k8s-host.com", "192.168.122.100", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) // Agent config file assertions configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "agent.yaml") @@ -409,22 +1297,31 @@ func TestConfigureKubernetes_SuccessfulMultiNodeK3sCluster(t *testing.T) { configContents = map[string]any{} // clear the map require.NoError(t, yaml.Unmarshal(b, configContents)) + assert.Equal(t, "canal", configContents["cni"]) assert.Equal(t, "123", configContents["token"]) - assert.Equal(t, "https://192.168.122.100:6443", configContents["server"]) + assert.Equal(t, "https://192.168.122.100:9345", configContents["server"]) assert.Nil(t, configContents["tls-san"]) - assert.Nil(t, configContents["disable"]) - assert.Nil(t, configContents["cluster-init"]) } -func TestConfigureKubernetes_SuccessfulSingleNodeRKE2Cluster(t *testing.T) { +func TestConfigureKubernetes_SuccessfulMultiNodeRKE2ClusterIPv6(t *testing.T) { ctx, teardown := setupContext(t) defer teardown() ctx.ImageDefinition.Kubernetes = image.Kubernetes{ Version: "v1.30.3+rke2r1", Network: image.Network{ - APIVIP4: "192.168.122.100", APIHost: "api.cluster01.hosted.on.edge.suse.com", + APIVIP6: "fd12:3456:789a::21", + }, + Nodes: []image.Node{ + { + Hostname: "node1.suse.com", + Type: "server", + }, + { + Hostname: "node2.suse.com", + Type: "agent", + }, }, } @@ -441,6 +1338,21 @@ func TestConfigureKubernetes_SuccessfulSingleNodeRKE2Cluster(t *testing.T) { }, } + serverConfig := map[string]any{ + "token": "123", + "cni": "canal", + "tls-san": []string{ + "k8s-host.com", + }, + } + + b, err := yaml.Marshal(serverConfig) + require.NoError(t, err) + + configDir := filepath.Join(ctx.ImageConfigDir, k8sDir, k8sConfigDir) + require.NoError(t, os.MkdirAll(configDir, os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(configDir, "server.yaml"), b, os.ModePerm)) + scripts, err := c.configureKubernetes(ctx) require.NoError(t, err) require.Len(t, scripts, 1) @@ -453,18 +1365,22 @@ func TestConfigureKubernetes_SuccessfulSingleNodeRKE2Cluster(t *testing.T) { assert.Equal(t, fileio.ExecutablePerms, info.Mode()) - b, err := os.ReadFile(scriptPath) + b, err = os.ReadFile(scriptPath) require.NoError(t, err) contents := string(b) + assert.Contains(t, contents, "hosts[node1.suse.com]=server") + assert.Contains(t, contents, "hosts[node2.suse.com]=agent") assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/images/* /var/lib/rancher/rke2/agent/images/") - assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/server.yaml /etc/rancher/rke2/config.yaml") - assert.Contains(t, contents, "echo \"192.168.122.100 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "cp $CONFIGFILE /etc/rancher/rke2/config.yaml") + assert.Contains(t, contents, "if [ \"$HOSTNAME\" = node1.suse.com ]; then") + assert.Contains(t, contents, "echo \"fd12:3456:789a::21 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") assert.Contains(t, contents, "export INSTALL_RKE2_ARTIFACT_PATH=$ARTEFACTS_DIR/kubernetes/install") assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-kubernetes.sh") - assert.Contains(t, contents, "systemctl enable rke2-server.service") + assert.Contains(t, contents, "systemctl enable rke2-$NODETYPE.service") + assert.Contains(t, contents, "sh set-node-ip.sh") - // Config file assertions + // Server config file assertions configPath := filepath.Join(ctx.ArtefactsDir, "kubernetes", "server.yaml") info, err = os.Stat(configPath) @@ -478,13 +1394,41 @@ func TestConfigureKubernetes_SuccessfulSingleNodeRKE2Cluster(t *testing.T) { var configContents map[string]any require.NoError(t, yaml.Unmarshal(b, &configContents)) - require.Contains(t, configContents, "cni") - assert.Equal(t, "cilium", configContents["cni"], "default CNI is not set") + assert.Equal(t, "canal", configContents["cni"]) + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, "https://[fd12:3456:789a::21]:9345", configContents["server"]) + assert.Equal(t, []any{"k8s-host.com", "fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + + // Initialising server config file assertions + configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "init_server.yaml") + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + configContents = map[string]any{} // clear the map + require.NoError(t, yaml.Unmarshal(b, configContents)) + + assert.Equal(t, "canal", configContents["cni"]) + assert.Equal(t, "123", configContents["token"]) assert.Equal(t, nil, configContents["server"]) - assert.Equal(t, []any{"192.168.122.100", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + assert.Equal(t, []any{"k8s-host.com", "fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + + // Agent config file assertions + configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "agent.yaml") + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + configContents = map[string]any{} // clear the map + require.NoError(t, yaml.Unmarshal(b, configContents)) + + assert.Equal(t, "canal", configContents["cni"]) + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, "https://[fd12:3456:789a::21]:9345", configContents["server"]) + assert.Nil(t, configContents["tls-san"]) } -func TestConfigureKubernetes_SuccessfulMultiNodeRKE2Cluster(t *testing.T) { +func TestConfigureKubernetes_SuccessfulMultiNodeRKE2ClusterDualstackPrioIPv4(t *testing.T) { ctx, teardown := setupContext(t) defer teardown() @@ -493,6 +1437,7 @@ func TestConfigureKubernetes_SuccessfulMultiNodeRKE2Cluster(t *testing.T) { Network: image.Network{ APIHost: "api.cluster01.hosted.on.edge.suse.com", APIVIP4: "192.168.122.100", + APIVIP6: "fd12:3456:789a::21", }, Nodes: []image.Node{ { @@ -523,8 +1468,10 @@ func TestConfigureKubernetes_SuccessfulMultiNodeRKE2Cluster(t *testing.T) { "token": "123", "cni": "canal", "tls-san": []string{ - "192-168-122-100.sslip.io", + "k8s-host.com", }, + "cluster-cidr": "10.42.0.0/16,fd12:3456:789b::/48", + "service-cidr": "10.43.0.0/16,fd12:3456:789c::/112", } b, err := yaml.Marshal(serverConfig) @@ -556,9 +1503,11 @@ func TestConfigureKubernetes_SuccessfulMultiNodeRKE2Cluster(t *testing.T) { assert.Contains(t, contents, "cp $CONFIGFILE /etc/rancher/rke2/config.yaml") assert.Contains(t, contents, "if [ \"$HOSTNAME\" = node1.suse.com ]; then") assert.Contains(t, contents, "echo \"192.168.122.100 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "echo \"fd12:3456:789a::21 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") assert.Contains(t, contents, "export INSTALL_RKE2_ARTIFACT_PATH=$ARTEFACTS_DIR/kubernetes/install") assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-kubernetes.sh") assert.Contains(t, contents, "systemctl enable rke2-$NODETYPE.service") + assert.Contains(t, contents, "sh set-node-ip.sh") // Server config file assertions configPath := filepath.Join(ctx.ArtefactsDir, "kubernetes", "server.yaml") @@ -577,7 +1526,7 @@ func TestConfigureKubernetes_SuccessfulMultiNodeRKE2Cluster(t *testing.T) { assert.Equal(t, "canal", configContents["cni"]) assert.Equal(t, "123", configContents["token"]) assert.Equal(t, "https://192.168.122.100:9345", configContents["server"]) - assert.Equal(t, []any{"192-168-122-100.sslip.io", "192.168.122.100", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + assert.ElementsMatch(t, []any{"k8s-host.com", "192.168.122.100", "fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) // Initialising server config file assertions configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "init_server.yaml") @@ -591,7 +1540,7 @@ func TestConfigureKubernetes_SuccessfulMultiNodeRKE2Cluster(t *testing.T) { assert.Equal(t, "canal", configContents["cni"]) assert.Equal(t, "123", configContents["token"]) assert.Equal(t, nil, configContents["server"]) - assert.Equal(t, []any{"192-168-122-100.sslip.io", "192.168.122.100", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + assert.ElementsMatch(t, []any{"k8s-host.com", "192.168.122.100", "fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) // Agent config file assertions configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "agent.yaml") @@ -608,6 +1557,135 @@ func TestConfigureKubernetes_SuccessfulMultiNodeRKE2Cluster(t *testing.T) { assert.Nil(t, configContents["tls-san"]) } +func TestConfigureKubernetes_SuccessfulMultiNodeRKE2ClusterDualstackPrioIPv6(t *testing.T) { + ctx, teardown := setupContext(t) + defer teardown() + + ctx.ImageDefinition.Kubernetes = image.Kubernetes{ + Version: "v1.30.3+rke2r1", + Network: image.Network{ + APIHost: "api.cluster01.hosted.on.edge.suse.com", + APIVIP4: "192.168.122.100", + APIVIP6: "fd12:3456:789a::21", + }, + Nodes: []image.Node{ + { + Hostname: "node1.suse.com", + Type: "server", + }, + { + Hostname: "node2.suse.com", + Type: "agent", + }, + }, + } + + c := Combustion{ + KubernetesScriptDownloader: mockKubernetesScriptDownloader{ + downloadScript: func(distribution, destPath string) (string, error) { + return kubernetesScriptInstaller, nil + }, + }, + KubernetesArtefactDownloader: mockKubernetesArtefactDownloader{ + downloadRKE2Artefacts: func(arch image.Arch, version, cni string, multusEnabled bool, installPath, imagesPath string) error { + return nil + }, + }, + } + + serverConfig := map[string]any{ + "token": "123", + "cni": "canal", + "tls-san": []string{ + "k8s-host.com", + }, + "cluster-cidr": "fd12:3456:789b::/48,10.42.0.0/16", + "service-cidr": "fd12:3456:789c::/112,10.43.0.0/16", + } + + b, err := yaml.Marshal(serverConfig) + require.NoError(t, err) + + configDir := filepath.Join(ctx.ImageConfigDir, k8sDir, k8sConfigDir) + require.NoError(t, os.MkdirAll(configDir, os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(configDir, "server.yaml"), b, os.ModePerm)) + + scripts, err := c.configureKubernetes(ctx) + require.NoError(t, err) + require.Len(t, scripts, 1) + + // Script file assertions + scriptPath := filepath.Join(ctx.CombustionDir, scripts[0]) + + info, err := os.Stat(scriptPath) + require.NoError(t, err) + + assert.Equal(t, fileio.ExecutablePerms, info.Mode()) + + b, err = os.ReadFile(scriptPath) + require.NoError(t, err) + + contents := string(b) + assert.Contains(t, contents, "hosts[node1.suse.com]=server") + assert.Contains(t, contents, "hosts[node2.suse.com]=agent") + assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/images/* /var/lib/rancher/rke2/agent/images/") + assert.Contains(t, contents, "cp $CONFIGFILE /etc/rancher/rke2/config.yaml") + assert.Contains(t, contents, "if [ \"$HOSTNAME\" = node1.suse.com ]; then") + assert.Contains(t, contents, "echo \"192.168.122.100 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "echo \"fd12:3456:789a::21 api.cluster01.hosted.on.edge.suse.com\" >> /etc/hosts") + assert.Contains(t, contents, "export INSTALL_RKE2_ARTIFACT_PATH=$ARTEFACTS_DIR/kubernetes/install") + assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-kubernetes.sh") + assert.Contains(t, contents, "systemctl enable rke2-$NODETYPE.service") + assert.Contains(t, contents, "sh set-node-ip.sh") + + // Server config file assertions + configPath := filepath.Join(ctx.ArtefactsDir, "kubernetes", "server.yaml") + + info, err = os.Stat(configPath) + require.NoError(t, err) + + assert.Equal(t, fileio.NonExecutablePerms, info.Mode()) + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + var configContents map[string]any + require.NoError(t, yaml.Unmarshal(b, &configContents)) + + assert.Equal(t, "canal", configContents["cni"]) + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, "https://[fd12:3456:789a::21]:9345", configContents["server"]) + assert.ElementsMatch(t, []any{"k8s-host.com", "192.168.122.100", "fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + + // Initialising server config file assertions + configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "init_server.yaml") + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + configContents = map[string]any{} // clear the map + require.NoError(t, yaml.Unmarshal(b, configContents)) + + assert.Equal(t, "canal", configContents["cni"]) + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, nil, configContents["server"]) + assert.ElementsMatch(t, []any{"k8s-host.com", "192.168.122.100", "fd12:3456:789a::21", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"]) + + // Agent config file assertions + configPath = filepath.Join(ctx.ArtefactsDir, "kubernetes", "agent.yaml") + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + configContents = map[string]any{} // clear the map + require.NoError(t, yaml.Unmarshal(b, configContents)) + + assert.Equal(t, "canal", configContents["cni"]) + assert.Equal(t, "123", configContents["token"]) + assert.Equal(t, "https://[fd12:3456:789a::21]:9345", configContents["server"]) + assert.Nil(t, configContents["tls-san"]) +} + func TestConfigureManifests_NoSetup(t *testing.T) { ctx, teardown := setupContext(t) defer teardown() @@ -835,13 +1913,15 @@ func TestKubernetesVIPManifestValidIPV4(t *testing.T) { assert.Contains(t, manifest, "- 192.168.1.1/32") assert.Contains(t, manifest, "- name: rke2-api") + assert.NotContains(t, manifest, "ipFamilies:\n - IPv6") + assert.NotContains(t, manifest, "ipFamilyPolicy: SingleStack") } func TestKubernetesVIPManifestValidIPV6(t *testing.T) { k8s := &image.Kubernetes{ Version: "v1.30.3+k3s1", Network: image.Network{ - APIVIP4: "fd12:3456:789a::21", + APIVIP6: "fd12:3456:789a::21", }, } @@ -850,17 +1930,26 @@ func TestKubernetesVIPManifestValidIPV6(t *testing.T) { assert.Contains(t, manifest, "- fd12:3456:789a::21/128") assert.Contains(t, manifest, "- name: k8s-api") + assert.Contains(t, manifest, "ipFamilies:\n - IPv6") + assert.Contains(t, manifest, "ipFamilyPolicy: SingleStack") assert.NotContains(t, manifest, "rke2") } -func TestKubernetesVIPManifestInvalidIP(t *testing.T) { +func TestKubernetesVIPManifestDualstack(t *testing.T) { k8s := &image.Kubernetes{ Version: "v1.30.3+k3s1", Network: image.Network{ - APIVIP4: "1111", + APIVIP4: "192.168.1.1", + APIVIP6: "fd12:3456:789a::21", }, } - _, err := kubernetesVIPManifest(k8s) - require.ErrorContains(t, err, "parsing kubernetes apiVIP address: ParseAddr(\"1111\"): unable to parse IP") + manifest, err := kubernetesVIPManifest(k8s) + require.NoError(t, err) + + assert.Contains(t, manifest, "- 192.168.1.1/32") + assert.Contains(t, manifest, "- fd12:3456:789a::21/128") + assert.Contains(t, manifest, "- name: k8s-api") + assert.NotContains(t, manifest, "ipFamilies:\n - IPv6") + assert.NotContains(t, manifest, "ipFamilyPolicy: SingleStack") } diff --git a/pkg/combustion/templates/k3s-multi-node-installer.sh.tpl b/pkg/combustion/templates/k3s-multi-node-installer.sh.tpl index 7ce34974..0559c732 100644 --- a/pkg/combustion/templates/k3s-multi-node-installer.sh.tpl +++ b/pkg/combustion/templates/k3s-multi-node-installer.sh.tpl @@ -102,7 +102,6 @@ cp $CONFIGFILE /etc/rancher/k3s/config.yaml if [ "$NODETYPE" = "server" ]; then {{- if .apiVIP6 }} -chmod +x {{ .setNodeIPScript }} sh {{ .setNodeIPScript }} {{- end }} fi diff --git a/pkg/combustion/templates/k3s-single-node-installer.sh.tpl b/pkg/combustion/templates/k3s-single-node-installer.sh.tpl index 6c2d9650..d270e790 100644 --- a/pkg/combustion/templates/k3s-single-node-installer.sh.tpl +++ b/pkg/combustion/templates/k3s-single-node-installer.sh.tpl @@ -74,7 +74,6 @@ mkdir -p /etc/rancher/k3s/ cp {{ .configFilePath }}/{{ .configFile }} /etc/rancher/k3s/config.yaml {{- if .apiVIP6 }} -chmod +x {{ .setNodeIPScript }} sh {{ .setNodeIPScript }} {{- end }} diff --git a/pkg/combustion/templates/k8s-vip.yaml.tpl b/pkg/combustion/templates/k8s-vip.yaml.tpl index de650e5a..7ec5207a 100644 --- a/pkg/combustion/templates/k8s-vip.yaml.tpl +++ b/pkg/combustion/templates/k8s-vip.yaml.tpl @@ -43,10 +43,10 @@ spec: - IPv4 - IPv6 {{- end }} - {{- if and (eq .APIAddress4 "false") (eq .APIAddress6 "true") }} - ipFamilyPolicy: SingleStack - ipFamilies: - - IPv6 + {{- if .OnlyIPv6 }} + ipFamilyPolicy: SingleStack + ipFamilies: + - IPv6 {{- end }} ports: {{- if .RKE2 }} diff --git a/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl b/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl index 80814246..46f6e8ed 100644 --- a/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl +++ b/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl @@ -104,7 +104,6 @@ cp $CONFIGFILE /etc/rancher/rke2/config.yaml if [ "$NODETYPE" = "server" ]; then {{- if .apiVIP6 }} -chmod +x {{ .setNodeIPScript }} sh {{ .setNodeIPScript }} {{- end }} fi diff --git a/pkg/combustion/templates/rke2-single-node-installer.sh.tpl b/pkg/combustion/templates/rke2-single-node-installer.sh.tpl index 3f63a181..b3cca90c 100644 --- a/pkg/combustion/templates/rke2-single-node-installer.sh.tpl +++ b/pkg/combustion/templates/rke2-single-node-installer.sh.tpl @@ -76,7 +76,6 @@ mkdir -p /etc/rancher/rke2/ cp {{ .configFilePath }}/{{ .configFile }} /etc/rancher/rke2/config.yaml {{- if .apiVIP6 }} -chmod +x {{ .setNodeIPScript }} sh {{ .setNodeIPScript }} {{- end }} diff --git a/pkg/combustion/templates/set-node-ip.sh.tpl b/pkg/combustion/templates/set-node-ip.sh.tpl index 1ee96897..12f4d25c 100644 --- a/pkg/combustion/templates/set-node-ip.sh.tpl +++ b/pkg/combustion/templates/set-node-ip.sh.tpl @@ -63,11 +63,11 @@ try_get_ipv4() { return 0 fi + ((attempt++)) if [ $attempt -lt $MAX_ATTEMPTS ]; then echo "Waiting $DELAY seconds before next attempt..." sleep $DELAY fi - ((attempt++)) done echo "Failed to get IPv4 address after $MAX_ATTEMPTS attempts" return 1 @@ -83,11 +83,11 @@ try_get_ipv6() { return 0 fi + ((attempt++)) if [ $attempt -lt $MAX_ATTEMPTS ]; then echo "Waiting $DELAY seconds before next attempt..." sleep $DELAY fi - ((attempt++)) done echo "Failed to get IPv6 address after $MAX_ATTEMPTS attempts" return 1 @@ -106,14 +106,10 @@ update_config() { if [ "${IPv4}" = "true" ] && [ "${IPv6}" = "true" ]; then if [ "$prioritizeIPv6" = "false" ]; then echo "node-ip: ${IPv4_ADDRESS},${IPv6_ADDRESS}" >> "$CONFIG_FILE" - echo "Added IPv4,IPv6 addresses to config (IPv4 prioritized)" + echo "Added IPv4 and IPv6 addresses to config (IPv4 prioritized)" else echo "node-ip: ${IPv6_ADDRESS},${IPv4_ADDRESS}" >> "$CONFIG_FILE" - echo "Added IPv6,IPv4 addresses to config (IPv6 prioritized)" -# if [ "$IS_SERVER" = "true" ]; then -# echo "kube-apiserver-arg:" >> "$CONFIG_FILE" -# echo " - \"advertise-address=${IPv6_ADDRESS}\"" >> "$CONFIG_FILE" -# fi + echo "Added IPv6 and IPv4 addresses to config (IPv6 prioritized)" fi elif [ "${IPv4}" = "true" ]; then echo "node-ip: ${IPv4_ADDRESS}" >> "$CONFIG_FILE" @@ -121,10 +117,6 @@ update_config() { elif [ "${IPv6}" = "true" ]; then echo "node-ip: ${IPv6_ADDRESS}" >> "$CONFIG_FILE" echo "Added IPv6 address to config" -# if [ "$IS_SERVER" = "true" ]; then -# echo "kube-apiserver-arg:" >> "$CONFIG_FILE" -# echo " - \"advertise-address=${IPv6_ADDRESS}\"" >> "$CONFIG_FILE" -# fi fi } diff --git a/pkg/image/validation/kubernetes.go b/pkg/image/validation/kubernetes.go index 89c54a0d..2a0a2494 100644 --- a/pkg/image/validation/kubernetes.go +++ b/pkg/image/validation/kubernetes.go @@ -3,6 +3,8 @@ package validation import ( "errors" "fmt" + "gopkg.in/yaml.v3" + "net/netip" "net/url" "os" "path/filepath" @@ -14,10 +16,13 @@ import ( ) const ( - k8sComponent = "Kubernetes" - httpScheme = "http" - httpsScheme = "https" - ociScheme = "oci" + k8sComponent = "Kubernetes" + httpScheme = "http" + httpsScheme = "https" + ociScheme = "oci" + k8sDir = "kubernetes" + k8sConfigDir = "config" + serverConfigFile = "server.yaml" ) var validNodeTypes = []string{image.KubernetesNodeTypeServer, image.KubernetesNodeTypeAgent} @@ -114,6 +119,7 @@ func validateNodes(k8s *image.Kubernetes) []FailedValidation { func validateNetwork(k8s *image.Kubernetes) []FailedValidation { var failures []FailedValidation + var err error if k8s.Network.APIVIP4 == "" && k8s.Network.APIVIP6 == "" { if len(k8s.Nodes) > 1 { @@ -125,34 +131,133 @@ func validateNetwork(k8s *image.Kubernetes) []FailedValidation { return failures } - //parsedIP, err := netip.ParseAddr(k8s.Network.APIVIP4) - //if err != nil { - // failures = append(failures, FailedValidation{ - // UserMessage: fmt.Sprintf("Invalid address value %q for field 'apiVIP'.", k8s.Network.APIVIP4), - // Error: err, - // }) - // - // return failures - //} - - //if !parsedIP.Is4() && !parsedIP.Is6() { - // failures = append(failures, FailedValidation{ - // UserMessage: "Only IPv4 and IPv6 addresses are valid values for field 'apiVIP'.", - // }) - // - // return failures - //} - // - //if !parsedIP.IsGlobalUnicast() { - // msg := fmt.Sprintf("Invalid non-unicast cluster API address (%s) for field 'apiVIP'.", k8s.Network.APIVIP4) - // failures = append(failures, FailedValidation{ - // UserMessage: msg, - // }) - //} + var ip4 netip.Addr + if k8s.Network.APIVIP4 != "" { + ip4, err = netip.ParseAddr(k8s.Network.APIVIP4) + if err != nil { + failures = append(failures, FailedValidation{ + UserMessage: fmt.Sprintf("Invalid address value %q for field 'apiVIP'.", k8s.Network.APIVIP4), + Error: err, + }) + + return failures + } + } + + var ip6 netip.Addr + if k8s.Network.APIVIP6 != "" { + ip6, err = netip.ParseAddr(k8s.Network.APIVIP6) + if err != nil { + failures = append(failures, FailedValidation{ + UserMessage: fmt.Sprintf("Invalid address value %q for field 'apiVIP6'.", k8s.Network.APIVIP6), + Error: err, + }) + + return failures + } + } + + if k8s.Network.APIVIP4 != "" && !ip4.Is4() { + failures = append(failures, FailedValidation{ + UserMessage: "Only IPv4 addresses are valid for field 'apiVIP'.", + }) + } + + if k8s.Network.APIVIP6 != "" && !ip6.Is6() { + failures = append(failures, FailedValidation{ + UserMessage: "Only a IPv6 addresses are valid for field 'apiVIP6'.", + }) + } + + if k8s.Network.APIVIP4 != "" && !ip4.IsGlobalUnicast() { + msg := fmt.Sprintf("Invalid non-unicast cluster API address (%s) for field 'apiVIP'.", k8s.Network.APIVIP4) + failures = append(failures, FailedValidation{ + UserMessage: msg, + }) + } + + if k8s.Network.APIVIP6 != "" && !ip6.IsGlobalUnicast() { + msg := fmt.Sprintf("Invalid non-unicast cluster API address (%s) for field 'apiVIP6'.", k8s.Network.APIVIP6) + failures = append(failures, FailedValidation{ + UserMessage: msg, + }) + } + + return failures +} + +func validateConfig(k8s *image.Kubernetes, ctx *image.Context) []FailedValidation { + var failures []FailedValidation + + if k8s.Network.APIVIP4 == "" && k8s.Network.APIVIP6 == "" { + return failures + } + + serverConfigPath := filepath.Join(ctx.ImageConfigDir, k8sDir, k8sConfigDir, serverConfigFile) + configFile, err := os.ReadFile(serverConfigPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + failures = append(failures, FailedValidation{ + UserMessage: fmt.Sprintf("Kubernetes server config could not be found at '%s,' dualstack configuration requires a defined cluster-cidr and service-cidr.", serverConfigPath), + }) + } else { + failures = append(failures, FailedValidation{ + UserMessage: "Kubernetes server config could not be read", + Error: err, + }) + } + } + + serverConfig := map[string]any{} + if err = yaml.Unmarshal(configFile, &serverConfig); err != nil { + failures = append(failures, FailedValidation{ + UserMessage: "Parsing kubernetes server config file", + Error: err, + }) + + return failures + } + + if clusterCIDR, ok := serverConfig["cluster-cidr"].(string); ok { + if len(cidrs) == 2 { + err = checkCIDR(clusterCIDR) + } else { + failures = append(failures, FailedValidation{ + UserMessage: "Kubernetes server config cluster-cidr should have both IPv4 and IPv6 configured", + }) + } + } else { + failures = append(failures, FailedValidation{ + UserMessage: "Kubernetes server config must contain cluster-cidr when configuring dualstack", + }) + } return failures } +func checkCIDR(cidr string) error { + cidrs := strings.Split(cidr, ",") + firstCIDR, err := netip.ParseAddr(cidrs[0]) + if err != nil { + return fmt.Errorf("parsing first CIDR %w", err) + } + + secondCIDR, err := netip.ParseAddr(cidrs[1]) + if err != nil { + return fmt.Errorf("parsing second CIDR %w", err) + } + + if firstCIDR.Is4() && secondCIDR.Is4() { + return fmt.Errorf("both CIDRs cannot be IPv4, one must be IPv6") + } + + if firstCIDR.Is6() && secondCIDR.Is6() { + return fmt.Errorf("both CIDRs cannot be IPv6, one must be IPv6") + } + + return nil +} + func validateManifestURLs(k8s *image.Kubernetes) []FailedValidation { var failures []FailedValidation diff --git a/pkg/image/validation/kubernetes_test.go b/pkg/image/validation/kubernetes_test.go index b2e0a691..4090dd1a 100644 --- a/pkg/image/validation/kubernetes_test.go +++ b/pkg/image/validation/kubernetes_test.go @@ -15,7 +15,7 @@ import ( var validNetwork = image.Network{ APIHost: "host.com", - APIVIP: "192.168.1.1", + APIVIP4: "192.168.100.1", } func TestValidateKubernetes(t *testing.T) { @@ -80,7 +80,8 @@ func TestValidateKubernetes(t *testing.T) { Version: "v1.30.3", Network: image.Network{ APIHost: "host.com", - APIVIP: "127.0.0.1", + APIVIP4: "127.0.0.1", + APIVIP6: "ff02::1", }, Nodes: []image.Node{ { @@ -120,6 +121,7 @@ func TestValidateKubernetes(t *testing.T) { "Helm repository 'name' field for \"apache-repo\" must match the 'repositoryName' field in at least one defined Helm chart.", "Helm chart 'repositoryName' \"another-apache-repo\" for Helm chart \"\" does not match the name of any defined repository.", "Invalid non-unicast cluster API address (127.0.0.1) for field 'apiVIP'.", + "Invalid non-unicast cluster API address (ff02::1) for field 'apiVIP6'.", }, }, } @@ -1096,58 +1098,190 @@ func TestValidateNetwork(t *testing.T) { "The 'apiVIP' field is required in the 'network' section for multi node clusters.", }, }, - `valid ipv4`: { + `valid IPv4`: { K8s: image.Kubernetes{ Network: image.Network{ - APIVIP: "192.168.1.1", + APIVIP4: "192.168.1.1", }, }, }, - `valid ipv6`: { + `invalid IPv4`: { K8s: image.Kubernetes{ Network: image.Network{ - APIVIP: "fd12:3456:789a::21", + APIVIP4: "500.168.1.1", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid address value \"500.168.1.1\" for field 'apiVIP'.", + }, + }, + `valid IPv6`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIVIP6: "fd12:3456:789a::21", + }, + }, + }, + `invalid IPv6`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIVIP6: "xxxx:3456:789a::21", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid address value \"xxxx:3456:789a::21\" for field 'apiVIP6'.", + }, + }, + `valid dualstack`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIVIP4: "192.168.1.1", + APIVIP6: "fd12:3456:789a::21", }, }, }, - `invalid ipv4`: { + `invalid dualstack IPv4 non unicast`: { K8s: image.Kubernetes{ Network: image.Network{ - APIVIP: "500.168.1.1", + APIVIP4: "127.0.0.1", + APIVIP6: "fd12:3456:789a::21", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid non-unicast cluster API address (127.0.0.1) for field 'apiVIP'.", + }, + }, + `invalid dualstack IPv6 non unicast`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIVIP4: "192.168.1.1", + APIVIP6: "ff02::1", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid non-unicast cluster API address (ff02::1) for field 'apiVIP6'.", + }, + }, + `invalid dualstack both non unicast`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIVIP4: "127.0.0.1", + APIVIP6: "ff02::1", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid non-unicast cluster API address (127.0.0.1) for field 'apiVIP'.", + "Invalid non-unicast cluster API address (ff02::1) for field 'apiVIP6'.", + }, + }, + `invalid dualstack IPv4 not valid`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIVIP4: "500.168.1.1", + APIVIP6: "fd12:3456:789a::21", }, }, ExpectedFailedMessages: []string{ "Invalid address value \"500.168.1.1\" for field 'apiVIP'.", }, }, - `non-unicast ipv4`: { + `invalid dualstack IPv6 not valid`: { K8s: image.Kubernetes{ Network: image.Network{ - APIVIP: "127.0.0.1", + APIVIP4: "192.168.1.1", + APIVIP6: "xxxx:3456:789a::21", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid address value \"xxxx:3456:789a::21\" for field 'apiVIP6'.", + }, + }, + `undefined v4 VIP`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIHost: "host.com", + APIVIP4: "0.0.0.0", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid non-unicast cluster API address (0.0.0.0) for field 'apiVIP'.", + }, + }, + `undefined v6 VIP`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIHost: "host.com", + APIVIP6: "::", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid non-unicast cluster API address (::) for field 'apiVIP6'.", + }, + }, + `loopback v4 VIP`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIHost: "host.com", + APIVIP4: "127.0.0.1", }, }, ExpectedFailedMessages: []string{ "Invalid non-unicast cluster API address (127.0.0.1) for field 'apiVIP'.", }, }, - `invalid ipv6`: { + `loopback v6 VIP`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIHost: "host.com", + APIVIP6: "::1", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid non-unicast cluster API address (::1) for field 'apiVIP6'.", + }, + }, + `multicast v4 VIP`: { K8s: image.Kubernetes{ Network: image.Network{ - APIVIP: "xxxx:3456:789a::21", + APIHost: "host.com", + APIVIP4: "224.224.224.224", }, }, ExpectedFailedMessages: []string{ - "Invalid address value \"xxxx:3456:789a::21\" for field 'apiVIP'.", + "Invalid non-unicast cluster API address (224.224.224.224) for field 'apiVIP'.", }, }, - `non-unicast ipv6`: { + `multicast v6 VIP`: { K8s: image.Kubernetes{ Network: image.Network{ - APIVIP: "ff02::1", + APIHost: "host.com", + APIVIP6: "FF01::1", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid non-unicast cluster API address (FF01::1) for field 'apiVIP6'.", + }, + }, + `link-local v4 VIP`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIHost: "host.com", + APIVIP4: "169.254.1.1", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid non-unicast cluster API address (169.254.1.1) for field 'apiVIP'.", + }, + }, + `link-local v6 VIP`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIHost: "host.com", + APIVIP6: "FE80::1", }, }, ExpectedFailedMessages: []string{ - "Invalid non-unicast cluster API address (ff02::1) for field 'apiVIP'.", + "Invalid non-unicast cluster API address (FE80::1) for field 'apiVIP6'.", }, }, } diff --git a/pkg/kubernetes/cluster.go b/pkg/kubernetes/cluster.go index d75d7bcb..57e586e9 100644 --- a/pkg/kubernetes/cluster.go +++ b/pkg/kubernetes/cluster.go @@ -73,7 +73,7 @@ func NewCluster(kubernetes *image.Kubernetes, configPath string) (*Cluster, erro ip6 = &ip6Holder } - prioritizeIPv6 := isIPv6Priority(serverConfig) + prioritizeIPv6 := IsIPv6Priority(serverConfig) setMultiNodeConfigDefaults(kubernetes, serverConfig, ip4, ip6, prioritizeIPv6) agentConfigPath := filepath.Join(configPath, agentConfigFile) @@ -167,6 +167,10 @@ func setSingleNodeConfigDefaults(kubernetes *image.Kubernetes, config map[string if kubernetes.Network.APIVIP6 != "" { appendClusterTLSSAN(config, kubernetes.Network.APIVIP6) + + if strings.Contains(kubernetes.Version, image.KubernetesDistroK3S) && kubernetes.Network.APIVIP4 == "" { + appendDisabledServices(config, "servicelb") + } } if kubernetes.Network.APIHost != "" { @@ -232,9 +236,12 @@ func setClusterAPIAddress(config map[string]any, ip4 *netip.Addr, ip6 *netip.Add return } - if ip6 != nil && prioritizeIPv6 { + switch { + case ip6 != nil && prioritizeIPv6: config[serverKey] = fmt.Sprintf("https://%s", netip.AddrPortFrom(*ip6, port).String()) - } else if ip4 != nil { + case ip6 != nil && ip4 == nil: + config[serverKey] = fmt.Sprintf("https://%s", netip.AddrPortFrom(*ip6, port).String()) + default: config[serverKey] = fmt.Sprintf("https://%s", netip.AddrPortFrom(*ip4, port).String()) } } @@ -323,11 +330,12 @@ func ServersCount(nodes []image.Node) int { return servers } -func isIPv6Priority(serverConfig map[string]any) bool { +func IsIPv6Priority(serverConfig map[string]any) bool { if clusterCIDR, ok := serverConfig["cluster-cidr"].(string); ok { cidrs := strings.Split(clusterCIDR, ",") if len(cidrs) > 0 { - return strings.Contains(cidrs[0], ":") + fmt.Println(strings.Contains(cidrs[0], ":")) + return strings.Contains(cidrs[0], "::") } } diff --git a/pkg/kubernetes/cluster_test.go b/pkg/kubernetes/cluster_test.go index debb2ef7..140b4c1a 100644 --- a/pkg/kubernetes/cluster_test.go +++ b/pkg/kubernetes/cluster_test.go @@ -1,7 +1,9 @@ package kubernetes import ( + "fmt" "net/netip" + "path/filepath" "testing" "github.com/google/uuid" @@ -15,7 +17,7 @@ func TestNewCluster_SingleNodeRKE2_MissingConfig(t *testing.T) { Version: "v1.30.3+rke2r1", Network: image.Network{ APIHost: "api.suse.edge.com", - APIVIP: "192.168.122.50", + APIVIP4: "192.168.122.50", }, } @@ -40,7 +42,7 @@ func TestNewCluster_SingleNodeK3s_MissingConfig(t *testing.T) { Version: "v1.30.3+k3s1", Network: image.Network{ APIHost: "api.suse.edge.com", - APIVIP: "192.168.122.50", + APIVIP4: "192.168.122.50", }, } @@ -64,17 +66,64 @@ func TestNewCluster_SingleNode_ExistingConfig(t *testing.T) { kubernetes := &image.Kubernetes{ Network: image.Network{ APIHost: "api.suse.edge.com", - APIVIP: "192.168.122.50", + APIVIP4: "192.168.122.50", }, } - cluster, err := NewCluster(kubernetes, "testdata") + cluster, err := NewCluster(kubernetes, filepath.Join("testdata", "default")) require.NoError(t, err) require.NotNil(t, cluster.ServerConfig) assert.Equal(t, "calico", cluster.ServerConfig["cni"]) assert.Equal(t, "totally-not-generated-one", cluster.ServerConfig["token"]) - assert.Equal(t, []string{"192.168.122.50", "api.suse.edge.com"}, cluster.ServerConfig["tls-san"]) + assert.ElementsMatch(t, []string{"192.168.122.50", "api.suse.edge.com"}, cluster.ServerConfig["tls-san"]) + assert.Equal(t, true, cluster.ServerConfig["selinux"]) + assert.Nil(t, cluster.ServerConfig["server"]) + + assert.Empty(t, cluster.InitialiserName) + assert.Nil(t, cluster.InitialiserConfig) + assert.Nil(t, cluster.AgentConfig) +} + +func TestNewCluster_SingleNode_ExistingConfigIPv6(t *testing.T) { + kubernetes := &image.Kubernetes{ + Network: image.Network{ + APIHost: "api.suse.edge.com", + APIVIP6: "fd12:3456:789a::21", + }, + } + + cluster, err := NewCluster(kubernetes, filepath.Join("testdata", "default")) + require.NoError(t, err) + + require.NotNil(t, cluster.ServerConfig) + assert.Equal(t, "calico", cluster.ServerConfig["cni"]) + assert.Equal(t, "totally-not-generated-one", cluster.ServerConfig["token"]) + assert.ElementsMatch(t, []string{"fd12:3456:789a::21", "api.suse.edge.com"}, cluster.ServerConfig["tls-san"]) + assert.Equal(t, true, cluster.ServerConfig["selinux"]) + assert.Nil(t, cluster.ServerConfig["server"]) + + assert.Empty(t, cluster.InitialiserName) + assert.Nil(t, cluster.InitialiserConfig) + assert.Nil(t, cluster.AgentConfig) +} + +func TestNewCluster_SingleNode_ExistingConfigDualstack(t *testing.T) { + kubernetes := &image.Kubernetes{ + Network: image.Network{ + APIHost: "api.suse.edge.com", + APIVIP4: "192.168.122.50", + APIVIP6: "fd12:3456:789a::21", + }, + } + + cluster, err := NewCluster(kubernetes, filepath.Join("testdata", "dualstack-prio-ipv4")) + require.NoError(t, err) + + require.NotNil(t, cluster.ServerConfig) + assert.Equal(t, "calico", cluster.ServerConfig["cni"]) + assert.Equal(t, "totally-not-generated-one", cluster.ServerConfig["token"]) + assert.ElementsMatch(t, []string{"fd12:3456:789a::21", "api.suse.edge.com", "192.168.122.50"}, cluster.ServerConfig["tls-san"]) assert.Equal(t, true, cluster.ServerConfig["selinux"]) assert.Nil(t, cluster.ServerConfig["server"]) @@ -88,7 +137,7 @@ func TestNewCluster_MultiNodeRKE2_MissingConfig(t *testing.T) { Version: "v1.30.3+rke2r1", Network: image.Network{ APIHost: "api.suse.edge.com", - APIVIP: "192.168.122.50", + APIVIP4: "192.168.122.50", }, Nodes: []image.Node{ { @@ -138,7 +187,7 @@ func TestNewCluster_MultiNodeRKE2_ExistingConfig(t *testing.T) { Version: "v1.30.3+rke2r1", Network: image.Network{ APIHost: "api.suse.edge.com", - APIVIP: "192.168.122.50", + APIVIP4: "192.168.122.50", }, Nodes: []image.Node{ { @@ -152,7 +201,7 @@ func TestNewCluster_MultiNodeRKE2_ExistingConfig(t *testing.T) { }, } - cluster, err := NewCluster(kubernetes, "testdata") + cluster, err := NewCluster(kubernetes, filepath.Join("testdata", "default")) require.NoError(t, err) assert.Equal(t, "node1.suse.com", cluster.InitialiserName) @@ -182,6 +231,156 @@ func TestNewCluster_MultiNodeRKE2_ExistingConfig(t *testing.T) { assert.Nil(t, cluster.AgentConfig["tls-san"]) } +func TestNewCluster_MultiNodeRKE2_ExistingConfigIPv6(t *testing.T) { + kubernetes := &image.Kubernetes{ + Version: "v1.30.3+rke2r1", + Network: image.Network{ + APIHost: "api.suse.edge.com", + APIVIP6: "fd12:3456:789a::21", + }, + Nodes: []image.Node{ + { + Hostname: "node1.suse.com", + Type: "server", + }, + { + Hostname: "node2.suse.com", + Type: "agent", + }, + }, + } + + cluster, err := NewCluster(kubernetes, filepath.Join("testdata", "default")) + require.NoError(t, err) + + assert.Equal(t, "node1.suse.com", cluster.InitialiserName) + + require.NotNil(t, cluster.InitialiserConfig) + assert.Equal(t, "calico", cluster.InitialiserConfig["cni"]) + assert.Equal(t, []string{"fd12:3456:789a::21", "api.suse.edge.com"}, cluster.InitialiserConfig["tls-san"]) + assert.Equal(t, "totally-not-generated-one", cluster.InitialiserConfig["token"]) + assert.Nil(t, cluster.InitialiserConfig["server"]) + assert.Equal(t, true, cluster.InitialiserConfig["selinux"]) + assert.Nil(t, cluster.InitialiserConfig["debug"]) + + require.NotNil(t, cluster.ServerConfig) + assert.Equal(t, "calico", cluster.ServerConfig["cni"]) + assert.Equal(t, []string{"fd12:3456:789a::21", "api.suse.edge.com"}, cluster.ServerConfig["tls-san"]) + assert.Equal(t, "totally-not-generated-one", cluster.ServerConfig["token"]) + assert.Equal(t, "https://[fd12:3456:789a::21]:9345", cluster.ServerConfig["server"]) + assert.Equal(t, true, cluster.ServerConfig["selinux"]) + assert.Nil(t, cluster.ServerConfig["debug"]) + + require.NotNil(t, cluster.AgentConfig) + assert.Equal(t, "calico", cluster.AgentConfig["cni"]) + assert.Equal(t, "totally-not-generated-one", cluster.AgentConfig["token"]) + assert.Equal(t, "https://[fd12:3456:789a::21]:9345", cluster.AgentConfig["server"]) + assert.Equal(t, true, cluster.AgentConfig["debug"]) + assert.Equal(t, true, cluster.AgentConfig["selinux"]) + assert.Nil(t, cluster.AgentConfig["tls-san"]) +} + +func TestNewCluster_MultiNodeRKE2_ExistingConfigDualstackPrioritizeIPv6(t *testing.T) { + kubernetes := &image.Kubernetes{ + Version: "v1.30.3+rke2r1", + Network: image.Network{ + APIHost: "api.suse.edge.com", + APIVIP4: "192.168.122.50", + APIVIP6: "fd12:3456:789a::21", + }, + Nodes: []image.Node{ + { + Hostname: "node1.suse.com", + Type: "server", + }, + { + Hostname: "node2.suse.com", + Type: "agent", + }, + }, + } + + cluster, err := NewCluster(kubernetes, filepath.Join("testdata", "dualstack-prio-ipv6")) + require.NoError(t, err) + + assert.Equal(t, "node1.suse.com", cluster.InitialiserName) + + require.NotNil(t, cluster.InitialiserConfig) + assert.Equal(t, "calico", cluster.InitialiserConfig["cni"]) + assert.ElementsMatch(t, []string{"192.168.122.50", "fd12:3456:789a::21", "api.suse.edge.com"}, cluster.InitialiserConfig["tls-san"]) + assert.Equal(t, "totally-not-generated-one", cluster.InitialiserConfig["token"]) + assert.Nil(t, cluster.InitialiserConfig["server"]) + assert.Equal(t, true, cluster.InitialiserConfig["selinux"]) + assert.Nil(t, cluster.InitialiserConfig["debug"]) + + require.NotNil(t, cluster.ServerConfig) + assert.Equal(t, "calico", cluster.ServerConfig["cni"]) + assert.ElementsMatch(t, []string{"192.168.122.50", "fd12:3456:789a::21", "api.suse.edge.com"}, cluster.ServerConfig["tls-san"]) + assert.Equal(t, "totally-not-generated-one", cluster.ServerConfig["token"]) + assert.Equal(t, "https://[fd12:3456:789a::21]:9345", cluster.ServerConfig["server"]) + assert.Equal(t, true, cluster.ServerConfig["selinux"]) + assert.Nil(t, cluster.ServerConfig["debug"]) + + require.NotNil(t, cluster.AgentConfig) + assert.Equal(t, "calico", cluster.AgentConfig["cni"]) + assert.Equal(t, "totally-not-generated-one", cluster.AgentConfig["token"]) + assert.Equal(t, "https://[fd12:3456:789a::21]:9345", cluster.AgentConfig["server"]) + assert.Equal(t, true, cluster.AgentConfig["debug"]) + assert.Equal(t, true, cluster.AgentConfig["selinux"]) + assert.Nil(t, cluster.AgentConfig["tls-san"]) +} + +func TestNewCluster_MultiNodeRKE2_ExistingConfigDualstackPrioritizeIPv4(t *testing.T) { + kubernetes := &image.Kubernetes{ + Version: "v1.30.3+rke2r1", + Network: image.Network{ + APIHost: "api.suse.edge.com", + APIVIP4: "192.168.122.50", + APIVIP6: "fd12:3456:789a::21", + }, + Nodes: []image.Node{ + { + Hostname: "node1.suse.com", + Type: "server", + }, + { + Hostname: "node2.suse.com", + Type: "agent", + }, + }, + } + + cluster, err := NewCluster(kubernetes, filepath.Join("testdata", "dualstack-prio-ipv4")) + require.NoError(t, err) + + assert.Equal(t, "node1.suse.com", cluster.InitialiserName) + + require.NotNil(t, cluster.InitialiserConfig) + assert.Equal(t, "calico", cluster.InitialiserConfig["cni"]) + assert.ElementsMatch(t, []string{"192.168.122.50", "fd12:3456:789a::21", "api.suse.edge.com"}, cluster.InitialiserConfig["tls-san"]) + assert.Equal(t, "totally-not-generated-one", cluster.InitialiserConfig["token"]) + assert.Nil(t, cluster.InitialiserConfig["server"]) + assert.Equal(t, true, cluster.InitialiserConfig["selinux"]) + assert.Nil(t, cluster.InitialiserConfig["debug"]) + + require.NotNil(t, cluster.ServerConfig) + assert.Equal(t, "calico", cluster.ServerConfig["cni"]) + assert.ElementsMatch(t, []string{"192.168.122.50", "fd12:3456:789a::21", "api.suse.edge.com"}, cluster.ServerConfig["tls-san"]) + assert.Equal(t, "totally-not-generated-one", cluster.ServerConfig["token"]) + assert.Equal(t, "https://192.168.122.50:9345", cluster.ServerConfig["server"]) + fmt.Println(cluster.ServerConfig["server"]) + assert.Equal(t, true, cluster.ServerConfig["selinux"]) + assert.Nil(t, cluster.ServerConfig["debug"]) + + require.NotNil(t, cluster.AgentConfig) + assert.Equal(t, "calico", cluster.AgentConfig["cni"]) + assert.Equal(t, "totally-not-generated-one", cluster.AgentConfig["token"]) + assert.Equal(t, "https://192.168.122.50:9345", cluster.AgentConfig["server"]) + assert.Equal(t, true, cluster.AgentConfig["debug"]) + assert.Equal(t, true, cluster.AgentConfig["selinux"]) + assert.Nil(t, cluster.AgentConfig["tls-san"]) +} + func TestNewCluster_MultiNode_MissingInitialiser(t *testing.T) { kubernetes := &image.Kubernetes{ Nodes: []image.Node{ @@ -277,19 +476,28 @@ func TestIdentifyInitialiserNode(t *testing.T) { func TestSetClusterAPIAddress(t *testing.T) { config := map[string]any{} - setClusterAPIAddress(config, nil, 9345) + setClusterAPIAddress(config, nil, nil, 9345, false) assert.NotContains(t, config, "server") ip4, err := netip.ParseAddr("192.168.122.50") assert.NoError(t, err) - setClusterAPIAddress(config, &ip4, 9345) - assert.Equal(t, "https://192.168.122.50:9345", config["server"]) - ip6, err := netip.ParseAddr("fd12:3456:789a::21") assert.NoError(t, err) - setClusterAPIAddress(config, &ip6, 9345) + setClusterAPIAddress(config, &ip4, nil, 9345, false) + assert.Equal(t, "https://192.168.122.50:9345", config["server"]) + + setClusterAPIAddress(config, &ip4, &ip6, 9345, false) + assert.Equal(t, "https://192.168.122.50:9345", config["server"]) + + setClusterAPIAddress(config, &ip4, &ip6, 9345, true) + assert.Equal(t, "https://[fd12:3456:789a::21]:9345", config["server"]) + + setClusterAPIAddress(config, nil, &ip6, 9345, true) + assert.Equal(t, "https://[fd12:3456:789a::21]:9345", config["server"]) + + setClusterAPIAddress(config, &ip4, &ip6, 9345, true) assert.Equal(t, "https://[fd12:3456:789a::21]:9345", config["server"]) } diff --git a/pkg/kubernetes/testdata/agent.yaml b/pkg/kubernetes/testdata/default/agent.yaml similarity index 100% rename from pkg/kubernetes/testdata/agent.yaml rename to pkg/kubernetes/testdata/default/agent.yaml diff --git a/pkg/kubernetes/testdata/server.yaml b/pkg/kubernetes/testdata/default/server.yaml similarity index 100% rename from pkg/kubernetes/testdata/server.yaml rename to pkg/kubernetes/testdata/default/server.yaml diff --git a/pkg/kubernetes/testdata/dualstack-prio-ipv4/agent.yaml b/pkg/kubernetes/testdata/dualstack-prio-ipv4/agent.yaml new file mode 100644 index 00000000..a50015a4 --- /dev/null +++ b/pkg/kubernetes/testdata/dualstack-prio-ipv4/agent.yaml @@ -0,0 +1,3 @@ +cni: calico +token: totally-not-generated-one +debug: true diff --git a/pkg/kubernetes/testdata/dualstack-prio-ipv4/server.yaml b/pkg/kubernetes/testdata/dualstack-prio-ipv4/server.yaml new file mode 100644 index 00000000..118d6822 --- /dev/null +++ b/pkg/kubernetes/testdata/dualstack-prio-ipv4/server.yaml @@ -0,0 +1,5 @@ +cni: calico +token: totally-not-generated-one +selinux: true +cluster-cidr: 10.42.0.0/16,fd12:3456:789b::/48 +service-cidr: 10.43.0.0/16,fd12:3456:789c::/112 \ No newline at end of file diff --git a/pkg/kubernetes/testdata/dualstack-prio-ipv6/agent.yaml b/pkg/kubernetes/testdata/dualstack-prio-ipv6/agent.yaml new file mode 100644 index 00000000..a50015a4 --- /dev/null +++ b/pkg/kubernetes/testdata/dualstack-prio-ipv6/agent.yaml @@ -0,0 +1,3 @@ +cni: calico +token: totally-not-generated-one +debug: true diff --git a/pkg/kubernetes/testdata/dualstack-prio-ipv6/server.yaml b/pkg/kubernetes/testdata/dualstack-prio-ipv6/server.yaml new file mode 100644 index 00000000..1dbe3bd3 --- /dev/null +++ b/pkg/kubernetes/testdata/dualstack-prio-ipv6/server.yaml @@ -0,0 +1,5 @@ +cni: calico +token: totally-not-generated-one +selinux: true +cluster-cidr: fd12:3456:789b::/48,10.42.0.0/16 +service-cidr: fd12:3456:789c::/112,10.43.0.0/16 \ No newline at end of file