Dual-stack Handling

RELEASE_NOTES.md

@@ -4,6 +4,8 @@
 
 ## General
 
+* Added support for IPv6 addresses in the Kubernetes 'apiVIP' field
+
 ## API
 
 ### Image Definition Changes

docs/building-images.md

@@ -261,7 +261,7 @@ kubernetes:
 * `network` - Required for multi-node clusters, optional for single-node clusters; Defines the network configuration 
 for bootstrapping a cluster.
   * `apiVIP` - Required for multi-node clusters, optional for single-node clusters; Specifies the IP address which
-  will serve as the cluster LoadBalancer, backed by MetalLB.
+  will serve as the cluster LoadBalancer, backed by MetalLB. Supports IPv4 and IPv6 addresses.
   * `apiHost` - Optional; Specifies the domain address for accessing the cluster.
 * `nodes` - Required for multi-node clusters; Defines a list of all nodes that form the cluster.
   * `hostname` - Required; Indicates the fully qualified domain name (FQDN) to identify the particular node on which

pkg/combustion/helm.go

@@ -22,7 +22,7 @@ func ComponentHelmCharts(ctx *image.Context) ([]image.HelmChart, []image.HelmRep
 	var charts []image.HelmChart
 	var repos []image.HelmRepository
 
-	if ctx.ImageDefinition.Kubernetes.Network.APIVIP != "" {
+	if ctx.ImageDefinition.Kubernetes.Network.APIVIP4 != "" || ctx.ImageDefinition.Kubernetes.Network.APIVIP6 != "" {
 		metalLBChart := image.HelmChart{
 			Name:                  ctx.ArtifactSources.MetalLB.Chart,
 			RepositoryName:        metallbRepositoryName,

pkg/combustion/kubernetes.go

@@ -3,6 +3,7 @@ package combustion
 import (
 	_ "embed"
 	"fmt"
+	"net/netip"
 	"os"
 	"path/filepath"
 	"strings"
@@ -34,6 +35,7 @@ const (
 	k8sAgentConfigFile      = "agent.yaml"
 
 	k8sInstallScript = "20-k8s-install.sh"
+	setNodeIPScript  = "set-node-ip.sh"
 )
 
 var (
@@ -51,6 +53,9 @@ var (
 
 	//go:embed templates/k8s-vip.yaml.tpl
 	k8sVIPManifest string
+
+	//go:embed templates/set-node-ip.sh.tpl
+	nodeIPScriptTemplate string
 )
 
 func (c *Combustion) configureKubernetes(ctx *image.Context) ([]string, error) {
@@ -85,6 +90,10 @@ func (c *Combustion) configureKubernetes(ctx *image.Context) ([]string, error) {
 		return nil, fmt.Errorf("initialising cluster config: %w", err)
 	}
 
+	if err = createNodeIPScript(ctx, cluster.ServerConfig); err != nil {
+		return nil, fmt.Errorf("creating set node IP script: %w", err)
+	}
+
 	artefactsPath := kubernetesArtefactsPath(ctx)
 	if err = os.MkdirAll(artefactsPath, os.ModePerm); err != nil {
 		return nil, fmt.Errorf("creating kubernetes artefacts path: %w", err)
@@ -147,18 +156,20 @@ func (c *Combustion) configureK3S(ctx *image.Context, cluster *kubernetes.Cluste
 
 	templateValues := map[string]any{
 		"installScript":   installScript,
-		"apiVIP":          ctx.ImageDefinition.Kubernetes.Network.APIVIP,
+		"apiVIP4":         ctx.ImageDefinition.Kubernetes.Network.APIVIP4,
+		"apiVIP6":         ctx.ImageDefinition.Kubernetes.Network.APIVIP6,
 		"apiHost":         ctx.ImageDefinition.Kubernetes.Network.APIHost,
 		"binaryPath":      binaryPath,
 		"imagesPath":      imagesPath,
 		"manifestsPath":   manifestsPath,
 		"configFilePath":  prependArtefactPath(k8sDir),
 		"registryMirrors": prependArtefactPath(filepath.Join(k8sDir, registryMirrorsFileName)),
+		"setNodeIPScript": setNodeIPScript,
 	}
 
 	singleNode := len(ctx.ImageDefinition.Kubernetes.Nodes) < 2
 	if singleNode {
-		if ctx.ImageDefinition.Kubernetes.Network.APIVIP == "" {
+		if ctx.ImageDefinition.Kubernetes.Network.APIVIP4 == "" || ctx.ImageDefinition.Kubernetes.Network.APIVIP6 == "" {
 			zap.S().Info("Virtual IP address for k3s cluster is not provided and will not be configured")
 		} else {
 			log.Audit("WARNING: A Virtual IP address for the k3s cluster has been provided. " +
@@ -242,18 +253,20 @@ func (c *Combustion) configureRKE2(ctx *image.Context, cluster *kubernetes.Clust
 
 	templateValues := map[string]any{
 		"installScript":   installScript,
-		"apiVIP":          ctx.ImageDefinition.Kubernetes.Network.APIVIP,
+		"apiVIP4":         ctx.ImageDefinition.Kubernetes.Network.APIVIP4,
+		"apiVIP6":         ctx.ImageDefinition.Kubernetes.Network.APIVIP6,
 		"apiHost":         ctx.ImageDefinition.Kubernetes.Network.APIHost,
 		"installPath":     installPath,
 		"imagesPath":      imagesPath,
 		"manifestsPath":   manifestsPath,
 		"configFilePath":  prependArtefactPath(k8sDir),
 		"registryMirrors": prependArtefactPath(filepath.Join(k8sDir, registryMirrorsFileName)),
+		"setNodeIPScript": setNodeIPScript,
 	}
 
 	singleNode := len(ctx.ImageDefinition.Kubernetes.Nodes) < 2
 	if singleNode {
-		if ctx.ImageDefinition.Kubernetes.Network.APIVIP == "" {
+		if ctx.ImageDefinition.Kubernetes.Network.APIVIP4 == "" || ctx.ImageDefinition.Kubernetes.Network.APIVIP6 == "" {
 			zap.S().Info("Virtual IP address for RKE2 cluster is not provided and will not be configured")
 		}
 
@@ -316,17 +329,77 @@ func (c *Combustion) downloadRKE2Artefacts(ctx *image.Context, cluster *kubernet
 }
 
 func kubernetesVIPManifest(k *image.Kubernetes) (string, error) {
+	var err error
+
+	var ip4 netip.Addr
+	if k.Network.APIVIP4 != "" {
+		ip4, err = netip.ParseAddr(k.Network.APIVIP4)
+		if err != nil {
+			return "", fmt.Errorf("parsing kubernetes apiVIP address: %w", err)
+		}
+	}
+
+	var ip6 netip.Addr
+	if k.Network.APIVIP6 != "" {
+		ip6, err = netip.ParseAddr(k.Network.APIVIP6)
+		if err != nil {
+			return "", fmt.Errorf("parsing kubernetes apiVIP address: %w", err)
+		}
+	}
+
 	manifest := struct {
-		APIAddress string
-		RKE2       bool
+		APIAddress4 string
+		APIAddress6 string
+		IsIPV4      bool
+		IsIPV6      bool
+		RKE2        bool
 	}{
-		APIAddress: k.Network.APIVIP,
-		RKE2:       strings.Contains(k.Version, image.KubernetesDistroRKE2),
+		APIAddress4: k.Network.APIVIP4,
+		APIAddress6: k.Network.APIVIP6,
+		IsIPV4:      ip4.Is4(),
+		IsIPV6:      ip6.Is6(),
+		RKE2:        strings.Contains(k.Version, image.KubernetesDistroRKE2),
 	}
 
 	return template.Parse("k8s-vip", k8sVIPManifest, &manifest)
 }
 
+func createNodeIPScript(ctx *image.Context, serverConfig map[string]any) error {
+	// Setting the Node IP only matters if we're doing dual-stack or single-stack IPv6
+	if ctx.ImageDefinition.Kubernetes.Network.APIVIP6 == "" {
+		return nil
+	}
+
+	var isIPv4Enabled bool
+	if ctx.ImageDefinition.Kubernetes.Network.APIVIP4 != "" {
+		isIPv4Enabled = true
+	}
+
+	manifest := struct {
+		IPv4Enabled    bool
+		IPv6Enabled    bool
+		PrioritizeIPv6 bool
+		RKE2           bool
+	}{
+		IPv4Enabled:    isIPv4Enabled,
+		IPv6Enabled:    true,
+		PrioritizeIPv6: isIPv6Priority(serverConfig),
+		RKE2:           strings.Contains(ctx.ImageDefinition.Kubernetes.Version, image.KubernetesDistroRKE2),
+	}
+
+	data, err := template.Parse("set-node-ip", nodeIPScriptTemplate, &manifest)
+	if err != nil {
+		return fmt.Errorf("parsing '%s' template: %w", setNodeIPScript, err)
+	}
+
+	nodeIPScript := filepath.Join(ctx.CombustionDir, setNodeIPScript)
+	if err = os.WriteFile(nodeIPScript, []byte(data), fileio.ExecutablePerms); err != nil {
+		return fmt.Errorf("writing set node IP script: %w", err)
+	}
+
+	return nil
+}
+
 func storeKubernetesClusterConfig(cluster *kubernetes.Cluster, destPath string) error {
 	serverConfig := filepath.Join(destPath, k8sServerConfigFile)
 	if err := storeKubernetesConfig(cluster.ServerConfig, serverConfig); err != nil {
@@ -367,7 +440,7 @@ func (c *Combustion) configureManifests(ctx *image.Context) (string, error) {
 	manifestsPath := localKubernetesManifestsPath()
 	manifestDestDir := filepath.Join(ctx.ArtefactsDir, manifestsPath)
 
-	if ctx.ImageDefinition.Kubernetes.Network.APIVIP != "" {
+	if ctx.ImageDefinition.Kubernetes.Network.APIVIP4 != "" || ctx.ImageDefinition.Kubernetes.Network.APIVIP6 != "" {
 		if err := os.MkdirAll(manifestDestDir, os.ModePerm); err != nil {
 			return "", fmt.Errorf("creating manifests destination dir: %w", err)
 		}
@@ -450,3 +523,14 @@ func HelmCertsPath(ctx *image.Context) string {
 func kubernetesArtefactsPath(ctx *image.Context) string {
 	return filepath.Join(ctx.ArtefactsDir, k8sDir)
 }
+
+func isIPv6Priority(serverConfig map[string]any) bool {
+	if clusterCIDR, ok := serverConfig["cluster-cidr"].(string); ok {
+		cidrs := strings.Split(clusterCIDR, ",")
+		if len(cidrs) > 0 {
+			return strings.Contains(cidrs[0], ":")
+		}
+	}
+
+	return false
+}

pkg/combustion/kubernetes_test.go

@@ -220,7 +220,7 @@ func TestConfigureKubernetes_SuccessfulSingleNodeK3sCluster(t *testing.T) {
 	ctx.ImageDefinition.Kubernetes = image.Kubernetes{
 		Version: "v1.30.3+k3s1",
 		Network: image.Network{
-			APIVIP:  "192.168.122.100",
+			APIVIP4: "192.168.122.100",
 			APIHost: "api.cluster01.hosted.on.edge.suse.com",
 		},
 	}
@@ -293,7 +293,7 @@ func TestConfigureKubernetes_SuccessfulMultiNodeK3sCluster(t *testing.T) {
 		Version: "v1.30.3+k3s1",
 		Network: image.Network{
 			APIHost: "api.cluster01.hosted.on.edge.suse.com",
-			APIVIP:  "192.168.122.100",
+			APIVIP4: "192.168.122.100",
 		},
 		Nodes: []image.Node{
 			{
@@ -423,7 +423,7 @@ func TestConfigureKubernetes_SuccessfulSingleNodeRKE2Cluster(t *testing.T) {
 	ctx.ImageDefinition.Kubernetes = image.Kubernetes{
 		Version: "v1.30.3+rke2r1",
 		Network: image.Network{
-			APIVIP:  "192.168.122.100",
+			APIVIP4: "192.168.122.100",
 			APIHost: "api.cluster01.hosted.on.edge.suse.com",
 		},
 	}
@@ -492,7 +492,7 @@ func TestConfigureKubernetes_SuccessfulMultiNodeRKE2Cluster(t *testing.T) {
 		Version: "v1.30.3+rke2r1",
 		Network: image.Network{
 			APIHost: "api.cluster01.hosted.on.edge.suse.com",
-			APIVIP:  "192.168.122.100",
+			APIVIP4: "192.168.122.100",
 		},
 		Nodes: []image.Node{
 			{
@@ -734,7 +734,7 @@ func TestConfigureKubernetes_SuccessfulRKE2ServerWithManifests(t *testing.T) {
 	ctx.ImageDefinition.Kubernetes = image.Kubernetes{
 		Version: "v1.30.3+rke2r1",
 		Network: image.Network{
-			APIVIP:  "192.168.122.100",
+			APIVIP4: "192.168.122.100",
 			APIHost: "api.cluster01.hosted.on.edge.suse.com",
 		},
 	}
@@ -821,3 +821,46 @@ func TestConfigureKubernetes_SuccessfulRKE2ServerWithManifests(t *testing.T) {
 	assert.Contains(t, contents, "name: my-nginx")
 	assert.Contains(t, contents, "image: nginx:1.14.2")
 }
+
+func TestKubernetesVIPManifestValidIPV4(t *testing.T) {
+	k8s := &image.Kubernetes{
+		Version: "v1.30.3+rke2r1",
+		Network: image.Network{
+			APIVIP4: "192.168.1.1",
+		},
+	}
+
+	manifest, err := kubernetesVIPManifest(k8s)
+	require.NoError(t, err)
+
+	assert.Contains(t, manifest, "- 192.168.1.1/32")
+	assert.Contains(t, manifest, "- name: rke2-api")
+}
+
+func TestKubernetesVIPManifestValidIPV6(t *testing.T) {
+	k8s := &image.Kubernetes{
+		Version: "v1.30.3+k3s1",
+		Network: image.Network{
+			APIVIP4: "fd12:3456:789a::21",
+		},
+	}
+
+	manifest, err := kubernetesVIPManifest(k8s)
+	require.NoError(t, err)
+
+	assert.Contains(t, manifest, "- fd12:3456:789a::21/128")
+	assert.Contains(t, manifest, "- name: k8s-api")
+	assert.NotContains(t, manifest, "rke2")
+}
+
+func TestKubernetesVIPManifestInvalidIP(t *testing.T) {
+	k8s := &image.Kubernetes{
+		Version: "v1.30.3+k3s1",
+		Network: image.Network{
+			APIVIP4: "1111",
+		},
+	}
+
+	_, err := kubernetesVIPManifest(k8s)
+	require.ErrorContains(t, err, "parsing kubernetes apiVIP address: ParseAddr(\"1111\"): unable to parse IP")
+}

pkg/combustion/templates/k3s-multi-node-installer.sh.tpl

@@ -89,13 +89,24 @@ systemctl enable kubernetes-resources-install.service
 {{- end }}
 fi
 
-{{- if and .apiVIP .apiHost }}
-echo "{{ .apiVIP }} {{ .apiHost }}" >> /etc/hosts
+{{- if and .apiVIP4 .apiHost }}
+echo "{{ .apiVIP4 }} {{ .apiHost }}" >> /etc/hosts
+{{- end }}
+
+{{- if and .apiVIP6 .apiHost }}
+echo "{{ .apiVIP6 }} {{ .apiHost }}" >> /etc/hosts
 {{- end }}
 
 mkdir -p /etc/rancher/k3s/
 cp $CONFIGFILE /etc/rancher/k3s/config.yaml
 
+if [ "$NODETYPE" = "server" ]; then
+{{- if .apiVIP6 }}
+chmod +x {{ .setNodeIPScript }}
+sh {{ .setNodeIPScript }}
+{{- end }}
+fi
+
 if [ -f {{ .registryMirrors }} ]; then
 cp {{ .registryMirrors }} /etc/rancher/k3s/registries.yaml
 fi

pkg/combustion/templates/k3s-single-node-installer.sh.tpl

@@ -62,13 +62,22 @@ EOF
 systemctl enable kubernetes-resources-install.service
 {{- end }}
 
-{{- if and .apiVIP .apiHost }}
-echo "{{ .apiVIP }} {{ .apiHost }}" >> /etc/hosts
+{{- if and .apiVIP4 .apiHost }}
+echo "{{ .apiVIP4 }} {{ .apiHost }}" >> /etc/hosts
+{{- end }}
+
+{{- if and .apiVIP6 .apiHost }}
+echo "{{ .apiVIP6 }} {{ .apiHost }}" >> /etc/hosts
 {{- end }}
 
 mkdir -p /etc/rancher/k3s/
 cp {{ .configFilePath }}/{{ .configFile }} /etc/rancher/k3s/config.yaml
 
+{{- if .apiVIP6 }}
+chmod +x {{ .setNodeIPScript }}
+sh {{ .setNodeIPScript }}
+{{- end }}
+
 if [ -f {{ .registryMirrors }} ]; then
 cp {{ .registryMirrors }} /etc/rancher/k3s/registries.yaml
 fi

pkg/combustion/templates/k8s-vip.yaml.tpl

@@ -6,7 +6,12 @@ metadata:
   namespace: metallb-system
 spec:
   addresses:
-  - {{ .APIAddress }}/32
+  {{- if .APIAddress4 }}
+    - {{ .APIAddress4 }}/32
+  {{- end }}
+  {{- if .APIAddress6 }}
+    - {{ .APIAddress6 }}/128
+  {{- end }}
   avoidBuggyIPs: true
   serviceAllocation:
     namespaces:
@@ -32,6 +37,17 @@ metadata:
   labels:
     serviceType: kubernetes-vip
 spec:
+  {{- if and .APIAddress4 .APIAddress6 }}
+  ipFamilyPolicy: RequireDualStack
+  ipFamilies:
+    - IPv4
+    - IPv6
+  {{- end }}
+  {{- if and (eq .APIAddress4 "false") (eq .APIAddress6 "true") }}
+    ipFamilyPolicy: SingleStack
+    ipFamilies:
+      - IPv6
+  {{- end }}
   ports:
 {{- if .RKE2 }}
   - name: rke2-api