Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dual-stack Handling #612

Open
wants to merge 18 commits into
base: main
Choose a base branch
from
Open

Dual-stack Handling #612

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
bcafb09
pre-testing-updates
dbw7 Oct 15, 2024
26e8725
beginning to write tests
dbw7 Oct 16, 2024
1e82f22
ipv6 handling
dbw7 Oct 18, 2024
9a6cd09
remove unnecessary if
dbw7 Oct 18, 2024
090d385
update cluster_test
dbw7 Oct 18, 2024
d66ad0b
feedback updates
dbw7 Oct 18, 2024
38dc6b7
add release notes and doc
dbw7 Oct 18, 2024
cdca128
fix validation bug
dbw7 Oct 18, 2024
7582178
feedback updates
dbw7 Oct 21, 2024
86843c2
feedback updates
dbw7 Oct 21, 2024
e100194
saving this state
dbw7 Nov 14, 2024
0c6e1d6
feedback updates + almost done
dbw7 Nov 22, 2024
b4f9022
code updates
dbw7 Dec 4, 2024
e87c034
update docs
dbw7 Dec 4, 2024
b89d867
Merge branch 'main' into dualstack
dbw7 Dec 4, 2024
cbe2af3
feedback updates
dbw7 Dec 12, 2024
c284c6e
more feedback updates
dbw7 Dec 13, 2024
b33df73
updates
dbw7 Dec 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions RELEASE_NOTES.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@

## General

* Added 'apiVIP6' in Kubernetes field for IPv6 addresses
atanasdinov marked this conversation as resolved.
Show resolved Hide resolved
* Implemented IPv6 Single and Dual-Stack handling for Kubernetes
atanasdinov marked this conversation as resolved.
Show resolved Hide resolved

## API

### Image Definition Changes
Expand Down
2 changes: 1 addition & 1 deletion docs/building-images.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@ kubernetes:
* `network` - Required for multi-node clusters, optional for single-node clusters; Defines the network configuration
for bootstrapping a cluster.
* `apiVIP` - Required for multi-node clusters, optional for single-node clusters; Specifies the IP address which
atanasdinov marked this conversation as resolved.
Show resolved Hide resolved
will serve as the cluster LoadBalancer, backed by MetalLB.
will serve as the cluster LoadBalancer, backed by MetalLB. Supports IPv4 and IPv6 addresses.
* `apiHost` - Optional; Specifies the domain address for accessing the cluster.
* `nodes` - Required for multi-node clusters; Defines a list of all nodes that form the cluster.
* `hostname` - Required; Indicates the fully qualified domain name (FQDN) to identify the particular node on which
Expand Down
2 changes: 1 addition & 1 deletion pkg/combustion/helm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ func ComponentHelmCharts(ctx *image.Context) ([]image.HelmChart, []image.HelmRep
var charts []image.HelmChart
var repos []image.HelmRepository

if ctx.ImageDefinition.Kubernetes.Network.APIVIP != "" {
if ctx.ImageDefinition.Kubernetes.Network.APIVIP4 != "" || ctx.ImageDefinition.Kubernetes.Network.APIVIP6 != "" {
metalLBChart := image.HelmChart{
Name: ctx.ArtifactSources.MetalLB.Chart,
RepositoryName: metallbRepositoryName,
Expand Down
85 changes: 71 additions & 14 deletions pkg/combustion/kubernetes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ const (
k8sAgentConfigFile = "agent.yaml"

k8sInstallScript = "20-k8s-install.sh"
setNodeIPScript = "set-node-ip.sh"
)

var (
Expand All @@ -51,6 +52,9 @@ var (

//go:embed templates/k8s-vip.yaml.tpl
k8sVIPManifest string

//go:embed templates/set-node-ip.sh.tpl
nodeIPScriptTemplate string
)

func (c *Combustion) configureKubernetes(ctx *image.Context) ([]string, error) {
Expand Down Expand Up @@ -85,6 +89,10 @@ func (c *Combustion) configureKubernetes(ctx *image.Context) ([]string, error) {
return nil, fmt.Errorf("initialising cluster config: %w", err)
}

if err = createNodeIPScript(ctx, cluster.ServerConfig); err != nil {
return nil, fmt.Errorf("creating set node IP script: %w", err)
}

artefactsPath := kubernetesArtefactsPath(ctx)
if err = os.MkdirAll(artefactsPath, os.ModePerm); err != nil {
return nil, fmt.Errorf("creating kubernetes artefacts path: %w", err)
Expand Down Expand Up @@ -147,23 +155,25 @@ func (c *Combustion) configureK3S(ctx *image.Context, cluster *kubernetes.Cluste

templateValues := map[string]any{
"installScript": installScript,
"apiVIP": ctx.ImageDefinition.Kubernetes.Network.APIVIP,
"apiVIP4": ctx.ImageDefinition.Kubernetes.Network.APIVIP4,
"apiVIP6": ctx.ImageDefinition.Kubernetes.Network.APIVIP6,
"apiHost": ctx.ImageDefinition.Kubernetes.Network.APIHost,
"binaryPath": binaryPath,
"imagesPath": imagesPath,
"manifestsPath": manifestsPath,
"configFilePath": prependArtefactPath(k8sDir),
"registryMirrors": prependArtefactPath(filepath.Join(k8sDir, registryMirrorsFileName)),
"setNodeIPScript": setNodeIPScript,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we get this value from somewhere else? It's better to "tie" it with the rest of the code, where this is a variable that's returned by the function which has created it. Not an arbitrary constant.

}

singleNode := len(ctx.ImageDefinition.Kubernetes.Nodes) < 2
if singleNode {
if ctx.ImageDefinition.Kubernetes.Network.APIVIP == "" {
zap.S().Info("Virtual IP address for k3s cluster is not provided and will not be configured")
if ctx.ImageDefinition.Kubernetes.Network.APIVIP4 == "" && ctx.ImageDefinition.Kubernetes.Network.APIVIP6 == "" {
zap.S().Info("Virtual IP address(es) for k3s cluster is not provided and will not be configured")
} else {
log.Audit("WARNING: A Virtual IP address for the k3s cluster has been provided. " +
log.Audit("WARNING: Virtual IP address(es) for the k3s cluster provided. " +
"An external IP address for the Ingress Controller (Traefik) must be manually configured.")
zap.S().Warn("Virtual IP address for k3s cluster is requested and will invalidate Traefik configuration")
zap.S().Warn("Virtual IP address(es) for k3s cluster requested and will invalidate Traefik configuration")
}

templateValues["configFile"] = k8sServerConfigFile
Expand All @@ -172,7 +182,7 @@ func (c *Combustion) configureK3S(ctx *image.Context, cluster *kubernetes.Cluste
}

log.Audit("WARNING: An external IP address for the Ingress Controller (Traefik) must be manually configured in multi-node clusters.")
zap.S().Warn("Virtual IP address for k3s cluster is necessary for multi node clusters and will invalidate Traefik configuration")
zap.S().Warn("Virtual IP address(es) for k3s cluster necessary for multi node clusters and will invalidate Traefik configuration")

templateValues["nodes"] = ctx.ImageDefinition.Kubernetes.Nodes
templateValues["initialiser"] = cluster.InitialiserName
Expand Down Expand Up @@ -242,19 +252,21 @@ func (c *Combustion) configureRKE2(ctx *image.Context, cluster *kubernetes.Clust

templateValues := map[string]any{
"installScript": installScript,
"apiVIP": ctx.ImageDefinition.Kubernetes.Network.APIVIP,
"apiVIP4": ctx.ImageDefinition.Kubernetes.Network.APIVIP4,
"apiVIP6": ctx.ImageDefinition.Kubernetes.Network.APIVIP6,
"apiHost": ctx.ImageDefinition.Kubernetes.Network.APIHost,
"installPath": installPath,
"imagesPath": imagesPath,
"manifestsPath": manifestsPath,
"configFilePath": prependArtefactPath(k8sDir),
"registryMirrors": prependArtefactPath(filepath.Join(k8sDir, registryMirrorsFileName)),
"setNodeIPScript": setNodeIPScript,
}

singleNode := len(ctx.ImageDefinition.Kubernetes.Nodes) < 2
if singleNode {
if ctx.ImageDefinition.Kubernetes.Network.APIVIP == "" {
zap.S().Info("Virtual IP address for RKE2 cluster is not provided and will not be configured")
if ctx.ImageDefinition.Kubernetes.Network.APIVIP4 == "" && ctx.ImageDefinition.Kubernetes.Network.APIVIP6 == "" {
zap.S().Info("Virtual IP address(es) for RKE2 cluster is not provided and will not be configured")
}

templateValues["configFile"] = k8sServerConfigFile
Expand Down Expand Up @@ -316,17 +328,62 @@ func (c *Combustion) downloadRKE2Artefacts(ctx *image.Context, cluster *kubernet
}

func kubernetesVIPManifest(k *image.Kubernetes) (string, error) {
onlyIPv6 := false
if k.Network.APIVIP4 == "" && k.Network.APIVIP6 != "" {
onlyIPv6 = true
}

manifest := struct {
APIAddress string
RKE2 bool
APIAddress4 string
APIAddress6 string
RKE2 bool
OnlyIPv6 bool
}{
APIAddress: k.Network.APIVIP,
RKE2: strings.Contains(k.Version, image.KubernetesDistroRKE2),
APIAddress4: k.Network.APIVIP4,
APIAddress6: k.Network.APIVIP6,
RKE2: strings.Contains(k.Version, image.KubernetesDistroRKE2),
OnlyIPv6: onlyIPv6,
}

return template.Parse("k8s-vip", k8sVIPManifest, &manifest)
}

func createNodeIPScript(ctx *image.Context, serverConfig map[string]any) error {
// Setting the Node IP only matters if we're doing dual-stack or single-stack IPv6
if ctx.ImageDefinition.Kubernetes.Network.APIVIP6 == "" {
return nil
}

var isIPv4Enabled bool
if ctx.ImageDefinition.Kubernetes.Network.APIVIP4 != "" {
isIPv4Enabled = true
}

manifest := struct {
IPv4Enabled bool
IPv6Enabled bool
PrioritizeIPv6 bool
RKE2 bool
}{
IPv4Enabled: isIPv4Enabled,
IPv6Enabled: true,
atanasdinov marked this conversation as resolved.
Show resolved Hide resolved
PrioritizeIPv6: kubernetes.IsIPv6Priority(serverConfig),
RKE2: strings.Contains(ctx.ImageDefinition.Kubernetes.Version, image.KubernetesDistroRKE2),
}

data, err := template.Parse("set-node-ip", nodeIPScriptTemplate, &manifest)
if err != nil {
return fmt.Errorf("parsing '%s' template: %w", setNodeIPScript, err)
}

nodeIPScript := filepath.Join(ctx.CombustionDir, setNodeIPScript)
if err = os.WriteFile(nodeIPScript, []byte(data), fileio.ExecutablePerms); err != nil {
return fmt.Errorf("writing set node IP script: %w", err)
}

return nil
}

func storeKubernetesClusterConfig(cluster *kubernetes.Cluster, destPath string) error {
serverConfig := filepath.Join(destPath, k8sServerConfigFile)
if err := storeKubernetesConfig(cluster.ServerConfig, serverConfig); err != nil {
Expand Down Expand Up @@ -367,7 +424,7 @@ func (c *Combustion) configureManifests(ctx *image.Context) (string, error) {
manifestsPath := localKubernetesManifestsPath()
manifestDestDir := filepath.Join(ctx.ArtefactsDir, manifestsPath)

if ctx.ImageDefinition.Kubernetes.Network.APIVIP != "" {
if ctx.ImageDefinition.Kubernetes.Network.APIVIP4 != "" || ctx.ImageDefinition.Kubernetes.Network.APIVIP6 != "" {
if err := os.MkdirAll(manifestDestDir, os.ModePerm); err != nil {
return "", fmt.Errorf("creating manifests destination dir: %w", err)
}
Expand Down
Loading
Loading