Skip to content

Commit

Permalink
more feedback updates
Browse files Browse the repository at this point in the history
  • Loading branch information
@dbw7
dbw7 committed Dec 13, 2024
1 parent cbe2af3 commit c284c6e
Show file tree
Hide file tree
Showing 10 changed files with 187 additions and 180 deletions.
68 changes: 36 additions & 32 deletions pkg/combustion/kubernetes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ import (
const (
k8sComponentName = "kubernetes"

K8sDir = "kubernetes"
K8sConfigDir = "config"
k8sDir = "kubernetes"
k8sConfigDir = "config"
k8sInstallDir = "install"
k8sImagesDir = "images"
k8sManifestsDir = "manifests"
Expand Down Expand Up @@ -80,19 +80,15 @@ func (c *Combustion) configureKubernetes(ctx *image.Context) ([]string, error) {
zap.S().Warn("Kubernetes cluster of two server nodes has been requested")
}

configDir := generateComponentPath(ctx, K8sDir)
configPath := filepath.Join(configDir, K8sConfigDir)
configDir := generateComponentPath(ctx, k8sDir)
configPath := filepath.Join(configDir, k8sConfigDir)

cluster, err := kubernetes.NewCluster(&ctx.ImageDefinition.Kubernetes, configPath)
if err != nil {
log.AuditComponentFailed(k8sComponentName)
return nil, fmt.Errorf("initialising cluster config: %w", err)
}

if err = createNodeIPScript(ctx, cluster.ServerConfig); err != nil {
return nil, fmt.Errorf("creating set node IP script: %w", err)
}

artefactsPath := kubernetesArtefactsPath(ctx)
if err = os.MkdirAll(artefactsPath, os.ModePerm); err != nil {
return nil, fmt.Errorf("creating kubernetes artefacts path: %w", err)
Expand Down Expand Up @@ -132,7 +128,7 @@ func (c *Combustion) downloadKubernetesInstallScript(ctx *image.Context, distrib
return "", fmt.Errorf("downloading install script: %w", err)
}

return prependArtefactPath(filepath.Join(K8sDir, installScript)), nil
return prependArtefactPath(filepath.Join(k8sDir, installScript)), nil
}

func (c *Combustion) configureK3S(ctx *image.Context, cluster *kubernetes.Cluster) (string, error) {
Expand All @@ -153,6 +149,11 @@ func (c *Combustion) configureK3S(ctx *image.Context, cluster *kubernetes.Cluste
return "", fmt.Errorf("configuring kubernetes manifests: %w", err)
}

nodeIPScript, err := createNodeIPScript(ctx, cluster.ServerConfig)
if err != nil {
return "", fmt.Errorf("creating set node IP script: %w", err)
}

templateValues := map[string]any{
"installScript": installScript,
"apiVIP4": ctx.ImageDefinition.Kubernetes.Network.APIVIP4,
Expand All @@ -161,10 +162,9 @@ func (c *Combustion) configureK3S(ctx *image.Context, cluster *kubernetes.Cluste
"binaryPath": binaryPath,
"imagesPath": imagesPath,
"manifestsPath": manifestsPath,
"configFilePath": prependArtefactPath(K8sDir),
"registryMirrors": prependArtefactPath(filepath.Join(K8sDir, registryMirrorsFileName)),
"setNodeIPScript": setNodeIPScript,
"getNodeIP": kubernetes.GetNodeIP(cluster.ServerConfig),
"configFilePath": prependArtefactPath(k8sDir),
"registryMirrors": prependArtefactPath(filepath.Join(k8sDir, registryMirrorsFileName)),
"setNodeIPScript": nodeIPScript,
}

singleNode := len(ctx.ImageDefinition.Kubernetes.Nodes) < 2
Expand Down Expand Up @@ -193,13 +193,13 @@ func (c *Combustion) configureK3S(ctx *image.Context, cluster *kubernetes.Cluste
}

func (c *Combustion) downloadK3sArtefacts(ctx *image.Context) (binaryPath, imagesPath string, err error) {
imagesPath = filepath.Join(K8sDir, k8sImagesDir)
imagesPath = filepath.Join(k8sDir, k8sImagesDir)
imagesDestination := filepath.Join(ctx.ArtefactsDir, imagesPath)
if err = os.MkdirAll(imagesDestination, os.ModePerm); err != nil {
return "", "", fmt.Errorf("creating kubernetes images dir: %w", err)
}

installPath := filepath.Join(K8sDir, k8sInstallDir)
installPath := filepath.Join(k8sDir, k8sInstallDir)
installDestination := filepath.Join(ctx.ArtefactsDir, installPath)
if err = os.MkdirAll(installDestination, os.ModePerm); err != nil {
return "", "", fmt.Errorf("creating kubernetes install dir: %w", err)
Expand Down Expand Up @@ -251,6 +251,11 @@ func (c *Combustion) configureRKE2(ctx *image.Context, cluster *kubernetes.Clust
return "", fmt.Errorf("configuring kubernetes manifests: %w", err)
}

nodeIPScript, err := createNodeIPScript(ctx, cluster.ServerConfig)
if err != nil {
return "", fmt.Errorf("creating set node IP script: %w", err)
}

templateValues := map[string]any{
"installScript": installScript,
"apiVIP4": ctx.ImageDefinition.Kubernetes.Network.APIVIP4,
Expand All @@ -259,10 +264,9 @@ func (c *Combustion) configureRKE2(ctx *image.Context, cluster *kubernetes.Clust
"installPath": installPath,
"imagesPath": imagesPath,
"manifestsPath": manifestsPath,
"configFilePath": prependArtefactPath(K8sDir),
"registryMirrors": prependArtefactPath(filepath.Join(K8sDir, registryMirrorsFileName)),
"setNodeIPScript": setNodeIPScript,
"getNodeIP": kubernetes.GetNodeIP(cluster.ServerConfig),
"configFilePath": prependArtefactPath(k8sDir),
"registryMirrors": prependArtefactPath(filepath.Join(k8sDir, registryMirrorsFileName)),
"setNodeIPScript": nodeIPScript,
}

singleNode := len(ctx.ImageDefinition.Kubernetes.Nodes) < 2
Expand Down Expand Up @@ -303,13 +307,13 @@ func (c *Combustion) downloadRKE2Artefacts(ctx *image.Context, cluster *kubernet
return "", "", fmt.Errorf("extracting CNI from cluster config: %w", err)
}

imagesPath = filepath.Join(K8sDir, k8sImagesDir)
imagesPath = filepath.Join(k8sDir, k8sImagesDir)
imagesDestination := filepath.Join(ctx.ArtefactsDir, imagesPath)
if err = os.MkdirAll(imagesDestination, os.ModePerm); err != nil {
return "", "", fmt.Errorf("creating kubernetes images dir: %w", err)
}

installPath = filepath.Join(K8sDir, k8sInstallDir)
installPath = filepath.Join(k8sDir, k8sInstallDir)
installDestination := filepath.Join(ctx.ArtefactsDir, installPath)
if err = os.MkdirAll(installDestination, os.ModePerm); err != nil {
return "", "", fmt.Errorf("creating kubernetes install dir: %w", err)
Expand Down Expand Up @@ -343,10 +347,10 @@ func kubernetesVIPManifest(k *image.Kubernetes) (string, error) {
return template.Parse("k8s-vip", k8sVIPManifest, &manifest)
}

func createNodeIPScript(ctx *image.Context, serverConfig map[string]any) error {
func createNodeIPScript(ctx *image.Context, serverConfig map[string]any) (string, error) {
// Setting the Node IP only matters if we're doing dual-stack or single-stack IPv6
if ctx.ImageDefinition.Kubernetes.Network.APIVIP6 == "" {
return nil
if ctx.ImageDefinition.Kubernetes.Network.APIVIP6 == "" || !kubernetes.GetNodeIP(serverConfig) {
return "", nil
}

var isIPv4Enabled bool
Expand All @@ -366,15 +370,15 @@ func createNodeIPScript(ctx *image.Context, serverConfig map[string]any) error {

data, err := template.Parse("set-node-ip", nodeIPScriptTemplate, &manifest)
if err != nil {
return fmt.Errorf("parsing '%s' template: %w", setNodeIPScript, err)
return "", fmt.Errorf("parsing '%s' template: %w", setNodeIPScript, err)
}

nodeIPScript := filepath.Join(ctx.CombustionDir, setNodeIPScript)
if err = os.WriteFile(nodeIPScript, []byte(data), fileio.ExecutablePerms); err != nil {
return fmt.Errorf("writing set node IP script: %w", err)
return "", fmt.Errorf("writing set node IP script: %w", err)
}

return nil
return setNodeIPScript, nil
}

func storeKubernetesClusterConfig(cluster *kubernetes.Cluster, destPath string) error {
Expand Down Expand Up @@ -478,25 +482,25 @@ func (c *Combustion) configureManifests(ctx *image.Context) (string, error) {
}

func KubernetesConfigPath(ctx *image.Context) string {
return filepath.Join(ctx.ImageConfigDir, K8sDir, K8sConfigDir, k8sServerConfigFile)
return filepath.Join(ctx.ImageConfigDir, k8sDir, k8sConfigDir, k8sServerConfigFile)
}

func localKubernetesManifestsPath() string {
return filepath.Join(K8sDir, k8sManifestsDir)
return filepath.Join(k8sDir, k8sManifestsDir)
}

func KubernetesManifestsPath(ctx *image.Context) string {
return filepath.Join(ctx.ImageConfigDir, localKubernetesManifestsPath())
}

func HelmValuesPath(ctx *image.Context) string {
return filepath.Join(ctx.ImageConfigDir, K8sDir, helmDir, helmValuesDir)
return filepath.Join(ctx.ImageConfigDir, k8sDir, helmDir, helmValuesDir)
}

func HelmCertsPath(ctx *image.Context) string {
return filepath.Join(ctx.ImageConfigDir, K8sDir, helmDir, helmCertsDir)
return filepath.Join(ctx.ImageConfigDir, k8sDir, helmDir, helmCertsDir)
}

func kubernetesArtefactsPath(ctx *image.Context) string {
return filepath.Join(ctx.ArtefactsDir, K8sDir)
return filepath.Join(ctx.ArtefactsDir, k8sDir)
}
64 changes: 44 additions & 20 deletions pkg/combustion/kubernetes_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -479,7 +479,7 @@ func TestConfigureKubernetes_SuccessfulMultiNodeK3sClusterIPv4(t *testing.T) {
b, err := yaml.Marshal(serverConfig)
require.NoError(t, err)

configDir := filepath.Join(ctx.ImageConfigDir, K8sDir, K8sConfigDir)
configDir := filepath.Join(ctx.ImageConfigDir, k8sDir, k8sConfigDir)
require.NoError(t, os.MkdirAll(configDir, os.ModePerm))
require.NoError(t, os.WriteFile(filepath.Join(configDir, "server.yaml"), b, os.ModePerm))

Expand Down Expand Up @@ -611,7 +611,7 @@ func TestConfigureKubernetes_SuccessfulMultiNodeK3sClusterIPv6(t *testing.T) {
b, err := yaml.Marshal(serverConfig)
require.NoError(t, err)

configDir := filepath.Join(ctx.ImageConfigDir, K8sDir, K8sConfigDir)
configDir := filepath.Join(ctx.ImageConfigDir, k8sDir, k8sConfigDir)
require.NoError(t, os.MkdirAll(configDir, os.ModePerm))
require.NoError(t, os.WriteFile(filepath.Join(configDir, "server.yaml"), b, os.ModePerm))

Expand Down Expand Up @@ -746,7 +746,7 @@ func TestConfigureKubernetes_SuccessfulMultiNodeK3sClusterDualstackPrioIPv4(t *t
b, err := yaml.Marshal(serverConfig)
require.NoError(t, err)

configDir := filepath.Join(ctx.ImageConfigDir, K8sDir, K8sConfigDir)
configDir := filepath.Join(ctx.ImageConfigDir, k8sDir, k8sConfigDir)
require.NoError(t, os.MkdirAll(configDir, os.ModePerm))
require.NoError(t, os.WriteFile(filepath.Join(configDir, "server.yaml"), b, os.ModePerm))

Expand Down Expand Up @@ -882,7 +882,7 @@ func TestConfigureKubernetes_SuccessfulMultiNodeK3sClusterDualstackPrioIPv6(t *t
b, err := yaml.Marshal(serverConfig)
require.NoError(t, err)

configDir := filepath.Join(ctx.ImageConfigDir, K8sDir, K8sConfigDir)
configDir := filepath.Join(ctx.ImageConfigDir, k8sDir, k8sConfigDir)
require.NoError(t, os.MkdirAll(configDir, os.ModePerm))
require.NoError(t, os.WriteFile(filepath.Join(configDir, "server.yaml"), b, os.ModePerm))

Expand Down Expand Up @@ -1088,7 +1088,7 @@ func TestConfigureKubernetes_SuccessfulMultiNodeRKE2ClusterDualstackPrioIPv6With
b, err := yaml.Marshal(serverConfig)
require.NoError(t, err)

configDir := filepath.Join(ctx.ImageConfigDir, K8sDir, K8sConfigDir)
configDir := filepath.Join(ctx.ImageConfigDir, k8sDir, k8sConfigDir)
require.NoError(t, os.MkdirAll(configDir, os.ModePerm))
require.NoError(t, os.WriteFile(filepath.Join(configDir, "server.yaml"), b, os.ModePerm))

Expand Down Expand Up @@ -1218,7 +1218,7 @@ func TestConfigureKubernetes_SuccessfulMultiNodeRKE2ClusterDualstackPrioIPv6With
b, err := yaml.Marshal(serverConfig)
require.NoError(t, err)

configDir := filepath.Join(ctx.ImageConfigDir, K8sDir, K8sConfigDir)
configDir := filepath.Join(ctx.ImageConfigDir, k8sDir, k8sConfigDir)
require.NoError(t, os.MkdirAll(configDir, os.ModePerm))
require.NoError(t, os.WriteFile(filepath.Join(configDir, "server.yaml"), b, os.ModePerm))

Expand Down Expand Up @@ -1382,7 +1382,7 @@ values: content`, "oci://registry-1.docker.io/bitnamicharts"),

assert.Equal(t, "$ARTEFACTS_DIR/kubernetes/manifests", manifestsPath)

manifestPath := filepath.Join(ctx.ArtefactsDir, K8sDir, k8sManifestsDir, "sample-crd.yaml")
manifestPath := filepath.Join(ctx.ArtefactsDir, k8sDir, k8sManifestsDir, "sample-crd.yaml")

b, err := os.ReadFile(manifestPath)
require.NoError(t, err)
Expand All @@ -1393,7 +1393,7 @@ values: content`, "oci://registry-1.docker.io/bitnamicharts"),
assert.Contains(t, contents, "name: my-nginx")
assert.Contains(t, contents, "image: nginx:1.14.2")

chartPath := filepath.Join(ctx.ArtefactsDir, K8sDir, k8sManifestsDir, "apache.yaml")
chartPath := filepath.Join(ctx.ArtefactsDir, k8sDir, k8sManifestsDir, "apache.yaml")
chartContent := `apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
Expand Down Expand Up @@ -1497,7 +1497,7 @@ func TestConfigureKubernetes_SuccessfulRKE2ServerWithManifests(t *testing.T) {
assert.Equal(t, []any{"192.168.122.100", "api.cluster01.hosted.on.edge.suse.com"}, configContents["tls-san"])

// Manifest assertions
manifest := filepath.Join(ctx.ArtefactsDir, K8sDir, k8sManifestsDir, "sample-crd.yaml")
manifest := filepath.Join(ctx.ArtefactsDir, k8sDir, k8sManifestsDir, "sample-crd.yaml")
info, err = os.Stat(manifest)
require.NoError(t, err)
assert.Equal(t, fileio.NonExecutablePerms, info.Mode())
Expand Down Expand Up @@ -1583,11 +1583,12 @@ func TestCreateNodeIPScriptDualstackIPv6PrioK3s(t *testing.T) {
"service-cidr": "fd12:3456:789c::/112,10.43.0.0/16",
}

err := createNodeIPScript(ctx, serverConfig)
nodeIPScript, err := createNodeIPScript(ctx, serverConfig)
require.NoError(t, err)
assert.Contains(t, nodeIPScript, "set-node-ip.sh")

nodeIPScript := filepath.Join(ctx.CombustionDir, setNodeIPScript)
b, err := os.ReadFile(nodeIPScript)
nodeIPScriptPath := filepath.Join(ctx.CombustionDir, setNodeIPScript)
b, err := os.ReadFile(nodeIPScriptPath)
require.NoError(t, err)

contents := string(b)
Expand Down Expand Up @@ -1615,11 +1616,12 @@ func TestCreateNodeIPScriptDualstackIPv4PrioRke2(t *testing.T) {
"service-cidr": "10.43.0.0/16,fd12:3456:789c::/112",
}

err := createNodeIPScript(ctx, serverConfig)
nodeIPScript, err := createNodeIPScript(ctx, serverConfig)
require.NoError(t, err)
assert.Contains(t, nodeIPScript, "set-node-ip.sh")

nodeIPScript := filepath.Join(ctx.CombustionDir, setNodeIPScript)
b, err := os.ReadFile(nodeIPScript)
nodeIPScriptPath := filepath.Join(ctx.CombustionDir, setNodeIPScript)
b, err := os.ReadFile(nodeIPScriptPath)
require.NoError(t, err)

contents := string(b)
Expand All @@ -1643,8 +1645,9 @@ func TestCreateNodeIPScriptIPv4Only(t *testing.T) {

serverConfig := map[string]any{}

err := createNodeIPScript(ctx, serverConfig)
require.Nil(t, err)
nodeIPScript, err := createNodeIPScript(ctx, serverConfig)
require.NoError(t, err)
assert.Contains(t, nodeIPScript, "")
}

func TestCreateNodeIPScriptIPv6Only(t *testing.T) {
Expand All @@ -1663,15 +1666,36 @@ func TestCreateNodeIPScriptIPv6Only(t *testing.T) {
"service-cidr": "fd12:3456:789c::/112",
}

err := createNodeIPScript(ctx, serverConfig)
nodeIPScript, err := createNodeIPScript(ctx, serverConfig)
require.NoError(t, err)
assert.Contains(t, nodeIPScript, "set-node-ip.sh")

nodeIPScript := filepath.Join(ctx.CombustionDir, setNodeIPScript)
b, err := os.ReadFile(nodeIPScript)
nodeIPScriptPath := filepath.Join(ctx.CombustionDir, setNodeIPScript)
b, err := os.ReadFile(nodeIPScriptPath)
require.NoError(t, err)

contents := string(b)

assert.Contains(t, contents, "IPv6=true")
assert.Contains(t, contents, "CONFIG_FILE=\"/etc/rancher/rke2/config.yaml\"")
}

func TestCreateNodeIPScriptNodeIPSpecified(t *testing.T) {
ctx, teardown := setupContext(t)
defer teardown()

ctx.ImageDefinition.Kubernetes = image.Kubernetes{
Version: "v1.30.3+rke2r1",
Network: image.Network{
APIVIP4: "192.168.1.1",
},
}

serverConfig := map[string]any{
"node-ip": "192.168.100.100",
}

nodeIPScript, err := createNodeIPScript(ctx, serverConfig)
require.NoError(t, err)
assert.Contains(t, nodeIPScript, "")
}
2 changes: 1 addition & 1 deletion pkg/combustion/registry_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ func TestWriteRegistryMirrorsValid(t *testing.T) {
// Verify
require.NoError(t, err)

manifestFileName := filepath.Join(ctx.ArtefactsDir, K8sDir, registryMirrorsFileName)
manifestFileName := filepath.Join(ctx.ArtefactsDir, k8sDir, registryMirrorsFileName)

foundBytes, err := os.ReadFile(manifestFileName)
require.NoError(t, err)
Expand Down
2 changes: 1 addition & 1 deletion pkg/combustion/templates/k3s-multi-node-installer.sh.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ mkdir -p /etc/rancher/k3s/
cp $CONFIGFILE /etc/rancher/k3s/config.yaml

if [ "$NODETYPE" = "server" ]; then
{{- if and .apiVIP6 .getNodeIP}}
{{- if .setNodeIPScript }}
sh {{ .setNodeIPScript }}
{{- end }}
fi
Expand Down
2 changes: 1 addition & 1 deletion pkg/combustion/templates/k3s-single-node-installer.sh.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ echo "{{ .apiVIP6 }} {{ .apiHost }}" >> /etc/hosts
mkdir -p /etc/rancher/k3s/
cp {{ .configFilePath }}/{{ .configFile }} /etc/rancher/k3s/config.yaml

{{- if and .apiVIP6 .getNodeIP}}
{{- if .setNodeIPScript }}
sh {{ .setNodeIPScript }}
{{- end }}

Expand Down
2 changes: 1 addition & 1 deletion pkg/combustion/templates/rke2-multi-node-installer.sh.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ mkdir -p /etc/rancher/rke2/
cp $CONFIGFILE /etc/rancher/rke2/config.yaml

if [ "$NODETYPE" = "server" ]; then
{{- if and .apiVIP6 .getNodeIP}}
{{- if .setNodeIPScript }}
sh {{ .setNodeIPScript }}
{{- end }}
fi
Expand Down
2 changes: 1 addition & 1 deletion pkg/combustion/templates/rke2-single-node-installer.sh.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ echo "{{ .apiVIP6 }} {{ .apiHost }}" >> /etc/hosts
mkdir -p /etc/rancher/rke2/
cp {{ .configFilePath }}/{{ .configFile }} /etc/rancher/rke2/config.yaml

{{- if and .apiVIP6 .getNodeIP}}
{{- if .setNodeIPScript }}
sh {{ .setNodeIPScript }}
{{- end }}

Expand Down
Loading

0 comments on commit c284c6e

Please sign in to comment.