cilium: remove appArmorProfile for k8s<v1.30.0

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/hashicorp/go-getter v1.7.6
 	github.com/hashicorp/go-retryablehttp v0.7.7
 	github.com/hooklift/iso9660 v1.0.0
+	github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0
 	github.com/jmoiron/sqlx v1.4.0
 	github.com/johanneswuerbach/nfsexports v0.0.0-20200318065542-c48c3734757f
 	github.com/juju/clock v1.1.1

go.sum

@@ -1032,6 +1032,8 @@ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
+github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0 h1:nHoRIX8iXob3Y2kdt9KsjyIb7iApSvb3vgsd93xb5Ow=
+github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0/go.mod h1:c1tRKs5Tx7E2+uHGSyyncziFjvGpgv4H2HrqXeUQ/Uk=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=

pkg/minikube/cni/cilium.go

@@ -19,21 +19,24 @@ package cni
 import (
 	"bytes"
 	_ "embed"
+	"fmt"
 	"os/exec"
 	"text/template"
 
+	"github.com/blang/semver/v4"
+	"github.com/icza/dyno"
 	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
 	"k8s.io/klog/v2"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/util"
 )
 
 // Generated by running `make update-cilium-version`
 //
 //go:embed cilium.yaml
 var ciliumYaml string
 
-var ciliumTmpl = template.Must(template.New("name").Parse(ciliumYaml))
-
 // Cilium is the Cilium CNI manager
 type Cilium struct {
 	cc config.ClusterConfig
@@ -50,7 +53,15 @@ func (c Cilium) CIDR() string {
 }
 
 // GenerateCiliumYAML generates the .yaml file
-func GenerateCiliumYAML() ([]byte, error) {
+func (c Cilium) GenerateCiliumYAML() ([]byte, error) {
+
+	// see issue #19683, older Kubernetes versions cannot recognize appArmorProfile fields
+	k8sVersion, err := util.ParseKubernetesVersion(c.cc.KubernetesConfig.KubernetesVersion)
+	if err == nil && k8sVersion.LT(semver.MustParse("1.30.0")) {
+		if ciliumYaml, err = removeAppArorProfile(ciliumYaml); err != nil {
+			return nil, err
+		}
+	}
 
 	podCIDR := DefaultPodCIDR
 
@@ -61,7 +72,7 @@ func GenerateCiliumYAML() ([]byte, error) {
 	}{
 		PodSubnet: podCIDR,
 	}
-
+	ciliumTmpl := template.Must(template.New("name").Parse(ciliumYaml))
 	b := bytes.Buffer{}
 	configTmpl := ciliumTmpl
 
@@ -80,10 +91,35 @@ func (c Cilium) Apply(r Runner) error {
 		return errors.Wrap(err, "bpf mount")
 	}
 
-	ciliumCfg, err := GenerateCiliumYAML()
+	ciliumCfg, err := c.GenerateCiliumYAML()
 	if err != nil {
 		return errors.Wrap(err, "generating cilium cfg")
 	}
 
 	return applyManifest(c.cc, r, manifestAsset(ciliumCfg))
 }
+
+func removeAppArorProfile(ciliumConfig string) (string, error) {
+	// remove all appArmorProfile fields
+	decoder := yaml.NewDecoder(bytes.NewBufferString(ciliumConfig))
+	var buffer bytes.Buffer
+	encoder := yaml.NewEncoder(&buffer)
+	for {
+		obj := map[string]interface{}{}
+		err := decoder.Decode(&obj)
+		if err != nil && err.Error() == "EOF" {
+			// we have unmarshaled all objects
+			break
+		} else if err != nil {
+			return "", fmt.Errorf("failed to unmarshal yaml: %v", err)
+		}
+		if err := dyno.Delete(obj, "appArmorProfile", "spec", "template", "spec", "securityContext"); err != nil {
+			return "", fmt.Errorf("failed to remove securityContext yaml: %v", err)
+		}
+		if err := encoder.Encode(obj); err != nil {
+			return "", fmt.Errorf("failed to encode yaml")
+		}
+
+	}
+	return buffer.String(), nil
+}