From 6e61e68c817810e5c1c8f38d6b84110256bd4d76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=94=A6=E5=8D=97=E8=B7=AF=E4=B9=8B=E8=8A=B1?=
 <tangjiaming1999@gmail.com>
Date: Wed, 30 Oct 2024 23:18:57 +0100
Subject: [PATCH 1/2] cilium: remove appArmorProfile for k8s<v1.30.0

---
 go.mod                     |  1 +
 go.sum                     |  2 ++
 pkg/minikube/cni/cilium.go | 46 +++++++++++++++++++++++++++++++++-----
 3 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/go.mod b/go.mod
index 41926fa35844..a296c8b58c51 100644
--- a/go.mod
+++ b/go.mod
@@ -34,6 +34,7 @@ require (
 	github.com/hashicorp/go-getter v1.7.6
 	github.com/hashicorp/go-retryablehttp v0.7.7
 	github.com/hooklift/iso9660 v1.0.0
+	github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0
 	github.com/jmoiron/sqlx v1.4.0
 	github.com/johanneswuerbach/nfsexports v0.0.0-20200318065542-c48c3734757f
 	github.com/juju/clock v1.1.1
diff --git a/go.sum b/go.sum
index ca9ec78f8e87..ba0ca2c1838d 100644
--- a/go.sum
+++ b/go.sum
@@ -1032,6 +1032,8 @@ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
+github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0 h1:nHoRIX8iXob3Y2kdt9KsjyIb7iApSvb3vgsd93xb5Ow=
+github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0/go.mod h1:c1tRKs5Tx7E2+uHGSyyncziFjvGpgv4H2HrqXeUQ/Uk=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
diff --git a/pkg/minikube/cni/cilium.go b/pkg/minikube/cni/cilium.go
index c0f635b6ca0c..02c97da0de61 100644
--- a/pkg/minikube/cni/cilium.go
+++ b/pkg/minikube/cni/cilium.go
@@ -19,12 +19,17 @@ package cni
 import (
 	"bytes"
 	_ "embed"
+	"fmt"
 	"os/exec"
 	"text/template"
 
+	"github.com/blang/semver/v4"
+	"github.com/icza/dyno"
 	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
 	"k8s.io/klog/v2"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/util"
 )
 
 // Generated by running `make update-cilium-version`
@@ -32,8 +37,6 @@ import (
 //go:embed cilium.yaml
 var ciliumYaml string
 
-var ciliumTmpl = template.Must(template.New("name").Parse(ciliumYaml))
-
 // Cilium is the Cilium CNI manager
 type Cilium struct {
 	cc config.ClusterConfig
@@ -50,7 +53,15 @@ func (c Cilium) CIDR() string {
 }
 
 // GenerateCiliumYAML generates the .yaml file
-func GenerateCiliumYAML() ([]byte, error) {
+func (c Cilium) GenerateCiliumYAML() ([]byte, error) {
+
+	// see issue #19683, older Kubernetes versions cannot recognize appArmorProfile fields
+	k8sVersion, err := util.ParseKubernetesVersion(c.cc.KubernetesConfig.KubernetesVersion)
+	if err == nil && k8sVersion.LT(semver.MustParse("1.30.0")) {
+		if ciliumYaml, err = removeAppArorProfile(ciliumYaml); err != nil {
+			return nil, err
+		}
+	}
 
 	podCIDR := DefaultPodCIDR
 
@@ -61,7 +72,7 @@ func GenerateCiliumYAML() ([]byte, error) {
 	}{
 		PodSubnet: podCIDR,
 	}
-
+	ciliumTmpl := template.Must(template.New("name").Parse(ciliumYaml))
 	b := bytes.Buffer{}
 	configTmpl := ciliumTmpl
 
@@ -80,10 +91,35 @@ func (c Cilium) Apply(r Runner) error {
 		return errors.Wrap(err, "bpf mount")
 	}
 
-	ciliumCfg, err := GenerateCiliumYAML()
+	ciliumCfg, err := c.GenerateCiliumYAML()
 	if err != nil {
 		return errors.Wrap(err, "generating cilium cfg")
 	}
 
 	return applyManifest(c.cc, r, manifestAsset(ciliumCfg))
 }
+
+func removeAppArorProfile(ciliumConfig string) (string, error) {
+	// remove all appArmorProfile fields
+	decoder := yaml.NewDecoder(bytes.NewBufferString(ciliumConfig))
+	var buffer bytes.Buffer
+	encoder := yaml.NewEncoder(&buffer)
+	for {
+		obj := map[string]interface{}{}
+		err := decoder.Decode(&obj)
+		if err != nil && err.Error() == "EOF" {
+			// we have unmarshaled all objects
+			break
+		} else if err != nil {
+			return "", fmt.Errorf("failed to unmarshal yaml: %v", err)
+		}
+		if err := dyno.Delete(obj, "appArmorProfile", "spec", "template", "spec", "securityContext"); err != nil {
+			return "", fmt.Errorf("failed to remove securityContext yaml: %v", err)
+		}
+		if err := encoder.Encode(obj); err != nil {
+			return "", fmt.Errorf("failed to encode yaml")
+		}
+
+	}
+	return buffer.String(), nil
+}

From b240f9d77986126e9714444475c34e6cc49a474f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=8C=A6=E5=8D=97=E8=B7=AF=E4=B9=8B=E8=8A=B1?=
 <46831212+ComradeProgrammer@users.noreply.github.com>
Date: Mon, 9 Dec 2024 22:54:14 +0100
Subject: [PATCH 2/2] Apply suggestions from code review

Co-authored-by: Steven Powell <44844360+spowelljr@users.noreply.github.com>
---
 pkg/minikube/cni/cilium.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/pkg/minikube/cni/cilium.go b/pkg/minikube/cni/cilium.go
index 02c97da0de61..f02728f6b902 100644
--- a/pkg/minikube/cni/cilium.go
+++ b/pkg/minikube/cni/cilium.go
@@ -20,6 +20,7 @@ import (
 	"bytes"
 	_ "embed"
 	"fmt"
+	"io"
 	"os/exec"
 	"text/template"
 
@@ -58,7 +59,7 @@ func (c Cilium) GenerateCiliumYAML() ([]byte, error) {
 	// see issue #19683, older Kubernetes versions cannot recognize appArmorProfile fields
 	k8sVersion, err := util.ParseKubernetesVersion(c.cc.KubernetesConfig.KubernetesVersion)
 	if err == nil && k8sVersion.LT(semver.MustParse("1.30.0")) {
-		if ciliumYaml, err = removeAppArorProfile(ciliumYaml); err != nil {
+		if ciliumYaml, err = removeAppArmorProfile(ciliumYaml); err != nil {
 			return nil, err
 		}
 	}
@@ -99,7 +100,7 @@ func (c Cilium) Apply(r Runner) error {
 	return applyManifest(c.cc, r, manifestAsset(ciliumCfg))
 }
 
-func removeAppArorProfile(ciliumConfig string) (string, error) {
+func removeAppArmorProfile(ciliumConfig string) (string, error) {
 	// remove all appArmorProfile fields
 	decoder := yaml.NewDecoder(bytes.NewBufferString(ciliumConfig))
 	var buffer bytes.Buffer
@@ -107,7 +108,7 @@ func removeAppArorProfile(ciliumConfig string) (string, error) {
 	for {
 		obj := map[string]interface{}{}
 		err := decoder.Decode(&obj)
-		if err != nil && err.Error() == "EOF" {
+		if err == io.EOF {
 			// we have unmarshaled all objects
 			break
 		} else if err != nil {