Add credential provider option to kubeadm (#533)

* Add credential provider option to kubeadm

* add unit test

* fix string manipulation

cluster/kubeadm/kubeadm.go

@@ -0,0 +1,50 @@
+package kubeadm
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/openconfig/kne/exec/run"
+	log "k8s.io/klog/v2"
+)
+
+var (
+	kubeadmFlagPath = "/var/lib/kubelet/kubeadm-flags.env"
+)
+
+// EnableCredentialProvider enables a credential provider according
+// to the specified config file on the kubelet.
+func EnableCredentialProvider(cfgPath string) error {
+	log.Infof("Enabling credential provider with config %q...", cfgPath)
+	if _, err := os.Stat(cfgPath); err != nil {
+		return fmt.Errorf("config file not found: %v", err)
+	}
+	if err := run.LogCommand("sudo", "kubeadm", "upgrade", "node", "phase", "kubelet-config"); err != nil {
+		return err
+	}
+	b, err := os.ReadFile(kubeadmFlagPath)
+	if err != nil {
+		return fmt.Errorf("failed to read kubeadm flag file: %v", err)
+	}
+	s, ok := strings.CutSuffix(string(b), "\"\n")
+	if !ok {
+		return fmt.Errorf("kubeadm flag file %q does not have expected contents: %q", kubeadmFlagPath, s)
+	}
+	s = fmt.Sprintf("%s --image-credential-provider-config=%s --image-credential-provider-bin-dir=/etc/kubernetes/bin\"\n", s, cfgPath)
+	f, err := os.CreateTemp("", "kne-kubeadm-flag.env")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(f.Name())
+	if _, err := f.WriteString(s); err != nil {
+		return err
+	}
+	if err := run.LogCommand("sudo", "cp", f.Name(), kubeadmFlagPath); err != nil {
+		return err
+	}
+	if err := run.LogCommand("sudo", "systemctl", "restart", "kubelet"); err != nil {
+		return err
+	}
+	return nil
+}

cluster/kubeadm/kubeadm_test.go

@@ -0,0 +1,96 @@
+package kubeadm
+
+import (
+	"os"
+	"testing"
+
+	"github.com/openconfig/gnmi/errdiff"
+	kexec "github.com/openconfig/kne/exec"
+	fexec "github.com/openconfig/kne/exec/fake"
+)
+
+func TestEnableCredentialProvider(t *testing.T) {
+	f, err := os.CreateTemp(t.TempDir(), "flags.env")
+	if err != nil {
+		t.Fatalf("Failed to create temp file for test: %v", err)
+	}
+	if _, err := f.WriteString("KUBELET_KUBEADM_ARGS=\"--container-runtime-endpoint=unix:///var/run/cri-dockerd.sock\"\n"); err != nil {
+		t.Fatalf("Failed to write temp file for test: %v", err)
+	}
+
+	origKubeadmFlagPath := kubeadmFlagPath
+	defer func() {
+		kubeadmFlagPath = origKubeadmFlagPath
+	}()
+	kubeadmFlagPath = f.Name()
+
+	cfg, err := os.CreateTemp(t.TempDir(), "cfg.yaml")
+	if err != nil {
+		t.Fatalf("Failed to create temp cfg file for test: %v", err)
+	}
+
+	tests := []struct {
+		desc    string
+		cfgPath string
+		resp    []fexec.Response
+		wantErr string
+	}{{
+		desc:    "success",
+		cfgPath: cfg.Name(),
+		resp: []fexec.Response{
+			{Cmd: "sudo", Args: []string{"kubeadm", "upgrade", "node", "phase", "kubelet-config"}},
+			{Cmd: "sudo", Args: []string{"cp", ".*", kubeadmFlagPath}},
+			{Cmd: "sudo", Args: []string{"systemctl", "restart", "kubelet"}},
+		},
+	}, {
+		desc:    "config file not found",
+		cfgPath: "dne",
+		wantErr: "config file not found",
+	}, {
+		desc:    "failed to upgrade kubelet",
+		cfgPath: cfg.Name(),
+		resp: []fexec.Response{
+			{Cmd: "sudo", Args: []string{"kubeadm", "upgrade", "node", "phase", "kubelet-config"}, Err: "failed to upgrade kubelet"},
+		},
+		wantErr: "failed to upgrade kubelet",
+	}, {
+		desc:    "failed to copy flag config",
+		cfgPath: cfg.Name(),
+		resp: []fexec.Response{
+			{Cmd: "sudo", Args: []string{"kubeadm", "upgrade", "node", "phase", "kubelet-config"}},
+			{Cmd: "sudo", Args: []string{"cp", ".*", kubeadmFlagPath}, Err: "failed to copy"},
+		},
+		wantErr: "failed to copy",
+	}, {
+		desc:    "failed to restart kubelet",
+		cfgPath: cfg.Name(),
+		resp: []fexec.Response{
+			{Cmd: "sudo", Args: []string{"kubeadm", "upgrade", "node", "phase", "kubelet-config"}},
+			{Cmd: "sudo", Args: []string{"cp", ".*", kubeadmFlagPath}},
+			{Cmd: "sudo", Args: []string{"systemctl", "restart", "kubelet"}, Err: "failed to restart kubelet"},
+		},
+		wantErr: "failed to restart kubelet",
+	}}
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			fexec.LogCommand = func(s string) {
+				t.Logf("%s: %s", tt.desc, s)
+			}
+			cmds := fexec.Commands(tt.resp)
+			kexec.Command = cmds.Command
+			defer checkCmds(t, cmds)
+
+			err := EnableCredentialProvider(tt.cfgPath)
+			if s := errdiff.Substring(err, tt.wantErr); s != "" {
+				t.Fatalf("unexpected error: %s", s)
+			}
+		})
+	}
+}
+
+func checkCmds(t *testing.T, cmds *fexec.Command) {
+	t.Helper()
+	if err := cmds.Done(); err != nil {
+		t.Errorf("%v", err)
+	}
+}

controller/server/main.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	log "github.com/golang/glog"
+	"github.com/openconfig/kne/cluster/kubeadm"
 	"github.com/openconfig/kne/deploy"
 	"github.com/openconfig/kne/exec/run"
 	cpb "github.com/openconfig/kne/proto/controller"
@@ -129,6 +130,7 @@ func newDeployment(req *cpb.CreateClusterRequest) (*deploy.Deployment, error) {
 			TokenTTL:                    req.GetKubeadm().TokenTtl,
 			Network:                     req.GetKubeadm().Network,
 			AllowControlPlaneScheduling: req.GetKubeadm().AllowControlPlaneScheduling,
+			CredentialProviderConfig:    req.GetKubeadm().CredentialProviderConfig,
 		}
 		switch t := req.GetKubeadm().GetPodNetworkAddOnManifest().GetManifestData().(type) {
 		case *cpb.Manifest_Data:
@@ -146,6 +148,13 @@ func newDeployment(req *cpb.CreateClusterRequest) (*deploy.Deployment, error) {
 		default:
 			return nil, fmt.Errorf("manifest data type not supported: %T", t)
 		}
+		if k.CredentialProviderConfig != "" {
+			p, err := validatePath(k.CredentialProviderConfig)
+			if err != nil {
+				return nil, fmt.Errorf("failed to validate path %q", p)
+			}
+			k.CredentialProviderConfig = p
+		}
 		d.Cluster = k
 	case *cpb.CreateClusterRequest_External:
 		d.Cluster = &deploy.ExternalSpec{
@@ -588,6 +597,11 @@ func (s *server) JoinCluster(ctx context.Context, req *cpb.JoinClusterRequest) (
 	if err := run.LogCommand("sudo", args...); err != nil {
 		return nil, status.Errorf(codes.Internal, "failed to join kubeadm cluster: %v", err)
 	}
+	if req.GetCredentialProviderConfig() != "" {
+		if err := kubeadm.EnableCredentialProvider(req.GetCredentialProviderConfig()); err != nil {
+			return nil, err
+		}
+	}
 	return &cpb.JoinClusterResponse{}, nil
 }
 

deploy/deploy.go

@@ -19,6 +19,7 @@ import (
 	"github.com/openconfig/gnmi/errlist"
 	metallbclientv1 "github.com/openconfig/kne/api/metallb/clientset/v1beta1"
 	"github.com/openconfig/kne/cluster/kind"
+	"github.com/openconfig/kne/cluster/kubeadm"
 	"github.com/openconfig/kne/events"
 	"github.com/openconfig/kne/exec/run"
 	"github.com/openconfig/kne/load"
@@ -420,6 +421,7 @@ type KubeadmSpec struct {
 	PodNetworkCIDR              string `yaml:"podNetworkCIDR"`
 	PodNetworkAddOnManifest     string `yaml:"podNetworkAddOnManifest" kne:"yaml"`
 	PodNetworkAddOnManifestData []byte
+	CredentialProviderConfig    string `yaml:"credentialProviderConfig" kne:"yaml"`
 	TokenTTL                    string `yaml:"tokenTTL"`
 	Network                     string `yaml:"network"`
 	AllowControlPlaneScheduling bool   `yaml:"allowControlPlaneScheduling"`
@@ -493,7 +495,12 @@ func (k *KubeadmSpec) Deploy(ctx context.Context) error {
 			return err
 		}
 	}
-
+	// If credential provider config provided, apply it.
+	if k.CredentialProviderConfig != "" {
+		if err := kubeadm.EnableCredentialProvider(k.CredentialProviderConfig); err != nil {
+			return err
+		}
+	}
 	// Create a new docker network if not specified.
 	if k.Network == "" {
 		k.Network = "kne-kubeadm-" + uuid.New()

manifests/kube/credential-provider-config.yaml

@@ -0,0 +1,14 @@
+kind: CredentialProviderConfig
+apiVersion: kubelet.config.k8s.io/v1
+providers:
+- name: auth-provider-gcp
+  apiVersion: credentialprovider.kubelet.k8s.io/v1
+  matchImages:
+  - "container.cloud.google.com"
+  - "gcr.io"
+  - "*.gcr.io"
+  - "*.pkg.dev"
+  args:
+  - get-credentials
+  - --v=3
+  defaultCacheDuration: 1m

proto/controller.proto

@@ -72,6 +72,7 @@ message KubeadmSpec {
   string token_ttl = 4;
   string network = 5; // name of the docker network to use for ingress
   bool allow_control_plane_scheduling = 6;
+  string credential_provider_config = 7;
 }
 
 // External cluster specifications
@@ -286,6 +287,7 @@ message JoinClusterRequest {
   string token = 2;
   string discovery_token_ca_cert_hash = 3;
   string cri_socket = 4;
+  string credential_provider_config = 5;
 }
 
 // Returns join cluster response.