Merge pull request #15927 from d4l3k/master

container-runtime: add nvidia-docker
kubernetes · Sep 20, 2023 · 075f1a1 · 075f1a1
2 parents e68882d + 2a1f5b9
commit 075f1a1
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 16 deletions.
diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go
@@ -50,7 +50,7 @@ func (cs ContainerState) String() string {
 
 // ValidRuntimes lists the supported container runtimes
 func ValidRuntimes() []string {
-	return []string{"docker", "cri-o", "containerd"}
+	return []string{"docker", "nvidia-docker", "cri-o", "containerd"}
 }
 
 // CommandRunner is the subset of command.Runner this package consumes
@@ -210,7 +210,7 @@ func New(c Config) (Manager, error) {
 	sm := sysinit.New(c.Runner)
 
 	switch c.Type {
-	case "", "docker":
+	case "", "docker", "nvidia-docker":
 		sp := c.Socket
 		cs := ""
 		// There is no more dockershim socket, in Kubernetes version 1.24 and beyond
@@ -219,6 +219,7 @@ func New(c Config) (Manager, error) {
 			cs = "cri-docker.socket"
 		}
 		return &Docker{
+			Type:              c.Type,
 			Socket:            sp,
 			Runner:            c.Runner,
 			NetworkPlugin:     c.NetworkPlugin,

diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go
@@ -40,6 +40,7 @@ func TestName(t *testing.T) {
 	}{
 		{"", "Docker"},
 		{"docker", "Docker"},
+		{"nvidia-docker", "Docker"},
 		{"crio", "CRI-O"},
 		{"cri-o", "CRI-O"},
 		{"containerd", "containerd"},
@@ -124,6 +125,7 @@ func TestCGroupDriver(t *testing.T) {
 		want    string
 	}{
 		{"docker", "cgroupfs"},
+		{"nvidia-docker", "cgroupfs"},
 		{"crio", "cgroupfs"},
 		{"containerd", "cgroupfs"},
 	}
@@ -155,6 +157,12 @@ func TestKubeletOptions(t *testing.T) {
 		{"docker", "1.24.0", map[string]string{
 			"container-runtime-endpoint": "unix:///var/run/cri-dockerd.sock",
 		}},
+		{"nvidia-docker", "1.23.0", map[string]string{
+			"container-runtime": "docker",
+		}},
+		{"nvidia-docker", "1.25.0", map[string]string{
+			"container-runtime-endpoint": "unix:///var/run/cri-dockerd.sock",
+		}},
 		{"crio", "1.25.0", map[string]string{
 			"container-runtime-endpoint": "unix:///var/run/crio/crio.sock",
 		}},
@@ -680,6 +688,13 @@ func TestEnable(t *testing.T) {
 				"crio":          SvcExited,
 				"crio-shutdown": SvcExited,
 			}},
+		{"nvidia-docker", defaultServices,
+			map[string]serviceState{
+				"docker":        SvcRestarted,
+				"containerd":    SvcExited,
+				"crio":          SvcExited,
+				"crio-shutdown": SvcExited,
+			}},
 		{"containerd", defaultServices,
 			map[string]serviceState{
 				"docker":        SvcExited,
@@ -721,6 +736,7 @@ func TestContainerFunctions(t *testing.T) {
 		runtime string
 	}{
 		{"docker"},
+		{"nvidia-docker"},
 		{"crio"},
 		{"containerd"},
 	}
@@ -730,7 +746,7 @@ func TestContainerFunctions(t *testing.T) {
 		t.Run(tc.runtime, func(t *testing.T) {
 			runner := NewFakeRunner(t)
 			prefix := ""
-			if tc.runtime == "docker" {
+			if tc.runtime == "docker" || tc.runtime == "nvidia-docker" {
 				prefix = "k8s_"
 			}
 			runner.containers = map[string]string{

diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
@@ -67,6 +67,7 @@ func (e *ErrISOFeature) Error() string {
 
 // Docker contains Docker runtime state
 type Docker struct {
+	Type              string
 	Socket            string
 	Runner            CommandRunner
 	NetworkPlugin     string
@@ -156,7 +157,7 @@ func (r *Docker) Enable(disOthers bool, cgroupDriver string, inUserNamespace boo
 		klog.ErrorS(err, "Failed to enable", "service", "docker.socket")
 	}
 
-	if err := r.setCGroup(cgroupDriver); err != nil {
+	if err := r.configureDocker(cgroupDriver); err != nil {
 		return err
 	}
 
@@ -525,24 +526,51 @@ func (r *Docker) SystemLogCmd(len int) string {
 	return fmt.Sprintf("sudo journalctl -u docker -u cri-docker -n %d", len)
 }
 
-// setCGroup configures the docker daemon to use driver as cgroup manager
+type dockerDaemonConfig struct {
+	ExecOpts       []string              `json:"exec-opts"`
+	LogDriver      string                `json:"log-driver"`
+	LogOpts        dockerDaemonLogOpts   `json:"log-opts"`
+	StorageDriver  string                `json:"storage-driver"`
+	DefaultRuntime string                `json:"default-runtime,omitempty"`
+	Runtimes       *dockerDaemonRuntimes `json:"runtimes,omitempty"`
+}
+type dockerDaemonLogOpts struct {
+	MaxSize string `json:"max-size"`
+}
+type dockerDaemonRuntimes struct {
+	Nvidia struct {
+		Path        string        `json:"path"`
+		RuntimeArgs []interface{} `json:"runtimeArgs"`
+	} `json:"nvidia"`
+}
+
+// configureDocker configures the docker daemon to use driver as cgroup manager
 // ref: https://docs.docker.com/engine/reference/commandline/dockerd/#options-for-the-runtime
-func (r *Docker) setCGroup(driver string) error {
+func (r *Docker) configureDocker(driver string) error {
 	if driver == constants.UnknownCgroupDriver {
 		return fmt.Errorf("unable to configure docker to use unknown cgroup driver")
 	}
 
 	klog.Infof("configuring docker to use %q as cgroup driver...", driver)
-	daemonConfig := fmt.Sprintf(`{
-"exec-opts": ["native.cgroupdriver=%s"],
-"log-driver": "json-file",
-"log-opts": {
-	"max-size": "100m"
-},
-"storage-driver": "overlay2"
-}
-`, driver)
-	ma := assets.NewMemoryAsset([]byte(daemonConfig), "/etc/docker", "daemon.json", "0644")
+	daemonConfig := dockerDaemonConfig{
+		ExecOpts:  []string{"native.cgroupdriver=" + driver},
+		LogDriver: "json-file",
+		LogOpts: dockerDaemonLogOpts{
+			MaxSize: "100m",
+		},
+		StorageDriver: "overlay2",
+	}
+	if r.Type == "nvidia-docker" {
+		daemonConfig.DefaultRuntime = "nvidia"
+		runtimes := &dockerDaemonRuntimes{}
+		runtimes.Nvidia.Path = "/usr/bin/nvidia-container-runtime"
+		daemonConfig.Runtimes = runtimes
+	}
+	daemonConfigBytes, err := json.Marshal(daemonConfig)
+	if err != nil {
+		return err
+	}
+	ma := assets.NewMemoryAsset(daemonConfigBytes, "/etc/docker", "daemon.json", "0644")
 	return r.Runner.Copy(ma)
 }