diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 6edd86945fe6..85488a7eea77 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -50,7 +50,7 @@ func (cs ContainerState) String() string { // ValidRuntimes lists the supported container runtimes func ValidRuntimes() []string { - return []string{"docker", "cri-o", "containerd"} + return []string{"docker", "nvidia-docker", "cri-o", "containerd"} } // CommandRunner is the subset of command.Runner this package consumes @@ -210,7 +210,7 @@ func New(c Config) (Manager, error) { sm := sysinit.New(c.Runner) switch c.Type { - case "", "docker": + case "", "docker", "nvidia-docker": sp := c.Socket cs := "" // There is no more dockershim socket, in Kubernetes version 1.24 and beyond @@ -219,6 +219,7 @@ func New(c Config) (Manager, error) { cs = "cri-docker.socket" } return &Docker{ + Type: c.Type, Socket: sp, Runner: c.Runner, NetworkPlugin: c.NetworkPlugin, diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go index 60e0687321d4..ed7e5d6b5561 100644 --- a/pkg/minikube/cruntime/cruntime_test.go +++ b/pkg/minikube/cruntime/cruntime_test.go @@ -40,6 +40,7 @@ func TestName(t *testing.T) { }{ {"", "Docker"}, {"docker", "Docker"}, + {"nvidia-docker", "Docker"}, {"crio", "CRI-O"}, {"cri-o", "CRI-O"}, {"containerd", "containerd"}, @@ -124,6 +125,7 @@ func TestCGroupDriver(t *testing.T) { want string }{ {"docker", "cgroupfs"}, + {"nvidia-docker", "cgroupfs"}, {"crio", "cgroupfs"}, {"containerd", "cgroupfs"}, } @@ -155,6 +157,12 @@ func TestKubeletOptions(t *testing.T) { {"docker", "1.24.0", map[string]string{ "container-runtime-endpoint": "unix:///var/run/cri-dockerd.sock", }}, + {"nvidia-docker", "1.23.0", map[string]string{ + "container-runtime": "docker", + }}, + {"nvidia-docker", "1.25.0", map[string]string{ + "container-runtime-endpoint": "unix:///var/run/cri-dockerd.sock", + }}, {"crio", "1.25.0", map[string]string{ "container-runtime-endpoint": "unix:///var/run/crio/crio.sock", }}, @@ -680,6 +688,13 @@ func TestEnable(t *testing.T) { "crio": SvcExited, "crio-shutdown": SvcExited, }}, + {"nvidia-docker", defaultServices, + map[string]serviceState{ + "docker": SvcRestarted, + "containerd": SvcExited, + "crio": SvcExited, + "crio-shutdown": SvcExited, + }}, {"containerd", defaultServices, map[string]serviceState{ "docker": SvcExited, @@ -721,6 +736,7 @@ func TestContainerFunctions(t *testing.T) { runtime string }{ {"docker"}, + {"nvidia-docker"}, {"crio"}, {"containerd"}, } @@ -730,7 +746,7 @@ func TestContainerFunctions(t *testing.T) { t.Run(tc.runtime, func(t *testing.T) { runner := NewFakeRunner(t) prefix := "" - if tc.runtime == "docker" { + if tc.runtime == "docker" || tc.runtime == "nvidia-docker" { prefix = "k8s_" } runner.containers = map[string]string{ diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 044ce41c809f..d083f94a3f65 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -67,6 +67,7 @@ func (e *ErrISOFeature) Error() string { // Docker contains Docker runtime state type Docker struct { + Type string Socket string Runner CommandRunner NetworkPlugin string @@ -156,7 +157,7 @@ func (r *Docker) Enable(disOthers bool, cgroupDriver string, inUserNamespace boo klog.ErrorS(err, "Failed to enable", "service", "docker.socket") } - if err := r.setCGroup(cgroupDriver); err != nil { + if err := r.configureDocker(cgroupDriver); err != nil { return err } @@ -525,24 +526,51 @@ func (r *Docker) SystemLogCmd(len int) string { return fmt.Sprintf("sudo journalctl -u docker -u cri-docker -n %d", len) } -// setCGroup configures the docker daemon to use driver as cgroup manager +type dockerDaemonConfig struct { + ExecOpts []string `json:"exec-opts"` + LogDriver string `json:"log-driver"` + LogOpts dockerDaemonLogOpts `json:"log-opts"` + StorageDriver string `json:"storage-driver"` + DefaultRuntime string `json:"default-runtime,omitempty"` + Runtimes *dockerDaemonRuntimes `json:"runtimes,omitempty"` +} +type dockerDaemonLogOpts struct { + MaxSize string `json:"max-size"` +} +type dockerDaemonRuntimes struct { + Nvidia struct { + Path string `json:"path"` + RuntimeArgs []interface{} `json:"runtimeArgs"` + } `json:"nvidia"` +} + +// configureDocker configures the docker daemon to use driver as cgroup manager // ref: https://docs.docker.com/engine/reference/commandline/dockerd/#options-for-the-runtime -func (r *Docker) setCGroup(driver string) error { +func (r *Docker) configureDocker(driver string) error { if driver == constants.UnknownCgroupDriver { return fmt.Errorf("unable to configure docker to use unknown cgroup driver") } klog.Infof("configuring docker to use %q as cgroup driver...", driver) - daemonConfig := fmt.Sprintf(`{ -"exec-opts": ["native.cgroupdriver=%s"], -"log-driver": "json-file", -"log-opts": { - "max-size": "100m" -}, -"storage-driver": "overlay2" -} -`, driver) - ma := assets.NewMemoryAsset([]byte(daemonConfig), "/etc/docker", "daemon.json", "0644") + daemonConfig := dockerDaemonConfig{ + ExecOpts: []string{"native.cgroupdriver=" + driver}, + LogDriver: "json-file", + LogOpts: dockerDaemonLogOpts{ + MaxSize: "100m", + }, + StorageDriver: "overlay2", + } + if r.Type == "nvidia-docker" { + daemonConfig.DefaultRuntime = "nvidia" + runtimes := &dockerDaemonRuntimes{} + runtimes.Nvidia.Path = "/usr/bin/nvidia-container-runtime" + daemonConfig.Runtimes = runtimes + } + daemonConfigBytes, err := json.Marshal(daemonConfig) + if err != nil { + return err + } + ma := assets.NewMemoryAsset(daemonConfigBytes, "/etc/docker", "daemon.json", "0644") return r.Runner.Copy(ma) }