From 2c3b69807081fe4317a155f10b66523b85a8d6b7 Mon Sep 17 00:00:00 2001 From: Tristan Rice Date: Sat, 25 Feb 2023 23:00:33 -0800 Subject: [PATCH 1/2] container-runtime: add nvidia-docker --- pkg/minikube/cruntime/cruntime.go | 5 ++- pkg/minikube/cruntime/cruntime_test.go | 13 ++++++- pkg/minikube/cruntime/docker.go | 54 +++++++++++++++++++------- 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index be04e87d8763..c45a780592d4 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -50,7 +50,7 @@ func (cs ContainerState) String() string { // ValidRuntimes lists the supported container runtimes func ValidRuntimes() []string { - return []string{"docker", "cri-o", "containerd"} + return []string{"docker", "nvidia-docker", "cri-o", "containerd"} } // CommandRunner is the subset of command.Runner this package consumes @@ -208,7 +208,7 @@ func New(c Config) (Manager, error) { sm := sysinit.New(c.Runner) switch c.Type { - case "", "docker": + case "", "docker", "nvidia-docker": sp := c.Socket cs := "" // There is no more dockershim socket, in Kubernetes version 1.24 and beyond @@ -217,6 +217,7 @@ func New(c Config) (Manager, error) { cs = "cri-docker.socket" } return &Docker{ + Type: c.Type, Socket: sp, Runner: c.Runner, ImageRepository: c.ImageRepository, diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go index 134806039054..963b8e129c8c 100644 --- a/pkg/minikube/cruntime/cruntime_test.go +++ b/pkg/minikube/cruntime/cruntime_test.go @@ -40,6 +40,7 @@ func TestName(t *testing.T) { }{ {"", "Docker"}, {"docker", "Docker"}, + {"nvidia-docker", "Docker"}, {"crio", "CRI-O"}, {"cri-o", "CRI-O"}, {"containerd", "containerd"}, @@ -124,6 +125,7 @@ func TestCGroupDriver(t *testing.T) { want string }{ {"docker", "cgroupfs"}, + {"nvidia-docker", "cgroupfs"}, {"crio", "cgroupfs"}, {"containerd", "cgroupfs"}, } @@ -151,6 +153,7 @@ func TestKubeletOptions(t *testing.T) { want map[string]string }{ {"docker", map[string]string{"container-runtime": "docker"}}, + {"nvidia-docker", map[string]string{"container-runtime": "docker"}}, {"crio", map[string]string{ "container-runtime": "remote", "container-runtime-endpoint": "/var/run/crio/crio.sock", @@ -675,6 +678,13 @@ func TestEnable(t *testing.T) { "crio": SvcExited, "crio-shutdown": SvcExited, }}, + {"nvidia-docker", defaultServices, + map[string]serviceState{ + "docker": SvcRestarted, + "containerd": SvcExited, + "crio": SvcExited, + "crio-shutdown": SvcExited, + }}, {"containerd", defaultServices, map[string]serviceState{ "docker": SvcExited, @@ -716,6 +726,7 @@ func TestContainerFunctions(t *testing.T) { runtime string }{ {"docker"}, + {"nvidia-docker"}, {"crio"}, {"containerd"}, } @@ -725,7 +736,7 @@ func TestContainerFunctions(t *testing.T) { t.Run(tc.runtime, func(t *testing.T) { runner := NewFakeRunner(t) prefix := "" - if tc.runtime == "docker" { + if tc.runtime == "docker" || tc.runtime == "nvidia-docker" { prefix = "k8s_" } runner.containers = map[string]string{ diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 2a200cd52068..5bf3c7c48025 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -67,6 +67,7 @@ func (e *ErrISOFeature) Error() string { // Docker contains Docker runtime state type Docker struct { + Type string Socket string Runner CommandRunner ImageRepository string @@ -149,7 +150,7 @@ func (r *Docker) Enable(disOthers bool, cgroupDriver string, inUserNamespace boo klog.ErrorS(err, "Failed to enable", "service", "docker.socket") } - if err := r.setCGroup(cgroupDriver); err != nil { + if err := r.configureDocker(cgroupDriver); err != nil { return err } @@ -519,24 +520,51 @@ func (r *Docker) SystemLogCmd(len int) string { return fmt.Sprintf("sudo journalctl -u docker -n %d", len) } -// setCGroup configures the docker daemon to use driver as cgroup manager +type dockerDaemonConfig struct { + ExecOpts []string `json:"exec-opts"` + LogDriver string `json:"log-driver"` + LogOpts dockerDaemonLogOpts `json:"log-opts"` + StorageDriver string `json:"storage-driver"` + DefaultRuntime string `json:"default-runtime,omitempty"` + Runtimes *dockerDaemonRuntimes `json:"runtimes,omitempty"` +} +type dockerDaemonLogOpts struct { + MaxSize string `json:"max-size"` +} +type dockerDaemonRuntimes struct { + Nvidia struct { + Path string `json:"path"` + RuntimeArgs []interface{} `json:"runtimeArgs"` + } `json:"nvidia"` +} + +// configureDocker configures the docker daemon to use driver as cgroup manager // ref: https://docs.docker.com/engine/reference/commandline/dockerd/#options-for-the-runtime -func (r *Docker) setCGroup(driver string) error { +func (r *Docker) configureDocker(driver string) error { if driver == constants.UnknownCgroupDriver { return fmt.Errorf("unable to configure docker to use unknown cgroup driver") } klog.Infof("configuring docker to use %q as cgroup driver...", driver) - daemonConfig := fmt.Sprintf(`{ -"exec-opts": ["native.cgroupdriver=%s"], -"log-driver": "json-file", -"log-opts": { - "max-size": "100m" -}, -"storage-driver": "overlay2" -} -`, driver) - ma := assets.NewMemoryAsset([]byte(daemonConfig), "/etc/docker", "daemon.json", "0644") + daemonConfig := dockerDaemonConfig{ + ExecOpts: []string{"native.cgroupdriver=" + driver}, + LogDriver: "json-file", + LogOpts: dockerDaemonLogOpts{ + MaxSize: "100m", + }, + StorageDriver: "overlay2", + } + if r.Type == "nvidia-docker" { + daemonConfig.DefaultRuntime = "nvidia" + runtimes := &dockerDaemonRuntimes{} + runtimes.Nvidia.Path = "/usr/bin/nvidia-container-runtime" + daemonConfig.Runtimes = runtimes + } + daemonConfigBytes, err := json.Marshal(daemonConfig) + if err != nil { + return err + } + ma := assets.NewMemoryAsset(daemonConfigBytes, "/etc/docker", "daemon.json", "0644") return r.Runner.Copy(ma) } From 2a1f5b9114b915ff9d9f0d64032af1cf94003b94 Mon Sep 17 00:00:00 2001 From: Steven Powell Date: Tue, 19 Sep 2023 09:29:34 -0700 Subject: [PATCH 2/2] fix rebased test --- pkg/minikube/cruntime/cruntime_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go index e967d177654d..ed7e5d6b5561 100644 --- a/pkg/minikube/cruntime/cruntime_test.go +++ b/pkg/minikube/cruntime/cruntime_test.go @@ -157,7 +157,12 @@ func TestKubeletOptions(t *testing.T) { {"docker", "1.24.0", map[string]string{ "container-runtime-endpoint": "unix:///var/run/cri-dockerd.sock", }}, - {"nvidia-docker", "1.25.0", map[string]string{"container-runtime": "docker"}}, + {"nvidia-docker", "1.23.0", map[string]string{ + "container-runtime": "docker", + }}, + {"nvidia-docker", "1.25.0", map[string]string{ + "container-runtime-endpoint": "unix:///var/run/cri-dockerd.sock", + }}, {"crio", "1.25.0", map[string]string{ "container-runtime-endpoint": "unix:///var/run/crio/crio.sock", }},