Add support for AMD GPU via --gpu=amd for docker linux amd64.

.github/workflows/update-amd-gpu-device-plugin-version.yml

@@ -0,0 +1,48 @@
+name: "update-amd-gpu-device-plugin-version"
+on:
+  workflow_dispatch:
+  schedule:
+    # every Monday at around 3 am pacific/10 am UTC
+    - cron: "0 10 * * 1"
+env:
+  GOPROXY: https://proxy.golang.org
+  GO_VERSION: '1.23.0'
+permissions:
+  contents: read
+
+jobs:
+  bump-amd-gpu-device-plugin-version:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32
+        with:
+          go-version: ${{env.GO_VERSION}}
+      - name: Bump amd-gpu-device-plugin version
+        id: bumpAmdDevicePlugin
+        run: |
+          echo "OLD_VERSION=$(DEP=amd-gpu-device-plugin make get-dependency-version)" >> "$GITHUB_OUTPUT"
+          make update-amd-gpu-device-plugin-version
+          echo "NEW_VERSION=$(DEP=amd-gpu-device-plugin make get-dependency-version)" >> "$GITHUB_OUTPUT"
+          # The following is to support multiline with GITHUB_OUTPUT, see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
+          echo "changes<<EOF" >> "$GITHUB_OUTPUT"
+          echo "$(git status --porcelain)" >> "$GITHUB_OUTPUT"
+          echo "EOF" >> "$GITHUB_OUTPUT"
+      - name: Create PR
+        if: ${{ steps.bumpAmdDevicePlugin.outputs.changes != '' }}
+        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f
+        with:
+          token: ${{ secrets.MINIKUBE_BOT_PAT }}
+          commit-message: 'Addon amd-gpu-device-plugin: Update amd/k8s-device-plugin image from ${{ steps.bumpAmdDevicePlugin.outputs.OLD_VERSION }} to ${{ steps.bumpAmdDevicePlugin.outputs.NEW_VERSION }}'
+          committer: minikube-bot <[email protected]>
+          author: minikube-bot <[email protected]>
+          branch: auto_bump_amd_device_plugin_version
+          push-to-fork: minikube-bot/minikube
+          base: master
+          delete-branch: true
+          title: 'Addon amd-gpu-device-plugin: Update amd/k8s-device-plugin image from ${{ steps.bumpAmdDevicePlugin.outputs.OLD_VERSION }} to ${{ steps.bumpAmdDevicePlugin.outputs.NEW_VERSION }}'
+          labels: ok-to-test
+          body: |
+            The [k8s-device-plugin](https://github.com/ROCm/k8s-device-plugin) project released a new k8s-device-plugin image
+
+            This PR was auto-generated by `make update-amd-gpu-device-plugin-version` using [update-amd-gpu-device-plugin-version.yml](https://github.com/kubernetes/minikube/tree/master/.github/workflows/update-amd-gpu-device-plugin-version.yml) CI Workflow.

Makefile

@@ -1222,6 +1222,11 @@ update-nvidia-device-plugin-version:
 	(cd hack/update/nvidia_device_plugin_version && \
 	 go run update_nvidia_device_plugin_version.go)
 
+.PHONY: update-amd-gpu-device-plugin-version
+update-amd-gpu-device-plugin-version:
+	(cd hack/update/amd_device_plugin_version && \
+	 go run update_amd_device_plugin_version.go)
+
 .PHONY: update-nerctld-version
 update-nerdctld-version:
 	(cd hack/update/nerdctld_version && \

README.md

@@ -35,6 +35,7 @@ As well as developer-friendly features:
 
 * [Addons](https://minikube.sigs.k8s.io/docs/handbook/deploying/#addons) - a marketplace for developers to share configurations for running services on minikube
 * [NVIDIA GPU support](https://minikube.sigs.k8s.io/docs/tutorials/nvidia/) - for machine learning
+* [AMD GPU support](https://minikube.sigs.k8s.io/docs/tutorials/amd/) - for machine learning
 * [Filesystem mounts](https://minikube.sigs.k8s.io/docs/handbook/mount/)
 
 **For more information, see the official [minikube website](https://minikube.sigs.k8s.io)**

cmd/minikube/cmd/start.go

@@ -1462,8 +1462,8 @@ func validateGPUs(value, drvName, rtime string) error {
 	if err := validateGPUsArch(); err != nil {
 		return err
 	}
-	if value != "nvidia" && value != "all" {
-		return errors.Errorf(`The gpus flag must be passed a value of "nvidia" or "all"`)
+	if value != "nvidia" && value != "all" && value != "amd" {
+		return errors.Errorf(`The gpus flag must be passed a value of "nvidia", "amd" or "all"`)
 	}
 	if drvName == constants.Docker && (rtime == constants.Docker || rtime == constants.DefaultContainerRuntime) {
 		return nil

cmd/minikube/cmd/start_flags.go

@@ -206,7 +206,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().Bool(disableOptimizations, false, "If set, disables optimizations that are set for local Kubernetes. Including decreasing CoreDNS replicas from 2 to 1. Defaults to false.")
 	startCmd.Flags().Bool(disableMetrics, false, "If set, disables metrics reporting (CPU and memory usage), this can improve CPU usage. Defaults to false.")
 	startCmd.Flags().String(staticIP, "", "Set a static IP for the minikube cluster, the IP must be: private, IPv4, and the last octet must be between 2 and 254, for example 192.168.200.200 (Docker and Podman drivers only)")
-	startCmd.Flags().StringP(gpus, "g", "", "Allow pods to use your NVIDIA GPUs. Options include: [all,nvidia] (Docker driver with Docker container-runtime only)")
+	startCmd.Flags().StringP(gpus, "g", "", "Allow pods to use your GPUs. Options include: [all,nvidia,amd] (Docker driver with Docker container-runtime only)")
 	startCmd.Flags().Duration(autoPauseInterval, time.Minute*1, "Duration of inactivity before the minikube VM is paused (default 1m0s)")
 }
 

cmd/minikube/cmd/start_test.go

@@ -814,7 +814,10 @@ func TestValidateGPUs(t *testing.T) {
 		{"nvidia", "docker", "", ""},
 		{"all", "kvm", "docker", "The gpus flag can only be used with the docker driver and docker container-runtime"},
 		{"nvidia", "docker", "containerd", "The gpus flag can only be used with the docker driver and docker container-runtime"},
-		{"cat", "docker", "docker", `The gpus flag must be passed a value of "nvidia" or "all"`},
+		{"cat", "docker", "docker", `The gpus flag must be passed a value of "nvidia", "amd" or "all"`},
+		{"amd", "docker", "docker", ""},
+		{"amd", "docker", "", ""},
+		{"amd", "docker", "containerd", "The gpus flag can only be used with the docker driver and docker container-runtime"},
 	}
 
 	for _, tc := range tests {

deploy/addons/assets.go

@@ -107,6 +107,10 @@ var (
 	//go:embed gpu/nvidia-gpu-device-plugin.yaml.tmpl
 	NvidiaGpuDevicePluginAssets embed.FS
 
+	// AmdGpuDevicePluginAssets assets for amd-gpu-device-plugin addon
+	//go:embed gpu/amd-gpu-device-plugin.yaml.tmpl
+	AmdGpuDevicePluginAssets embed.FS
+
 	// LogviewerAssets assets for logviewer addon
 	//go:embed logviewer/*.tmpl logviewer/*.yaml
 	LogviewerAssets embed.FS

deploy/addons/gpu/amd-gpu-device-plugin.yaml.tmpl

@@ -0,0 +1,60 @@
+# Copyright 2024 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: amd-gpu-device-plugin
+  namespace: kube-system
+  labels:
+    k8s-app: amd-gpu-device-plugin
+    kubernetes.io/minikube-addons: amd-gpu-device-plugin
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  selector:
+    matchLabels:
+      k8s-app: amd-gpu-device-plugin
+  template:
+    metadata:
+      labels:
+        name: amd-gpu-device-plugin
+        k8s-app: amd-gpu-device-plugin
+    spec:
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      priorityClassName: system-node-critical
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      volumes:
+      - name: dp
+        hostPath:
+          path: /var/lib/kubelet/device-plugins
+      - name: sys
+        hostPath:
+          path: /sys
+      containers:
+      - image: {{.CustomRegistries.AmdDevicePlugin  | default .ImageRepository | default .Registries.AmdDevicePlugin }}{{.Images.AmdDevicePlugin}}
+        name: amd-gpu-device-plugin
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: ["ALL"]
+        volumeMounts:
+        - name: dp
+          mountPath: /var/lib/kubelet/device-plugins
+        - name: sys
+          mountPath: /sys
+  updateStrategy:
+    type: RollingUpdate

hack/update/amd_device_plugin_version/update_amd_device_plugin_version.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2024 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"k8s.io/klog/v2"
+	"k8s.io/minikube/hack/update"
+)
+
+var schema = map[string]update.Item{
+	"pkg/minikube/assets/addons.go": {
+		Replace: map[string]string{
+			`rocm/k8s-device-plugin:.*`: `rocm/k8s-device-plugin:{{.Version}}@{{.SHA}}",`,
+		},
+	},
+}
+
+type Data struct {
+	Version string
+	SHA     string
+}
+
+func main() {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+
+	stable, _, _, err := update.GHReleases(ctx, "ROCm", "k8s-device-plugin")
+	if err != nil {
+		klog.Fatalf("Unable to get stable version: %v", err)
+	}
+	sha, err := update.GetImageSHA(fmt.Sprintf("rocm/k8s-device-plugin:%s", stable.Tag))
+	if err != nil {
+		klog.Fatalf("failed to get image SHA: %v", err)
+	}
+
+	data := Data{Version: stable.Tag, SHA: sha}
+
+	update.Apply(schema, data)
+}

hack/update/get_version/get_version.go

@@ -33,6 +33,7 @@ type dependency struct {
 }
 
 var dependencies = map[string]dependency{
+	"amd-gpu-device-plugin":   {addonsFile, `rocm/k8s-device-plugin:(.*)@`},
 	"buildkit":                {"deploy/iso/minikube-iso/arch/x86_64/package/buildkit-bin/buildkit-bin.mk", `BUILDKIT_BIN_VERSION = (.*)`},
 	"calico":                  {"pkg/minikube/bootstrapper/images/images.go", `calicoVersion = "(.*)"`},
 	"cilium":                  {"pkg/minikube/cni/cilium.yaml", `quay.io/cilium/cilium:(.*)@`},

kubeadm.new.yaml

@@ -0,0 +1,71 @@
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: InitConfiguration
+bootstrapTokens:
+- groups:
+  - system:bootstrappers:kubeadm:default-node-token
+  token: r3db6d.ut3qb84zr8ngrbhf
+  ttl: 24h0m0s
+  usages:
+  - signing
+  - authentication
+localAPIEndpoint:
+  advertiseAddress: 192.168.49.2
+  bindPort: 8443
+nodeRegistration:
+  criSocket: unix:///var/run/cri-dockerd.sock
+  name: minikube
+  imagePullPolicy: IfNotPresent
+  imagePullSerial: true
+  kubeletExtraArgs:
+  - name: node-ip
+    value: 192.168.49.2
+  taints: []
+timeouts:
+  controlPlaneComponentHealthCheck: 4m0s
+  discovery: 5m0s
+  etcdAPICall: 2m0s
+  kubeletHealthCheck: 4m0s
+  kubernetesAPICall: 1m0s
+  tlsBootstrap: 5m0s
+  upgradeManifests: 5m0s
+---
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: ClusterConfiguration
+apiServer:
+  certSANs:
+  - 127.0.0.1
+  - localhost
+  - 192.168.49.2
+  extraArgs:
+  - name: enable-admission-plugins
+    value: NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota
+caCertificateValidityPeriod: 87600h0m0s
+certificateValidityPeriod: 8760h0m0s
+certificatesDir: /var/lib/minikube/certs
+clusterName: mk
+controlPlaneEndpoint: control-plane.minikube.internal:8443
+controllerManager:
+  extraArgs:
+  - name: allocate-node-cidrs
+    value: "true"
+  - name: leader-elect
+    value: "false"
+dns: {}
+encryptionAlgorithm: RSA-2048
+etcd:
+  local:
+    dataDir: /var/lib/minikube/etcd
+    extraArgs:
+    - name: proxy-refresh-interval
+      value: "70000"
+imageRepository: registry.k8s.io
+kubernetesVersion: v1.31.0
+networking:
+  dnsDomain: cluster.dev.local
+  podSubnet: 10.244.0.0/16
+  serviceSubnet: 10.96.0.0/12
+proxy: {}
+scheduler:
+  extraArgs:
+  - name: leader-elect
+    value: "false"

kubeadm.yaml

@@ -0,0 +1,75 @@
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: InitConfiguration
+bootstrapTokens:
+  - groups:
+      - system:bootstrappers:kubeadm:default-node-token
+    ttl: 24h0m0s
+    usages:
+      - signing
+      - authentication
+localAPIEndpoint:
+  advertiseAddress: 192.168.49.2
+  bindPort: 8443
+nodeRegistration:
+  criSocket: unix:///var/run/cri-dockerd.sock
+  name: "minikube"
+  kubeletExtraArgs:
+    node-ip: 192.168.49.2
+  taints: []
+---
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: ClusterConfiguration
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "192.168.49.2"]
+  extraArgs:
+    enable-admission-plugins: NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota
+certificatesDir: /var/lib/minikube/certs
+clusterName: mk
+controlPlaneEndpoint: control-plane.minikube.internal:8443
+controllerManager:
+  extraArgs:
+    allocate-node-cidrs: "true"
+    leader-elect: "false"
+scheduler:
+  extraArgs:
+    leader-elect: "false"
+etcd:
+  local:
+    dataDir: /var/lib/minikube/etcd
+    extraArgs:
+      proxy-refresh-interval: "70000"
+kubernetesVersion: v1.31.0
+networking:
+  dnsDomain: cluster.dev.local
+  podSubnet: "10.244.0.0/16"
+  serviceSubnet: 10.96.0.0/12
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
+containerRuntimeEndpoint: unix:///var/run/cri-dockerd.sock
+hairpinMode: hairpin-veth
+runtimeRequestTimeout: 15m
+clusterDomain: "cluster.dev.local"
+# disable disk resource management by default
+imageGCHighThresholdPercent: 100
+evictionHard:
+  nodefs.available: "0%"
+  nodefs.inodesFree: "0%"
+  imagefs.available: "0%"
+failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+clusterCIDR: "10.244.0.0/16"
+metricsBindAddress: 0.0.0.0:10249
+conntrack:
+  maxPerCore: 0
+# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established"
+  tcpEstablishedTimeout: 0s
+# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_close"
+  tcpCloseWaitTimeout: 0s

pkg/addons/config.go

@@ -131,6 +131,11 @@ var Addons = []*Addon{
 		validations: []setFn{isKVMDriverForNVIDIA},
 		callbacks:   []setFn{EnableOrDisableAddon},
 	},
+	{
+		name:      "amd-gpu-device-plugin",
+		set:       SetBool,
+		callbacks: []setFn{EnableOrDisableAddon},
+	},
 	{
 		name:      "olm",
 		set:       SetBool,