From b05b464e0c5fb6ea2897fdde667d855be9814954 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Sat, 18 Nov 2023 21:15:48 +0000 Subject: [PATCH 1/3] handle kubernetes versions before v1.24 with docker as container runtime --- test/integration/start_stop_delete_test.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index cedb952c182f..5a2b1f34a975 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -352,7 +352,23 @@ func testPulledImages(ctx context.Context, t *testing.T, profile, version string t.Helper() defer PostMortemLogs(t, profile) - rr, err := Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "sudo crictl images -o json")) + cmd := "sudo crictl images -o json" + + // handle old kubernetes versions (before v1.24) with docker as container runtime + // avoid 'validate service connection: validate CRI v1 image API for endpoint "unix:///var/run/dockershim.sock": rpc error: code = Unimplemented desc = unknown service runtime.v1.ImageService' error + // as newer crictl needs cri-dockerd.sock instead of dockershim.sock + // and if unspecified, crictl will try dockershim.sock first and cri-dockerd.sock last + // ref: https://github.com/shannonxtreme/cri-tools/blob/17484cda811c93b69e61448835db9559c7f3ab9c/cmd/crictl/main_unix.go#L30 + if v, _ := util.ParseKubernetesVersion(version); v.LT(semver.Version{Major: 1, Minor: 24}) { + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "kubectl", "--ssh", "--", "get nodes -o wide")) + if err == nil && strings.Contains(rr.Stdout.String(), "docker://") { + cmd = "sudo crictl --runtime-endpoint unix:///var/run/cri-dockerd.sock images -o json" + // try to ensure active "cri-docker.socket", fallthrough on error + _, _ = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "sudo", "systemctl", "restart", "cri-docker.socket")) + } + } + + rr, err := Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, cmd)) if err != nil { t.Errorf("failed to get images inside minikube. args %q: %v", rr.Command(), err) } From 44821c0dee9e42668c40434bc869b6968f9fa365 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Tue, 21 Nov 2023 23:39:24 +0000 Subject: [PATCH 2/3] use 'minikube image list' instead of 'crictl images' --- test/integration/start_stop_delete_test.go | 30 +++++----------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index 5a2b1f34a975..c3d0dd47c8bb 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -352,45 +352,29 @@ func testPulledImages(ctx context.Context, t *testing.T, profile, version string t.Helper() defer PostMortemLogs(t, profile) - cmd := "sudo crictl images -o json" - - // handle old kubernetes versions (before v1.24) with docker as container runtime - // avoid 'validate service connection: validate CRI v1 image API for endpoint "unix:///var/run/dockershim.sock": rpc error: code = Unimplemented desc = unknown service runtime.v1.ImageService' error - // as newer crictl needs cri-dockerd.sock instead of dockershim.sock - // and if unspecified, crictl will try dockershim.sock first and cri-dockerd.sock last - // ref: https://github.com/shannonxtreme/cri-tools/blob/17484cda811c93b69e61448835db9559c7f3ab9c/cmd/crictl/main_unix.go#L30 - if v, _ := util.ParseKubernetesVersion(version); v.LT(semver.Version{Major: 1, Minor: 24}) { - rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "kubectl", "--ssh", "--", "get nodes -o wide")) - if err == nil && strings.Contains(rr.Stdout.String(), "docker://") { - cmd = "sudo crictl --runtime-endpoint unix:///var/run/cri-dockerd.sock images -o json" - // try to ensure active "cri-docker.socket", fallthrough on error - _, _ = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "sudo", "systemctl", "restart", "cri-docker.socket")) - } - } - - rr, err := Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, cmd)) + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "list", "--format=json")) if err != nil { t.Errorf("failed to get images inside minikube. args %q: %v", rr.Command(), err) } - jv := map[string][]struct { + jv := []struct { Tags []string `json:"repoTags"` }{} stdout := rr.Stdout.String() - err = json.Unmarshal([]byte(stdout), &jv) - if err != nil { + if err := json.Unmarshal([]byte(stdout), &jv); err != nil { t.Errorf("failed to decode images json %v. output:\n%s", err, stdout) } + found := map[string]bool{} - for _, img := range jv["images"] { + for _, img := range jv { for _, i := range img.Tags { i = trimImageName(i) if defaultImage(i) { found[i] = true - } else { - t.Logf("Found non-minikube image: %s", i) + continue } + t.Logf("Found non-minikube image: %s", i) } } From 251fdc5c9deed15bd59c544a825b5d97d5aa4e80 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Tue, 28 Nov 2023 01:21:53 +0000 Subject: [PATCH 3/3] todo: switch from 'minikube image list' to 'crictl images' --- test/integration/start_stop_delete_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index c3d0dd47c8bb..18b724381f2d 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -352,6 +352,10 @@ func testPulledImages(ctx context.Context, t *testing.T, profile, version string t.Helper() defer PostMortemLogs(t, profile) + // TODO(prezha): once we bump the minimum supported k8s version to v1.24+ + // (where dockershim is deprecated, while cri-tools we use support cri v1 api), + // we can revert back to the "crictl" to check images here - eg: + // rr, err := Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "sudo crictl images -o json")) rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "list", "--format=json")) if err != nil { t.Errorf("failed to get images inside minikube. args %q: %v", rr.Command(), err)