Expand testing of k8s versions (#320)

gavinbunney · Nov 30, 2024 · 4449e71 · 4449e71
1 parent 1ce5762
commit 4449e71
Show file tree

Hide file tree

Showing 6 changed files with 66 additions and 20 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -45,6 +45,16 @@ jobs:
         run: make test
 
   testacc:
+    strategy:
+      matrix:
+        k3s_version:
+          - v1.31.2-k3s1
+          - v1.30.6-k3s1
+          - v1.29.10-k3s1
+          - v1.28.15-k3s1
+          - v1.27.16-k3s1
+          - v1.26.15-k3s1
+          - v1.25.16-k3s4
     runs-on: ubuntu-latest
     env:
       GO111MODULE: on
@@ -61,6 +71,8 @@ jobs:
         run: make ci-build-setup
       - name: Start k3s
         run: make k3s-start
+        env:
+          K3S_VERSION: ${{ matrix.k3s_version }}
       - name: Run acceptance tests
         run: make testacc
       - name: Stop k3s

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -3,14 +3,21 @@ name: Cache Images
 on:
   push:
     paths:
-      - 'scripts/docker-compose.yaml'
       - '.github/workflows/docker.yml'
 
-env:
-  IMAGES: "rancher/k3s:v1.31.2-k3s1,registry:2"
-
 jobs:
   cache-images:
+    strategy:
+      matrix:
+        image:
+          - rancher/k3s:v1.31.2-k3s1
+          - rancher/k3s:v1.30.6-k3s1
+          - rancher/k3s:v1.29.10-k3s1
+          - rancher/k3s:v1.28.15-k3s1
+          - rancher/k3s:v1.27.16-k3s1
+          - rancher/k3s:v1.26.15-k3s1
+          - rancher/k3s:v1.25.16-k3s4
+          - registry:2
     if: github.event.pull_request.user.login != 'dependabot[bot]' && github.actor != 'dependabot[bot]'
     runs-on: ubuntu-latest
     permissions:
@@ -39,17 +46,15 @@ jobs:
 
       - name: Cache images
         run: |
-          for image in ${IMAGES//,/ }; do
-            new_image="ghcr.io/gavinbunney/terraform-provider-kubectl/testacc-$(echo $image | sed 's/\//-/')"
-            echo "Caching $image as $new_image"
-            docker rmi $image --force
-            docker pull $image --platform linux/amd64
-            docker tag $image $new_image-amd64
-            docker push $new_image-amd64
-            docker tag $image $new_image-x86_64
-            docker push $new_image-x86_64
-            docker rmi $image --force
-            docker pull $image --platform linux/arm64
-            docker tag $image $new_image-arm64
-            docker push $new_image-arm64
-          done
+          new_image_tag="ghcr.io/gavinbunney/terraform-provider-kubectl/testacc-$(echo '${{matrix.image}}' | sed 's/\//-/')"
+          echo "Caching ${{matrix.image}} as $new_image_tag"
+          docker rmi ${{matrix.image}} --force
+          docker pull ${{matrix.image}} --platform linux/amd64
+          docker tag ${{matrix.image}} $new_image_tag-amd64
+          docker push $new_image_tag-amd64
+          docker tag ${{matrix.image}} $new_image_tag-x86_64
+          docker push $new_image_tag-x86_64
+          docker rmi ${{matrix.image}} --force
+          docker pull ${{matrix.image}} --platform linux/arm64
+          docker tag ${{matrix.image}} $new_image_tag-arm64
+          docker push $new_image_tag-arm64
diff --git a/kubernetes/provider.go b/kubernetes/provider.go
@@ -7,16 +7,20 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"reflect"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/mitchellh/go-homedir"
+
 	"k8s.io/apimachinery/pkg/api/meta"
 	apimachineryschema "k8s.io/apimachinery/pkg/runtime/schema"
+	k8sruntime "k8s.io/apimachinery/pkg/util/runtime"
 	k8sresource "k8s.io/cli-runtime/pkg/resource"
 	"k8s.io/client-go/discovery"
 	diskcached "k8s.io/client-go/discovery/cached/disk"
@@ -265,6 +269,27 @@ func providerConfigure(d *schema.ResourceData, terraformVersion string) (interfa
 		return nil, diag.FromErr(fmt.Errorf("failed to configure: %s", err))
 	}
 
+	// inject our own error handler into the k8s runtime so we can log correctly into provider logs
+	// and also ignore some background cache refresh logs which don't relate to the user's actions
+	const defaultLogHandlerFunc = "k8s.io/apimachinery/pkg/util/runtime.logError"
+	for idx, handler := range k8sruntime.ErrorHandlers {
+		handlerName := runtime.FuncForPC(reflect.ValueOf(handler).Pointer()).Name()
+		if handlerName == defaultLogHandlerFunc {
+			k8sruntime.ErrorHandlers[idx] = func(ctx context.Context, err error, msg string, keysAndValues ...interface{}) {
+				// silence discovery cache refresh errors
+				if caller, _, _, ok := runtime.Caller(3); ok {
+					callerFunc := runtime.FuncForPC(caller)
+					if strings.HasPrefix(callerFunc.Name(), "k8s.io/client-go/discovery/cached/memory") {
+						log.Printf("[DEBUG] %s - %s, %v", callerFunc.Name(), msg, err)
+						return
+					}
+				}
+
+				log.Printf("[ERROR] %s %v", msg, err)
+			}
+		}
+	}
+
 	// dereference config to create a shallow copy, allowing each func
 	// to manipulate the state without affecting another func
 	return &KubeProvider{

diff --git a/scripts/docker-compose.yaml b/scripts/docker-compose.yaml
@@ -1,6 +1,6 @@
 services:
   server:
-    image: ghcr.io/gavinbunney/terraform-provider-kubectl/testacc-rancher-k3s:v1.31.2-k3s1-${ARCH:-amd64}
+    image: ghcr.io/gavinbunney/terraform-provider-kubectl/testacc-rancher-k3s:${K3S_VERSION}-${ARCH}
     command: server --disable-agent --tls-san 172.17.0.1 --agent-token somethingtotallyrandom
     privileged: true
     environment:
@@ -20,7 +20,7 @@ services:
       - 6443:6443
 
   node:
-    image: ghcr.io/gavinbunney/terraform-provider-kubectl/testacc-rancher-k3s:v1.31.2-k3s1-${ARCH:-amd64}
+    image: ghcr.io/gavinbunney/terraform-provider-kubectl/testacc-rancher-k3s:${K3S_VERSION}-${ARCH}
     command: agent --token somethingtotallyrandom --server https://server:6443
     privileged: true
     restart: always

diff --git a/scripts/start-k3s.sh b/scripts/start-k3s.sh
@@ -9,6 +9,8 @@ export COMPOSE_PROJECT_NAME=k3s
 export ARCH=$(uname -m | tr '[:upper:]' '[:lower:]')
 export DOCKER_DEFAULT_PLATFORM=linux/${ARCH}
 
+export K3S_VERSION=${K3S_VERSION:-v1.31.2-k3s1}
+
 echo "--> Tearing down k3s in docker-compose"
 docker-compose down -v &>/dev/null || true
 rm -rf ${KUBECONFIG}

diff --git a/scripts/stop-k3s.sh b/scripts/stop-k3s.sh
@@ -8,6 +8,8 @@ export COMPOSE_PROJECT_NAME=k3s
 export ARCH=$(uname -m | tr '[:upper:]' '[:lower:]')
 export DOCKER_DEFAULT_PLATFORM=linux/${ARCH}
 
+export K3S_VERSION=${K3S_VERSION:-v1.31.2-k3s1}
+
 echo "--> Stopping k3s in docker-compose"
 docker-compose down -v
 rm -rf kubeconfig.yaml