From d101c00e037dc393fdf4a0942f452fc3849cee31 Mon Sep 17 00:00:00 2001 From: Karl Isenberg Date: Mon, 22 May 2023 15:40:46 -0700 Subject: [PATCH] Add e2e test for HorizontalPodAutoscaler --- e2e/nomostest/client.go | 2 + e2e/testcases/hpa_test.go | 104 ++++++++ e2e/testdata/hpa/deployment-helloworld.yaml | 35 +++ e2e/testdata/hpa/hpa-helloworld.yaml | 33 +++ e2e/testdata/hpa/ns-helloworld.yaml | 18 ++ .../metrics-server/components-v0.7.2.yaml | 233 ++++++++++++++++++ pkg/kinds/kinds.go | 6 + 7 files changed, 431 insertions(+) create mode 100644 e2e/testcases/hpa_test.go create mode 100644 e2e/testdata/hpa/deployment-helloworld.yaml create mode 100644 e2e/testdata/hpa/hpa-helloworld.yaml create mode 100644 e2e/testdata/hpa/ns-helloworld.yaml create mode 100644 e2e/testdata/metrics-server/components-v0.7.2.yaml diff --git a/e2e/nomostest/client.go b/e2e/nomostest/client.go index 330fe45640..22b4325c70 100644 --- a/e2e/nomostest/client.go +++ b/e2e/nomostest/client.go @@ -22,6 +22,7 @@ import ( admissionv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -95,6 +96,7 @@ func newScheme(t testing.NTB) *runtime.Scheme { resourcegroupv1alpha1.SchemeBuilder.SchemeBuilder, apiregistrationv1.SchemeBuilder, hubv1.SchemeBuilder, + autoscalingv2.SchemeBuilder, } for _, b := range builders { err := b.AddToScheme(s) diff --git a/e2e/testcases/hpa_test.go b/e2e/testcases/hpa_test.go new file mode 100644 index 0000000000..b704646d2e --- /dev/null +++ b/e2e/testcases/hpa_test.go @@ -0,0 +1,104 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "kpt.dev/configsync/e2e/nomostest" + "kpt.dev/configsync/e2e/nomostest/ntopts" + nomostesting "kpt.dev/configsync/e2e/nomostest/testing" + "kpt.dev/configsync/e2e/nomostest/testwatcher" + "kpt.dev/configsync/pkg/kinds" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestHPA(t *testing.T) { + nt := nomostest.New(t, nomostesting.Reconciliation2, + ntopts.SyncWithGitSource(nomostest.DefaultRootSyncID, ntopts.Unstructured)) + rootSyncGitRepo := nt.SyncSourceGitReadWriteRepository(nomostest.DefaultRootSyncID) + + nt.T.Log("Checking if metrics-server is installed") + found := false + for _, ns := range []string{"kube-system", "metrics-system"} { + metricsServerObj := &appsv1.Deployment{} + if err := nt.KubeClient.Get("metrics-server", ns, metricsServerObj); err != nil { + if apierrors.IsNotFound(err) { + continue + } + nt.T.Fatal(err) + } + found = true + break + } + + if !found { + nt.T.Log("Deploy metrics-server Deployment & APIService") + nt.Must(rootSyncGitRepo.Copy(fmt.Sprintf("%s/metrics-server/components-v0.7.2.yaml", yamlDir), "acme/namespaces/metrics-system/components-v0.7.2.yaml")) + nt.Must(rootSyncGitRepo.CommitAndPush("Add metrics-server")) + nt.Must(nt.WatchForAllSyncs()) + + nt.T.Log("Wait for metrics-server Deployment to be available") + nt.Must(nt.Watcher.WatchForCurrentStatus(kinds.Deployment(), "metrics-server", "metrics-system")) + } + + nt.T.Log("Deploy hello-world Deployment") + nt.Must(rootSyncGitRepo.Copy(fmt.Sprintf("%s/hpa/ns-helloworld.yaml", yamlDir), "acme/namespaces/helloworld/ns-helloworld.yaml")) + nt.Must(rootSyncGitRepo.Copy(fmt.Sprintf("%s/hpa/deployment-helloworld.yaml", yamlDir), "acme/namespaces/helloworld/deployment-helloworld.yaml")) + nt.Must(rootSyncGitRepo.CommitAndPush("Add deployment")) + nt.Must(nt.WatchForAllSyncs()) + + nt.T.Log("Wait for hello-world Deployment to be available") + nt.Must(nt.Watcher.WatchForCurrentStatus(kinds.Deployment(), "hello-world", "hello-world")) + + nt.T.Log("Deploy hello-world HPA") + nt.Must(rootSyncGitRepo.Copy(fmt.Sprintf("%s/hpa/hpa-helloworld.yaml", yamlDir), "acme/namespaces/helloworld/hpa-helloworld.yaml")) + nt.Must(rootSyncGitRepo.CommitAndPush("Add HPA")) + nt.Must(nt.WatchForAllSyncs()) + + nt.T.Log("Wait for hello-world HPA to be available") + nt.Must(nt.Watcher.WatchForCurrentStatus(kinds.HorizontalPodAutoscaler(), "hello-world", "hello-world")) + + oldHPPObj := &autoscalingv2.HorizontalPodAutoscaler{} + nt.Must(nt.KubeClient.Get("hello-world", "hello-world", oldHPPObj)) + + metricsHaveChanged := func(obj client.Object) error { + newHPAObj := obj.(*autoscalingv2.HorizontalPodAutoscaler) + if !equality.Semantic.DeepEqual(oldHPPObj.Spec.Metrics, newHPAObj.Spec.Metrics) { + return errors.Errorf("metrics have changed:\n%s", + cmp.Diff(oldHPPObj.Spec.Metrics, newHPAObj.Spec.Metrics)) + } + return errors.New("metrics have not changed") + } + + // Once available, the spec.metrics should not change + err := nt.Watcher.WatchObject(kinds.HorizontalPodAutoscaler(), "hello-world", "hello-world", + testwatcher.WatchPredicates(metricsHaveChanged), + testwatcher.WatchTimeout(30*time.Second)) + if err == nil { + nt.T.Fatal("unexpected result: WatchObject with metricsHaveChanged should always error") + } else if !errors.Is(err, context.DeadlineExceeded) { + nt.T.Fatal(err) + } +} diff --git a/e2e/testdata/hpa/deployment-helloworld.yaml b/e2e/testdata/hpa/deployment-helloworld.yaml new file mode 100644 index 0000000000..4b6c33c508 --- /dev/null +++ b/e2e/testdata/hpa/deployment-helloworld.yaml @@ -0,0 +1,35 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello-world + namespace: hello-world +spec: + selector: + matchLabels: + app: hello-world + replicas: 1 + template: + metadata: + labels: + app: hello-world + spec: + containers: + - name: hello + image: "gcr.io/google-samples/hello-app:2.0" + env: + - name: "PORT" + value: "50000" diff --git a/e2e/testdata/hpa/hpa-helloworld.yaml b/e2e/testdata/hpa/hpa-helloworld.yaml new file mode 100644 index 0000000000..2c85355734 --- /dev/null +++ b/e2e/testdata/hpa/hpa-helloworld.yaml @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: hello-world + namespace: hello-world +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: hello-world + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 diff --git a/e2e/testdata/hpa/ns-helloworld.yaml b/e2e/testdata/hpa/ns-helloworld.yaml new file mode 100644 index 0000000000..6f2a6345dd --- /dev/null +++ b/e2e/testdata/hpa/ns-helloworld.yaml @@ -0,0 +1,18 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: hello-world diff --git a/e2e/testdata/metrics-server/components-v0.7.2.yaml b/e2e/testdata/metrics-server/components-v0.7.2.yaml new file mode 100644 index 0000000000..4d91f85d9f --- /dev/null +++ b/e2e/testdata/metrics-server/components-v0.7.2.yaml @@ -0,0 +1,233 @@ +# Copyright The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Source: https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.2/components.yaml +# +# Modifications: +# - Add depends-on annotation APIService -> Deployment +# - Change namespace "kube-system" -> "metrics-system" +# - Add namespace "metrics-system" +# - Add --kubelet-insecure-tls + +apiVersion: v1 +kind: Namespace +metadata: + labels: + k8s-app: metrics-server + name: metrics-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: metrics-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader +rules: +- apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get +- apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system # must be kube-system, because the location of the extension-apiserver-authentication ConfigMap is not configurable +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: metrics-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: metrics-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: metrics-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: metrics-system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: metrics-system +spec: + selector: + matchLabels: + k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=10250 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --kubelet-insecure-tls + image: registry.k8s.io/metrics-server/metrics-server:v0.7.2 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 10250 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + volumes: + - emptyDir: {} + name: tmp-dir +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + annotations: + config.kubernetes.io/depends-on: apps/namespaces/metrics-system/Deployment/metrics-server + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: metrics-system + version: v1beta1 + versionPriority: 100 diff --git a/pkg/kinds/kinds.go b/pkg/kinds/kinds.go index 20ddfb8123..7ef983d22b 100644 --- a/pkg/kinds/kinds.go +++ b/pkg/kinds/kinds.go @@ -18,6 +18,7 @@ import ( kptfilev1 "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1" admissionv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -268,3 +269,8 @@ func APIService() schema.GroupVersionKind { func ValidatingWebhookConfiguration() schema.GroupVersionKind { return admissionv1.SchemeGroupVersion.WithKind("ValidatingWebhookConfiguration") } + +// HorizontalPodAutoscaler returns the canonical HorizontalPodAutoscaler GroupVersionKind. +func HorizontalPodAutoscaler() schema.GroupVersionKind { + return autoscalingv2.SchemeGroupVersion.WithKind("HorizontalPodAutoscaler") +}