diff --git a/balancer/Dockerfile b/balancer/Dockerfile new file mode 100644 index 000000000000..8813d320418a --- /dev/null +++ b/balancer/Dockerfile @@ -0,0 +1,6 @@ +FROM gcr.io/distroless/static:latest +MAINTAINER Marcin Wielgus "mwielgus@google.com" + +COPY balancer / + +ENTRYPOINT ["/balancer"] diff --git a/balancer/Makefile b/balancer/Makefile new file mode 100644 index 000000000000..5a8a144c3fde --- /dev/null +++ b/balancer/Makefile @@ -0,0 +1,59 @@ +all: build + +FLAGS= +OFFICIAL_NAME=balancer + +build: clean + go build . + +build-linux-amd64: clean + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build . + +test-unit: clean + go test --test.short -race ./... $(FLAGS) + +docker-build: +ifndef REGISTRY + ERR = $(error REGISTRY is undefined) + $(ERR) +endif +ifndef TAG + ERR = $(error TAG is undefined) + $(ERR) +endif + docker build --pull -t ${REGISTRY}/${OFFICIAL_NAME}:${TAG} . + +docker-push: +ifndef REGISTRY + ERR = $(error REGISTRY is undefined) + $(ERR) +endif +ifndef TAG + ERR = $(error TAG is undefined) + $(ERR) +endif + docker push ${REGISTRY}/${OFFICIAL_NAME}:${TAG} + +docker-builder: + docker build -t vpa-autoscaling-builder ../vertical-pod-autoscaler/builder + +build-in-docker: clean docker-builder + docker run -v `pwd`/..:/gopath/src/k8s.io/autoscaler vpa-autoscaling-builder:latest bash -c 'cd /gopath/src/k8s.io/autoscaler/balancer && make build-linux-amd64' + +build-image-in-docker: build-in-docker docker-build + +test-in-docker: build-in-docker + docker run -v `pwd`/..:/gopath/src/k8s.io/autoscaler vpa-autoscaling-builder:latest bash -c 'cd /gopath/src/k8s.io/autoscaler/balancer && make test-unit' + +release: build-image-in-docker docker-push + @echo "Full in-docker release ${OFFICIAL_NAME}:${TAG} completed" + +clean: + rm -f balancer + +format: + test -z "$$(find . -path ./vendor -prune -type f -o -name '*.go' -exec gofmt -s -d {} + | tee /dev/stderr)" || \ + test -z "$$(find . -path ./vendor -prune -type f -o -name '*.go' -exec gofmt -s -w {} + | tee /dev/stderr)" + +.PHONY: all build test-unit clean format release + diff --git a/balancer/deploy/controller.yaml b/balancer/deploy/controller.yaml new file mode 100644 index 000000000000..9dfc079ee0fe --- /dev/null +++ b/balancer/deploy/controller.yaml @@ -0,0 +1,89 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: balancer-controller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: balancer-controller + namespace: kube-system +rules: + - apiGroups: + - balancer.x-k8s.io + resources: + - balancers + verbs: + - get + - list + - watch + - patch + - update + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - deployments/scale + verbs: + - get + - list + - watch + - patch + - update + - apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: balancer-controller + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: balancer-controller +subjects: + - kind: ServiceAccount + name: balancer-controller + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: balancer-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: balancer-controller + template: + metadata: + labels: + app: balancer-controller + spec: + serviceAccountName: balancer-controller + containers: + - name: controller + image: gcr.io/gke-autoscaling-gcr/balancer:0.1.1 + imagePullPolicy: Always + args: ["-v","4"] + resources: + requests: + cpu: 100m diff --git a/balancer/deploy/crd.yaml b/balancer/deploy/crd.yaml new file mode 100644 index 000000000000..d6a971081e56 --- /dev/null +++ b/balancer/deploy/crd.yaml @@ -0,0 +1,320 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + creationTimestamp: null + name: balancers.balancer.x-k8s.io +spec: + group: balancer.x-k8s.io + names: + kind: Balancer + listKind: BalancerList + plural: balancers + singular: balancer + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Balancer is an object used to automatically keep the desired + number of replicas (pods) distributed among the specified set of targets + (deployments or other objects that expose the Scale subresource). + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'Specification of the Balancer behavior. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status.' + properties: + policy: + description: Policy defines how the balancer should distribute replicas + among targets. + properties: + fallback: + description: Fallback contains specification of how to recognize + and what to do if some replicas fail to start in one or more + targets. No fallback happens if not-set. + properties: + startupTimeoutSeconds: + description: StartupTimeoutSeconds defines how long will the + Balancer wait before considering a pending/not-started pod + as blocked and starting another replica in some other target. + Once the replica is finally started, replicas in other targets + may be stopped. + format: int32 + minimum: 0 + type: integer + required: + - startupTimeoutSeconds + type: object + policyName: + description: PolicyName decides how to balance replicas across + the targets. Depending on the name one of the fields Priorities + or Proportions must be set. + type: string + priorities: + description: Priorities contains detailed specification of how + to balance when balancer policy name is set to Priority. + properties: + targetOrder: + description: TargetOrder is the priority-based list of Balancer + targets names. The first target on the list gets the replicas + until its maxReplicas is reached (or replicas fail to start). + Then the replicas go to the second target and so on. MinReplicas + is guaranteed to be fulfilled, irrespective of the order, + presence on the list, and/or total Balancer's replica count. + items: + type: string + minItems: 2 + type: array + required: + - targetOrder + type: object + proportions: + description: Proportions contains detailed specification of how + to balance when balancer policy name is set to Proportional. + properties: + targetProportions: + additionalProperties: + format: int32 + type: integer + description: TargetProportions is a map from Balancer targets + names to rates. Replicas are distributed so that the max + difference between the current replica share and the desired + replica share is minimized. Once a target reaches maxReplicas + it is removed from the calculations and replicas are distributed + with the updated proportions. MinReplicas is guaranteed + for a target, irrespective of the total Balancer's replica + count, proportions or the presence in the map. + minProperties: 2 + type: object + required: + - targetProportions + type: object + required: + - policyName + type: object + replicas: + description: Replicas is the number of pods that should be distributed + among the declared targets according to the specified policy. + format: int32 + minimum: 0 + type: integer + selector: + description: Selector that groups the pods from all targets together + (and only those). Ideally it should match the selector used by the + Service built on top of the Balancer. All pods selectable by targets' + selector must match to this selector, however target's selector + don't have to be a superset of this one (although it is recommended). + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + targets: + description: Targets is a list of targets between which Balancer tries + to distribute replicas. + items: + description: BalancerTarget is the declaration of one of the targets + between which the balancer tries to distribute replicas. + properties: + maxReplicas: + description: MaxReplicas is the maximum number of replicas inside + of this target. Balancer will set at most this amount on the + target, even if the total desired number of replicas for the + Balancer is higher. There will be no limit if not provided. + format: int32 + minimum: 0 + type: integer + minReplicas: + description: MinReplicas is the minimum number of replicas inside + of this target. Balancer will set at least this amount on + the target, even if the total desired number of replicas for + Balancer is lower. 0 will be used (no min) if not provided. + format: int32 + minimum: 0 + type: integer + name: + description: Name of the target. + minLength: 1 + type: string + scaleTargetRef: + description: ScaleTargetRef is a reference that points to a + target resource to balance. The target needs to expose the + Scale subresource. + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - kind + - name + type: object + required: + - name + - scaleTargetRef + type: object + minItems: 2 + type: array + required: + - policy + - replicas + - selector + - targets + type: object + status: + description: Current information about the Balancer. + properties: + conditions: + description: Conditions is the set of conditions required for this + Balancer to work properly, and indicates whether or not those conditions + are met. + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + replicas: + description: Replicas is an actual number of observed pods matching + Balancer selector. + format: int32 + type: integer + selector: + description: 'Selector is a query over pods that should match the + replicas count. This is same as the label selector but in the string + format to avoid introspection by clients. The string will be in + the same format as the query-param syntax. More info about label + selectors: http://kubernetes.io/docs/user-guide/labels#label-selectors' + type: string + required: + - replicas + - selector + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: {} diff --git a/balancer/examples/nginx-priority-hpa.yaml b/balancer/examples/nginx-priority-hpa.yaml new file mode 100644 index 000000000000..abcada74e1f2 --- /dev/null +++ b/balancer/examples/nginx-priority-hpa.yaml @@ -0,0 +1,119 @@ +# +# Balancer scaling 2 deployments using priority policy and hpa. +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-1 + labels: + app: nginx-1 + srv: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx-1 + srv: nginx + template: + metadata: + labels: + app: nginx-1 + srv: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + resources: + requests: + cpu: 100m +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-2 + labels: + app: nginx-2 + srv: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx-2 + srv: nginx + template: + metadata: + labels: + app: nginx-2 + srv: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + resources: + requests: + cpu: 100m +--- +apiVersion: balancer.x-k8s.io/v1alpha1 +kind: Balancer +metadata: + name: nginx +spec: + replicas: 5 + selector: + matchLabels: + srv: nginx + policy: + policyName: priority + priorities: + targetOrder: [nginx-1,nginx-2] + fallback: + startupTimeoutSeconds: 180 + targets: + - name: nginx-1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx-1 + minReplicas: 1 + maxReplicas: 7 + - name: nginx-2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx-2 + minReplicas: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + srv: nginx +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: nginx +spec: + minReplicas: 2 + maxReplicas: 10 + metrics: + - resource: + name: cpu + target: + averageUtilization: 50 + type: Utilization + type: Resource + scaleTargetRef: + apiVersion: balancer.x-k8s.io/v1alpha1 + kind: Balancer + name: nginx diff --git a/balancer/examples/nginx-priority.yaml b/balancer/examples/nginx-priority.yaml new file mode 100644 index 000000000000..0fde69f35802 --- /dev/null +++ b/balancer/examples/nginx-priority.yaml @@ -0,0 +1,94 @@ +# +# Balancer scaling 2 deployments using priority policy. +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-1 + labels: + app: nginx-1 + srv: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx-1 + srv: nginx + template: + metadata: + labels: + app: nginx-1 + srv: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-2 + labels: + app: nginx-2 + srv: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx-2 + srv: nginx + template: + metadata: + labels: + app: nginx-2 + srv: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 +--- +apiVersion: balancer.x-k8s.io/v1alpha1 +kind: Balancer +metadata: + name: nginx +spec: + replicas: 5 + selector: + matchLabels: + srv: nginx + policy: + policyName: priority + priorities: + targetOrder: [nginx-1,nginx-2] + fallback: + startupTimeoutSeconds: 180 + targets: + - name: nginx-1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx-1 + minReplicas: 1 + maxReplicas: 7 + - name: nginx-2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx-2 + minReplicas: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + srv: nginx diff --git a/balancer/examples/nginx-proportional.yaml b/balancer/examples/nginx-proportional.yaml new file mode 100644 index 000000000000..543f395162cf --- /dev/null +++ b/balancer/examples/nginx-proportional.yaml @@ -0,0 +1,96 @@ +# +# Balancer scaling 2 deployments using 50/50 proportional policy. +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-1 + labels: + app: nginx-1 + srv: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx-1 + srv: nginx + template: + metadata: + labels: + app: nginx-1 + srv: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-2 + labels: + app: nginx-2 + srv: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx-2 + srv: nginx + template: + metadata: + labels: + app: nginx-2 + srv: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 +--- +apiVersion: balancer.x-k8s.io/v1alpha1 +kind: Balancer +metadata: + name: nginx +spec: + replicas: 10 + selector: + matchLabels: + srv: nginx + policy: + policyName: proportional + proportions: + targetProportions: + nginx-1: 50 + nginx-2: 50 + fallback: + startupTimeoutSeconds: 180 + targets: + - name: nginx-1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx-1 + minReplicas: 1 + maxReplicas: 7 + - name: nginx-2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx-2 + minReplicas: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + srv: nginx diff --git a/balancer/main.go b/balancer/main.go new file mode 100644 index 000000000000..38530e528781 --- /dev/null +++ b/balancer/main.go @@ -0,0 +1,109 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "flag" + "k8s.io/klog/v2" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + balancerclientset "k8s.io/autoscaler/balancer/pkg/client/clientset/versioned" + balancerinformers "k8s.io/autoscaler/balancer/pkg/client/informers/externalversions" + "k8s.io/autoscaler/balancer/pkg/controller" + cacheddiscovery "k8s.io/client-go/discovery/cached" + "k8s.io/client-go/dynamic" + kubeinformers "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/restmapper" + scaleclient "k8s.io/client-go/scale" + "k8s.io/client-go/tools/clientcmd" +) + +var ( + masterURL string + kubeconfig string + balancerReprocessPeriodSec int + concurrency int +) + +const ( + defaultResyncPeriod = 60 * time.Second + mapperResetPeriod = 30 * time.Second +) + +func init() { + flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") + flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.") + flag.IntVar(&balancerReprocessPeriodSec, "reprocess-period-sec", 15, "How often (in second) balancers are processed") + flag.IntVar(&concurrency, "concurrency", 3, "How many balancers can be processed in parallel") +} + +func main() { + klog.InitFlags(nil) + flag.Parse() + + // set up signals so we handle the first shutdown signal gracefully + // TODO: handle sigints + stopCh := make(chan struct{}) + + cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) + if err != nil { + klog.Fatalf("Error building kubeconfig: %s", err.Error()) + } + klog.V(1).Infof("Starting Balancer for %v", cfg) + + kubeClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + klog.Fatalf("Error building kubernetes clientset: %s", err.Error()) + } + + balancerClient, err := balancerclientset.NewForConfig(cfg) + if err != nil { + klog.Fatalf("Error building Balancer clientset: %s", err.Error()) + } + + kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeClient, defaultResyncPeriod) + balancerInformerFactory := balancerinformers.NewSharedInformerFactory(balancerClient, defaultResyncPeriod) + + cachedClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery()) + restMapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedClient) + + // Synchronize mapper until stopCh is closed. + go wait.Until(func() { + restMapper.Reset() + }, mapperResetPeriod, stopCh) + + scaleKindResolver := scaleclient.NewDiscoveryScaleKindResolver(kubeClient.Discovery()) + scaleClient, err := scaleclient.NewForConfig(cfg, restMapper, dynamic.LegacyAPIPathResolverFunc, scaleKindResolver) + + podInformer := kubeInformerFactory.Core().V1().Pods() + core := controller.NewCore(controller.NewScaleClient(context.TODO(), scaleClient, restMapper), podInformer) + + controller := controller.NewController(balancerClient, + balancerInformerFactory.Balancer().V1alpha1().Balancers(), + kubeClient.CoreV1().Events(""), + core, + time.Duration(balancerReprocessPeriodSec)*time.Second) + + // Start method is non-blocking and runs all registered informers in a dedicated goroutine. + kubeInformerFactory.Start(stopCh) + balancerInformerFactory.Start(stopCh) + + controller.Run(concurrency, stopCh) +} diff --git a/balancer/pkg/apis/balancer.x-k8s.io/v1alpha1/zz_generated.deepcopy.go b/balancer/pkg/apis/balancer.x-k8s.io/v1alpha1/zz_generated.deepcopy.go index de3ccee97d7b..e7ef76cbd44d 100644 --- a/balancer/pkg/apis/balancer.x-k8s.io/v1alpha1/zz_generated.deepcopy.go +++ b/balancer/pkg/apis/balancer.x-k8s.io/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,6 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + /* Copyright The Kubernetes Authors. @@ -14,9 +17,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - // Code generated by deepcopy-gen. DO NOT EDIT. package v1alpha1