From 44a8887b902a4c505ee4dd94123b1b60d71351ac Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Tue, 24 Oct 2023 16:47:48 +0300 Subject: [PATCH] deployment/helm: initial chart for balloons policy. Signed-off-by: Krisztian Litkey --- .../crds/config.nri_balloonsconfigs.yaml | 334 ++++++++++++++++++ .../helm/balloons/templates/clusterrole.yaml | 11 + .../helm/balloons/templates/config.yaml | 11 + .../helm/balloons/templates/configmap.yaml | 10 - .../helm/balloons/templates/daemonset.yaml | 7 - deployment/helm/balloons/values.yaml | 33 +- 6 files changed, 385 insertions(+), 21 deletions(-) create mode 100644 deployment/helm/balloons/crds/config.nri_balloonsconfigs.yaml create mode 100644 deployment/helm/balloons/templates/config.yaml delete mode 100644 deployment/helm/balloons/templates/configmap.yaml diff --git a/deployment/helm/balloons/crds/config.nri_balloonsconfigs.yaml b/deployment/helm/balloons/crds/config.nri_balloonsconfigs.yaml new file mode 100644 index 000000000..ee25e171f --- /dev/null +++ b/deployment/helm/balloons/crds/config.nri_balloonsconfigs.yaml @@ -0,0 +1,334 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: balloonsconfigs.config.nri +spec: + group: config.nri + names: + kind: BalloonsConfig + listKind: BalloonsConfigList + plural: balloonsconfigs + singular: balloonsconfig + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: BalloonsConfig represents the configuration for the balloons + policy. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + balloons: + properties: + PreferSpreadOnPhysicalCores: + description: 'PreferSpreadOnPhysicalCores prefers allocating logical + CPUs (possibly hyperthreads) for a balloon from separate physical + CPU cores. This prevents workloads in the balloon from interfering + with themselves as they do not compete on the resources of the same + CPU cores. On the other hand, it allows more interference between + workloads in different balloons. The default is false: balloons + are packed tightly to a minimum number of physical CPU cores. The + value set here is the default for all balloon types, but it can + be overridden with the balloon type specific setting with the same + name.' + type: boolean + allocatorTopologyBalancing: + description: 'If AllocatorTopologyBalancing is true, balloons are + allocated and resized so that all topology elements (packages, dies, + numa nodes, cores) have roughly same amount of allocations. The + default is false: balloons are packed tightly to optimize power + efficiency. The value set here can be overridden with the balloon + type specific setting with the same name.' + type: boolean + availableResources: + additionalProperties: + type: string + description: Available/allowed (CPU) resources to use. + type: object + balloonTypes: + description: BallonDefs contains balloon type definitions. + items: + description: BalloonDef contains a balloon definition. + properties: + AllocatorTopologyBalancing: + description: AllocatorTopologyBalancing is the balloon type + specific parameter of the policy level parameter with the + same name. + type: boolean + PreferSpreadOnPhysicalCores: + description: PreferSpreadOnPhysicalCores is the balloon type + specific parameter of the policy level parameter with the + same name. + type: boolean + allocatorPriority: + default: high + description: AllocatorPriority (High, Normal, Low, None) This + parameter is passed to CPU allocator when creating or resizing + a balloon. At init, balloons with highest priority CPUs are + allocated first. + enum: + - high + - normal + - low + - none + type: string + cpuClass: + description: CpuClass controls how CPUs of a balloon are (re)configured + whenever a balloon is created, inflated or deflated. + type: string + maxBalloons: + description: MaxBalloons is the maximum number of balloon instances + that is allowed to co-exist. If reached, new balloons cannot + be created anymore. + type: integer + maxCPUs: + description: MaxCpus specifies the maximum number of CPUs exclusively + usable by containers in a balloon. Balloon size will not be + inflated larger than MaxCpus. + type: integer + minBalloons: + description: MinBalloons is the number of balloon instances + that always exist even if they would become empty. At init + this number of instances will be created before assigning + any containers. + type: integer + minCPUs: + description: MinCpus specifies the minimum number of CPUs exclusively + usable by containers in a balloon. When new balloon is created, + this will be the number of CPUs reserved for it even if a + container would request less. + type: integer + name: + description: Name of the balloon definition. + type: string + namespaces: + description: Namespaces control which namespaces are assigned + into balloon instances from this definition. This is used + by namespace assign methods. + items: + type: string + type: array + preferNewBalloons: + description: 'PreferNewBalloons: prefer creating new balloons + over adding containers to existing balloons. The default is + false: prefer using filling free capacity and possibly inflating + existing balloons before creating new ones.' + type: boolean + preferPerNamespaceBalloon: + description: 'PreferPerNamespaceBalloon: if true, containers + in different namespaces are preferrably placed in separate + balloons, even if the balloon type is the same for all of + them. On the other hand, containers in the same namespace + will be placed in the same balloon instances. The default + is false: namespaces have no effect on placement.' + type: boolean + preferSpreadingPods: + description: 'PreferSpreadingPods: containers of the same pod + may be placed on separate balloons. The default is false: + prefer placing containers of a pod to the same balloon(s).' + type: boolean + shareIdleCPUsInSame: + description: 'ShareIdleCpusInSame : if there + are idle CPUs, that is CPUs not in any balloon, in the same + as any CPU in the balloon, then allow workloads + to run on those (shared) CPUs in addition to the (dedicated) + CPUs of the balloon.' + enum: + - "" + - system + - package + - die + - numa + - core + - thread + type: string + required: + - allocatorPriority + - minCPUs + - name + - namespaces + type: object + type: array + idleCPUClass: + description: IdleCpuClass controls how unusded CPUs outside any a + balloons are (re)configured. + type: string + pinCPU: + default: true + description: PinCPU controls pinning containers to CPUs. + type: boolean + pinMemory: + default: true + description: PinMemory controls pinning containers to memory nodes. + type: boolean + reservedPoolNamespaces: + description: ReservedPoolNamespaces is a list of namespace globs that + will be allocated to reserved CPUs. + items: + type: string + type: array + reservedResources: + additionalProperties: + type: string + description: Reserved (CPU) resources for kube-system namespace. + type: object + required: + - allocatorTopologyBalancing + - idleCPUClass + - reservedResources + type: object + common: + properties: + control: + properties: + cpu: + properties: + classes: + additionalProperties: + properties: + energyPerformancePreference: + type: integer + maxFreq: + type: integer + minFreq: + type: integer + uncoreMaxFreq: + type: integer + uncoreMinFreq: + type: integer + required: + - energyPerformancePreference + - maxFreq + - minFreq + - uncoreMaxFreq + - uncoreMinFreq + type: object + type: object + required: + - classes + type: object + enable: + items: + type: string + type: array + type: object + instrumentation: + description: Config is the runtime configuration for instrumentation. + properties: + httpEndpoint: + description: HTTPEndpoint, used among other things to expose metrics + to Prometheus. + type: string + prometheusExport: + description: PrometheusExport enables /metrics exporting for Prometheus. + type: boolean + reportPeriod: + description: ReportPeriod for our metrics view. + format: duration + type: string + samplingRatePerMillion: + description: Number of samples to collect per million spans. + type: integer + tracingCollector: + description: TracingCollector is the endpoint for tracing data + collection. + type: string + type: object + log: + properties: + debug: + additionalProperties: + type: boolean + description: Debug controls which log sources produce debug messages. + type: object + klog: + description: Klog configures klog-specific options. + properties: + add_dir_header: + type: boolean + alsologtostderr: + type: boolean + log_backtrace_at: + type: string + log_dir: + type: string + log_file: + type: string + log_file_max_size: + format: int64 + type: integer + logtostderr: + type: boolean + one_output: + type: boolean + skip_headers: + type: boolean + skip_log_headers: + type: boolean + stderrthreshold: + type: string + v: + type: integer + vmodule: + type: string + type: object + logSource: + description: LogSource controls if messages are prefixed with + the logger source. + type: boolean + type: object + type: object + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + properties: + nodes: + additionalProperties: + properties: + errors: + description: Error can provide further details of a configuration + error. + type: string + generation: + description: Generation is the generation the configuration + this status was set for. + format: int64 + type: integer + status: + description: Status of activating the configuration on this + node. + enum: + - Success + - Failure + type: string + timestamp: + description: Last time of success/failure. + format: date-time + type: string + required: + - generation + - status + type: object + type: object + required: + - nodes + type: object + required: + - balloons + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deployment/helm/balloons/templates/clusterrole.yaml b/deployment/helm/balloons/templates/clusterrole.yaml index a657cf620..dfe8773d7 100644 --- a/deployment/helm/balloons/templates/clusterrole.yaml +++ b/deployment/helm/balloons/templates/clusterrole.yaml @@ -22,3 +22,14 @@ rules: - list - update - delete +- apiGroups: + - config.nri + resources: + - balloonsconfigs + verbs: + - create + - get + - list + - watch + - update + - delete diff --git a/deployment/helm/balloons/templates/config.yaml b/deployment/helm/balloons/templates/config.yaml new file mode 100644 index 000000000..60519698e --- /dev/null +++ b/deployment/helm/balloons/templates/config.yaml @@ -0,0 +1,11 @@ +apiVersion: config.nri/v1alpha1 +kind: BalloonsConfig +metadata: + name: default + namespace: {{ .Release.Namespace }} + labels: + {{- include "balloons-plugin.labels" . | nindent 4 }} +balloons: + {{- toYaml .Values.policyConfig | nindent 2 }} +common: + {{- toYaml .Values.commonConfig | nindent 2 }} diff --git a/deployment/helm/balloons/templates/configmap.yaml b/deployment/helm/balloons/templates/configmap.yaml deleted file mode 100644 index b3b70d3a8..000000000 --- a/deployment/helm/balloons/templates/configmap.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: nri-resource-policy-config.default - namespace: {{ .Release.Namespace }} - labels: - {{- include "balloons-plugin.labels" . | nindent 4 }} -data: - nri-resource-policy.cfg: | - {{- toYaml .Values.config | nindent 4 }} diff --git a/deployment/helm/balloons/templates/daemonset.yaml b/deployment/helm/balloons/templates/daemonset.yaml index 7697a328f..58bd8c34e 100644 --- a/deployment/helm/balloons/templates/daemonset.yaml +++ b/deployment/helm/balloons/templates/daemonset.yaml @@ -41,8 +41,6 @@ spec: args: - --host-root - /host - - --fallback-config - - /etc/nri-resource-policy/nri-resource-policy.cfg - --pid-file - /tmp/nri-resource-policy.pid - -metrics-interval @@ -74,8 +72,6 @@ spec: mountPath: /host/sys - name: resource-policysockets mountPath: /var/run/nri-resource-policy - - name: resource-policyconfig - mountPath: /etc/nri-resource-policy - name: nrisockets mountPath: /var/run/nri volumes: @@ -90,9 +86,6 @@ spec: - name: resource-policysockets hostPath: path: /var/run/nri-resource-policy - - name: resource-policyconfig - configMap: - name: nri-resource-policy-config.default - name: nrisockets hostPath: path: /var/run/nri diff --git a/deployment/helm/balloons/values.yaml b/deployment/helm/balloons/values.yaml index 2524dbf89..f7c695b4d 100644 --- a/deployment/helm/balloons/values.yaml +++ b/deployment/helm/balloons/values.yaml @@ -8,10 +8,35 @@ image: #tag: unstable pullPolicy: Always -config: - policy: - ReservedResources: - cpu: 750m +policyConfig: + reservedResources: + cpu: 750m + idleCPUClass: normal + allocatorTopologyBalancing: true + balloonTypes: + - name: default + namespaces: + - default + minCPUs: 1 + minBalloons: 0 + allocatorPriority: normal + shareIdleCPUsInSame: system + reservedPoolNamespaces: + - kube-system + +commonConfig: + control: + enable: + - cpu + log: + debug: + "*": false + logSource: true + klog: + skip_headers: true + instrumentation: + reportPeriod: 60s + samplingRatePerMillion: 0 hostPort: 8891