From b134ab1f97fcc644dcbd62b42df5a6a6aad76eea Mon Sep 17 00:00:00 2001 From: Antti Kervinen Date: Mon, 30 Sep 2024 12:43:19 +0300 Subject: [PATCH 1/3] balloons: add the preserve option to policy configuration The preserve option enables specifying containers whose resource pinning must not be modified in policy configuration. The meaning is the same as with the preserve annotation for both CPU and memory. Signed-off-by: Antti Kervinen --- .../balloons/policy/balloons-policy.go | 11 +++++ .../bases/config.nri_balloonspolicies.yaml | 48 +++++++++++++++++++ .../crds/config.nri_balloonspolicies.yaml | 48 +++++++++++++++++++ .../v1alpha1/resmgr/policy/balloons/config.go | 27 +++++++++++ .../policy/balloons/zz_generated.deepcopy.go | 27 +++++++++++ 5 files changed, 161 insertions(+) diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index a5e4415f3..532eadfb0 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -227,6 +227,17 @@ func (p *balloons) AllocateResources(c cache.Container) error { log.Infof("not handling resources of container %s, preserving CPUs %q and memory %q", c.PrettyName(), c.GetCpusetCpus(), c.GetCpusetMems()) return nil } + + if p.bpoptions.Preserve != nil { + rule, err := p.bpoptions.Preserve.MatchContainer(c) + if err != nil { + log.Errorf("error in matching container %s to preserve conditions: %s", c, err) + } else if rule != "" { + log.Debugf("preserve container %s due to matching %s", c, rule) + return nil + } + } + log.Debug("allocating resources for container %s (request %d mCPU, limit %d mCPU)...", c.PrettyName(), p.containerRequestedMilliCpus(c.GetID()), diff --git a/config/crd/bases/config.nri_balloonspolicies.yaml b/config/crd/bases/config.nri_balloonspolicies.yaml index 16f1d3872..f672f734f 100644 --- a/config/crd/bases/config.nri_balloonspolicies.yaml +++ b/config/crd/bases/config.nri_balloonspolicies.yaml @@ -393,6 +393,54 @@ spec: overridden with the balloon type specific setting with the same name. type: boolean + preserve: + description: |- + Preserve specifies containers whose resource pinning must not be + modified by the policy. + properties: + matchExpressions: + description: MatchExpressions specifies one or more expressions. + items: + description: |- + Expression describes some runtime-evaluated condition. An expression + consist of a key, an operator and a set of values. An expressions is + evaluated against an object which implements the Evaluable interface. + Evaluating an expression consists of looking up the value for the key + in the object, then using the operator to check it agains the values + of the expression. The result is a single boolean value. An object is + said to satisfy the evaluated expression if this value is true. An + expression can contain 0, 1 or more values depending on the operator. + properties: + key: + description: Key is the expression key. + type: string + operator: + description: Op is the expression operator. + enum: + - Equals + - NotEqual + - In + - NotIn + - Exists + - NotExist + - AlwaysTrue + - Matches + - MatchesNot + - MatchesAny + - MatchesNone + type: string + values: + description: Values contains the values the key value is + evaluated against. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object reservedPoolNamespaces: description: |- ReservedPoolNamespaces is a list of namespace globs that diff --git a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml index 16f1d3872..f672f734f 100644 --- a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml +++ b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml @@ -393,6 +393,54 @@ spec: overridden with the balloon type specific setting with the same name. type: boolean + preserve: + description: |- + Preserve specifies containers whose resource pinning must not be + modified by the policy. + properties: + matchExpressions: + description: MatchExpressions specifies one or more expressions. + items: + description: |- + Expression describes some runtime-evaluated condition. An expression + consist of a key, an operator and a set of values. An expressions is + evaluated against an object which implements the Evaluable interface. + Evaluating an expression consists of looking up the value for the key + in the object, then using the operator to check it agains the values + of the expression. The result is a single boolean value. An object is + said to satisfy the evaluated expression if this value is true. An + expression can contain 0, 1 or more values depending on the operator. + properties: + key: + description: Key is the expression key. + type: string + operator: + description: Op is the expression operator. + enum: + - Equals + - NotEqual + - In + - NotIn + - Exists + - NotExist + - AlwaysTrue + - Matches + - MatchesNot + - MatchesAny + - MatchesNone + type: string + values: + description: Values contains the values the key value is + evaluated against. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object reservedPoolNamespaces: description: |- ReservedPoolNamespaces is a list of namespace globs that diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go index 04b582e63..31414a3dc 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go @@ -21,6 +21,7 @@ import ( policy "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" resmgr "github.com/containers/nri-plugins/pkg/apis/resmgr/v1alpha1" "github.com/containers/nri-plugins/pkg/cpuallocator" + "github.com/containers/nri-plugins/pkg/resmgr/cache" ) type ( @@ -78,6 +79,9 @@ type Config struct { // Reserved (CPU) resources for kube-system namespace. // +kubebuilder:validation:Required ReservedResources Constraints `json:"reservedResources"` + // Preserve specifies containers whose resource pinning must not be + // modified by the policy. + Preserve *ContainerMatchConfig `json:"preserve,omitempty"` } type CPUTopologyLevel string @@ -252,8 +256,31 @@ func (p CPUPriority) Value() cpuallocator.CPUPriority { return cpuallocator.PriorityNone } +// ContainerMatchConfig contains container matching configurations. +// +k8s:deepcopy-gen=true +type ContainerMatchConfig struct { + // MatchExpressions specifies one or more expressions. + MatchExpressions []resmgr.Expression `json:"matchExpressions,omitempty"` +} + +func (cmc *ContainerMatchConfig) MatchContainer(c cache.Container) (string, error) { + for _, expr := range cmc.MatchExpressions { + if expr.Evaluate(c) { + return expr.String(), nil + } + } + return "", nil +} + func (c *Config) Validate() error { errs := []error{} + if c.Preserve != nil { + for _, expr := range c.Preserve.MatchExpressions { + if err := expr.Validate(); err != nil { + errs = append(errs, err) + } + } + } for _, blnDef := range c.BalloonDefs { for _, expr := range blnDef.MatchExpressions { if err := expr.Validate(); err != nil { diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go index ed78429e7..38cc957ac 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go @@ -118,6 +118,11 @@ func (in *Config) DeepCopyInto(out *Config) { (*out)[key] = val } } + if in.Preserve != nil { + in, out := &in.Preserve, &out.Preserve + *out = new(ContainerMatchConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Config. @@ -129,3 +134,25 @@ func (in *Config) DeepCopy() *Config { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContainerMatchConfig) DeepCopyInto(out *ContainerMatchConfig) { + *out = *in + if in.MatchExpressions != nil { + in, out := &in.MatchExpressions, &out.MatchExpressions + *out = make([]v1alpha1.Expression, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerMatchConfig. +func (in *ContainerMatchConfig) DeepCopy() *ContainerMatchConfig { + if in == nil { + return nil + } + out := new(ContainerMatchConfig) + in.DeepCopyInto(out) + return out +} From cdcfe549bef976dca823445cbca0527fafee53e0 Mon Sep 17 00:00:00 2001 From: Antti Kervinen Date: Mon, 30 Sep 2024 12:43:57 +0300 Subject: [PATCH 2/3] e2e: add a test for "preserve" in the balloons configuration Signed-off-by: Antti Kervinen --- test/e2e/policies.test-suite/balloons/match-config.yaml | 9 +++++++++ .../balloons/n4c16/test11-match-expression/code.var.sh | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/test/e2e/policies.test-suite/balloons/match-config.yaml b/test/e2e/policies.test-suite/balloons/match-config.yaml index 29d68649e..8ec1c1032 100644 --- a/test/e2e/policies.test-suite/balloons/match-config.yaml +++ b/test/e2e/policies.test-suite/balloons/match-config.yaml @@ -5,6 +5,15 @@ config: pinMemory: true idleCPUClass: normal allocatorTopologyBalancing: true + preserve: + matchExpressions: + - key: name + operator: In + values: + - pod1cX + - pod2c1 + - pod2c2 + - pod3cX balloonTypes: - name: special matchExpressions: diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh index 70b065532..551a6b7eb 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh @@ -21,4 +21,13 @@ CONTCOUNT=1 create balloons-busybox report allowed verify 'len(cpus["pod1c0"]) == 1' +# pod2: run ordinary workload where pod2c1 and pod2c2 match the +# preserve matchexpression in policy configuration. +CONTCOUNT=4 create balloons-busybox +report allowed +verify 'len(cpus["pod2c0"]) == 1' \ + 'len(cpus["pod2c1"]) == 16' \ + 'len(cpus["pod2c2"]) == 16' \ + 'len(cpus["pod2c3"]) == 1' + cleanup From 0846c569d3712787940fee89cdcb30eaecb913c1 Mon Sep 17 00:00:00 2001 From: Antti Kervinen Date: Mon, 30 Sep 2024 14:38:18 +0300 Subject: [PATCH 3/3] doc: document the preserve balloons policy option Signed-off-by: Antti Kervinen --- docs/resource-policy/policy/balloons.md | 29 +++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/docs/resource-policy/policy/balloons.md b/docs/resource-policy/policy/balloons.md index 86f781ba7..3a46f2d7e 100644 --- a/docs/resource-policy/policy/balloons.md +++ b/docs/resource-policy/policy/balloons.md @@ -82,6 +82,23 @@ Balloons policy parameters: cause kernel to kill containers due to out-of-memory error when closest NUMA nodes do not have enough memory. In this situation consider switching this option `false`. +- `preserve` specifies containers whose resource pinning must not be + modified by the policy. + - `matchExpressions` if a container matches an expression in this + list, the policy will preserve container's resource pinning. If + there is no resource pinning, the policy will not change that + either. Example: preserve containers named "a" and "b". As a + result, the policy will not modify CPU or memory pinning of + matching containers. + ``` + ignore: + matchExpressions: + - key: name + operator: In + values: + - a + - b + ``` - `idleCPUClass` specifies the CPU class of those CPUs that do not belong to any balloon. - `reservedPoolNamespaces` is a list of namespaces (wildcards allowed) @@ -315,8 +332,16 @@ not defined, a built-in `default` balloon type is used. ## Disabling CPU or Memory Pinning of a Container Some containers may need to run on all CPUs or access all memories -without restrictions. Annotate these pods and containers to prevent -the resource policy from touching their CPU or memory pinning. +without restrictions. There are two alternatives to achieve this: +policy configuration and pod annotations. + +The resource policy will not touch allowed resources of containers +that match `preserve` criteria. See policy configuration options +above. + +Alternatively, pod annotations can opt-out all or selected containers +in the pod from CPU or memory pinning by preserving whatever existing +or non-existing pinning configuration: ```yaml cpu.preserve.resource-policy.nri.io/container.CONTAINER_NAME: "true"