Merge pull request #368 from askervin/5XG_balloons_policy_ignore

balloons: add "preserve" option to match containers whose pinning must not be modified

cmd/plugins/balloons/policy/balloons-policy.go

@@ -227,6 +227,17 @@ func (p *balloons) AllocateResources(c cache.Container) error {
 		log.Infof("not handling resources of container %s, preserving CPUs %q and memory %q", c.PrettyName(), c.GetCpusetCpus(), c.GetCpusetMems())
 		return nil
 	}
+
+	if p.bpoptions.Preserve != nil {
+		rule, err := p.bpoptions.Preserve.MatchContainer(c)
+		if err != nil {
+			log.Errorf("error in matching container %s to preserve conditions: %s", c, err)
+		} else if rule != "" {
+			log.Debugf("preserve container %s due to matching %s", c, rule)
+			return nil
+		}
+	}
+
 	log.Debug("allocating resources for container %s (request %d mCPU, limit %d mCPU)...",
 		c.PrettyName(),
 		p.containerRequestedMilliCpus(c.GetID()),

config/crd/bases/config.nri_balloonspolicies.yaml

@@ -393,6 +393,54 @@ spec:
                   overridden with the balloon type specific setting with the same
                   name.
                 type: boolean
+              preserve:
+                description: |-
+                  Preserve specifies containers whose resource pinning must not be
+                  modified by the policy.
+                properties:
+                  matchExpressions:
+                    description: MatchExpressions specifies one or more expressions.
+                    items:
+                      description: |-
+                        Expression describes some runtime-evaluated condition. An expression
+                        consist of a key, an operator and a set of values. An expressions is
+                        evaluated against an object which implements the Evaluable interface.
+                        Evaluating an expression consists of looking up the value for the key
+                        in the object, then using the operator to check it agains the values
+                        of the expression. The result is a single boolean value. An object is
+                        said to satisfy the evaluated expression if this value is true. An
+                        expression can contain 0, 1 or more values depending on the operator.
+                      properties:
+                        key:
+                          description: Key is the expression key.
+                          type: string
+                        operator:
+                          description: Op is the expression operator.
+                          enum:
+                          - Equals
+                          - NotEqual
+                          - In
+                          - NotIn
+                          - Exists
+                          - NotExist
+                          - AlwaysTrue
+                          - Matches
+                          - MatchesNot
+                          - MatchesAny
+                          - MatchesNone
+                          type: string
+                        values:
+                          description: Values contains the values the key value is
+                            evaluated against.
+                          items:
+                            type: string
+                          type: array
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                type: object
               reservedPoolNamespaces:
                 description: |-
                   ReservedPoolNamespaces is a list of namespace globs that

deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml

@@ -393,6 +393,54 @@ spec:
                   overridden with the balloon type specific setting with the same
                   name.
                 type: boolean
+              preserve:
+                description: |-
+                  Preserve specifies containers whose resource pinning must not be
+                  modified by the policy.
+                properties:
+                  matchExpressions:
+                    description: MatchExpressions specifies one or more expressions.
+                    items:
+                      description: |-
+                        Expression describes some runtime-evaluated condition. An expression
+                        consist of a key, an operator and a set of values. An expressions is
+                        evaluated against an object which implements the Evaluable interface.
+                        Evaluating an expression consists of looking up the value for the key
+                        in the object, then using the operator to check it agains the values
+                        of the expression. The result is a single boolean value. An object is
+                        said to satisfy the evaluated expression if this value is true. An
+                        expression can contain 0, 1 or more values depending on the operator.
+                      properties:
+                        key:
+                          description: Key is the expression key.
+                          type: string
+                        operator:
+                          description: Op is the expression operator.
+                          enum:
+                          - Equals
+                          - NotEqual
+                          - In
+                          - NotIn
+                          - Exists
+                          - NotExist
+                          - AlwaysTrue
+                          - Matches
+                          - MatchesNot
+                          - MatchesAny
+                          - MatchesNone
+                          type: string
+                        values:
+                          description: Values contains the values the key value is
+                            evaluated against.
+                          items:
+                            type: string
+                          type: array
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                type: object
               reservedPoolNamespaces:
                 description: |-
                   ReservedPoolNamespaces is a list of namespace globs that

docs/resource-policy/policy/balloons.md

@@ -82,6 +82,23 @@ Balloons policy parameters:
   cause kernel to kill containers due to out-of-memory error when
   closest NUMA nodes do not have enough memory. In this situation
   consider switching this option `false`.
+- `preserve` specifies containers whose resource pinning must not be
+  modified by the policy.
+  - `matchExpressions` if a container matches an expression in this
+    list, the policy will preserve container's resource pinning. If
+    there is no resource pinning, the policy will not change that
+    either. Example: preserve containers named "a" and "b". As a
+    result, the policy will not modify CPU or memory pinning of
+    matching containers.
+    ```
+    ignore:
+      matchExpressions:
+        - key: name
+          operator: In
+          values:
+            - a
+            - b
+    ```
 - `idleCPUClass` specifies the CPU class of those CPUs that do not
   belong to any balloon.
 - `reservedPoolNamespaces` is a list of namespaces (wildcards allowed)
@@ -315,8 +332,16 @@ not defined, a built-in `default` balloon type is used.
 ## Disabling CPU or Memory Pinning of a Container
 
 Some containers may need to run on all CPUs or access all memories
-without restrictions. Annotate these pods and containers to prevent
-the resource policy from touching their CPU or memory pinning.
+without restrictions. There are two alternatives to achieve this:
+policy configuration and pod annotations.
+
+The resource policy will not touch allowed resources of containers
+that match `preserve` criteria. See policy configuration options
+above.
+
+Alternatively, pod annotations can opt-out all or selected containers
+in the pod from CPU or memory pinning by preserving whatever existing
+or non-existing pinning configuration:
 
 ```yaml
 cpu.preserve.resource-policy.nri.io/container.CONTAINER_NAME: "true"

pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go

@@ -21,6 +21,7 @@ import (
 	policy "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy"
 	resmgr "github.com/containers/nri-plugins/pkg/apis/resmgr/v1alpha1"
 	"github.com/containers/nri-plugins/pkg/cpuallocator"
+	"github.com/containers/nri-plugins/pkg/resmgr/cache"
 )
 
 type (
@@ -78,6 +79,9 @@ type Config struct {
 	// Reserved (CPU) resources for kube-system namespace.
 	// +kubebuilder:validation:Required
 	ReservedResources Constraints `json:"reservedResources"`
+	// Preserve specifies containers whose resource pinning must not be
+	// modified by the policy.
+	Preserve *ContainerMatchConfig `json:"preserve,omitempty"`
 }
 
 type CPUTopologyLevel string
@@ -252,8 +256,31 @@ func (p CPUPriority) Value() cpuallocator.CPUPriority {
 	return cpuallocator.PriorityNone
 }
 
+// ContainerMatchConfig contains container matching configurations.
+// +k8s:deepcopy-gen=true
+type ContainerMatchConfig struct {
+	// MatchExpressions specifies one or more expressions.
+	MatchExpressions []resmgr.Expression `json:"matchExpressions,omitempty"`
+}
+
+func (cmc *ContainerMatchConfig) MatchContainer(c cache.Container) (string, error) {
+	for _, expr := range cmc.MatchExpressions {
+		if expr.Evaluate(c) {
+			return expr.String(), nil
+		}
+	}
+	return "", nil
+}
+
 func (c *Config) Validate() error {
 	errs := []error{}
+	if c.Preserve != nil {
+		for _, expr := range c.Preserve.MatchExpressions {
+			if err := expr.Validate(); err != nil {
+				errs = append(errs, err)
+			}
+		}
+	}
 	for _, blnDef := range c.BalloonDefs {
 		for _, expr := range blnDef.MatchExpressions {
 			if err := expr.Validate(); err != nil {

test/e2e/policies.test-suite/balloons/match-config.yaml

@@ -5,6 +5,15 @@ config:
   pinMemory: true
   idleCPUClass: normal
   allocatorTopologyBalancing: true
+  preserve:
+    matchExpressions:
+      - key: name
+        operator: In
+        values:
+          - pod1cX
+          - pod2c1
+          - pod2c2
+          - pod3cX
   balloonTypes:
   - name: special
     matchExpressions:

test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh

@@ -21,4 +21,13 @@ CONTCOUNT=1 create balloons-busybox
 report allowed
 verify 'len(cpus["pod1c0"]) == 1'
 
+# pod2: run ordinary workload where pod2c1 and pod2c2 match the
+# preserve matchexpression in policy configuration.
+CONTCOUNT=4 create balloons-busybox
+report allowed
+verify 'len(cpus["pod2c0"]) == 1' \
+       'len(cpus["pod2c1"]) == 16' \
+       'len(cpus["pod2c2"]) == 16' \
+       'len(cpus["pod2c3"]) == 1'
+
 cleanup