Merge pull request #260 from klihub/devel/assign-balloons-by-expression

balloons: allow assigning containers to balloons by runtime-evaluated expressions.

cmd/plugins/balloons/policy/balloons-policy.go

@@ -354,24 +354,33 @@ func (p *balloons) balloonDefByName(defName string) *BalloonDef {
 }
 
 func (p *balloons) chooseBalloonDef(c cache.Container) (*BalloonDef, error) {
-	var blnDef *BalloonDef
 	// Case 1: BalloonDef is defined by annotation.
 	if blnDefName, ok := c.GetEffectiveAnnotation(balloonKey); ok {
-		blnDef = p.balloonDefByName(blnDefName)
+		blnDef := p.balloonDefByName(blnDefName)
 		if blnDef == nil {
 			return nil, balloonsError("no balloon for annotation %q", blnDefName)
 		}
 		return blnDef, nil
 	}
 
-	// Case 2: BalloonDef is defined by the namespace.
 	for _, blnDef := range p.bpoptions.BalloonDefs {
+		// Case 2: BalloonDef is defined by a match expression.
+		for _, expr := range blnDef.MatchExpressions {
+			log.Debugf("- checking expression %s of balloon %q against container %s...",
+				expr.String(), blnDef.Name, c.PrettyName())
+			if expr.Evaluate(c) {
+				log.Debugf("  => matches")
+				return blnDef, nil
+			}
+		}
+
+		// Case 3: BalloonDef is defined by the namespace.
 		if namespaceMatches(c.GetNamespace(), blnDef.Namespaces) {
 			return blnDef, nil
 		}
 	}
 
-	// Case 3: Fallback to the default balloon.
+	// Case 4: Fallback to the default balloon.
 	return p.defaultBalloonDef, nil
 }
 

config/crd/bases/config.nri_balloonspolicies.yaml

@@ -83,6 +83,51 @@ spec:
                         CpuClass controls how CPUs of a balloon are (re)configured
                         whenever a balloon is created, inflated or deflated.
                       type: string
+                    matchExpressions:
+                      description: |-
+                        MatchExpressions specifies one or more expressions which are evaluated
+                        to see if a container should be assigned into balloon instances from
+                        this definition.
+                      items:
+                        description: |-
+                          Expression describes some runtime-evaluated condition. An expression
+                          consist of a key, an operator and a set of values. An expressions is
+                          evaluated against an object which implements the Evaluable interface.
+                          Evaluating an expression consists of looking up the value for the key
+                          in the object, then using the operator to check it agains the values
+                          of the expression. The result is a single boolean value. An object is
+                          said to satisfy the evaluated expression if this value is true. An
+                          expression can contain 0, 1 or more values depending on the operator.
+                        properties:
+                          key:
+                            description: Key is the expression key.
+                            type: string
+                          operator:
+                            description: Op is the expression operator.
+                            enum:
+                            - Equals
+                            - NotEqual
+                            - In
+                            - NotIn
+                            - Exists
+                            - NotExist
+                            - AlwaysTrue
+                            - Matches
+                            - MatchesNot
+                            - MatchesAny
+                            - MatchesNone
+                            type: string
+                          values:
+                            description: Values contains the values the key value
+                              is evaluated against.
+                            items:
+                              type: string
+                            type: array
+                        required:
+                        - key
+                        - operator
+                        type: object
+                      type: array
                     maxBalloons:
                       description: |-
                         MaxBalloons is the maximum number of balloon instances that

deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml

@@ -83,6 +83,51 @@ spec:
                         CpuClass controls how CPUs of a balloon are (re)configured
                         whenever a balloon is created, inflated or deflated.
                       type: string
+                    matchExpressions:
+                      description: |-
+                        MatchExpressions specifies one or more expressions which are evaluated
+                        to see if a container should be assigned into balloon instances from
+                        this definition.
+                      items:
+                        description: |-
+                          Expression describes some runtime-evaluated condition. An expression
+                          consist of a key, an operator and a set of values. An expressions is
+                          evaluated against an object which implements the Evaluable interface.
+                          Evaluating an expression consists of looking up the value for the key
+                          in the object, then using the operator to check it agains the values
+                          of the expression. The result is a single boolean value. An object is
+                          said to satisfy the evaluated expression if this value is true. An
+                          expression can contain 0, 1 or more values depending on the operator.
+                        properties:
+                          key:
+                            description: Key is the expression key.
+                            type: string
+                          operator:
+                            description: Op is the expression operator.
+                            enum:
+                            - Equals
+                            - NotEqual
+                            - In
+                            - NotIn
+                            - Exists
+                            - NotExist
+                            - AlwaysTrue
+                            - Matches
+                            - MatchesNot
+                            - MatchesAny
+                            - MatchesNone
+                            type: string
+                          values:
+                            description: Values contains the values the key value
+                              is evaluated against.
+                            items:
+                              type: string
+                            type: array
+                        required:
+                        - key
+                        - operator
+                        type: object
+                      type: array
                     maxBalloons:
                       description: |-
                         MaxBalloons is the maximum number of balloon instances that

docs/resource-policy/policy/balloons.md

@@ -115,6 +115,15 @@ Balloons policy parameters:
   - `namespaces` is a list of namespaces (wildcards allowed) whose
     pods should be assigned to this balloon type, unless overridden by
     pod annotations.
+  - `matchExpressions` is a list of container match expressions. These
+    expressions are evaluated for all containers which have not been
+    assigned otherwise to other balloons. If an expression matches,
+    IOW it evaluates to true, the container gets assigned to this
+    balloon type. Container mach expressions have the same syntax and
+    semantics as the scope and match expressions in container affinity
+    annotations for the topology-aware policy.
+    See the [affinity documentation](./topology-aware.md#affinity-semantics)
+    for a detailed description of expressions.
   - `minBalloons` is the minimum number of balloons of this type that
     is always present, even if the balloons would not have any
     containers. The default is 0: if a balloon has no containers, it

pkg/apis/config/v1alpha1/balloons-policy.go

@@ -35,3 +35,10 @@ func (c *BalloonsPolicy) PolicyConfig() interface{} {
 	}
 	return &c.Spec.Config
 }
+
+func (c *BalloonsPolicy) Validate() error {
+	if c == nil {
+		return nil
+	}
+	return c.Spec.Config.Validate()
+}

pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go

@@ -15,9 +15,11 @@
 package balloons
 
 import (
+	"errors"
 	"strings"
 
 	policy "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy"
+	resmgr "github.com/containers/nri-plugins/pkg/apis/resmgr/v1alpha1"
 	"github.com/containers/nri-plugins/pkg/cpuallocator"
 )
 
@@ -124,6 +126,10 @@ type BalloonDef struct {
 	// balloon instances from this definition. This is used by
 	// namespace assign methods.
 	Namespaces []string `json:"namespaces,omitempty"`
+	// MatchExpressions specifies one or more expressions which are evaluated
+	// to see if a container should be assigned into balloon instances from
+	// this definition.
+	MatchExpressions []resmgr.Expression `json:"matchExpressions,omitempty"`
 	// MaxCpus specifies the maximum number of CPUs exclusively
 	// usable by containers in a balloon. Balloon size will not be
 	// inflated larger than MaxCpus.
@@ -218,3 +224,15 @@ func (p CPUPriority) Value() cpuallocator.CPUPriority {
 	}
 	return cpuallocator.PriorityNone
 }
+
+func (c *Config) Validate() error {
+	errs := []error{}
+	for _, blnDef := range c.BalloonDefs {
+		for _, expr := range blnDef.MatchExpressions {
+			if err := expr.Validate(); err != nil {
+				errs = append(errs, err)
+			}
+		}
+	}
+	return errors.Join(errs...)
+}

test/e2e/policies.test-suite/balloons/balloons-busybox.yaml.in

@@ -8,6 +8,9 @@ metadata:
   "; fi)
   labels:
     app: ${NAME}
+  $(if [ -n "$POD_LABEL" ]; then echo "
+    $POD_LABEL
+  "; fi)
 spec:
   containers:
   $(for contnum in $(seq 1 ${CONTCOUNT}); do echo "

test/e2e/policies.test-suite/balloons/match-config.yaml

@@ -0,0 +1,27 @@
+config:
+  reservedResources:
+    cpu: 750m
+  pinCPU: true
+  pinMemory: true
+  idleCPUClass: normal
+  allocatorTopologyBalancing: true
+  balloonTypes:
+  - name: special
+    matchExpressions:
+      - key: pod/labels/app.kubernetes.io/component
+        operator: Equals
+        values: [ "precious" ]
+    minCPUs: 2
+    maxCPUs: 2
+  - name: default
+    namespaces:
+      - default
+    minCPUs: 1
+    maxCPUs: 1
+    allocatorPriority: normal
+  reservedPoolNamespaces:
+    - kube-system
+  log:
+    debug:
+      - policy
+      - expression

test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh

@@ -0,0 +1,24 @@
+# Test placing containers with and without annotations to correct balloons
+# reserved and shared CPUs.
+
+helm-terminate
+helm_config=${TEST_DIR}/../../match-config.yaml helm-launch balloons
+
+cleanup() {
+    vm-command "kubectl delete pods --all --now"
+    return 0
+}
+
+cleanup
+
+# pod0: run precious workload
+POD_LABEL="app.kubernetes.io/component: precious" CONTCOUNT=1 create balloons-busybox
+report allowed
+verify 'len(cpus["pod0c0"]) == 2'
+
+# pod1: run ordinary workload
+CONTCOUNT=1 create balloons-busybox
+report allowed
+verify 'len(cpus["pod1c0"]) == 1'
+
+cleanup