Merge pull request #278 from askervin/5Tk_balloons_groupby_stacked

balloons: implement groupBy option

cmd/plugins/balloons/policy/balloons-policy.go

@@ -90,7 +90,10 @@ type Balloon struct {
 	// - len(PodIDs) is the number of pods in the balloon.
 	// - len(PodIDs[podID]) is the number of containers of podID
 	//   currently assigned to the balloon.
-	PodIDs       map[string][]string
+	PodIDs map[string][]string
+	// Groups is a multiset (group-by-value -> appearance-count)
+	// of evaluated GroupBy expressions on containers in the balloon.
+	Groups       map[string]int
 	cpuTreeAlloc *cpuTreeAllocator
 }
 
@@ -304,6 +307,18 @@ func (p *balloons) balloonByContainer(c cache.Container) *Balloon {
 	return nil
 }
 
+// balloonsByGroup returns balloons that contain containers on which
+// balloon's GroupBy expression evaluates given group.
+func (p *balloons) balloonsByGroup(group string) []*Balloon {
+	blns := []*Balloon{}
+	for _, bln := range p.balloons {
+		if bln.Groups[group] > 0 {
+			blns = append(blns, bln)
+		}
+	}
+	return blns
+}
+
 // balloonsByNamespace returns balloons that contain containers in a
 // namespace.
 func (p *balloons) balloonsByNamespace(namespace string) []*Balloon {
@@ -563,6 +578,7 @@ func (p *balloons) newBalloon(blnDef *BalloonDef, confCpus bool) (*Balloon, erro
 	bln := &Balloon{
 		Def:            blnDef,
 		Instance:       freeInstance,
+		Groups:         make(map[string]int),
 		PodIDs:         make(map[string][]string),
 		Cpus:           cpus,
 		SharedIdleCpus: cpuset.New(),
@@ -669,6 +685,18 @@ func (p *balloons) chooseBalloonInstance(blnDef *BalloonDef, fm FillMethod, c ca
 			p.updatePinning(p.shareIdleCpus(p.freeCpus, newBln.Cpus)...)
 		}
 		return newBln, nil
+	case FillSameGroup:
+		group, err := c.Expand(blnDef.GroupBy, true)
+		if err != nil {
+			log.Errorf("error choosing balloon for container %q based on groupBy: %s", c.PrettyName(), err)
+			return nil, nil
+		}
+		for _, bln := range p.balloonsByGroup(group) {
+			if bln.Def == blnDef && p.maxFreeMilliCpus(bln) >= reqMilliCpus {
+				return bln, nil
+			}
+		}
+		return nil, nil
 	case FillSameNamespace:
 		for _, bln := range p.balloonsByNamespace(c.GetNamespace()) {
 			if bln.Def == blnDef && p.maxFreeMilliCpus(bln) >= reqMilliCpus {
@@ -754,6 +782,9 @@ func (p *balloons) allocateBalloon(c cache.Container) (*Balloon, error) {
 // definition for a container.
 func (p *balloons) allocateBalloonOfDef(blnDef *BalloonDef, c cache.Container) (*Balloon, error) {
 	fillChain := []FillMethod{}
+	if blnDef.GroupBy != "" {
+		fillChain = append(fillChain, FillSameGroup)
+	}
 	if !blnDef.PreferSpreadingPods {
 		fillChain = append(fillChain, FillSamePod)
 	}
@@ -1346,11 +1377,20 @@ func (p *balloons) shareIdleCpus(addCpus, removeCpus cpuset.CPUSet) []*Balloon {
 	return updatedBalloons
 }
 
+// updateGroups updates the number of groups present in the balloon.
+func (bln *Balloon) updateGroups(c cache.Container, delta int) {
+	if bln.Def.GroupBy != "" {
+		group, _ := c.Expand(bln.Def.GroupBy, false)
+		bln.Groups[group] += delta
+	}
+}
+
 // assignContainer adds a container to a balloon
 func (p *balloons) assignContainer(c cache.Container, bln *Balloon) {
 	log.Info("assigning container %s to balloon %s", c.PrettyName(), bln)
 	podID := c.GetPodID()
 	bln.PodIDs[podID] = append(bln.PodIDs[podID], c.GetID())
+	bln.updateGroups(c, 1)
 	p.updatePinning(bln)
 }
 
@@ -1361,6 +1401,7 @@ func (p *balloons) dismissContainer(c cache.Container, bln *Balloon) {
 	if len(bln.PodIDs[podID]) == 0 {
 		delete(bln.PodIDs, podID)
 	}
+	bln.updateGroups(c, -1)
 }
 
 // pinCpuMem pins container to CPUs and memory nodes if flagged

cmd/plugins/balloons/policy/fillmethod.go

@@ -40,6 +40,10 @@ const (
 	// it minimizes the amount of unused CPUs if the balloon is
 	// inflated to the maximum size.
 	FillPackedInflate
+	// FillSameGroup: put a container into a balloon with other
+	// containers that share the same evaluated GroupBy expression
+	// result, that is, the same group.
+	FillSameGroup
 	// FillSameNamespace: put a container into a balloon that already
 	// includes another container from the same namespace
 	FillSameNamespace
@@ -61,6 +65,7 @@ var fillMethodNames = map[FillMethod]string{
 	FillBalancedInflate: "balanced-inflate",
 	FillPacked:          "packed",
 	FillPackedInflate:   "packed-inflate",
+	FillSameGroup:       "same-group",
 	FillSameNamespace:   "same-namespace",
 	FillSamePod:         "same-pod",
 	FillNewBalloon:      "new-balloon",

cmd/plugins/balloons/policy/metrics.go

@@ -39,6 +39,7 @@ var descriptors = []*prometheus.Desc{
 			"cpus_min",
 			"cpus_max",
 			"balloon",
+			"groups",
 			"cpus",
 			"cpus_count",
 			"numas",
@@ -72,6 +73,7 @@ type BalloonMetrics struct {
 	MaxCpus  int
 	// Balloon instance metrics
 	PrettyName            string
+	Groups                string
 	Cpus                  cpuset.CPUSet
 	CpusCount             int
 	Numas                 []string
@@ -108,6 +110,14 @@ func (p *balloons) PollMetrics() policy.Metrics {
 		bm.MinCpus = bln.Def.MinCpus
 		bm.MaxCpus = bln.Def.MaxCpus
 		bm.PrettyName = bln.PrettyName()
+		groups := []string{}
+		for group, cCount := range bln.Groups {
+			if cCount > 0 {
+				groups = append(groups, group)
+			}
+		}
+		sort.Strings(groups)
+		bm.Groups = strings.Join(groups, ",")
 		bm.Cpus = bln.Cpus
 		bm.CpusCount = bm.Cpus.Size()
 		if len(cpuLoc) > 3 {
@@ -158,6 +168,7 @@ func (p *balloons) CollectMetrics(m policy.Metrics) ([]prometheus.Metric, error)
 			strconv.Itoa(bm.MinCpus),
 			strconv.Itoa(bm.MaxCpus),
 			bm.PrettyName,
+			bm.Groups,
 			bm.Cpus.String(),
 			strconv.Itoa(bm.CpusCount),
 			strings.Join(bm.Numas, ","),

config/crd/bases/config.nri_balloonspolicies.yaml

@@ -83,6 +83,14 @@ spec:
                         CpuClass controls how CPUs of a balloon are (re)configured
                         whenever a balloon is created, inflated or deflated.
                       type: string
+                    groupBy:
+                      description: |-
+                        GroupBy groups containers into same balloon instances if
+                        their GroupBy expressions evaluate to the same group.
+                        Expressions are strings where key references like
+                        ${pod/labels/mylabel} will be substituted with
+                        corresponding values.
+                      type: string
                     matchExpressions:
                       description: |-
                         MatchExpressions specifies one or more expressions which are evaluated

deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml

@@ -83,6 +83,14 @@ spec:
                         CpuClass controls how CPUs of a balloon are (re)configured
                         whenever a balloon is created, inflated or deflated.
                       type: string
+                    groupBy:
+                      description: |-
+                        GroupBy groups containers into same balloon instances if
+                        their GroupBy expressions evaluate to the same group.
+                        Expressions are strings where key references like
+                        ${pod/labels/mylabel} will be substituted with
+                        corresponding values.
+                      type: string
                     matchExpressions:
                       description: |-
                         MatchExpressions specifies one or more expressions which are evaluated

docs/resource-policy/policy/balloons.md

@@ -121,6 +121,11 @@ Balloons policy parameters:
   - `namespaces` is a list of namespaces (wildcards allowed) whose
     pods should be assigned to this balloon type, unless overridden by
     pod annotations.
+  - `groupBy` groups containers into same balloon instances if
+    their GroupBy expressions evaluate to the same group.
+    Expressions are strings where key references like
+    `${pod/labels/mylabel}` will be substituted with corresponding
+    values.
   - `matchExpressions` is a list of container match expressions. These
     expressions are evaluated for all containers which have not been
     assigned otherwise to other balloons. If an expression matches,

pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go

@@ -126,6 +126,12 @@ type BalloonDef struct {
 	// balloon instances from this definition. This is used by
 	// namespace assign methods.
 	Namespaces []string `json:"namespaces,omitempty"`
+	// GroupBy groups containers into same balloon instances if
+	// their GroupBy expressions evaluate to the same group.
+	// Expressions are strings where key references like
+	// ${pod/labels/mylabel} will be substituted with
+	// corresponding values.
+	GroupBy string `json:"groupBy,omitempty"`
 	// MatchExpressions specifies one or more expressions which are evaluated
 	// to see if a container should be assigned into balloon instances from
 	// this definition.

test/e2e/policies.test-suite/balloons/n4c16/test02-prometheus-metrics/code.var.sh

@@ -25,7 +25,7 @@ POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: full-core" CONTCOUNT=2
 report allowed
 verify-metrics-has-line 'balloon="default\[0\]"'
 verify-metrics-has-line 'balloon="reserved\[0\]"'
-verify-metrics-has-line 'balloons{balloon="full-core\[0\]",balloon_type="full-core",containers="pod0/pod0c0,pod0/pod0c1",cpu_class="normal",cpus=".*",cpus_allowed=".*",cpus_allowed_count="2",cpus_count="2",cpus_max="2",cpus_min="2",dies="p[01]d0",dies_count="1",mems="[0-3]",numas="p[01]d0n[0-3]",numas_count="1",packages="p[01]",packages_count="1",sharedidlecpus="",sharedidlecpus_count="0",tot_req_millicpu="(199|200)"} 2'
+verify-metrics-has-line 'balloons{balloon="full-core\[0\]",balloon_type="full-core",containers="pod0/pod0c0,pod0/pod0c1",cpu_class="normal",cpus=".*",cpus_allowed=".*",cpus_allowed_count="2",cpus_count="2",cpus_max="2",cpus_min="2",dies="p[01]d0",dies_count="1",groups="",mems="[0-3]",numas="p[01]d0n[0-3]",numas_count="1",packages="p[01]",packages_count="1",sharedidlecpus="",sharedidlecpus_count="0",tot_req_millicpu="(199|200)"} 2'
 
 # pod1 in fast-dualcore[0]
 CPUREQ="200m" MEMREQ="" CPULIM="200m" MEMLIM=""

test/e2e/policies.test-suite/balloons/n4c16/test04-groupby/balloons-groupby.cfg

@@ -0,0 +1,18 @@
+config:
+  balloonTypes:
+    - name: grouped-by-label
+      groupBy: ${pod/namespace}-${pod/labels/balloon-instance}
+      minCPUs: 2
+      minBalloons: 2
+      preferNewBalloons: true
+    - name: default
+      groupBy: ns=$pod/namespace
+      preferNewBalloons: true
+  instrumentation:
+    httpEndpoint: :8891
+    prometheusExport: true
+  log:
+    debug:
+      - policy
+    klog:
+      skip_headers: true

test/e2e/policies.test-suite/balloons/n4c16/test04-groupby/code.var.sh

@@ -0,0 +1,62 @@
+# This test verifies that the groupby expression in a balloon type
+# affects grouping containers into balloon instances of that type.
+
+helm-terminate
+helm_config=$TEST_DIR/balloons-groupby.cfg helm-launch balloons
+
+testns=e2e-balloons-test04
+
+cleanup() {
+    vm-command "kubectl delete pods --all --now; \
+        kubectl delete pods -n $testns --all --now; \
+        kubectl delete namespace $testns; \
+        true"
+}
+
+cleanup
+
+POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: grouped-by-label"
+# pod0c0
+POD_LABEL='balloon-instance: g1'
+create balloons-busybox
+
+# pod1c0
+POD_LABEL='balloon-instance: g2'
+create balloons-busybox
+
+# pod2c0
+POD_LABEL='balloon-instance: g1'
+create balloons-busybox
+
+# pod3c0
+POD_LABEL='balloon-instance: g1'
+vm-command "kubectl create namespace $testns"
+namespace=$testns create balloons-busybox
+report allowed
+
+verify 'cpus["pod0c0"] == cpus["pod2c0"]' \
+       'disjoint_sets(cpus["pod0c0"], cpus["pod1c0"], cpus["pod3c0"])'
+
+# Test that pods are grouped by namespaces in separate default
+# balloon instances.
+POD_ANNOTATION=""
+# pod4c0
+namespace=$testns create balloons-busybox
+# pod5c0
+create balloons-busybox
+# pod6c0
+create balloons-busybox
+# pod7c0
+namespace=$testns create balloons-busybox
+report allowed
+verify 'cpus["pod4c0"] == cpus["pod7c0"]' \
+       'cpus["pod5c0"] == cpus["pod6c0"]' \
+       'disjoint_sets(cpus["pod4c0"], cpus["pod5c0"])'
+
+verify-metrics-has-line 'groups="e2e-balloons-test04-g1"'
+verify-metrics-has-line 'groups="default-g2"'
+verify-metrics-has-line 'groups="default-g1"'
+
+cleanup
+
+helm-terminate