From d9a5c90ff6ba27001b9d8972ce5c1dca31e0e381 Mon Sep 17 00:00:00 2001
From: Antti Kervinen <antti.kervinen@intel.com>
Date: Fri, 18 Oct 2024 13:12:17 +0300
Subject: [PATCH] balloons: align CPU allocations based on L2 caches

This implementation expects CPUs sharing the same L2 cache also
belonging into the same NUMA node, and that hyperthreads of the same
CPU core always share L2 cache.

Signed-off-by: Antti Kervinen <antti.kervinen@intel.com>
---
 cmd/plugins/balloons/policy/cputree.go        | 48 ++++++++++++-------
 cmd/plugins/balloons/policy/flags.go          |  1 +
 .../v1alpha1/resmgr/policy/balloons/config.go |  6 ++-
 3 files changed, 36 insertions(+), 19 deletions(-)

diff --git a/cmd/plugins/balloons/policy/cputree.go b/cmd/plugins/balloons/policy/cputree.go
index 634291fb7..6f8cfc02a 100644
--- a/cmd/plugins/balloons/policy/cputree.go
+++ b/cmd/plugins/balloons/policy/cputree.go
@@ -248,7 +248,7 @@ func (t *cpuTreeNode) CpuLocations(cpus cpuset.CPUSet) [][]string {
 // NewCpuTreeFromSystem returns the root node of the topology tree
 // constructed from the underlying system.
 func NewCpuTreeFromSystem() (*cpuTreeNode, error) {
-	sys, err := system.DiscoverSystem(system.DiscoverCPUTopology)
+	sys, err := system.DiscoverSystem(system.DiscoverCPUTopology | system.DiscoverCache)
 	if err != nil {
 		return nil, err
 	}
@@ -262,30 +262,44 @@ func NewCpuTreeFromSystem() (*cpuTreeNode, error) {
 		cpuPackage := sys.Package(packageID)
 		sysTree.AddChild(packageTree)
 		for _, dieID := range cpuPackage.DieIDs() {
-			dieTree := NewCpuTree(fmt.Sprintf("p%dd%d", packageID, dieID))
+			dieTree := NewCpuTree(fmt.Sprintf("%sd%d", packageTree.name, dieID))
 			dieTree.level = CPUTopologyLevelDie
 			packageTree.AddChild(dieTree)
 			for _, nodeID := range cpuPackage.DieNodeIDs(dieID) {
-				nodeTree := NewCpuTree(fmt.Sprintf("p%dd%dn%d", packageID, dieID, nodeID))
+				nodeTree := NewCpuTree(fmt.Sprintf("%sn%d", dieTree.name, nodeID))
 				nodeTree.level = CPUTopologyLevelNuma
 				dieTree.AddChild(nodeTree)
 				node := sys.Node(nodeID)
-				threadsSeen := map[int]struct{}{}
+
+				// Find all level 2 caches (l2c) shared by CPUs of this node.
+				l2cs := map[*system.Cache]struct{}{}
 				for _, cpuID := range node.CPUSet().List() {
-					if _, alreadySeen := threadsSeen[cpuID]; alreadySeen {
-						continue
+					for _, cache := range sys.CPU(cpuID).GetCachesByLevel(2) {
+						l2cs[cache] = struct{}{}
 					}
-					cpuTree := NewCpuTree(fmt.Sprintf("p%dd%dn%dcpu%d", packageID, dieID, nodeID, cpuID))
-
-					cpuTree.level = CPUTopologyLevelCore
-					nodeTree.AddChild(cpuTree)
-					cpu := sys.CPU(cpuID)
-					for _, threadID := range cpu.ThreadCPUSet().List() {
-						threadsSeen[threadID] = struct{}{}
-						threadTree := NewCpuTree(fmt.Sprintf("p%dd%dn%dcpu%dt%d", packageID, dieID, nodeID, cpuID, threadID))
-						threadTree.level = CPUTopologyLevelThread
-						cpuTree.AddChild(threadTree)
-						threadTree.AddCpus(cpuset.New(threadID))
+				}
+
+				for cache := range l2cs {
+					l2cTree := NewCpuTree(fmt.Sprintf("%s$%d", nodeTree.name, cache.ID()))
+					l2cTree.level = CPUTopologyLevelL2Cache
+					nodeTree.AddChild(l2cTree)
+
+					threadsSeen := map[int]struct{}{}
+					for _, cpuID := range cache.SharedCPUSet().List() {
+						if _, alreadySeen := threadsSeen[cpuID]; alreadySeen {
+							continue
+						}
+						cpu := sys.CPU(cpuID)
+						coreTree := NewCpuTree(fmt.Sprintf("%scpu%d", nodeTree.name, cpuID))
+						coreTree.level = CPUTopologyLevelCore
+						l2cTree.AddChild(coreTree)
+						for _, threadID := range cpu.ThreadCPUSet().List() {
+							threadsSeen[threadID] = struct{}{}
+							threadTree := NewCpuTree(fmt.Sprintf("%st%d", coreTree.name, threadID))
+							threadTree.level = CPUTopologyLevelThread
+							coreTree.AddChild(threadTree)
+							threadTree.AddCpus(cpuset.New(threadID))
+						}
 					}
 				}
 			}
diff --git a/cmd/plugins/balloons/policy/flags.go b/cmd/plugins/balloons/policy/flags.go
index 9129c5c5f..bd66b2366 100644
--- a/cmd/plugins/balloons/policy/flags.go
+++ b/cmd/plugins/balloons/policy/flags.go
@@ -38,6 +38,7 @@ const (
 	CPUTopologyLevelPackage   = cfgapi.CPUTopologyLevelPackage
 	CPUTopologyLevelDie       = cfgapi.CPUTopologyLevelDie
 	CPUTopologyLevelNuma      = cfgapi.CPUTopologyLevelNuma
+	CPUTopologyLevelL2Cache   = cfgapi.CPUTopologyLevelL2Cache
 	CPUTopologyLevelCore      = cfgapi.CPUTopologyLevelCore
 	CPUTopologyLevelThread    = cfgapi.CPUTopologyLevelThread
 )
diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go
index e94748eaf..1449a7017 100644
--- a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go
+++ b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go
@@ -92,6 +92,7 @@ const (
 	CPUTopologyLevelPackage   = "package"
 	CPUTopologyLevelDie       = "die"
 	CPUTopologyLevelNuma      = "numa"
+	CPUTopologyLevelL2Cache   = "l2cache"
 	CPUTopologyLevelCore      = "core"
 	CPUTopologyLevelThread    = "thread"
 )
@@ -103,8 +104,9 @@ var (
 		CPUTopologyLevelPackage:   2,
 		CPUTopologyLevelDie:       3,
 		CPUTopologyLevelNuma:      4,
-		CPUTopologyLevelCore:      5,
-		CPUTopologyLevelThread:    6,
+		CPUTopologyLevelL2Cache:   5,
+		CPUTopologyLevelCore:      6,
+		CPUTopologyLevelThread:    7,
 	}
 
 	CPUTopologyLevelCount = len(cpuTopologyLevelValues)