diff --git a/cmd/plugins/topology-aware/policy/cache_test.go b/cmd/plugins/topology-aware/policy/cache_test.go index effb9bca6..a3696d0cd 100644 --- a/cmd/plugins/topology-aware/policy/cache_test.go +++ b/cmd/plugins/topology-aware/policy/cache_test.go @@ -88,11 +88,11 @@ func TestAllocationMarshalling(t *testing.T) { }{ { name: "non-zero Exclusive", - data: []byte(`{"key1":{"Exclusive":"1","Part":1,"CPUType":0,"Container":"1","Pool":"testnode","MemoryPool":"testnode","MemType":"DRAM,PMEM,HBM","Memset":"","MemoryLimit":{},"ColdStart":0}}`), + data: []byte(`{"key1":{"Exclusive":"1","Part":1,"CPUType":0,"Container":"1","Pool":"testnode","MemoryPool":0,"MemType":"DRAM,PMEM,HBM","MemSize":0,"ColdStart":0}}`), }, { name: "zero Exclusive", - data: []byte(`{"key1":{"Exclusive":"","Part":1,"CPUType":0,"Container":"1","Pool":"testnode","MemoryPool":"testnode","MemType":"DRAM,PMEM,HBM","Memset":"","MemoryLimit":{},"ColdStart":0}}`), + data: []byte(`{"key1":{"Exclusive":"","Part":1,"CPUType":0,"Container":"1","Pool":"testnode","MemoryPool":0,"MemType":"DRAM,PMEM,HBM","MemSize":0,"ColdStart":0}}`), }, } for _, tc := range tcases { @@ -104,8 +104,8 @@ func TestAllocationMarshalling(t *testing.T) { node: node{ name: "testnode", kind: UnknownNode, - noderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(0, 0, 0), createMemoryMap(0, 0, 0)), - freeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(0, 0, 0), createMemoryMap(0, 0, 0)), + noderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0), + freeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0), }, }, }, diff --git a/cmd/plugins/topology-aware/policy/coldstart_test.go b/cmd/plugins/topology-aware/policy/coldstart_test.go index 945cf0867..2ce630c47 100644 --- a/cmd/plugins/topology-aware/policy/coldstart_test.go +++ b/cmd/plugins/topology-aware/policy/coldstart_test.go @@ -22,6 +22,7 @@ import ( "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/events" + libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" system "github.com/containers/nri-plugins/pkg/sysfs" idset "github.com/intel/goresctrl/pkg/utils" @@ -62,8 +63,8 @@ func TestColdStart(t *testing.T) { { name: "three node cold start", numaNodes: []system.Node{ - &mockSystemNode{id: 1, memFree: 10000, memTotal: 10000, memType: system.MemoryTypeDRAM, distance: []int{5, 5, 1}}, - &mockSystemNode{id: 2, memFree: 50000, memTotal: 50000, memType: system.MemoryTypePMEM, distance: []int{5, 1, 5}}, + &mockSystemNode{id: 0, memFree: 10000, memTotal: 10000, memType: system.MemoryTypeDRAM, distance: []int{1, 5}}, + &mockSystemNode{id: 1, memFree: 50000, memTotal: 50000, memType: system.MemoryTypePMEM, distance: []int{5, 1}}, }, container: &mockContainer{ name: "demo-coldstart-container", @@ -77,8 +78,8 @@ func TestColdStart(t *testing.T) { }, expectedColdStartTimeout: 1000 * time.Millisecond, expectedDRAMNodeID: 101, - expectedDRAMSystemNodeID: idset.ID(1), - expectedPMEMSystemNodeID: idset.ID(2), + expectedDRAMSystemNodeID: 0, + expectedPMEMSystemNodeID: 1, expectedPMEMNodeID: 102, }, } @@ -100,6 +101,11 @@ func TestColdStart(t *testing.T) { } policy.allocations.policy = policy policy.options.SendEvent = sendEvent + ma, err := libmem.NewAllocator(libmem.WithSystemNodes(policy.sys)) + if err != nil { + panic(err) + } + policy.memAllocator = ma if err := policy.buildPoolsByTopology(); err != nil { t.Errorf("failed to build topology pool") @@ -115,8 +121,8 @@ func TestColdStart(t *testing.T) { policy.allocations.grants[tc.container.GetID()] = grant - mems := grant.Memset() - if len(mems) != 1 || mems.Members()[0] != tc.expectedPMEMSystemNodeID { + mems := grant.GetMemoryZone() + if mems.Size() != 1 || mems.Slice()[0] != tc.expectedPMEMSystemNodeID { t.Errorf("Expected one memory controller %v, got: %v", tc.expectedPMEMSystemNodeID, mems) } @@ -135,11 +141,11 @@ func TestColdStart(t *testing.T) { time.Sleep(tc.expectedColdStartTimeout * 2) - newMems := grant.Memset() - if len(newMems) != 2 { - t.Errorf("Expected two memory controllers, got %d: %v", len(newMems), newMems) + newMems := grant.GetMemoryZone() + if newMems.Size() != 2 { + t.Errorf("Expected two memory controllers, got %d: %s", newMems.Size(), newMems) } - if !newMems.Has(tc.expectedPMEMSystemNodeID) || !newMems.Has(tc.expectedDRAMSystemNodeID) { + if !newMems.Contains(tc.expectedPMEMSystemNodeID) || !newMems.Contains(tc.expectedDRAMSystemNodeID) { t.Errorf("Didn't get all expected system nodes in mems, got: %v", newMems) } }) diff --git a/cmd/plugins/topology-aware/policy/mocks_test.go b/cmd/plugins/topology-aware/policy/mocks_test.go index e60b852dd..96ff268b2 100644 --- a/cmd/plugins/topology-aware/policy/mocks_test.go +++ b/cmd/plugins/topology-aware/policy/mocks_test.go @@ -251,6 +251,9 @@ func (fake *mockSystem) OfflineCPUs() cpuset.CPUSet { func (fake *mockSystem) CoreKindCPUs(sysfs.CoreKind) cpuset.CPUSet { return cpuset.New() } +func (fake *mockSystem) CoreKinds() []sysfs.CoreKind { + return nil +} func (fake *mockSystem) AllThreadsForCPUs(cpuset.CPUSet) cpuset.CPUSet { return cpuset.New() } diff --git a/cmd/plugins/topology-aware/policy/pools_test.go b/cmd/plugins/topology-aware/policy/pools_test.go index 9c8e6ad00..b4111b3a2 100644 --- a/cmd/plugins/topology-aware/policy/pools_test.go +++ b/cmd/plugins/topology-aware/policy/pools_test.go @@ -22,15 +22,10 @@ import ( "testing" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" - "github.com/containers/nri-plugins/pkg/resmgr/cache" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" - v1 "k8s.io/api/core/v1" - resapi "k8s.io/apimachinery/pkg/api/resource" - system "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/utils" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) func findNodeWithID(id int, nodes []Node) Node { @@ -74,215 +69,6 @@ func setLinks(nodes []Node, tree map[int][]int) { } } -func TestMemoryLimitFiltering(t *testing.T) { - - // Test the scoring algorithm with synthetic data. The assumptions are: - - // 1. The first node in "nodes" is the root of the tree. - - tcases := []struct { - name string - nodes []Node - numaNodes []system.Node - req Request - affinities map[int]int32 - tree map[int][]int - expectedRemainingNodes []int - }{ - { - name: "single node memory limit (fits)", - nodes: []Node{ - &numanode{ - node: node{ - id: 100, - name: "testnode0", - kind: UnknownNode, - noderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(10001, 0, 0), createMemoryMap(0, 0, 0)), - freeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(10001, 0, 0), createMemoryMap(0, 0, 0)), - }, - id: 0, // system node id - }, - }, - numaNodes: []system.Node{ - &mockSystemNode{id: 0, memFree: 10001, memTotal: 10001}, - }, - req: &request{ - memReq: 10000, - memLim: 10000, - memType: defaultMemoryType, - container: &mockContainer{}, - }, - expectedRemainingNodes: []int{100}, - tree: map[int][]int{100: {}}, - }, - { - name: "single node memory limit (doesn't fit)", - nodes: []Node{ - &numanode{ - node: node{ - id: 100, - name: "testnode0", - kind: UnknownNode, - noderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(9999, 0, 0), createMemoryMap(0, 0, 0)), - freeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(9999, 0, 0), createMemoryMap(0, 0, 0)), - }, - id: 0, // system node id - }, - }, - numaNodes: []system.Node{ - &mockSystemNode{id: 0, memFree: 9999, memTotal: 9999}, - }, - req: &request{ - memReq: 10000, - memLim: 10000, - memType: defaultMemoryType, - container: &mockContainer{}, - }, - expectedRemainingNodes: []int{}, - tree: map[int][]int{100: {}}, - }, - { - name: "two node memory limit (fits to leaf)", - nodes: []Node{ - &virtualnode{ - node: node{ - id: 100, - name: "testnode0", - kind: UnknownNode, - noderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(10001, 0, 0), createMemoryMap(0, 0, 0)), - freeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(10001, 0, 0), createMemoryMap(0, 0, 0)), - }, - }, - &numanode{ - node: node{ - id: 101, - name: "testnode1", - kind: UnknownNode, - noderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(10001, 0, 0), createMemoryMap(0, 0, 0)), - freeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(10001, 0, 0), createMemoryMap(0, 0, 0)), - }, - id: 0, // system node id - }, - }, - numaNodes: []system.Node{ - &mockSystemNode{id: 0, memFree: 10001, memTotal: 10001}, - }, - req: &request{ - memReq: 10000, - memLim: 10000, - memType: defaultMemoryType, - container: &mockContainer{}, - }, - expectedRemainingNodes: []int{100, 101}, - tree: map[int][]int{100: {101}, 101: {}}, - }, - { - name: "three node memory limit (fits to root)", - nodes: []Node{ - &virtualnode{ - node: node{ - id: 100, - name: "testnode0", - kind: UnknownNode, - noderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(12000, 0, 0), createMemoryMap(0, 0, 0)), - freeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(12000, 0, 0), createMemoryMap(0, 0, 0)), - }, - }, - &numanode{ - node: node{ - id: 101, - name: "testnode1", - kind: UnknownNode, - noderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(6000, 0, 0), createMemoryMap(0, 0, 0)), - freeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(6000, 0, 0), createMemoryMap(0, 0, 0)), - }, - id: 0, // system node id - }, - &numanode{ - node: node{ - id: 102, - name: "testnode2", - kind: UnknownNode, - noderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(6000, 0, 0), createMemoryMap(0, 0, 0)), - freeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(6000, 0, 0), createMemoryMap(0, 0, 0)), - }, - id: 1, // system node id - }, - }, - numaNodes: []system.Node{ - &mockSystemNode{id: 0, memFree: 6000, memTotal: 6000}, - &mockSystemNode{id: 1, memFree: 6000, memTotal: 6000}, - }, - req: &request{ - memReq: 10000, - memLim: 10000, - memType: defaultMemoryType, - container: &mockContainer{}, - }, - expectedRemainingNodes: []int{100}, - tree: map[int][]int{100: {101, 102}, 101: {}, 102: {}}, - }, - } - for _, tc := range tcases { - t.Run(tc.name, func(t *testing.T) { - setLinks(tc.nodes, tc.tree) - policy := &policy{ - sys: &mockSystem{ - nodes: tc.numaNodes, - }, - pools: tc.nodes, - cache: &mockCache{}, - root: tc.nodes[0], - nodeCnt: len(tc.nodes), - allocations: allocations{}, - cpuAllocator: &mockCPUAllocator{}, - } - // back pointers - for _, node := range tc.nodes { - switch node.(type) { - case *numanode: - numaNode := node.(*numanode) - numaNode.self.node = numaNode - noderes := numaNode.noderes.(*supply) - noderes.node = node - freeres := numaNode.freeres.(*supply) - freeres.node = node - numaNode.policy = policy - case *virtualnode: - virtualNode := node.(*virtualnode) - virtualNode.self.node = virtualNode - noderes := virtualNode.noderes.(*supply) - noderes.node = node - freeres := virtualNode.freeres.(*supply) - freeres.node = node - virtualNode.policy = policy - } - } - policy.allocations.policy = policy - - scores, filteredPools := policy.sortPoolsByScore(tc.req, tc.affinities) - fmt.Printf("scores: %v, remaining pools: %v\n", scores, filteredPools) - - if len(filteredPools) != len(tc.expectedRemainingNodes) { - t.Errorf("Wrong nodes in the filtered pool: expected %v but got %v", tc.expectedRemainingNodes, filteredPools) - } - - for _, id := range tc.expectedRemainingNodes { - found := false - for _, node := range filteredPools { - if node.NodeID() == id { - found = true - break - } - } - if !found { - t.Errorf("Did not find id %d in filtered pools: %v", id, filteredPools) - } - } - }) - } -} - func TestPoolCreation(t *testing.T) { // Test pool creation with "real" sysfs data. @@ -553,175 +339,6 @@ func TestWorkloadPlacement(t *testing.T) { } } -func TestContainerMove(t *testing.T) { - - // In case there's not enough memory to guarantee that the - // containers running on child nodes won't get OOM killed, they need - // to be moved upwards in the tree. - - // Create a temporary directory for the test data. - dir, err := ioutil.TempDir("", "nri-resource-policy-test-sysfs-") - if err != nil { - panic(err) - } - defer os.RemoveAll(dir) - - // Uncompress the test data to the directory. - err = utils.UncompressTbz2(path.Join("testdata", "sysfs.tar.bz2"), dir) - if err != nil { - panic(err) - } - - tcases := []struct { - path string - name string - container1 cache.Container - container2 cache.Container - container3 cache.Container - affinities map[int]int32 - expectedLeafNodeForContainer1 bool - expectedLeafNodeForContainer2 bool - expectedLeafNodeForContainer3 bool - expectedChangeForContainer1 bool - expectedChangeForContainer2 bool - expectedChangeForContainer3 bool - }{ - { - path: path.Join(dir, "sysfs", "server", "sys"), - name: "workload placement on a server system leaf node", - container1: &mockContainer{ - returnValueForGetResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: resapi.MustParse("2"), - v1.ResourceMemory: resapi.MustParse("1000"), - }, - }, - returnValueForGetID: "first", - }, - container2: &mockContainer{ - returnValueForGetResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: resapi.MustParse("2"), - v1.ResourceMemory: resapi.MustParse("1000"), - }, - }, - returnValueForGetID: "second", - }, - container3: &mockContainer{ - returnValueForGetResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: resapi.MustParse("2"), - v1.ResourceMemory: resapi.MustParse("1000"), - }, - }, - returnValueForGetID: "third", - }, - expectedLeafNodeForContainer1: true, - expectedLeafNodeForContainer2: true, - expectedLeafNodeForContainer3: true, - }, - { - path: path.Join(dir, "sysfs", "server", "sys"), - name: "workload placement on a server system non-leaf node", - container1: &mockContainer{ - name: "c1", - returnValueForGetResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: resapi.MustParse("2"), - v1.ResourceMemory: resapi.MustParse("1000"), - }, - }, - returnValueForGetID: "first", - }, - container2: &mockContainer{ - name: "c2", - returnValueForGetResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: resapi.MustParse("2"), - v1.ResourceMemory: resapi.MustParse("190000000000"), // 180 GB - }, - }, - returnValueForGetID: "second", - }, - container3: &mockContainer{ - name: "c3", - returnValueForGetResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: resapi.MustParse("2"), - v1.ResourceMemory: resapi.MustParse("140000000000"), // 130 GB - }, - }, - returnValueForGetID: "third", - }, - expectedLeafNodeForContainer1: false, - expectedLeafNodeForContainer2: false, - expectedLeafNodeForContainer3: false, - expectedChangeForContainer1: true, - }, - } - for _, tc := range tcases { - t.Run(tc.name, func(t *testing.T) { - sys, err := system.DiscoverSystemAt(tc.path) - if err != nil { - panic(err) - } - - policyOptions := &policyapi.BackendOptions{ - Cache: &mockCache{}, - System: sys, - Config: &cfgapi.Config{ - ReservedResources: cfgapi.Constraints{ - cfgapi.CPU: "750m", - }, - }, - } - - log.EnableDebug(true) - policy := New().(*policy) - policy.Setup(policyOptions) - log.EnableDebug(false) - - grant1, err := policy.allocatePool(tc.container1, "") - if err != nil { - panic(err) - } - fmt.Printf("grant 1 memsets: dram %s, pmem %s\n", grant1.GetMemoryNode().GetMemset(memoryDRAM), grant1.GetMemoryNode().GetMemset(memoryPMEM)) - - grant2, err := policy.allocatePool(tc.container2, "") - if err != nil { - panic(err) - } - fmt.Printf("grant 2 memsets: dram %s, pmem %s\n", grant2.GetMemoryNode().GetMemset(memoryDRAM), grant2.GetMemoryNode().GetMemset(memoryPMEM)) - - grant3, err := policy.allocatePool(tc.container3, "") - if err != nil { - panic(err) - } - fmt.Printf("grant 3 memsets: dram %s, pmem %s\n", grant3.GetMemoryNode().GetMemset(memoryDRAM), grant3.GetMemoryNode().GetMemset(memoryPMEM)) - - if (grant1.GetCPUNode().IsSameNode(grant1.GetMemoryNode())) && tc.expectedChangeForContainer1 { - t.Errorf("Workload 1 should have been relocated: %t, node: %s", tc.expectedChangeForContainer1, grant1.GetMemoryNode().Name()) - } - if (grant2.GetCPUNode().IsSameNode(grant2.GetMemoryNode())) && tc.expectedChangeForContainer2 { - t.Errorf("Workload 2 should have been relocated: %t, node: %s", tc.expectedChangeForContainer2, grant2.GetMemoryNode().Name()) - } - if (grant3.GetCPUNode().IsSameNode(grant3.GetMemoryNode())) && tc.expectedChangeForContainer3 { - t.Errorf("Workload 3 should have been relocated: %t, node: %s", tc.expectedChangeForContainer3, grant3.GetMemoryNode().Name()) - } - - if grant1.GetMemoryNode().IsLeafNode() != tc.expectedLeafNodeForContainer1 { - t.Errorf("Workload 1 should have been placed in a leaf node: %t, node: %s", tc.expectedLeafNodeForContainer1, grant1.GetMemoryNode().Name()) - } - if grant2.GetMemoryNode().IsLeafNode() != tc.expectedLeafNodeForContainer2 { - t.Errorf("Workload 2 should have been placed in a leaf node: %t, node: %s", tc.expectedLeafNodeForContainer2, grant2.GetMemoryNode().Name()) - } - if grant3.GetMemoryNode().IsLeafNode() != tc.expectedLeafNodeForContainer3 { - t.Errorf("Workload 3 should have been placed in a leaf node: %t, node: %s", tc.expectedLeafNodeForContainer3, grant3.GetMemoryNode().Name()) - } - }) - } -} - func TestAffinities(t *testing.T) { // // Test how (already pre-calculated) affinities affect workload placement.