Skip to content

Commit

Permalink
fix big memory overflow
Browse files Browse the repository at this point in the history
  • Loading branch information
@leiwingqueen
leiwingqueen committed Nov 1, 2024
1 parent 3c6bc83 commit 31f7931
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 9 deletions.
29 changes: 20 additions & 9 deletions internal/store/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ package store

import (
"context"
"k8s.io/apimachinery/pkg/api/resource"
"math"
"strings"

basemetrics "k8s.io/component-base/metrics"
Expand Down Expand Up @@ -353,7 +355,7 @@ func createNodeStatusAllocatableFamilyGenerator() generator.FamilyGenerator {
SanitizeLabelName(string(resourceName)),
string(constant.UnitByte),
},
Value: float64(val.MilliValue()) / 1000,
Value: convertFloat64(&val),
})
}
if isAttachableVolumeResourceName(resourceName) {
Expand All @@ -362,7 +364,7 @@ func createNodeStatusAllocatableFamilyGenerator() generator.FamilyGenerator {
SanitizeLabelName(string(resourceName)),
string(constant.UnitByte),
},
Value: float64(val.MilliValue()) / 1000,
Value: convertFloat64(&val),
})
}
if isExtendedResourceName(resourceName) {
Expand All @@ -371,7 +373,7 @@ func createNodeStatusAllocatableFamilyGenerator() generator.FamilyGenerator {
SanitizeLabelName(string(resourceName)),
string(constant.UnitInteger),
},
Value: float64(val.MilliValue()) / 1000,
Value: convertFloat64(&val),
})
}
}
Expand Down Expand Up @@ -407,7 +409,7 @@ func createNodeStatusCapacityFamilyGenerator() generator.FamilyGenerator {
SanitizeLabelName(string(resourceName)),
string(constant.UnitCore),
},
Value: float64(val.MilliValue()) / 1000,
Value: convertFloat64(&val),
})
case v1.ResourceStorage:
fallthrough
Expand All @@ -419,15 +421,15 @@ func createNodeStatusCapacityFamilyGenerator() generator.FamilyGenerator {
SanitizeLabelName(string(resourceName)),
string(constant.UnitByte),
},
Value: float64(val.MilliValue()) / 1000,
Value: convertFloat64(&val),
})
case v1.ResourcePods:
ms = append(ms, &metric.Metric{
LabelValues: []string{
SanitizeLabelName(string(resourceName)),
string(constant.UnitInteger),
},
Value: float64(val.MilliValue()) / 1000,
Value: convertFloat64(&val),
})
default:
if isHugePageResourceName(resourceName) {
Expand All @@ -436,7 +438,7 @@ func createNodeStatusCapacityFamilyGenerator() generator.FamilyGenerator {
SanitizeLabelName(string(resourceName)),
string(constant.UnitByte),
},
Value: float64(val.MilliValue()) / 1000,
Value: convertFloat64(&val),
})
}
if isAttachableVolumeResourceName(resourceName) {
Expand All @@ -445,7 +447,7 @@ func createNodeStatusCapacityFamilyGenerator() generator.FamilyGenerator {
SanitizeLabelName(string(resourceName)),
string(constant.UnitByte),
},
Value: float64(val.MilliValue()) / 1000,
Value: convertFloat64(&val),
})
}
if isExtendedResourceName(resourceName) {
Expand All @@ -454,7 +456,7 @@ func createNodeStatusCapacityFamilyGenerator() generator.FamilyGenerator {
SanitizeLabelName(string(resourceName)),
string(constant.UnitInteger),
},
Value: float64(val.MilliValue()) / 1000,
Value: convertFloat64(&val),
})
}
}
Expand Down Expand Up @@ -530,3 +532,12 @@ func createNodeListWatch(kubeClient clientset.Interface, _ string, _ string) cac
},
}
}

const maxValue = math.MaxInt64 / 1000

func convertFloat64(q *resource.Quantity) float64 {
if q.Value() > maxValue {
return float64(q.Value())
}
return float64(q.MilliValue()) / 1000
}
88 changes: 88 additions & 0 deletions internal/store/node_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,94 @@ func TestNodeStore(t *testing.T) {
`,
MetricNames: []string{"kube_node_status_addresses"},
},
// memory overflow test
{
Obj: &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "127.0.0.1",
CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)},
Labels: map[string]string{
"node-role.kubernetes.io/master": "",
},
},
Spec: v1.NodeSpec{
Unschedulable: true,
ProviderID: "provider://i-randomidentifier",
PodCIDR: "172.24.10.0/24",
},
Status: v1.NodeStatus{
NodeInfo: v1.NodeSystemInfo{
KernelVersion: "kernel",
KubeletVersion: "kubelet",
KubeProxyVersion: "kubeproxy",
OSImage: "osimage",
ContainerRuntimeVersion: "rkt",
SystemUUID: "6a934e21-5207-4a84-baea-3a952d926c80",
},
Addresses: []v1.NodeAddress{
{Type: "InternalIP", Address: "1.2.3.4"},
},
Capacity: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("4.3"),
// overflow test
v1.ResourceMemory: resource.MustParse("10000T"),
v1.ResourcePods: resource.MustParse("1000"),
v1.ResourceStorage: resource.MustParse("3G"),
v1.ResourceEphemeralStorage: resource.MustParse("4G"),
v1.ResourceName("nvidia.com/gpu"): resource.MustParse("4"),
},
Allocatable: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("3"),
v1.ResourceMemory: resource.MustParse("1G"),
v1.ResourcePods: resource.MustParse("555"),
v1.ResourceStorage: resource.MustParse("2G"),
v1.ResourceEphemeralStorage: resource.MustParse("3G"),
v1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"),
},
},
},
Want: `
# HELP kube_node_created [STABLE] Unix creation timestamp
# HELP kube_node_info [STABLE] Information about a cluster node.
# HELP kube_node_labels [STABLE] Kubernetes labels converted to Prometheus labels.
# HELP kube_node_role The role of a cluster node.
# HELP kube_node_spec_unschedulable [STABLE] Whether a node can schedule new pods.
# HELP kube_node_status_allocatable [STABLE] The allocatable for different resources of a node that are available for scheduling.
# HELP kube_node_status_capacity [STABLE] The capacity for different resources of a node.
# TYPE kube_node_created gauge
# TYPE kube_node_info gauge
# TYPE kube_node_labels gauge
# TYPE kube_node_role gauge
# TYPE kube_node_spec_unschedulable gauge
# TYPE kube_node_status_allocatable gauge
# TYPE kube_node_status_capacity gauge
kube_node_created{node="127.0.0.1"} 1.5e+09
kube_node_info{container_runtime_version="rkt",kernel_version="kernel",kubelet_version="kubelet",kubeproxy_version="deprecated",node="127.0.0.1",os_image="osimage",pod_cidr="172.24.10.0/24",provider_id="provider://i-randomidentifier",internal_ip="1.2.3.4",system_uuid="6a934e21-5207-4a84-baea-3a952d926c80"} 1
kube_node_role{node="127.0.0.1",role="master"} 1
kube_node_spec_unschedulable{node="127.0.0.1"} 1
kube_node_status_allocatable{node="127.0.0.1",resource="cpu",unit="core"} 3
kube_node_status_allocatable{node="127.0.0.1",resource="ephemeral_storage",unit="byte"} 3e+09
kube_node_status_allocatable{node="127.0.0.1",resource="memory",unit="byte"} 1e+09
kube_node_status_allocatable{node="127.0.0.1",resource="nvidia_com_gpu",unit="integer"} 1
kube_node_status_allocatable{node="127.0.0.1",resource="pods",unit="integer"} 555
kube_node_status_allocatable{node="127.0.0.1",resource="storage",unit="byte"} 2e+09
kube_node_status_capacity{node="127.0.0.1",resource="cpu",unit="core"} 4.3
kube_node_status_capacity{node="127.0.0.1",resource="ephemeral_storage",unit="byte"} 4e+09
kube_node_status_capacity{node="127.0.0.1",resource="memory",unit="byte"} 1e+16
kube_node_status_capacity{node="127.0.0.1",resource="nvidia_com_gpu",unit="integer"} 4
kube_node_status_capacity{node="127.0.0.1",resource="pods",unit="integer"} 1000
kube_node_status_capacity{node="127.0.0.1",resource="storage",unit="byte"} 3e+09
`,
MetricNames: []string{
"kube_node_status_capacity",
"kube_node_status_allocatable",
"kube_node_spec_unschedulable",
"kube_node_labels",
"kube_node_role",
"kube_node_info",
"kube_node_created",
},
},
}
for i, c := range cases {
c.Func = generator.ComposeMetricGenFuncs(nodeMetricFamilies(nil, nil))
Expand Down

0 comments on commit 31f7931

Please sign in to comment.