Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix the fallback for container metrics logic to query both container and pod metrics #789

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Fix the fallback for container metrics logic to query both container and pod metrics #789

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fbfa0e7
Use one_of operator for the fallback for container metrics logic
nabil-dbz Nov 5, 2024
a6e7f58
Add resource.labels to the allowlist of prefixes for custom metrics
nabil-dbz Nov 5, 2024
d25ecd2
Add ContainerType in addition to PodContainerType
nabil-dbz Nov 5, 2024
f85193b
Allow the label resource.labels.container_name instead of all labels …
nabil-dbz Nov 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion custom-metrics-stackdriver-adapter/pkg/adapter/coreprovider/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func (p *stackdriverCoreClient) getPodMetric(podsNames []string, metricName, met
segmentEnd := min((i+1)*translator.MaxNumOfArgsInOneOfFilter, len(podsNames))

stackdriverRequest, err := translator.NewQueryBuilder(p.translator, metricName).
AsContainerType().
EnforceContainerType().
WithPodNames(podsNames[segmentBeg:segmentEnd]).
WithMetricKind(metricKind).
WithMetricValueType(metricValueType).
Expand Down
87 changes: 35 additions & 52 deletions custom-metrics-stackdriver-adapter/pkg/adapter/provider/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,9 @@ func (p *StackdriverProvider) getNamespacedMetricByName(groupResource schema.Gro
return nil, err
}
queryBuilder := translator.NewQueryBuilder(p.translator, metricName)
if p.fallbackForContainerMetrics {
queryBuilder = queryBuilder.EnforcePodContainerType()
}

pods := &v1.PodList{Items: []v1.Pod{*matchingPod}}
stackdriverRequest, err := queryBuilder.
Expand All @@ -206,27 +209,9 @@ func (p *StackdriverProvider) getNamespacedMetricByName(groupResource schema.Gro
return nil, err
}

if p.fallbackForContainerMetrics && len(stackdriverResponse.TimeSeries) == 0 {
stackdriverRequest, err := queryBuilder.
AsContainerType().
WithPods(pods).
WithMetricKind(metricKind).
WithMetricValueType(metricValueType).
WithMetricSelector(metricSelector).
WithNamespace(namespace).
Build()
if err != nil {
return nil, err
}
stackdriverResponse, err = stackdriverRequest.Do()
if err != nil {
return nil, err
}

err = p.translator.CheckMetricUniquenessForPod(stackdriverResponse, escapedMetricName)
if err != nil {
return nil, err
}
stackdriverResponse, err = p.PostProcessPodContainerResp(stackdriverResponse, escapedMetricName)
if err != nil {
return nil, err
}
return p.translator.GetRespForSingleObject(stackdriverResponse, groupResource, escapedMetricName, metricSelector, namespace, name)
}
Expand Down Expand Up @@ -255,6 +240,10 @@ func (p *StackdriverProvider) getNamespacedMetricBySelector(groupResource schema
WithMetricValueType(metricValueType).
WithNamespace(namespace)

if p.fallbackForContainerMetrics {
queryBuilder = queryBuilder.EnforcePodContainerType()
}

result := []custom_metrics.MetricValue{}
for i := 0; i < len(matchingPods.Items); i += translator.MaxNumOfArgsInOneOfFilter {
sliceSegmentEnd := min(i+translator.MaxNumOfArgsInOneOfFilter, len(matchingPods.Items))
Expand All @@ -267,44 +256,17 @@ func (p *StackdriverProvider) getNamespacedMetricBySelector(groupResource schema
if err != nil {
return nil, err
}
stackdriverResponse, err = p.PostProcessPodContainerResp(stackdriverResponse, escapedMetricName)
if err != nil {
return nil, err
}
slice, err := p.translator.GetRespForMultipleObjects(stackdriverResponse, p.translator.GetPodItems(matchingPods), groupResource, escapedMetricName, metricSelector)
if err != nil {
return nil, err
}
result = append(result, slice...)
}

if p.fallbackForContainerMetrics && len(result) == 0 {
for i := 0; i < len(matchingPods.Items); i += translator.MaxNumOfArgsInOneOfFilter {
sliceSegmentEnd := min(i+translator.MaxNumOfArgsInOneOfFilter, len(matchingPods.Items))
podsSlice := &v1.PodList{Items: matchingPods.Items[i:sliceSegmentEnd]}
stackdriverRequest, err := translator.NewQueryBuilder(p.translator, metricName).
AsContainerType().
WithPods(podsSlice).
WithMetricKind(metricKind).
WithMetricValueType(metricValueType).
WithMetricSelector(metricSelector).
WithNamespace(namespace).
Build()
if err != nil {
return nil, err
}
stackdriverResponse, err := stackdriverRequest.Do()
if err != nil {
return nil, err
}
err = p.translator.CheckMetricUniquenessForPod(stackdriverResponse, escapedMetricName)
if err != nil {
return nil, err
}
slice, err := p.translator.GetRespForMultipleObjects(stackdriverResponse, p.translator.GetPodItems(matchingPods), groupResource, escapedMetricName, metricSelector)
if err != nil {
return nil, err
}
result = append(result, slice...)
}
}

return &custom_metrics.MetricValueList{Items: result}, nil
}

Expand Down Expand Up @@ -349,6 +311,27 @@ func (p *StackdriverProvider) ListAllExternalMetrics() []provider.ExternalMetric
}
}

func (p *StackdriverProvider) PostProcessPodContainerResp(response *stackdriver.ListTimeSeriesResponse, metricName string) (*stackdriver.ListTimeSeriesResponse, error) {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should really document what this does because modifying a response object here could be really confusing.

I was initially going to suggest not returning a ListTimeSeriesResponse here but I see that this would cause a bunch of issues with the follow-up calls.

podTimeSeries := []*stackdriver.TimeSeries{}
containerTimeSeries := []*stackdriver.TimeSeries{}
for _, series := range response.TimeSeries {
if series.Resource.Type == "k8s_pod" {
podTimeSeries = append(podTimeSeries, series)
continue
}
if series.Resource.Type == "k8s_container" {
containerTimeSeries = append(podTimeSeries, series)
continue
}
}
response.TimeSeries = podTimeSeries
if len(podTimeSeries) != 0 || !p.fallbackForContainerMetrics {
return response, nil
}
response.TimeSeries = containerTimeSeries
return response, p.translator.CheckMetricUniquenessForPod(response, metricName)
}

func min(a, b int) int {
if a < b {
return a
Expand Down
40 changes: 27 additions & 13 deletions custom-metrics-stackdriver-adapter/pkg/adapter/translator/query_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ var (
allowedExternalMetricsFullLabelNames = []string{"resource.type", "reducer"}
// allowedCustomMetricsLabelPrefixes and allowedCustomMetricsFullLabelNames specify all metric labels allowed for querying
allowedCustomMetricsLabelPrefixes = []string{"metric.labels"}
allowedCustomMetricsFullLabelNames = []string{"reducer"}
allowedCustomMetricsFullLabelNames = []string{"resource.labels.container_name", "reducer"}
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you can omit resource.labels here, can't you? since it will be picked up by the prefix above?

allowedReducers = map[string]bool{
"REDUCE_NONE": true,
"REDUCE_MEAN": true,
Expand Down Expand Up @@ -169,15 +169,16 @@ func (nc nodeValues) getNodeNames() []string {
//
// use NewQueryBuilder() to initialize
type QueryBuilder struct {
translator *Translator // translator provides configurations to filter
metricName string // metricName is the metric name to filter
metricKind string // metricKind is the metric kind to filter
metricValueType string // metricValueType is the metric value type to filter
metricSelector labels.Selector // metricSelector is the metric selector to filtere
namespace string // namespace is the namespace to filter (mutually exclusive with nodes)
pods podValues // pods is the pods to filter (mutually exclusive with nodes)
nodes nodeValues // nodes is the nodes to filter (mutually exclusive with namespace, pods)
enforceContainerType bool // enforceContainerType decides whether to enforce using container type filter schema
translator *Translator // translator provides configurations to filter
metricName string // metricName is the metric name to filter
metricKind string // metricKind is the metric kind to filter
metricValueType string // metricValueType is the metric value type to filter
metricSelector labels.Selector // metricSelector is the metric selector to filtere
namespace string // namespace is the namespace to filter (mutually exclusive with nodes)
pods podValues // pods is the pods to filter (mutually exclusive with nodes)
nodes nodeValues // nodes is the nodes to filter (mutually exclusive with namespace, pods)
enforceContainerType bool // enforceContainerType decides whether to enforce using container type filter schema
enforcePodContainerType bool // enforcePodContainerType decides wether to to enforce using pod_container type filter schema
}

// NewQueryBuilder is the initiator for QueryBuilder
Expand Down Expand Up @@ -314,14 +315,22 @@ func (qb QueryBuilder) getResourceNames() []string {
return qb.pods.getPodIDs()
}

// AsContainerType enforces to query k8s_container type metrics
// EnforceContainerType enforces to query k8s_container type metrics
//
// it it valid only when useNewResourceModel is true
func (qb QueryBuilder) AsContainerType() QueryBuilder {
func (qb QueryBuilder) EnforceContainerType() QueryBuilder {
qb.enforceContainerType = true
return qb
}

// EnforcePodContainerType enforces to query both k8s_pod and k8s_container type metrics
//
// it it valid only when useNewResourceModel is true
func (qb QueryBuilder) EnforcePodContainerType() QueryBuilder {
qb.enforcePodContainerType = true
return qb
}

// validate is an internal helper function for checking prerequisits before Build
//
// Criteria:
Expand Down Expand Up @@ -366,7 +375,7 @@ func (qb QueryBuilder) validate() error {
return apierr.NewBadRequest("distributions are not supported")
}

if qb.enforceContainerType && !qb.translator.useNewResourceModel {
if !qb.translator.useNewResourceModel && (qb.enforceContainerType || qb.enforcePodContainerType) {
return apierr.NewInternalError(fmt.Errorf("illegal state! Container metrics works only with new resource model"))
}

Expand All @@ -392,6 +401,11 @@ func (qb QueryBuilder) getFilterBuilder() utils.FilterBuilder {
return utils.NewFilterBuilder(utils.SchemaTypes[utils.ContainerSchemaKey])
}

// pod_container type FilterBuilder
if qb.enforcePodContainerType {
return utils.NewFilterBuilder(utils.SchemaTypes[utils.PodContainerSchemaKey])
}

// prometheus type FilterBuilder
if strings.HasPrefix(qb.metricName, PrometheusMetricPrefix) {
return utils.NewFilterBuilder(utils.SchemaTypes[utils.PrometheusSchemaKey])
Expand Down
Loading
Loading