Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MQE: Add support for histogram_stdvar #10211

Merged
merged 1 commit into from
Dec 12, 2024
Merged

MQE: Add support for histogram_stdvar #10211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pkg/streamingpromql/engine_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2671,6 +2671,7 @@ func TestCompareVariousMixedMetricsFunctions(t *testing.T) {
expressions = append(expressions, fmt.Sprintf(`histogram_quantile(0.8, series{label=~"(%s)"})`, labelRegex))
expressions = append(expressions, fmt.Sprintf(`histogram_quantile(scalar(series{label="i"}), series{label=~"(%s)"})`, labelRegex))
expressions = append(expressions, fmt.Sprintf(`histogram_stddev(series{label=~"(%s)"})`, labelRegex))
expressions = append(expressions, fmt.Sprintf(`histogram_stdvar(series{label=~"(%s)"})`, labelRegex))
expressions = append(expressions, fmt.Sprintf(`histogram_sum(series{label=~"(%s)"})`, labelRegex))
}

Expand Down
3 changes: 2 additions & 1 deletion pkg/streamingpromql/functions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,8 @@ var instantVectorFunctionOperatorFactories = map[string]InstantVectorFunctionOpe
"histogram_count": InstantVectorTransformationFunctionOperatorFactory("histogram_count", functions.HistogramCount),
"histogram_fraction": HistogramFractionFunctionOperatorFactory(),
"histogram_quantile": HistogramQuantileFunctionOperatorFactory(),
"histogram_stddev": InstantVectorTransformationFunctionOperatorFactory("histogram_stddev", functions.HistogramStdDev),
"histogram_stddev": InstantVectorTransformationFunctionOperatorFactory("histogram_stddev", functions.HistogramStdDevStdVar(true)),
"histogram_stdvar": InstantVectorTransformationFunctionOperatorFactory("histogram_stdvar", functions.HistogramStdDevStdVar(false)),
"histogram_sum": InstantVectorTransformationFunctionOperatorFactory("histogram_sum", functions.HistogramSum),
"increase": FunctionOverRangeVectorOperatorFactory("increase", functions.Increase),
"label_replace": LabelReplaceFunctionOperatorFactory(),
Expand Down
81 changes: 43 additions & 38 deletions pkg/streamingpromql/operators/functions/native_histograms.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,51 +89,56 @@ func HistogramFraction(seriesData types.InstantVectorSeriesData, scalarArgsData
return data, nil
}

// HistogramStdDev returns the estimated standard deviation of observations in a native histogram
// Float values are ignored.
func HistogramStdDev(seriesData types.InstantVectorSeriesData, _ []types.ScalarData, _ types.QueryTimeRange, memoryConsumptionTracker *limiting.MemoryConsumptionTracker) (types.InstantVectorSeriesData, error) {
fPoints, err := types.FPointSlicePool.Get(len(seriesData.Histograms), memoryConsumptionTracker)
if err != nil {
return types.InstantVectorSeriesData{}, err
}
func HistogramStdDevStdVar(isStdDev bool) InstantVectorSeriesFunction {
// returns either the standard deviation, or standard variance of a native histogram.
// Float values are ignored.
Comment on lines +93 to +94
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this comment in the right place? Looks like it should be the docstring for HistogramStdDevStdVar.

return func(seriesData types.InstantVectorSeriesData, _ []types.ScalarData, _ types.QueryTimeRange, memoryConsumptionTracker *limiting.MemoryConsumptionTracker) (types.InstantVectorSeriesData, error) {
fPoints, err := types.FPointSlicePool.Get(len(seriesData.Histograms), memoryConsumptionTracker)
if err != nil {
return types.InstantVectorSeriesData{}, err
}

data := types.InstantVectorSeriesData{
Floats: fPoints,
}
data := types.InstantVectorSeriesData{
Floats: fPoints,
}

for _, histogram := range seriesData.Histograms {
mean := histogram.H.Sum / histogram.H.Count
var variance, cVariance float64
it := histogram.H.AllBucketIterator()
for it.Next() {
bucket := it.At()
if bucket.Count == 0 {
continue
}
var val float64
if bucket.Lower <= 0 && 0 <= bucket.Upper {
val = 0
} else {
val = math.Sqrt(bucket.Upper * bucket.Lower)
if bucket.Upper < 0 {
val = -val
for _, histogram := range seriesData.Histograms {
mean := histogram.H.Sum / histogram.H.Count
var variance, cVariance float64
it := histogram.H.AllBucketIterator()
for it.Next() {
bucket := it.At()
if bucket.Count == 0 {
continue
}
var val float64
if bucket.Lower <= 0 && 0 <= bucket.Upper {
val = 0
} else {
val = math.Sqrt(bucket.Upper * bucket.Lower)
if bucket.Upper < 0 {
val = -val
}
}
delta := val - mean
variance, cVariance = floats.KahanSumInc(bucket.Count*delta*delta, variance, cVariance)
}
variance += cVariance
variance /= histogram.H.Count
if isStdDev {
variance = math.Sqrt(variance)
}
delta := val - mean
variance, cVariance = floats.KahanSumInc(bucket.Count*delta*delta, variance, cVariance)
}
variance += cVariance
variance /= histogram.H.Count

data.Floats = append(data.Floats, promql.FPoint{
T: histogram.T,
F: math.Sqrt(variance),
})
}
data.Floats = append(data.Floats, promql.FPoint{
T: histogram.T,
F: variance,
})
}

types.PutInstantVectorSeriesData(seriesData, memoryConsumptionTracker)
types.PutInstantVectorSeriesData(seriesData, memoryConsumptionTracker)

return data, nil
return data, nil
}
}

func HistogramSum(seriesData types.InstantVectorSeriesData, _ []types.ScalarData, _ types.QueryTimeRange, memoryConsumptionTracker *limiting.MemoryConsumptionTracker) (types.InstantVectorSeriesData, error) {
Expand Down
17 changes: 17 additions & 0 deletions pkg/streamingpromql/testdata/ours/native_histograms.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ eval range from 0 to 5m step 1m histogram_fraction(0, 2, single_histogram)
eval range from 0 to 5m step 1m histogram_stddev(single_histogram)
{} 0.842629429717281 0.842629429717281 0.842629429717281 0.842629429717281 0.842629429717281 2.986282214238901

eval range from 0 to 5m step 1m histogram_stdvar(single_histogram)
{} 0.7100243558256704 0.7100243558256704 0.7100243558256704 0.7100243558256704 0.7100243558256704 8.917881463079594

# histogram_sum extracts the sum property from the histogram.
eval range from 0 to 5m step 1m histogram_sum(single_histogram)
{} 5 5 5 5 5 20
Expand All @@ -52,6 +55,9 @@ eval instant at 3m histogram_fraction(0, 1, mixed_metric)
eval instant at 4m histogram_stddev(mixed_metric)
{} 0.6650352854715079

eval instant at 4m histogram_stdvar(mixed_metric)
{} 0.44227193092217004

eval instant at 4m histogram_sum(mixed_metric)
{} 8

Expand All @@ -73,6 +79,9 @@ eval instant at 2m histogram_fraction(0, 1, mixed_metric)
# histogram_stddev ignores any float values
eval instant at 2m histogram_stddev(mixed_metric)

# histogram_stdvar ignores any float values
eval instant at 2m histogram_stdvar(mixed_metric)

# histogram_sum ignores any float values
eval instant at 2m histogram_sum(mixed_metric)

Expand Down Expand Up @@ -109,6 +118,11 @@ eval instant at 0 histogram_stddev(route)
{path="two"} 0.8415900492770793
{path="three"} 1.1865698706402301

eval instant at 0 histogram_stdvar(route)
{path="one"} 0.7100243558256704
{path="two"} 0.7082738110421968
{path="three"} 1.4079480579111723

eval instant at 0 histogram_sum(route)
{path="one"} 5
{path="two"} 10
Expand Down Expand Up @@ -149,6 +163,9 @@ eval range from 0 to 8m step 1m histogram_fraction(0, 1, mixed_metric)
eval range from 0 to 8m step 1m histogram_stddev(mixed_metric)
{} _ _ _ _ _ _ _ 0.8574122997574659 0.8574122997574659

eval range from 0 to 8m step 1m histogram_stdvar(mixed_metric)
{} _ _ _ _ _ _ _ 0.7351558517753866 0.7351558517753866

eval range from 0 to 8m step 1m histogram_sum(mixed_metric)
{} _ _ _ _ _ _ _ 18 18

Expand Down
14 changes: 6 additions & 8 deletions pkg/streamingpromql/testdata/upstream/histograms.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,16 +99,14 @@ eval instant at 50m histogram_avg(testhistogram3)
{start="negative"} 4

# Test histogram_stddev. This has no classic equivalent.
# Unsupported by streaming engine.
# eval instant at 50m histogram_stddev(testhistogram3)
# {start="positive"} 2.8189265757336734
# {start="negative"} 4.182715937754936
eval instant at 50m histogram_stddev(testhistogram3)
{start="positive"} 2.8189265757336734
{start="negative"} 4.182715937754936

# Test histogram_stdvar. This has no classic equivalent.
# Unsupported by streaming engine.
# eval instant at 50m histogram_stdvar(testhistogram3)
# {start="positive"} 7.946347039377573
# {start="negative"} 17.495112615949154
eval instant at 50m histogram_stdvar(testhistogram3)
{start="positive"} 7.946347039377573
{start="negative"} 17.495112615949154

# Test histogram_fraction.

Expand Down
40 changes: 16 additions & 24 deletions pkg/streamingpromql/testdata/upstream/native_histograms.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,9 +320,8 @@ load 10m
eval instant at 10m histogram_stddev(histogram_stddev_stdvar_1)
{} 1.0787993180043811

# Unsupported by streaming engine.
# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_1)
# {} 1.163807968526718
eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_1)
{} 1.163807968526718

clear

Expand All @@ -333,9 +332,8 @@ load 10m
eval instant at 10m histogram_stddev(histogram_stddev_stdvar_2)
{} 0.0048960313898237465

# Unsupported by streaming engine.
# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_2)
# {} 2.3971123370139447e-05
eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_2)
{} 2.3971123370139447e-05

clear

Expand All @@ -346,9 +344,8 @@ load 10m
eval instant at 10m histogram_stddev(histogram_stddev_stdvar_3)
{} 42.947236400258

# Unsupported by streaming engine.
# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_3)
# {} 1844.4651144196398
eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_3)
{} 1844.4651144196398

clear

Expand All @@ -359,9 +356,8 @@ load 10m
eval instant at 10m histogram_stddev(histogram_stddev_stdvar_4)
{} 27556.344499842

# Unsupported by streaming engine.
# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_4)
# {} 759352122.1939945
eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_4)
{} 759352122.1939945

clear

Expand All @@ -372,9 +368,8 @@ load 10m
eval instant at 10m histogram_stddev(histogram_stddev_stdvar_5)
{} 1.3137084989848

# Unsupported by streaming engine.
# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_5)
# {} 1.725830020304794
eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_5)
{} 1.725830020304794

clear

Expand All @@ -385,9 +380,8 @@ load 10m
eval instant at 10m histogram_stddev(histogram_stddev_stdvar_6)
{} NaN

# Unsupported by streaming engine.
# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_6)
# {} NaN
eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_6)
{} NaN

clear

Expand All @@ -398,9 +392,8 @@ load 10m
eval instant at 10m histogram_stddev(histogram_stddev_stdvar_7)
{} Inf

# Unsupported by streaming engine.
# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_7)
# {} Inf
eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_7)
{} Inf

clear

Expand Down Expand Up @@ -1102,9 +1095,8 @@ eval instant at 5m histogram_stddev(rate(const_histogram[5m]))
{} NaN

# Zero buckets mean no observations, so there is no standard variance.
# Unsupported by streaming engine.
# eval instant at 5m histogram_stdvar(rate(const_histogram[5m]))
# {} NaN
eval instant at 5m histogram_stdvar(rate(const_histogram[5m]))
{} NaN

clear

Expand Down
Loading