diff --git a/pkg/streamingpromql/engine_test.go b/pkg/streamingpromql/engine_test.go index d3e4ff2dfc..0cce9cc607 100644 --- a/pkg/streamingpromql/engine_test.go +++ b/pkg/streamingpromql/engine_test.go @@ -2671,6 +2671,7 @@ func TestCompareVariousMixedMetricsFunctions(t *testing.T) { expressions = append(expressions, fmt.Sprintf(`histogram_quantile(0.8, series{label=~"(%s)"})`, labelRegex)) expressions = append(expressions, fmt.Sprintf(`histogram_quantile(scalar(series{label="i"}), series{label=~"(%s)"})`, labelRegex)) expressions = append(expressions, fmt.Sprintf(`histogram_stddev(series{label=~"(%s)"})`, labelRegex)) + expressions = append(expressions, fmt.Sprintf(`histogram_stdvar(series{label=~"(%s)"})`, labelRegex)) expressions = append(expressions, fmt.Sprintf(`histogram_sum(series{label=~"(%s)"})`, labelRegex)) } diff --git a/pkg/streamingpromql/functions.go b/pkg/streamingpromql/functions.go index 77ed149dd7..e4229de917 100644 --- a/pkg/streamingpromql/functions.go +++ b/pkg/streamingpromql/functions.go @@ -371,7 +371,8 @@ var instantVectorFunctionOperatorFactories = map[string]InstantVectorFunctionOpe "histogram_count": InstantVectorTransformationFunctionOperatorFactory("histogram_count", functions.HistogramCount), "histogram_fraction": HistogramFractionFunctionOperatorFactory(), "histogram_quantile": HistogramQuantileFunctionOperatorFactory(), - "histogram_stddev": InstantVectorTransformationFunctionOperatorFactory("histogram_stddev", functions.HistogramStdDev), + "histogram_stddev": InstantVectorTransformationFunctionOperatorFactory("histogram_stddev", functions.HistogramStdDevStdVar(true)), + "histogram_stdvar": InstantVectorTransformationFunctionOperatorFactory("histogram_stdvar", functions.HistogramStdDevStdVar(false)), "histogram_sum": InstantVectorTransformationFunctionOperatorFactory("histogram_sum", functions.HistogramSum), "increase": FunctionOverRangeVectorOperatorFactory("increase", functions.Increase), "label_replace": LabelReplaceFunctionOperatorFactory(), diff --git a/pkg/streamingpromql/operators/functions/native_histograms.go b/pkg/streamingpromql/operators/functions/native_histograms.go index fe902acaec..65db6f158c 100644 --- a/pkg/streamingpromql/operators/functions/native_histograms.go +++ b/pkg/streamingpromql/operators/functions/native_histograms.go @@ -89,51 +89,56 @@ func HistogramFraction(seriesData types.InstantVectorSeriesData, scalarArgsData return data, nil } -// HistogramStdDev returns the estimated standard deviation of observations in a native histogram -// Float values are ignored. -func HistogramStdDev(seriesData types.InstantVectorSeriesData, _ []types.ScalarData, _ types.QueryTimeRange, memoryConsumptionTracker *limiting.MemoryConsumptionTracker) (types.InstantVectorSeriesData, error) { - fPoints, err := types.FPointSlicePool.Get(len(seriesData.Histograms), memoryConsumptionTracker) - if err != nil { - return types.InstantVectorSeriesData{}, err - } +func HistogramStdDevStdVar(isStdDev bool) InstantVectorSeriesFunction { + // returns either the standard deviation, or standard variance of a native histogram. + // Float values are ignored. + return func(seriesData types.InstantVectorSeriesData, _ []types.ScalarData, _ types.QueryTimeRange, memoryConsumptionTracker *limiting.MemoryConsumptionTracker) (types.InstantVectorSeriesData, error) { + fPoints, err := types.FPointSlicePool.Get(len(seriesData.Histograms), memoryConsumptionTracker) + if err != nil { + return types.InstantVectorSeriesData{}, err + } - data := types.InstantVectorSeriesData{ - Floats: fPoints, - } + data := types.InstantVectorSeriesData{ + Floats: fPoints, + } - for _, histogram := range seriesData.Histograms { - mean := histogram.H.Sum / histogram.H.Count - var variance, cVariance float64 - it := histogram.H.AllBucketIterator() - for it.Next() { - bucket := it.At() - if bucket.Count == 0 { - continue - } - var val float64 - if bucket.Lower <= 0 && 0 <= bucket.Upper { - val = 0 - } else { - val = math.Sqrt(bucket.Upper * bucket.Lower) - if bucket.Upper < 0 { - val = -val + for _, histogram := range seriesData.Histograms { + mean := histogram.H.Sum / histogram.H.Count + var variance, cVariance float64 + it := histogram.H.AllBucketIterator() + for it.Next() { + bucket := it.At() + if bucket.Count == 0 { + continue } + var val float64 + if bucket.Lower <= 0 && 0 <= bucket.Upper { + val = 0 + } else { + val = math.Sqrt(bucket.Upper * bucket.Lower) + if bucket.Upper < 0 { + val = -val + } + } + delta := val - mean + variance, cVariance = floats.KahanSumInc(bucket.Count*delta*delta, variance, cVariance) + } + variance += cVariance + variance /= histogram.H.Count + if isStdDev { + variance = math.Sqrt(variance) } - delta := val - mean - variance, cVariance = floats.KahanSumInc(bucket.Count*delta*delta, variance, cVariance) - } - variance += cVariance - variance /= histogram.H.Count - data.Floats = append(data.Floats, promql.FPoint{ - T: histogram.T, - F: math.Sqrt(variance), - }) - } + data.Floats = append(data.Floats, promql.FPoint{ + T: histogram.T, + F: variance, + }) + } - types.PutInstantVectorSeriesData(seriesData, memoryConsumptionTracker) + types.PutInstantVectorSeriesData(seriesData, memoryConsumptionTracker) - return data, nil + return data, nil + } } func HistogramSum(seriesData types.InstantVectorSeriesData, _ []types.ScalarData, _ types.QueryTimeRange, memoryConsumptionTracker *limiting.MemoryConsumptionTracker) (types.InstantVectorSeriesData, error) { diff --git a/pkg/streamingpromql/testdata/ours/native_histograms.test b/pkg/streamingpromql/testdata/ours/native_histograms.test index b943a715bf..cd80e98449 100644 --- a/pkg/streamingpromql/testdata/ours/native_histograms.test +++ b/pkg/streamingpromql/testdata/ours/native_histograms.test @@ -27,6 +27,9 @@ eval range from 0 to 5m step 1m histogram_fraction(0, 2, single_histogram) eval range from 0 to 5m step 1m histogram_stddev(single_histogram) {} 0.842629429717281 0.842629429717281 0.842629429717281 0.842629429717281 0.842629429717281 2.986282214238901 +eval range from 0 to 5m step 1m histogram_stdvar(single_histogram) + {} 0.7100243558256704 0.7100243558256704 0.7100243558256704 0.7100243558256704 0.7100243558256704 8.917881463079594 + # histogram_sum extracts the sum property from the histogram. eval range from 0 to 5m step 1m histogram_sum(single_histogram) {} 5 5 5 5 5 20 @@ -52,6 +55,9 @@ eval instant at 3m histogram_fraction(0, 1, mixed_metric) eval instant at 4m histogram_stddev(mixed_metric) {} 0.6650352854715079 +eval instant at 4m histogram_stdvar(mixed_metric) + {} 0.44227193092217004 + eval instant at 4m histogram_sum(mixed_metric) {} 8 @@ -73,6 +79,9 @@ eval instant at 2m histogram_fraction(0, 1, mixed_metric) # histogram_stddev ignores any float values eval instant at 2m histogram_stddev(mixed_metric) +# histogram_stdvar ignores any float values +eval instant at 2m histogram_stdvar(mixed_metric) + # histogram_sum ignores any float values eval instant at 2m histogram_sum(mixed_metric) @@ -109,6 +118,11 @@ eval instant at 0 histogram_stddev(route) {path="two"} 0.8415900492770793 {path="three"} 1.1865698706402301 +eval instant at 0 histogram_stdvar(route) + {path="one"} 0.7100243558256704 + {path="two"} 0.7082738110421968 + {path="three"} 1.4079480579111723 + eval instant at 0 histogram_sum(route) {path="one"} 5 {path="two"} 10 @@ -149,6 +163,9 @@ eval range from 0 to 8m step 1m histogram_fraction(0, 1, mixed_metric) eval range from 0 to 8m step 1m histogram_stddev(mixed_metric) {} _ _ _ _ _ _ _ 0.8574122997574659 0.8574122997574659 +eval range from 0 to 8m step 1m histogram_stdvar(mixed_metric) +{} _ _ _ _ _ _ _ 0.7351558517753866 0.7351558517753866 + eval range from 0 to 8m step 1m histogram_sum(mixed_metric) {} _ _ _ _ _ _ _ 18 18 diff --git a/pkg/streamingpromql/testdata/upstream/histograms.test b/pkg/streamingpromql/testdata/upstream/histograms.test index 100c2447f2..88ff72053e 100644 --- a/pkg/streamingpromql/testdata/upstream/histograms.test +++ b/pkg/streamingpromql/testdata/upstream/histograms.test @@ -99,16 +99,14 @@ eval instant at 50m histogram_avg(testhistogram3) {start="negative"} 4 # Test histogram_stddev. This has no classic equivalent. -# Unsupported by streaming engine. -# eval instant at 50m histogram_stddev(testhistogram3) -# {start="positive"} 2.8189265757336734 -# {start="negative"} 4.182715937754936 +eval instant at 50m histogram_stddev(testhistogram3) + {start="positive"} 2.8189265757336734 + {start="negative"} 4.182715937754936 # Test histogram_stdvar. This has no classic equivalent. -# Unsupported by streaming engine. -# eval instant at 50m histogram_stdvar(testhistogram3) -# {start="positive"} 7.946347039377573 -# {start="negative"} 17.495112615949154 +eval instant at 50m histogram_stdvar(testhistogram3) + {start="positive"} 7.946347039377573 + {start="negative"} 17.495112615949154 # Test histogram_fraction. diff --git a/pkg/streamingpromql/testdata/upstream/native_histograms.test b/pkg/streamingpromql/testdata/upstream/native_histograms.test index e983e1ff55..3616d45591 100644 --- a/pkg/streamingpromql/testdata/upstream/native_histograms.test +++ b/pkg/streamingpromql/testdata/upstream/native_histograms.test @@ -320,9 +320,8 @@ load 10m eval instant at 10m histogram_stddev(histogram_stddev_stdvar_1) {} 1.0787993180043811 -# Unsupported by streaming engine. -# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_1) -# {} 1.163807968526718 +eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_1) + {} 1.163807968526718 clear @@ -333,9 +332,8 @@ load 10m eval instant at 10m histogram_stddev(histogram_stddev_stdvar_2) {} 0.0048960313898237465 -# Unsupported by streaming engine. -# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_2) -# {} 2.3971123370139447e-05 +eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_2) + {} 2.3971123370139447e-05 clear @@ -346,9 +344,8 @@ load 10m eval instant at 10m histogram_stddev(histogram_stddev_stdvar_3) {} 42.947236400258 -# Unsupported by streaming engine. -# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_3) -# {} 1844.4651144196398 +eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_3) + {} 1844.4651144196398 clear @@ -359,9 +356,8 @@ load 10m eval instant at 10m histogram_stddev(histogram_stddev_stdvar_4) {} 27556.344499842 -# Unsupported by streaming engine. -# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_4) -# {} 759352122.1939945 +eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_4) + {} 759352122.1939945 clear @@ -372,9 +368,8 @@ load 10m eval instant at 10m histogram_stddev(histogram_stddev_stdvar_5) {} 1.3137084989848 -# Unsupported by streaming engine. -# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_5) -# {} 1.725830020304794 +eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_5) + {} 1.725830020304794 clear @@ -385,9 +380,8 @@ load 10m eval instant at 10m histogram_stddev(histogram_stddev_stdvar_6) {} NaN -# Unsupported by streaming engine. -# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_6) -# {} NaN +eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_6) + {} NaN clear @@ -398,9 +392,8 @@ load 10m eval instant at 10m histogram_stddev(histogram_stddev_stdvar_7) {} Inf -# Unsupported by streaming engine. -# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_7) -# {} Inf +eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_7) + {} Inf clear @@ -1102,9 +1095,8 @@ eval instant at 5m histogram_stddev(rate(const_histogram[5m])) {} NaN # Zero buckets mean no observations, so there is no standard variance. -# Unsupported by streaming engine. -# eval instant at 5m histogram_stdvar(rate(const_histogram[5m])) -# {} NaN +eval instant at 5m histogram_stdvar(rate(const_histogram[5m])) + {} NaN clear