diff --git a/src/OpenTelemetry/CHANGELOG.md b/src/OpenTelemetry/CHANGELOG.md index 396db7aee78..b5f8fb24a5f 100644 --- a/src/OpenTelemetry/CHANGELOG.md +++ b/src/OpenTelemetry/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +* **Experimental Feature** Added an opt-in feature to aggregate any metric + measurements that were dropped due to reaching the [max MetricPoints + limit](https://github.com/open-telemetry/opentelemetry-dotnet/tree/core-1.6.0-alpha.1/docs/metrics/customizing-the-sdk). + When this feature is enabled, SDK would aggregate such measurements using a + reserved MetricPoint with a single tag with key as `otel.metric.overflow` and + value as `true`. The feature is turned-off by default. You can enable it by + setting the environment variable + `OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE` to `true` before + setting up the `MeterProvider`. + ([#4737](https://github.com/open-telemetry/opentelemetry-dotnet/pull/4737)) + ## 1.6.0-alpha.1 Released 2023-Jul-12 diff --git a/src/OpenTelemetry/Metrics/AggregatorStore.cs b/src/OpenTelemetry/Metrics/AggregatorStore.cs index b0c4f673114..489f3b63c29 100644 --- a/src/OpenTelemetry/Metrics/AggregatorStore.cs +++ b/src/OpenTelemetry/Metrics/AggregatorStore.cs @@ -22,9 +22,11 @@ namespace OpenTelemetry.Metrics; internal sealed class AggregatorStore { - private static readonly string MetricPointCapHitFixMessage = "Modify instrumentation to reduce the number of unique key/value pair combinations. Or use Views to drop unwanted tags. Or use MeterProviderBuilder.SetMaxMetricPointsPerMetricStream to set higher limit."; + private static readonly string MetricPointCapHitFixMessage = "Consider opting in for the experimental SDK feature to emit all the throttled metrics under the overflow attribute by setting env variable OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE = true. You could also modify instrumentation to reduce the number of unique key/value pair combinations. Or use Views to drop unwanted tags. Or use MeterProviderBuilder.SetMaxMetricPointsPerMetricStream to set higher limit."; private static readonly Comparison> DimensionComparisonDelegate = (x, y) => x.Key.CompareTo(y.Key); + private readonly object lockZeroTags = new(); + private readonly object lockOverflowTag = new(); private readonly HashSet tagKeysInteresting; private readonly int tagsKeysInterestingCount; @@ -43,17 +45,21 @@ internal sealed class AggregatorStore private readonly UpdateLongDelegate updateLongCallback; private readonly UpdateDoubleDelegate updateDoubleCallback; private readonly int maxMetricPoints; + private readonly bool emitOverflowAttribute; private readonly ExemplarFilter exemplarFilter; + private int metricPointIndex = 0; private int batchSize = 0; private int metricCapHitMessageLogged; private bool zeroTagMetricPointInitialized; + private bool overflowTagMetricPointInitialized; internal AggregatorStore( MetricStreamIdentity metricStreamIdentity, AggregationType aggType, AggregationTemporality temporality, int maxMetricPoints, + bool emitOverflowAttribute, ExemplarFilter exemplarFilter = null) { this.name = metricStreamIdentity.InstrumentName; @@ -81,6 +87,15 @@ internal AggregatorStore( this.tagKeysInteresting = hs; this.tagsKeysInterestingCount = hs.Count; } + + this.emitOverflowAttribute = emitOverflowAttribute; + + if (emitOverflowAttribute) + { + // Setting metricPointIndex to 1 as we would reserve the metricPoints[1] for overflow attribute. + // Newer attributes should be added starting at the index: 2 + this.metricPointIndex = 1; + } } private delegate void UpdateLongDelegate(long value, ReadOnlySpan> tags); @@ -197,6 +212,22 @@ private void InitializeZeroTagPointIfNotInitialized() } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void InitializeOverflowTagPointIfNotInitialized() + { + if (!this.overflowTagMetricPointInitialized) + { + lock (this.lockOverflowTag) + { + if (!this.overflowTagMetricPointInitialized) + { + this.metricPoints[1] = new MetricPoint(this, this.aggType, new KeyValuePair[] { new("otel.metric.overflow", true) }, this.histogramBounds, this.exponentialHistogramMaxSize, this.exponentialHistogramMaxScale); + this.overflowTagMetricPointInitialized = true; + } + } + } + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int LookupAggregatorStore(KeyValuePair[] tagKeysAndValues, int length) { @@ -329,12 +360,21 @@ private void UpdateLong(long value, ReadOnlySpan> t var index = this.FindMetricAggregatorsDefault(tags); if (index < 0) { - if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0) + if (this.emitOverflowAttribute) { - OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage); + this.InitializeOverflowTagPointIfNotInitialized(); + this.metricPoints[1].Update(value); + return; } + else + { + if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0) + { + OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage); + } - return; + return; + } } // TODO: can special case built-in filters to be bit faster. @@ -361,12 +401,21 @@ private void UpdateLongCustomTags(long value, ReadOnlySpan instrumentations = new(); private readonly List> viewConfigs; private readonly object collectLock = new(); @@ -48,6 +51,9 @@ internal MeterProviderSdk( var state = serviceProvider!.GetRequiredService(); state.RegisterProvider(this); + var config = serviceProvider!.GetRequiredService(); + _ = config.TryGetBoolValue(EmitOverFlowAttributeConfigKey, out bool isEmitOverflowAttributeKeySet); + this.ServiceProvider = serviceProvider!; if (ownsServiceProvider) @@ -79,7 +85,7 @@ internal MeterProviderSdk( reader.SetParentProvider(this); reader.SetMaxMetricStreams(state.MaxMetricStreams); - reader.SetMaxMetricPointsPerMetricStream(state.MaxMetricPointsPerMetricStream); + reader.SetMaxMetricPointsPerMetricStream(state.MaxMetricPointsPerMetricStream, isEmitOverflowAttributeKeySet); reader.SetExemplarFilter(state.ExemplarFilter); if (this.reader == null) diff --git a/src/OpenTelemetry/Metrics/Metric.cs b/src/OpenTelemetry/Metrics/Metric.cs index bdfd73368e7..cc42f96ffc2 100644 --- a/src/OpenTelemetry/Metrics/Metric.cs +++ b/src/OpenTelemetry/Metrics/Metric.cs @@ -35,6 +35,7 @@ internal Metric( MetricStreamIdentity instrumentIdentity, AggregationTemporality temporality, int maxMetricPointsPerMetricStream, + bool emitOverflowAttribute, ExemplarFilter exemplarFilter = null) { this.InstrumentIdentity = instrumentIdentity; @@ -141,7 +142,7 @@ internal Metric( throw new NotSupportedException($"Unsupported Instrument Type: {instrumentIdentity.InstrumentType.FullName}"); } - this.aggStore = new AggregatorStore(instrumentIdentity, aggType, temporality, maxMetricPointsPerMetricStream, exemplarFilter); + this.aggStore = new AggregatorStore(instrumentIdentity, aggType, temporality, maxMetricPointsPerMetricStream, emitOverflowAttribute, exemplarFilter); this.Temporality = temporality; this.InstrumentDisposed = false; } diff --git a/src/OpenTelemetry/Metrics/MetricReaderExt.cs b/src/OpenTelemetry/Metrics/MetricReaderExt.cs index fdb2781f99a..f10119dd7cb 100644 --- a/src/OpenTelemetry/Metrics/MetricReaderExt.cs +++ b/src/OpenTelemetry/Metrics/MetricReaderExt.cs @@ -33,6 +33,7 @@ public abstract partial class MetricReader private Metric[] metrics; private Metric[] metricsCurrentBatch; private int metricIndex = -1; + private bool emitOverflowAttribute; private ExemplarFilter exemplarFilter; @@ -71,7 +72,7 @@ internal Metric AddMetricWithNoViews(Instrument instrument) Metric metric = null; try { - metric = new Metric(metricStreamIdentity, this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), this.maxMetricPointsPerMetricStream, exemplarFilter: this.exemplarFilter); + metric = new Metric(metricStreamIdentity, this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), this.maxMetricPointsPerMetricStream, this.emitOverflowAttribute, this.exemplarFilter); } catch (NotSupportedException nse) { @@ -156,7 +157,7 @@ internal List AddMetricsListWithViews(Instrument instrument, List 1) + { + this.emitOverflowAttribute = true; + } + } } private Batch GetMetricsBatch() diff --git a/src/Shared/Options/ConfigurationExtensions.cs b/src/Shared/Options/ConfigurationExtensions.cs index 7a00025a104..c99e5d8d7d6 100644 --- a/src/Shared/Options/ConfigurationExtensions.cs +++ b/src/Shared/Options/ConfigurationExtensions.cs @@ -95,6 +95,26 @@ public static bool TryGetIntValue( return true; } + public static bool TryGetBoolValue( + this IConfiguration configuration, + string key, + out bool value) + { + if (!configuration.TryGetStringValue(key, out var stringValue)) + { + value = default; + return false; + } + + if (!bool.TryParse(stringValue, out value)) + { + LogInvalidEnvironmentVariable?.Invoke(key, stringValue!); + return false; + } + + return true; + } + public static bool TryGetValue( this IConfiguration configuration, string key, diff --git a/test/OpenTelemetry.Tests/Metrics/AggregatorTest.cs b/test/OpenTelemetry.Tests/Metrics/AggregatorTestsBase.cs similarity index 94% rename from test/OpenTelemetry.Tests/Metrics/AggregatorTest.cs rename to test/OpenTelemetry.Tests/Metrics/AggregatorTestsBase.cs index 7d52f3cad1d..969c6890a87 100644 --- a/test/OpenTelemetry.Tests/Metrics/AggregatorTest.cs +++ b/test/OpenTelemetry.Tests/Metrics/AggregatorTestsBase.cs @@ -1,4 +1,4 @@ -// +// // Copyright The OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,13 +19,27 @@ namespace OpenTelemetry.Metrics.Tests; -public class AggregatorTest +#pragma warning disable SA1402 + +public abstract class AggregatorTestsBase { private static readonly Meter Meter = new("testMeter"); private static readonly Instrument Instrument = Meter.CreateHistogram("testInstrument"); private static readonly ExplicitBucketHistogramConfiguration HistogramConfiguration = new() { Boundaries = Metric.DefaultHistogramBounds }; private static readonly MetricStreamIdentity MetricStreamIdentity = new(Instrument, HistogramConfiguration); - private readonly AggregatorStore aggregatorStore = new(MetricStreamIdentity, AggregationType.HistogramWithBuckets, AggregationTemporality.Cumulative, 1024); + + private readonly bool emitOverflowAttribute; + private readonly AggregatorStore aggregatorStore; + + protected AggregatorTestsBase(bool emitOverflowAttribute) + { + if (emitOverflowAttribute) + { + this.emitOverflowAttribute = emitOverflowAttribute; + } + + this.aggregatorStore = new(MetricStreamIdentity, AggregationType.HistogramWithBuckets, AggregationTemporality.Cumulative, 1024, emitOverflowAttribute); + } [Fact] public void HistogramDistributeToAllBucketsDefault() @@ -284,6 +298,7 @@ internal void ExponentialHistogramTests(AggregationType aggregationType, Aggrega aggregationType, aggregationTemporality, maxMetricPoints: 1024, + this.emitOverflowAttribute, exemplarsEnabled ? new AlwaysOnExemplarFilter() : null); var expectedHistogram = new Base2ExponentialBucketHistogram(); @@ -391,7 +406,8 @@ internal void ExponentialMaxScaleConfigWorks(int? maxScale) metricStreamIdentity, AggregationType.Base2ExponentialHistogram, AggregationTemporality.Cumulative, - maxMetricPoints: 1024); + maxMetricPoints: 1024, + this.emitOverflowAttribute); aggregatorStore.Update(10, Array.Empty>()); @@ -463,3 +479,19 @@ private class ThreadArguments public double SumOfDelta; } } + +public class AggregatorTests : AggregatorTestsBase +{ + public AggregatorTests() + : base(false) + { + } +} + +public class AggregatorTestsWithOverflowAttribute : AggregatorTestsBase +{ + public AggregatorTestsWithOverflowAttribute() + : base(true) + { + } +} diff --git a/test/OpenTelemetry.Tests/Metrics/MetricAPITest.cs b/test/OpenTelemetry.Tests/Metrics/MetricApiTestsBase.cs similarity index 98% rename from test/OpenTelemetry.Tests/Metrics/MetricAPITest.cs rename to test/OpenTelemetry.Tests/Metrics/MetricApiTestsBase.cs index 5f403456a05..bfe2229dcf5 100644 --- a/test/OpenTelemetry.Tests/Metrics/MetricAPITest.cs +++ b/test/OpenTelemetry.Tests/Metrics/MetricApiTestsBase.cs @@ -1,4 +1,4 @@ -// +// // Copyright The OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,7 +24,9 @@ namespace OpenTelemetry.Metrics.Tests; -public class MetricApiTest : MetricTestsBase +#pragma warning disable SA1402 + +public abstract class MetricApiTestsBase : MetricTestsBase, IDisposable { private const int MaxTimeToAllowForFlush = 10000; private static readonly int NumberOfThreads = Environment.ProcessorCount; @@ -33,9 +35,14 @@ public class MetricApiTest : MetricTestsBase private static readonly int NumberOfMetricUpdateByEachThread = 100000; private readonly ITestOutputHelper output; - public MetricApiTest(ITestOutputHelper output) + protected MetricApiTestsBase(ITestOutputHelper output, bool emitOverflowAttribute) { this.output = output; + + if (emitOverflowAttribute) + { + Environment.SetEnvironmentVariable(EmitOverFlowAttributeConfigKey, "true"); + } } [Fact] @@ -1518,6 +1525,11 @@ public void UnsupportedMetricInstrument() Assert.Empty(exportedItems); } + public void Dispose() + { + Environment.SetEnvironmentVariable(EmitOverFlowAttributeConfigKey, null); + } + private static void CounterUpdateThread(object obj) where T : struct, IComparable { @@ -1689,3 +1701,19 @@ private class UpdateThreadArguments public T[] ValuesToRecord; } } + +public class MetricApiTest : MetricApiTestsBase +{ + public MetricApiTest(ITestOutputHelper output) + : base(output, false) + { + } +} + +public class MetricApiTestWithOverflowAttribute : MetricApiTestsBase +{ + public MetricApiTestWithOverflowAttribute(ITestOutputHelper output) + : base(output, true) + { + } +} diff --git a/test/OpenTelemetry.Tests/Metrics/MetricOverflowAttributeTests.cs b/test/OpenTelemetry.Tests/Metrics/MetricOverflowAttributeTests.cs new file mode 100644 index 00000000000..98f7fa05392 --- /dev/null +++ b/test/OpenTelemetry.Tests/Metrics/MetricOverflowAttributeTests.cs @@ -0,0 +1,409 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System.Diagnostics.Metrics; +using System.Reflection; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using OpenTelemetry.Tests; +using Xunit; + +namespace OpenTelemetry.Metrics.Tests; + +public class MetricOverflowAttributeTests +{ + [Theory] + [InlineData("false", false)] + [InlineData("False", false)] + [InlineData("FALSE", false)] + [InlineData("true", true)] + [InlineData("True", true)] + [InlineData("TRUE", true)] + public void TestEmitOverflowAttributeConfigWithEnvVar(string value, bool isEmitOverflowAttributeKeySet) + { + try + { + Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, value); + + var exportedItems = new List(); + + var meter = new Meter(Utils.GetCurrentMethodName()); + var counter = meter.CreateCounter("TestCounter"); + + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter(meter.Name) + .AddInMemoryExporter(exportedItems) + .Build(); + + counter.Add(10); + + meterProvider.ForceFlush(); + + Assert.Single(exportedItems); + var metric = exportedItems[0]; + + var aggregatorStore = typeof(Metric).GetField("aggStore", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(metric) as AggregatorStore; + var emitOverflowAttribute = (bool)typeof(AggregatorStore).GetField("emitOverflowAttribute", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(aggregatorStore); + + Assert.Equal(isEmitOverflowAttributeKeySet, emitOverflowAttribute); + } + finally + { + Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, null); + } + } + + [Theory] + [InlineData("false", false)] + [InlineData("False", false)] + [InlineData("FALSE", false)] + [InlineData("true", true)] + [InlineData("True", true)] + [InlineData("TRUE", true)] + public void TestEmitOverflowAttributeConfigWithOtherConfigProvider(string value, bool isEmitOverflowAttributeKeySet) + { + try + { + var exportedItems = new List(); + + var meter = new Meter(Utils.GetCurrentMethodName()); + var counter = meter.CreateCounter("TestCounter"); + + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .ConfigureServices(services => + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { [MetricTestsBase.EmitOverFlowAttributeConfigKey] = value }) + .Build(); + + services.AddSingleton(configuration); + }) + .AddMeter(meter.Name) + .AddInMemoryExporter(exportedItems) + .Build(); + + counter.Add(10); + + meterProvider.ForceFlush(); + + Assert.Single(exportedItems); + var metric = exportedItems[0]; + + var aggregatorStore = typeof(Metric).GetField("aggStore", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(metric) as AggregatorStore; + var emitOverflowAttribute = (bool)typeof(AggregatorStore).GetField("emitOverflowAttribute", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(aggregatorStore); + + Assert.Equal(isEmitOverflowAttributeKeySet, emitOverflowAttribute); + } + finally + { + Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, null); + } + } + + [Theory] + [InlineData(1, false)] + [InlineData(2, true)] + [InlineData(10, true)] + public void EmitOverflowAttributeIsOnlySetWhenMaxMetricPointsIsGreaterThanOne(int maxMetricPoints, bool isEmitOverflowAttributeKeySet) + { + try + { + Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, "true"); + + var exportedItems = new List(); + + var meter = new Meter(Utils.GetCurrentMethodName()); + var counter = meter.CreateCounter("TestCounter"); + + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .SetMaxMetricPointsPerMetricStream(maxMetricPoints) + .AddMeter(meter.Name) + .AddInMemoryExporter(exportedItems) + .Build(); + + counter.Add(10); + + meterProvider.ForceFlush(); + + Assert.Single(exportedItems); + var metric = exportedItems[0]; + + var aggregatorStore = typeof(Metric).GetField("aggStore", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(metric) as AggregatorStore; + var emitOverflowAttribute = (bool)typeof(AggregatorStore).GetField("emitOverflowAttribute", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(aggregatorStore); + + Assert.Equal(isEmitOverflowAttributeKeySet, emitOverflowAttribute); + } + finally + { + Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, null); + } + } + + [Theory] + [InlineData(MetricReaderTemporalityPreference.Delta)] + [InlineData(MetricReaderTemporalityPreference.Cumulative)] + public void MetricOverflowAttributeIsRecordedCorrectlyForCounter(MetricReaderTemporalityPreference temporalityPreference) + { + try + { + Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, "true"); + + var exportedItems = new List(); + + var meter = new Meter(Utils.GetCurrentMethodName()); + var counter = meter.CreateCounter("TestCounter"); + + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter(meter.Name) + .AddInMemoryExporter(exportedItems, metricReaderOptions => metricReaderOptions.TemporalityPreference = temporalityPreference) + .Build(); + + // There are two reserved MetricPoints + // 1. For zero tags + // 2. For metric overflow attribute when user opts-in for this feature + + // Max number for MetricPoints available for use when emitted with tags + int maxMetricPointsForUse = MeterProviderBuilderSdk.MaxMetricPointsPerMetricDefault - 2; + + for (int i = 0; i < maxMetricPointsForUse; i++) + { + // Emit unique key-value pairs to use up the available MetricPoints + // Once this loop is run, we have used up all available MetricPoints for metrics emitted with tags + counter.Add(10, new KeyValuePair("Key", i)); + } + + meterProvider.ForceFlush(); + + Assert.Single(exportedItems); + var metric = exportedItems[0]; + + var metricPoints = new List(); + foreach (ref readonly var mp in metric.GetMetricPoints()) + { + metricPoints.Add(mp); + } + + MetricPoint overflowMetricPoint; + + // We still have not exceeded the max MetricPoint limit + Assert.DoesNotContain(metricPoints, mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow"); + + exportedItems.Clear(); + metricPoints.Clear(); + + counter.Add(5, new KeyValuePair("Key", 9999)); // Emit a metric to exceed the max MetricPoint limit + + meterProvider.ForceFlush(); + metric = exportedItems[0]; + foreach (ref readonly var mp in metric.GetMetricPoints()) + { + metricPoints.Add(mp); + } + + overflowMetricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow"); + Assert.Equal(true, overflowMetricPoint.Tags.KeyAndValues[0].Value); + Assert.Equal(1, overflowMetricPoint.Tags.Count); + Assert.Equal(5, overflowMetricPoint.GetSumLong()); + + exportedItems.Clear(); + metricPoints.Clear(); + + // Emit 50 more newer MetricPoints with distinct dimension combinations + for (int i = 10000; i < 10050; i++) + { + counter.Add(5, new KeyValuePair("Key", i)); + } + + meterProvider.ForceFlush(); + metric = exportedItems[0]; + foreach (ref readonly var mp in metric.GetMetricPoints()) + { + metricPoints.Add(mp); + } + + overflowMetricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow"); + if (temporalityPreference == MetricReaderTemporalityPreference.Delta) + { + Assert.Equal(250, overflowMetricPoint.GetSumLong()); // 50 * 5 + } + else + { + Assert.Equal(255, overflowMetricPoint.GetSumLong()); // 5 + (50 * 5) + } + + exportedItems.Clear(); + metricPoints.Clear(); + + // Test that the SDK continues to correctly aggregate the previously registered measurements even after overflow has occurred + counter.Add(15, new KeyValuePair("Key", 0)); + + meterProvider.ForceFlush(); + metric = exportedItems[0]; + foreach (ref readonly var mp in metric.GetMetricPoints()) + { + metricPoints.Add(mp); + } + + var metricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "Key" && (int)mp.Tags.KeyAndValues[0].Value == 0); + + if (temporalityPreference == MetricReaderTemporalityPreference.Delta) + { + Assert.Equal(15, metricPoint.GetSumLong()); + } + else + { + overflowMetricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow"); + + Assert.Equal(25, metricPoint.GetSumLong()); // 10 + 15 + Assert.Equal(255, overflowMetricPoint.GetSumLong()); + } + } + finally + { + Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, null); + } + } + + [Theory] + [InlineData(MetricReaderTemporalityPreference.Delta)] + [InlineData(MetricReaderTemporalityPreference.Cumulative)] + public void MetricOverflowAttributeIsRecordedCorrectlyForHistogram(MetricReaderTemporalityPreference temporalityPreference) + { + try + { + Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, "true"); + + var exportedItems = new List(); + + var meter = new Meter(Utils.GetCurrentMethodName()); + var histogram = meter.CreateHistogram("TestHistogram"); + + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter(meter.Name) + .AddInMemoryExporter(exportedItems, metricReaderOptions => metricReaderOptions.TemporalityPreference = temporalityPreference) + .Build(); + + // There are two reserved MetricPoints + // 1. For zero tags + // 2. For metric overflow attribute when user opts-in for this feature + + // Max number for MetricPoints available for use when emitted with tags + int maxMetricPointsForUse = MeterProviderBuilderSdk.MaxMetricPointsPerMetricDefault - 2; + + for (int i = 0; i < maxMetricPointsForUse; i++) + { + // Emit unique key-value pairs to use up the available MetricPoints + // Once this loop is run, we have used up all available MetricPoints for metrics emitted with tags + histogram.Record(10, new KeyValuePair("Key", i)); + } + + meterProvider.ForceFlush(); + + Assert.Single(exportedItems); + var metric = exportedItems[0]; + + var metricPoints = new List(); + foreach (ref readonly var mp in metric.GetMetricPoints()) + { + metricPoints.Add(mp); + } + + MetricPoint overflowMetricPoint; + + // We still have not exceeded the max MetricPoint limit + Assert.DoesNotContain(metricPoints, mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow"); + + exportedItems.Clear(); + metricPoints.Clear(); + + histogram.Record(5, new KeyValuePair("Key", 9999)); // Emit a metric to exceed the max MetricPoint limit + + meterProvider.ForceFlush(); + metric = exportedItems[0]; + foreach (ref readonly var mp in metric.GetMetricPoints()) + { + metricPoints.Add(mp); + } + + overflowMetricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow"); + Assert.Equal(true, overflowMetricPoint.Tags.KeyAndValues[0].Value); + Assert.Equal(1, overflowMetricPoint.GetHistogramCount()); + Assert.Equal(5, overflowMetricPoint.GetHistogramSum()); + + exportedItems.Clear(); + metricPoints.Clear(); + + // Emit 50 more newer MetricPoints with distinct dimension combinations + for (int i = 10000; i < 10050; i++) + { + histogram.Record(5, new KeyValuePair("Key", i)); + } + + meterProvider.ForceFlush(); + metric = exportedItems[0]; + foreach (ref readonly var mp in metric.GetMetricPoints()) + { + metricPoints.Add(mp); + } + + overflowMetricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow"); + if (temporalityPreference == MetricReaderTemporalityPreference.Delta) + { + Assert.Equal(50, overflowMetricPoint.GetHistogramCount()); + Assert.Equal(250, overflowMetricPoint.GetHistogramSum()); // 50 * 5 + } + else + { + Assert.Equal(51, overflowMetricPoint.GetHistogramCount()); + Assert.Equal(255, overflowMetricPoint.GetHistogramSum()); // 5 + (50 * 5) + } + + exportedItems.Clear(); + metricPoints.Clear(); + + // Test that the SDK continues to correctly aggregate the previously registered measurements even after overflow has occurred + histogram.Record(15, new KeyValuePair("Key", 0)); + + meterProvider.ForceFlush(); + metric = exportedItems[0]; + foreach (ref readonly var mp in metric.GetMetricPoints()) + { + metricPoints.Add(mp); + } + + var metricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "Key" && (int)mp.Tags.KeyAndValues[0].Value == 0); + + if (temporalityPreference == MetricReaderTemporalityPreference.Delta) + { + Assert.Equal(1, metricPoint.GetHistogramCount()); + Assert.Equal(15, metricPoint.GetHistogramSum()); + } + else + { + overflowMetricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow"); + + Assert.Equal(2, metricPoint.GetHistogramCount()); + Assert.Equal(25, metricPoint.GetHistogramSum()); // 10 + 15 + + Assert.Equal(255, overflowMetricPoint.GetHistogramSum()); + } + } + finally + { + Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, null); + } + } +} diff --git a/test/OpenTelemetry.Tests/Metrics/MetricSnapshotTests.cs b/test/OpenTelemetry.Tests/Metrics/MetricSnapshotTestsBase.cs similarity index 89% rename from test/OpenTelemetry.Tests/Metrics/MetricSnapshotTests.cs rename to test/OpenTelemetry.Tests/Metrics/MetricSnapshotTestsBase.cs index 37043198aef..dc0810fd338 100644 --- a/test/OpenTelemetry.Tests/Metrics/MetricSnapshotTests.cs +++ b/test/OpenTelemetry.Tests/Metrics/MetricSnapshotTestsBase.cs @@ -1,4 +1,4 @@ -// +// // Copyright The OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,8 +22,18 @@ namespace OpenTelemetry.Metrics.Tests; -public class MetricSnapshotTests +#pragma warning disable SA1402 + +public abstract class MetricSnapshotTestsBase : IDisposable { + protected MetricSnapshotTestsBase(bool emitOverflowAttribute) + { + if (emitOverflowAttribute) + { + Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, "true"); + } + } + [Fact] public void VerifySnapshot_Counter() { @@ -87,7 +97,9 @@ public void VerifySnapshot_Counter() // Verify Snapshot 2 Assert.Equal(2, exportedSnapshots.Count); var snapshot2 = exportedSnapshots[1]; + Assert.Single(snapshot2.MetricPoints); + Assert.Equal(15, snapshot2.MetricPoints[0].GetSumLong()); } @@ -214,7 +226,7 @@ public void VerifySnapshot_ExponentialHistogram() metricPoint1.TryGetHistogramMinMaxValues(out var min, out var max); Assert.Equal(10, min); Assert.Equal(10, max); - AggregatorTest.AssertExponentialBucketsAreCorrect(expectedHistogram, metricPoint1.GetExponentialHistogramData()); + AggregatorTestsBase.AssertExponentialBucketsAreCorrect(expectedHistogram, metricPoint1.GetExponentialHistogramData()); // Verify Snapshot 1 Assert.Single(exportedSnapshots); @@ -225,7 +237,7 @@ public void VerifySnapshot_ExponentialHistogram() snapshot1.MetricPoints[0].TryGetHistogramMinMaxValues(out min, out max); Assert.Equal(10, min); Assert.Equal(10, max); - AggregatorTest.AssertExponentialBucketsAreCorrect(expectedHistogram, snapshot1.MetricPoints[0].GetExponentialHistogramData()); + AggregatorTestsBase.AssertExponentialBucketsAreCorrect(expectedHistogram, snapshot1.MetricPoints[0].GetExponentialHistogramData()); // Verify Metric == Snapshot Assert.Equal(metric1.Name, snapshot1.Name); @@ -259,7 +271,7 @@ public void VerifySnapshot_ExponentialHistogram() metricPoint1.TryGetHistogramMinMaxValues(out min, out max); Assert.Equal(5, min); Assert.Equal(10, max); - AggregatorTest.AssertExponentialBucketsAreCorrect(expectedHistogram, metricPoint2.GetExponentialHistogramData()); + AggregatorTestsBase.AssertExponentialBucketsAreCorrect(expectedHistogram, metricPoint2.GetExponentialHistogramData()); // Verify Snapshot 1 after second export // This value is expected to be unchanged. @@ -278,6 +290,27 @@ public void VerifySnapshot_ExponentialHistogram() snapshot2.MetricPoints[0].TryGetHistogramMinMaxValues(out min, out max); Assert.Equal(5, min); Assert.Equal(10, max); - AggregatorTest.AssertExponentialBucketsAreCorrect(expectedHistogram, snapshot2.MetricPoints[0].GetExponentialHistogramData()); + AggregatorTestsBase.AssertExponentialBucketsAreCorrect(expectedHistogram, snapshot2.MetricPoints[0].GetExponentialHistogramData()); + } + + public void Dispose() + { + Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, null); + } +} + +public class MetricSnapshotTests : MetricSnapshotTestsBase +{ + public MetricSnapshotTests() + : base(false) + { + } +} + +public class MetricSnapshotTestsWithOverflowAttribute : MetricSnapshotTestsBase +{ + public MetricSnapshotTestsWithOverflowAttribute() + : base(true) + { } } diff --git a/test/OpenTelemetry.Tests/Metrics/MetricTestsBase.cs b/test/OpenTelemetry.Tests/Metrics/MetricTestsBase.cs index 181027e37a2..85c1b2c0817 100644 --- a/test/OpenTelemetry.Tests/Metrics/MetricTestsBase.cs +++ b/test/OpenTelemetry.Tests/Metrics/MetricTestsBase.cs @@ -20,6 +20,8 @@ namespace OpenTelemetry.Metrics.Tests; public class MetricTestsBase { + public const string EmitOverFlowAttributeConfigKey = "OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE"; + public static void ValidateMetricPointTags(List> expectedTags, ReadOnlyTagCollection actualTags) { int tagIndex = 0; diff --git a/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs b/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs index 82cc2c57aff..df98fc6cc2a 100644 --- a/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs +++ b/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs @@ -636,7 +636,7 @@ public void ViewToProduceExponentialHistogram() var count = metricPoint.GetHistogramCount(); var sum = metricPoint.GetHistogramSum(); - AggregatorTest.AssertExponentialBucketsAreCorrect(expectedHistogram, metricPoint.GetExponentialHistogramData()); + AggregatorTestsBase.AssertExponentialBucketsAreCorrect(expectedHistogram, metricPoint.GetExponentialHistogramData()); Assert.Equal(50, sum); Assert.Equal(6, count); }