From 9beacb747a19945569a36d19107f1fe6fe31d6fb Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Thu, 6 Jun 2024 18:22:20 +0300 Subject: [PATCH] Added instrumentation via System.Diagnostics.Activity --- README.md | 9 +++ .../PeriodicTasksActivitySource.cs | 54 ++++++++++++++++ .../Extensions/ActivityExtensions.cs | 57 +++++++++++++++++ .../Internal/PeriodicTaskRunner.cs | 64 +++++++++++++++++-- .../PeriodicTaskExecutionContext.cs | 6 +- 5 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 src/Tingle.PeriodicTasks/Diagnostics/PeriodicTasksActivitySource.cs create mode 100644 src/Tingle.PeriodicTasks/Extensions/ActivityExtensions.cs diff --git a/README.md b/README.md index 2e3b9dd..574eeb7 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,15 @@ services.AddSingleton(provider => }); ``` +### Instrumentation and observability + +This core library (`Tingle.PeriodicTasks`) is instrumented using `System.Diagnostics.Activity` and `System.Diagnostics.ActivitySource`. +This makes it easy to use with OpenTelemetry by listening to the `Tingle.PeriodicTasks` activity source. + +```cs +services.AddOpenTelemetry().WithTracing().AddSource("Tingle.PeriodicTasks"); +``` + ### Management via endpoints in AspNetCore You can choose to manage the periodic tasks in using endpoints in AspNetCore. Update your application setup as follows. diff --git a/src/Tingle.PeriodicTasks/Diagnostics/PeriodicTasksActivitySource.cs b/src/Tingle.PeriodicTasks/Diagnostics/PeriodicTasksActivitySource.cs new file mode 100644 index 0000000..3611aab --- /dev/null +++ b/src/Tingle.PeriodicTasks/Diagnostics/PeriodicTasksActivitySource.cs @@ -0,0 +1,54 @@ +using System.Diagnostics; +using System.Reflection; + +namespace Tingle.PeriodicTasks.Diagnostics; + +/// +public static class PeriodicTasksActivitySource +{ + private static readonly AssemblyName AssemblyName = typeof(PeriodicTasksActivitySource).Assembly.GetName(); + private static readonly string ActivitySourceName = AssemblyName.Name!; + private static readonly Version Version = AssemblyName.Version!; + private static readonly ActivitySource ActivitySource = new(ActivitySourceName, Version.ToString()); + + /// + /// Creates a new activity if there are active listeners for it, using the specified + /// name, activity kind, and parent Id. + /// + /// + /// + /// + /// + public static Activity? StartActivity(string name, ActivityKind kind = ActivityKind.Internal, string? parentId = null) + { + return parentId is not null + ? ActivitySource.StartActivity(name: name, kind: kind, parentId: parentId) + : ActivitySource.StartActivity(name: name, kind: kind); + } +} + +/// +/// Names for activities generated by PeriodicTasks +/// +public static class ActivityNames +{ +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + public const string Execute = "Tingle.PeriodicTasks.Execute"; + public const string ExecuteAttempt = "Tingle.PeriodicTasks.ExecuteAttempt"; +#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member +} + +/// +/// Names for tags added to activities generated by PeriodicTasks +/// +public static class ActivityTagNames +{ +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + public const string PeriodicTaskType = "periodictask.type"; + public const string PeriodicTaskName = "periodictask.name"; + public const string PeriodicTaskSchedule = "periodictask.schedule"; + public const string PeriodicTaskTimezone = "periodictask.timezone"; + public const string PeriodicTaskDeadline = "periodictask.deadline"; + public const string PeriodicTaskAttemptNumber = "periodictask.attempt_number"; +#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member +} diff --git a/src/Tingle.PeriodicTasks/Extensions/ActivityExtensions.cs b/src/Tingle.PeriodicTasks/Extensions/ActivityExtensions.cs new file mode 100644 index 0000000..69d2fbe --- /dev/null +++ b/src/Tingle.PeriodicTasks/Extensions/ActivityExtensions.cs @@ -0,0 +1,57 @@ +namespace System.Diagnostics; + +internal static class ActivityExtensions +{ + // Copied from https://github.com/dotnet/runtime/pull/102905/files +#if !NET9_0_OR_GREATER + public static Activity AddException(this Activity activity, Exception exception, TagList tags = default, DateTimeOffset timestamp = default) + { + ArgumentNullException.ThrowIfNull(activity); + ArgumentNullException.ThrowIfNull(exception); + + TagList exceptionTags = tags; + + const string ExceptionEventName = "exception"; + const string ExceptionMessageTag = "exception.message"; + const string ExceptionStackTraceTag = "exception.stacktrace"; + const string ExceptionTypeTag = "exception.type"; + + bool hasMessage = false; + bool hasStackTrace = false; + bool hasType = false; + + for (int i = 0; i < exceptionTags.Count; i++) + { + if (exceptionTags[i].Key == ExceptionMessageTag) + { + hasMessage = true; + } + else if (exceptionTags[i].Key == ExceptionStackTraceTag) + { + hasStackTrace = true; + } + else if (exceptionTags[i].Key == ExceptionTypeTag) + { + hasType = true; + } + } + + if (!hasMessage) + { + exceptionTags.Add(new KeyValuePair(ExceptionMessageTag, exception.Message)); + } + + if (!hasStackTrace) + { + exceptionTags.Add(new KeyValuePair(ExceptionStackTraceTag, exception.ToString())); + } + + if (!hasType) + { + exceptionTags.Add(new KeyValuePair(ExceptionTypeTag, exception.GetType().ToString())); + } + + return activity.AddEvent(new ActivityEvent(ExceptionEventName, timestamp)); + } +#endif +} diff --git a/src/Tingle.PeriodicTasks/Internal/PeriodicTaskRunner.cs b/src/Tingle.PeriodicTasks/Internal/PeriodicTaskRunner.cs index 02e4c03..b1118c4 100644 --- a/src/Tingle.PeriodicTasks/Internal/PeriodicTaskRunner.cs +++ b/src/Tingle.PeriodicTasks/Internal/PeriodicTaskRunner.cs @@ -3,7 +3,9 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using Tingle.PeriodicTasks.Diagnostics; namespace Tingle.PeriodicTasks.Internal; @@ -65,13 +67,26 @@ public async Task RunAsync(string name, CancellationToken cancellationToken = de { var options = optionsMonitor.Get(name); + // Instrumentation + using var activity = PeriodicTasksActivitySource.StartActivity(ActivityNames.Execute, ActivityKind.Consumer); + if (activity is not null) + { + activity.DisplayName = $"Periodic Task: {name}"; + activity.AddTag(ActivityTagNames.PeriodicTaskName, name); + activity.AddTag(ActivityTagNames.PeriodicTaskType, typeof(TTask).FullName); + activity.AddTag(ActivityTagNames.PeriodicTaskSchedule, options.Schedule!.ToString()); + activity.AddTag(ActivityTagNames.PeriodicTaskTimezone, TimeZoneInfo.FindSystemTimeZoneById(options.Timezone!).Id); + activity.AddTag(ActivityTagNames.PeriodicTaskDeadline, options.Deadline!.ToString()); + } + // create linked CancellationTokenSource and attach deadline if not null var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(options.Deadline!.Value); // execute the task var id = idGenerator.Generate(name, options.ExecutionIdFormat!.Value); - var t = ExecuteInnerAsync(executionId: id, + var t = ExecuteInnerAsync(activity: activity, + executionId: id, name: name, options: options, throwOnError: throwOnError, @@ -97,7 +112,7 @@ public async Task RunAsync(string name, CancellationToken cancellationToken = de return null; } - internal async Task ExecuteInnerAsync(string executionId, string name, PeriodicTaskOptions options, bool throwOnError, CancellationToken cancellationToken = default) + internal async Task ExecuteInnerAsync(Activity? activity, string executionId, string name, PeriodicTaskOptions options, bool throwOnError, CancellationToken cancellationToken = default) { var start = DateTimeOffset.UtcNow; var lockName = options.LockName!; @@ -126,24 +141,37 @@ public async Task RunAsync(string name, CancellationToken cancellationToken = de { var task = ActivatorUtilities.GetServiceOrCreateInstance(provider); - var context = new PeriodicTaskExecutionContext(name, executionId) { TaskType = typeof(TTask), }; + var context = new PeriodicTaskExecutionContext(name, executionId, typeof(TTask)); // Invoke handler method, with resilience pipeline if specified var resiliencePipeline = options.ResiliencePipeline; - if (resiliencePipeline != null) + if (resiliencePipeline is not null) { var contextData = new Dictionary { ["context"] = context, }; - await resiliencePipeline.ExecuteAsync(async (ctx, ct) => await task.ExecuteAsync(context, cts.Token).ConfigureAwait(false), contextData, cancellationToken).ConfigureAwait(false); + var attemptNumber = 0; + await resiliencePipeline.ExecuteAsync( + async (ctx, ct) => + { + attemptNumber++; + await ExecuteTrackedAsync(task, context, attemptNumber, cts.Token).ConfigureAwait(false); + }, + contextData, + cancellationToken + ).ConfigureAwait(false); } else { - await task.ExecuteAsync(context, cts.Token).ConfigureAwait(false); + await ExecuteTrackedAsync(task, context, 1, cts.Token).ConfigureAwait(false); } } catch (Exception ex) { Interlocked.Exchange(ref caught, ex); logger.ExceptionInPeriodicTask(ex, executionId); + + // record the exception in the activity and set the status to error + activity?.SetStatus(ActivityStatusCode.Error); + activity?.AddException(ex); } var end = DateTimeOffset.UtcNow; @@ -203,4 +231,28 @@ public async Task RunAsync(string name, CancellationToken cancellationToken = de return attempt; } + + private async Task ExecuteTrackedAsync(IPeriodicTask task, PeriodicTaskExecutionContext context, int attemptNumber, CancellationToken cancellationToken) + { + using var activity = PeriodicTasksActivitySource.StartActivity(ActivityNames.ExecuteAttempt, ActivityKind.Consumer); + if (activity is not null) + { + activity.DisplayName = $"Periodic Task: {context.Name} (Attempt: {attemptNumber})"; + activity.AddTag(ActivityTagNames.PeriodicTaskName, context.Name); + activity.AddTag(ActivityTagNames.PeriodicTaskType, typeof(TTask).FullName); + activity.AddTag(ActivityTagNames.PeriodicTaskAttemptNumber, attemptNumber); + } + + try + { + await task.ExecuteAsync(context, cancellationToken).ConfigureAwait(false); + activity?.SetStatus(ActivityStatusCode.Ok); + } + catch (Exception ex) + { + activity?.SetStatus(ActivityStatusCode.Error); + activity?.AddException(ex); + throw; + } + } } diff --git a/src/Tingle.PeriodicTasks/PeriodicTaskExecutionContext.cs b/src/Tingle.PeriodicTasks/PeriodicTaskExecutionContext.cs index 25806c2..0ef1e88 100644 --- a/src/Tingle.PeriodicTasks/PeriodicTaskExecutionContext.cs +++ b/src/Tingle.PeriodicTasks/PeriodicTaskExecutionContext.cs @@ -11,11 +11,13 @@ public class PeriodicTaskExecutionContext /// This value is different for tasks of the same type but multiple registrations. /// /// Unique identifier of the execution. + /// Periodic task type. /// - public PeriodicTaskExecutionContext(string name, string executionId) + public PeriodicTaskExecutionContext(string name, string executionId, Type taskType) { Name = name ?? throw new ArgumentNullException(nameof(name)); ExecutionId = executionId ?? throw new ArgumentNullException(nameof(executionId)); + TaskType = taskType ?? throw new ArgumentNullException(nameof(taskType)); } /// @@ -30,5 +32,5 @@ public PeriodicTaskExecutionContext(string name, string executionId) public string ExecutionId { get; } /// Periodic task type. - public Type? TaskType { get; init; } + public Type TaskType { get; } }