From b9b9f9ae18658311202f4d110b26e3f8ec2a9c70 Mon Sep 17 00:00:00 2001 From: Macocian Alexandru Victor Date: Tue, 28 Nov 2023 03:15:18 +0100 Subject: [PATCH] Log db operations (#37) --- GuildWarsPartySearch/Config.Debug.json | 27 +- GuildWarsPartySearch/Config.Release.json | 27 +- .../Extensions/ServiceCollectionExtensions.cs | 67 ++++ .../Launch/ServerConfiguration.cs | 10 +- .../Options/ContentOptions.cs | 3 +- .../Options/IAzureBlobStorageOptions.cs | 6 + .../Options/IAzureTableStorageOptions.cs | 6 + .../Options/PartySearchTableOptions.cs | 6 + .../Azure/InterceptingAsyncPageable.cs | 35 ++ .../Services/Azure/InterceptingPageable.cs | 35 ++ .../Azure/NamedBlobContainerClient.cs | 40 ++ .../Services/Azure/NamedTableClient.cs | 363 ++++++++++++++++++ .../Content/ContentRetrievalService.cs | 14 +- .../Services/Database/TableStorageDatabase.cs | 23 +- 14 files changed, 630 insertions(+), 32 deletions(-) create mode 100644 GuildWarsPartySearch/Extensions/ServiceCollectionExtensions.cs create mode 100644 GuildWarsPartySearch/Options/IAzureBlobStorageOptions.cs create mode 100644 GuildWarsPartySearch/Options/IAzureTableStorageOptions.cs create mode 100644 GuildWarsPartySearch/Options/PartySearchTableOptions.cs create mode 100644 GuildWarsPartySearch/Services/Azure/InterceptingAsyncPageable.cs create mode 100644 GuildWarsPartySearch/Services/Azure/InterceptingPageable.cs create mode 100644 GuildWarsPartySearch/Services/Azure/NamedBlobContainerClient.cs create mode 100644 GuildWarsPartySearch/Services/Azure/NamedTableClient.cs diff --git a/GuildWarsPartySearch/Config.Debug.json b/GuildWarsPartySearch/Config.Debug.json index 2b99c43..9e8a175 100644 --- a/GuildWarsPartySearch/Config.Debug.json +++ b/GuildWarsPartySearch/Config.Debug.json @@ -1,4 +1,25 @@ { + "IpRateLimitOptions": { + "EnableEndpointRateLimiting": false, + "StackBlockedRequests": false, + "GeneralRules": [ + { + "Endpoint": "*", + "Period": "1s", + "Limit": 20 + }, + { + "Endpoint": "*", + "Period": "10m", + "Limit": 100 + }, + { + "Endpoint": "*", + "Period": "12h", + "Limit": 3000 + } + ] + }, "Logging": { "LogLevel": { "Default": "Debug", @@ -16,11 +37,13 @@ "HeartbeatFrequency": "0:0:1" }, "StorageAccountOptions": { - "TableName": "searches", - "ContainerName": "content", "ConnectionString": "[AZURE_TABLESTORAGE_CONNECTIONSTRING]" }, + "PartySearchTableOptions": { + "TableName": "searches" + }, "ContentOptions": { + "ContainerName": "content", "UpdateFrequency": "0:5:0", "StagingFolder": "Content" } diff --git a/GuildWarsPartySearch/Config.Release.json b/GuildWarsPartySearch/Config.Release.json index c714f07..b6130be 100644 --- a/GuildWarsPartySearch/Config.Release.json +++ b/GuildWarsPartySearch/Config.Release.json @@ -1,4 +1,25 @@ { + "IpRateLimitOptions": { + "EnableEndpointRateLimiting": false, + "StackBlockedRequests": false, + "GeneralRules": [ + { + "Endpoint": "*", + "Period": "1s", + "Limit": 20 + }, + { + "Endpoint": "*", + "Period": "10m", + "Limit": 100 + }, + { + "Endpoint": "*", + "Period": "12h", + "Limit": 3000 + } + ] + }, "Logging": { "LogLevel": { "Default": "Information", @@ -16,11 +37,13 @@ "HeartbeatFrequency": "0:0:1" }, "StorageAccountOptions": { - "TableName": "searches", - "ContainerName": "content", "ConnectionString": "[AZURE_TABLESTORAGE_CONNECTIONSTRING]" }, + "PartySearchTableOptions": { + "TableName": "searches" + }, "ContentOptions": { + "ContainerName": "content", "UpdateFrequency": "0:1:0", "StagingFolder": "Content" } diff --git a/GuildWarsPartySearch/Extensions/ServiceCollectionExtensions.cs b/GuildWarsPartySearch/Extensions/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..63f06e8 --- /dev/null +++ b/GuildWarsPartySearch/Extensions/ServiceCollectionExtensions.cs @@ -0,0 +1,67 @@ +using GuildWarsPartySearch.Server.Options; +using GuildWarsPartySearch.Server.Services.Azure; +using Microsoft.Extensions.Options; +using System.Core.Extensions; + +namespace GuildWarsPartySearch.Server.Extensions; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddSingletonTableClient(this IServiceCollection services) + where TOptions : class, IAzureTableStorageOptions, new() + { + services.ThrowIfNull() + .AddSingleton(sp => + { + var storageOptions = sp.GetRequiredService>(); + var clientOptions = sp.GetRequiredService>(); + var logger = sp.GetRequiredService>>(); + return new NamedTableClient(logger, storageOptions.Value.ConnectionString!, clientOptions.Value.TableName); + }); + + return services; + } + + public static IServiceCollection AddScopedTableClient(this IServiceCollection services) + where TOptions : class, IAzureTableStorageOptions, new() + { + services.ThrowIfNull() + .AddScoped(sp => + { + var storageOptions = sp.GetRequiredService>(); + var clientOptions = sp.GetRequiredService>(); + var logger = sp.GetRequiredService>>(); + return new NamedTableClient(logger, storageOptions.Value.ConnectionString!, clientOptions.Value.TableName); + }); + + return services; + } + + public static IServiceCollection AddSingletonBlobContainerClient(this IServiceCollection services) + where TOptions : class, IAzureBlobStorageOptions, new() + { + services.ThrowIfNull() + .AddSingleton(sp => + { + var storageOptions = sp.GetRequiredService>(); + var clientOptions = sp.GetRequiredService>(); + return new NamedBlobContainerClient(storageOptions.Value.ConnectionString!, clientOptions.Value.ContainerName); + }); + + return services; + } + + public static IServiceCollection AddScopedBlobContainerClient(this IServiceCollection services) + where TOptions : class, IAzureBlobStorageOptions, new() + { + services.ThrowIfNull() + .AddScoped(sp => + { + var storageOptions = sp.GetRequiredService>(); + var clientOptions = sp.GetRequiredService>(); + return new NamedBlobContainerClient(storageOptions.Value.ConnectionString!, clientOptions.Value.ContainerName); + }); + + return services; + } +} diff --git a/GuildWarsPartySearch/Launch/ServerConfiguration.cs b/GuildWarsPartySearch/Launch/ServerConfiguration.cs index 7f1209e..ad784f1 100644 --- a/GuildWarsPartySearch/Launch/ServerConfiguration.cs +++ b/GuildWarsPartySearch/Launch/ServerConfiguration.cs @@ -1,4 +1,5 @@ using AspNetCoreRateLimit; +using Azure.Data.Tables; using GuildWarsPartySearch.Common.Converters; using GuildWarsPartySearch.Server.Endpoints; using GuildWarsPartySearch.Server.Extensions; @@ -10,6 +11,8 @@ using GuildWarsPartySearch.Server.Services.Feed; using GuildWarsPartySearch.Server.Services.Lifetime; using GuildWarsPartySearch.Server.Services.PartySearch; +using Microsoft.Extensions.Azure; +using Microsoft.Extensions.Options; using System.Core.Extensions; using System.Extensions; using System.Text.Json.Serialization; @@ -61,10 +64,11 @@ public static WebApplicationBuilder SetupOptions(this WebApplicationBuilder buil builder.ThrowIfNull() .Services.Configure(builder.Configuration.GetSection(nameof(EnvironmentOptions))) .Configure(builder.Configuration.GetSection(nameof(ContentOptions))) + .Configure(builder.Configuration.GetSection(nameof(PartySearchTableOptions))) .Configure(builder.Configuration.GetSection(nameof(StorageAccountOptions))) .Configure(builder.Configuration.GetSection(nameof(ServerOptions))) - .Configure(builder.Configuration.GetSection("IpRateLimiting")) - .Configure(builder.Configuration.GetSection("IpRateLimitPolicies")); + .Configure(builder.Configuration.GetSection(nameof(IpRateLimitOptions))) + .Configure(builder.Configuration.GetSection(nameof(IpRateLimitPolicies))); return builder; } @@ -82,6 +86,8 @@ public static IServiceCollection SetupServices(this IServiceCollection services) services.AddScoped(); services.AddSingleton(); services.AddSingleton(); + services.AddScopedTableClient(); + services.AddSingletonBlobContainerClient(); return services; } diff --git a/GuildWarsPartySearch/Options/ContentOptions.cs b/GuildWarsPartySearch/Options/ContentOptions.cs index a946d0a..c467976 100644 --- a/GuildWarsPartySearch/Options/ContentOptions.cs +++ b/GuildWarsPartySearch/Options/ContentOptions.cs @@ -1,7 +1,8 @@ namespace GuildWarsPartySearch.Server.Options; -public sealed class ContentOptions +public sealed class ContentOptions : IAzureBlobStorageOptions { public TimeSpan UpdateFrequency { get; set; } = TimeSpan.FromMinutes(5); public string StagingFolder { get; set; } = "Content"; + public string ContainerName { get; set; } = default!; } diff --git a/GuildWarsPartySearch/Options/IAzureBlobStorageOptions.cs b/GuildWarsPartySearch/Options/IAzureBlobStorageOptions.cs new file mode 100644 index 0000000..6855a56 --- /dev/null +++ b/GuildWarsPartySearch/Options/IAzureBlobStorageOptions.cs @@ -0,0 +1,6 @@ +namespace GuildWarsPartySearch.Server.Options; + +public interface IAzureBlobStorageOptions +{ + string ContainerName { get; set; } +} diff --git a/GuildWarsPartySearch/Options/IAzureTableStorageOptions.cs b/GuildWarsPartySearch/Options/IAzureTableStorageOptions.cs new file mode 100644 index 0000000..66c6e4e --- /dev/null +++ b/GuildWarsPartySearch/Options/IAzureTableStorageOptions.cs @@ -0,0 +1,6 @@ +namespace GuildWarsPartySearch.Server.Options; + +public interface IAzureTableStorageOptions +{ + string TableName { get; set; } +} diff --git a/GuildWarsPartySearch/Options/PartySearchTableOptions.cs b/GuildWarsPartySearch/Options/PartySearchTableOptions.cs new file mode 100644 index 0000000..271c252 --- /dev/null +++ b/GuildWarsPartySearch/Options/PartySearchTableOptions.cs @@ -0,0 +1,6 @@ +namespace GuildWarsPartySearch.Server.Options; + +public class PartySearchTableOptions : IAzureTableStorageOptions +{ + public string TableName { get; set; } = default!; +} diff --git a/GuildWarsPartySearch/Services/Azure/InterceptingAsyncPageable.cs b/GuildWarsPartySearch/Services/Azure/InterceptingAsyncPageable.cs new file mode 100644 index 0000000..f2ced41 --- /dev/null +++ b/GuildWarsPartySearch/Services/Azure/InterceptingAsyncPageable.cs @@ -0,0 +1,35 @@ +using Azure; + +namespace GuildWarsPartySearch.Server.Services.Azure; + +public class InterceptingAsyncPageable : AsyncPageable where T : notnull +{ + private readonly Action> interceptPage; + private readonly Action interceptSuccess; + private readonly AsyncPageable originalPageable; + + public InterceptingAsyncPageable(AsyncPageable originalPageable, Action> interceptPage, Action interceptSuccess) + { + this.originalPageable = originalPageable; + this.interceptPage = interceptPage; + this.interceptSuccess = interceptSuccess; + } + + public override async IAsyncEnumerable> AsPages(string? continuationToken = null, int? pageSizeHint = null) + { + await foreach (var page in this.originalPageable.AsPages(continuationToken, pageSizeHint)) + { + this.interceptPage(page); + yield return page; + } + } + + public async override IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + await foreach(var item in this.originalPageable) + { + this.interceptSuccess(); + yield return item; + } + } +} diff --git a/GuildWarsPartySearch/Services/Azure/InterceptingPageable.cs b/GuildWarsPartySearch/Services/Azure/InterceptingPageable.cs new file mode 100644 index 0000000..9a72d2e --- /dev/null +++ b/GuildWarsPartySearch/Services/Azure/InterceptingPageable.cs @@ -0,0 +1,35 @@ +using Azure; + +namespace GuildWarsPartySearch.Server.Services.Azure; + +public sealed class InterceptingPageable : Pageable where T: notnull +{ + private readonly Action> interceptPage; + private readonly Action interceptSuccess; + private readonly Pageable originalPageable; + + public InterceptingPageable(Pageable originalPageable, Action> interceptPage, Action interceptSuccess) + { + this.originalPageable = originalPageable; + this.interceptPage = interceptPage; + this.interceptSuccess = interceptSuccess; + } + + public override IEnumerable> AsPages(string? continuationToken = null, int? pageSizeHint = null) + { + foreach (var page in this.originalPageable.AsPages(continuationToken, pageSizeHint)) + { + this.interceptPage(page); + yield return page; + } + } + + public override IEnumerator GetEnumerator() + { + foreach (var item in this.originalPageable) + { + this.interceptSuccess(); + yield return item; + } + } +} diff --git a/GuildWarsPartySearch/Services/Azure/NamedBlobContainerClient.cs b/GuildWarsPartySearch/Services/Azure/NamedBlobContainerClient.cs new file mode 100644 index 0000000..69d1c52 --- /dev/null +++ b/GuildWarsPartySearch/Services/Azure/NamedBlobContainerClient.cs @@ -0,0 +1,40 @@ +using Azure; +using Azure.Core; +using Azure.Storage; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using GuildWarsPartySearch.Server.Options; + +namespace GuildWarsPartySearch.Server.Services.Azure; + +public class NamedBlobContainerClient : BlobContainerClient + where TOptions : IAzureBlobStorageOptions +{ + public NamedBlobContainerClient(string connectionString, string blobContainerName) : base(connectionString, blobContainerName) + { + } + + public NamedBlobContainerClient(Uri blobContainerUri, BlobClientOptions? options = null) : base(blobContainerUri, options) + { + } + + public NamedBlobContainerClient(string connectionString, string blobContainerName, BlobClientOptions options) : base(connectionString, blobContainerName, options) + { + } + + public NamedBlobContainerClient(Uri blobContainerUri, StorageSharedKeyCredential credential, BlobClientOptions? options = null) : base(blobContainerUri, credential, options) + { + } + + public NamedBlobContainerClient(Uri blobContainerUri, AzureSasCredential credential, BlobClientOptions? options = null) : base(blobContainerUri, credential, options) + { + } + + public NamedBlobContainerClient(Uri blobContainerUri, TokenCredential credential, BlobClientOptions? options = null) : base(blobContainerUri, credential, options) + { + } + + protected NamedBlobContainerClient() + { + } +} diff --git a/GuildWarsPartySearch/Services/Azure/NamedTableClient.cs b/GuildWarsPartySearch/Services/Azure/NamedTableClient.cs new file mode 100644 index 0000000..e5a4a39 --- /dev/null +++ b/GuildWarsPartySearch/Services/Azure/NamedTableClient.cs @@ -0,0 +1,363 @@ +using Azure; +using Azure.Core; +using Azure.Data.Tables; +using Azure.Data.Tables.Models; +using Microsoft.AspNetCore.Mvc.RazorPages; +using Newtonsoft.Json; +using System.Collections.Concurrent; +using System.Core.Extensions; +using System.Diagnostics; +using System.Extensions; +using System.Linq.Expressions; +using System.Logging; + +namespace GuildWarsPartySearch.Server.Services.Azure; + +public class NamedTableClient : TableClient +{ + private readonly ILogger> logger; + + public NamedTableClient(ILogger> logger) + { + this.logger = logger.ThrowIfNull(); + } + + public NamedTableClient(ILogger> logger, Uri endpoint, TableClientOptions? options = null) : base(endpoint, options) + { + this.logger = logger.ThrowIfNull(); + } + + public NamedTableClient(ILogger> logger, string connectionString, string tableName) : base(connectionString, tableName) + { + this.logger = logger.ThrowIfNull(); + } + + public NamedTableClient(ILogger> logger, Uri endpoint, AzureSasCredential credential, TableClientOptions? options = null) : base(endpoint, credential, options) + { + this.logger = logger.ThrowIfNull(); + } + + public NamedTableClient(ILogger> logger, Uri endpoint, string tableName, TableSharedKeyCredential credential) : base(endpoint, tableName, credential) + { + this.logger = logger.ThrowIfNull(); + } + + public NamedTableClient(ILogger> logger, string connectionString, string tableName, TableClientOptions? options = null) : base(connectionString, tableName, options) + { + this.logger = logger.ThrowIfNull(); + } + + public NamedTableClient(ILogger> logger, Uri endpoint, string tableName, TableSharedKeyCredential credential, TableClientOptions? options = null) : base(endpoint, tableName, credential, options) + { + this.logger = logger.ThrowIfNull(); + } + + public NamedTableClient(ILogger> logger, Uri endpoint, string tableName, TokenCredential tokenCredential, TableClientOptions? options = null) : base(endpoint, tableName, tokenCredential, options) + { + this.logger = logger.ThrowIfNull(); + } + + public override Response Create(CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.Create(cancellationToken), nameof(this.Create)); + } + + public override Response CreateIfNotExists(CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.CreateIfNotExists(cancellationToken), nameof(this.CreateIfNotExists)); + } + + public override Response AddEntity(T entity, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.AddEntity(entity, cancellationToken), nameof(this.AddEntity)); + } + + public override Response Delete(CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.Delete(cancellationToken), nameof(this.Delete)); + } + + public override Response DeleteEntity(string partitionKey, string rowKey, ETag ifMatch = default, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.DeleteEntity(partitionKey, rowKey, ifMatch, cancellationToken), nameof(this.DeleteEntity)); + } + + public override Response UpdateEntity(T entity, ETag ifMatch, TableUpdateMode mode = TableUpdateMode.Merge, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.UpdateEntity(entity, ifMatch, mode, cancellationToken), nameof(this.UpdateEntity)); + } + + public override Response UpsertEntity(T entity, TableUpdateMode mode = TableUpdateMode.Merge, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.UpsertEntity(entity, mode, cancellationToken), nameof(this.UpsertEntity)); + } + + public override Response GetEntity(string partitionKey, string rowKey, IEnumerable? select = null, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.GetEntity(partitionKey, rowKey, select, cancellationToken), nameof(this.GetEntity)); + } + + public override NullableResponse GetEntityIfExists(string partitionKey, string rowKey, IEnumerable? select = null, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.GetEntityIfExists(partitionKey, rowKey, select, cancellationToken), nameof(GetEntityIfExists)); + } + + public override Pageable Query(string? filter = null, int? maxPerPage = null, IEnumerable? select = null, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.Query(filter, maxPerPage, select, cancellationToken), nameof(this.Query)); + } + + public override Pageable Query(Expression> filter, int? maxPerPage = null, IEnumerable? select = null, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.Query(filter, maxPerPage, select, cancellationToken), nameof(this.Query)); + } + + public override Response> SubmitTransaction(IEnumerable transactionActions, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.SubmitTransaction(transactionActions, cancellationToken), nameof(this.SubmitTransaction)); + } + + public override Task> CreateAsync(CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.CreateAsync(cancellationToken), nameof(this.CreateAsync)); + } + + public override Task> CreateIfNotExistsAsync(CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.CreateIfNotExistsAsync(cancellationToken), nameof(this.CreateIfNotExistsAsync)); + } + + public override Task AddEntityAsync(T entity, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.AddEntityAsync(entity, cancellationToken), nameof(this.AddEntityAsync)); + } + + public override Task DeleteAsync(CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.DeleteAsync(cancellationToken), nameof(this.DeleteAsync)); + } + + public override Task DeleteEntityAsync(string partitionKey, string rowKey, ETag ifMatch = default, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.DeleteEntityAsync(partitionKey, rowKey, ifMatch, cancellationToken), nameof(this.DeleteEntityAsync)); + } + + public override Task UpdateEntityAsync(T entity, ETag ifMatch, TableUpdateMode mode = TableUpdateMode.Merge, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.UpdateEntityAsync(entity, ifMatch, mode, cancellationToken), nameof(this.UpdateEntityAsync)); + } + + public override Task UpsertEntityAsync(T entity, TableUpdateMode mode = TableUpdateMode.Merge, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.UpsertEntityAsync(entity, mode, cancellationToken), nameof(this.UpsertEntityAsync)); + } + + public override Task> GetEntityAsync(string partitionKey, string rowKey, IEnumerable? select = null, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.GetEntityAsync(partitionKey, rowKey, select, cancellationToken), nameof(this.GetEntityAsync)); + } + + public override Task> GetEntityIfExistsAsync(string partitionKey, string rowKey, IEnumerable? select = null, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.GetEntityIfExistsAsync(partitionKey, rowKey, select, cancellationToken), nameof(this.GetEntityIfExistsAsync)); + } + + public override AsyncPageable QueryAsync(Expression> filter, int? maxPerPage = null, IEnumerable? select = null, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.QueryAsync(filter, maxPerPage, select, cancellationToken), nameof(this.QueryAsync)); + } + + public override AsyncPageable QueryAsync(string? filter = null, int? maxPerPage = null, IEnumerable? select = null, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.QueryAsync(filter, maxPerPage, select, cancellationToken), nameof(this.QueryAsync)); + } + + public override Task>> SubmitTransactionAsync(IEnumerable transactionActions, CancellationToken cancellationToken = default) + { + return this.LogOperation(() => base.SubmitTransactionAsync(transactionActions, cancellationToken), nameof(this.SubmitTransactionAsync)); + } + + private Response LogOperation(Func func, string operationName) + { + var scopedLogger = this.logger.CreateScopedLogger(operationName, string.Empty); + var sw = Stopwatch.StartNew(); + scopedLogger.LogInformation(">> {0}", this.Uri); + try + { + var response = func(); + scopedLogger.LogInformation("<< {0} {1}ms", response.Status, sw.ElapsedMilliseconds); + return response; + } + catch(RequestFailedException ex) + { + scopedLogger.LogInformation("<< {0} {1}ms", ex.Status, sw.ElapsedMilliseconds); + throw; + } + } + + private Response LogOperation(Func> func, string operationName) + { + var scopedLogger = this.logger.CreateScopedLogger(operationName, string.Empty); + var sw = Stopwatch.StartNew(); + scopedLogger.LogInformation(">> {0}", this.Uri); + try + { + var response = func(); + scopedLogger.LogInformation("<< {0} {1}ms", response.GetRawResponse().Status, sw.ElapsedMilliseconds); + return response; + } + catch (RequestFailedException ex) + { + scopedLogger.LogInformation("<< {0} {1}ms", ex.Status, sw.ElapsedMilliseconds); + throw; + } + } + + private NullableResponse LogOperation(Func> func, string operationName) + { + var scopedLogger = this.logger.CreateScopedLogger(operationName, string.Empty); + var sw = Stopwatch.StartNew(); + scopedLogger.LogInformation(">> {0}", this.Uri); + try + { + var response = func(); + scopedLogger.LogInformation("<< {0} {1}ms", response.GetRawResponse().Status, sw.ElapsedMilliseconds); + return response; + } + catch (RequestFailedException ex) + { + scopedLogger.LogInformation("<< {0} {1}ms", ex.Status, sw.ElapsedMilliseconds); + throw; + } + } + + private Pageable LogOperation(Func> func, string operationName) + where T : notnull + { + var scopedLogger = this.logger.CreateScopedLogger(operationName, string.Empty); + var sw = Stopwatch.StartNew(); + scopedLogger.LogInformation(">> {0}", this.Uri); + var response = func(); + return new InterceptingPageable(response, page => + { + scopedLogger.LogInformation("<< {0} {1}ms", page.GetRawResponse().Status, sw.ElapsedMilliseconds); + }, () => + { + scopedLogger.LogInformation("<< 200 {1}ms", sw.ElapsedMilliseconds); + }); + } + + private Response> LogOperation(Func>> func, string operationName) + { + var scopedLogger = this.logger.CreateScopedLogger(operationName, string.Empty); + var sw = Stopwatch.StartNew(); + scopedLogger.LogInformation(">> {0}", this.Uri); + try + { + var response = func(); + foreach (var action in response.Value) + { + scopedLogger.LogInformation("<< {0} {1}ms", action.Status, sw.ElapsedMilliseconds); + } + + return response; + } + catch (RequestFailedException ex) + { + scopedLogger.LogInformation("<< {0} {1}ms", ex.Status, sw.ElapsedMilliseconds); + throw; + } + } + + private async Task LogOperation(Func> func, string operationName) + { + var scopedLogger = this.logger.CreateScopedLogger(operationName, string.Empty); + var sw = Stopwatch.StartNew(); + scopedLogger.LogInformation(">> {0}", this.Uri); + try + { + var response = await func(); + scopedLogger.LogInformation("<< {0} {1}ms", response.Status, sw.ElapsedMilliseconds); + return response; + } + catch (RequestFailedException ex) + { + scopedLogger.LogInformation("<< {0} {1}ms", ex.Status, sw.ElapsedMilliseconds); + throw; + } + } + + private async Task> LogOperation(Func>> func, string operationName) + { + var scopedLogger = this.logger.CreateScopedLogger(operationName, string.Empty); + var sw = Stopwatch.StartNew(); + scopedLogger.LogInformation(">> {0}", this.Uri); + try + { + var response = await func(); + scopedLogger.LogInformation("<< {0} {1}ms", response.GetRawResponse().Status, sw.ElapsedMilliseconds); + return response; + } + catch (RequestFailedException ex) + { + scopedLogger.LogInformation("<< {0} {1}ms", ex.Status, sw.ElapsedMilliseconds); + throw; + } + } + + private async Task> LogOperation(Func>> func, string operationName) + { + var scopedLogger = this.logger.CreateScopedLogger(operationName, string.Empty); + var sw = Stopwatch.StartNew(); + scopedLogger.LogInformation(">> {0}", this.Uri); + try + { + var response = await func(); + scopedLogger.LogInformation("<< {0} {1}ms", response.GetRawResponse().Status, sw.ElapsedMilliseconds); + return response; + } + catch (RequestFailedException ex) + { + scopedLogger.LogInformation("<< {0} {1}ms", ex.Status, sw.ElapsedMilliseconds); + throw; + } + } + + private AsyncPageable LogOperation(Func> func, string operationName) + where T : notnull + { + var scopedLogger = this.logger.CreateScopedLogger(operationName, string.Empty); + var sw = Stopwatch.StartNew(); + scopedLogger.LogInformation(">> {0}", this.Uri); + var response = func(); + return new InterceptingAsyncPageable(response, page => + { + scopedLogger.LogInformation("<< {0} {1}ms", page.GetRawResponse().Status, sw.ElapsedMilliseconds); + }, () => + { + scopedLogger.LogInformation("<< 200 {1}ms", sw.ElapsedMilliseconds); + }); + } + + private async Task>> LogOperation(Func>>> func, string operationName) + { + var scopedLogger = this.logger.CreateScopedLogger(operationName, string.Empty); + var sw = Stopwatch.StartNew(); + scopedLogger.LogInformation(">> {0}", this.Uri); + try + { + var response = await func(); + foreach (var action in response.Value) + { + scopedLogger.LogInformation("<< {0} {1}ms", action.Status, sw.ElapsedMilliseconds); + } + + return response; + } + catch (RequestFailedException ex) + { + scopedLogger.LogInformation("<< {0} {1}ms", ex.Status, sw.ElapsedMilliseconds); + throw; + } + } +} diff --git a/GuildWarsPartySearch/Services/Content/ContentRetrievalService.cs b/GuildWarsPartySearch/Services/Content/ContentRetrievalService.cs index 16f46fc..ea8160d 100644 --- a/GuildWarsPartySearch/Services/Content/ContentRetrievalService.cs +++ b/GuildWarsPartySearch/Services/Content/ContentRetrievalService.cs @@ -1,9 +1,9 @@ using Azure.Storage.Blobs.Models; -using Azure.Storage.Blobs; using GuildWarsPartySearch.Server.Options; using System.Extensions; using Microsoft.Extensions.Options; using System.Core.Extensions; +using GuildWarsPartySearch.Server.Services.Azure; namespace GuildWarsPartySearch.Server.Services.Content; @@ -11,17 +11,17 @@ public sealed class ContentRetrievalService : BackgroundService { private readonly Dictionary fileMetadatas = []; + private readonly NamedBlobContainerClient namedBlobContainerClient; private readonly ContentOptions contentOptions; - private readonly StorageAccountOptions storageAccountOptions; private readonly ILogger logger; public ContentRetrievalService( + NamedBlobContainerClient namedBlobContainerClient, IOptions contentOptions, - IOptions storageAccountOptions, ILogger logger) { + this.namedBlobContainerClient = namedBlobContainerClient.ThrowIfNull(); this.contentOptions = contentOptions.Value.ThrowIfNull(); - this.storageAccountOptions = storageAccountOptions.Value.ThrowIfNull(); this.logger = logger.ThrowIfNull(); } @@ -47,9 +47,7 @@ private async Task UpdateContent(CancellationToken cancellationToken) { var scopedLogger = logger.CreateScopedLogger(nameof(this.UpdateContent), string.Empty); scopedLogger.LogDebug("Checking content to retrieve"); - var serviceBlobClient = new BlobServiceClient(storageAccountOptions.ConnectionString); - var blobContainerClient = serviceBlobClient.GetBlobContainerClient(storageAccountOptions.ContainerName); - var blobs = blobContainerClient.GetBlobsAsync(cancellationToken: cancellationToken); + var blobs = this.namedBlobContainerClient.GetBlobsAsync(cancellationToken: cancellationToken); if (!Directory.Exists(contentOptions.StagingFolder)) { @@ -90,7 +88,7 @@ private async Task UpdateContent(CancellationToken cancellationToken) continue; } - var blobClient = blobContainerClient.GetBlobClient(blob.Name); + var blobClient = this.namedBlobContainerClient.GetBlobClient(blob.Name); using var fileStream = new FileStream(finalPath, FileMode.Create); using var blobStream = await blobClient.OpenReadAsync(new BlobOpenReadOptions(false) { diff --git a/GuildWarsPartySearch/Services/Database/TableStorageDatabase.cs b/GuildWarsPartySearch/Services/Database/TableStorageDatabase.cs index e21363c..6dc7984 100644 --- a/GuildWarsPartySearch/Services/Database/TableStorageDatabase.cs +++ b/GuildWarsPartySearch/Services/Database/TableStorageDatabase.cs @@ -2,8 +2,8 @@ using GuildWarsPartySearch.Common.Models.GuildWars; using GuildWarsPartySearch.Server.Models; using GuildWarsPartySearch.Server.Options; +using GuildWarsPartySearch.Server.Services.Azure; using GuildWarsPartySearch.Server.Services.Database.Models; -using Microsoft.Extensions.Options; using System.Core.Extensions; using System.Extensions; @@ -11,23 +11,15 @@ namespace GuildWarsPartySearch.Server.Services.Database; public sealed class TableStorageDatabase : IPartySearchDatabase { - private readonly TableClient tableClient; - private readonly IOptions options; + private readonly NamedTableClient client; private readonly ILogger logger; public TableStorageDatabase( - IOptions options, + NamedTableClient namedTableClient, ILogger logger) { - var tableClientOptions = new TableClientOptions(); - tableClientOptions.Diagnostics.IsLoggingEnabled = true; - - this.options = options.ThrowIfNull(); + this.client = namedTableClient.ThrowIfNull(); this.logger = logger.ThrowIfNull(); - - var tableServiceClient = new TableServiceClient(this.options?.Value.ConnectionString ?? throw new InvalidOperationException("Config contains no connection string")); - var tableName = options.Value.TableName?.ThrowIfNull() ?? throw new InvalidOperationException("Config contains no table name"); - this.tableClient = tableServiceClient.GetTableClient(tableName); } public async Task> GetAllPartySearches(CancellationToken cancellationToken) @@ -144,7 +136,6 @@ public async Task SetPartySearches(Campaign campaign, Continent continent, var scopedLogger = this.logger.CreateScopedLogger(nameof(this.SetPartySearches), partitionKey); try { - scopedLogger.LogInformation("Retrieving existing entries"); var existingEntries = await this.GetPartySearches(campaign, continent, region, map, district, cancellationToken); var entries = partySearch.Select(e => { @@ -169,7 +160,6 @@ public async Task SetPartySearches(Campaign campaign, Continent continent, var actions = new List(); if (existingEntries is not null) { - scopedLogger.LogInformation("Patching nonexisting entries"); // Find all existing entries that don't exist in the update. For those, queue a delete transaction actions.AddRange(existingEntries .Where(e => entries.FirstOrDefault(e2 => e.CharName == e2.CharName) is null) @@ -180,7 +170,6 @@ public async Task SetPartySearches(Campaign campaign, Continent continent, }))); } - scopedLogger.LogInformation("Batch transaction"); actions.AddRange(entries .Where(e => { @@ -203,7 +192,7 @@ public async Task SetPartySearches(Campaign campaign, Continent continent, return true; } - var responses = await this.tableClient.SubmitTransactionAsync(actions, cancellationToken); + var responses = await this.client.SubmitTransactionAsync(actions, cancellationToken); foreach(var response in responses.Value) { scopedLogger.LogInformation($"[{response.Status}] {response.ReasonPhrase}"); @@ -221,7 +210,7 @@ public async Task SetPartySearches(Campaign campaign, Continent continent, private async Task> QuerySearches(string query, CancellationToken cancellationToken) { var responseList = new Dictionary)>(); - var response = this.tableClient.QueryAsync(query, cancellationToken: cancellationToken); + var response = this.client.QueryAsync(query, cancellationToken: cancellationToken); await foreach (var entry in response) { if (!responseList.TryGetValue(entry.PartitionKey, out var tuple))