From a7dcf7e58996a2a808a36e772875fdd429d221c3 Mon Sep 17 00:00:00 2001 From: Ricardo Boss Date: Sat, 1 Jul 2023 13:18:29 +0200 Subject: [PATCH 1/4] Added endpoint for causing a reindex --- .../Controllers/SearchController.cs | 24 ++++++++++- TRENZ.Docs.API/Program.cs | 3 +- TRENZ.Docs.API/Worker.cs | 41 +++++++++++-------- TRENZ.Docs.API/appsettings.example.json | 3 +- 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/TRENZ.Docs.API/Controllers/SearchController.cs b/TRENZ.Docs.API/Controllers/SearchController.cs index 36dc08d..9826bd4 100644 --- a/TRENZ.Docs.API/Controllers/SearchController.cs +++ b/TRENZ.Docs.API/Controllers/SearchController.cs @@ -9,10 +9,14 @@ namespace TRENZ.Docs.API.Controllers; public class SearchController : ControllerBase { private readonly IIndexingService _indexingService; + private readonly IConfiguration _configuration; + private readonly Worker _worker; - public SearchController(IIndexingService indexingService) + public SearchController(IIndexingService indexingService, IConfiguration configuration, Worker worker) { _indexingService = indexingService; + _configuration = configuration; + _worker = worker; } [HttpGet] @@ -26,4 +30,20 @@ public async Task Stats() { return await _indexingService.GetStats(); } -} \ No newline at end of file + + [HttpGet] + public async Task Reindex([FromQuery] string? key, CancellationToken cancellationToken = default) + { + if (_configuration["ReindexPassword"] != key) + return; + +#if !DEBUG + if (DateTime.Now - _lastReindex < TimeSpan.FromMinutes(30)) + return; + + _lastReindex = DateTime.Now; +#endif + + await _worker.DoReindex(cancellationToken); + } +} diff --git a/TRENZ.Docs.API/Program.cs b/TRENZ.Docs.API/Program.cs index a868d26..e1fc30a 100644 --- a/TRENZ.Docs.API/Program.cs +++ b/TRENZ.Docs.API/Program.cs @@ -38,7 +38,8 @@ builder.Services.AddAuthAdapter(); } -builder.Services.AddHostedService(); +builder.Services.AddSingleton(); +builder.Services.AddHostedService(sp => sp.GetRequiredService()); builder.Services.AddControllers(); diff --git a/TRENZ.Docs.API/Worker.cs b/TRENZ.Docs.API/Worker.cs index 1a1a9c4..da25df6 100644 --- a/TRENZ.Docs.API/Worker.cs +++ b/TRENZ.Docs.API/Worker.cs @@ -29,23 +29,9 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { - _logger.LogInformation("Running at {Time}", DateTimeOffset.Now); + _logger.LogInformation("Worker running at {Time}", DateTimeOffset.Now); - var sourceFiles = await GatherFiles(stoppingToken).ToListAsync(stoppingToken); - var tree = await _navTreeProvider.RebuildAsync(sourceFiles, stoppingToken); - - var indexTree = tree.WithoutHiddenNodes().WithoutChildlessContentlessNodes(); - var markdownFiles = sourceFiles - .Where(file => file.RelativePath.EndsWith(".md")) - .Select(file => (file, node: indexTree.FindNodeByLocation(file.Location))) - .Where(tup => tup.node is not null) - .Select(tup => tup.file); - - var indexFiles = await _fileProcessingService - .ProcessAsync(markdownFiles, stoppingToken) - .ToListAsync(stoppingToken); - - await _indexingService.IndexAsync(indexFiles, stoppingToken); + await DoReindex(stoppingToken); if (_configuration["TrenzDocsApi:OneShot"] == "true") { @@ -64,6 +50,29 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) _lifetime.StopApplication(); } + public async Task DoReindex(CancellationToken cancellationToken) + { + _logger.LogInformation("Reindexing..."); + + var sourceFiles = await GatherFiles(cancellationToken).ToListAsync(cancellationToken); + var tree = await _navTreeProvider.RebuildAsync(sourceFiles, cancellationToken); + + var indexTree = tree.WithoutHiddenNodes().WithoutChildlessContentlessNodes(); + var markdownFiles = sourceFiles + .Where(file => file.RelativePath.EndsWith(".md")) + .Select(file => (file, node: indexTree.FindNodeByLocation(file.Location))) + .Where(tup => tup.node is not null) + .Select(tup => tup.file); + + var indexFiles = await _fileProcessingService + .ProcessAsync(markdownFiles, cancellationToken) + .ToListAsync(cancellationToken); + + await _indexingService.IndexAsync(indexFiles, cancellationToken); + + _logger.LogInformation("Reindexing done"); + } + private async IAsyncEnumerable GatherFiles([EnumeratorCancellation] CancellationToken cancellationToken) { _logger.LogInformation("Gathering files..."); diff --git a/TRENZ.Docs.API/appsettings.example.json b/TRENZ.Docs.API/appsettings.example.json index be08875..4a63561 100644 --- a/TRENZ.Docs.API/appsettings.example.json +++ b/TRENZ.Docs.API/appsettings.example.json @@ -20,5 +20,6 @@ "Endpoint": "https://my-secure-provider.com/SignIn", "ClientId": "my-client-id", "ClientSecret": "my-client-secret" - } + }, + "ReindexPassword": "$ome.$3cur3_Key" } \ No newline at end of file From fa2afd0f38dee59070f463f4a3eeea321e9ed4a6 Mon Sep 17 00:00:00 2001 From: Ricardo Boss Date: Sat, 1 Jul 2023 13:26:47 +0200 Subject: [PATCH 2/4] Added missing field --- TRENZ.Docs.API/Controllers/SearchController.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TRENZ.Docs.API/Controllers/SearchController.cs b/TRENZ.Docs.API/Controllers/SearchController.cs index 9826bd4..88008ab 100644 --- a/TRENZ.Docs.API/Controllers/SearchController.cs +++ b/TRENZ.Docs.API/Controllers/SearchController.cs @@ -8,6 +8,8 @@ namespace TRENZ.Docs.API.Controllers; [Route("api/[controller]/[action]")] public class SearchController : ControllerBase { + private static DateTime? _lastReindex; + private readonly IIndexingService _indexingService; private readonly IConfiguration _configuration; private readonly Worker _worker; From b6c3f405933f90aa73e9deefbc48ca389d947613 Mon Sep 17 00:00:00 2001 From: Ricardo Boss Date: Sat, 1 Jul 2023 14:59:00 +0200 Subject: [PATCH 3/4] Rename Worker to IndexWorker --- TRENZ.Docs.API/Controllers/SearchController.cs | 8 ++++---- TRENZ.Docs.API/{Worker.cs => IndexWorker.cs} | 6 +++--- TRENZ.Docs.API/Program.cs | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) rename TRENZ.Docs.API/{Worker.cs => IndexWorker.cs} (89%) diff --git a/TRENZ.Docs.API/Controllers/SearchController.cs b/TRENZ.Docs.API/Controllers/SearchController.cs index 88008ab..c7233c8 100644 --- a/TRENZ.Docs.API/Controllers/SearchController.cs +++ b/TRENZ.Docs.API/Controllers/SearchController.cs @@ -12,13 +12,13 @@ public class SearchController : ControllerBase private readonly IIndexingService _indexingService; private readonly IConfiguration _configuration; - private readonly Worker _worker; + private readonly IndexWorker _indexWorker; - public SearchController(IIndexingService indexingService, IConfiguration configuration, Worker worker) + public SearchController(IIndexingService indexingService, IConfiguration configuration, IndexWorker indexWorker) { _indexingService = indexingService; _configuration = configuration; - _worker = worker; + _indexWorker = indexWorker; } [HttpGet] @@ -46,6 +46,6 @@ public async Task Reindex([FromQuery] string? key, CancellationToken cancellatio _lastReindex = DateTime.Now; #endif - await _worker.DoReindex(cancellationToken); + await _indexWorker.DoReindex(cancellationToken); } } diff --git a/TRENZ.Docs.API/Worker.cs b/TRENZ.Docs.API/IndexWorker.cs similarity index 89% rename from TRENZ.Docs.API/Worker.cs rename to TRENZ.Docs.API/IndexWorker.cs index da25df6..9b0845c 100644 --- a/TRENZ.Docs.API/Worker.cs +++ b/TRENZ.Docs.API/IndexWorker.cs @@ -4,9 +4,9 @@ namespace TRENZ.Docs.API { - public class Worker : BackgroundService + public class IndexWorker : BackgroundService { - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly IConfiguration _configuration; private readonly IHostApplicationLifetime _lifetime; private readonly IIndexingService _indexingService; @@ -14,7 +14,7 @@ public class Worker : BackgroundService private readonly IFileProcessingService _fileProcessingService; private readonly INavTreeProvider _navTreeProvider; - public Worker(ILogger logger, IConfiguration configuration, IHostApplicationLifetime lifetime, IIndexingService indexingService, ISourcesProvider sourcesProvider, IFileProcessingService fileProcessingService, INavTreeProvider navTreeProvider) + public IndexWorker(ILogger logger, IConfiguration configuration, IHostApplicationLifetime lifetime, IIndexingService indexingService, ISourcesProvider sourcesProvider, IFileProcessingService fileProcessingService, INavTreeProvider navTreeProvider) { _logger = logger; _configuration = configuration; diff --git a/TRENZ.Docs.API/Program.cs b/TRENZ.Docs.API/Program.cs index e1fc30a..d69b268 100644 --- a/TRENZ.Docs.API/Program.cs +++ b/TRENZ.Docs.API/Program.cs @@ -38,8 +38,8 @@ builder.Services.AddAuthAdapter(); } -builder.Services.AddSingleton(); -builder.Services.AddHostedService(sp => sp.GetRequiredService()); +builder.Services.AddSingleton(); +builder.Services.AddHostedService(sp => sp.GetRequiredService()); builder.Services.AddControllers(); From 7f21e2fffca1198d187f474a47dbcd296588fc94 Mon Sep 17 00:00:00 2001 From: Ricardo Boss Date: Tue, 5 Dec 2023 09:47:21 +0100 Subject: [PATCH 4/4] Added response codes and configurable reindex throttle --- TRENZ.Docs.API/Controllers/SearchController.cs | 10 ++++++---- TRENZ.Docs.API/appsettings.json | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/TRENZ.Docs.API/Controllers/SearchController.cs b/TRENZ.Docs.API/Controllers/SearchController.cs index c7233c8..c59a02c 100644 --- a/TRENZ.Docs.API/Controllers/SearchController.cs +++ b/TRENZ.Docs.API/Controllers/SearchController.cs @@ -34,18 +34,20 @@ public async Task Stats() } [HttpGet] - public async Task Reindex([FromQuery] string? key, CancellationToken cancellationToken = default) + public async Task Reindex([FromQuery] string? key, CancellationToken cancellationToken = default) { if (_configuration["ReindexPassword"] != key) - return; + return Unauthorized(); #if !DEBUG - if (DateTime.Now - _lastReindex < TimeSpan.FromMinutes(30)) - return; + if (DateTime.Now - _lastReindex < TimeSpan.FromSeconds(_configuration.GetValue("ReindexThrottling"))) + return new StatusCodeResult(429); _lastReindex = DateTime.Now; #endif await _indexWorker.DoReindex(cancellationToken); + + return Ok(); } } diff --git a/TRENZ.Docs.API/appsettings.json b/TRENZ.Docs.API/appsettings.json index 10f68b8..34d4eae 100644 --- a/TRENZ.Docs.API/appsettings.json +++ b/TRENZ.Docs.API/appsettings.json @@ -5,5 +5,6 @@ "Microsoft.AspNetCore": "Warning" } }, - "AllowedHosts": "*" + "AllowedHosts": "*", + "ReindexThrottling": 600 }