From d8168589131ef4bbdf01a03712ffb78a5193c90e Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 25 Nov 2024 15:19:54 +0700 Subject: [PATCH] make api to call fw headless sync (#1252) * add crdt sync passthrough endpoint to call crdt sync on fw headless, setup service discovery to simplify configuration * fix issue with cookie port number not working when in the host parameter * log errors from fw headless, and return a problem from the lexbox api * Add button for triggering CRDT sync * increase request timeout on the sync endpoint, change the path to not be redundant --------- Co-authored-by: Tim Haasdyk --- Tiltfile | 2 +- backend/FwHeadless/HttpClientAuthHandler.cs | 5 ++- backend/FwHeadless/Program.cs | 5 ++- .../CrdtFwdataProjectSyncService.cs | 3 +- .../LexBoxApi/Controllers/CrdtController.cs | 18 +++++++- backend/LexBoxApi/LexBoxApi.csproj | 1 + backend/LexBoxApi/LexBoxKernel.cs | 3 ++ .../LexBoxApi/Services/FwHeadlessClient.cs | 19 ++++++++ .../LexBoxApi/appsettings.Development.json | 7 +++ backend/LexBoxApi/appsettings.json | 5 +++ backend/LexCore/Sync/SyncResult.cs | 3 ++ backend/LexData/SeedingData.cs | 1 + deployment/base/lexbox-deployment.yaml | 3 ++ frontend/src/lib/forms/Button.svelte | 7 ++- frontend/src/lib/icons/Icon.svelte | 4 +- .../project/[project_code]/+page.svelte | 2 + .../[project_code]/CrdtSyncButton.svelte | 43 +++++++++++++++++++ 17 files changed, 120 insertions(+), 11 deletions(-) create mode 100644 backend/LexBoxApi/Services/FwHeadlessClient.cs create mode 100644 backend/LexCore/Sync/SyncResult.cs create mode 100644 frontend/src/routes/(authenticated)/project/[project_code]/CrdtSyncButton.svelte diff --git a/Tiltfile b/Tiltfile index 24b320e75..061bdb63d 100644 --- a/Tiltfile +++ b/Tiltfile @@ -28,7 +28,7 @@ docker_build( context='backend', dockerfile='./backend/FwHeadless/dev.Dockerfile', only=['.'], - ignore=['LexBoxApi'], + ignore=['LexBoxApi', '**/Mercurial', '**/MercurialExtensions'], live_update=[ sync('backend', '/src/backend') ] diff --git a/backend/FwHeadless/HttpClientAuthHandler.cs b/backend/FwHeadless/HttpClientAuthHandler.cs index cc3efa799..41d364334 100644 --- a/backend/FwHeadless/HttpClientAuthHandler.cs +++ b/backend/FwHeadless/HttpClientAuthHandler.cs @@ -34,7 +34,10 @@ protected override async Task SendAsync(HttpRequestMessage private async Task SetAuthHeader(HttpRequestMessage request, CancellationToken cancellationToken, Uri lexboxUrl) { var cookieContainer = new CookieContainer(); - cookieContainer.Add(new Cookie(LexAuthConstants.AuthCookieName, await GetToken(cancellationToken), null, lexboxUrl.Authority)); + cookieContainer.Add(new Cookie(LexAuthConstants.AuthCookieName, await GetToken(cancellationToken), null, lexboxUrl.Host) + { + Port = $"\"{lexboxUrl.Port}\"" + }); request.Headers.Add("Cookie", cookieContainer.GetCookieHeader(lexboxUrl)); } diff --git a/backend/FwHeadless/Program.cs b/backend/FwHeadless/Program.cs index e4e897016..95e175463 100644 --- a/backend/FwHeadless/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -4,6 +4,7 @@ using FwLiteProjectSync; using LcmCrdt; using LcmCrdt.RemoteSync; +using LexCore.Sync; using LexData; using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Extensions.Options; @@ -51,7 +52,7 @@ app.Run(); -static async Task, NotFound, ProblemHttpResult>> ExecuteMergeRequest( +static async Task, NotFound, ProblemHttpResult>> ExecuteMergeRequest( ILogger logger, IServiceProvider services, SendReceiveService srService, @@ -69,7 +70,7 @@ if (dryRun) { logger.LogInformation("Dry run, not actually syncing"); - return TypedResults.Ok(new CrdtFwdataProjectSyncService.SyncResult(0, 0)); + return TypedResults.Ok(new SyncResult(0, 0)); } var projectCode = await projectLookupService.GetProjectCode(projectId); diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs index 12258dcee..ee42e1026 100644 --- a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs +++ b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs @@ -1,6 +1,7 @@ using System.Text.Json; using FwDataMiniLcmBridge.Api; using LcmCrdt; +using LexCore.Sync; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MiniLcm; @@ -13,8 +14,6 @@ namespace FwLiteProjectSync; public class CrdtFwdataProjectSyncService(IOptions lcmCrdtConfig, MiniLcmImport miniLcmImport, ILogger logger) { - public record SyncResult(int CrdtChanges, int FwdataChanges); - public async Task Sync(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataApi, bool dryRun = false) { if (crdtApi is CrdtMiniLcmApi crdt && crdt.ProjectData.FwProjectId != fwdataApi.ProjectId) diff --git a/backend/LexBoxApi/Controllers/CrdtController.cs b/backend/LexBoxApi/Controllers/CrdtController.cs index 4c4e98bb0..4051d2b52 100644 --- a/backend/LexBoxApi/Controllers/CrdtController.cs +++ b/backend/LexBoxApi/Controllers/CrdtController.cs @@ -1,11 +1,14 @@ using System.Text.Json.Serialization; -using LexBoxApi.Auth; using SIL.Harmony.Core; +using LexBoxApi.Auth; +using LexBoxApi.Auth.Attributes; using LexBoxApi.Hub; using LexBoxApi.Services; using LexCore.Entities; using LexCore.ServiceInterfaces; +using LexCore.Sync; using LexData; +using Microsoft.AspNetCore.Http.Timeouts; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; @@ -21,7 +24,8 @@ public class CrdtController( IHubContext hubContext, IPermissionService permissionService, LoggedInContext loggedInContext, - ProjectService projectService) : ControllerBase + ProjectService projectService, + FwHeadlessClient fwHeadlessClient) : ControllerBase { private DbSet ServerCommits => dbContext.Set(); @@ -90,4 +94,14 @@ public async Task> GetProjectId(string code) return Ok(projectId); } + + [HttpPost("sync/{projectId}")] + [AdminRequired] + [RequestTimeout(300_000)]//5 minutes + public async Task> ExecuteMerge(Guid projectId) + { + var result = await fwHeadlessClient.CrdtSync(projectId); + if (result is null) return Problem("Failed to sync CRDT"); + return result; + } } diff --git a/backend/LexBoxApi/LexBoxApi.csproj b/backend/LexBoxApi/LexBoxApi.csproj index baffc9fb9..238fc7e57 100644 --- a/backend/LexBoxApi/LexBoxApi.csproj +++ b/backend/LexBoxApi/LexBoxApi.csproj @@ -33,6 +33,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/backend/LexBoxApi/LexBoxKernel.cs b/backend/LexBoxApi/LexBoxKernel.cs index 347449122..a92217cf2 100644 --- a/backend/LexBoxApi/LexBoxKernel.cs +++ b/backend/LexBoxApi/LexBoxKernel.cs @@ -49,6 +49,9 @@ public static void AddLexBoxApi(this IServiceCollection services, .ValidateDataAnnotations() .ValidateOnStart(); services.AddHttpClient(); + services.AddServiceDiscovery(); + services.AddHttpClient(client => client.BaseAddress = new ("http://fwHeadless")) + .AddServiceDiscovery();//service discovery means that we lookup the hostname in Services__fwHeadless__http in config services.AddHttpContextAccessor(); services.AddMemoryCache(); services.AddScoped(); diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs new file mode 100644 index 000000000..48324f218 --- /dev/null +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -0,0 +1,19 @@ +using LexCore.Sync; + +namespace LexBoxApi.Services; + +public class FwHeadlessClient(HttpClient httpClient, ILogger logger) +{ + public async Task CrdtSync(Guid projectId) + { + var response = await httpClient.PostAsync($"/api/crdt-sync?projectId={projectId}", null); + if (response.IsSuccessStatusCode) + return await response.Content.ReadFromJsonAsync(); + logger.LogError("Failed to sync CRDT: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", + response.StatusCode, + response.ReasonPhrase, + projectId, + await response.Content.ReadAsStringAsync()); + return null; + } +} diff --git a/backend/LexBoxApi/appsettings.Development.json b/backend/LexBoxApi/appsettings.Development.json index 35f3ab4dc..1705db6f7 100644 --- a/backend/LexBoxApi/appsettings.Development.json +++ b/backend/LexBoxApi/appsettings.Development.json @@ -77,5 +77,12 @@ "From": "Lexbox ", "EmailRenderHost": "localhost:3000", "BaseUrl": "http://localhost:3000" + }, + "Services": { + "fwHeadless": { + "http": [ + "localhost:5275" + ] + } } } diff --git a/backend/LexBoxApi/appsettings.json b/backend/LexBoxApi/appsettings.json index 6f3e5ef2e..3ae6b1f17 100644 --- a/backend/LexBoxApi/appsettings.json +++ b/backend/LexBoxApi/appsettings.json @@ -70,5 +70,10 @@ }, "Email": { "CreateProjectEmailDestination": "lexbox_support@groups.sil.org" + }, + "Services": { + "fwHeadless": { + "http": ["fw-headless"] + } } } diff --git a/backend/LexCore/Sync/SyncResult.cs b/backend/LexCore/Sync/SyncResult.cs new file mode 100644 index 000000000..70b7d0aa6 --- /dev/null +++ b/backend/LexCore/Sync/SyncResult.cs @@ -0,0 +1,3 @@ +namespace LexCore.Sync; + +public record SyncResult(int CrdtChanges, int FwdataChanges); diff --git a/backend/LexData/SeedingData.cs b/backend/LexData/SeedingData.cs index bb7e2fb44..553699e8b 100644 --- a/backend/LexData/SeedingData.cs +++ b/backend/LexData/SeedingData.cs @@ -156,6 +156,7 @@ private async Task SeedUserData(CancellationToken cancellationToken = default) Description = "Eastern Lawa project", Code = "elawa-dev-flex", Type = ProjectType.FLEx, + FlexProjectMetadata = new(), ProjectOrigin = ProjectMigrationStatus.Migrated, LastCommit = DateTimeOffset.UtcNow, RetentionPolicy = RetentionPolicy.Dev, diff --git a/deployment/base/lexbox-deployment.yaml b/deployment/base/lexbox-deployment.yaml index 15f25a185..a7c4da81a 100644 --- a/deployment/base/lexbox-deployment.yaml +++ b/deployment/base/lexbox-deployment.yaml @@ -205,6 +205,9 @@ spec: value: /tmp/tus-test-upload - name: Tus__ResetUploadPath value: /tmp/tus-reset-upload + - name: Services__fwHeadless__http__0 + value: fw-headless + - name: otel-collector image: otel/opentelemetry-collector-contrib:0.101.0 diff --git a/frontend/src/lib/forms/Button.svelte b/frontend/src/lib/forms/Button.svelte index b2f8c1e9c..595e951ba 100644 --- a/frontend/src/lib/forms/Button.svelte +++ b/frontend/src/lib/forms/Button.svelte @@ -7,13 +7,16 @@ export let type: undefined | 'submit' = undefined; export let size: undefined | 'btn-sm' = undefined; export let disabled = false; + export let customLoader = false; diff --git a/frontend/src/lib/icons/Icon.svelte b/frontend/src/lib/icons/Icon.svelte index 99fa52775..874b49d13 100644 --- a/frontend/src/lib/icons/Icon.svelte +++ b/frontend/src/lib/icons/Icon.svelte @@ -10,6 +10,8 @@ export let size: IconSize = 'text-lg'; export let color: `text-${string}` | undefined = undefined; export let pale = false; + export let spin = false; + export let spinReverse = false; // For pixel perfect text alignment, because the svgs often contain vertical white-space export let y: string | undefined = undefined; @@ -17,5 +19,5 @@ {#if icon} - + {/if} diff --git a/frontend/src/routes/(authenticated)/project/[project_code]/+page.svelte b/frontend/src/routes/(authenticated)/project/[project_code]/+page.svelte index ec4a4bb70..76ea3ef35 100644 --- a/frontend/src/routes/(authenticated)/project/[project_code]/+page.svelte +++ b/frontend/src/routes/(authenticated)/project/[project_code]/+page.svelte @@ -55,6 +55,7 @@ import { onMount } from 'svelte'; import { getSearchParamValues } from '$lib/util/query-params'; import FlexModelVersionText from '$lib/components/Projects/FlexModelVersionText.svelte'; + import CrdtSyncButton from './CrdtSyncButton.svelte'; export let data: PageData; $: user = data.user; @@ -312,6 +313,7 @@ {/if} {#if project.type === ProjectType.FlEx && $isDev} + {:else} diff --git a/frontend/src/routes/(authenticated)/project/[project_code]/CrdtSyncButton.svelte b/frontend/src/routes/(authenticated)/project/[project_code]/CrdtSyncButton.svelte new file mode 100644 index 000000000..324c40126 --- /dev/null +++ b/frontend/src/routes/(authenticated)/project/[project_code]/CrdtSyncButton.svelte @@ -0,0 +1,43 @@ + + +