make api to call fw headless sync (#1252)

* add crdt sync passthrough endpoint to call crdt sync on fw headless, setup service discovery to simplify configuration

* fix issue with cookie port number not working when in the host parameter

* log errors from fw headless, and return a problem from the lexbox api

* Add button for triggering CRDT sync

* increase request timeout on the sync endpoint, change the path to not be redundant

---------

Co-authored-by: Tim Haasdyk <[email protected]>

Tiltfile

@@ -28,7 +28,7 @@ docker_build(
     context='backend',
     dockerfile='./backend/FwHeadless/dev.Dockerfile',
     only=['.'],
-    ignore=['LexBoxApi'],
+    ignore=['LexBoxApi', '**/Mercurial', '**/MercurialExtensions'],
     live_update=[
         sync('backend', '/src/backend')
     ]

backend/FwHeadless/HttpClientAuthHandler.cs

@@ -34,7 +34,10 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
     private async Task SetAuthHeader(HttpRequestMessage request, CancellationToken cancellationToken, Uri lexboxUrl)
     {
         var cookieContainer = new CookieContainer();
-        cookieContainer.Add(new Cookie(LexAuthConstants.AuthCookieName, await GetToken(cancellationToken), null, lexboxUrl.Authority));
+        cookieContainer.Add(new Cookie(LexAuthConstants.AuthCookieName, await GetToken(cancellationToken), null, lexboxUrl.Host)
+        {
+            Port = $"\"{lexboxUrl.Port}\""
+        });
         request.Headers.Add("Cookie", cookieContainer.GetCookieHeader(lexboxUrl));
     }
 

backend/FwHeadless/Program.cs

@@ -4,6 +4,7 @@
 using FwLiteProjectSync;
 using LcmCrdt;
 using LcmCrdt.RemoteSync;
+using LexCore.Sync;
 using LexData;
 using Microsoft.AspNetCore.Http.HttpResults;
 using Microsoft.Extensions.Options;
@@ -51,7 +52,7 @@
 
 app.Run();
 
-static async Task<Results<Ok<CrdtFwdataProjectSyncService.SyncResult>, NotFound, ProblemHttpResult>> ExecuteMergeRequest(
+static async Task<Results<Ok<SyncResult>, NotFound, ProblemHttpResult>> ExecuteMergeRequest(
     ILogger<Program> logger,
     IServiceProvider services,
     SendReceiveService srService,
@@ -69,7 +70,7 @@
     if (dryRun)
     {
         logger.LogInformation("Dry run, not actually syncing");
-        return TypedResults.Ok(new CrdtFwdataProjectSyncService.SyncResult(0, 0));
+        return TypedResults.Ok(new SyncResult(0, 0));
     }
 
     var projectCode = await projectLookupService.GetProjectCode(projectId);

backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs

@@ -1,6 +1,7 @@
 using System.Text.Json;
 using FwDataMiniLcmBridge.Api;
 using LcmCrdt;
+using LexCore.Sync;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using MiniLcm;
@@ -13,8 +14,6 @@ namespace FwLiteProjectSync;
 
 public class CrdtFwdataProjectSyncService(IOptions<LcmCrdtConfig> lcmCrdtConfig, MiniLcmImport miniLcmImport, ILogger<CrdtFwdataProjectSyncService> logger)
 {
-    public record SyncResult(int CrdtChanges, int FwdataChanges);
-
     public async Task<SyncResult> Sync(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataApi, bool dryRun = false)
     {
         if (crdtApi is CrdtMiniLcmApi crdt && crdt.ProjectData.FwProjectId != fwdataApi.ProjectId)

backend/LexBoxApi/Controllers/CrdtController.cs

@@ -1,11 +1,14 @@
 using System.Text.Json.Serialization;
-using LexBoxApi.Auth;
 using SIL.Harmony.Core;
+using LexBoxApi.Auth;
+using LexBoxApi.Auth.Attributes;
 using LexBoxApi.Hub;
 using LexBoxApi.Services;
 using LexCore.Entities;
 using LexCore.ServiceInterfaces;
+using LexCore.Sync;
 using LexData;
+using Microsoft.AspNetCore.Http.Timeouts;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.SignalR;
 using Microsoft.EntityFrameworkCore;
@@ -21,7 +24,8 @@ public class CrdtController(
     IHubContext<CrdtProjectChangeHub, IProjectChangeListener> hubContext,
     IPermissionService permissionService,
     LoggedInContext loggedInContext,
-    ProjectService projectService) : ControllerBase
+    ProjectService projectService,
+    FwHeadlessClient fwHeadlessClient) : ControllerBase
 {
     private DbSet<ServerCommit> ServerCommits => dbContext.Set<ServerCommit>();
 
@@ -90,4 +94,14 @@ public async Task<ActionResult<Guid>> GetProjectId(string code)
 
         return Ok(projectId);
     }
+
+    [HttpPost("sync/{projectId}")]
+    [AdminRequired]
+    [RequestTimeout(300_000)]//5 minutes
+    public async Task<ActionResult<SyncResult?>> ExecuteMerge(Guid projectId)
+    {
+        var result = await fwHeadlessClient.CrdtSync(projectId);
+        if (result is null) return Problem("Failed to sync CRDT");
+        return result;
+    }
 }

backend/LexBoxApi/LexBoxApi.csproj

@@ -33,6 +33,7 @@
           <PrivateAssets>all</PrivateAssets>
           <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
+        <PackageReference Include="Microsoft.Extensions.ServiceDiscovery" Version="9.0.0" />
         <PackageReference Include="Microsoft.IdentityModel.Protocols.OpenIdConnect" Version="8.0.2" />
         <PackageReference Include="Nito.AsyncEx.Coordination" Version="5.1.2" />
         <PackageReference Include="Npgsql.OpenTelemetry" Version="8.0.3" />

backend/LexBoxApi/LexBoxKernel.cs

@@ -49,6 +49,9 @@ public static void AddLexBoxApi(this IServiceCollection services,
             .ValidateDataAnnotations()
             .ValidateOnStart();
         services.AddHttpClient();
+        services.AddServiceDiscovery();
+        services.AddHttpClient<FwHeadlessClient>(client => client.BaseAddress = new ("http://fwHeadless"))
+            .AddServiceDiscovery();//service discovery means that we lookup the hostname in Services__fwHeadless__http in config
         services.AddHttpContextAccessor();
         services.AddMemoryCache();
         services.AddScoped<LoggedInContext>();

backend/LexBoxApi/Services/FwHeadlessClient.cs

@@ -0,0 +1,19 @@
+using LexCore.Sync;
+
+namespace LexBoxApi.Services;
+
+public class FwHeadlessClient(HttpClient httpClient, ILogger<FwHeadlessClient> logger)
+{
+    public async Task<SyncResult?> CrdtSync(Guid projectId)
+    {
+        var response = await httpClient.PostAsync($"/api/crdt-sync?projectId={projectId}", null);
+        if (response.IsSuccessStatusCode)
+            return await response.Content.ReadFromJsonAsync<SyncResult>();
+        logger.LogError("Failed to sync CRDT: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}",
+            response.StatusCode,
+            response.ReasonPhrase,
+            projectId,
+            await response.Content.ReadAsStringAsync());
+        return null;
+    }
+}

backend/LexBoxApi/appsettings.Development.json

@@ -77,5 +77,12 @@
     "From": "Lexbox <[email protected]>",
     "EmailRenderHost": "localhost:3000",
     "BaseUrl": "http://localhost:3000"
+  },
+  "Services": {
+    "fwHeadless": {
+      "http": [
+        "localhost:5275"
+      ]
+    }
   }
 }

backend/LexBoxApi/appsettings.json

@@ -70,5 +70,10 @@
   },
   "Email": {
     "CreateProjectEmailDestination": "[email protected]"
+  },
+  "Services": {
+    "fwHeadless": {
+      "http": ["fw-headless"]
+    }
   }
 }

backend/LexCore/Sync/SyncResult.cs

@@ -0,0 +1,3 @@
+namespace LexCore.Sync;
+
+public record SyncResult(int CrdtChanges, int FwdataChanges);

backend/LexData/SeedingData.cs

@@ -156,6 +156,7 @@ private async Task SeedUserData(CancellationToken cancellationToken = default)
             Description = "Eastern Lawa project",
             Code = "elawa-dev-flex",
             Type = ProjectType.FLEx,
+            FlexProjectMetadata = new(),
             ProjectOrigin = ProjectMigrationStatus.Migrated,
             LastCommit = DateTimeOffset.UtcNow,
             RetentionPolicy = RetentionPolicy.Dev,

deployment/base/lexbox-deployment.yaml

@@ -205,6 +205,9 @@ spec:
             value: /tmp/tus-test-upload
           - name: Tus__ResetUploadPath
             value: /tmp/tus-reset-upload
+          - name: Services__fwHeadless__http__0
+            value: fw-headless
+
 
       - name: otel-collector
         image: otel/opentelemetry-collector-contrib:0.101.0

frontend/src/lib/forms/Button.svelte

@@ -7,13 +7,16 @@
   export let type: undefined | 'submit' = undefined;
   export let size: undefined | 'btn-sm' = undefined;
   export let disabled = false;
+  export let customLoader = false;
 </script>
 
 <!-- https://daisyui.com/components/button -->
 <button on:click {...$$restProps} class="btn whitespace-nowrap {variant ?? ''} {$$restProps.class ?? ''} {size ?? ''}" {type}
   class:btn-outline={outline}
   disabled={disabled && !loading}
-  class:pointer-events-none={loading}>
-  <Loader {loading} />
+  class:pointer-events-none={loading || $$restProps.class?.includes('pointer-events-none')}>
+  {#if !customLoader}
+    <Loader {loading} />
+  {/if}
   <slot />
 </button>

frontend/src/lib/icons/Icon.svelte

@@ -10,12 +10,14 @@
   export let size: IconSize = 'text-lg';
   export let color: `text-${string}` | undefined = undefined;
   export let pale = false;
+  export let spin = false;
+  export let spinReverse = false;
   // For pixel perfect text alignment, because the svgs often contain vertical white-space
   export let y: string | undefined = undefined;
 
   $: transform = y ? `translateY(${y})` : '';
 </script>
 
 {#if icon}
-  <span class="{icon} {size} {color ?? ''} shrink-0" class:pale style:transform />
+  <span class="{icon} {size} {color ?? ''} shrink-0 {spinReverse ? 'transform rotate-180' : ''}" class:pale style:transform class:animate-spin={spin} />
 {/if}

frontend/src/routes/(authenticated)/project/[project_code]/+page.svelte

@@ -55,6 +55,7 @@
   import { onMount } from 'svelte';
   import { getSearchParamValues } from '$lib/util/query-params';
   import FlexModelVersionText from '$lib/components/Projects/FlexModelVersionText.svelte';
+  import CrdtSyncButton from './CrdtSyncButton.svelte';
 
   export let data: PageData;
   $: user = data.user;
@@ -312,6 +313,7 @@
         </a>
       {/if}
       {#if project.type === ProjectType.FlEx && $isDev}
+        <CrdtSyncButton projectId={project.id} />
         <OpenInFlexModal bind:this={openInFlexModal} {project}/>
         <OpenInFlexButton projectId={project.id} on:click={openInFlexModal.open}/>
       {:else}

frontend/src/routes/(authenticated)/project/[project_code]/CrdtSyncButton.svelte

@@ -0,0 +1,43 @@
+<script lang="ts">
+  import {Button} from '$lib/forms';
+  import {Icon} from '$lib/icons';
+  import {useNotifications} from '$lib/notify';
+
+  export let projectId: string;
+
+  const {notifySuccess, notifyWarning} = useNotifications();
+
+  let syncing = false;
+
+  async function triggerSync(): Promise<void> {
+    syncing = true;
+    try {
+      const response = await fetch(`/api/crdt/sync/${projectId}`, {
+        method: 'POST',
+      });
+
+      if (response.ok) {
+        const { crdtChanges, fwdataChanges } = await response.json();
+        notifySuccess(`Synced successfully (${fwdataChanges} FwData changes. ${crdtChanges} CRDT changes)`);
+      } else {
+        const error = `Failed to sync: ${response.statusText} (${response.status})`;
+        notifyWarning(error);
+        console.error(error, await response.text());
+      }
+    } finally {
+      syncing = false;
+    }
+  }
+</script>
+
+<Button
+  variant="btn-primary"
+  class="gap-1"
+  on:click={triggerSync}
+  loading={syncing}
+  customLoader
+>
+  FwData
+  <Icon icon="i-mdi-sync" spin={syncing} spinReverse />
+  CRDT
+</Button>