Merge branch 'develop' into bug/optimize-middleware

.github/workflows/integration-test.yaml

@@ -71,9 +71,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
-      - uses: testspace-com/[email protected]
-        with:
-          domain: ${{ github.repository_owner }}
       - name: Setup self-hosted dependencies
         if: ${{ inputs.runs-on == 'self-hosted' }}
         run: |
@@ -143,9 +140,6 @@ jobs:
         with:
           check_name: Integration Tests ${{ inputs.runs-on }} for Mercurial ${{ inputs.hg-version }}
           files: ./test-results/*.trx
-      - name: Publish results to testspace
-        if: always()
-        run: testspace "[.Net Integration/${{ inputs.runs-on }} HG ${{ inputs.hg-version }}]./test-results/*.trx"
 
   playwright-test:
     if: ${{ inputs.run-playwright }}
@@ -160,9 +154,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
-      - uses: testspace-com/[email protected]
-        with:
-          domain: ${{ github.repository_owner }}
       - name: Setup self-hosted dependencies
         if: ${{ inputs.runs-on == 'self-hosted' }}
         run: |
@@ -212,9 +203,6 @@ jobs:
         env:
           ZIP_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
         run: 7z a ./playwright-traces.7z -mx=0 -mmt=off ./frontend/test-results -p"$ZIP_PASSWORD"
-      - name: Publish results to testspace
-        if: always()
-        run: testspace "[Playwright]./frontend/test-results/results.xml"
       - name: Upload playwright results
         if: ${{ always() && steps.password_protect_test_results.outcome == 'success' }}
         uses: actions/upload-artifact@v4

.vscode/settings.json

@@ -23,6 +23,10 @@
     "sort-imports.on-save": true,
     "editor.detectIndentation": false,
     "eslint.validate": [
+      "javascript",
+      "typescript",
+      "html",
+      "json",
       "svelte"
     ],
     "yaml.schemas": {

README.md

@@ -27,13 +27,15 @@ There are some exceptions:
    * linux: `sudo snap install task --classic` or other options on their website
    * mac: `brew install go-task/tap/go-task`
    * via npm: `npm install -g @go-task/cli`
- * install [Tilt](https://docs.tilt.dev/) and add it to your path
-   * on Linux, a good practice is to create `$HOME/.local/bin` and put binaries there; most distributions automatically add `$HOME/.local/bin` to your path if it exists
-     * don't forget to run `chmod +x $HOME/.local/bin/tilt`
-   * on Windows, we suggest creating a `bin` folder in your home folder. Put the Tilt binary there, then do the following:
+ * install [Tilt](https://docs.tilt.dev/) and add it to your path (don't forget to read the script before running it)
+   * on Linux, the script will install tilt into `$HOME/.local/bin`, creating it if it doesn't exist
+     * most Linux distributions put `$HOME/.local/bin` in your PATH automatically. If `tilt version` doesn't work, log out and log back in and it should work; otherwise you'll need to add it to your PATH in `$HOME/.bashrc` or equivalent.
+   * on Windows, the Tilt installer will create a `bin` folder in your home folder and put the Tilt binary there
+     * you will then need to do the following to make sure the Tilt binary is in your PATH:
      * go to your System properties, click the **Advanced** tab, and click **Environment Variables...**
      * Click the Path variable (in either User or System, User is recommended) and click the **Edit...** button
-     * Add `C:\Users\YOUR_USER_NAME\bin` to the list and click **OK**
+     * Add `C:\Users\YOUR_USER_NAME\bin` to the list (if it's not already there) and click **OK**
+ * run `tilt version` to check that Tilt is installed correctly
  * clone the repo
  * run `git push` to make sure your GitHub credentials are set up
    * on Windows, allow the Git Credential Manager to log in to GitHub via your browser

backend/FwHeadless/CrdtSyncService.cs

@@ -0,0 +1,27 @@
+using LcmCrdt;
+using LcmCrdt.RemoteSync;
+using SIL.Harmony;
+
+namespace FwHeadless;
+
+public class CrdtSyncService(
+    CrdtHttpSyncService httpSyncService,
+    IHttpClientFactory httpClientFactory,
+    CurrentProjectService currentProjectService,
+    DataModel dataModel,
+    ILogger<CrdtSyncService> logger)
+{
+    public async Task Sync()
+    {
+        var lexboxRemoteServer = await httpSyncService.CreateProjectSyncable(
+            currentProjectService.ProjectData,
+            httpClientFactory.CreateClient(FwHeadlessKernel.LexboxHttpClientName)
+        );
+        var syncResults = await dataModel.SyncWith(lexboxRemoteServer);
+        if (!syncResults.IsSynced) throw new InvalidOperationException("Sync failed");
+        logger.LogInformation(
+            "Synced with Lexbox, Downloaded changes: {MissingFromLocal}, Uploaded changes: {MissingFromRemote}",
+            syncResults.MissingFromLocal.Length,
+            syncResults.MissingFromRemote.Length);
+    }
+}

backend/FwHeadless/FwHeadless.csproj

@@ -24,6 +24,8 @@
   </ItemGroup>
 
   <ItemGroup>
+      <!-- json files get imported by default because this is a web project, so exclude those here so they aren't imported again below -->
+      <Content Remove="Mercurial\contrib\asv.conf.json"/>
       <Content Include="Mercurial\**" CopyToOutputDirectory="Always" Watch="false" />
       <Content Include="MercurialExtensions\**" CopyToOutputDirectory="Always" Watch="false" />
   </ItemGroup>

backend/FwHeadless/FwHeadlessKernel.cs

@@ -2,11 +2,13 @@
 using FwHeadless.Services;
 using FwLiteProjectSync;
 using LcmCrdt;
+using Microsoft.Extensions.Options;
 
 namespace FwHeadless;
 
 public static class FwHeadlessKernel
 {
+    public const string LexboxHttpClientName = "LexboxHttpClient";
     public static void AddFwHeadless(this IServiceCollection services)
     {
         services
@@ -23,5 +25,12 @@ public static void AddFwHeadless(this IServiceCollection services)
             .AddLcmCrdtClient()
             .AddFwDataBridge()
             .AddFwLiteProjectSync();
+        services.AddScoped<CrdtSyncService>();
+        services.AddTransient<HttpClientAuthHandler>();
+        services.AddHttpClient(LexboxHttpClientName,
+            (provider, client) =>
+            {
+                client.BaseAddress = new Uri(provider.GetRequiredService<IOptions<FwHeadlessConfig>>().Value.LexboxUrl);
+            }).AddHttpMessageHandler<HttpClientAuthHandler>();
     }
-};
+}

backend/FwHeadless/HttpClientAuthHandler.cs

@@ -0,0 +1,73 @@
+using System.Net;
+using LexCore;
+using LexCore.Auth;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Options;
+
+namespace FwHeadless;
+
+public class HttpClientAuthHandler(IOptions<FwHeadlessConfig> config, IMemoryCache cache, ILogger<HttpClientAuthHandler> logger) : DelegatingHandler
+{
+    protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken cancellationToken)
+    {
+        throw new NotSupportedException("use async apis");
+    }
+
+    protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+    {
+        var lexboxUrl = new Uri(config.Value.LexboxUrl);
+        if (request.RequestUri?.Authority != lexboxUrl.Authority)
+        {
+            return await base.SendAsync(request, cancellationToken);
+        }
+        try
+        {
+            await SetAuthHeader(request, cancellationToken, lexboxUrl);
+        }
+        catch (Exception e)
+        {
+            throw new InvalidOperationException("Unable to set auth header", e);
+        }
+        return await base.SendAsync(request, cancellationToken);
+    }
+
+    private async Task SetAuthHeader(HttpRequestMessage request, CancellationToken cancellationToken, Uri lexboxUrl)
+    {
+        var cookieContainer = new CookieContainer();
+        cookieContainer.Add(new Cookie(LexAuthConstants.AuthCookieName, await GetToken(cancellationToken), null, lexboxUrl.Authority));
+        request.Headers.Add("Cookie", cookieContainer.GetCookieHeader(lexboxUrl));
+    }
+
+    private async ValueTask<string> GetToken(CancellationToken cancellationToken)
+    {
+        try
+        {
+            return await cache.GetOrCreateAsync("LexboxAuthToken",
+                async entry =>
+                {
+                    if (InnerHandler is null) throw new InvalidOperationException("InnerHandler is null");
+                    logger.LogInformation("Getting auth token");
+                    var client = new HttpClient(InnerHandler);
+                    client.BaseAddress = new Uri(config.Value.LexboxUrl);
+                    var response = await client.PostAsJsonAsync("/api/login",
+                        new LoginRequest(config.Value.LexboxPassword, config.Value.LexboxUsername),
+                        cancellationToken);
+                    response.EnsureSuccessStatusCode();
+                    var cookies = response.Headers.GetValues("Set-Cookie");
+                    var cookieContainer = new CookieContainer();
+                    cookieContainer.SetCookies(response.RequestMessage!.RequestUri!, cookies.Single());
+                    var authCookie = cookieContainer.GetAllCookies()
+                        .FirstOrDefault(c => c.Name == LexAuthConstants.AuthCookieName);
+                    if (authCookie is null) throw new InvalidOperationException("Auth cookie not found");
+                    entry.SetValue(authCookie.Value);
+                    entry.AbsoluteExpiration = authCookie.Expires;
+                    logger.LogInformation("Got auth token: {AuthToken}", authCookie.Value);
+                    return authCookie.Value;
+                }) ?? throw new NullReferenceException("unable to get the login token");
+        }
+        catch (Exception e)
+        {
+            throw new InvalidOperationException("Unable to get auth token", e);
+        }
+    }
+}

backend/FwHeadless/Program.cs

@@ -1,7 +1,9 @@
 using FwHeadless;
 using FwDataMiniLcmBridge;
+using FwDataMiniLcmBridge.Api;
 using FwLiteProjectSync;
 using LcmCrdt;
+using LcmCrdt.RemoteSync;
 using LexData;
 using Microsoft.AspNetCore.Http.HttpResults;
 using Microsoft.Extensions.Options;
@@ -45,11 +47,11 @@
 
 app.MapHealthChecks("/api/healthz");
 
-app.MapPost("/sync", ExecuteMergeRequest);
+app.MapPost("/api/crdt-sync", ExecuteMergeRequest);
 
 app.Run();
 
-static async Task<Results<Ok<CrdtFwdataProjectSyncService.SyncResult>, NotFound>> ExecuteMergeRequest(
+static async Task<Results<Ok<CrdtFwdataProjectSyncService.SyncResult>, NotFound, ProblemHttpResult>> ExecuteMergeRequest(
     ILogger<Program> logger,
     IServiceProvider services,
     SendReceiveService srService,
@@ -58,6 +60,8 @@
     ProjectsService projectsService,
     ProjectLookupService projectLookupService,
     CrdtFwdataProjectSyncService syncService,
+    CrdtHttpSyncService crdtHttpSyncService,
+    IHttpClientFactory httpClientFactory,
     Guid projectId,
     bool dryRun = false)
 {
@@ -75,6 +79,11 @@
         return TypedResults.NotFound();
     }
     logger.LogInformation("Project code is {projectCode}", projectCode);
+    //if we can't sync with lexbox fail fast
+    if (!await crdtHttpSyncService.TestAuth(httpClientFactory.CreateClient(FwHeadlessKernel.LexboxHttpClientName)))
+    {
+        return TypedResults.Problem("Unable to authenticate with Lexbox");
+    }
 
     var projectFolder = Path.Join(config.Value.ProjectStorageRoot, $"{projectCode}-{projectId}");
     if (!Directory.Exists(projectFolder)) Directory.CreateDirectory(projectFolder);
@@ -85,25 +94,71 @@
     logger.LogDebug("crdtFile: {crdtFile}", crdtFile);
     logger.LogDebug("fwDataFile: {fwDataFile}", fwDataProject.FilePath);
 
+    var fwdataApi = await SetupFwData(fwDataProject, srService, projectCode, logger, fwDataFactory);
+    using var deferCloseFwData = fwDataFactory.DeferClose(fwDataProject);
+    var crdtProject = await SetupCrdtProject(crdtFile, projectLookupService, projectId, projectsService, projectFolder, fwdataApi.ProjectId, config.Value.LexboxUrl);
+
+    var miniLcmApi = await services.OpenCrdtProject(crdtProject);
+    var crdtSyncService = services.GetRequiredService<CrdtSyncService>();
+    await crdtSyncService.Sync();
+
+
+    var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun);
+    logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges);
+
+    await crdtSyncService.Sync();
+    var srResult2 = await srService.SendReceive(fwDataProject, projectCode);
+    logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output);
+    return TypedResults.Ok(result);
+}
+
+static async Task<FwDataMiniLcmApi> SetupFwData(FwDataProject fwDataProject,
+    SendReceiveService srService,
+    string projectCode,
+    ILogger<Program> logger,
+    FwDataFactory fwDataFactory)
+{
     if (File.Exists(fwDataProject.FilePath))
     {
-        var srResult = srService.SendReceive(fwDataProject, projectCode);
+        var srResult = await srService.SendReceive(fwDataProject, projectCode);
         logger.LogInformation("Send/Receive result: {srResult}", srResult.Output);
     }
     else
     {
-        var srResult = srService.Clone(fwDataProject, projectCode);
+        var srResult = await srService.Clone(fwDataProject, projectCode);
         logger.LogInformation("Send/Receive result: {srResult}", srResult.Output);
     }
+
     var fwdataApi = fwDataFactory.GetFwDataMiniLcmApi(fwDataProject, true);
-    var crdtProject = File.Exists(crdtFile) ?
-        new CrdtProject("crdt", crdtFile) :
-        await projectsService.CreateProject(new("crdt", SeedNewProjectData: false, Path: projectFolder, FwProjectId: fwdataApi.ProjectId));
-    var miniLcmApi = await services.OpenCrdtProject(crdtProject);
-    var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun);
-    logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges);
-    var srResult2 = srService.SendReceive(fwDataProject, projectCode);
-    logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output);
-    return TypedResults.Ok(result);
+    return fwdataApi;
+}
+
+static async Task<CrdtProject> SetupCrdtProject(string crdtFile,
+    ProjectLookupService projectLookupService,
+    Guid projectId,
+    ProjectsService projectsService,
+    string projectFolder,
+    Guid fwProjectId,
+    string lexboxUrl)
+{
+    if (File.Exists(crdtFile))
+    {
+        return new CrdtProject("crdt", crdtFile);
+    }
+    else
+    {
+        if (await projectLookupService.IsCrdtProject(projectId))
+        {
+            //todo determine what to do in this case, maybe we just download the project?
+            throw new InvalidOperationException("Project already exists, not sure why it's not on the server");
+        }
+        return await projectsService.CreateProject(new("crdt",
+            SeedNewProjectData: false,
+            Id: projectId,
+            Path: projectFolder,
+            FwProjectId: fwProjectId,
+            Domain: new Uri(lexboxUrl)));
+    }
+
 }
 

backend/FwHeadless/ProjectLookupService.cs

@@ -1,5 +1,6 @@
 using LexData;
 using Microsoft.EntityFrameworkCore;
+using SIL.Harmony.Core;
 
 namespace FwHeadless;
 
@@ -13,4 +14,9 @@ public class ProjectLookupService(LexBoxDbContext dbContext)
             .FirstOrDefaultAsync();
         return projectCode;
     }
+
+    public async Task<bool> IsCrdtProject(Guid projectId)
+    {
+        return await dbContext.Set<ServerCommit>().AnyAsync(c => c.ProjectId == projectId);
+    }
 }