Merge pull request #5 from digirati-co-uk/storage-pres-refactor

Storage pres refactor
digirati-co-uk · Sep 9, 2024 · 3b36225 · 3b36225
2 parents 07687ab + 8623516
commit 3b36225
Show file tree

Hide file tree

Showing 111 changed files with 24,848 additions and 184 deletions.
diff --git a/LeedsExperiment/Dashboard/Controllers/ImportExportController.cs b/LeedsExperiment/Dashboard/Controllers/ImportExportController.cs
@@ -68,21 +68,31 @@ public async Task<IActionResult> ExportExecuteAsync(
         public async Task<IActionResult> ImportStartAsync(
             [FromRoute] string path,
             [FromQuery] string? source = null,
-            [FromQuery] string? name = null) // name if creating a new one; can't rename atm
-        {            
+            [FromQuery] string? name = null,
+            [FromQuery] string? validateSource = null) // name if creating a new one; can't rename atm
+        {
             var resourceInfo = await storage.GetResourceInfo(path);
             var model = new ImportModel { Path = path, ResourceInfo = resourceInfo };
-            if(resourceInfo.Type == nameof(ArchivalGroup))
+            if (resourceInfo.Type == nameof(ArchivalGroup))
             {
                 // This is an update to an existing archival group
                 var existingAg = await storage.GetArchivalGroup(path, null);
                 model.ArchivalGroup = existingAg;
-            } 
+            }
             else if (resourceInfo.StatusCode != 404)
             {
                 ViewBag.Problem = "Invalid Status code: " + resourceInfo.StatusCode;
                 return View("ImportStart", model);
             }
+            if (validateSource == "on")
+            {
+                model.ImportSource = await storage.GetImportSource(source);
+                if(model.ImportSource != null && model.ImportSource.Files.Any(x => string.IsNullOrWhiteSpace(x.Digest)))
+                {
+                    ViewBag.Problem = "At least one source file lacks a Digest";
+                }
+                return View("ImportStart", model);
+            }
             model.Name = name;
             if (!string.IsNullOrWhiteSpace(source))
             {
@@ -102,7 +112,23 @@ public async Task<IActionResult> ImportStartAsync(
             // form prevents conflicting path;
             // doesn't appear under AG
         }
-
+
+
+
+        [HttpPost]
+        [ActionName("CopySource")]
+        [Route("copysource/{*path}")]
+        public async Task<IActionResult> CopyToNewSourceWithChecksums(
+            [FromRoute] string path,
+            [FromForm] string? source = null,
+            [FromForm] string? name = null,
+            [FromForm] string? copySource = null)
+        {
+            var newImportSource = await storage.CopyToNewSourceWithChecksums(source);
+            return await ImportStartAsync(path, newImportSource?.Source.ToString(), name, "on");
+        }
+
+
 
         [HttpPost]
         [ActionName("ImportExecute")]

diff --git a/LeedsExperiment/Dashboard/Dashboard.csproj b/LeedsExperiment/Dashboard/Dashboard.csproj
@@ -9,17 +9,18 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="iiif-net" Version="0.2.5" />
-    <PackageReference Include="Microsoft.AspNetCore.Components.QuickGrid.EntityFrameworkAdapter" Version="8.0.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.19.5" />
-    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="8.0.0" />
+    <PackageReference Include="iiif-net" Version="0.2.8" />
+    <PackageReference Include="Microsoft.AspNetCore.Components.QuickGrid.EntityFrameworkAdapter" Version="8.0.8" />
+    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="8.0.4" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Dlcs\Dlcs.csproj" />
     <ProjectReference Include="..\Fedora\Fedora.csproj" />
     <ProjectReference Include="..\PreservationApiClient\StorageApiClient.csproj" />
     <ProjectReference Include="..\Preservation\Storage.csproj" />
+    <ProjectReference Include="..\Utils\Utils.csproj" />
   </ItemGroup>
 
 </Project>
diff --git a/LeedsExperiment/Dashboard/Models/ImportModel.cs b/LeedsExperiment/Dashboard/Models/ImportModel.cs
@@ -1,5 +1,6 @@
 using Dashboard.Helpers;
 using Fedora.Abstractions;
+using Fedora.Abstractions.Transfer;
 using Storage;
 
 namespace Dashboard.Models;
@@ -17,4 +18,6 @@ public class ImportModel
     ///  Archival Group.
     /// </summary>
     public string? Name { get; internal set; }
+
+    public ImportSource? ImportSource { get; set; }
 }
diff --git a/LeedsExperiment/Dashboard/Views/Browse/Binary.cshtml b/LeedsExperiment/Dashboard/Views/Browse/Binary.cshtml
@@ -1,5 +1,4 @@
-@using Dashboard.Borrowed
-@using Dashboard.Helpers
+@using Dashboard.Helpers
 @using Utils
 @model Fedora.Abstractions.Binary
 

diff --git a/LeedsExperiment/Dashboard/Views/ImportExport/ExportResult.cshtml b/LeedsExperiment/Dashboard/Views/ImportExport/ExportResult.cshtml
@@ -1,5 +1,4 @@
-@using Dashboard.Borrowed
-@using Dashboard.Helpers
+@using Dashboard.Helpers
 @using Utils
 @model Storage.ExportResult
 

diff --git a/LeedsExperiment/Dashboard/Views/ImportExport/ImportJob.cshtml b/LeedsExperiment/Dashboard/Views/ImportExport/ImportJob.cshtml
@@ -1,5 +1,4 @@
-@using Dashboard.Borrowed
-@using Dashboard.Helpers
+@using Dashboard.Helpers
 @using System.Text.Json
 @using Utils
 @model Dashboard.Models.ImportModel

diff --git a/LeedsExperiment/Dashboard/Views/ImportExport/ImportResult.cshtml b/LeedsExperiment/Dashboard/Views/ImportExport/ImportResult.cshtml
@@ -1,5 +1,4 @@
-@using Dashboard.Borrowed
-@using Dashboard.Helpers
+@using Dashboard.Helpers
 @using System.Text.Json
 @using Utils
 @model Dashboard.Models.ImportModel

diff --git a/LeedsExperiment/Dashboard/Views/ImportExport/ImportStart.cshtml b/LeedsExperiment/Dashboard/Views/ImportExport/ImportStart.cshtml
@@ -7,6 +7,15 @@
     <h1 class="display-4">Import 📦 - @Model.DisplayName</h1>
     <p>Path: @Model.Path</p>
 
+    @if(ViewBag.Problem != null)
+    {
+        <div class="row mt-2">
+            <div class="alert alert-danger" role="alert">
+                <p>@ViewBag.Problem</p>
+            </div>
+        </div>        
+    }
+
     @if(Model.ArchivalGroup == null)
     {
         <p>Specify a source location to create a new Archival Group at @Model.Path</p>
@@ -22,9 +31,15 @@
     <form method="get" action="/import/@(Model.Path)">
         <div class="mb-3">
             <label for="sourceKey" class="form-label">Source</label>
-            <input type="text" class="form-control" id="sourceKey" name="source" aria-describedby="sourceHelp" value="s3://uol-expts-staging-01/<...key...>" />
+            <input type="text" class="form-control" id="sourceKey" name="source" aria-describedby="sourceHelp"
+                value="@(Model.ImportSource?.Source.ToString() ?? "s3://uol-expts-staging-01/<...key...>")" />
             <div id="sourceHelp" class="form-text">Later you can choose other locations outside this bucket!</div>
         </div>
+        <div class="mb-3">
+            <input type="checkbox" class="form-check-input" id="validateSource" name="validateSource" aria-describedby="validateSourceHelp" />
+            <label for="validateSource" class="form-check-label">Validate source first</label>
+            <div id="validateSourceHelp" class="form-text">Verify that the files in source (if S3) all have SHA-256 checksums, and offer help if not.</div>
+        </div>
         @if(Model.ArchivalGroup == null)
         {
             <div class="mb-3">
@@ -36,6 +51,60 @@
         <button type="submit" class="btn btn-primary">Submit</button>
     </form>
 
+    @if(Model.ImportSource != null)
+    {
+        if (Model.ImportSource.Files.Count > 0)
+        {
+            <table class="table">
+                <thead>
+                    <tr>
+                        <th>Hash</th>
+                        <th>Uri</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    @foreach (var bf in Model.ImportSource.Files)
+                    {
+                        <tr>
+                            <td>
+                                @if (bf.Digest.HasText())
+                                {
+                                    <span title="@bf.Digest" style="color:darkgreen">@bf.Digest.Substring(0, 8)</span>
+                                }
+                                else
+                                {
+                                    <span style="color:red">⚠ NO HASH</span>
+                                }
+                            </td>
+                            <td>
+                                @bf.ExternalLocation
+                            </td>
+                        </tr>
+                    }
+                </tbody>
+            </table>
+
+            @if(Model.ImportSource.Files.Any(x => string.IsNullOrWhiteSpace(x.Digest)))
+            {
+                <form method="post" action="/copysource/@(Model.Path)">
+                    <input type="hidden" name="source" value="@Model.ImportSource.Source" />
+                    <input type="hidden" name="name" value="@Model.Name" />
+                    <input type="hidden" name="copySource" value="copy" />
+
+                    <button type="submit" class="btn btn-primary" aria-describedby="copyhelp">Create new source</button>
+                    <div id="copyhelp" class="form-text">Create a new source location alongside the old, with hash (digest) applied.</div>
+                </form>
+            }
+        } 
+        else
+        {
+            <div class="row mt-2">
+                <div class="alert alert-danger" role="alert">
+                    <p>There are no files at the source!</p>
+                </div>   
+            </div>
+        }
+    }
 
 
 </div>
diff --git a/LeedsExperiment/Dlcs/Dlcs.csproj b/LeedsExperiment/Dlcs/Dlcs.csproj
@@ -7,8 +7,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Options" Version="8.0.2" />
     <PackageReference Include="System.Text.Json" Version="8.0.4" />
   </ItemGroup>
 

diff --git a/LeedsExperiment/Fedora/Abstractions/Transfer/ResourceWithParentUri.cs b/LeedsExperiment/Fedora/Abstractions/Transfer/ResourceWithParentUri.cs
@@ -45,4 +45,13 @@ public string GetDisplayName()
         }
         return $"{Name} (/{Slug})";
     }
+
+    public override string ToString()
+    {
+        if(Path != null)
+        {
+            return $"{Path} [{Name}]";
+        }
+        return base.ToString(); 
+    }
 }
diff --git a/LeedsExperiment/FedoraTests/FedoraTests.csproj b/LeedsExperiment/FedoraTests/FedoraTests.csproj
@@ -10,13 +10,13 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
-    <PackageReference Include="xunit" Version="2.6.5" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.6">
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+    <PackageReference Include="xunit" Version="2.9.0" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="coverlet.collector" Version="6.0.0">
+    <PackageReference Include="coverlet.collector" Version="6.0.2">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>

diff --git a/LeedsExperiment/LeedsExperiment.sln b/LeedsExperiment/LeedsExperiment.sln
@@ -26,6 +26,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Dlcs", "Dlcs\Dlcs.csproj",
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Preservation.API", "Preservation.API\Preservation.API.csproj", "{65F1F519-256D-4C11-B3F8-53F5779113EE}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MetsParser", "MetsParser\MetsParser.csproj", "{54AB5F7D-A565-4565-938D-4630B42F10C9}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{5CC04298-37E7-4932-BEC9-3EB1A5BDCA69}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MetsParserTests", "tests\MetsParserTests\MetsParserTests.csproj", "{76121C1B-6B49-4F33-95EF-ADD22D2F27E8}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Utils", "Utils\Utils.csproj", "{992C93C1-46D1-4DCB-B95D-C303BB4F9CAA}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -68,10 +76,25 @@ Global
 		{65F1F519-256D-4C11-B3F8-53F5779113EE}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{65F1F519-256D-4C11-B3F8-53F5779113EE}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{65F1F519-256D-4C11-B3F8-53F5779113EE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{54AB5F7D-A565-4565-938D-4630B42F10C9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{54AB5F7D-A565-4565-938D-4630B42F10C9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{54AB5F7D-A565-4565-938D-4630B42F10C9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{54AB5F7D-A565-4565-938D-4630B42F10C9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{76121C1B-6B49-4F33-95EF-ADD22D2F27E8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{76121C1B-6B49-4F33-95EF-ADD22D2F27E8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{76121C1B-6B49-4F33-95EF-ADD22D2F27E8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{76121C1B-6B49-4F33-95EF-ADD22D2F27E8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{992C93C1-46D1-4DCB-B95D-C303BB4F9CAA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{992C93C1-46D1-4DCB-B95D-C303BB4F9CAA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{992C93C1-46D1-4DCB-B95D-C303BB4F9CAA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{992C93C1-46D1-4DCB-B95D-C303BB4F9CAA}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{76121C1B-6B49-4F33-95EF-ADD22D2F27E8} = {5CC04298-37E7-4932-BEC9-3EB1A5BDCA69}
+	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {F79BC6C1-C8F5-4204-B8B7-14CD40FDE97B}
 	EndGlobalSection

diff --git a/LeedsExperiment/MetsParser/MetsFile.cs b/LeedsExperiment/MetsParser/MetsFile.cs
@@ -0,0 +1,22 @@
+using Fedora.Abstractions.Transfer;
+
+namespace MetsParser
+{    
+    public class MetsFile
+    {
+        // The title of the object the METS file describes
+        public string? Name { get; set; }
+
+        // The location of this METS file's parent directory; the METS file should be at its root
+        public Uri? Root {  get; set; }
+
+        // An entry describing the METS file itself, because it is not (typically) included in itself
+        public BinaryFile? Self { get; set; }
+
+        // A list of all the directories mentioned, with their names
+        public List<ContainerDirectory> Directories { get; set; } = [];
+
+        // A list of all the files mentioned, with their names and hashes (digests)
+        public List<BinaryFile> Files { get; set; } = [];
+    }
+}
diff --git a/LeedsExperiment/MetsParser/MetsParser.csproj b/LeedsExperiment/MetsParser/MetsParser.csproj
@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="AWSSDK.S3" Version="3.7.402.5" />
+    <PackageReference Include="MimeTypes" Version="2.5.2">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Fedora\Fedora.csproj" />
+    <ProjectReference Include="..\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>