Add goal Review Deferred Duplicates to handle skipped merge sets (#2610)

Co-authored-by: D. Ror <[email protected]>
sillsdev · Sep 29, 2023 · a9c0f82 · a9c0f82
1 parent 1d09bc3
commit a9c0f82
Show file tree

Hide file tree

Showing 45 changed files with 1,167 additions and 186 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -50,6 +50,7 @@
     "Dups",
     "endcap",
     "globaltool",
+    "graylist",
     "Guids",
     "kubeconfig",
     "langtags",

diff --git a/Backend.Tests/Controllers/MergeControllerTests.cs b/Backend.Tests/Controllers/MergeControllerTests.cs
@@ -12,6 +12,7 @@ namespace Backend.Tests.Controllers
     public class MergeControllerTests : IDisposable
     {
         private IMergeBlacklistRepository _mergeBlacklistRepo = null!;
+        private IMergeGraylistRepository _mergeGraylistRepo = null!;
         private IWordRepository _wordRepo = null!;
         private IMergeService _mergeService = null!;
         private IPermissionService _permissionService = null!;
@@ -38,9 +39,10 @@ protected virtual void Dispose(bool disposing)
         public void Setup()
         {
             _mergeBlacklistRepo = new MergeBlacklistRepositoryMock();
+            _mergeGraylistRepo = new MergeGraylistRepositoryMock();
             _wordRepo = new WordRepositoryMock();
             _wordService = new WordService(_wordRepo);
-            _mergeService = new MergeService(_mergeBlacklistRepo, _wordRepo, _wordService);
+            _mergeService = new MergeService(_mergeBlacklistRepo, _mergeGraylistRepo, _wordRepo, _wordService);
             _permissionService = new PermissionServiceMock();
             _mergeController = new MergeController(_mergeService, _permissionService);
         }
@@ -54,16 +56,39 @@ public void BlacklistAddTest()
 
             // Add two Lists of wordIds.
             _ = _mergeController.BlacklistAdd(ProjId, wordIdsA).Result;
-            var result = _mergeBlacklistRepo.GetAllEntries(ProjId).Result;
+            var result = _mergeBlacklistRepo.GetAllSets(ProjId).Result;
             Assert.That(result, Has.Count.EqualTo(1));
             Assert.That(result.First().WordIds, Is.EqualTo(wordIdsA));
             _ = _mergeController.BlacklistAdd(ProjId, wordIdsB).Result;
-            result = _mergeBlacklistRepo.GetAllEntries(ProjId).Result;
+            result = _mergeBlacklistRepo.GetAllSets(ProjId).Result;
             Assert.That(result, Has.Count.EqualTo(2));
 
             // Add a List of wordIds that contains both previous lists.
             _ = _mergeController.BlacklistAdd(ProjId, wordIdsC).Result;
-            result = _mergeBlacklistRepo.GetAllEntries(ProjId).Result;
+            result = _mergeBlacklistRepo.GetAllSets(ProjId).Result;
+            Assert.That(result, Has.Count.EqualTo(1));
+            Assert.That(result.First().WordIds, Is.EqualTo(wordIdsC));
+        }
+
+        [Test]
+        public void GreylistAddTest()
+        {
+            var wordIdsA = new List<string> { "1", "2" };
+            var wordIdsB = new List<string> { "3", "1" };
+            var wordIdsC = new List<string> { "1", "2", "3" };
+
+            // Add two Lists of wordIds.
+            _ = _mergeController.GraylistAdd(ProjId, wordIdsA).Result;
+            var result = _mergeGraylistRepo.GetAllSets(ProjId).Result;
+            Assert.That(result, Has.Count.EqualTo(1));
+            Assert.That(result.First().WordIds, Is.EqualTo(wordIdsA));
+            _ = _mergeController.GraylistAdd(ProjId, wordIdsB).Result;
+            result = _mergeGraylistRepo.GetAllSets(ProjId).Result;
+            Assert.That(result, Has.Count.EqualTo(2));
+
+            // Add a List of wordIds that contains both previous lists.
+            _ = _mergeController.GraylistAdd(ProjId, wordIdsC).Result;
+            result = _mergeGraylistRepo.GetAllSets(ProjId).Result;
             Assert.That(result, Has.Count.EqualTo(1));
             Assert.That(result.First().WordIds, Is.EqualTo(wordIdsC));
         }

diff --git a/Backend.Tests/Helper/DuplicateFinderTests.cs b/Backend.Tests/Helper/DuplicateFinderTests.cs
@@ -12,7 +12,7 @@ public class DuplicateFinderTests
     {
         private DuplicateFinder _dupFinder = null!;
         private List<Word> _frontier = null!;
-        private Func<List<string>, Task<bool>> _isInBlacklist = null!;
+        private Func<List<string>, Task<bool>> _isUnavailableSet = null!;
 
         private const int MaxInList = 4;
         private const int MaxLists = 3;
@@ -26,7 +26,7 @@ public void Setup()
         {
             _dupFinder = new DuplicateFinder(MaxInList, MaxLists, MaxScore);
             _frontier = new List<Word>();
-            _isInBlacklist = _ => Task.FromResult(false);
+            _isUnavailableSet = _ => Task.FromResult(false);
         }
 
         [Test]
@@ -40,7 +40,7 @@ public void GetIdenticalVernToWordTest()
             _frontier.ElementAt(1).Vernacular = vern;
             _frontier.ElementAt(2).Vernacular = vern;
             _frontier.ElementAt(5).Vernacular = vern;
-            var wordLists = _dupFinder.GetIdenticalVernWords(_frontier, _isInBlacklist).Result;
+            var wordLists = _dupFinder.GetIdenticalVernWords(_frontier, _isUnavailableSet).Result;
             Assert.That(wordLists, Has.Count.EqualTo(1));
             Assert.That(wordLists.First(), Has.Count.EqualTo(3));
         }
@@ -50,7 +50,7 @@ public void GetSimilarWordsAndMaxInListAndMaxListsTest()
         {
             _frontier = Util.RandomWordList(MaxInList * MaxLists, ProjId);
             _dupFinder = new DuplicateFinder(MaxInList, MaxLists, NoMaxScore);
-            var wordLists = _dupFinder.GetSimilarWords(_frontier, _isInBlacklist).Result;
+            var wordLists = _dupFinder.GetSimilarWords(_frontier, _isUnavailableSet).Result;
             Assert.That(wordLists, Has.Count.EqualTo(MaxLists));
             Assert.That(wordLists.First(), Has.Count.EqualTo(MaxInList));
             Assert.That(wordLists.Last(), Has.Count.EqualTo(MaxInList));
@@ -63,7 +63,7 @@ public void GetSimilarWordsAndMaxScoreTest()
             // Ensure at least one set of similar words, in case MaxScore is too low.
             _frontier.Last().Vernacular = _frontier.First().Vernacular;
 
-            var wordLists = _dupFinder.GetSimilarWords(_frontier, _isInBlacklist).Result;
+            var wordLists = _dupFinder.GetSimilarWords(_frontier, _isUnavailableSet).Result;
             var firstList = wordLists.First();
             var firstMin = _dupFinder.GetWordScore(firstList.First(), firstList.ElementAt(1));
             var firstMax = _dupFinder.GetWordScore(firstList.First(), firstList.Last());
@@ -87,13 +87,13 @@ public void GetSimilarWordsAndMaxScoreTest()
         }
 
         [Test]
-        public void GetSimilarWordsBlacklistTest()
+        public void GetSimilarWordsBlacklistOrGraylistTest()
         {
             _frontier = Util.RandomWordList(MaxInList + 1, ProjId);
-            // Make sure the first set only is blacklisted, so all but the first word end up in a lone list.
-            _isInBlacklist = wordList => Task.FromResult(wordList.First() == _frontier.First().Vernacular);
+            // Make sure the first set only is black/gray-listed, so all but the first word end up in a lone list.
+            _isUnavailableSet = wordList => Task.FromResult(wordList.First() == _frontier.First().Vernacular);
             _dupFinder = new DuplicateFinder(MaxInList, MaxLists, NoMaxScore);
-            var wordLists = _dupFinder.GetSimilarWords(_frontier, _isInBlacklist).Result;
+            var wordLists = _dupFinder.GetSimilarWords(_frontier, _isUnavailableSet).Result;
             Assert.That(wordLists, Has.Count.EqualTo(1));
             Assert.That(wordLists.First(), Has.Count.EqualTo(MaxInList));
         }

diff --git a/Backend.Tests/Mocks/MergeBlacklistRepositoryMock.cs b/Backend.Tests/Mocks/MergeBlacklistRepositoryMock.cs
@@ -10,14 +10,14 @@ namespace Backend.Tests.Mocks
 {
     public class MergeBlacklistRepositoryMock : IMergeBlacklistRepository
     {
-        private readonly List<MergeBlacklistEntry> _mergeBlacklist;
+        private readonly List<MergeWordSet> _mergeBlacklist;
 
         public MergeBlacklistRepositoryMock()
         {
-            _mergeBlacklist = new List<MergeBlacklistEntry>();
+            _mergeBlacklist = new List<MergeWordSet>();
         }
 
-        public Task<List<MergeBlacklistEntry>> GetAllEntries(string projectId, string? userId = null)
+        public Task<List<MergeWordSet>> GetAllSets(string projectId, string? userId = null)
         {
             var cloneList = _mergeBlacklist.Select(e => e.Clone()).ToList();
             var enumerable = userId is null ?
@@ -26,27 +26,27 @@ public Task<List<MergeBlacklistEntry>> GetAllEntries(string projectId, string? u
             return Task.FromResult(enumerable.ToList());
         }
 
-        public Task<MergeBlacklistEntry?> GetEntry(string projectId, string entryId)
+        public Task<MergeWordSet?> GetSet(string projectId, string entryId)
         {
             try
             {
                 var foundMergeBlacklist = _mergeBlacklist.Single(entry => entry.Id == entryId);
-                return Task.FromResult<MergeBlacklistEntry?>(foundMergeBlacklist.Clone());
+                return Task.FromResult<MergeWordSet?>(foundMergeBlacklist.Clone());
             }
             catch (InvalidOperationException)
             {
-                return Task.FromResult<MergeBlacklistEntry?>(null);
+                return Task.FromResult<MergeWordSet?>(null);
             }
         }
 
-        public Task<MergeBlacklistEntry> Create(MergeBlacklistEntry blacklistEntry)
+        public Task<MergeWordSet> Create(MergeWordSet wordSetEntry)
         {
-            blacklistEntry.Id = Guid.NewGuid().ToString();
-            _mergeBlacklist.Add(blacklistEntry.Clone());
-            return Task.FromResult(blacklistEntry.Clone());
+            wordSetEntry.Id = Guid.NewGuid().ToString();
+            _mergeBlacklist.Add(wordSetEntry.Clone());
+            return Task.FromResult(wordSetEntry.Clone());
         }
 
-        public Task<bool> DeleteAllEntries(string projectId)
+        public Task<bool> DeleteAllSets(string projectId)
         {
             _mergeBlacklist.Clear();
             return Task.FromResult(true);
@@ -58,17 +58,17 @@ public Task<bool> Delete(string projectId, string entryId)
             return Task.FromResult(_mergeBlacklist.Remove(foundMergeBlacklist));
         }
 
-        public Task<ResultOfUpdate> Update(MergeBlacklistEntry blacklistEntry)
+        public Task<ResultOfUpdate> Update(MergeWordSet wordSetEntry)
         {
             var foundEntry = _mergeBlacklist.Single(
-                e => e.ProjectId == blacklistEntry.ProjectId && e.Id == blacklistEntry.Id);
+                e => e.ProjectId == wordSetEntry.ProjectId && e.Id == wordSetEntry.Id);
             var success = _mergeBlacklist.Remove(foundEntry);
             if (!success)
             {
                 return Task.FromResult(ResultOfUpdate.NotFound);
             }
 
-            _mergeBlacklist.Add(blacklistEntry.Clone());
+            _mergeBlacklist.Add(wordSetEntry.Clone());
             return Task.FromResult(ResultOfUpdate.Updated);
         }
     }

diff --git a/Backend.Tests/Mocks/MergeGraylistRepositoryMock.cs b/Backend.Tests/Mocks/MergeGraylistRepositoryMock.cs
@@ -0,0 +1,75 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using BackendFramework.Helper;
+using BackendFramework.Interfaces;
+using BackendFramework.Models;
+
+namespace Backend.Tests.Mocks
+{
+    public class MergeGraylistRepositoryMock : IMergeGraylistRepository
+    {
+        private readonly List<MergeWordSet> _mergeGraylist;
+
+        public MergeGraylistRepositoryMock()
+        {
+            _mergeGraylist = new List<MergeWordSet>();
+        }
+
+        public Task<List<MergeWordSet>> GetAllSets(string projectId, string? userId = null)
+        {
+            var cloneList = _mergeGraylist.Select(e => e.Clone()).ToList();
+            var enumerable = userId is null ?
+                cloneList.Where(e => e.ProjectId == projectId) :
+                cloneList.Where(e => e.ProjectId == projectId && e.UserId == userId);
+            return Task.FromResult(enumerable.ToList());
+        }
+
+        public Task<MergeWordSet?> GetSet(string projectId, string entryId)
+        {
+            try
+            {
+                var foundMergeGraylist = _mergeGraylist.Single(entry => entry.Id == entryId);
+                return Task.FromResult<MergeWordSet?>(foundMergeGraylist.Clone());
+            }
+            catch (InvalidOperationException)
+            {
+                return Task.FromResult<MergeWordSet?>(null);
+            }
+        }
+
+        public Task<MergeWordSet> Create(MergeWordSet wordSetEntry)
+        {
+            wordSetEntry.Id = Guid.NewGuid().ToString();
+            _mergeGraylist.Add(wordSetEntry.Clone());
+            return Task.FromResult(wordSetEntry.Clone());
+        }
+
+        public Task<bool> DeleteAllSets(string projectId)
+        {
+            _mergeGraylist.Clear();
+            return Task.FromResult(true);
+        }
+
+        public Task<bool> Delete(string projectId, string entryId)
+        {
+            var foundMergeGraylist = _mergeGraylist.Single(entry => entry.Id == entryId);
+            return Task.FromResult(_mergeGraylist.Remove(foundMergeGraylist));
+        }
+
+        public Task<ResultOfUpdate> Update(MergeWordSet wordSetEntry)
+        {
+            var foundEntry = _mergeGraylist.Single(
+                e => e.ProjectId == wordSetEntry.ProjectId && e.Id == wordSetEntry.Id);
+            var success = _mergeGraylist.Remove(foundEntry);
+            if (!success)
+            {
+                return Task.FromResult(ResultOfUpdate.NotFound);
+            }
+
+            _mergeGraylist.Add(wordSetEntry.Clone());
+            return Task.FromResult(ResultOfUpdate.Updated);
+        }
+    }
+}
diff --git a/....Tests/Models/MergeBlacklistEntryTests.cs → Backend.Tests/Models/MergeWordSetTests.cs b/....Tests/Models/MergeBlacklistEntryTests.cs → Backend.Tests/Models/MergeWordSetTests.cs
@@ -4,18 +4,18 @@
 
 namespace Backend.Tests.Models
 {
-    public class MergeBlacklistEntryTests
+    public class MergeWordSetTests
     {
-        private const string EntryId = "MergeBlacklistEntryTestId";
-        private const string ProjId = "MergeBlacklistEntryTestProjectId";
-        private const string UserId = "MergeBlacklistEntryTestUserId";
+        private const string EntryId = "MergeWordSetTestId";
+        private const string ProjId = "MergeWordSetTestProjectId";
+        private const string UserId = "MergeWordSetTestUserId";
         private readonly List<string> _wordIds = new() { "word1", "word2" };
         private readonly List<string> _wordIdsReversed = new() { "word2", "word1" };
 
         [Test]
         public void TestClone()
         {
-            var entryA = new MergeBlacklistEntry
+            var entryA = new MergeWordSet
             {
                 Id = EntryId,
                 ProjectId = ProjId,
@@ -29,14 +29,14 @@ public void TestClone()
         [Test]
         public void TestEquals()
         {
-            var entryA = new MergeBlacklistEntry
+            var entryA = new MergeWordSet
             {
                 Id = EntryId,
                 ProjectId = ProjId,
                 UserId = UserId,
                 WordIds = _wordIds
             };
-            var entryB = new MergeBlacklistEntry
+            var entryB = new MergeWordSet
             {
                 Id = EntryId,
                 ProjectId = ProjId,
@@ -49,8 +49,8 @@ public void TestEquals()
         [Test]
         public void TestEqualsFalse()
         {
-            var entryA = new MergeBlacklistEntry();
-            var entryB = new MergeBlacklistEntry();
+            var entryA = new MergeWordSet();
+            var entryB = new MergeWordSet();
             entryA.Id = EntryId;
             Assert.That(entryA.Equals(entryB), Is.False);
 
@@ -70,21 +70,21 @@ public void TestEqualsFalse()
         [Test]
         public void TestEqualsNull()
         {
-            var edit = new MergeBlacklistEntry { ProjectId = ProjId };
+            var edit = new MergeWordSet { ProjectId = ProjId };
             Assert.That(edit.Equals(null), Is.False);
         }
 
         [Test]
         public void TestHashCode()
         {
-            var entryA = new MergeBlacklistEntry
+            var entryA = new MergeWordSet
             {
                 Id = EntryId,
                 ProjectId = ProjId,
                 UserId = UserId,
                 WordIds = _wordIdsReversed
             };
-            var entryB = new MergeBlacklistEntry
+            var entryB = new MergeWordSet
             {
                 Id = "DifferentTestId",
                 ProjectId = ProjId,