refactor: reimplement anime↔character/creator mappings for AniDB

Reimplemented the AniDB Character and Creator mappings for AniDB anime. Making sure we store all the mappings, and in the correct order they appear in the XMLs. This is done by dropping the existing tables, followed by creating new ones and restoring the data from the XMLs if possible, or queueing the anime for a forced online lookup if it's not in the local XML cache or is otherwise unusable. Dropped the `AnimeStaff` and `CrossRef_Anime_Staff` tables used by the API, which were almost a 1:1 mapping for `AniDB_Anime_Staff` combined with `AniDB_Anime_Character`, except sometimes out-of-sync data. All usage of the tables has been directed at the updated `AniDB_Character`, `AniDB_Anime_Staff`, `AniDB_Anime_Character`, or `AniDB_Anime_Character_Creator` tables where appropriate. Fixes #1183 Closes #1183
ShokoAnime · Dec 30, 2024 · 1709eb3 · 1709eb3
1 parent 8c82241
commit 1709eb3
Show file tree

Hide file tree

Showing 138 changed files with 2,133 additions and 3,092 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -26,6 +26,7 @@
     "muxed",
     "muxing",
     "mylist",
+    "NutzCode",
     "outro",
     "ova",
     "ovas",

diff --git a/Shoko.Commons b/Shoko.Commons
diff --git a/Shoko.Server/API/AuthenticationController.cs b/Shoko.Server/API/AuthenticationController.cs
@@ -48,7 +48,7 @@ public ActionResult<List<ApikeyResult>> GetApikeys()
     public ActionResult<string> GenerateApikey([FromBody]string device)
     {
         if (string.IsNullOrWhiteSpace(device)) return BadRequest("device cannot be empty");
-        return RepoFactory.AuthTokens.CreateNewApikey(User, device);
+        return RepoFactory.AuthTokens.CreateNewApiKey(User, device);
     }
 
     /// <summary>

diff --git a/Shoko.Server/API/v1/Implementations/ShokoServiceImplementation/ShokoServiceImplementation.cs b/Shoko.Server/API/v1/Implementations/ShokoServiceImplementation/ShokoServiceImplementation.cs
@@ -1278,15 +1278,15 @@ public List<CL_AniDB_Character> GetCharactersForSeiyuu(int seiyuuID)
                 return chars;
             }
 
-            var links = RepoFactory.AniDB_Character_Creator.GetByCreatorID(seiyuu.CreatorID);
+            var links = RepoFactory.AniDB_Anime_Character_Creator.GetByCreatorID(seiyuu.CreatorID);
 
             foreach (var chrSei in links)
             {
                 var chr = RepoFactory.AniDB_Character.GetByID(chrSei.CharacterID);
                 if (chr != null)
                 {
                     var aniChars =
-                        RepoFactory.AniDB_Anime_Character.GetByCharID(chr.CharID);
+                        RepoFactory.AniDB_Anime_Character.GetByCharacterID(chr.CharacterID);
                     if (aniChars.Count > 0)
                     {
                         var anime = RepoFactory.AniDB_Anime.GetByAnimeID(aniChars[0].AnimeID);

diff --git a/.../API/v1/Implementations/ShokoServiceImplementation/ShokoServiceImplementation_Entities.cs b/.../API/v1/Implementations/ShokoServiceImplementation/ShokoServiceImplementation_Entities.cs
@@ -500,8 +500,7 @@ public string RemoveAssociationOnFile(int videoLocalID, int animeEpisodeID)
                 }
 
                 animeSeriesID = ep.AnimeSeriesID;
-                var xref =
-                    RepoFactory.CrossRef_File_Episode.GetByHashAndEpisodeID(vid.Hash, ep.AniDB_EpisodeID);
+                var xref = vid.EpisodeCrossReferences.FirstOrDefault(x => x.EpisodeID == ep.AniDB_EpisodeID);
                 if (xref != null)
                 {
                     if (xref.CrossRefSource == (int)CrossRefSource.AniDB)
@@ -586,7 +585,7 @@ public string SetVariationStatusOnFile(int videoLocalID, bool isVariation)
     private static void RemoveXRefsForFile(int videoLocalID)
     {
         var vlocal = RepoFactory.VideoLocal.GetByID(videoLocalID);
-        var fileEps = RepoFactory.CrossRef_File_Episode.GetByHash(vlocal.Hash);
+        var fileEps = RepoFactory.CrossRef_File_Episode.GetByEd2k(vlocal.Hash);
 
         foreach (var fileEp in fileEps)
         {
@@ -1193,7 +1192,7 @@ public List<CL_VideoDetailed> GetFilesForEpisode(int episodeID, int userID)
             var ep = RepoFactory.AnimeEpisode.GetByID(episodeID);
             if (ep != null)
             {
-                var files = ep.VideoLocals;
+                var files = ep.VideoLocals.ToList();
                 files.Sort(FileQualityFilter.CompareTo);
                 return files.Select(a => _videoLocalService.GetV1DetailedContract(a, userID)).ToList();
             }
@@ -3348,10 +3347,8 @@ public string DeleteCustomTagCrossRef(int customTagID, int crossRefType, int cro
     {
         try
         {
-            var xrefs =
-                RepoFactory.CrossRef_CustomTag.GetByUniqueID(customTagID, crossRefType, crossRefID);
-
-            if (xrefs is null || xrefs.Count == 0)
+            var xrefs = RepoFactory.CrossRef_CustomTag.GetByUniqueID(customTagID, (CustomTagCrossRefType)crossRefType, crossRefID);
+            if (xrefs.Count == 0)
             {
                 return "Custom Tag not found";
             }

diff --git a/...API/v1/Implementations/ShokoServiceImplementation/ShokoServiceImplementation_Providers.cs b/...API/v1/Implementations/ShokoServiceImplementation/ShokoServiceImplementation_Providers.cs
@@ -57,7 +57,7 @@ public CL_AniDB_AnimeCrossRefs GetCrossRefDetails(int animeID)
             }
 
             // Trakt
-            foreach (var xref in anime.GetCrossRefTraktV2())
+            foreach (var xref in anime.TraktShowCrossReferences)
             {
                 result.CrossRef_AniDB_Trakt.Add(xref);
 
@@ -81,7 +81,7 @@ public CL_AniDB_AnimeCrossRefs GetCrossRefDetails(int animeID)
             }
 
             // MAL
-            var xrefMAL = anime.GetCrossRefMAL();
+            var xrefMAL = anime.MalCrossReferences;
             if (xrefMAL == null)
             {
                 result.CrossRef_AniDB_MAL = null;

diff --git a/...API/v1/Implementations/ShokoServiceImplementation/ShokoServiceImplementation_Utilities.cs b/...API/v1/Implementations/ShokoServiceImplementation/ShokoServiceImplementation_Utilities.cs
@@ -146,7 +146,7 @@ public bool DeleteMultipleFilesWithPreferences(int userID)
 
             foreach (var ep in eps)
             {
-                var videoLocals = ep.VideoLocals;
+                var videoLocals = ep.VideoLocals.ToList();
                 videoLocals.Sort(FileQualityFilter.CompareTo);
                 var keep = videoLocals
                     .Take(FileQualityFilter.Settings.MaxNumberOfFilesToKeep)
@@ -196,7 +196,7 @@ public List<CL_VideoLocal> PreviewDeleteMultipleFilesWithPreferences(int userID)
 
         foreach (var ep in eps)
         {
-            var videoLocals = ep.VideoLocals;
+            var videoLocals = ep.VideoLocals.ToList();
             videoLocals.Sort(FileQualityFilter.CompareTo);
             var keep = videoLocals
                 .Take(FileQualityFilter.Settings.MaxNumberOfFilesToKeep)
@@ -222,7 +222,7 @@ public List<CL_VideoDetailed> GetMultipleFilesForDeletionByPreferences(int userI
 
         foreach (var ep in eps)
         {
-            var videoLocals = ep.VideoLocals;
+            var videoLocals = ep.VideoLocals.ToList();
             videoLocals.Sort(FileQualityFilter.CompareTo);
             var keep = videoLocals
                 .Take(FileQualityFilter.Settings.MaxNumberOfFilesToKeep)
@@ -275,7 +275,7 @@ public List<CL_VideoLocal> SearchForFiles(int searchType, string searchCriteria,
                     break;
 
                 case FileSearchCriteria.ED2KHash:
-                    var vidl = RepoFactory.VideoLocal.GetByHash(searchCriteria.Trim());
+                    var vidl = RepoFactory.VideoLocal.GetByEd2k(searchCriteria.Trim());
                     if (vidl != null)
                         vids.Add(_videoLocalService.GetV1Contract(vidl, userID));
                     break;
@@ -598,7 +598,7 @@ public List<CL_MissingFile> GetMyListFilesForRemoval(int userID)
                     else
                     {
                         // now check if the file actually exists on disk
-                        var v = RepoFactory.VideoLocal.GetByHash(hash);
+                        var v = RepoFactory.VideoLocal.GetByEd2k(hash);
                         fileMissing = true;
                         if (v == null) break;
                         foreach (var p in v.Places)
@@ -850,7 +850,7 @@ public List<CL_DuplicateFile> GetAllDuplicateFiles()
                 var vl = first.VideoLocal;
                 SVR_AniDB_Anime anime = null;
                 SVR_AniDB_Episode episode = null;
-                var xref = RepoFactory.CrossRef_File_Episode.GetByHash(vl.Hash);
+                var xref = RepoFactory.CrossRef_File_Episode.GetByEd2k(vl.Hash);
                 if (xref.Count > 0)
                 {
                     if (xref.FirstOrDefault(x => x.AnimeID is not 0)?.AnimeID is { } animeId)
@@ -1138,7 +1138,7 @@ public List<CL_GroupVideoQuality> GetGroupVideoQualitySummary(int animeID)
     {
         var vidQuals = new List<CL_GroupVideoQuality>();
 
-        var files = RepoFactory.VideoLocal.GetByAniDBAnimeID(animeID);
+        var files = RepoFactory.VideoLocal.GetByAniDBAnimeID(animeID).ToList();
         files.Sort(FileQualityFilter.CompareTo);
         var lookup = files.ToLookup(a =>
         {

diff --git a/Shoko.Server/API/v1/Implementations/ShokoServiceImplementationMetro.cs b/Shoko.Server/API/v1/Implementations/ShokoServiceImplementationMetro.cs
@@ -136,7 +136,7 @@ public Metro_CommunityLinks GetCommunityLinks(int animeID)
             contract.AniDB_DiscussURL = string.Format(Constants.URLS.AniDB_SeriesDiscussion, animeID);
 
             // MAL
-            var malRef = anime.GetCrossRefMAL();
+            var malRef = anime.MalCrossReferences;
             if (malRef is not null && malRef.Count > 0)
             {
                 contract.MAL_ID = malRef[0].MALID.ToString();
@@ -145,7 +145,7 @@ public Metro_CommunityLinks GetCommunityLinks(int animeID)
             }
 
             // Trakt
-            var traktRef = anime.GetCrossRefTraktV2();
+            var traktRef = anime.TraktShowCrossReferences;
             if (traktRef is not null && traktRef.Count > 0)
             {
                 contract.Trakt_ID = traktRef[0].TraktID;
@@ -644,46 +644,29 @@ public List<Metro_Anime_Summary> SearchAnime(int userID, string queryText, int m
                 return retAnime;
             }
 
-
-            var allAnime = RepoFactory.AniDB_Anime.SearchByName(queryText);
-            foreach (var anidb_anime in allAnime)
+            var allAnime = SeriesSearch.SearchSeries(user, queryText, maxRecords, SeriesSearch.SearchFlags.Titles);
+            foreach (var result in allAnime)
             {
-                if (!user.AllowedAnime(anidb_anime))
-                {
+                var ser = result.Result;
+                var anidb_anime = ser.AniDB_Anime!;
+                if (anidb_anime is null || !user.AllowedSeries(ser))
                     continue;
-                }
-
-                var ser = RepoFactory.AnimeSeries.GetByAnimeID(anidb_anime.AnimeID);
 
+                var imgDet = anidb_anime.PreferredOrDefaultPoster;
                 var summary = new Metro_Anime_Summary
                 {
                     AirDateAsSeconds = anidb_anime.GetAirDateAsSeconds(),
-                    AnimeID = anidb_anime.AnimeID
+                    AnimeID = anidb_anime.AnimeID,
+                    AnimeName = ser.PreferredTitle,
+                    AnimeSeriesID = ser.AnimeSeriesID,
+                    BeginYear = anidb_anime.BeginYear,
+                    EndYear = anidb_anime.EndYear,
+                    PosterName = imgDet.LocalPath,
+                    ImageType = (int)imgDet.ImageType.ToClient(imgDet.Source),
+                    ImageID = imgDet.ID,
                 };
-                if (ser is not null)
-                {
-                    summary.AnimeName = ser.PreferredTitle;
-                    summary.AnimeSeriesID = ser.AnimeSeriesID;
-                }
-                else
-                {
-                    summary.AnimeName = anidb_anime.MainTitle;
-                    summary.AnimeSeriesID = 0;
-                }
-
-                summary.BeginYear = anidb_anime.BeginYear;
-                summary.EndYear = anidb_anime.EndYear;
-
-                var imgDet = anidb_anime.PreferredOrDefaultPoster;
-                summary.PosterName = imgDet.LocalPath;
-                summary.ImageType = (int)imgDet.ImageType.ToClient(imgDet.Source);
-                summary.ImageID = imgDet.ID;
 
                 retAnime.Add(summary);
-                if (retAnime.Count == maxRecords)
-                {
-                    break;
-                }
             }
         }
         catch (Exception ex)
@@ -924,7 +907,7 @@ public List<Metro_AniDB_Character> GetCharactersForAnime(int animeID, int maxRec
         try
         {
             var animeChars = RepoFactory.AniDB_Anime_Character.GetByAnimeID(animeID)
-                .OrderByDescending(item => item.CharType.Equals("main character in", StringComparison.InvariantCultureIgnoreCase))
+                .OrderByDescending(item => item.AppearanceType is Server.CharacterAppearanceType.Main_Character)
                 .ToList();
             if (animeChars.Count == 0)
             {
@@ -937,7 +920,7 @@ public List<Metro_AniDB_Character> GetCharactersForAnime(int animeID, int maxRec
             foreach (var animeChar in animeChars)
             {
                 index++;
-                var character = RepoFactory.AniDB_Character.GetByID(animeChar.CharID);
+                var character = RepoFactory.AniDB_Character.GetByID(animeChar.CharacterID);
                 if (character is not null)
                 {
                     var contract = new Metro_AniDB_Character();

diff --git a/Shoko.Server/API/v2/Models/common/Group.cs b/Shoko.Server/API/v2/Models/common/Group.cs
@@ -104,25 +104,26 @@ public static Group GenerateFromAnimeGroup(HttpContext ctx, SVR_AnimeGroup ag, i
 
         if (!noCast)
         {
-            var xrefAnimeStaff = RepoFactory.CrossRef_Anime_Staff.GetByAnimeIDAndRoleType(anime.AnimeID, StaffRoleType.Seiyuu);
+            var xrefAnimeStaff = RepoFactory.AniDB_Anime_Character_Creator.GetByAnimeID(anime.AnimeID);
             foreach (var xref in xrefAnimeStaff)
             {
-                if (xref.RoleID == null) continue;
-
-                var character = RepoFactory.AnimeCharacter.GetByID(xref.RoleID.Value);
+                var character = RepoFactory.AniDB_Character.GetByID(xref.CharacterID);
                 if (character == null) continue;
 
-                var staff = RepoFactory.AnimeStaff.GetByID(xref.StaffID);
+                var staff = RepoFactory.AniDB_Creator.GetByID(xref.CreatorID);
                 if (staff == null) continue;
 
+                var xref2 = xref.CharacterCrossReference;
+                if (xref2 == null) continue;
+
                 var role = new Role
                 {
                     character = character.Name,
-                    character_image = APIHelper.ConstructImageLinkFromTypeAndId(ctx, ImageEntityType.Character, DataSourceEnum.Shoko, xref.RoleID.Value),
+                    character_image = APIHelper.ConstructImageLinkFromTypeAndId(ctx, ImageEntityType.Character, DataSourceEnum.Shoko, xref.CharacterID),
                     staff = staff.Name,
-                    staff_image = APIHelper.ConstructImageLinkFromTypeAndId(ctx, ImageEntityType.Person, DataSourceEnum.Shoko, xref.StaffID),
-                    role = xref.Role,
-                    type = ((StaffRoleType)xref.RoleType).ToString()
+                    staff_image = APIHelper.ConstructImageLinkFromTypeAndId(ctx, ImageEntityType.Person, DataSourceEnum.Shoko, xref.CreatorID),
+                    role = xref2.AppearanceType.ToString().Replace("_", " "),
+                    type = "Seiyuu",
                 };
                 g.roles ??= [];
 

diff --git a/Shoko.Server/API/v2/Models/common/RawFile.cs b/Shoko.Server/API/v2/Models/common/RawFile.cs
@@ -159,7 +159,7 @@ public RawFile(HttpContext ctx, SVR_VideoLocal vl, int level, int uid, AnimeEpis
         url = APIV2Helper.ConstructVideoLocalStream(ctx, uid, vl.VideoLocalID.ToString(),
             "file" + Path.GetExtension(filename), false);
 
-        recognized = e != null || vl.EpisodeCrossRefs.Count != 0;
+        recognized = e != null || vl.EpisodeCrossReferences.Count != 0;
 
         if (vl.MediaInfo?.GeneralStream == null || level < 0)
         {

diff --git a/Shoko.Server/API/v2/Models/common/Serie.cs b/Shoko.Server/API/v2/Models/common/Serie.cs
@@ -131,29 +131,26 @@ public static Serie GenerateFromAniDBAnime(HttpContext ctx, SVR_AniDB_Anime anim
 
         if (!noCast)
         {
-            var xrefAnimeStaff = RepoFactory.CrossRef_Anime_Staff.GetByAnimeIDAndRoleType(anime.AnimeID,
-                StaffRoleType.Seiyuu);
+            var xrefAnimeStaff = RepoFactory.AniDB_Anime_Character_Creator.GetByAnimeID(anime.AnimeID);
             foreach (var xref in xrefAnimeStaff)
             {
-                if (!xref.RoleID.HasValue)
-                    continue;
+                var character = RepoFactory.AniDB_Character.GetByID(xref.CharacterID);
+                if (character == null) continue;
 
-                var character = RepoFactory.AnimeCharacter.GetByID(xref.RoleID.Value);
-                if (character is null)
-                    continue;
+                var staff = RepoFactory.AniDB_Creator.GetByID(xref.CreatorID);
+                if (staff == null) continue;
 
-                var staff = RepoFactory.AnimeStaff.GetByID(xref.StaffID);
-                if (staff is null)
-                    continue;
+                var xref2 = xref.CharacterCrossReference;
+                if (xref2 == null) continue;
 
                 var role = new Role
                 {
                     character = character.Name,
-                    character_image = APIHelper.ConstructImageLinkFromTypeAndId(ctx, ImageEntityType.Character, DataSourceEnum.Shoko, xref.RoleID.Value),
+                    character_image = APIHelper.ConstructImageLinkFromTypeAndId(ctx, ImageEntityType.Character, DataSourceEnum.AniDB, xref.CharacterID),
                     staff = staff.Name,
-                    staff_image = APIHelper.ConstructImageLinkFromTypeAndId(ctx, ImageEntityType.Person, DataSourceEnum.Shoko, xref.StaffID),
-                    role = xref.Role,
-                    type = ((StaffRoleType)xref.RoleType).ToString()
+                    staff_image = APIHelper.ConstructImageLinkFromTypeAndId(ctx, ImageEntityType.Person, DataSourceEnum.AniDB, xref.CreatorID),
+                    role = xref2.AppearanceType.ToString().Replace("_", " "),
+                    type = "Seiyuu",
                 };
                 sr.roles ??= [];
                 sr.roles.Add(role);