release(patch): Version 4.2.1

Small round of bug fixes.

 # Highlights

Here are some of the key features and improvements since the last stable release (4.2.0):

 ## Bug Fixes

- Don't clean up VFS if it doesn't exist. (b0cc8d4)

- Automagically remove old media folder mappings. (c940504)

- Fix partial episodes (b1280fb, 83c4745)

- Ensure all media folder configurations are available before cleaning up the VFS roots, since we rely on the metadata being available to fix them properly. (be15776)

- Scan all known libraries if series is gone. (e404314)

 ## Repository Changes

- Update read-me. (74418da)

For the full list of changes, please check out the [complete changelog](4.2.0...4.2.1) here on GitHub.

README.md

@@ -31,14 +31,15 @@ GitHub repository, or you can build it from source.
 Below is a version compatibility matrix for which version of Shokofin is
 compatible with what.
 
-| Shokofin   | Jellyfin | Shoko Server  |
-|------------|----------|---------------|
-| `0.x.x`    | `10.7`   | `4.0.0-4.1.2` |
-| `1.x.x`    | `10.7`   | `4.1.0-4.1.2` |
-| `2.x.x`    | `10.8`   | `4.1.2`       |
-| `3.x.x`    | `10.8`   | `4.2.0`       |
-| `4.x.x`    | `10.9`   | `4.2.2`       |
-| `dev`      | `10.9`   | `dev`       |
+| Shokofin          | Jellyfin | Shoko Server  |
+|-------------------|----------|---------------|
+| `0.x.x`           | `10.7`   | `4.0.0` — `4.1.2` |
+| `1.x.x`           | `10.7`   | `4.1.0` — `4.1.2` |
+| `2.x.x`           | `10.8`   | `4.1.2`           |
+| `3.x.x`           | `10.8`   | `4.2.0`           |
+| `4.0.0` — `4.1.1` | `10.9`   | `4.2.2`           |
+| `4.2.0` — `4.x.x` | `10.9`   | `4.2.2` — `5.0.0` |
+| `dev`             | `10.9`   | `dev`             |
 
 ### Official Repository
 

Shokofin/API/Models/CrossReference.cs

@@ -35,6 +35,16 @@ public class EpisodeCrossReferenceIDs
 
         public int? ReleaseGroup { get; set; }
 
+        /// <summary>
+        /// ED2K hash.
+        /// </summary>
+        public string ED2K { get; set; } = string.Empty;
+
+        /// <summary>
+        /// File size.
+        /// </summary>
+        public long FileSize { get; set; }
+
         /// <summary>
         /// Percentage file is matched to the episode.
         /// </summary>

Shokofin/API/ShokoAPIClient.cs

@@ -276,6 +276,11 @@ public Task<File> GetFile(string id)
         return Get<File>($"/api/v3/File/{id}?include=XRefs&includeDataFrom=AniDB");
     }
 
+    public Task<File> GetFileByEd2kAndFileSize(string ed2k, long fileSize)
+    {
+        return Get<File>($"/api/v3/File/Hash/ED2K?hash={Uri.EscapeDataString(ed2k)}&size={fileSize}&includeDataFrom=AniDB");
+    }
+
     public Task<List<File>> GetFileByPath(string path)
     {
         return Get<List<File>>($"/api/v3/File/PathEndsWith?path={Uri.EscapeDataString(path)}&includeDataFrom=AniDB&limit=1");

Shokofin/Configuration/MediaFolderConfigurationService.cs

@@ -209,11 +209,17 @@ private async void OnLibraryManagerItemRemoved(object? sender, ItemChangeEventAr
 
     #region Media Folder Mapping
 
-    public IReadOnlyList<(string vfsPath, string mainMediaFolderPath, CollectionType? collectionType, IReadOnlyList<MediaFolderConfiguration> mediaList)> GetAvailableMediaFoldersForLibrariesForEvents(Func<MediaFolderConfiguration, bool>? filter = null)
+    public async Task<IReadOnlyList<(string vfsPath, string mainMediaFolderPath, CollectionType? collectionType, IReadOnlyList<MediaFolderConfiguration> mediaList)>> GetAvailableMediaFoldersForLibraries(Func<MediaFolderConfiguration, bool>? filter = null)
     {
-        LockObj.Wait();
+        await LockObj.WaitAsync();
         try {
             var virtualFolders = LibraryManager.GetVirtualFolders();
+            if (ShouldGenerateAllConfigurations)
+            {
+                ShouldGenerateAllConfigurations = false;
+                await GenerateAllConfigurations(virtualFolders).ConfigureAwait(false);
+            }
+
             var attachRoot = Plugin.Instance.Configuration.VFS_AttachRoot;
             return Plugin.Instance.Configuration.MediaFolders
                 .Where(config => config.IsMapped && !config.IsVirtualRoot && (filter is null || filter(config)) && LibraryManager.GetItemById(config.MediaFolderId) is Folder)
@@ -312,7 +318,7 @@ private async Task GenerateAllConfigurations(List<VirtualFolderInfo> allVirtualF
             foreach (var mediaFolderPath in virtualFolder.Locations) {
                 if (LibraryManager.FindByPath(mediaFolderPath, true) is not Folder secondFolder)
                 {
-                    Logger.LogTrace("Unable to find database entry for {Path}", mediaFolderPath);
+                    Logger.LogTrace("Unable to find database entry for {Path} (Library={LibraryId})", mediaFolderPath, libraryId);
                     continue;
                 }
 
@@ -353,6 +359,15 @@ private async Task GenerateAllConfigurations(List<VirtualFolderInfo> allVirtualF
                     edits.remove.Add(location);
             }
         }
+
+        var mediaFoldersToRemove = config.MediaFolders
+            .Where(c => !filteredVirtualFolders.Any(v => Guid.Parse(v.ItemId) == c.LibraryId))
+            .ToList();
+        Logger.LogDebug("Found {Count} out of {TotalCount} media folders to remove.", mediaFoldersToRemove.Count, config.MediaFolders.Count);
+        foreach (var mediaFolder in mediaFoldersToRemove) {
+            Logger.LogTrace("Removing config for media folder at path {Path} (Library={LibraryId})", mediaFolder.MediaFolderPath, mediaFolder.LibraryId);
+            config.MediaFolders.Remove(mediaFolder);
+        }
     }
 
     private async Task<MediaFolderConfiguration> CreateConfigurationForPath(Guid libraryId, Folder mediaFolder, MediaFolderConfiguration? libraryConfig)

Shokofin/Events/EventDispatchService.cs

@@ -211,7 +211,7 @@ private async Task ProcessFileEvents(int fileId, List<(UpdateReason Reason, int
             var locationsToNotify = new List<string>();
             var mediaFoldersToNotify = new Dictionary<string, (string pathToReport, Folder mediaFolder)>();
             var seriesIds = await GetSeriesIdsForFile(fileId, changes.Select(t => t.Event).LastOrDefault(e => e.HasCrossReferences));
-            var libraries = ConfigurationService.GetAvailableMediaFoldersForLibrariesForEvents(c => c.IsFileEventsEnabled);
+            var libraries = await ConfigurationService.GetAvailableMediaFoldersForLibraries(c => c.IsFileEventsEnabled).ConfigureAwait(false);
             var (reason, importFolderId, relativePath, lastEvent) = changes.Last();
             if (reason is not UpdateReason.Removed) {
                 Logger.LogTrace("Processing file changed. (File={FileId})", fileId);
@@ -281,6 +281,19 @@ private async Task ProcessFileEvents(int fileId, List<(UpdateReason Reason, int
             }
             // Something was removed, so assume the location is gone.
             else if (changes.FirstOrDefault(t => t.Reason is UpdateReason.Removed).Event is IFileEventArgs firstRemovedEvent) {
+                // If we don't know which series to remove, then add all of them to be scanned.
+                if (seriesIds.Count is 0) {
+                    Logger.LogTrace("No series found for file. Adding all libraries. (File={FileId})", fileId);
+                    foreach (var (vfsPath, mainMediaFolderPath, collectionType, mediaConfigs) in libraries) {
+                        // Give the core logic _any_ file or folder placed directly in the media folder, so it will schedule the media folder to be refreshed.
+                        var fileOrFolder = FileSystem.GetFileSystemEntryPaths(mainMediaFolderPath, false).FirstOrDefault();
+                        if (!string.IsNullOrEmpty(fileOrFolder))
+                            mediaFoldersToNotify.TryAdd(mainMediaFolderPath, (fileOrFolder, mainMediaFolderPath.GetFolderForPath()));
+                    }
+
+                    goto aLabelToReduceNesting;
+                }
+
                 Logger.LogTrace("Processing file removed. (File={FileId})", fileId);
                 relativePath = firstRemovedEvent.RelativePath;
                 importFolderId = firstRemovedEvent.ImportFolderId;
@@ -351,6 +364,7 @@ private async Task ProcessFileEvents(int fileId, List<(UpdateReason Reason, int
                 }
             }
 
+            aLabelToReduceNesting:;
             if (LibraryScanWatcher.IsScanRunning) {
                 Logger.LogDebug("Skipped notifying Jellyfin about {LocationCount} changes because a library scan is running. (File={FileId})", locationsToNotify.Count, fileId.ToString());
                 return;
@@ -362,6 +376,7 @@ private async Task ProcessFileEvents(int fileId, List<(UpdateReason Reason, int
                 LibraryMonitor.ReportFileSystemChanged(location);
             if (mediaFoldersToNotify.Count > 0)
                 await Task.WhenAll(mediaFoldersToNotify.Values.Select(tuple => ReportMediaFolderChanged(tuple.mediaFolder, tuple.pathToReport))).ConfigureAwait(false);
+            Logger.LogDebug("Notified Jellyfin about {LocationCount} changes. (File={FileId})", locationsToNotify.Count + mediaFoldersToNotify.Count, fileId.ToString());
         }
         catch (Exception ex) {
             Logger.LogError(ex, "Error processing {EventCount} file change events. (File={FileId})", changes.Count, fileId);
@@ -397,15 +412,21 @@ private async Task<IReadOnlySet<string>> GetSeriesIdsForFile(int fileId, IFileEv
 
         var filteredSeriesIds = new HashSet<string>();
         foreach (var seriesId in seriesIds) {
-            var (primaryId, extraIds) = await ApiManager.GetSeriesIdsForSeason(seriesId);
-            var seriesPathSet = await ApiManager.GetPathSetForSeries(primaryId, extraIds);
-            if (seriesPathSet.Count > 0) {
-                filteredSeriesIds.Add(seriesId);
+            try {
+                var (primaryId, extraIds) = await ApiManager.GetSeriesIdsForSeason(seriesId);
+                var seriesPathSet = await ApiManager.GetPathSetForSeries(primaryId, extraIds);
+                if (seriesPathSet.Count > 0) {
+                    filteredSeriesIds.Add(seriesId);
+                }
+            }
+            // If we fail to find the series data (most likely because it's already gone) then just abort early. We'll handle it elsewhere.
+            catch (ApiException ex) when (ex.StatusCode is System.Net.HttpStatusCode.NotFound) {
+                return new HashSet<string>();
             }
         }
 
         // Return all series if we only have this file for all of them,
-        // otherwise return only the series were we have other files that are
+        // otherwise return only the series where we have other files that are
         // not linked to other series.
         return filteredSeriesIds.Count is 0 ? seriesIds : filteredSeriesIds;
     }
@@ -434,7 +455,7 @@ private async Task<IReadOnlySet<string>> GetSeriesIdsForFile(int fileId, IFileEv
 
     private void RemoveSymbolicLink(string filePath)
     {
-        // TODO: If this works better, the move it to an utility and also use it in the VFS if needed, or remove this comment if it's not needed.
+        // TODO: If this works better, then move it to an utility and also use it in the VFS if needed, or remove this comment if that's not needed.
         try {
             var fileExists = File.Exists(filePath);
             var fileInfo = new System.IO.FileInfo(filePath);

Shokofin/Resolvers/VirtualFileSystemService.cs

@@ -793,9 +793,8 @@ await Task.WhenAll(allFiles.Select(async (tuple) => {
             ExtraType.Sample => ["samples"],
             _ => ["extras"],
         };
-        var filePartSuffix = (episodeXref.Percentage?.Group ?? 1) is not 1
-            ? $".pt{episode.Shoko.CrossReferences.Where(xref => xref.ReleaseGroup == episodeXref.ReleaseGroup && xref.Percentage!.Group == episodeXref.Percentage!.Group).ToList().FindIndex(xref => xref.Percentage!.Start == episodeXref.Percentage!.Start && xref.Percentage!.End == episodeXref.Percentage!.End) + 1}"
-            : "";
+        var fileIdList = fileId;
+        var filePartSuffix = "";
         if (collectionType is CollectionType.movies || (collectionType is null && isMovieSeason)) {
             if (extrasFolders != null) {
                 foreach (var extrasFolder in extrasFolders)
@@ -823,7 +822,14 @@ await Task.WhenAll(allFiles.Select(async (tuple) => {
             }
             else {
                 folders.Add(Path.Join(vfsPath, showFolder, seasonFolder));
-                episodeName = $"{showName} S{(isSpecial ? 0 : seasonNumber).ToString().PadLeft(2, '0')}E{episodeNumber.ToString().PadLeft(show.EpisodePadding, '0')}{filePartSuffix}";
+                episodeName = $"{showName} S{(isSpecial ? 0 : seasonNumber).ToString().PadLeft(2, '0')}E{episodeNumber.ToString().PadLeft(show.EpisodePadding, '0')}";
+                if ((episodeXref.Percentage?.Group ?? 1) is not 1) {
+                    var list = episode.Shoko.CrossReferences.Where(xref => xref.ReleaseGroup == episodeXref.ReleaseGroup && xref.Percentage!.Group == episodeXref.Percentage!.Group).ToList();
+                    var files = await Task.WhenAll(list.Select(xref => ApiClient.GetFileByEd2kAndFileSize(xref.ED2K, xref.FileSize)));
+                    var index = list.FindIndex(xref => xref.Percentage!.Start == episodeXref.Percentage!.Start && xref.Percentage!.End == episodeXref.Percentage!.End);
+                    filePartSuffix = $".pt{index + 1}";
+                    fileIdList = files.Select(f => f.Id.ToString()).Join(",");
+                }
             }
         }
 
@@ -840,7 +846,7 @@ file.Shoko.AniDBData is not null
             );
         if (config.VFS_AddResolution && !string.IsNullOrEmpty(file.Shoko.Resolution))
             extraDetails.Add(file.Shoko.Resolution);
-        var fileName = $"{episodeName} {(extraDetails.Count is > 0 ? $"[{extraDetails.Select(a => a.ReplaceInvalidPathCharacters()).Join("] [")}] " : "")}[{ShokoSeriesId.Name}={seriesId}] [{ShokoFileId.Name}={fileId}]{Path.GetExtension(sourceLocation)}";
+        var fileName = $"{episodeName} {(extraDetails.Count is > 0 ? $"[{extraDetails.Select(a => a.ReplaceInvalidPathCharacters()).Join("] [")}] " : "")}[{ShokoSeriesId.Name}={seriesId}] [{ShokoFileId.Name}={fileIdList}]{filePartSuffix}{Path.GetExtension(sourceLocation)}";
         var symbolicLinks = folders
             .Select(folderPath => Path.Join(folderPath, fileName))
             .ToArray();
@@ -1001,6 +1007,12 @@ private List<string> FindSubtitlesForPath(string sourcePath)
 
     private LinkGenerationResult CleanupStructure(string vfsPath, string directoryToClean, IReadOnlyList<string> allKnownPaths, bool preview = false)
     {
+        if (!FileSystem.DirectoryExists(directoryToClean)) {
+            if (!preview)
+                Logger.LogDebug("Skipped cleaning up folder because it does not exist: {Path}", directoryToClean);
+            return new();
+        }
+
         if (!preview)
             Logger.LogDebug("Looking for files to remove in folder at {Path}", directoryToClean);
         var start = DateTime.Now;

Shokofin/StringExtensions.cs

@@ -4,10 +4,11 @@
 using System.Linq;
 using System.Text.RegularExpressions;
 using MediaBrowser.Common.Providers;
+using Shokofin.ExternalIds;
 
 namespace Shokofin;
 
-public static class StringExtensions
+public static partial class StringExtensions
 {
     public static string Replace(this string input, Regex regex, string replacement, int count, int startAt)
         =>  regex.Replace(input, replacement, count, startAt);
@@ -152,6 +153,21 @@ public static string ReplaceInvalidPathCharacters(this string path)
         return null;
     }
 
+    [GeneratedRegex(@"\.pt(?<partNumber>\d+)(?:\.[a-z0-9]+)?$", RegexOptions.IgnoreCase)]
+    private static partial Regex GetPartRegex();
+
     public static bool TryGetAttributeValue(this string text, string attribute, [NotNullWhen(true)] out string? value)
-        => !string.IsNullOrEmpty(value = GetAttributeValue(text, attribute));
+    {
+        value = GetAttributeValue(text, attribute);
+
+        // Select the correct id for the part number in the stringified list of file ids.
+        if (!string.IsNullOrEmpty(value) && attribute == ShokoFileId.Name && GetPartRegex().Match(text) is { Success: true } regexResult) {
+            var partNumber = int.Parse(regexResult.Groups["partNumber"].Value);
+            var index = partNumber - 1;
+            value = value.Split(',')[index];
+        }
+
+        return !string.IsNullOrEmpty(value);
+    }
+
 }

Shokofin/Tasks/CleanupVirtualRootTask.cs

@@ -9,14 +9,15 @@
 using MediaBrowser.Model.IO;
 using MediaBrowser.Model.Tasks;
 using Microsoft.Extensions.Logging;
+using Shokofin.Configuration;
 using Shokofin.Utils;
 
 namespace Shokofin.Tasks;
 
 /// <summary>
 /// Clean-up any old VFS roots leftover from an outdated install or failed removal of the roots.
 /// </summary>
-public class CleanupVirtualRootTask(ILogger<CleanupVirtualRootTask> logger, ILibraryManager libraryManager, IFileSystem fileSystem, LibraryScanWatcher scanWatcher) : IScheduledTask, IConfigurableScheduledTask
+public class CleanupVirtualRootTask(ILogger<CleanupVirtualRootTask> logger, ILibraryManager libraryManager, IFileSystem fileSystem, MediaFolderConfigurationService configurationService, LibraryScanWatcher scanWatcher) : IScheduledTask, IConfigurableScheduledTask
 {
     /// <inheritdoc />
     public string Name => "Clean-up Virtual File System Roots";
@@ -45,6 +46,8 @@ public class CleanupVirtualRootTask(ILogger<CleanupVirtualRootTask> logger, ILib
 
     private readonly IFileSystem FileSystem = fileSystem;
 
+    private readonly MediaFolderConfigurationService ConfigurationService = configurationService;
+
     private readonly LibraryScanWatcher ScanWatcher = scanWatcher;
 
     public IEnumerable<TaskTriggerInfo> GetDefaultTriggers()
@@ -54,11 +57,14 @@ public IEnumerable<TaskTriggerInfo> GetDefaultTriggers()
             },
         ];
 
-    public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken)
+    public async Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken)
     {
         if (ScanWatcher.IsScanRunning)
-            return Task.CompletedTask;
+            return;
 
+        var mediaFolders = (await ConfigurationService.GetAvailableMediaFoldersForLibraries().ConfigureAwait(false))
+            .SelectMany(x => x.mediaList)
+            .ToList();
         var start = DateTime.Now;
         var virtualRoots = Plugin.Instance.AllVirtualRoots
             .Except([Plugin.Instance.VirtualRoot])
@@ -73,7 +79,7 @@ public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellat
             Logger.LogTrace("Removed VFS root {Path} in {TimeSpan}.", virtualRoot, perFolderDeltaTime);
         }
 
-        var libraryIds = Plugin.Instance.Configuration.MediaFolders.ToList()
+        var libraryIds = mediaFolders.ToList()
             .Select(config => config.LibraryId.ToString())
             .Distinct()
             .ToList();
@@ -96,7 +102,7 @@ public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellat
             start = DateTime.Now;
             var addedCount = 0;
             var fixedCount = 0;
-            var vfsPaths = Plugin.Instance.Configuration.MediaFolders
+            var vfsPaths = mediaFolders
                 .DistinctBy(config => config.LibraryId)
                 .Select(config => LibraryManager.GetItemById(config.LibraryId) as Folder)
                 .Where(folder => folder is not null)
@@ -123,7 +129,5 @@ public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellat
             deltaTime = DateTime.Now - start;
             Logger.LogDebug("Added {AddedCount} missing and fixed {FixedCount} broken VFS roots in {TimeSpan}.", addedCount, fixedCount, deltaTime);
         }
-
-        return Task.CompletedTask;
     }
 }