diff --git a/README.md b/README.md index 50dc50b5..88dc9788 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,15 @@ GitHub repository, or you can build it from source. Below is a version compatibility matrix for which version of Shokofin is compatible with what. -| Shokofin | Jellyfin | Shoko Server | -|------------|----------|---------------| -| `0.x.x` | `10.7` | `4.0.0-4.1.2` | -| `1.x.x` | `10.7` | `4.1.0-4.1.2` | -| `2.x.x` | `10.8` | `4.1.2` | -| `3.x.x` | `10.8` | `4.2.0` | -| `4.x.x` | `10.9` | `4.2.2` | -| `dev` | `10.9` | `dev` | +| Shokofin | Jellyfin | Shoko Server | +|-------------------|----------|---------------| +| `0.x.x` | `10.7` | `4.0.0` — `4.1.2` | +| `1.x.x` | `10.7` | `4.1.0` — `4.1.2` | +| `2.x.x` | `10.8` | `4.1.2` | +| `3.x.x` | `10.8` | `4.2.0` | +| `4.0.0` — `4.1.1` | `10.9` | `4.2.2` | +| `4.2.0` — `4.x.x` | `10.9` | `4.2.2` — `5.0.0` | +| `dev` | `10.9` | `dev` | ### Official Repository diff --git a/Shokofin/API/Models/CrossReference.cs b/Shokofin/API/Models/CrossReference.cs index 244bea8b..8632e360 100644 --- a/Shokofin/API/Models/CrossReference.cs +++ b/Shokofin/API/Models/CrossReference.cs @@ -35,6 +35,16 @@ public class EpisodeCrossReferenceIDs public int? ReleaseGroup { get; set; } + /// + /// ED2K hash. + /// + public string ED2K { get; set; } = string.Empty; + + /// + /// File size. + /// + public long FileSize { get; set; } + /// /// Percentage file is matched to the episode. /// diff --git a/Shokofin/API/ShokoAPIClient.cs b/Shokofin/API/ShokoAPIClient.cs index 0d556b06..138c993d 100644 --- a/Shokofin/API/ShokoAPIClient.cs +++ b/Shokofin/API/ShokoAPIClient.cs @@ -276,6 +276,11 @@ public Task GetFile(string id) return Get($"/api/v3/File/{id}?include=XRefs&includeDataFrom=AniDB"); } + public Task GetFileByEd2kAndFileSize(string ed2k, long fileSize) + { + return Get($"/api/v3/File/Hash/ED2K?hash={Uri.EscapeDataString(ed2k)}&size={fileSize}&includeDataFrom=AniDB"); + } + public Task> GetFileByPath(string path) { return Get>($"/api/v3/File/PathEndsWith?path={Uri.EscapeDataString(path)}&includeDataFrom=AniDB&limit=1"); diff --git a/Shokofin/Configuration/MediaFolderConfigurationService.cs b/Shokofin/Configuration/MediaFolderConfigurationService.cs index 1f8a41d6..7994a5d2 100644 --- a/Shokofin/Configuration/MediaFolderConfigurationService.cs +++ b/Shokofin/Configuration/MediaFolderConfigurationService.cs @@ -209,11 +209,17 @@ private async void OnLibraryManagerItemRemoved(object? sender, ItemChangeEventAr #region Media Folder Mapping - public IReadOnlyList<(string vfsPath, string mainMediaFolderPath, CollectionType? collectionType, IReadOnlyList mediaList)> GetAvailableMediaFoldersForLibrariesForEvents(Func? filter = null) + public async Task mediaList)>> GetAvailableMediaFoldersForLibraries(Func? filter = null) { - LockObj.Wait(); + await LockObj.WaitAsync(); try { var virtualFolders = LibraryManager.GetVirtualFolders(); + if (ShouldGenerateAllConfigurations) + { + ShouldGenerateAllConfigurations = false; + await GenerateAllConfigurations(virtualFolders).ConfigureAwait(false); + } + var attachRoot = Plugin.Instance.Configuration.VFS_AttachRoot; return Plugin.Instance.Configuration.MediaFolders .Where(config => config.IsMapped && !config.IsVirtualRoot && (filter is null || filter(config)) && LibraryManager.GetItemById(config.MediaFolderId) is Folder) @@ -312,7 +318,7 @@ private async Task GenerateAllConfigurations(List allVirtualF foreach (var mediaFolderPath in virtualFolder.Locations) { if (LibraryManager.FindByPath(mediaFolderPath, true) is not Folder secondFolder) { - Logger.LogTrace("Unable to find database entry for {Path}", mediaFolderPath); + Logger.LogTrace("Unable to find database entry for {Path} (Library={LibraryId})", mediaFolderPath, libraryId); continue; } @@ -353,6 +359,15 @@ private async Task GenerateAllConfigurations(List allVirtualF edits.remove.Add(location); } } + + var mediaFoldersToRemove = config.MediaFolders + .Where(c => !filteredVirtualFolders.Any(v => Guid.Parse(v.ItemId) == c.LibraryId)) + .ToList(); + Logger.LogDebug("Found {Count} out of {TotalCount} media folders to remove.", mediaFoldersToRemove.Count, config.MediaFolders.Count); + foreach (var mediaFolder in mediaFoldersToRemove) { + Logger.LogTrace("Removing config for media folder at path {Path} (Library={LibraryId})", mediaFolder.MediaFolderPath, mediaFolder.LibraryId); + config.MediaFolders.Remove(mediaFolder); + } } private async Task CreateConfigurationForPath(Guid libraryId, Folder mediaFolder, MediaFolderConfiguration? libraryConfig) diff --git a/Shokofin/Events/EventDispatchService.cs b/Shokofin/Events/EventDispatchService.cs index d9199f60..91845b7b 100644 --- a/Shokofin/Events/EventDispatchService.cs +++ b/Shokofin/Events/EventDispatchService.cs @@ -211,7 +211,7 @@ private async Task ProcessFileEvents(int fileId, List<(UpdateReason Reason, int var locationsToNotify = new List(); var mediaFoldersToNotify = new Dictionary(); var seriesIds = await GetSeriesIdsForFile(fileId, changes.Select(t => t.Event).LastOrDefault(e => e.HasCrossReferences)); - var libraries = ConfigurationService.GetAvailableMediaFoldersForLibrariesForEvents(c => c.IsFileEventsEnabled); + var libraries = await ConfigurationService.GetAvailableMediaFoldersForLibraries(c => c.IsFileEventsEnabled).ConfigureAwait(false); var (reason, importFolderId, relativePath, lastEvent) = changes.Last(); if (reason is not UpdateReason.Removed) { Logger.LogTrace("Processing file changed. (File={FileId})", fileId); @@ -281,6 +281,19 @@ private async Task ProcessFileEvents(int fileId, List<(UpdateReason Reason, int } // Something was removed, so assume the location is gone. else if (changes.FirstOrDefault(t => t.Reason is UpdateReason.Removed).Event is IFileEventArgs firstRemovedEvent) { + // If we don't know which series to remove, then add all of them to be scanned. + if (seriesIds.Count is 0) { + Logger.LogTrace("No series found for file. Adding all libraries. (File={FileId})", fileId); + foreach (var (vfsPath, mainMediaFolderPath, collectionType, mediaConfigs) in libraries) { + // Give the core logic _any_ file or folder placed directly in the media folder, so it will schedule the media folder to be refreshed. + var fileOrFolder = FileSystem.GetFileSystemEntryPaths(mainMediaFolderPath, false).FirstOrDefault(); + if (!string.IsNullOrEmpty(fileOrFolder)) + mediaFoldersToNotify.TryAdd(mainMediaFolderPath, (fileOrFolder, mainMediaFolderPath.GetFolderForPath())); + } + + goto aLabelToReduceNesting; + } + Logger.LogTrace("Processing file removed. (File={FileId})", fileId); relativePath = firstRemovedEvent.RelativePath; importFolderId = firstRemovedEvent.ImportFolderId; @@ -351,6 +364,7 @@ private async Task ProcessFileEvents(int fileId, List<(UpdateReason Reason, int } } + aLabelToReduceNesting:; if (LibraryScanWatcher.IsScanRunning) { Logger.LogDebug("Skipped notifying Jellyfin about {LocationCount} changes because a library scan is running. (File={FileId})", locationsToNotify.Count, fileId.ToString()); return; @@ -362,6 +376,7 @@ private async Task ProcessFileEvents(int fileId, List<(UpdateReason Reason, int LibraryMonitor.ReportFileSystemChanged(location); if (mediaFoldersToNotify.Count > 0) await Task.WhenAll(mediaFoldersToNotify.Values.Select(tuple => ReportMediaFolderChanged(tuple.mediaFolder, tuple.pathToReport))).ConfigureAwait(false); + Logger.LogDebug("Notified Jellyfin about {LocationCount} changes. (File={FileId})", locationsToNotify.Count + mediaFoldersToNotify.Count, fileId.ToString()); } catch (Exception ex) { Logger.LogError(ex, "Error processing {EventCount} file change events. (File={FileId})", changes.Count, fileId); @@ -397,15 +412,21 @@ private async Task> GetSeriesIdsForFile(int fileId, IFileEv var filteredSeriesIds = new HashSet(); foreach (var seriesId in seriesIds) { - var (primaryId, extraIds) = await ApiManager.GetSeriesIdsForSeason(seriesId); - var seriesPathSet = await ApiManager.GetPathSetForSeries(primaryId, extraIds); - if (seriesPathSet.Count > 0) { - filteredSeriesIds.Add(seriesId); + try { + var (primaryId, extraIds) = await ApiManager.GetSeriesIdsForSeason(seriesId); + var seriesPathSet = await ApiManager.GetPathSetForSeries(primaryId, extraIds); + if (seriesPathSet.Count > 0) { + filteredSeriesIds.Add(seriesId); + } + } + // If we fail to find the series data (most likely because it's already gone) then just abort early. We'll handle it elsewhere. + catch (ApiException ex) when (ex.StatusCode is System.Net.HttpStatusCode.NotFound) { + return new HashSet(); } } // Return all series if we only have this file for all of them, - // otherwise return only the series were we have other files that are + // otherwise return only the series where we have other files that are // not linked to other series. return filteredSeriesIds.Count is 0 ? seriesIds : filteredSeriesIds; } @@ -434,7 +455,7 @@ private async Task> GetSeriesIdsForFile(int fileId, IFileEv private void RemoveSymbolicLink(string filePath) { - // TODO: If this works better, the move it to an utility and also use it in the VFS if needed, or remove this comment if it's not needed. + // TODO: If this works better, then move it to an utility and also use it in the VFS if needed, or remove this comment if that's not needed. try { var fileExists = File.Exists(filePath); var fileInfo = new System.IO.FileInfo(filePath); diff --git a/Shokofin/Resolvers/VirtualFileSystemService.cs b/Shokofin/Resolvers/VirtualFileSystemService.cs index 7978461c..3f5ee189 100644 --- a/Shokofin/Resolvers/VirtualFileSystemService.cs +++ b/Shokofin/Resolvers/VirtualFileSystemService.cs @@ -793,9 +793,8 @@ await Task.WhenAll(allFiles.Select(async (tuple) => { ExtraType.Sample => ["samples"], _ => ["extras"], }; - var filePartSuffix = (episodeXref.Percentage?.Group ?? 1) is not 1 - ? $".pt{episode.Shoko.CrossReferences.Where(xref => xref.ReleaseGroup == episodeXref.ReleaseGroup && xref.Percentage!.Group == episodeXref.Percentage!.Group).ToList().FindIndex(xref => xref.Percentage!.Start == episodeXref.Percentage!.Start && xref.Percentage!.End == episodeXref.Percentage!.End) + 1}" - : ""; + var fileIdList = fileId; + var filePartSuffix = ""; if (collectionType is CollectionType.movies || (collectionType is null && isMovieSeason)) { if (extrasFolders != null) { foreach (var extrasFolder in extrasFolders) @@ -823,7 +822,14 @@ await Task.WhenAll(allFiles.Select(async (tuple) => { } else { folders.Add(Path.Join(vfsPath, showFolder, seasonFolder)); - episodeName = $"{showName} S{(isSpecial ? 0 : seasonNumber).ToString().PadLeft(2, '0')}E{episodeNumber.ToString().PadLeft(show.EpisodePadding, '0')}{filePartSuffix}"; + episodeName = $"{showName} S{(isSpecial ? 0 : seasonNumber).ToString().PadLeft(2, '0')}E{episodeNumber.ToString().PadLeft(show.EpisodePadding, '0')}"; + if ((episodeXref.Percentage?.Group ?? 1) is not 1) { + var list = episode.Shoko.CrossReferences.Where(xref => xref.ReleaseGroup == episodeXref.ReleaseGroup && xref.Percentage!.Group == episodeXref.Percentage!.Group).ToList(); + var files = await Task.WhenAll(list.Select(xref => ApiClient.GetFileByEd2kAndFileSize(xref.ED2K, xref.FileSize))); + var index = list.FindIndex(xref => xref.Percentage!.Start == episodeXref.Percentage!.Start && xref.Percentage!.End == episodeXref.Percentage!.End); + filePartSuffix = $".pt{index + 1}"; + fileIdList = files.Select(f => f.Id.ToString()).Join(","); + } } } @@ -840,7 +846,7 @@ file.Shoko.AniDBData is not null ); if (config.VFS_AddResolution && !string.IsNullOrEmpty(file.Shoko.Resolution)) extraDetails.Add(file.Shoko.Resolution); - var fileName = $"{episodeName} {(extraDetails.Count is > 0 ? $"[{extraDetails.Select(a => a.ReplaceInvalidPathCharacters()).Join("] [")}] " : "")}[{ShokoSeriesId.Name}={seriesId}] [{ShokoFileId.Name}={fileId}]{Path.GetExtension(sourceLocation)}"; + var fileName = $"{episodeName} {(extraDetails.Count is > 0 ? $"[{extraDetails.Select(a => a.ReplaceInvalidPathCharacters()).Join("] [")}] " : "")}[{ShokoSeriesId.Name}={seriesId}] [{ShokoFileId.Name}={fileIdList}]{filePartSuffix}{Path.GetExtension(sourceLocation)}"; var symbolicLinks = folders .Select(folderPath => Path.Join(folderPath, fileName)) .ToArray(); @@ -1001,6 +1007,12 @@ private List FindSubtitlesForPath(string sourcePath) private LinkGenerationResult CleanupStructure(string vfsPath, string directoryToClean, IReadOnlyList allKnownPaths, bool preview = false) { + if (!FileSystem.DirectoryExists(directoryToClean)) { + if (!preview) + Logger.LogDebug("Skipped cleaning up folder because it does not exist: {Path}", directoryToClean); + return new(); + } + if (!preview) Logger.LogDebug("Looking for files to remove in folder at {Path}", directoryToClean); var start = DateTime.Now; diff --git a/Shokofin/StringExtensions.cs b/Shokofin/StringExtensions.cs index 5494e58f..1f21d7ee 100644 --- a/Shokofin/StringExtensions.cs +++ b/Shokofin/StringExtensions.cs @@ -4,10 +4,11 @@ using System.Linq; using System.Text.RegularExpressions; using MediaBrowser.Common.Providers; +using Shokofin.ExternalIds; namespace Shokofin; -public static class StringExtensions +public static partial class StringExtensions { public static string Replace(this string input, Regex regex, string replacement, int count, int startAt) => regex.Replace(input, replacement, count, startAt); @@ -152,6 +153,21 @@ public static string ReplaceInvalidPathCharacters(this string path) return null; } + [GeneratedRegex(@"\.pt(?\d+)(?:\.[a-z0-9]+)?$", RegexOptions.IgnoreCase)] + private static partial Regex GetPartRegex(); + public static bool TryGetAttributeValue(this string text, string attribute, [NotNullWhen(true)] out string? value) - => !string.IsNullOrEmpty(value = GetAttributeValue(text, attribute)); + { + value = GetAttributeValue(text, attribute); + + // Select the correct id for the part number in the stringified list of file ids. + if (!string.IsNullOrEmpty(value) && attribute == ShokoFileId.Name && GetPartRegex().Match(text) is { Success: true } regexResult) { + var partNumber = int.Parse(regexResult.Groups["partNumber"].Value); + var index = partNumber - 1; + value = value.Split(',')[index]; + } + + return !string.IsNullOrEmpty(value); + } + } \ No newline at end of file diff --git a/Shokofin/Tasks/CleanupVirtualRootTask.cs b/Shokofin/Tasks/CleanupVirtualRootTask.cs index d3bbf4e9..0f083837 100644 --- a/Shokofin/Tasks/CleanupVirtualRootTask.cs +++ b/Shokofin/Tasks/CleanupVirtualRootTask.cs @@ -9,6 +9,7 @@ using MediaBrowser.Model.IO; using MediaBrowser.Model.Tasks; using Microsoft.Extensions.Logging; +using Shokofin.Configuration; using Shokofin.Utils; namespace Shokofin.Tasks; @@ -16,7 +17,7 @@ namespace Shokofin.Tasks; /// /// Clean-up any old VFS roots leftover from an outdated install or failed removal of the roots. /// -public class CleanupVirtualRootTask(ILogger logger, ILibraryManager libraryManager, IFileSystem fileSystem, LibraryScanWatcher scanWatcher) : IScheduledTask, IConfigurableScheduledTask +public class CleanupVirtualRootTask(ILogger logger, ILibraryManager libraryManager, IFileSystem fileSystem, MediaFolderConfigurationService configurationService, LibraryScanWatcher scanWatcher) : IScheduledTask, IConfigurableScheduledTask { /// public string Name => "Clean-up Virtual File System Roots"; @@ -45,6 +46,8 @@ public class CleanupVirtualRootTask(ILogger logger, ILib private readonly IFileSystem FileSystem = fileSystem; + private readonly MediaFolderConfigurationService ConfigurationService = configurationService; + private readonly LibraryScanWatcher ScanWatcher = scanWatcher; public IEnumerable GetDefaultTriggers() @@ -54,11 +57,14 @@ public IEnumerable GetDefaultTriggers() }, ]; - public Task ExecuteAsync(IProgress progress, CancellationToken cancellationToken) + public async Task ExecuteAsync(IProgress progress, CancellationToken cancellationToken) { if (ScanWatcher.IsScanRunning) - return Task.CompletedTask; + return; + var mediaFolders = (await ConfigurationService.GetAvailableMediaFoldersForLibraries().ConfigureAwait(false)) + .SelectMany(x => x.mediaList) + .ToList(); var start = DateTime.Now; var virtualRoots = Plugin.Instance.AllVirtualRoots .Except([Plugin.Instance.VirtualRoot]) @@ -73,7 +79,7 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat Logger.LogTrace("Removed VFS root {Path} in {TimeSpan}.", virtualRoot, perFolderDeltaTime); } - var libraryIds = Plugin.Instance.Configuration.MediaFolders.ToList() + var libraryIds = mediaFolders.ToList() .Select(config => config.LibraryId.ToString()) .Distinct() .ToList(); @@ -96,7 +102,7 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat start = DateTime.Now; var addedCount = 0; var fixedCount = 0; - var vfsPaths = Plugin.Instance.Configuration.MediaFolders + var vfsPaths = mediaFolders .DistinctBy(config => config.LibraryId) .Select(config => LibraryManager.GetItemById(config.LibraryId) as Folder) .Where(folder => folder is not null) @@ -123,7 +129,5 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat deltaTime = DateTime.Now - start; Logger.LogDebug("Added {AddedCount} missing and fixed {FixedCount} broken VFS roots in {TimeSpan}.", addedCount, fixedCount, deltaTime); } - - return Task.CompletedTask; } }