From 09d4ae0e83050751cf234ba4fd6f4163f7380d39 Mon Sep 17 00:00:00 2001 From: Mikal Stordal Date: Sun, 14 Jul 2024 23:00:46 +0200 Subject: [PATCH 1/3] fix: add catch in try get method - Don't throw in the try get method for retrieving the file id. --- Shokofin/API/ShokoAPIManager.cs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Shokofin/API/ShokoAPIManager.cs b/Shokofin/API/ShokoAPIManager.cs index 70742a36..f8cf95be 100644 --- a/Shokofin/API/ShokoAPIManager.cs +++ b/Shokofin/API/ShokoAPIManager.cs @@ -562,10 +562,15 @@ public bool TryGetFileIdForPath(string path, [NotNullWhen(true)] out string? fil // Slow path; getting the show from cache or remote and finding the default season's id. Logger.LogDebug("Trying to find file id using the slow path. (Path={FullPath})", path); - if (GetFileInfoByPath(path).ConfigureAwait(false).GetAwaiter().GetResult() is { } tuple && tuple.Item1 is not null) { - var (fileInfo, _, _) = tuple; - fileId = fileInfo.Id; - return true; + try { + if (GetFileInfoByPath(path).ConfigureAwait(false).GetAwaiter().GetResult() is { } tuple && tuple.Item1 is not null) { + var (fileInfo, _, _) = tuple; + fileId = fileInfo.Id; + return true; + } + } + catch (Exception ex) { + Logger.LogError(ex, "Encountered an error while trying to lookup the file id for {Path}", path); } fileId = null; From eada3b4208df116441085756ab5b7c88af0fcb68 Mon Sep 17 00:00:00 2001 From: Mikal Stordal Date: Mon, 15 Jul 2024 01:40:16 +0200 Subject: [PATCH 2/3] fix: add episode user data migration task --- Shokofin/Sync/SyncExtensions.cs | 13 ++ Shokofin/Tasks/MigrateEpisodeUserDataTask.cs | 185 +++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 Shokofin/Tasks/MigrateEpisodeUserDataTask.cs diff --git a/Shokofin/Sync/SyncExtensions.cs b/Shokofin/Sync/SyncExtensions.cs index 936ea01d..ee0badac 100644 --- a/Shokofin/Sync/SyncExtensions.cs +++ b/Shokofin/Sync/SyncExtensions.cs @@ -22,6 +22,19 @@ public static File.UserStats ToFileUserStats(this UserItemData userData) }; } + public static void CopyFrom(this UserItemData userData, UserItemData otherUserData) + { + userData.LastPlayedDate = otherUserData.LastPlayedDate; + userData.IsFavorite = otherUserData.IsFavorite; + userData.AudioStreamIndex = otherUserData.AudioStreamIndex; + userData.Likes = otherUserData.Likes; + userData.PlaybackPositionTicks = otherUserData.PlaybackPositionTicks; + userData.PlayCount = otherUserData.PlayCount; + userData.Played = otherUserData.Played; + userData.Rating = otherUserData.Rating; + userData.SubtitleStreamIndex = otherUserData.SubtitleStreamIndex; + } + public static UserItemData MergeWithFileUserStats(this UserItemData userData, File.UserStats userStats) { userData.Played = userStats.LastWatchedAt.HasValue; diff --git a/Shokofin/Tasks/MigrateEpisodeUserDataTask.cs b/Shokofin/Tasks/MigrateEpisodeUserDataTask.cs new file mode 100644 index 00000000..ea49f4eb --- /dev/null +++ b/Shokofin/Tasks/MigrateEpisodeUserDataTask.cs @@ -0,0 +1,185 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Data.Enums; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Tasks; +using Microsoft.Extensions.Logging; +using Shokofin.ExternalIds; +using Shokofin.Providers; +using Shokofin.Sync; + +namespace Shokofin.Tasks; + +public class MigrateEpisodeUserDataTask : IScheduledTask, IConfigurableScheduledTask +{ + /// + public string Name => "Migrate Episode User Watch Data"; + + /// + public string Description => "Migrate user watch data for episodes store in Jellyfin to the newest id namespace."; + + /// + public string Category => "Shokofin"; + + /// + public string Key => "ShokoMigrateEpisodeUserDataTask"; + + /// + public bool IsHidden => false; + + /// + public bool IsEnabled => false; + + /// + public bool IsLogged => true; + + private readonly ILogger Logger; + + private readonly IUserDataManager UserDataManager; + + private readonly IUserManager UserManager; + + private readonly ILibraryManager LibraryManager; + + private readonly IIdLookup Lookup; + + public MigrateEpisodeUserDataTask( + ILogger logger, + IUserDataManager userDataManager, + IUserManager userManager, + ILibraryManager libraryManager, + IIdLookup lookup + ) + { + Logger = logger; + UserDataManager = userDataManager; + UserManager = userManager; + LibraryManager = libraryManager; + Lookup = lookup; + } + + public IEnumerable GetDefaultTriggers() + => Array.Empty(); + + public Task ExecuteAsync(IProgress progress, CancellationToken cancellationToken) + { + var foundEpisodeCount = 0; + var seriesDict = new Dictionary episodes)>(); + var users = UserManager.Users.ToList(); + var allEpisodes = LibraryManager.GetItemList(new InternalItemsQuery { + IncludeItemTypes = [BaseItemKind.Episode], + HasAnyProviderId = new Dictionary { { ShokoFileId.Name, string.Empty } }, + IsFolder = false, + Recursive = true, + DtoOptions = new(false) { + EnableImages = false + }, + SourceTypes = [SourceType.Library], + IsVirtualItem = false, + }) + .OfType() + .ToList(); + Logger.LogInformation("Attempting to migrate user watch data across {EpisodeCount} episodes and {UserCount} users.", allEpisodes.Count, users.Count); + foreach (var episode in allEpisodes) { + cancellationToken.ThrowIfCancellationRequested(); + + if (!episode.ParentIndexNumber.HasValue || !episode.IndexNumber.HasValue || + !Lookup.TryGetFileIdFor(episode, out var fileId) || episode.Series is not Series series) + continue; + if (!seriesDict.TryGetValue(series.Id, out var tuple)) + seriesDict[series.Id] = tuple = (series, []); + + tuple.episodes.Add(episode); + foundEpisodeCount++; + } + + Logger.LogInformation("Found {SeriesCount} series and {EpisodeCount} episodes across {AllEpisodeCount} initial episodes to search for user watch data.", foundEpisodeCount, allEpisodes.Count); + var savedCount = 0; + var numComplete = 0; + var numTotal = foundEpisodeCount * users.Count; + var userDataDict = users.ToDictionary(user => user, user => (UserDataManager.GetAllUserData(user.Id).DistinctBy(data => data.Key).ToDictionary(data => data.Key), new List())); + var userDataToRemove = new List(); + foreach (var (series, episodes) in seriesDict.Values) { + cancellationToken.ThrowIfCancellationRequested(); + + if (!Lookup.TryGetSeriesIdFor(series, out var seriesId)) + continue; + + SeriesProvider.AddProviderIds(series, seriesId); + var seriesUserKeys = series.GetUserDataKeys(); + if (seriesUserKeys.Count > 1) + seriesUserKeys = seriesUserKeys.TakeLast(1).ToList(); + + // 10.9 post-4.1 id format + var primaryKey = seriesUserKeys.First(); + var keysToSearch = seriesUserKeys.Skip(1) + // 10.9 pre-4.1 id format + .Prepend($"shoko://shoko-series={seriesId}") + // 10.8 id format + .Prepend($"INVALID-BUT-DO-NOT-TOUCH:{seriesId}") + .ToList(); + Logger.LogInformation("Migrating user watch data for series {SeriesName}. (Series={SeriesId},Primary={PrimaryKey},Search={SearchKeys})", series.Name, seriesId, primaryKey, keysToSearch); + foreach (var episode in episodes) { + cancellationToken.ThrowIfCancellationRequested(); + + if (!episode.TryGetProviderId(ShokoFileId.Name, out var fileId)) + continue; + + var suffix = episode.ParentIndexNumber!.Value.ToString("000", CultureInfo.InvariantCulture) + episode.IndexNumber!.Value.ToString("000", CultureInfo.InvariantCulture); + var videoUserDataKeys = (episode as Video).GetUserDataKeys(); + var episodeKeyToUse = primaryKey + suffix; + var episodeKeysToSearch = keysToSearch.Select(key => key + suffix).Concat(videoUserDataKeys).ToList(); + Logger.LogInformation("Migrating user watch data for season {SeasonNumber}, episode {EpisodeNumber} - {EpisodeName}. (Series={SeriesId},File={FileId},Primary={PrimaryKey},Search={SearchKeys})", episode.ParentIndexNumber, episode.IndexNumber, episode.Name, seriesId, fileId, episodeKeyToUse, episodeKeysToSearch); + foreach (var (user, (dataDict, dataList)) in userDataDict) { + var userData = UserDataManager.GetUserData(user, episode); + if (dataDict.TryGetValue(episodeKeyToUse, out var primaryUserData)) { + Logger.LogInformation("Found user data to migrate. (Key={SearchKey})", episodeKeyToUse); + userData.CopyFrom(primaryUserData); + dataList.Add(userData); + savedCount++; + } + else { + foreach (var secondaryKey in episodeKeysToSearch) { + if (!dataDict.TryGetValue(episodeKeyToUse, out var secondaryUserData)) + continue; + + Logger.LogInformation("Found user data to migrate. (Key={SearchKey})", secondaryKey); + userData.CopyFrom(secondaryUserData); + dataList.Add(userData); + savedCount++; + break; + } + } + + numComplete++; + double percent = numComplete; + percent /= numTotal; + + progress.Report(percent * 100); + } + } + } + + // Last attempt to cancel before we save all the changes. + cancellationToken.ThrowIfCancellationRequested(); + + Logger.LogInformation("Saving {UserDataCount} user watch data entries across {UserCount} users", savedCount, users.Count); + foreach (var (user, (dataDict, dataList)) in userDataDict) { + if (dataList.Count is 0) + continue; + + UserDataManager.SaveAllUserData(user.Id, dataList.ToArray(), CancellationToken.None); + } + Logger.LogInformation("Saved {UserDataCount} user watch data entries across {UserCount} users", savedCount, users.Count); + + progress.Report(100); + return Task.CompletedTask; + } +} From 4679c51b96bc775b7b43d95df1745eb4b45e184f Mon Sep 17 00:00:00 2001 From: Mikal Stordal Date: Tue, 16 Jul 2024 01:52:08 +0200 Subject: [PATCH 3/3] fix: improve logic in the episode user data migration task --- Shokofin/Sync/SyncExtensions.cs | 69 +++++++++++++++++--- Shokofin/Tasks/MigrateEpisodeUserDataTask.cs | 58 ++++++---------- 2 files changed, 80 insertions(+), 47 deletions(-) diff --git a/Shokofin/Sync/SyncExtensions.cs b/Shokofin/Sync/SyncExtensions.cs index ee0badac..81777bed 100644 --- a/Shokofin/Sync/SyncExtensions.cs +++ b/Shokofin/Sync/SyncExtensions.cs @@ -22,19 +22,68 @@ public static File.UserStats ToFileUserStats(this UserItemData userData) }; } - public static void CopyFrom(this UserItemData userData, UserItemData otherUserData) + public static bool CopyFrom(this UserItemData userData, UserItemData otherUserData) { - userData.LastPlayedDate = otherUserData.LastPlayedDate; - userData.IsFavorite = otherUserData.IsFavorite; - userData.AudioStreamIndex = otherUserData.AudioStreamIndex; - userData.Likes = otherUserData.Likes; - userData.PlaybackPositionTicks = otherUserData.PlaybackPositionTicks; - userData.PlayCount = otherUserData.PlayCount; - userData.Played = otherUserData.Played; - userData.Rating = otherUserData.Rating; - userData.SubtitleStreamIndex = otherUserData.SubtitleStreamIndex; + var updated = false; + + if (!userData.Rating.HasValue && otherUserData.Rating.HasValue || userData.Rating.HasValue && otherUserData.Rating.HasValue && userData.Rating != otherUserData.Rating) + { + userData.Rating = otherUserData.Rating; + updated = true; + } + + if (userData.PlaybackPositionTicks != otherUserData.PlaybackPositionTicks) + { + userData.PlaybackPositionTicks = otherUserData.PlaybackPositionTicks; + updated = true; + } + + if (userData.PlayCount != otherUserData.PlayCount) + { + userData.PlayCount = otherUserData.PlayCount; + updated = true; + } + + if (!userData.IsFavorite != otherUserData.IsFavorite) + { + userData.IsFavorite = otherUserData.IsFavorite; + updated = true; + } + + if (!userData.LastPlayedDate.HasValue && otherUserData.LastPlayedDate.HasValue || userData.LastPlayedDate.HasValue && otherUserData.LastPlayedDate.HasValue && userData.LastPlayedDate < otherUserData.LastPlayedDate) + { + userData.LastPlayedDate = otherUserData.LastPlayedDate; + updated = true; + } + + if (userData.Played != otherUserData.Played) + { + userData.Played = otherUserData.Played; + updated = true; + } + + if (!userData.AudioStreamIndex.HasValue && otherUserData.AudioStreamIndex.HasValue || userData.AudioStreamIndex.HasValue && otherUserData.AudioStreamIndex.HasValue && userData.AudioStreamIndex != otherUserData.AudioStreamIndex) + { + userData.AudioStreamIndex = otherUserData.AudioStreamIndex; + updated = true; + } + + if (!userData.SubtitleStreamIndex.HasValue && otherUserData.SubtitleStreamIndex.HasValue || userData.SubtitleStreamIndex.HasValue && otherUserData.SubtitleStreamIndex.HasValue && userData.SubtitleStreamIndex != otherUserData.SubtitleStreamIndex) + { + userData.SubtitleStreamIndex = otherUserData.SubtitleStreamIndex; + updated = true; + } + + if (!userData.Likes.HasValue && otherUserData.Likes.HasValue || userData.Likes.HasValue && otherUserData.Likes.HasValue && userData.Likes != otherUserData.Likes) + { + userData.Likes = otherUserData.Likes; + updated = true; + } + + return updated; } + public static UserItemData MergeWithFileUserStats(this UserItemData userData, File.UserStats userStats) { userData.Played = userStats.LastWatchedAt.HasValue; diff --git a/Shokofin/Tasks/MigrateEpisodeUserDataTask.cs b/Shokofin/Tasks/MigrateEpisodeUserDataTask.cs index ea49f4eb..0fac08ab 100644 --- a/Shokofin/Tasks/MigrateEpisodeUserDataTask.cs +++ b/Shokofin/Tasks/MigrateEpisodeUserDataTask.cs @@ -48,21 +48,17 @@ public class MigrateEpisodeUserDataTask : IScheduledTask, IConfigurableScheduled private readonly ILibraryManager LibraryManager; - private readonly IIdLookup Lookup; - public MigrateEpisodeUserDataTask( ILogger logger, IUserDataManager userDataManager, IUserManager userManager, - ILibraryManager libraryManager, - IIdLookup lookup + ILibraryManager libraryManager ) { Logger = logger; UserDataManager = userDataManager; UserManager = userManager; LibraryManager = libraryManager; - Lookup = lookup; } public IEnumerable GetDefaultTriggers() @@ -71,7 +67,7 @@ public IEnumerable GetDefaultTriggers() public Task ExecuteAsync(IProgress progress, CancellationToken cancellationToken) { var foundEpisodeCount = 0; - var seriesDict = new Dictionary episodes)>(); + var seriesDict = new Dictionary episodes)>(); var users = UserManager.Users.ToList(); var allEpisodes = LibraryManager.GetItemList(new InternalItemsQuery { IncludeItemTypes = [BaseItemKind.Episode], @@ -86,37 +82,33 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat }) .OfType() .ToList(); - Logger.LogInformation("Attempting to migrate user watch data across {EpisodeCount} episodes and {UserCount} users.", allEpisodes.Count, users.Count); + Logger.LogDebug("Attempting to migrate user watch data across {EpisodeCount} episodes and {UserCount} users.", allEpisodes.Count, users.Count); foreach (var episode in allEpisodes) { cancellationToken.ThrowIfCancellationRequested(); if (!episode.ParentIndexNumber.HasValue || !episode.IndexNumber.HasValue || - !Lookup.TryGetFileIdFor(episode, out var fileId) || episode.Series is not Series series) + !episode.TryGetProviderId(ShokoFileId.Name, out var fileId) || + episode.Series is not Series series || !series.TryGetProviderId(ShokoSeriesId.Name, out var seriesId)) continue; - if (!seriesDict.TryGetValue(series.Id, out var tuple)) - seriesDict[series.Id] = tuple = (series, []); + + if (!seriesDict.TryGetValue(seriesId, out var tuple)) + seriesDict[seriesId] = tuple = (series, []); tuple.episodes.Add(episode); foundEpisodeCount++; } - Logger.LogInformation("Found {SeriesCount} series and {EpisodeCount} episodes across {AllEpisodeCount} initial episodes to search for user watch data.", foundEpisodeCount, allEpisodes.Count); + Logger.LogInformation("Found {SeriesCount} series and {EpisodeCount} episodes across {AllEpisodeCount} total episodes to search for user watch data to migrate.", seriesDict.Count, foundEpisodeCount, allEpisodes.Count); var savedCount = 0; var numComplete = 0; var numTotal = foundEpisodeCount * users.Count; var userDataDict = users.ToDictionary(user => user, user => (UserDataManager.GetAllUserData(user.Id).DistinctBy(data => data.Key).ToDictionary(data => data.Key), new List())); var userDataToRemove = new List(); - foreach (var (series, episodes) in seriesDict.Values) { + foreach (var (seriesId, (series, episodes)) in seriesDict) { cancellationToken.ThrowIfCancellationRequested(); - if (!Lookup.TryGetSeriesIdFor(series, out var seriesId)) - continue; - SeriesProvider.AddProviderIds(series, seriesId); var seriesUserKeys = series.GetUserDataKeys(); - if (seriesUserKeys.Count > 1) - seriesUserKeys = seriesUserKeys.TakeLast(1).ToList(); - // 10.9 post-4.1 id format var primaryKey = seriesUserKeys.First(); var keysToSearch = seriesUserKeys.Skip(1) @@ -125,37 +117,29 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat // 10.8 id format .Prepend($"INVALID-BUT-DO-NOT-TOUCH:{seriesId}") .ToList(); - Logger.LogInformation("Migrating user watch data for series {SeriesName}. (Series={SeriesId},Primary={PrimaryKey},Search={SearchKeys})", series.Name, seriesId, primaryKey, keysToSearch); + Logger.LogTrace("Migrating user watch data for series {SeriesName}. (Series={SeriesId},Primary={PrimaryKey},Search={SearchKeys})", series.Name, seriesId, primaryKey, keysToSearch); foreach (var episode in episodes) { cancellationToken.ThrowIfCancellationRequested(); - + if (!episode.TryGetProviderId(ShokoFileId.Name, out var fileId)) continue; var suffix = episode.ParentIndexNumber!.Value.ToString("000", CultureInfo.InvariantCulture) + episode.IndexNumber!.Value.ToString("000", CultureInfo.InvariantCulture); var videoUserDataKeys = (episode as Video).GetUserDataKeys(); - var episodeKeyToUse = primaryKey + suffix; - var episodeKeysToSearch = keysToSearch.Select(key => key + suffix).Concat(videoUserDataKeys).ToList(); - Logger.LogInformation("Migrating user watch data for season {SeasonNumber}, episode {EpisodeNumber} - {EpisodeName}. (Series={SeriesId},File={FileId},Primary={PrimaryKey},Search={SearchKeys})", episode.ParentIndexNumber, episode.IndexNumber, episode.Name, seriesId, fileId, episodeKeyToUse, episodeKeysToSearch); + var episodeKeysToSearch = keysToSearch.Select(key => key + suffix).Prepend(primaryKey + suffix).Concat(videoUserDataKeys).ToList(); + Logger.LogTrace("Migrating user watch data for season {SeasonNumber}, episode {EpisodeNumber} - {EpisodeName}. (Series={SeriesId},File={FileId},Search={SearchKeys})", episode.ParentIndexNumber, episode.IndexNumber, episode.Name, seriesId, fileId, episodeKeysToSearch); foreach (var (user, (dataDict, dataList)) in userDataDict) { var userData = UserDataManager.GetUserData(user, episode); - if (dataDict.TryGetValue(episodeKeyToUse, out var primaryUserData)) { - Logger.LogInformation("Found user data to migrate. (Key={SearchKey})", episodeKeyToUse); - userData.CopyFrom(primaryUserData); - dataList.Add(userData); - savedCount++; - } - else { - foreach (var secondaryKey in episodeKeysToSearch) { - if (!dataDict.TryGetValue(episodeKeyToUse, out var secondaryUserData)) - continue; + foreach (var searchKey in episodeKeysToSearch) { + if (!dataDict.TryGetValue(searchKey, out var searchUserData)) + continue; - Logger.LogInformation("Found user data to migrate. (Key={SearchKey})", secondaryKey); - userData.CopyFrom(secondaryUserData); + if (userData.CopyFrom(searchUserData)) { + Logger.LogInformation("Found user data to migrate. (Series={SeriesId},File={FileId},Search={SearchKeys},Key={SearchKey},User={UserId})", seriesId, fileId, episodeKeysToSearch, searchKey, user.Id); dataList.Add(userData); savedCount++; - break; } + break; } numComplete++; @@ -170,7 +154,7 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat // Last attempt to cancel before we save all the changes. cancellationToken.ThrowIfCancellationRequested(); - Logger.LogInformation("Saving {UserDataCount} user watch data entries across {UserCount} users", savedCount, users.Count); + Logger.LogDebug("Saving {UserDataCount} user watch data entries across {UserCount} users", savedCount, users.Count); foreach (var (user, (dataDict, dataList)) in userDataDict) { if (dataList.Count is 0) continue;