From 1f19d95419a82a69194e4fe01fdf5ab09eda7139 Mon Sep 17 00:00:00 2001 From: Mikal Stordal Date: Sat, 23 Nov 2024 19:31:12 +0100 Subject: [PATCH] refactor: purge TMDB people before scheduling entities to be updated - Purge the TMDB people before we schedule any related shows or movies to be updated. - Update some variable names to better define what they are for. - Remove excess whitespaces and/or newlines. - Always throw an aggregate exception if we encountered any errors while processing the episodes. --- .../Providers/TMDB/TmdbMetadataService.cs | 195 +++++++++--------- 1 file changed, 101 insertions(+), 94 deletions(-) diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs index 16b4354e5..279b43b20 100644 --- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs @@ -615,7 +615,7 @@ private async Task UpdateMovieCastAndCrew(TMDB_Movie tmdbMovie, MovieCredi .ToHashSet(); foreach (var personId in peopleToKeep) { - var (added, updated) = await UpdatePerson(personId, forceRefresh, downloadImages); + var (added, updated) = await UpdatePerson(personId, forceRefresh, downloadImages, currentMovieId: tmdbMovie.Id); if (added) peopleAdded++; if (updated) @@ -1046,8 +1046,8 @@ public async Task UpdateShow(int showId, bool forceRefresh = false, bool d var episodesToSkip = new ConcurrentBag(); var episodesToSave = new ConcurrentBag(); var episodeEventsToEmit = new Dictionary(); - var allPeopleToCheck = new ConcurrentBag(); - var allPeopleToRemove = new ConcurrentBag(); + var allPeopleToAddOrKeep = new ConcurrentBag(); + var allPeopleToPotentiallyRemove = new ConcurrentBag(); foreach (var reducedSeason in show.Seasons) { _logger.LogDebug("Checking season {SeasonNumber} for show {ShowTitle} (Show={ShowId})", reducedSeason.SeasonNumber, show.Name, show.Id); @@ -1104,12 +1104,12 @@ await ProcessWithConcurrencyAsync(_maxConcurrency, season.Episodes, async (reduc // Update crew & cast. if (downloadCrewAndCast) { - var (castOrCrewUpdated, peopleToCheck, peopleToRemove) = UpdateEpisodeCastAndCrew(tmdbEpisode, episode.Credits); + var (castOrCrewUpdated, peopleToAddOrKeep, peopleToPotentiallyRemove) = UpdateEpisodeCastAndCrew(tmdbEpisode, episode.Credits); episodeUpdated |= castOrCrewUpdated; - foreach (var personId in peopleToCheck) - allPeopleToCheck.Add(personId); - foreach (var personId in peopleToRemove) - allPeopleToRemove.Add(personId); + foreach (var personId in peopleToAddOrKeep) + allPeopleToAddOrKeep.Add(personId); + foreach (var personId in peopleToPotentiallyRemove) + allPeopleToPotentiallyRemove.Add(personId); } } @@ -1175,11 +1175,11 @@ await ProcessWithConcurrencyAsync(_maxConcurrency, season.Episodes, async (reduc var peopleAdded = 0; var peopleUpdated = 0; var peoplePurged = 0; - var peopleToCheck = allPeopleToCheck.ToArray().Distinct().ToList(); - var peopleToPurge = allPeopleToRemove.ToArray().Distinct().Except(peopleToCheck).ToList(); + var peopleToCheck = allPeopleToAddOrKeep.ToArray().Distinct().ToList(); + var peopleToPurge = allPeopleToPotentiallyRemove.ToArray().Distinct().Except(peopleToCheck).ToList(); foreach (var personId in peopleToCheck) { - var (added, updated) = await UpdatePerson(personId, forceRefresh, downloadImages); + var (added, updated) = await UpdatePerson(personId, forceRefresh, downloadImages, currentShowId: show.Id); if (added) peopleAdded++; if (updated) @@ -1344,9 +1344,7 @@ private async Task UpdateShowAlternateOrdering(TvShow show) private (bool, IEnumerable, IEnumerable) UpdateEpisodeCastAndCrew(TMDB_Episode tmdbEpisode, CreditsWithGuestStars credits) { - var peopleToAdd = new HashSet(); - var knownPeopleDict = new Dictionary(); - + var peopleToAddOrKeep = new HashSet(); var counter = 0; var castToAdd = 0; var castToKeep = new HashSet(); @@ -1359,7 +1357,7 @@ private async Task UpdateShowAlternateOrdering(TvShow show) var ordering = counter++; var isGuestRole = ordering >= guestOffset; castToKeep.Add(cast.CreditId); - peopleToAdd.Add(cast.Id); + peopleToAddOrKeep.Add(cast.Id); var roleUpdated = false; if (!existingCastDict.TryGetValue(cast.CreditId, out var role)) @@ -1409,7 +1407,7 @@ private async Task UpdateShowAlternateOrdering(TvShow show) .ToDictionary(crew => crew.TmdbCreditID); foreach (var crew in credits.Crew) { - peopleToAdd.Add(crew.Id); + peopleToAddOrKeep.Add(crew.Id); crewToKeep.Add(crew.CreditId); var roleUpdated = false; if (!existingCrewDict.TryGetValue(crew.CreditId, out var role)) @@ -1456,11 +1454,10 @@ private async Task UpdateShowAlternateOrdering(TvShow show) _tmdbEpisodeCast.Delete(castToRemove); _tmdbEpisodeCrew.Delete(crewToRemove); - var peopleToCheck = existingCastDict.Values - .Select(cast => cast.TmdbPersonID) - .Concat(existingCrewDict.Values.Select(crew => crew.TmdbPersonID)) - .Except(knownPeopleDict.Keys) - .ToHashSet(); + var peopleToPotentiallyRemove = new HashSet([ + ..existingCastDict.Values.Select(cast => cast.TmdbPersonID), + ..existingCrewDict.Values.Select(crew => crew.TmdbPersonID), + ]); _logger.LogDebug( "Added/updated/removed/skipped {aa}/{au}/{ar}/{as} cast and {ra}/{ru}/{rr}/{rs} crew for episode {EpisodeTitle} (Show={ShowId},Season={SeasonId},Episode={EpisodeId})", @@ -1482,8 +1479,8 @@ private async Task UpdateShowAlternateOrdering(TvShow show) castToRemove.Count > 0 || crewToSave.Count > 0 || crewToRemove.Count > 0, - peopleToAdd, - peopleToCheck + peopleToAddOrKeep, + peopleToPotentiallyRemove ); } @@ -2036,40 +2033,37 @@ public async Task RepairMissingPeople() var missingIds = new HashSet(); var updateCount = 0; var skippedCount = 0; - var peopleIds = _tmdbPeople.GetAll().Select(person => person.TmdbPersonID).ToHashSet(); - foreach (var person in _tmdbEpisodeCast.GetAll()) if (!peopleIds.Contains(person.TmdbPersonID)) missingIds.Add(person.TmdbPersonID); foreach (var person in _tmdbEpisodeCrew.GetAll()) if (!peopleIds.Contains(person.TmdbPersonID)) missingIds.Add(person.TmdbPersonID); - + foreach (var person in _tmdbMovieCast.GetAll()) if (!peopleIds.Contains(person.TmdbPersonID)) missingIds.Add(person.TmdbPersonID); foreach (var person in _tmdbMovieCrew.GetAll()) if (!peopleIds.Contains(person.TmdbPersonID)) missingIds.Add(person.TmdbPersonID); - _logger.LogDebug("Found {@Count} unique missing TMDB People for Episode & Movie staff", missingIds.Count); - + _logger.LogDebug("Found {Count} unique missing TMDB People for Episode & Movie staff", missingIds.Count); foreach (var personId in missingIds) { var (_, updated) = await UpdatePerson(personId, forceRefresh: true); - if (updated) + if (updated) updateCount++; else skippedCount++; } - - _logger.LogInformation("Updated missing TMDB People: Found/Updated/Skipped {@Found}/{@Updated}/{@Skipped}", + + _logger.LogInformation("Updated missing TMDB People: Found/Updated/Skipped {Found}/{Updated}/{Skipped}", missingIds.Count, updateCount, skippedCount); } - - public async Task<(bool added, bool updated)> UpdatePerson(int personId, bool forceRefresh = false, bool downloadImages = false) + + public async Task<(bool added, bool updated)> UpdatePerson(int personId, bool forceRefresh = false, bool downloadImages = false, int? currentShowId = null, int? currentMovieId = null) { using (await GetLockForEntity(ForeignEntityType.Person, personId, "metadata & images", "Update").ConfigureAwait(false)) { var tmdbPerson = _tmdbPeople.GetByTmdbPersonID(personId) ?? new(personId); - if (!forceRefresh && tmdbPerson.LastUpdatedAt > DateTime.UtcNow.AddMinutes(-15)) + if (!forceRefresh && tmdbPerson.TMDB_PersonID is not 0 && tmdbPerson.LastUpdatedAt > DateTime.Now.AddHours(-1)) { _logger.LogDebug("Skipping update for staff. (Person={PersonId})", personId); return (false, false); @@ -2083,10 +2077,11 @@ public async Task RepairMissingPeople() var person = await UseClient(c => c.GetPersonAsync(personId, methods), $"Get person {personId}"); if (person is null) { - _logger.LogDebug("Unable to update staff; Scheduling refresh of related links. (Person={PersonId})", personId); - await ScheduleUpdateMoviesAndShowsByPerson(personId); - return (false, false); + _logger.LogWarning("Failed to find staff at remote. Purging local records and scheduling shows/movies to-be forcefully updated. (Person={PersonId})", personId); + await PurgePersonInternal(personId, currentShowId: currentShowId, currentMovieId: currentMovieId); + return (false, !newlyAdded); } + var updated = tmdbPerson.Populate(person); if (updated) { @@ -2101,28 +2096,6 @@ public async Task RepairMissingPeople() } } - private async Task ScheduleUpdateMoviesAndShowsByPerson(int tmdbPersonId) - { - var showIds = new HashSet(); - var movieIds = new HashSet(); - - foreach (var staff in _tmdbEpisodeCast.GetByTmdbPersonID(tmdbPersonId)) - showIds.Add(staff.TmdbShowID); - foreach (var staff in _tmdbEpisodeCrew.GetByTmdbPersonID(tmdbPersonId)) - showIds.Add(staff.TmdbShowID); - - foreach (var staff in _tmdbMovieCast.GetByTmdbPersonID(tmdbPersonId)) - movieIds.Add(staff.TmdbMovieID); - foreach (var staff in _tmdbMovieCrew.GetByTmdbPersonID(tmdbPersonId)) - movieIds.Add(staff.TmdbMovieID); - - foreach (var showId in showIds) - await ScheduleUpdateOfShow(showId, downloadCrewAndCast: true, forceRefresh: true); - - foreach (var movieId in movieIds) - await ScheduleUpdateOfMovie(movieId, downloadCrewAndCast: true, forceRefresh: true); - } - private async Task DownloadPersonImages(int personId, ProfileImages images, bool forceDownload = false) { var settings = _settingsProvider.GetSettings(); @@ -2140,47 +2113,78 @@ public async Task PurgePerson(int personId, bool removeImageFiles = true) if (IsPersonLinkedToOtherEntities(personId)) return false; - var person = _tmdbPeople.GetByTmdbPersonID(personId); - if (person != null) - { - _logger.LogDebug("Removing staff. (Person={PersonId})", personId); - _tmdbPeople.Delete(person); - } + await PurgePersonInternal(personId, removeImageFiles).ConfigureAwait(false); - var images = _tmdbImages.GetByTmdbPersonID(personId); - if (images.Count > 0) - foreach (var image in images) - _imageService.PurgeImage(image, ForeignEntityType.Person, removeImageFiles); + return true; + } + } - var movieCast = _tmdbMovieCast.GetByTmdbPersonID(personId); - if (movieCast.Count > 0) - { - _logger.LogDebug("Removing {count} movie cast roles for staff. (Person={PersonId})", movieCast.Count, personId); - _tmdbMovieCast.Delete(movieCast); - } + internal async Task PurgePersonInternal(int personId, bool removeImageFiles = true, int? currentShowId = null, int? currentMovieId = null) + { + var person = _tmdbPeople.GetByTmdbPersonID(personId); + if (person != null) + { + _logger.LogDebug("Removing staff. (Person={PersonId})", personId); + _tmdbPeople.Delete(person); + } - var movieCrew = _tmdbMovieCrew.GetByTmdbPersonID(personId); - if (movieCrew.Count > 0) - { - _logger.LogDebug("Removing {count} movie crew roles for staff. (Person={PersonId})", movieCrew.Count, personId); - _tmdbMovieCrew.Delete(movieCrew); - } + var images = _tmdbImages.GetByTmdbPersonID(personId); + if (images.Count > 0) + foreach (var image in images) + _imageService.PurgeImage(image, ForeignEntityType.Person, removeImageFiles); - var episodeCast = _tmdbEpisodeCast.GetByTmdbPersonID(personId); - if (episodeCast.Count > 0) - { - _logger.LogDebug("Removing {count} show cast roles for staff. (Person={PersonId})", episodeCast.Count, personId); - _tmdbEpisodeCast.Delete(episodeCast); - } + var movieCast = _tmdbMovieCast.GetByTmdbPersonID(personId); + if (movieCast.Count > 0) + { + _logger.LogDebug("Removing {count} movie cast roles for staff. (Person={PersonId})", movieCast.Count, personId); + _tmdbMovieCast.Delete(movieCast); + } - var episodeCrew = _tmdbEpisodeCrew.GetByTmdbPersonID(personId); - if (episodeCrew.Count > 0) - { - _logger.LogDebug("Removing {count} show crew roles for staff. (Person={PersonId})", episodeCrew.Count, personId); - _tmdbEpisodeCrew.Delete(episodeCrew); - } + var movieCrew = _tmdbMovieCrew.GetByTmdbPersonID(personId); + if (movieCrew.Count > 0) + { + _logger.LogDebug("Removing {count} movie crew roles for staff. (Person={PersonId})", movieCrew.Count, personId); + _tmdbMovieCrew.Delete(movieCrew); + } - return true; + var episodeCast = _tmdbEpisodeCast.GetByTmdbPersonID(personId); + if (episodeCast.Count > 0) + { + _logger.LogDebug("Removing {count} show cast roles for staff. (Person={PersonId})", episodeCast.Count, personId); + _tmdbEpisodeCast.Delete(episodeCast); + } + + var episodeCrew = _tmdbEpisodeCrew.GetByTmdbPersonID(personId); + if (episodeCrew.Count > 0) + { + _logger.LogDebug("Removing {count} show crew roles for staff. (Person={PersonId})", episodeCrew.Count, personId); + _tmdbEpisodeCrew.Delete(episodeCrew); + } + + var showIds = new HashSet([ + ..episodeCast.Select(x => x.TmdbShowID), + ..episodeCrew.Select(x => x.TmdbShowID), + ]); + if (currentShowId.HasValue) + showIds.Remove(currentShowId.Value); + if (showIds.Count > 0) + { + _logger.LogDebug("Scheduling {count} shows to be updated. (Person={PersonId})", showIds.Count, personId); + foreach (var showId in showIds) + await ScheduleUpdateOfShow(showId, downloadCrewAndCast: true); + } + + var movieIds = new HashSet([ + ..movieCast.Select(x => x.TmdbMovieID), + ..movieCrew.Select(x => x.TmdbMovieID), + ]); + if (currentMovieId.HasValue) + movieIds.Remove(currentMovieId.Value); + if (movieIds.Count > 0) + { + _logger.LogDebug("Scheduling {count} movies to be updated. (Person={PersonId})", movieIds.Count, personId); + foreach (var movieId in movieIds) + await ScheduleUpdateOfMovie(movieId, downloadCrewAndCast: true); } } @@ -2264,6 +2268,9 @@ Func processAsync continue; } } + + if (exceptions.Count > 0) + throw new AggregateException(exceptions); } #endregion