From b209931f4c570f77203d6c15dd262f4e6b070a82 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:14:17 +1100 Subject: [PATCH] Add caching of playlistitems requests - Partially fixes #545 - Results will be cached for 1 hour - Also re-sorts cached values so they match the original requested order --- .../youtube/helper/resource_manager.py | 431 ++++++++++-------- .../youtube_plugin/youtube/helper/yt_play.py | 129 +++--- .../lib/youtube_plugin/youtube/provider.py | 37 +- 3 files changed, 302 insertions(+), 295 deletions(-) diff --git a/resources/lib/youtube_plugin/youtube/helper/resource_manager.py b/resources/lib/youtube_plugin/youtube/helper/resource_manager.py index 60e3aa4b4..8e29b9ece 100644 --- a/resources/lib/youtube_plugin/youtube/helper/resource_manager.py +++ b/resources/lib/youtube_plugin/youtube/helper/resource_manager.py @@ -10,180 +10,220 @@ from __future__ import absolute_import, division, unicode_literals -from ..youtube_exceptions import YouTubeException -from ...kodion.utils import strip_html_from_text - class ResourceManager(object): def __init__(self, context, client): self._context = context self._client = client - self._channel_data = {} - self._video_data = {} - self._playlist_data = {} - self._enable_channel_fanart = context.get_settings().get_bool('youtube.channel.fanart.show', True) + self._data_cache = context.get_data_cache() + self._func_cache = context.get_function_cache() + self._show_fanart = context.get_settings().get_bool( + 'youtube.channel.fanart.show', True + ) + + @staticmethod + def _list_batch(input_list, n=50): + if not isinstance(input_list, (list, tuple)): + input_list = list(input_list) + for i in range(0, len(input_list), n): + yield input_list[i:i + n] def clear(self): - self._context.get_function_cache().clear() - self._context.get_data_cache().clear() - - def _get_channel_data(self, channel_id): - return self._channel_data.get(channel_id, {}) - - def _get_video_data(self, video_id): - return self._video_data.get(video_id, {}) - - def _get_playlist_data(self, playlist_id): - return self._playlist_data.get(playlist_id, {}) - - def _update_channels(self, channel_ids): - json_data = None - updated_channel_ids = [] - function_cache = self._context.get_function_cache() - - for channel_id in channel_ids: - if channel_id == 'mine': - json_data = function_cache.get(self._client.get_channel_by_username, - function_cache.ONE_DAY, - channel_id) - items = json_data.get('items', [{'id': 'mine'}]) - - try: - channel_id = items[0]['id'] - except IndexError: - self._context.log_debug('Channel "mine" not found: %s' % json_data) - channel_id = None - - json_data = None - - if channel_id: - updated_channel_ids.append(channel_id) - - channel_ids = updated_channel_ids - - data_cache = self._context.get_data_cache() - channel_data = data_cache.get_items(channel_ids, data_cache.ONE_MONTH) - - channel_ids = set(channel_ids) - channel_ids_cached = set(channel_data) - channel_ids_to_update = channel_ids - channel_ids_cached - channel_ids_cached = channel_ids & channel_ids_cached - - result = channel_data - if channel_ids_cached: - self._context.log_debug('Found cached data for channels |%s|' % ', '.join(channel_ids_cached)) - - if channel_ids_to_update: - self._context.log_debug('No data for channels |%s| cached' % ', '.join(channel_ids_to_update)) - json_data = [ - self._client.get_channels(list_of_50) - for list_of_50 in self._list_batch(channel_ids_to_update, n=50) - ] - channel_data = { + self._func_cache.clear() + self._data_cache.clear() + + def get_channels(self, ids): + updated = [] + for channel_id in ids: + if not channel_id: + continue + + if channel_id != 'mine': + updated.append(channel_id) + continue + + data = self._func_cache.get(self._client.get_channel_by_username, + self._func_cache.ONE_DAY, + channel_id) + items = data.get('items', [{'id': 'mine'}]) + + try: + channel_id = items[0]['id'] + updated.append(channel_id) + except IndexError: + self._context.log_error('Channel not found:\n\t{data}' + .format(data=data)) + + ids = updated + result = self._data_cache.get_items(ids, self._data_cache.ONE_MONTH) + to_update = [id_ for id_ in ids if id_ not in result] + + if result: + self._context.log_debug('Found cached data for channels:\n|{ids}|' + .format(ids=list(result))) + + if to_update: + new_data = [self._client.get_channels(list_of_50) + for list_of_50 in self._list_batch(to_update, n=50)] + if not any(new_data): + new_data = None + else: + new_data = None + + if new_data: + self._context.log_debug('Got data for channels:\n|{ids}|' + .format(ids=to_update)) + new_data = { yt_item['id']: yt_item - for batch in json_data + for batch in new_data for yt_item in batch.get('items', []) if yt_item } - result.update(channel_data) - data_cache.set_items(channel_data) - self._context.log_debug('Cached data for channels |%s|' % ', '.join(channel_data)) - - if self.handle_error(json_data): - return result - return {} - - def _update_videos(self, video_ids, live_details=False, suppress_errors=False): - json_data = None - data_cache = self._context.get_data_cache() - video_data = data_cache.get_items(video_ids, data_cache.ONE_MONTH) - - video_ids = set(video_ids) - video_ids_cached = set(video_data) - video_ids_to_update = video_ids - video_ids_cached - video_ids_cached = video_ids & video_ids_cached - - result = video_data - if video_ids_cached: - self._context.log_debug('Found cached data for videos |%s|' % ', '.join(video_ids_cached)) - - if video_ids_to_update: - self._context.log_debug('No data for videos |%s| cached' % ', '.join(video_ids_to_update)) - json_data = self._client.get_videos(video_ids_to_update, live_details) - video_data = dict.fromkeys(video_ids_to_update, {}) - video_data.update({ - yt_item['id']: yt_item or {} - for yt_item in json_data.get('items', []) - }) - result.update(video_data) - data_cache.set_items(video_data) - self._context.log_debug('Cached data for videos |%s|' % ', '.join(video_data)) - - if self._context.get_settings().use_local_history(): - playback_history = self._context.get_playback_history() - played_items = playback_history.get_items(video_ids) - for video_id, play_data in played_items.items(): - result[video_id]['play_data'] = play_data + result.update(new_data) + self._data_cache.set_items(new_data) + self._context.log_debug('Cached data for channels:\n|{ids}|' + .format(ids=list(new_data))) + + # Re-sort result to match order of requested IDs + # Will only work in Python v3.7+ + if list(result) != ids: + result = { + id: result[id] + for id in ids + if id in result + } - if self.handle_error(json_data, suppress_errors) or suppress_errors: - return result - return {} + return result - @staticmethod - def _list_batch(input_list, n=50): - if not isinstance(input_list, (list, tuple)): - input_list = list(input_list) - for i in range(0, len(input_list), n): - yield input_list[i:i + n] + def get_fanarts(self, channel_ids): + if not self._show_fanart: + return {} - def get_videos(self, video_ids, live_details=False, suppress_errors=False): - list_of_50s = self._list_batch(video_ids, n=50) + result = self.get_channels(channel_ids) + banners = ['bannerTvMediumImageUrl', 'bannerTvLowImageUrl', + 'bannerTvImageUrl', 'bannerExternalUrl'] + # transform + for key, item in result.items(): + images = item.get('brandingSettings', {}).get('image', {}) + for banner in banners: + image = images.get(banner) + if not image: + continue + result[key] = image + break + else: + # set an empty url + result[key] = '' - result = {} - for list_of_50 in list_of_50s: - result.update(self._update_videos(list_of_50, live_details, suppress_errors)) return result - def _update_playlists(self, playlists_ids): - json_data = None - data_cache = self._context.get_data_cache() - playlist_data = data_cache.get_items(playlists_ids, data_cache.ONE_MONTH) + def get_playlists(self, ids): + result = self._data_cache.get_items(ids, self._data_cache.ONE_MONTH) + to_update = [id_ for id_ in ids if id_ not in result] - playlists_ids = set(playlists_ids) - playlists_ids_cached = set(playlist_data) - playlist_ids_to_update = playlists_ids - playlists_ids_cached - playlists_ids_cached = playlists_ids & playlists_ids_cached + if result: + self._context.log_debug('Found cached data for playlists:\n|{ids}|' + .format(ids=list(result))) - result = playlist_data - if playlists_ids_cached: - self._context.log_debug('Found cached data for playlists |%s|' % ', '.join(playlists_ids_cached)) + if to_update: + new_data = [self._client.get_playlists(list_of_50) + for list_of_50 in self._list_batch(to_update, n=50)] + if not any(new_data): + new_data = None + else: + new_data = None - if playlist_ids_to_update: - self._context.log_debug('No data for playlists |%s| cached' % ', '.join(playlist_ids_to_update)) - json_data = self._client.get_playlists(playlist_ids_to_update) - playlist_data = { + if new_data: + self._context.log_debug('Got data for playlists:\n|{ids}|' + .format(ids=to_update)) + new_data = { yt_item['id']: yt_item - for yt_item in json_data.get('items', []) + for batch in new_data + for yt_item in batch.get('items', []) if yt_item } - result.update(playlist_data) - data_cache.set_items(playlist_data) - self._context.log_debug('Cached data for playlists |%s|' % ', '.join(playlist_data)) + result.update(new_data) + self._data_cache.set_items(new_data) + self._context.log_debug('Cached data for playlists:\n|{ids}|' + .format(ids=list(new_data))) + + # Re-sort result to match order of requested IDs + # Will only work in Python v3.7+ + if list(result) != ids: + result = { + id: result[id] + for id in ids + if id in result + } - if self.handle_error(json_data): - return result - return {} + return result - def get_playlists(self, playlists_ids): - list_of_50s = self._list_batch(playlists_ids, n=50) + def get_playlist_items(self, ids=None, batch_id=None): + if not ids and not batch_id: + return None + if batch_id: + ids = [batch_id[0]] + page_token = batch_id[1] + fetch_next = False + else: + page_token = None + fetch_next = True + + batch_ids = [] + to_update = [] result = {} - for list_of_50 in list_of_50s: - result.update(self._update_playlists(list_of_50)) + for playlist_id in ids: + page_token = page_token or 0 + while 1: + batch_id = (playlist_id, page_token) + batch_ids.append(batch_id) + batch = self._data_cache.get_item(batch_id, + self._data_cache.ONE_HOUR) + if not batch: + to_update.append(batch_id) + break + result[batch_id] = batch + page_token = batch.get('nextPageToken') if fetch_next else None + if page_token is None: + break + + if result: + self._context.log_debug('Found cached items for playlists:\n|{ids}|' + .format(ids=list(result))) + + new_data = {} + for playlist_id, page_token in to_update: + while 1: + batch_id = (playlist_id, page_token) + batch = self._client.get_playlist_items(*batch_id) + new_data[batch_id] = batch + page_token = batch.get('nextPageToken') if fetch_next else None + if page_token is None: + break + + if new_data: + to_update = list(new_data) + self._context.log_debug('Got items for playlists:\n|{ids}|' + .format(ids=to_update)) + self._data_cache.set_items(new_data) + result.update(new_data) + self._context.log_debug('Cached items for playlists:\n|{ids}|' + .format(ids=to_update)) + + # Re-sort result to match order of requested IDs + # Will only work in Python v3.7+ + if list(result) != batch_ids: + result = { + id: result[id] + for id in batch_ids + if id in result + } + return result def get_related_playlists(self, channel_id): - result = self._update_channels([channel_id]) + result = self.get_channels([channel_id]) # transform item = None @@ -199,65 +239,52 @@ def get_related_playlists(self, channel_id): return item.get('contentDetails', {}).get('relatedPlaylists', {}) - def get_channels(self, channel_ids): - list_of_50s = self._list_batch(channel_ids, n=50) - - result = {} - for list_of_50 in list_of_50s: - result.update(self._update_channels(list_of_50)) - return result + def get_videos(self, ids, live_details=False, suppress_errors=False): + result = self._data_cache.get_items(ids, self._data_cache.ONE_MONTH) + to_update = [id_ for id_ in ids if id_ not in result] + + if result: + self._context.log_debug('Found cached data for videos:\n|{ids}|' + .format(ids=list(result))) + + if to_update: + notify_and_raise = not suppress_errors + new_data = [self._client.get_videos(list_of_50, + live_details, + notify=notify_and_raise, + raise_exc=notify_and_raise) + for list_of_50 in self._list_batch(to_update, n=50)] + if not any(new_data): + new_data = None + else: + new_data = None - def get_fanarts(self, channel_ids): - if not self._enable_channel_fanart: - return {} + if new_data: + self._context.log_debug('Got data for videos:\n|{ids}|' + .format(ids=to_update)) + new_data = dict(dict.fromkeys(to_update, {}), **{ + yt_item['id']: yt_item or {} + for batch in new_data + for yt_item in batch.get('items', []) + }) + result.update(new_data) + self._data_cache.set_items(new_data) + self._context.log_debug('Cached data for videos:\n|{ids}|' + .format(ids=list(new_data))) + + # Re-sort result to match order of requested IDs + # Will only work in Python v3.7+ + if list(result) != ids: + result = { + id: result[id] + for id in ids + if id in result + } - result = self._update_channels(channel_ids) - banners = ['bannerTvMediumImageUrl', 'bannerTvLowImageUrl', - 'bannerTvImageUrl', 'bannerExternalUrl'] - # transform - for key, item in result.items(): - images = item.get('brandingSettings', {}).get('image', {}) - for banner in banners: - image = images.get(banner) - if not image: - continue - result[key] = image - break - else: - # set an empty url - result[key] = '' + if self._context.get_settings().use_local_history(): + playback_history = self._context.get_playback_history() + played_items = playback_history.get_items(ids) + for video_id, play_data in played_items.items(): + result[video_id]['play_data'] = play_data return result - - def handle_error(self, json_data, suppress_errors=False): - context = self._context - if json_data and 'error' in json_data: - ok_dialog = False - message_timeout = 5000 - message = json_data['error'].get('message', '') - message = strip_html_from_text(message) - reason = json_data['error']['errors'][0].get('reason', '') - title = '%s: %s' % (context.get_name(), reason) - error_message = 'Error reason: |%s| with message: |%s|' % (reason, message) - - context.log_error(error_message) - - if reason == 'accessNotConfigured': - message = context.localize('key.requirement.notification') - ok_dialog = True - - elif reason in {'quotaExceeded', 'dailyLimitExceeded'}: - message_timeout = 7000 - - if not suppress_errors: - if ok_dialog: - context.get_ui().on_ok(title, message) - else: - context.get_ui().show_notification(message, title, - time_ms=message_timeout) - - raise YouTubeException(error_message) - - return False - - return True diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_play.py b/resources/lib/youtube_plugin/youtube/helper/yt_play.py index 92ad9b9b1..8274f5b58 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_play.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_play.py @@ -146,35 +146,32 @@ def play_playlist(provider, context): if not playlist_ids: playlist_ids = [params.get('playlist_id')] - client = provider.get_client(context) + resource_manager = provider.get_resource_manager(context) ui = context.get_ui() - progress_dialog = ui.create_progress_dialog( + with ui.create_progress_dialog( context.localize('playlist.progress.updating'), context.localize('please_wait'), background=True - ) - - # start the loop and fill the list with video items - total = 0 - for playlist_id in playlist_ids: - page_token = 0 - while page_token is not None: - json_data = client.get_playlist_items(playlist_id, page_token) - if not v3.handle_error(context, json_data): - break - - if page_token == 0: - playlist_total = int(json_data.get('pageInfo', {}) - .get('totalResults', 0)) - if not playlist_total: - break - total += playlist_total - progress_dialog.set_total(total) + ) as progress_dialog: + json_data = resource_manager.get_playlist_items(playlist_ids) + + total = sum(len(chunk.get('items', [])) for chunk in json_data.values()) + progress_dialog.set_total(total) + progress_dialog.update( + steps=0, + text='{wait} {current}/{total}'.format( + wait=context.localize('please_wait'), + current=0, + total=total + ) + ) + # start the loop and fill the list with video items + for chunk in json_data.values(): result = v3.response_to_items(provider, context, - json_data, + chunk, process_next_page=False) videos.extend(result) @@ -187,51 +184,51 @@ def play_playlist(provider, context): ) ) - page_token = json_data.get('nextPageToken') or None - - # select order - order = params.get('order', '') - if not order: - order_list = ['default', 'reverse', 'shuffle'] - items = [(context.localize('playlist.play.%s' % order), order) - for order in order_list] - order = ui.on_select(context.localize('playlist.play.select'), items) - if order not in order_list: - order = 'default' - - # reverse the list - if order == 'reverse': - videos = videos[::-1] - elif order == 'shuffle': - # we have to shuffle the playlist by our self. - # The implementation of XBMC/KODI is quite weak :( - random.shuffle(videos) - - # clear the playlist - playlist = context.get_video_playlist() - playlist.clear() - - # select unshuffle - if order == 'shuffle': - playlist.unshuffle() - - # check if we have a video as starting point for the playlist - video_id = params.get('video_id', '') - # add videos to playlist - playlist_position = 0 - for idx, video in enumerate(videos): - playlist.add(video) - if video_id and not playlist_position and video_id in video.get_uri(): - playlist_position = idx - - # we use the shuffle implementation of the playlist - """ - if order == 'shuffle': - playlist.shuffle() - """ - - if progress_dialog: - progress_dialog.close() + if not videos: + return False + + # select order + order = params.get('order', '') + if not order: + order_list = ['default', 'reverse', 'shuffle'] + items = [(context.localize('playlist.play.%s' % order), order) + for order in order_list] + order = ui.on_select(context.localize('playlist.play.select'), + items) + if order not in order_list: + order = 'default' + + # reverse the list + if order == 'reverse': + videos = videos[::-1] + elif order == 'shuffle': + # we have to shuffle the playlist by our self. + # The implementation of XBMC/KODI is quite weak :( + random.shuffle(videos) + + # clear the playlist + playlist = context.get_video_playlist() + playlist.clear() + + # select unshuffle + if order == 'shuffle': + playlist.unshuffle() + + # check if we have a video as starting point for the playlist + video_id = params.get('video_id', '') + # add videos to playlist + playlist_position = 0 + for idx, video in enumerate(videos): + playlist.add(video) + if (video_id and not playlist_position + and video_id in video.get_uri()): + playlist_position = idx + + # we use the shuffle implementation of the playlist + """ + if order == 'shuffle': + playlist.shuffle() + """ if not params.get('play'): return videos diff --git a/resources/lib/youtube_plugin/youtube/provider.py b/resources/lib/youtube_plugin/youtube/provider.py index 4bdc07206..d5950382d 100644 --- a/resources/lib/youtube_plugin/youtube/provider.py +++ b/resources/lib/youtube_plugin/youtube/provider.py @@ -277,45 +277,28 @@ def on_uri2addon(self, context, re_match): return False - @RegisterProviderPath('^/playlist/(?P[^/]+)/$') - def _on_playlist(self, context, re_match): - self.set_content_type(context, constants.content_type.VIDEOS) - - result = [] - - playlist_id = re_match.group('playlist_id') - page_token = context.get_param('page_token', '') - - # no caching - json_data = self.get_client(context).get_playlist_items(playlist_id=playlist_id, page_token=page_token) - if not v3.handle_error(context, json_data): - return False - result.extend(v3.response_to_items(self, context, json_data)) - - return result - """ Lists the videos of a playlist. path : '/channel/(?P[^/]+)/playlist/(?P[^/]+)/' + or + path : '/playlist/(?P[^/]+)/' channel_id : ['mine'|] playlist_id: """ - @RegisterProviderPath('^/channel/(?P[^/]+)/playlist/(?P[^/]+)/$') - def _on_channel_playlist(self, context, re_match): + @RegisterProviderPath('^(?:/channel/(?P[^/]+))?/playlist/(?P[^/]+)/$') + def _on_playlist(self, context, re_match): self.set_content_type(context, constants.content_type.VIDEOS) client = self.get_client(context) - result = [] + resource_manager = self.get_resource_manager(context) - playlist_id = re_match.group('playlist_id') - page_token = context.get_param('page_token', '') + batch_id = (re_match.group('playlist_id'), + context.get_param('page_token') or 0) - # no caching - json_data = client.get_playlist_items(playlist_id=playlist_id, page_token=page_token) - if not v3.handle_error(context, json_data): + json_data = resource_manager.get_playlist_items(batch_id=batch_id) + if not json_data: return False - result.extend(v3.response_to_items(self, context, json_data)) - + result = v3.response_to_items(self, context, json_data[batch_id]) return result """