diff --git a/resources/lib/youtube_plugin/youtube/helper/resource_manager.py b/resources/lib/youtube_plugin/youtube/helper/resource_manager.py index dd3b13033..b9b7d6293 100644 --- a/resources/lib/youtube_plugin/youtube/helper/resource_manager.py +++ b/resources/lib/youtube_plugin/youtube/helper/resource_manager.py @@ -10,6 +10,8 @@ from __future__ import absolute_import, division, unicode_literals +from .utils import get_thumbnail + class ResourceManager(object): def __init__(self, provider, context): @@ -64,7 +66,7 @@ def get_channels(self, ids, defer_cache=False): .format(exc=exc, data=data)) ids = updated - if refresh: + if refresh or not ids: result = {} else: result = data_cache.get_items(ids, data_cache.ONE_MONTH) @@ -116,30 +118,99 @@ def get_channels(self, ids, defer_cache=False): return result - def get_fanarts(self, channel_ids, force=False, defer_cache=False): + def get_channel_info(self, + ids, + force=False, + channel_data=None, + defer_cache=False): if force: pass elif self._fanart_type != self._context.get_settings().FANART_CHANNEL: return {} - result = self.get_channels(channel_ids, defer_cache=defer_cache) + context = self._context + refresh = context.get_param('refresh') + if not refresh and channel_data: + result = channel_data + else: + result = {} + + to_check = [id_ for id_ in ids + if id_ not in result + or not result[id_] + or result[id_].get('_partial')] + if to_check: + data_cache = context.get_data_cache() + result.update(data_cache.get_items(to_check, data_cache.ONE_MONTH)) + to_update = [id_ for id_ in ids + if id_ not in result + or not result[id_] + or result[id_].get('_partial')] + + if result: + context.debug_log and context.log_debug( + 'ResourceManager.get_fanarts' + ' - Using cached data for channels' + '\n\tChannel IDs: {ids}' + .format(ids=list(result)) + ) + + if to_update: + client = self._provider.get_client(context) + new_data = [client.get_channels(list_of_50) + for list_of_50 in self._list_batch(to_update, n=50)] + if not any(new_data): + new_data = None + else: + new_data = None + + if new_data: + context.debug_log and context.log_debug( + 'ResourceManager.get_fanarts' + ' - Retrieved new data for channels' + '\n\tChannel IDs: {ids}' + .format(ids=to_update) + ) + new_data = { + yt_item['id']: yt_item + for batch in new_data + for yt_item in batch.get('items', []) + if yt_item + } + result.update(new_data) + self.cache_data(new_data, defer=defer_cache) + banners = ( 'bannerTvMediumImageUrl', 'bannerTvLowImageUrl', 'bannerTvImageUrl', 'bannerExternalUrl', ) + untitled = context.localize('untitled') + thumb_size = context.get_settings().get_thumbnail_size() + # transform for key, item in result.items(): + channel_info = { + 'name': None, + 'image': None, + 'fanart': None, + } images = item.get('brandingSettings', {}).get('image', {}) for banner in banners: image = images.get(banner) if image: - result[key] = image + channel_info['fanart'] = image break - else: - # set an empty url - result[key] = '' + snippet = item.get('snippet') + if snippet: + localised_info = snippet.get('localized') or {} + channel_info['name'] = (localised_info.get('title') + or snippet.get('title') + or untitled) + channel_info['image'] = get_thumbnail(thumb_size, + snippet.get('thumbnails')) + result[key] = channel_info return result @@ -147,7 +218,7 @@ def get_playlists(self, ids, defer_cache=False): context = self._context ids = tuple(ids) refresh = context.get_param('refresh') - if refresh: + if refresh or not ids: result = {} else: data_cache = context.get_data_cache() @@ -315,7 +386,7 @@ def get_videos(self, context = self._context ids = tuple(ids) refresh = context.get_param('refresh') - if refresh: + if refresh or not ids: result = {} else: data_cache = context.get_data_cache() diff --git a/resources/lib/youtube_plugin/youtube/helper/tv.py b/resources/lib/youtube_plugin/youtube/helper/tv.py index 85e90f3f7..f1f1dc86c 100644 --- a/resources/lib/youtube_plugin/youtube/helper/tv.py +++ b/resources/lib/youtube_plugin/youtube/helper/tv.py @@ -36,14 +36,14 @@ def tv_videos_to_items(provider, context, json_data): item_filter = context.get_settings().item_filter() - utils.update_video_infos( + utils.update_video_items( provider, context, video_id_dict, channel_items_dict=channel_items_dict, item_filter=item_filter, ) - utils.update_fanarts(provider, context, channel_items_dict) + utils.update_channel_info(provider, context, channel_items_dict) if item_filter: result = utils.filter_videos(video_id_dict.values(), **item_filter) @@ -102,11 +102,11 @@ def saved_playlists_to_items(provider, context, json_data): playlist_id_dict[playlist_id] = playlist_item channel_items_dict = {} - utils.update_playlist_infos(provider, + utils.update_playlist_items(provider, context, playlist_id_dict, - channel_items_dict) - utils.update_fanarts(provider, context, channel_items_dict) + channel_items_dict=channel_items_dict) + utils.update_channel_info(provider, context, channel_items_dict) # next page next_page_token = json_data.get('next_page_token') diff --git a/resources/lib/youtube_plugin/youtube/helper/url_to_item_converter.py b/resources/lib/youtube_plugin/youtube/helper/url_to_item_converter.py index 8dc19b7eb..8a1c16041 100644 --- a/resources/lib/youtube_plugin/youtube/helper/url_to_item_converter.py +++ b/resources/lib/youtube_plugin/youtube/helper/url_to_item_converter.py @@ -213,14 +213,14 @@ def get_video_items(self, provider, context, skip_title=False): if self._video_items: return self._video_items - channel_id_dict = {} - utils.update_video_infos( + channel_items_dict = {} + utils.update_video_items( provider, context, self._video_id_dict, - channel_items_dict=channel_id_dict, + channel_items_dict=channel_items_dict, ) - utils.update_fanarts(provider, context, channel_id_dict) + utils.update_channel_info(provider, context, channel_items_dict) self._video_items = [ video_item @@ -233,11 +233,11 @@ def get_playlist_items(self, provider, context, skip_title=False): if self._playlist_items: return self._playlist_items - channel_id_dict = {} - utils.update_playlist_infos(provider, context, + channel_items_dict = {} + utils.update_playlist_items(provider, context, self._playlist_id_dict, - channel_items_dict=channel_id_dict) - utils.update_fanarts(provider, context, channel_id_dict) + channel_items_dict=channel_items_dict) + utils.update_channel_info(provider, context, channel_items_dict) self._playlist_items = [ playlist_item @@ -246,13 +246,10 @@ def get_playlist_items(self, provider, context, skip_title=False): ] return self._playlist_items - def get_channel_items(self, provider, context, skip_title=False): + def get_channel_items(self, _provider, _context, skip_title=False): if self._channel_items: return self._channel_items - channel_id_dict = {} - utils.update_fanarts(provider, context, channel_id_dict) - self._channel_items = [ channel_item for channel_item in self._channel_id_dict.values() diff --git a/resources/lib/youtube_plugin/youtube/helper/utils.py b/resources/lib/youtube_plugin/youtube/helper/utils.py index 454fc5a55..980324d61 100644 --- a/resources/lib/youtube_plugin/youtube/helper/utils.py +++ b/resources/lib/youtube_plugin/youtube/helper/utils.py @@ -149,19 +149,23 @@ def make_comment_item(context, snippet, uri, reply_count=0): return comment_item -def update_channel_infos(provider, context, channel_id_dict, +def update_channel_items(provider, context, channel_id_dict, subscription_id_dict=None, channel_items_dict=None, data=None): - channel_ids = list(channel_id_dict) - if not channel_ids and not data: + if not channel_id_dict and not data and not channel_items_dict: return - if not data: + channel_ids = list(channel_id_dict) + if channel_ids and not data: resource_manager = provider.get_resource_manager(context) data = resource_manager.get_channels(channel_ids) if not data: + if channel_items_dict: + update_channel_info(provider, + context, + channel_items_dict=channel_items_dict) return if subscription_id_dict is None: @@ -170,11 +174,9 @@ def update_channel_infos(provider, context, channel_id_dict, logged_in = provider.is_logged_in() settings = context.get_settings() - channel_name_aliases = settings.get_channel_name_aliases() show_details = settings.show_detailed_description() localize = context.localize - channel_role = localize('channel') untitled = localize('untitled') path = context.get_path() @@ -206,7 +208,9 @@ def update_channel_infos(provider, context, channel_id_dict, continue snippet = yt_item['snippet'] - channel_item = channel_id_dict[channel_id] + channel_item = channel_id_dict.get(channel_id) + if not channel_item: + continue label_stats = [] stats = [] @@ -247,10 +251,6 @@ def update_channel_infos(provider, context, channel_id_dict, or untitled) channel_item.set_name(channel_name) channel_item.add_artist(channel_name) - if 'cast' in channel_name_aliases: - channel_item.add_cast(channel_name, role=channel_role) - if 'studio' in channel_name_aliases: - channel_item.add_studio(channel_name) # plot description = strip_html_from_text(localised_info.get('description') @@ -326,15 +326,21 @@ def update_channel_infos(provider, context, channel_id_dict, channel_items_dict[channel_id] = [] channel_items_dict[channel_id].append(channel_item) + if channel_items_dict: + update_channel_info(provider, + context, + channel_items_dict=channel_items_dict, + channel_data=data) + -def update_playlist_infos(provider, context, playlist_id_dict, +def update_playlist_items(provider, context, playlist_id_dict, channel_items_dict=None, data=None): - playlist_ids = list(playlist_id_dict) - if not playlist_ids and not data: + if not playlist_id_dict and not data: return - if not data: + playlist_ids = list(playlist_id_dict) + if playlist_ids and not data: resource_manager = provider.get_resource_manager(context) data = resource_manager.get_playlists(playlist_ids) @@ -348,12 +354,10 @@ def update_playlist_infos(provider, context, playlist_id_dict, settings = context.get_settings() thumb_size = settings.get_thumbnail_size() - channel_name_aliases = settings.get_channel_name_aliases() show_details = settings.show_detailed_description() item_count_color = settings.get_label_color('itemCount') localize = context.localize - channel_role = localize('channel') episode_count_label = localize('stats.itemCount') video_count_label = localize('stats.videoCount') podcast_label = context.localize('playlist.podcast') @@ -379,7 +383,9 @@ def update_playlist_infos(provider, context, playlist_id_dict, continue snippet = yt_item['snippet'] - playlist_item = playlist_id_dict[playlist_id] + playlist_item = playlist_id_dict.get(playlist_id) + if not playlist_item: + continue is_podcast = yt_item.get('status', {}).get('podcastStatus') == 'enabled' item_count_str, item_count = friendly_number( @@ -408,10 +414,6 @@ def update_playlist_infos(provider, context, playlist_id_dict, # channel name channel_name = snippet.get('channelTitle') or untitled playlist_item.add_artist(channel_name) - if 'cast' in channel_name_aliases: - playlist_item.add_cast(channel_name, role=channel_role) - if 'studio' in channel_name_aliases: - playlist_item.add_studio(channel_name) # plot with channel name, podcast status and item count description = strip_html_from_text(localised_info.get('description') @@ -518,24 +520,18 @@ def update_playlist_infos(provider, context, playlist_id_dict, if context_menu: playlist_item.add_context_menu(context_menu) - # update channel mapping - if channel_items_dict is not None: - if channel_id not in channel_items_dict: - channel_items_dict[channel_id] = [] - channel_items_dict[channel_id].append(playlist_item) - -def update_video_infos(provider, context, video_id_dict, +def update_video_items(provider, context, video_id_dict, playlist_item_id_dict=None, channel_items_dict=None, live_details=True, item_filter=None, data=None): - video_ids = list(video_id_dict) - if not video_ids and not data: + if not video_id_dict and not data: return - if not data: + video_ids = list(video_id_dict) + if video_ids and not data: resource_manager = provider.get_resource_manager(context) data = resource_manager.get_videos(video_ids, live_details=live_details, @@ -558,7 +554,6 @@ def update_video_infos(provider, context, video_id_dict, default_web_urls = settings.default_player_web_urls() ask_quality = not default_web_urls and settings.ask_for_video_quality() audio_only = settings.audio_only() - channel_name_aliases = settings.get_channel_name_aliases() show_details = settings.show_detailed_description() subtitles_prompt = settings.get_subtitle_selection() == 1 thumb_size = settings.get_thumbnail_size() @@ -566,7 +561,6 @@ def update_video_infos(provider, context, video_id_dict, use_play_data = settings.use_local_history() localize = context.localize - channel_role = localize('channel') untitled = localize('untitled') path = context.get_path() @@ -799,10 +793,6 @@ def update_video_infos(provider, context, video_id_dict, # channel name channel_name = snippet.get('channelTitle', '') or untitled media_item.add_artist(channel_name) - if 'cast' in channel_name_aliases: - media_item.add_cast(channel_name, role=channel_role) - if 'studio' in channel_name_aliases: - media_item.add_studio(channel_name) # plot description = strip_html_from_text(localised_info.get('description') @@ -1014,7 +1004,7 @@ def update_video_infos(provider, context, video_id_dict, def update_play_info(provider, context, video_id, media_item, video_stream): - update_video_infos(provider, context, {video_id: media_item}) + update_video_items(provider, context, {video_id: media_item}) settings = context.get_settings() ui = context.get_ui() @@ -1064,35 +1054,53 @@ def update_play_info(provider, context, video_id, media_item, video_stream): ui.set_property(LICENSE_TOKEN, license_token) -def update_fanarts(provider, context, channel_items_dict, data=None): +def update_channel_info(provider, + context, + channel_items_dict, + data=None, + channel_data=None): # at least we need one channel id - channel_ids = list(channel_items_dict) - if not channel_ids and not data: + if not channel_items_dict and not (data or channel_data): return - if not data: + channel_ids = list(channel_items_dict) + if channel_ids and not data: resource_manager = provider.get_resource_manager(context) - data = resource_manager.get_fanarts(channel_ids, force=True) + data = resource_manager.get_channel_info(channel_ids, + force=True, + channel_data=channel_data) if not data: return settings = context.get_settings() + channel_name_aliases = settings.get_channel_name_aliases() fanart_type = context.get_param('fanart_type') if fanart_type is None: fanart_type = settings.fanart_selection() use_channel_fanart = fanart_type == settings.FANART_CHANNEL use_thumb_fanart = fanart_type == settings.FANART_THUMBNAIL + channel_role = context.localize('channel') + for channel_id, channel_items in channel_items_dict.items(): - # only set not empty fanarts - fanart = data.get(channel_id) - if not fanart: + channel_info = data.get(channel_id) + if not channel_info: continue + for item in channel_items: if (use_channel_fanart or use_thumb_fanart and not item.get_fanart(default=False)): - item.set_fanart(fanart) + item.set_fanart(channel_info.get('fanart')) + + channel_name = channel_info.get('name') + if channel_name: + if 'cast' in channel_name_aliases: + item.add_cast(channel_name, + role=channel_role, + thumbnail=channel_info.get('image')) + if 'studio' in channel_name_aliases: + item.add_studio(channel_name) THUMB_TYPES = { diff --git a/resources/lib/youtube_plugin/youtube/helper/v3.py b/resources/lib/youtube_plugin/youtube/helper/v3.py index a4828407b..3f8f60370 100644 --- a/resources/lib/youtube_plugin/youtube/helper/v3.py +++ b/resources/lib/youtube_plugin/youtube/helper/v3.py @@ -19,10 +19,9 @@ filter_videos, get_thumbnail, make_comment_item, - update_channel_infos, - update_fanarts, - update_playlist_infos, - update_video_infos, + update_channel_items, + update_playlist_items, + update_video_items, ) from ...kodion import KodionException from ...kodion.constants import PATHS @@ -330,14 +329,16 @@ def _process_list_response(provider, resources = { 1: { 'fetcher': resource_manager.get_videos, - 'args': (video_id_dict,), + 'args': ( + video_id_dict, + ), 'kwargs': { 'live_details': True, 'suppress_errors': True, 'defer_cache': True, }, 'thread': None, - 'updater': update_video_infos, + 'updater': update_video_items, 'upd_args': ( provider, context, @@ -355,77 +356,80 @@ def _process_list_response(provider, }, 2: { 'fetcher': resource_manager.get_playlists, - 'args': (playlist_id_dict,), - 'kwargs': {'defer_cache': True}, + 'args': ( + playlist_id_dict, + ), + 'kwargs': { + 'defer_cache': True, + }, 'thread': None, - 'updater': update_playlist_infos, + 'updater': update_playlist_items, 'upd_args': ( provider, context, playlist_id_dict, channel_items_dict, ), - 'upd_kwargs': {'data': None}, + 'upd_kwargs': { + 'data': None, + }, 'complete': False, 'defer': False, }, 3: { 'fetcher': resource_manager.get_channels, - 'args': (channel_id_dict,), - 'kwargs': {'defer_cache': True}, - 'thread': None, - 'updater': update_channel_infos, - 'upd_args': ( - provider, - context, + 'args': ( channel_id_dict, - subscription_id_dict, - channel_items_dict, ), - 'upd_kwargs': {'data': None}, - 'complete': False, - 'defer': False, - }, - 4: { - 'fetcher': resource_manager.get_fanarts, - 'args': (channel_items_dict,), 'kwargs': { - 'force': bool(channel_id_dict or playlist_id_dict), + '_force_run': True, 'defer_cache': True, }, 'thread': None, - 'updater': update_fanarts, + 'updater': update_channel_items, 'upd_args': ( provider, context, + channel_id_dict, + subscription_id_dict, channel_items_dict, ), - 'upd_kwargs': {'data': None}, + 'upd_kwargs': { + '_force_run': True, + 'data': None, + }, 'complete': False, 'defer': True, }, - 5: { + 4: { 'fetcher': resource_manager.cache_data, 'args': (), - 'kwargs': {}, + 'kwargs': { + '_force_run': True, + }, 'thread': None, 'updater': None, 'upd_args': (), 'upd_kwargs': {}, 'complete': False, - 'defer': 4, + 'defer': 3, }, } def _fetch(resource): try: - data = resource['fetcher']( - *resource['args'], **resource['kwargs'] - ) - if data and resource['updater']: - resource['upd_kwargs']['data'] = data - resource['updater'](*resource['upd_args'], - **resource['upd_kwargs']) + data = resource['fetcher'](*resource['args'], **resource['kwargs']) + + updater = resource['updater'] + if not updater: + return + + kwargs = resource['upd_kwargs'] + if not kwargs.pop('_force_run', False) and not data: + return + kwargs['data'] = data + + updater(*resource['upd_args'], **kwargs) except Exception as exc: tb_obj = exc_info()[2] while tb_obj: @@ -443,9 +447,10 @@ def _fetch(resource): '\n\tStack trace (most recent call last):\n{stack}' .format(exc=exc, stack=stack)) context.log_error(msg) - resource['complete'] = True - threads['current'].discard(resource['thread']) - threads['loop'].set() + finally: + resource['complete'] = True + threads['current'].discard(resource['thread']) + threads['loop'].set() threads = { 'current': set(), @@ -493,12 +498,12 @@ def _fetch(resource): continue resource['defer'] = False - args = resource['args'] - if args and not args[0]: - resource['complete'] = True - continue - if not resource['thread']: + if (not resource['kwargs'].pop('_force_run', False) + and not any(resource['args'])): + resource['complete'] = True + continue + new_thread = threading.Thread(target=_fetch, args=(resource,)) new_thread.daemon = True threads['current'].add(new_thread) diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_specials.py b/resources/lib/youtube_plugin/youtube/helper/yt_specials.py index 355cf0e71..f37250128 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_specials.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_specials.py @@ -224,11 +224,7 @@ def _display_channels(channel_ids): ) channel_id_dict[channel_id] = channel_item - channel_item_dict = {} - utils.update_channel_infos(provider, - context, - channel_id_dict, - channel_items_dict=channel_item_dict) + utils.update_channel_items(provider, context, channel_id_dict) # clean up - remove empty entries return [channel_item @@ -251,12 +247,12 @@ def _display_playlists(playlist_ids): ) playlist_id_dict[playlist_id] = playlist_item - channel_item_dict = {} - utils.update_playlist_infos(provider, + channel_items_dict = {} + utils.update_playlist_items(provider, context, playlist_id_dict, - channel_items_dict=channel_item_dict) - utils.update_fanarts(provider, context, channel_item_dict) + channel_items_dict=channel_items_dict) + utils.update_channel_info(provider, context, channel_items_dict) # clean up - remove empty entries return [playlist_item diff --git a/resources/lib/youtube_plugin/youtube/provider.py b/resources/lib/youtube_plugin/youtube/provider.py index 3ae559bf3..321d951e6 100644 --- a/resources/lib/youtube_plugin/youtube/provider.py +++ b/resources/lib/youtube_plugin/youtube/provider.py @@ -405,9 +405,9 @@ def on_channel_playlists(provider, context, re_match): new_params['addon_id'] = addon_id resource_manager = provider.get_resource_manager(context) - fanart = resource_manager.get_fanarts( + channel_info = resource_manager.get_channel_info( (channel_id,), force=True - ).get(channel_id) + ).get(channel_id) or {} playlists = resource_manager.get_related_playlists(channel_id) playlist_id = playlists.get('uploads') @@ -420,7 +420,7 @@ def on_channel_playlists(provider, context, re_match): new_params, ), image='{media}/playlist.png', - fanart=fanart, + fanart=channel_info.get('fanart') or '', category_label=item_label, channel_id=channel_id, playlist_id=playlist_id, @@ -446,11 +446,15 @@ def on_channel_playlists(provider, context, re_match): ), # subscribe to the channel via the playlist item menu_items.subscribe_to_channel( - context, channel_id, + context, + channel_id, + channel_name=channel_info.get('name') or '', ) if provider.is_logged_in else None, # bookmark channel of the playlist menu_items.bookmark_add_channel( - context, channel_id, + context, + channel_id, + channel_name=channel_info.get('name') or '', ) )) @@ -597,9 +601,10 @@ def on_channel(provider, context, re_match): if not channel_id: return False - fanart = resource_manager.get_fanarts( + channel_info = resource_manager.get_channel_info( (channel_id,), force=True - ).get(channel_id) + ).get(channel_id) or {} + fanart = channel_info.get('fanart') or '' page = params.get('page', 1) page_token = params.get('page_token', '')