From 764caec393ff9f7fee777cfa9c8bf497573d9801 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Fri, 15 Dec 2023 14:47:28 +1100 Subject: [PATCH] Standardise input to SQL storage - Input/output will be always be JSON de(serialized) and (un)pickled - Also set ensure_ascii=False --- resources/lib/youtube_plugin/kodion/abstract_provider.py | 3 +-- .../lib/youtube_plugin/kodion/json_store/json_store.py | 6 +++--- .../youtube_plugin/kodion/player/xbmc/xbmc_playlist.py | 2 +- .../lib/youtube_plugin/kodion/sql_store/data_cache.py | 6 +++--- .../youtube_plugin/kodion/sql_store/playback_history.py | 2 +- resources/lib/youtube_plugin/kodion/sql_store/storage.py | 9 +++++---- resources/lib/youtube_plugin/youtube/client/youtube.py | 6 +++--- .../lib/youtube_plugin/youtube/helper/video_info.py | 7 +++---- resources/lib/youtube_plugin/youtube/helper/yt_play.py | 3 ++- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/abstract_provider.py b/resources/lib/youtube_plugin/kodion/abstract_provider.py index b346a196e..a35a8bad5 100644 --- a/resources/lib/youtube_plugin/kodion/abstract_provider.py +++ b/resources/lib/youtube_plugin/kodion/abstract_provider.py @@ -247,8 +247,7 @@ def _internal_search(self, context, re_match): channel_id = context.get_param('channel_id', '') self._data_cache.set_item('search_query', - json.dumps({'query': quote(query)}, - ensure_ascii=False)) + {'query': quote(query)}) if not incognito and not channel_id: try: diff --git a/resources/lib/youtube_plugin/kodion/json_store/json_store.py b/resources/lib/youtube_plugin/kodion/json_store/json_store.py index 1f5a98c29..ca6baad78 100644 --- a/resources/lib/youtube_plugin/kodion/json_store/json_store.py +++ b/resources/lib/youtube_plugin/kodion/json_store/json_store.py @@ -56,7 +56,7 @@ def save(self, data, update=False, process=None): try: if not data: raise ValueError - _data = json.loads(json.dumps(data)) + _data = json.loads(json.dumps(data, ensure_ascii=False)) with open(self.filename, mode='w', encoding='utf-8') as jsonfile: jsonfile.write(to_unicode(json.dumps(_data, ensure_ascii=False, @@ -98,12 +98,12 @@ def get_data(self, process=None): try: if not self._data: raise ValueError - _data = json.loads(json.dumps(self._data)) + _data = json.loads(json.dumps(self._data, ensure_ascii=False)) return process(_data) if process is not None else _data except (TypeError, ValueError): log_error('JSONStore.get_data - invalid data:\n|{data}|'.format( data=self._data )) self.set_defaults(reset=True) - _data = json.loads(json.dumps(self._data)) + _data = json.loads(json.dumps(self._data, ensure_ascii=False)) return process(_data) if process is not None else _data diff --git a/resources/lib/youtube_plugin/kodion/player/xbmc/xbmc_playlist.py b/resources/lib/youtube_plugin/kodion/player/xbmc/xbmc_playlist.py index 29e03de19..e6bdc14c2 100644 --- a/resources/lib/youtube_plugin/kodion/player/xbmc/xbmc_playlist.py +++ b/resources/lib/youtube_plugin/kodion/player/xbmc/xbmc_playlist.py @@ -64,7 +64,7 @@ def get_items(self, properties=None, dumps=False): result = response['result']['items'] else: result = [] - return json.dumps(result) if dumps else result + return json.dumps(result, ensure_ascii=False) if dumps else result if 'error' in response: message = response['error']['message'] diff --git a/resources/lib/youtube_plugin/kodion/sql_store/data_cache.py b/resources/lib/youtube_plugin/kodion/sql_store/data_cache.py index 8eb571ce0..f6f7d08b3 100644 --- a/resources/lib/youtube_plugin/kodion/sql_store/data_cache.py +++ b/resources/lib/youtube_plugin/kodion/sql_store/data_cache.py @@ -26,7 +26,7 @@ def is_empty(self): return self._is_empty() def get_items(self, content_ids, seconds): - query_result = self._get_by_ids(content_ids, process=json.loads) + query_result = self._get_by_ids(content_ids) if not query_result: return {} @@ -48,7 +48,7 @@ def get_item(self, content_id, seconds): if self.get_seconds_diff(query_result[1] or current_time) > seconds: return None - return json.loads(query_result[0]) + return query_result[0] def set_item(self, content_id, item): self._set(content_id, item) @@ -63,7 +63,7 @@ def remove(self, content_id): self._remove(content_id) def update(self, content_id, item): - self._set(str(content_id), json.dumps(item)) + self._set(str(content_id), item) def _optimize_item_count(self): pass diff --git a/resources/lib/youtube_plugin/kodion/sql_store/playback_history.py b/resources/lib/youtube_plugin/kodion/sql_store/playback_history.py index 0f9a263ad..ff4763b2b 100644 --- a/resources/lib/youtube_plugin/kodion/sql_store/playback_history.py +++ b/resources/lib/youtube_plugin/kodion/sql_store/playback_history.py @@ -21,7 +21,7 @@ def is_empty(self): @staticmethod def _process_item(item): - return item.split(',') + return item.strip('"').split(',') def get_items(self, keys): query_result = self._get_by_ids(keys, process=self._process_item) diff --git a/resources/lib/youtube_plugin/kodion/sql_store/storage.py b/resources/lib/youtube_plugin/kodion/sql_store/storage.py index d8a41b7d1..b41898429 100644 --- a/resources/lib/youtube_plugin/kodion/sql_store/storage.py +++ b/resources/lib/youtube_plugin/kodion/sql_store/storage.py @@ -165,7 +165,7 @@ def _set(self, item_id, item): # add 1 microsecond, required for dbapi2 now = since_epoch(datetime.now()) + 0.000001 self._open() - self._execute(True, self._set_query, values=[item_id, + self._execute(True, self._set_query, values=[str(item_id), now, self._encode(item)]) self._close() @@ -176,7 +176,7 @@ def _set_all(self, items): now = since_epoch(datetime.now()) + 0.000001 self._open() self._execute(True, self._set_query, - values=[(key, now, self._encode(json.dumps(item))) + values=[(str(key), now, self._encode(item)) for key, item in items.items()], many=True) self._close() @@ -222,12 +222,13 @@ def _decode(obj, process=None): decoded_obj = pickle.loads(obj) if process: return process(decoded_obj) - return decoded_obj + return json.loads(decoded_obj) @staticmethod def _encode(obj): return sqlite3.Binary(pickle.dumps( - obj, protocol=pickle.HIGHEST_PROTOCOL + json.dumps(obj, ensure_ascii=False), + protocol=pickle.HIGHEST_PROTOCOL )) def _get(self, item_id): diff --git a/resources/lib/youtube_plugin/youtube/client/youtube.py b/resources/lib/youtube_plugin/youtube/client/youtube.py index b6874c97b..002267b0f 100644 --- a/resources/lib/youtube_plugin/youtube/client/youtube.py +++ b/resources/lib/youtube_plugin/youtube/client/youtube.py @@ -381,7 +381,7 @@ def helper(video_id, responses): # Truncate items to keep it manageable, and cache items = items[:500] - cache.set_item(cache_items_key, json.dumps(items)) + cache.set_item(cache_items_key, items) # Build the result set items.sort( @@ -438,7 +438,7 @@ def _sort_by_date_time(item): } """ # Update cache - cache.set_item(cache_home_key, json.dumps(payload)) + cache.set_item(cache_home_key, payload) # If there are no sorted_items we fall back to default API behaviour return payload @@ -884,7 +884,7 @@ def _sort_by_date_time(item): _result['items'].sort(reverse=True, key=_sort_by_date_time) # Update cache - cache.set_item(cache_items_key, json.dumps(_result['items'])) + cache.set_item(cache_items_key, _result['items']) """ no cache, get uploads data from web """ # trim result diff --git a/resources/lib/youtube_plugin/youtube/helper/video_info.py b/resources/lib/youtube_plugin/youtube/helper/video_info.py index 2326c5f1b..3fc2bea5e 100644 --- a/resources/lib/youtube_plugin/youtube/helper/video_info.py +++ b/resources/lib/youtube_plugin/youtube/helper/video_info.py @@ -733,7 +733,7 @@ def _get_player_js(self): return '' js_url = self._normalize_url(js_url) - self._data_cache.set_item('player_js_url', json_dumps({'url': js_url})) + self._data_cache.set_item('player_js_url', {'url': js_url}) js_cache_key = quote(js_url) cached = self._data_cache.get_item(js_cache_key, @@ -752,7 +752,7 @@ def _get_player_js(self): return '' javascript = result.text - self._data_cache.set_item(js_cache_key, json_dumps({'js': javascript})) + self._data_cache.set_item(js_cache_key, {'js': javascript}) return javascript @staticmethod @@ -938,8 +938,7 @@ def _process_signature_cipher(self, stream_map): 'Failed to extract URL from signatureCipher' ) return None - self._data_cache.set_item(encrypted_signature, - json_dumps({'sig': signature})) + self._data_cache.set_item(encrypted_signature, {'sig': signature}) if signature: url = '{0}&{1}={2}'.format(url, query_var, signature) diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_play.py b/resources/lib/youtube_plugin/youtube/helper/yt_play.py index f6d4cdb76..a233a7bc0 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_play.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_play.py @@ -125,7 +125,8 @@ def play_video(provider, context): 'refresh_only': screensaver } - ui.set_property('playback_json', json.dumps(playback_json)) + ui.set_property('playback_json', json.dumps(playback_json, + ensure_ascii=False)) context.send_notification('PlaybackInit', { 'video_id': video_id, 'channel_id': playback_json.get('channel_id', ''),