diff --git a/resources/lib/youtube_plugin/kodion/sql_store/data_cache.py b/resources/lib/youtube_plugin/kodion/sql_store/data_cache.py index f6f7d08b3..defce41f6 100644 --- a/resources/lib/youtube_plugin/kodion/sql_store/data_cache.py +++ b/resources/lib/youtube_plugin/kodion/sql_store/data_cache.py @@ -10,7 +10,6 @@ from __future__ import absolute_import, division, unicode_literals -import json from datetime import datetime from .storage import Storage @@ -45,10 +44,10 @@ def get_item(self, content_id, seconds): return None current_time = datetime.now() - if self.get_seconds_diff(query_result[1] or current_time) > seconds: + if self.get_seconds_diff(query_result[0] or current_time) > seconds: return None - return query_result[0] + return query_result[1] def set_item(self, content_id, item): self._set(content_id, item) diff --git a/resources/lib/youtube_plugin/kodion/sql_store/function_cache.py b/resources/lib/youtube_plugin/kodion/sql_store/function_cache.py index 91839a64b..fa492ce8e 100644 --- a/resources/lib/youtube_plugin/kodion/sql_store/function_cache.py +++ b/resources/lib/youtube_plugin/kodion/sql_store/function_cache.py @@ -19,7 +19,8 @@ class FunctionCache(Storage): def __init__(self, filename, max_file_size_mb=5): max_file_size_kb = max_file_size_mb * 1024 - super(FunctionCache, self).__init__(filename, max_file_size_kb=max_file_size_kb) + super(FunctionCache, self).__init__(filename, + max_file_size_kb=max_file_size_kb) self._enabled = True @@ -67,10 +68,9 @@ def get_cached_only(self, func, *args, **keywords): # only return before cached data data, cache_id = self._get_cached_data(partial_func) - if data is not None: - return data[0] - - return None + if data is None: + return None + return data[1] def get(self, func, seconds, *args, **keywords): """ @@ -86,22 +86,14 @@ def get(self, func, seconds, *args, **keywords): if not self._enabled: return partial_func() - cached_data = None - cached_time = None data, cache_id = self._get_cached_data(partial_func) if data is not None: - cached_data = data[0] - cached_time = data[1] - - diff_seconds = 0 - - if cached_time is not None: - diff_seconds = self.get_seconds_diff(cached_time) - - if cached_data is None or diff_seconds > seconds: - cached_data = partial_func() - self._set(cache_id, cached_data) + cached_time, cached_data = data + if data is None or self.get_seconds_diff(cached_time) > seconds: + data = partial_func() + self._set(cache_id, data) + return data return cached_data def _optimize_item_count(self): diff --git a/resources/lib/youtube_plugin/kodion/sql_store/playback_history.py b/resources/lib/youtube_plugin/kodion/sql_store/playback_history.py index ff4763b2b..4a26265d0 100644 --- a/resources/lib/youtube_plugin/kodion/sql_store/playback_history.py +++ b/resources/lib/youtube_plugin/kodion/sql_store/playback_history.py @@ -19,23 +19,14 @@ def __init__(self, filename): def is_empty(self): return self._is_empty() - @staticmethod - def _process_item(item): - return item.strip('"').split(',') - def get_items(self, keys): - query_result = self._get_by_ids(keys, process=self._process_item) + query_result = self._get_by_ids(keys) if not query_result: return {} result = { - item[0]: { - 'play_count': int(item[2][0]), - 'total_time': float(item[2][1]), - 'played_time': float(item[2][2]), - 'played_percent': int(item[2][3]), - 'last_played': str(item[1]), - } for item in query_result + item[0]: dict(item[2], last_played=item[1]) + for item in query_result } return result @@ -44,14 +35,7 @@ def get_item(self, key): if not query_result: return {} - values = query_result[0].split(',') - result = {key: { - 'play_count': int(values[0]), - 'total_time': float(values[1]), - 'played_time': float(values[2]), - 'played_percent': int(values[3]), - 'last_played': str(query_result[1]), - }} + result = {key: dict(query_result[1], last_played=query_result[0])} return result def clear(self): @@ -60,9 +44,8 @@ def clear(self): def remove(self, video_id): self._remove(video_id) - def update(self, video_id, play_count, total_time, played_time, played_percent): - item = ','.join([str(play_count), str(total_time), str(played_time), str(played_percent)]) - self._set(str(video_id), item) + def update(self, video_id, play_data): + self._set(video_id, play_data) def _optimize_item_count(self): pass diff --git a/resources/lib/youtube_plugin/kodion/sql_store/storage.py b/resources/lib/youtube_plugin/kodion/sql_store/storage.py index b41898429..833517aeb 100644 --- a/resources/lib/youtube_plugin/kodion/sql_store/storage.py +++ b/resources/lib/youtube_plugin/kodion/sql_store/storage.py @@ -10,7 +10,6 @@ from __future__ import absolute_import, division, unicode_literals -import json import os import pickle import sqlite3 @@ -29,23 +28,19 @@ class Storage(object): ONE_WEEK = 7 * ONE_DAY ONE_MONTH = 4 * ONE_WEEK - _key = str('key') - _time = str('time') - _value = str('value') - _timestamp = str('timestamp') - - _table_name = 'storage' - _clear_query = 'DELETE FROM %s' % _table_name - _create_table_query = 'CREATE TABLE IF NOT EXISTS %s (key TEXT PRIMARY KEY, time TIMESTAMP, value BLOB)' % _table_name - _get_query = 'SELECT * FROM %s WHERE key = ?' % _table_name - _get_by_query = 'SELECT * FROM %s WHERE key in ({0})' % _table_name - _get_all_asc_query = 'SELECT * FROM %s ORDER BY time ASC LIMIT {0}' % _table_name - _get_all_desc_query = 'SELECT * FROM %s ORDER BY time DESC LIMIT {0}' % _table_name - _is_empty_query = 'SELECT EXISTS(SELECT 1 FROM %s LIMIT 1)' % _table_name - _optimize_item_query = 'SELECT key FROM %s ORDER BY time DESC LIMIT -1 OFFSET {0}' % _table_name - _remove_query = 'DELETE FROM %s WHERE key = ?' % _table_name - _remove_all_query = 'DELETE FROM %s WHERE key in ({0})' % _table_name - _set_query = 'REPLACE INTO %s (key, time, value) VALUES(?, ?, ?)' % _table_name + _table_name = 'storage_v2' + _clear_sql = 'DELETE FROM %s' % _table_name + _create_table_sql = 'CREATE TABLE IF NOT EXISTS %s (key TEXT PRIMARY KEY, time TIMESTAMP, value BLOB)' % _table_name + _drop_old_tables_sql = 'DELETE FROM sqlite_master WHERE type = "table" and name IS NOT "%s"' % _table_name + _get_sql = 'SELECT * FROM %s WHERE key = ?' % _table_name + _get_by_sql = 'SELECT * FROM %s WHERE key in ({0})' % _table_name + _get_all_asc_sql = 'SELECT * FROM %s ORDER BY time ASC LIMIT {0}' % _table_name + _get_all_desc_sql = 'SELECT * FROM %s ORDER BY time DESC LIMIT {0}' % _table_name + _is_empty_sql = 'SELECT EXISTS(SELECT 1 FROM %s LIMIT 1)' % _table_name + _optimize_item_sql = 'SELECT key FROM %s ORDER BY time DESC LIMIT -1 OFFSET {0}' % _table_name + _remove_sql = 'DELETE FROM %s WHERE key = ?' % _table_name + _remove_all_sql = 'DELETE FROM %s WHERE key in ({0})' % _table_name + _set_sql = 'REPLACE INTO %s (key, time, value) VALUES(?, ?, ?)' % _table_name def __init__(self, filename, max_item_count=-1, max_file_size_kb=-1): self._filename = filename @@ -59,7 +54,7 @@ def __init__(self, filename, max_item_count=-1, max_file_size_kb=-1): self._table_created = False self._needs_commit = False - sqlite3.register_converter(self._timestamp, self._convert_timestamp) + sqlite3.register_converter(str('timestamp'), self._convert_timestamp) def set_max_item_count(self, max_item_count): self._max_item_count = max_item_count @@ -68,10 +63,19 @@ def set_max_file_size_kb(self, max_file_size_kb): self._max_file_size_kb = max_file_size_kb def __del__(self): + self._close(True) + + def __enter__(self): + self._open() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): self._close() def _open(self): if self._db: + if not self._cursor: + self._cursor = self._db.cursor() return self._optimize_file_size() @@ -83,7 +87,6 @@ def _open(self): db = sqlite3.connect(self._filename, check_same_thread=False, detect_types=sqlite3.PARSE_DECLTYPES, timeout=1, isolation_level=None) - db.row_factory = sqlite3.Row cursor = db.cursor() # cursor.execute('PRAGMA journal_mode=MEMORY') cursor.execute('PRAGMA journal_mode=WAL') @@ -96,6 +99,14 @@ def _open(self): self._db = db self._cursor = cursor self._create_table() + self._drop_old_tables() + + def _drop_old_tables(self): + self._execute(True, 'PRAGMA writable_schema=1') + self._execute(True, self._drop_old_tables_sql) + self._execute(True, 'PRAGMA writable_schema=0') + self._sync() + self._execute(False, 'VACUUM') def _execute(self, needs_commit, query, values=None, many=False): if values is None: @@ -120,12 +131,13 @@ def _execute(self, needs_commit, query, values=None, many=False): time.sleep(0.1) return [] - def _close(self): - if self._db: + def _close(self, full=False): + if self._db and self._cursor: self._sync() self._db.commit() self._cursor.close() self._cursor = None + if full and self._db: self._db.close() self._db = None @@ -152,7 +164,7 @@ def _optimize_file_size(self): def _create_table(self): if self._table_created: return - self._execute(True, self._create_table_query) + self._execute(True, self._create_table_sql) self._table_created = True def _sync(self): @@ -164,22 +176,18 @@ def _sync(self): def _set(self, item_id, item): # add 1 microsecond, required for dbapi2 now = since_epoch(datetime.now()) + 0.000001 - self._open() - self._execute(True, self._set_query, values=[str(item_id), - now, - self._encode(item)]) - self._close() + with self as db: + db._execute(True, db._set_sql, + values=[str(item_id), now, db._encode(item)]) self._optimize_item_count() def _set_all(self, items): # add 1 microsecond, required for dbapi2 now = since_epoch(datetime.now()) + 0.000001 - self._open() - self._execute(True, self._set_query, - values=[(str(key), now, self._encode(item)) - for key, item in items.items()], - many=True) - self._close() + with self as db: + db._execute(True, db._set_sql, many=True, + values=[(str(item_id), now, db._encode(item)) + for item_id, item in items.items()]) self._optimize_item_count() def _optimize_item_count(self): @@ -189,32 +197,28 @@ def _optimize_item_count(self): return if self._max_item_count < 0: return - query = self._optimize_item_query.format(self._max_item_count) - self._open() - item_ids = self._execute(False, query) - key = self._key - item_ids = [item_id[key] for item_id in item_ids] - if item_ids: - self._remove_all(item_ids) - self._close() + query = self._optimize_item_sql.format(self._max_item_count) + with self as db: + item_ids = db._execute(False, query) + item_ids = [item_id[0] for item_id in item_ids] + if item_ids: + db._remove_all(item_ids) def _clear(self): - self._open() - self._execute(True, self._clear_query) - self._create_table() - self._sync() - self._execute(False, 'VACUUM') - self._close() + with self as db: + db._execute(True, db._clear_sql) + db._create_table() + db._sync() + db._execute(False, 'VACUUM') def _is_empty(self): - self._open() - result = self._execute(False, self._is_empty_query) - for item in result: - is_empty = item[0] == 0 - break - else: - is_empty = True - self._close() + with self as db: + result = db._execute(False, db._is_empty_sql) + for item in result: + is_empty = item[0] == 0 + break + else: + is_empty = True return is_empty @staticmethod @@ -222,58 +226,51 @@ def _decode(obj, process=None): decoded_obj = pickle.loads(obj) if process: return process(decoded_obj) - return json.loads(decoded_obj) + return decoded_obj @staticmethod def _encode(obj): return sqlite3.Binary(pickle.dumps( - json.dumps(obj, ensure_ascii=False), - protocol=pickle.HIGHEST_PROTOCOL + obj, protocol=pickle.HIGHEST_PROTOCOL )) - def _get(self, item_id): - self._open() - result = self._execute(False, self._get_query, [item_id]) - if result: - result = result.fetchone() - self._close() - if result: - return self._decode(result[self._value]), result[self._time] - return None + def _get(self, item_id, process=None): + with self as db: + result = db._execute(False, db._get_sql, [item_id]) + if result: + result = result.fetchone() + if not result: + return None + return result[1], self._decode(result[2], process) def _get_by_ids(self, item_ids=None, oldest_first=True, limit=-1, process=None): if not item_ids: if oldest_first: - query = self._get_all_asc_query + query = self._get_all_asc_sql else: - query = self._get_all_desc_query + query = self._get_all_desc_sql query = query.format(limit) else: num_ids = len(item_ids) - query = self._get_by_query.format(('?,' * (num_ids - 1)) + '?') + query = self._get_by_sql.format(('?,' * (num_ids - 1)) + '?') item_ids = tuple(item_ids) - self._open() - result = self._execute(False, query, item_ids) - key = self._key - time = self._time - value = self._value - result = [ - (item[key], item[time], self._decode(item[value], process)) - for item in result - ] - self._close() + with self as db: + result = db._execute(False, query, item_ids) + result = [ + (item[0], item[1], db._decode(item[2], process)) + for item in result + ] return result def _remove(self, item_id): - self._open() - self._execute(True, self._remove_query, [item_id]) + with self as db: + db._execute(True, db._remove_sql, [item_id]) def _remove_all(self, item_ids): num_ids = len(item_ids) - query = self._remove_all_query.format(('?,' * (num_ids - 1)) + '?') - self._open() + query = self._remove_all_sql.format(('?,' * (num_ids - 1)) + '?') self._execute(True, query, tuple(item_ids)) @staticmethod diff --git a/resources/lib/youtube_plugin/kodion/sql_store/watch_later_list.py b/resources/lib/youtube_plugin/kodion/sql_store/watch_later_list.py index cb8e39743..658424a30 100644 --- a/resources/lib/youtube_plugin/kodion/sql_store/watch_later_list.py +++ b/resources/lib/youtube_plugin/kodion/sql_store/watch_later_list.py @@ -13,7 +13,7 @@ from datetime import datetime from .storage import Storage -from .. import items +from ..items import to_json, from_json class WatchLaterList(Storage): @@ -28,13 +28,13 @@ def _sort_item(_item): return _item[2].get_date() def get_items(self): - result = self._get_by_ids(process=items.from_json) + result = self._get_by_ids(process=from_json) return sorted(result, key=self._sort_item, reverse=False) def add(self, base_item): base_item.set_date_from_datetime(datetime.now()) - item_json_data = items.to_json(base_item) + item_json_data = to_json(base_item) self._set(base_item.get_id(), item_json_data) def remove(self, base_item): diff --git a/resources/lib/youtube_plugin/kodion/utils/player.py b/resources/lib/youtube_plugin/kodion/utils/player.py index 8c3adc136..f68f8ec00 100644 --- a/resources/lib/youtube_plugin/kodion/utils/player.py +++ b/resources/lib/youtube_plugin/kodion/utils/player.py @@ -329,11 +329,14 @@ def run(self): refresh_only = True if use_local_history: + play_data = { + 'play_count': play_count, + 'total_time': self.total_time, + 'played_time': self.current_time, + 'played_percent': self.percent_complete, + } self._context.get_playback_history().update(self.video_id, - play_count, - self.total_time, - self.current_time, - self.percent_complete) + play_data) if not refresh_only and is_logged_in: if settings.get_bool('youtube.playlist.watchlater.autoremove', diff --git a/resources/lib/youtube_plugin/youtube/provider.py b/resources/lib/youtube_plugin/youtube/provider.py index d5950382d..2c40c1d3f 100644 --- a/resources/lib/youtube_plugin/youtube/provider.py +++ b/resources/lib/youtube_plugin/youtube/provider.py @@ -473,7 +473,7 @@ def _on_channel(self, context, re_match): upload_playlist, page_token=page_token) if not json_data: - return False + return result result.extend(v3.response_to_items(self, context, json_data)) @@ -1171,11 +1171,7 @@ def on_playback_history(self, context, re_match): play_data['played_time'] = 0 play_data['played_percent'] = 0 - playback_history.update(video_id, - play_data.get('play_count', 0), - play_data.get('total_time', 0), - play_data.get('played_time', 0), - play_data.get('played_percent', 0)) + playback_history.update(video_id, play_data) context.get_ui().refresh_container() return True