From bcc201b2df8924b2ba70c99c0d9bcf001f63496a Mon Sep 17 00:00:00 2001 From: Arjit Das Date: Thu, 14 Nov 2024 22:27:19 +0530 Subject: [PATCH] refactor: Refactor logic for retrieving special playlists and reduce repeated code in special tabs by adding TabView --- scraper/src/youtube2zim/schemas.py | 6 +- scraper/src/youtube2zim/scraper.py | 51 +++-- scraper/src/youtube2zim/youtube.py | 175 ++++-------------- zimui/src/assets/main.css | 1 + .../src/components/channel/tabs/LivesTab.vue | 15 +- .../components/channel/tabs/ShortsGridTab.vue | 61 ------ .../src/components/channel/tabs/ShortsTab.vue | 16 +- .../components/channel/tabs/VideosGridTab.vue | 61 ------ .../src/components/channel/tabs/VideosTab.vue | 15 +- .../LivesGridTab.vue => views/TabView.vue} | 47 +++-- 10 files changed, 94 insertions(+), 354 deletions(-) delete mode 100644 zimui/src/components/channel/tabs/ShortsGridTab.vue delete mode 100644 zimui/src/components/channel/tabs/VideosGridTab.vue rename zimui/src/{components/channel/tabs/LivesGridTab.vue => views/TabView.vue} (57%) mode change 100644 => 100755 diff --git a/scraper/src/youtube2zim/schemas.py b/scraper/src/youtube2zim/schemas.py index abfbb0e5..bdd340d7 100644 --- a/scraper/src/youtube2zim/schemas.py +++ b/scraper/src/youtube2zim/schemas.py @@ -107,9 +107,9 @@ class Channel(CamelModel): joined_date: str collection_type: str main_playlist: str | None = None - user_long_uploads_playlist: str | None=None - user_short_uploads_playlist: str | None=None - user_lives_playlist: str | None=None + user_long_uploads_playlist: str | None = None + user_short_uploads_playlist: str | None = None + user_lives_playlist: str | None = None playlist_count: int diff --git a/scraper/src/youtube2zim/scraper.py b/scraper/src/youtube2zim/scraper.py index 1d054139..36e02da6 100644 --- a/scraper/src/youtube2zim/scraper.py +++ b/scraper/src/youtube2zim/scraper.py @@ -1083,7 +1083,7 @@ def generate_video_object(video) -> Video: author = videos_channels[video_id] subtitles_list = get_subtitles(video_id) channel_data = get_channel_json(author["channelId"]) - + return Video( id=video_id, title=video["snippet"]["title"], @@ -1227,14 +1227,13 @@ def get_playlist_slug(playlist) -> str: playlist_obj.videos = playlist_obj.videos[:12] if playlist.playlist_id == self.user_long_uploads_playlist_id: - user_long_uploads_playlist_slug = (playlist_slug) - + user_long_uploads_playlist_slug = playlist_slug + if playlist.playlist_id == self.user_short_uploads_playlist_id: - user_short_uploads_playlist_slug = (playlist_slug) + user_short_uploads_playlist_slug = playlist_slug if playlist.playlist_id == self.user_lives_playlist_id: - user_lives_playlist_slug= (playlist_slug) - + user_lives_playlist_slug = playlist_slug if playlist.playlist_id == self.uploads_playlist_id: main_playlist_slug = ( @@ -1271,33 +1270,25 @@ def get_playlist_slug(playlist) -> str: # write channel.json file channel_data = get_channel_json(self.main_channel_id) - channel_data_dict = { - "id":str(self.main_channel_id), - "title":str(self.title), - "description":str(self.description), - "channel_name":channel_data["snippet"]["title"], - "channel_description":channel_data["snippet"]["description"], - "profile_path":"profile.jpg", - "banner_path":"banner.jpg", - "collection_type":self.collection_type, - "main_playlist":main_playlist_slug, - "playlist_count":len(self.playlists), - "joined_date":channel_data["snippet"]["publishedAt"], - } - - if user_long_uploads_playlist_slug is not None : - channel_data_dict["user_long_uploads_playlist"] = user_long_uploads_playlist_slug - - if user_short_uploads_playlist_slug is not None : - channel_data_dict["user_short_uploads_playlist"] = user_short_uploads_playlist_slug - - if user_lives_playlist_slug is not None : - channel_data_dict["user_lives_playlist"] = user_lives_playlist_slug - self.zim_file.add_item_for( path="channel.json", title=self.title, - content = Channel(**channel_data_dict).model_dump_json(by_alias=True, indent=2, exclude_none=True), + content=Channel( + id=str(self.main_channel_id), + title=str(self.title), + description=str(self.description), + channel_name=channel_data["snippet"]["title"], + channel_description=channel_data["snippet"]["description"], + profile_path="profile.jpg", + banner_path="banner.jpg", + collection_type=self.collection_type, + main_playlist=main_playlist_slug, + user_long_uploads_playlist=user_long_uploads_playlist_slug, + user_short_uploads_playlist=user_short_uploads_playlist_slug, + user_lives_playlist=user_lives_playlist_slug, + playlist_count=len(self.playlists), + joined_date=channel_data["snippet"]["publishedAt"], + ).model_dump_json(by_alias=True, indent=2), mimetype="application/json", is_front=False, ) diff --git a/scraper/src/youtube2zim/youtube.py b/scraper/src/youtube2zim/youtube.py index a4dc35f3..1e4ee5cd 100644 --- a/scraper/src/youtube2zim/youtube.py +++ b/scraper/src/youtube2zim/youtube.py @@ -45,6 +45,8 @@ def __init__( @classmethod def from_id(cls, playlist_id): playlist_json = get_playlist_json(playlist_id) + if playlist_json is None: + raise ValueError(f"Playlist with ID `{playlist_id}` was not found.") return Playlist( playlist_id=playlist_id, title=playlist_json["snippet"]["title"], @@ -81,7 +83,6 @@ def credentials_ok(): return False - def get_channel_json(channel_id): """fetch or retieve-save and return the Youtube ChannelResult JSON""" fname = f"channel_{channel_id}" @@ -165,9 +166,13 @@ def get_playlist_json(playlist_id): req.raise_for_status() try: playlist_json = req.json()["items"][0] + total_results = req.json().get("pageInfo", {}).get("totalResults", 0) + if total_results == 0: + logger.error(f"Playlist `{playlist_id}`: No Item Available") + return None except IndexError: logger.error(f"Invalid playlistId `{playlist_id}`: Not Found") - raise + return None save_json(YOUTUBE.cache_dir, fname, playlist_json) return playlist_json @@ -321,128 +326,6 @@ def skip_outofrange_videos(date_range, item): return dt_parser.parse(item["snippet"]["publishedAt"]).date() in date_range -def get_user_short_uploads_playlist_id(channel_id): - '''Return the user's uploaded short playlist ID, or None if shorts are not available or if an error occurs''' - - user_short_uploads_playlist_id = "UUSH" + channel_id[2:] # Generate the short playlist ID - - '''Make the API request to get the playlist details to determine whether shorts are available on the channel''' - - try: - req = requests.get( - PLAYLIST_API, - params={"id": user_short_uploads_playlist_id, "part": "snippet", "key": YOUTUBE.api_key}, - timeout=REQUEST_TIMEOUT, - ) - - # Check for HTTP error response - if req.status_code >= HTTPStatus.BAD_REQUEST: - logger.error(f"HTTP {req.status_code} Error response: {req.text}") - req.raise_for_status() # Raises an HTTPError if the status code is 4xx or 5xx - - # Parse the response - response_json = req.json() - total_results = response_json.get("pageInfo", {}).get("totalResults", 0) - playlist_items = response_json.get("items", []) - - # Check if there are no items or totalResults is 0 if yes then shorts not available - if total_results == 0 or not playlist_items: - logger.error(f"Short Playlist `{user_short_uploads_playlist_id}`: Not Found or No Shorts Available") - return None - - # If everything is successful, return the short playlist ID - return user_short_uploads_playlist_id - - except IndexError: - logger.error(f"User short uploads Playlist `{user_short_uploads_playlist_id}`: Not Found or No uploaded Shorts Available") - return None - - except requests.RequestException as e: - logger.error(f"Request failed: {e}") - return None - -def get_user_long_uploads_playlist_id(channel_id): - '''Return the user's uploaded long videos playlist ID, or None if long videos are not available or if an error occurs''' - - user_long_uploads_playlist_id = "UULF" + channel_id[2:] # Generate the long videos playlist ID - - '''Make the API request to get the playlist details to determine whether long videos are available on the channel''' - - try: - req = requests.get( - PLAYLIST_API, - params={"id": user_long_uploads_playlist_id, "part": "snippet", "key": YOUTUBE.api_key}, - timeout=REQUEST_TIMEOUT, - ) - - # Check for HTTP error response - if req.status_code >= HTTPStatus.BAD_REQUEST: - logger.error(f"HTTP {req.status_code} Error response: {req.text}") - req.raise_for_status() # Raises an HTTPError if the status code is 4xx or 5xx - - - # Parse the response - response_json = req.json() - total_results = response_json.get("pageInfo", {}).get("totalResults", 0) - playlist_items = response_json.get("items", []) - - # Check if there are no items or totalResults is 0 if yes then long videos not available - if total_results == 0 or not playlist_items: - logger.error(f"User Long uploads Playlist `{user_long_uploads_playlist_id}`: Not Found or No uploaded long videos Available") - return None - - # If everything is successful, return the long videos playlist ID - return user_long_uploads_playlist_id - - except IndexError: - logger.error(f"Long videos Playlist `{user_long_uploads_playlist_id}`: Not Found or No long videos Available") - return None - - except requests.RequestException as e: - logger.error(f"Request failed: {e}") - return None - -def get_user_lives_playlist_id(channel_id): - '''Return the user's lives playlist ID, or None if lives are not available or if an error occurs''' - - user_lives_playlist_id = "UULV" + channel_id[2:] # Generate the lives playlist ID - - '''Make the API request to get the playlist details to determine whether Lives are available on the channel''' - - try: - req = requests.get( - PLAYLIST_API, - params={"id": user_lives_playlist_id, "part": "snippet", "key": YOUTUBE.api_key}, - timeout=REQUEST_TIMEOUT, - ) - - # Check for HTTP error response - if req.status_code >= HTTPStatus.BAD_REQUEST: - logger.error(f"HTTP {req.status_code} Error response: {req.text}") - req.raise_for_status() # Raises an HTTPError if the status code is 4xx or 5xx - - # Parse the response - response_json = req.json() - total_results = response_json.get("pageInfo", {}).get("totalResults", 0) - playlist_items = response_json.get("items", []) - - # Check if there are no items or totalResults is 0 if yes then lives not available - if total_results == 0 or not playlist_items: - logger.error(f"User lives Playlist `{user_lives_playlist_id}`: Not Found or No lives Available") - return None - - # If everything is successful, return the live playlist ID - return user_lives_playlist_id - - except IndexError: - logger.error(f"Live Playlist `{user_lives_playlist_id}`: Not Found or No lives Available") - return None - - except requests.RequestException as e: - logger.error(f"Request failed: {e}") - return None - - def extract_playlists_details_from(collection_type, youtube_id): """prepare a list of Playlist from user request @@ -459,25 +342,31 @@ def extract_playlists_details_from(collection_type, youtube_id): # retrieve list of playlists for that channel playlist_ids = [p["id"] for p in get_channel_playlists_json(main_channel_id)] - - # Retrieve the shorts,long videos and lives playlist ID - user_long_uploads_playlist_id = get_user_long_uploads_playlist_id(main_channel_id) - user_short_uploads_playlist_id = get_user_short_uploads_playlist_id(main_channel_id) - user_lives_playlist_id = get_user_lives_playlist_id(main_channel_id) - - - if user_long_uploads_playlist_id is not None: - # include uploads long videos playlist (contains every long videos) - playlist_ids += [user_long_uploads_playlist_id] - - if user_short_uploads_playlist_id is not None: - # include uploads short playlist (contains every shorts) - playlist_ids += [user_short_uploads_playlist_id] - - if user_lives_playlist_id is not None: - # include lives playlist (contains every lives) - playlist_ids += [user_lives_playlist_id] - + + # Get playlist JSON objects + user_long_uploads_json = get_playlist_json("UULF" + main_channel_id[2:]) + user_short_uploads_json = get_playlist_json("UUSH" + main_channel_id[2:]) + user_lives_json = get_playlist_json("UULV" + main_channel_id[2:]) + + # Extract playlist IDs if the JSON objects are not None + user_long_uploads_playlist_id = ( + user_long_uploads_json["id"] if user_long_uploads_json else None + ) + user_short_uploads_playlist_id = ( + user_short_uploads_json["id"] if user_short_uploads_json else None + ) + user_lives_playlist_id = user_lives_json["id"] if user_lives_json else None + + # Add special playlists if they exists + playlist_ids += filter( + None, + [ + user_long_uploads_playlist_id, + user_short_uploads_playlist_id, + user_lives_playlist_id, + ], + ) + # we always include uploads playlist (contains everything) playlist_ids += [channel_json["contentDetails"]["relatedPlaylists"]["uploads"]] uploads_playlist_id = playlist_ids[-1] diff --git a/zimui/src/assets/main.css b/zimui/src/assets/main.css index 251eb294..b73d2399 100644 --- a/zimui/src/assets/main.css +++ b/zimui/src/assets/main.css @@ -6,6 +6,7 @@ html { overflow: auto !important; font-family: 'Roboto', sans-serif; + overflow-y: scroll !important; } body { diff --git a/zimui/src/components/channel/tabs/LivesTab.vue b/zimui/src/components/channel/tabs/LivesTab.vue index fbd8ba05..4165f7ca 100644 --- a/zimui/src/components/channel/tabs/LivesTab.vue +++ b/zimui/src/components/channel/tabs/LivesTab.vue @@ -1,18 +1,7 @@ diff --git a/zimui/src/components/channel/tabs/ShortsGridTab.vue b/zimui/src/components/channel/tabs/ShortsGridTab.vue deleted file mode 100644 index e0227dff..00000000 --- a/zimui/src/components/channel/tabs/ShortsGridTab.vue +++ /dev/null @@ -1,61 +0,0 @@ - - - diff --git a/zimui/src/components/channel/tabs/ShortsTab.vue b/zimui/src/components/channel/tabs/ShortsTab.vue index 96019048..e4fa6090 100644 --- a/zimui/src/components/channel/tabs/ShortsTab.vue +++ b/zimui/src/components/channel/tabs/ShortsTab.vue @@ -1,19 +1,7 @@ diff --git a/zimui/src/components/channel/tabs/VideosGridTab.vue b/zimui/src/components/channel/tabs/VideosGridTab.vue deleted file mode 100644 index 641ae984..00000000 --- a/zimui/src/components/channel/tabs/VideosGridTab.vue +++ /dev/null @@ -1,61 +0,0 @@ - - - diff --git a/zimui/src/components/channel/tabs/VideosTab.vue b/zimui/src/components/channel/tabs/VideosTab.vue index 357393a4..e6e0b3ee 100644 --- a/zimui/src/components/channel/tabs/VideosTab.vue +++ b/zimui/src/components/channel/tabs/VideosTab.vue @@ -1,18 +1,7 @@ diff --git a/zimui/src/components/channel/tabs/LivesGridTab.vue b/zimui/src/views/TabView.vue old mode 100644 new mode 100755 similarity index 57% rename from zimui/src/components/channel/tabs/LivesGridTab.vue rename to zimui/src/views/TabView.vue index 9234bed7..6e80576c --- a/zimui/src/components/channel/tabs/LivesGridTab.vue +++ b/zimui/src/views/TabView.vue @@ -1,6 +1,5 @@