Merge pull request #1092 from Stypox/channel-tabs-improvements

Channel tabs code improvements
TeamNewPipe · Aug 6, 2023 · 7936987 · 7936987
2 parents 95a3cc0 + 6d22271
commit 7936987
Show file tree

Hide file tree

Showing 29 changed files with 2,568 additions and 2,569 deletions.
diff --git a/...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java b/...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java
@@ -43,9 +43,9 @@ public static String resolveChannelId(@Nonnull final String idOrPath)
         }
 
         // If the URL is not a /channel URL, we need to use the navigation/resolve_url endpoint of
-        // the InnerTube API to get the channel id.
-        // Otherwise, we couldn't get information about the channel associated with this URL, if
-        // there is one.
+        // the InnerTube API to get the channel id. If this fails or if the URL is not a /channel
+        // URL, then no information about the channel associated with this URL was found,
+        // so the unresolved url will be returned.
         if (!channelId[0].equals("channel")) {
             final byte[] body = JsonWriter.string(
                     prepareDesktopJsonBuilder(Localization.DEFAULT, ContentCountry.DEFAULT)
@@ -78,6 +78,7 @@ public static String resolveChannelId(@Nonnull final String idOrPath)
             }
         }
 
+        // return the unresolved URL
         return channelId[1];
     }
 
@@ -110,11 +111,11 @@ private ChannelResponseData(@Nonnull final JsonObject jsonResponse,
      * Fetch a YouTube channel tab response, using the given channel ID and tab parameters.
      *
      * <p>
-     * Redirections to other channels such as are supported to up to 3 redirects, which could
-     * happen for instance for localized channels or auto-generated ones such as the {@code Movies
-     * and Shows} (channel IDs {@code UCuJcl0Ju-gPDoksRjK1ya-w}, {@code UChBfWrfBXL9wS6tQtgjt_OQ}
-     * and {@code UCok7UTQQEP1Rsctxiv3gwSQ} of this channel redirect to the
-     * {@code UClgRkhTL3_hImCAmdLfDE4g} one).
+     * Redirections to other channels are supported to up to 3 redirects, which could happen for
+     * instance for localized channels or for auto-generated ones. For instance, there are three IDs
+     * of the auto-generated "Movies and Shows" channel, i.e. {@code UCuJcl0Ju-gPDoksRjK1ya-w},
+     * {@code UChBfWrfBXL9wS6tQtgjt_OQ} and {@code UCok7UTQQEP1Rsctxiv3gwSQ}, and they all redirect
+     * to the {@code UClgRkhTL3_hImCAmdLfDE4g} one.
      * </p>
      *
      * @param channelId    a valid YouTube channel ID
@@ -177,7 +178,7 @@ public static ChannelResponseData getChannelResponse(@Nonnull final String chann
         }
 
         if (ajaxJson == null) {
-            throw new ExtractionException("Got no channel response");
+            throw new ExtractionException("Got no channel response after 3 redirects");
         }
 
         defaultAlertsCheck(ajaxJson);

diff --git a/...ava/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/...ava/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
@@ -57,6 +57,7 @@
 public class YoutubeChannelExtractor extends ChannelExtractor {
 
     private JsonObject jsonResponse;
+
     @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
     private Optional<YoutubeChannelHelper.ChannelHeader> channelHeader;
 
@@ -89,6 +90,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
                 "EgZ2aWRlb3PyBgQKAjoA", getExtractorLocalization(), getExtractorContentCountry());
 
         jsonResponse = data.jsonResponse;
+        channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse);
         channelId = data.channelId;
         channelAgeGateRenderer = getChannelAgeGateRenderer();
     }
@@ -115,12 +117,8 @@ private JsonObject getChannelAgeGateRenderer() {
     }
 
     @Nonnull
-    private Optional<YoutubeChannelHelper.ChannelHeader> getChannelHeader() {
-        //noinspection OptionalAssignedToNull
-        if (channelHeader == null) {
-            channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse);
-        }
-        return channelHeader;
+    private Optional<JsonObject> getChannelHeaderJson() {
+        return channelHeader.map(it -> it.json);
     }
 
     @Nonnull
@@ -136,9 +134,9 @@ public String getUrl() throws ParsingException {
     @Nonnull
     @Override
     public String getId() throws ParsingException {
-        return getChannelHeader()
-                .flatMap(header -> Optional.ofNullable(header.json.getString("channelId"))
-                        .or(() -> Optional.ofNullable(header.json.getObject("navigationEndpoint")
+        return getChannelHeaderJson()
+                .flatMap(header -> Optional.ofNullable(header.getString("channelId"))
+                        .or(() -> Optional.ofNullable(header.getObject("navigationEndpoint")
                                 .getObject("browseEndpoint")
                                 .getString("browseId"))
                 ))
@@ -160,8 +158,8 @@ public String getName() throws ParsingException {
             return metadataRendererTitle;
         }
 
-        return getChannelHeader().flatMap(header -> {
-            final Object title = header.json.get("title");
+        return getChannelHeaderJson().flatMap(header -> {
+            final Object title = header.get("title");
             if (title instanceof String) {
                 return Optional.of((String) title);
             } else if (title instanceof JsonObject) {
@@ -180,7 +178,7 @@ public String getAvatarUrl() throws ParsingException {
         if (channelAgeGateRenderer != null) {
             avatarJsonObjectContainer = channelAgeGateRenderer;
         } else {
-            avatarJsonObjectContainer = getChannelHeader().map(header -> header.json)
+            avatarJsonObjectContainer = getChannelHeaderJson()
                     .orElseThrow(() -> new ParsingException("Could not get avatar URL"));
         }
 
@@ -196,8 +194,8 @@ public String getBannerUrl() throws ParsingException {
             return "";
         }
 
-        return getChannelHeader().flatMap(header -> Optional.ofNullable(
-                        header.json.getObject("banner")
+        return getChannelHeaderJson().flatMap(header -> Optional.ofNullable(
+                        header.getObject("banner")
                                 .getArray("thumbnails")
                                 .getObject(0)
                                 .getString("url")))
@@ -226,9 +224,9 @@ public long getSubscriberCount() throws ParsingException {
             return UNKNOWN_SUBSCRIBER_COUNT;
         }
 
-        final Optional<YoutubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
+        final Optional<JsonObject> headerOpt = getChannelHeaderJson();
         if (headerOpt.isPresent()) {
-            final JsonObject header = headerOpt.get().json;
+            final JsonObject header = headerOpt.get();
             JsonObject textObject = null;
 
             if (header.has("subscriberCountText")) {
@@ -285,9 +283,8 @@ public boolean isVerified() throws ParsingException {
             return false;
         }
 
-        final Optional<YoutubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
-        if (headerOpt.isPresent()) {
-            final YoutubeChannelHelper.ChannelHeader header = headerOpt.get();
+        if (channelHeader.isPresent()) {
+            final YoutubeChannelHelper.ChannelHeader header = channelHeader.get();
 
             // The CarouselHeaderRenderer does not contain any verification badges.
             // Since it is only shown on YT-internal channels or on channels of large organizations

diff --git a/.../org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java b/.../org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
@@ -23,7 +23,6 @@
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Optional;
-import java.util.function.Consumer;
 
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
@@ -299,20 +298,20 @@ private Optional<JsonObject> collectItem(@Nonnull final MultiInfoItemsCollector
                     .getObject("content");
 
             if (richItem.has("videoRenderer")) {
-                getCommitVideoConsumer(collector, timeAgoParser, channelIds).accept(
+                getCommitVideoConsumer(collector, timeAgoParser, channelIds,
                         richItem.getObject("videoRenderer"));
             } else if (richItem.has("reelItemRenderer")) {
-                getCommitReelItemConsumer(collector, timeAgoParser, channelIds).accept(
+                getCommitReelItemConsumer(collector, timeAgoParser, channelIds,
                         richItem.getObject("reelItemRenderer"));
             } else if (richItem.has("playlistRenderer")) {
-                getCommitPlaylistConsumer(collector, channelIds).accept(
+                getCommitPlaylistConsumer(collector, channelIds,
                         item.getObject("playlistRenderer"));
             }
         } else if (item.has("gridVideoRenderer")) {
-            getCommitVideoConsumer(collector, timeAgoParser, channelIds).accept(
+            getCommitVideoConsumer(collector, timeAgoParser, channelIds,
                     item.getObject("gridVideoRenderer"));
         } else if (item.has("gridPlaylistRenderer")) {
-            getCommitPlaylistConsumer(collector, channelIds).accept(
+            getCommitPlaylistConsumer(collector, channelIds,
                     item.getObject("gridPlaylistRenderer"));
         } else if (item.has("gridChannelRenderer")) {
             collector.commit(new YoutubeChannelInfoItemExtractor(
@@ -336,13 +335,12 @@ private Optional<JsonObject> collectItem(@Nonnull final MultiInfoItemsCollector
         return Optional.empty();
     }
 
-    @Nonnull
-    private Consumer<JsonObject> getCommitVideoConsumer(
-            @Nonnull final MultiInfoItemsCollector collector,
-            @Nonnull final TimeAgoParser timeAgoParser,
-            @Nonnull final List<String> channelIds) {
-        return videoRenderer -> collector.commit(
-                new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser) {
+    private void getCommitVideoConsumer(@Nonnull final MultiInfoItemsCollector collector,
+                                        @Nonnull final TimeAgoParser timeAgoParser,
+                                        @Nonnull final List<String> channelIds,
+                                        @Nonnull final JsonObject jsonObject) {
+        collector.commit(
+                new YoutubeStreamInfoItemExtractor(jsonObject, timeAgoParser) {
                     @Override
                     public String getUploaderName() throws ParsingException {
                         if (channelIds.size() >= 2) {
@@ -361,13 +359,12 @@ public String getUploaderUrl() throws ParsingException {
                 });
     }
 
-    @Nonnull
-    private Consumer<JsonObject> getCommitReelItemConsumer(
-            @Nonnull final MultiInfoItemsCollector collector,
-            @Nonnull final TimeAgoParser timeAgoParser,
-            @Nonnull final List<String> channelIds) {
-        return reelItemRenderer -> collector.commit(
-                new YoutubeReelInfoItemExtractor(reelItemRenderer, timeAgoParser) {
+    private void getCommitReelItemConsumer(@Nonnull final MultiInfoItemsCollector collector,
+                                           @Nonnull final TimeAgoParser timeAgoParser,
+                                           @Nonnull final List<String> channelIds,
+                                           @Nonnull final JsonObject jsonObject) {
+        collector.commit(
+                new YoutubeReelInfoItemExtractor(jsonObject, timeAgoParser) {
                     @Override
                     public String getUploaderName() throws ParsingException {
                         if (channelIds.size() >= 2) {
@@ -386,12 +383,11 @@ public String getUploaderUrl() throws ParsingException {
                 });
     }
 
-    @Nonnull
-    private Consumer<JsonObject> getCommitPlaylistConsumer(
-            @Nonnull final MultiInfoItemsCollector collector,
-            @Nonnull final List<String> channelIds) {
-        return playlistRenderer -> collector.commit(
-                new YoutubePlaylistInfoItemExtractor(playlistRenderer) {
+    private void getCommitPlaylistConsumer(@Nonnull final MultiInfoItemsCollector collector,
+                                           @Nonnull final List<String> channelIds,
+                                           @Nonnull final JsonObject jsonObject) {
+        collector.commit(
+                new YoutubePlaylistInfoItemExtractor(jsonObject) {
                     @Override
                     public String getUploaderName() throws ParsingException {
                         if (channelIds.size() >= 2) {

diff --git a/...abi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabPlaylistExtractor.java b/...abi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabPlaylistExtractor.java
@@ -1,6 +1,7 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
-import org.schabi.newpipe.extractor.InfoItem;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
@@ -12,14 +13,11 @@
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
-import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
-import javax.annotation.Nonnull;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
 
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+import javax.annotation.Nonnull;
 
 /**
  * A {@link ChannelTabExtractor} for YouTube system playlists using a
@@ -74,35 +72,19 @@ public void onFetchPage(@Nonnull final Downloader downloader)
 
     @Nonnull
     @Override
-    public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
+    public InfoItemsPage getInitialPage() throws IOException, ExtractionException {
         if (!playlistExisting) {
             return InfoItemsPage.emptyPage();
         }
-
-        final InfoItemsPage<StreamInfoItem> playlistInitialPage =
-                playlistExtractorInstance.getInitialPage();
-
-        // We can't provide the playlist page as it is due to a type conflict, we need to wrap the
-        // page items and provide a new InfoItemsPage
-        final List<InfoItem> infoItems = new ArrayList<>(playlistInitialPage.getItems());
-        return new InfoItemsPage<>(infoItems, playlistInitialPage.getNextPage(),
-                playlistInitialPage.getErrors());
+        return playlistExtractorInstance.getInitialPage();
     }
 
     @Override
-    public InfoItemsPage<InfoItem> getPage(final Page page)
-            throws IOException, ExtractionException {
+    public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException {
         if (!playlistExisting) {
             return InfoItemsPage.emptyPage();
         }
-
-        final InfoItemsPage<StreamInfoItem> playlistPage = playlistExtractorInstance.getPage(page);
-
-        // We can't provide the playlist page as it is due to a type conflict, we need to wrap the
-        // page items and provide a new InfoItemsPage
-        final List<InfoItem> infoItems = new ArrayList<>(playlistPage.getItems());
-        return new InfoItemsPage<>(infoItems, playlistPage.getNextPage(),
-                playlistPage.getErrors());
+        return playlistExtractorInstance.getPage(page);
     }
 
     /**

diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java
@@ -151,8 +151,8 @@ public static void assertContains(
                 "'" + shouldBeContained + "' should be contained inside '" + container + "'");
     }
 
-    public static void assertTabsContained(@Nonnull final List<ListLinkHandler> tabs,
-                                           @Nonnull final String... expectedTabs) {
+    public static void assertTabsContain(@Nonnull final List<ListLinkHandler> tabs,
+                                         @Nonnull final String... expectedTabs) {
         final Set<String> tabSet = tabs.stream()
                 .map(linkHandler -> linkHandler.getContentFilters().get(0))
                 .collect(Collectors.toUnmodifiableSet());

diff --git a/...est/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/...est/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java
@@ -11,7 +11,7 @@
 import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
 
 import static org.junit.jupiter.api.Assertions.*;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContained;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContain;
 import static org.schabi.newpipe.extractor.ServiceList.Bandcamp;
 
 public class BandcampChannelExtractorTest implements BaseChannelExtractorTest {
@@ -84,7 +84,7 @@ public void testOriginalUrl() throws Exception {
     @Test
     @Override
     public void testTabs() throws Exception {
-        assertTabsContained(extractor.getTabs(), ChannelTabs.ALBUMS);
+        assertTabsContain(extractor.getTabs(), ChannelTabs.ALBUMS);
     }
 
     @Test