From d85454186a9766c73c2937a349f1b4e1d16c5f3d Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 12 Aug 2023 22:56:25 +0200 Subject: [PATCH 01/35] Add an Image class to the extractor Objects of this serializable class contains four properties: a URL (as a string), a width, a height (represented as integers) and an estimated resolution level, which can be constructed from a given height. Possible resolution levels are: - UNKNOWN: for unknown heights or heights <= 0; - LOW: for heights > 0 & < 175; - MEDIUM: for heights >= 175 & < 720; - HIGH: for heights >= 720. Getters of these properties are available and the constructor needs these four properties. --- .../org/schabi/newpipe/extractor/Image.java | 211 ++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/Image.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Image.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Image.java new file mode 100644 index 0000000000..e0d56e14ad --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Image.java @@ -0,0 +1,211 @@ +package org.schabi.newpipe.extractor; + +import javax.annotation.Nonnull; +import java.io.Serializable; +import java.util.Objects; + +/** + * Class representing images in the extractor. + * + *

+ * An image has four properties: its URL, its height, its width and its estimated quality level. + *

+ * + *

+ * Depending of the services, the height, the width or both properties may be not known. + * Implementations must use the relevant unknown constants in this case + * ({@link #HEIGHT_UNKNOWN} and {@link #WIDTH_UNKNOWN}), to ensure properly the lack of knowledge + * of one or both of these properties to extractor clients. + *

+ * + *

+ * They should also respect the ranges defined in the estimated image resolution levels as much as + * possible, to ensure consistency to extractor clients. + *

+ */ +public final class Image implements Serializable { + + /** + * Constant representing that the height of an {@link Image} is unknown. + */ + public static final int HEIGHT_UNKNOWN = -1; + + /** + * Constant representing that the width of an {@link Image} is unknown. + */ + public static final int WIDTH_UNKNOWN = -1; + + @Nonnull + private final String url; + private final int height; + private final int width; + @Nonnull + private final ResolutionLevel estimatedResolutionLevel; + + /** + * Construct an {@link Image} instance. + * + * @param url the URL to the image, which should be not null or empty + * @param height the image's height + * @param width the image's width + * @param estimatedResolutionLevel the image's estimated resolution level, which must not be + * null + * @throws NullPointerException if {@code estimatedResolutionLevel} is null + */ + public Image(@Nonnull final String url, + final int height, + final int width, + @Nonnull final ResolutionLevel estimatedResolutionLevel) + throws NullPointerException { + this.url = url; + this.height = height; + this.width = width; + this.estimatedResolutionLevel = Objects.requireNonNull( + estimatedResolutionLevel, "estimatedResolutionLevel is null"); + } + + /** + * Get the URL of this {@link Image}. + * + * @return the {@link Image}'s URL. + */ + @Nonnull + public String getUrl() { + return url; + } + + /** + * Get the height of this {@link Image}. + * + *

+ * If it is unknown, {@link #HEIGHT_UNKNOWN} is returned instead. + *

+ * + * @return the {@link Image}'s height or {@link #HEIGHT_UNKNOWN} + */ + public int getHeight() { + return height; + } + + /** + * Get the width of this {@link Image}. + * + *

+ * If it is unknown, {@link #WIDTH_UNKNOWN} is returned instead. + *

+ * + * @return the {@link Image}'s width or {@link #WIDTH_UNKNOWN} + */ + public int getWidth() { + return width; + } + + /** + * Get the estimated resolution level of this image. + * + *

+ * If it is unknown, {@link ResolutionLevel#UNKNOWN} is returned instead. + *

+ * + * @return the estimated resolution level, which is never {@code null} + * @see ResolutionLevel + */ + @Nonnull + public ResolutionLevel getEstimatedResolutionLevel() { + return estimatedResolutionLevel; + } + + /** + * Get a string representation of this {@link Image} instance. + * + *

+ * The representation will be in the following format, where {@code url}, {@code height}, + * {@code width} and {@code estimatedResolutionLevel} represent the corresponding properties: + *
+ *
+ * {@code Image {url=url, height='height, width=width, + * estimatedResolutionLevel=estimatedResolutionLevel}'} + *

+ * + * @return a string representation of this {@link Image} instance + */ + @Nonnull + @Override + public String toString() { + return "Image {" + "url=" + url + ", height=" + height + ", width=" + width + + ", estimatedResolutionLevel=" + estimatedResolutionLevel + "}"; + } + + /** + * The estimated resolution level of an {@link Image}. + * + *

+ * Some services don't return the size of their images, but we may know for a specific image + * type that a service returns, according to real data, an approximation of the resolution + * level. + *

+ */ + public enum ResolutionLevel { + + /** + * The high resolution level. + * + *

+ * This level applies to images with a height greater than or equal to 720px. + *

+ */ + HIGH, + + /** + * The medium resolution level. + * + *

+ * This level applies to images with a height between 175px inclusive and 720px exclusive. + *

+ */ + MEDIUM, + + /** + * The low resolution level. + * + *

+ * This level applies to images with a height between 1px inclusive and 175px exclusive. + *

+ */ + LOW, + + /** + * The unknown resolution level. + * + *

+ * This value is returned when the extractor doesn't know what resolution level an image + * could have, for example if the extractor loops in an array of images with different + * resolution levels without knowing the height. + *

+ */ + UNKNOWN; + + /** + * Get a {@link ResolutionLevel} based from the given height. + * + * @param heightPx the height from which returning the good {@link ResolutionLevel} + * @return the {@link ResolutionLevel} corresponding to the height provided. See the + * {@link ResolutionLevel} values for details about what value is returned. + */ + public static ResolutionLevel fromHeight(final int heightPx) { + if (heightPx <= 0) { + return UNKNOWN; + } + + if (heightPx < 175) { + return LOW; + } + + if (heightPx < 720) { + return MEDIUM; + } + + return HIGH; + } + } +} From 78ce65769fe39bcdba7f9532b8f240ca2e8b4837 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 12 Aug 2023 22:56:25 +0200 Subject: [PATCH 02/35] Add an ImageSuffix class to the extractor The goal of this utility class is to simply store suffixes which need to be appended to image URLs, in order to get images at the suffix resolution. This class contains four properties: the suffix (as a string), the height, the width (as integers) and the estimated resolution level of the image corresponding to the one represented by the suffix. --- .../newpipe/extractor/utils/ImageSuffix.java | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/ImageSuffix.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ImageSuffix.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ImageSuffix.java new file mode 100644 index 0000000000..d1ba7359c9 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ImageSuffix.java @@ -0,0 +1,104 @@ +package org.schabi.newpipe.extractor.utils; + +import org.schabi.newpipe.extractor.Image.ResolutionLevel; + +import javax.annotation.Nonnull; +import java.io.Serializable; +import java.util.Objects; + +/** + * Serializable class representing a suffix (including its format extension, such as {@code .jpg}) + * which needs to be added to get an image/thumbnail URL with its corresponding height, width and + * estimated resolution level. + * + *

+ * This class is used to construct {@link org.schabi.newpipe.extractor.Image Image} + * instances from a single base URL/path, in order to get all or most image resolutions provided, + * depending of the service and the resolutions provided. + *

+ * + *

+ * Note that this class is not intended to be used externally and so should only be used when + * interfacing with the extractor. + *

+ */ +public final class ImageSuffix implements Serializable { + @Nonnull + private final String suffix; + private final int height; + private final int width; + @Nonnull + private final ResolutionLevel resolutionLevel; + + /** + * Create a new {@link ImageSuffix} instance. + * + * @param suffix the suffix string + * @param height the height corresponding to the image suffix + * @param width the width corresponding to the image suffix + * @param estimatedResolutionLevel the {@link ResolutionLevel} of the image suffix, which must + * not be null + * @throws NullPointerException if {@code estimatedResolutionLevel} is {@code null} + */ + public ImageSuffix(@Nonnull final String suffix, + final int height, + final int width, + @Nonnull final ResolutionLevel estimatedResolutionLevel) + throws NullPointerException { + this.suffix = suffix; + this.height = height; + this.width = width; + this.resolutionLevel = Objects.requireNonNull(estimatedResolutionLevel, + "estimatedResolutionLevel is null"); + } + + /** + * @return the suffix which needs to be appended to get the full image URL + */ + @Nonnull + public String getSuffix() { + return suffix; + } + + /** + * @return the height corresponding to the image suffix, which may be unknown + */ + public int getHeight() { + return height; + } + + /** + * @return the width corresponding to the image suffix, which may be unknown + */ + public int getWidth() { + return width; + } + + /** + * @return the estimated {@link ResolutionLevel} of the suffix, which is never null. + */ + @Nonnull + public ResolutionLevel getResolutionLevel() { + return resolutionLevel; + } + + /** + * Get a string representation of this {@link ImageSuffix} instance. + * + *

+ * The representation will be in the following format, where {@code suffix}, {@code height}, + * {@code width} and {@code resolutionLevel} represent the corresponding properties: + *
+ *
+ * {@code ImageSuffix {url=url, height=height, width=width, resolutionLevel=resolutionLevel}'} + *

+ * + * @return a string representation of this {@link ImageSuffix} instance + */ + @Nonnull + @Override + public String toString() { + return "ImageSuffix {" + "suffix=" + suffix + ", height=" + height + ", width=" + + width + ", resolutionLevel=" + resolutionLevel + "}"; + } +} From 2f3ee8a3f2adf1cfd0f99542054c5becb8fa9d98 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 20 Jul 2022 23:28:06 +0200 Subject: [PATCH 03/35] Replace avatar and thumbnail URLs attributes and methods to List in InfoItems --- .../schabi/newpipe/extractor/InfoItem.java | 26 ++++++++------- .../extractor/comments/CommentsInfoItem.java | 19 +++++++---- .../extractor/stream/StreamInfoItem.java | 32 +++++++++++-------- 3 files changed, 45 insertions(+), 32 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java index cbcab0a50c..3925911d73 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java @@ -1,33 +1,36 @@ -package org.schabi.newpipe.extractor; - /* * Created by Christian Schabesberger on 11.02.17. * * Copyright (C) Christian Schabesberger 2017 - * InfoItem.java is part of NewPipe. + * InfoItem.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ +package org.schabi.newpipe.extractor; + +import javax.annotation.Nonnull; import java.io.Serializable; +import java.util.List; public abstract class InfoItem implements Serializable { private final InfoType infoType; private final int serviceId; private final String url; private final String name; - private String thumbnailUrl; + @Nonnull + private List thumbnails = List.of(); public InfoItem(final InfoType infoType, final int serviceId, @@ -55,12 +58,13 @@ public String getName() { return name; } - public void setThumbnailUrl(final String thumbnailUrl) { - this.thumbnailUrl = thumbnailUrl; + public void setThumbnails(@Nonnull final List thumbnails) { + this.thumbnails = thumbnails; } - public String getThumbnailUrl() { - return thumbnailUrl; + @Nonnull + public List getThumbnails() { + return thumbnails; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java index 0752e9b745..1b679f2cf2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java @@ -1,18 +1,22 @@ package org.schabi.newpipe.extractor.comments; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.Description; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; public class CommentsInfoItem extends InfoItem { private String commentId; private Description commentText; private String uploaderName; - private String uploaderAvatarUrl; + @Nonnull + private List uploaderAvatars = List.of(); private String uploaderUrl; private boolean uploaderVerified; private String textualUploadDate; @@ -60,12 +64,13 @@ public void setUploaderName(final String uploaderName) { this.uploaderName = uploaderName; } - public String getUploaderAvatarUrl() { - return uploaderAvatarUrl; + @Nonnull + public List getUploaderAvatars() { + return uploaderAvatars; } - public void setUploaderAvatarUrl(final String uploaderAvatarUrl) { - this.uploaderAvatarUrl = uploaderAvatarUrl; + public void setUploaderAvatars(@Nonnull final List uploaderAvatars) { + this.uploaderAvatars = uploaderAvatars; } public String getUploaderUrl() { @@ -94,8 +99,8 @@ public void setUploadDate(@Nullable final DateWrapper uploadDate) { } /** - * @return the comment's like count - * or {@link CommentsInfoItem#NO_LIKE_COUNT} if it is unavailable + * @return the comment's like count or {@link CommentsInfoItem#NO_LIKE_COUNT} if it is + * unavailable */ public int getLikeCount() { return likeCount; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java index 37adb475f4..a478a6994b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java @@ -1,32 +1,35 @@ -package org.schabi.newpipe.extractor.stream; - /* * Created by Christian Schabesberger on 26.08.15. * * Copyright (C) Christian Schabesberger 2016 - * StreamInfoItem.java is part of NewPipe. + * StreamInfoItem.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ +package org.schabi.newpipe.extractor.stream; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.localization.DateWrapper; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; /** - * Info object for previews of unopened videos, eg search results, related videos + * Info object for previews of unopened videos, e.g. search results, related videos. */ public class StreamInfoItem extends InfoItem { private final StreamType streamType; @@ -40,7 +43,8 @@ public class StreamInfoItem extends InfoItem { private long duration = -1; private String uploaderUrl = null; - private String uploaderAvatarUrl = null; + @Nonnull + private List uploaderAvatars = List.of(); private boolean uploaderVerified = false; private boolean shortFormContent = false; @@ -88,13 +92,13 @@ public void setUploaderUrl(final String uploaderUrl) { this.uploaderUrl = uploaderUrl; } - @Nullable - public String getUploaderAvatarUrl() { - return uploaderAvatarUrl; + @Nonnull + public List getUploaderAvatars() { + return uploaderAvatars; } - public void setUploaderAvatarUrl(final String uploaderAvatarUrl) { - this.uploaderAvatarUrl = uploaderAvatarUrl; + public void setUploaderAvatars(@Nonnull final List uploaderAvatars) { + this.uploaderAvatars = uploaderAvatars; } public String getShortDescription() { @@ -152,7 +156,7 @@ public String toString() { + ", serviceId=" + getServiceId() + ", url='" + getUrl() + '\'' + ", name='" + getName() + '\'' - + ", thumbnailUrl='" + getThumbnailUrl() + '\'' + + ", thumbnails='" + getThumbnails() + '\'' + ", uploaderVerified='" + isUploaderVerified() + '\'' + '}'; } From ca1d4a6fa4e3c679dba514098c409a463d676af8 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 20 Jul 2022 23:31:47 +0200 Subject: [PATCH 04/35] Replace avatar and thumbnail URLs attributes and methods to List in InfoItemExtractors --- .../newpipe/extractor/InfoItemExtractor.java | 6 +- .../comments/CommentsInfoItemExtractor.java | 8 ++- .../stream/StreamInfoItemExtractor.java | 55 ++++++++++--------- 3 files changed, 41 insertions(+), 28 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemExtractor.java index 0e38ceecda..ed59741764 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemExtractor.java @@ -2,8 +2,12 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; +import javax.annotation.Nonnull; +import java.util.List; + public interface InfoItemExtractor { String getName() throws ParsingException; String getUrl() throws ParsingException; - String getThumbnailUrl() throws ParsingException; + @Nonnull + List getThumbnails() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java index 695478764f..dfff3bda4f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.comments; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.InfoItemExtractor; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -8,7 +9,9 @@ import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamExtractor; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; public interface CommentsInfoItemExtractor extends InfoItemExtractor { @@ -77,8 +80,9 @@ default String getUploaderName() throws ParsingException { return ""; } - default String getUploaderAvatarUrl() throws ParsingException { - return ""; + @Nonnull + default List getUploaderAvatars() throws ParsingException { + return List.of(); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java index 77a633e2dc..62e69a4338 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java @@ -1,39 +1,41 @@ -package org.schabi.newpipe.extractor.stream; - -import org.schabi.newpipe.extractor.InfoItemExtractor; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.localization.DateWrapper; - -import javax.annotation.Nullable; - /* * Created by Christian Schabesberger on 28.02.16. * * Copyright (C) Christian Schabesberger 2016 - * StreamInfoItemExtractor.java is part of NewPipe. + * StreamInfoItemExtractor.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ -public interface StreamInfoItemExtractor extends InfoItemExtractor { +package org.schabi.newpipe.extractor.stream; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.InfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; + +public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Get the stream type * * @return the stream type - * @throws ParsingException thrown if there is an error in the extraction + * @throws ParsingException if there is an error in the extraction */ StreamType getStreamType() throws ParsingException; @@ -41,7 +43,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { * Check if the stream is an ad. * * @return {@code true} if the stream is an ad. - * @throws ParsingException thrown if there is an error in the extraction + * @throws ParsingException if there is an error in the extraction */ boolean isAd() throws ParsingException; @@ -49,7 +51,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { * Get the stream duration in seconds * * @return the stream duration in seconds - * @throws ParsingException thrown if there is an error in the extraction + * @throws ParsingException if there is an error in the extraction */ long getDuration() throws ParsingException; @@ -57,7 +59,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { * Parses the number of views * * @return the number of views or -1 for live streams - * @throws ParsingException thrown if there is an error in the extraction + * @throws ParsingException if there is an error in the extraction */ long getViewCount() throws ParsingException; @@ -65,27 +67,29 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { * Get the uploader name * * @return the uploader name - * @throws ParsingException if parsing fails + * @throws ParsingException if there is an error in the extraction */ String getUploaderName() throws ParsingException; String getUploaderUrl() throws ParsingException; /** - * Get the uploader's avatar + * Get the uploader avatars. * - * @return The uploader's avatar url or {@code null} if not provided by the service. + * @return the uploader avatars or an empty list if not provided by the service * @throws ParsingException if there is an error in the extraction */ - @Nullable - String getUploaderAvatarUrl() throws ParsingException; + @Nonnull + default List getUploaderAvatars() throws ParsingException { + return List.of(); + } /** * Whether the uploader has been verified by the service's provider. * If there is no verification implemented, return false. * * @return whether the uploader has been verified by the service's provider - * @throws ParsingException + * @throws ParsingException if there is an error in the extraction */ boolean isUploaderVerified() throws ParsingException; @@ -109,8 +113,8 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { *

* * @return The date and time (can be approximated) this item was uploaded or {@code null}. - * @throws ParsingException if there is an error in the extraction - * or the extracted date couldn't be parsed. + * @throws ParsingException if there is an error in the extraction or the extracted date + * couldn't be parsed * @see #getTextualUploadDate() */ @Nullable @@ -137,6 +141,7 @@ default String getShortDescription() throws ParsingException { *

* * @return whether the stream is a short-form content + * @throws ParsingException if there is an error in the extraction */ default boolean isShortFormContent() throws ParsingException { return false; From 0f4a5a81841df6c03352a4c9778b36d808d779c0 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 20 Jul 2022 23:36:27 +0200 Subject: [PATCH 05/35] Replace avatar and thumbnail URLs attributes and methods to List in InfoItemsCollectors --- .../channel/ChannelInfoItemsCollector.java | 20 ++++++------ .../comments/CommentsInfoItemsCollector.java | 4 +-- .../playlist/PlaylistInfoItemsCollector.java | 2 +- .../stream/StreamInfoItemsCollector.java | 31 +++++++++---------- 4 files changed, 28 insertions(+), 29 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java index 5b085f8b86..5b8ff6918c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java @@ -1,28 +1,28 @@ -package org.schabi.newpipe.extractor.channel; - -import org.schabi.newpipe.extractor.InfoItemsCollector; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - /* * Created by Christian Schabesberger on 12.02.17. * * Copyright (C) Christian Schabesberger 2017 - * ChannelInfoItemsCollector.java is part of NewPipe. + * ChannelInfoItemsCollector.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ +package org.schabi.newpipe.extractor.channel; + +import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + public final class ChannelInfoItemsCollector extends InfoItemsCollector { public ChannelInfoItemsCollector(final int serviceId) { @@ -47,7 +47,7 @@ public ChannelInfoItem extract(final ChannelInfoItemExtractor extractor) addError(e); } try { - resultItem.setThumbnailUrl(extractor.getThumbnailUrl()); + resultItem.setThumbnails(extractor.getThumbnails()); } catch (final Exception e) { addError(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java index 3afeb04554..fca5bdf59b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java @@ -36,7 +36,7 @@ public CommentsInfoItem extract(final CommentsInfoItemExtractor extractor) addError(e); } try { - resultItem.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl()); + resultItem.setUploaderAvatars(extractor.getUploaderAvatars()); } catch (final Exception e) { addError(e); } @@ -66,7 +66,7 @@ public CommentsInfoItem extract(final CommentsInfoItemExtractor extractor) addError(e); } try { - resultItem.setThumbnailUrl(extractor.getThumbnailUrl()); + resultItem.setThumbnails(extractor.getThumbnails()); } catch (final Exception e) { addError(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java index 4fe35e40e8..0b854a999f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java @@ -32,7 +32,7 @@ public PlaylistInfoItem extract(final PlaylistInfoItemExtractor extractor) addError(e); } try { - resultItem.setThumbnailUrl(extractor.getThumbnailUrl()); + resultItem.setThumbnails(extractor.getThumbnails()); } catch (final Exception e) { addError(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java index 35cf7fd328..c0e1ac1e68 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java @@ -1,31 +1,31 @@ -package org.schabi.newpipe.extractor.stream; - -import org.schabi.newpipe.extractor.InfoItemsCollector; -import org.schabi.newpipe.extractor.exceptions.FoundAdException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -import java.util.Comparator; - /* * Created by Christian Schabesberger on 28.02.16. * * Copyright (C) Christian Schabesberger 2016 - * StreamInfoItemsCollector.java is part of NewPipe. + * StreamInfoItemsCollector.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ +package org.schabi.newpipe.extractor.stream; + +import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.exceptions.FoundAdException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import java.util.Comparator; + public class StreamInfoItemsCollector extends InfoItemsCollector { @@ -74,7 +74,7 @@ public StreamInfoItem extract(final StreamInfoItemExtractor extractor) throws Pa addError(e); } try { - resultItem.setThumbnailUrl(extractor.getThumbnailUrl()); + resultItem.setThumbnails(extractor.getThumbnails()); } catch (final Exception e) { addError(e); } @@ -84,7 +84,7 @@ public StreamInfoItem extract(final StreamInfoItemExtractor extractor) throws Pa addError(e); } try { - resultItem.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl()); + resultItem.setUploaderAvatars(extractor.getUploaderAvatars()); } catch (final Exception e) { addError(e); } @@ -111,8 +111,7 @@ public StreamInfoItem extract(final StreamInfoItemExtractor extractor) throws Pa public void commit(final StreamInfoItemExtractor extractor) { try { addItem(extract(extractor)); - } catch (final FoundAdException ae) { - //System.out.println("AD_WARNING: " + ae.getMessage()); + } catch (final FoundAdException ignored) { } catch (final Exception e) { addError(e); } From 9d8098576ea3da32277cf417ff4e52acfeeb176a Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Thu, 21 Jul 2022 00:17:45 +0200 Subject: [PATCH 06/35] Replace avatar and thumbnail URLs attributes and methods to List in Extractors --- .../extractor/channel/ChannelExtractor.java | 38 +++++++------ .../extractor/playlist/PlaylistExtractor.java | 21 ++++---- .../extractor/stream/StreamExtractor.java | 53 ++++++++++++------- 3 files changed, 66 insertions(+), 46 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index db0333647a..d5587ec4ed 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -1,33 +1,34 @@ -package org.schabi.newpipe.extractor.channel; - -import org.schabi.newpipe.extractor.Extractor; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; - -import javax.annotation.Nonnull; -import java.util.List; - /* * Created by Christian Schabesberger on 25.07.16. * * Copyright (C) Christian Schabesberger 2016 - * ChannelExtractor.java is part of NewPipe. + * ChannelExtractor.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ +package org.schabi.newpipe.extractor.channel; + +import org.schabi.newpipe.extractor.Extractor; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; + +import javax.annotation.Nonnull; +import java.util.List; + public abstract class ChannelExtractor extends Extractor { public static final long UNKNOWN_SUBSCRIBER_COUNT = -1; @@ -36,14 +37,17 @@ protected ChannelExtractor(final StreamingService service, final ListLinkHandler super(service, linkHandler); } - public abstract String getAvatarUrl() throws ParsingException; - public abstract String getBannerUrl() throws ParsingException; + @Nonnull + public abstract List getAvatars() throws ParsingException; + @Nonnull + public abstract List getBanners() throws ParsingException; public abstract String getFeedUrl() throws ParsingException; public abstract long getSubscriberCount() throws ParsingException; public abstract String getDescription() throws ParsingException; public abstract String getParentChannelName() throws ParsingException; public abstract String getParentChannelUrl() throws ParsingException; - public abstract String getParentChannelAvatarUrl() throws ParsingException; + @Nonnull + public abstract List getParentChannelAvatars() throws ParsingException; public abstract boolean isVerified() throws ParsingException; @Nonnull public abstract List getTabs() throws ParsingException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index bc4eee467c..a714deadb7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.playlist; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -9,6 +10,9 @@ import javax.annotation.Nonnull; +import java.util.Collections; +import java.util.List; + public abstract class PlaylistExtractor extends ListExtractor { public PlaylistExtractor(final StreamingService service, final ListLinkHandler linkHandler) { @@ -17,7 +21,8 @@ public PlaylistExtractor(final StreamingService service, final ListLinkHandler l public abstract String getUploaderUrl() throws ParsingException; public abstract String getUploaderName() throws ParsingException; - public abstract String getUploaderAvatarUrl() throws ParsingException; + @Nonnull + public abstract List getUploaderAvatars() throws ParsingException; public abstract boolean isUploaderVerified() throws ParsingException; public abstract long getStreamCount() throws ParsingException; @@ -26,15 +31,13 @@ public PlaylistExtractor(final StreamingService service, final ListLinkHandler l public abstract Description getDescription() throws ParsingException; @Nonnull - public String getThumbnailUrl() throws ParsingException { - return ""; + public List getThumbnails() throws ParsingException { + return Collections.emptyList(); } @Nonnull - public String getBannerUrl() throws ParsingException { - // Banner can't be handled by frontend right now. - // Whoever is willing to implement this should also implement it in the frontend. - return ""; + public List getBanners() throws ParsingException { + return List.of(); } @Nonnull @@ -48,8 +51,8 @@ public String getSubChannelUrl() throws ParsingException { } @Nonnull - public String getSubChannelAvatarUrl() throws ParsingException { - return ""; + public List getSubChannelAvatars() throws ParsingException { + return List.of(); } public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 6fbcf8fbb5..f974cade0e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -1,25 +1,26 @@ -package org.schabi.newpipe.extractor.stream; - /* * Created by Christian Schabesberger on 10.08.18. * * Copyright (C) Christian Schabesberger 2016 - * StreamExtractor.java is part of NewPipe. + * StreamExtractor.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ +package org.schabi.newpipe.extractor.stream; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.InfoItemExtractor; @@ -87,13 +88,12 @@ public DateWrapper getUploadDate() throws ParsingException { } /** - * This will return the url to the thumbnail of the stream. Try to return the medium resolution - * here. + * This will return the thumbnails of the stream. * - * @return The url of the thumbnail. + * @return the thumbnails of the stream */ @Nonnull - public abstract String getThumbnailUrl() throws ParsingException; + public abstract List getThumbnails() throws ParsingException; /** * This is the stream description. @@ -208,14 +208,18 @@ public long getUploaderSubscriberCount() throws ParsingException { } /** - * The url to the image file/profile picture/avatar of the creator/uploader of the stream. - * If the url is not available you can return an empty String. + * The image files/profile pictures/avatars of the creator/uploader of the stream. * - * @return The url of the image file of the uploader or an empty String + *

+ * If they are not available in the stream on specific cases, you must return an empty list for + * these ones, like it is made by default. + *

+ * + * @return the avatars of the sub-channel of the stream or an empty list (default) */ @Nonnull - public String getUploaderAvatarUrl() throws ParsingException { - return ""; + public List getUploaderAvatars() throws ParsingException { + return List.of(); } /** @@ -243,14 +247,23 @@ public String getSubChannelName() throws ParsingException { } /** - * The url to the image file/profile picture/avatar of the sub-channel of the stream. - * If the url is not available you can return an empty String. + * The avatars of the sub-channel of the stream. * - * @return The url of the image file of the sub-channel or an empty String + *

+ * If they are not available in the stream on specific cases, you must return an empty list for + * these ones, like it is made by default. + *

+ * + *

+ * If the concept of sub-channels doesn't apply to the stream's service, keep the default + * implementation. + *

+ * + * @return the avatars of the sub-channel of the stream or an empty list (default) */ @Nonnull - public String getSubChannelAvatarUrl() throws ParsingException { - return ""; + public List getSubChannelAvatars() throws ParsingException { + return List.of(); } /** From d56b880cae7393ed71f31deec770182352f8e412 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 22 Jul 2022 15:22:14 +0200 Subject: [PATCH 07/35] Replace avatar and thumbnail URLs attributes and methods to List in Infos --- .../extractor/channel/ChannelInfo.java | 77 ++++++------ .../extractor/playlist/PlaylistInfo.java | 67 +++++----- .../newpipe/extractor/stream/StreamInfo.java | 119 ++++++++---------- 3 files changed, 133 insertions(+), 130 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 4c05de6920..502901e29c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -1,36 +1,37 @@ -package org.schabi.newpipe.extractor.channel; - -import org.schabi.newpipe.extractor.Info; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; - -import java.io.IOException; -import java.util.List; - -import javax.annotation.Nonnull; - /* * Created by Christian Schabesberger on 31.07.16. * * Copyright (C) Christian Schabesberger 2016 - * ChannelInfo.java is part of NewPipe. + * ChannelInfo.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ +package org.schabi.newpipe.extractor.channel; + +import org.schabi.newpipe.extractor.Info; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; + +import java.io.IOException; +import java.util.List; + +import javax.annotation.Nonnull; + public class ChannelInfo extends Info { public ChannelInfo(final int serviceId, @@ -64,13 +65,13 @@ public static ChannelInfo getInfo(final ChannelExtractor extractor) final ChannelInfo info = new ChannelInfo(serviceId, id, url, originalUrl, name); try { - info.setAvatarUrl(extractor.getAvatarUrl()); + info.setAvatars(extractor.getAvatars()); } catch (final Exception e) { info.addError(e); } try { - info.setBannerUrl(extractor.getBannerUrl()); + info.setBanners(extractor.getBanners()); } catch (final Exception e) { info.addError(e); } @@ -106,7 +107,7 @@ public static ChannelInfo getInfo(final ChannelExtractor extractor) } try { - info.setParentChannelAvatarUrl(extractor.getParentChannelAvatarUrl()); + info.setParentChannelAvatars(extractor.getParentChannelAvatars()); } catch (final Exception e) { info.addError(e); } @@ -132,15 +133,18 @@ public static ChannelInfo getInfo(final ChannelExtractor extractor) return info; } - private String avatarUrl; private String parentChannelName; private String parentChannelUrl; - private String parentChannelAvatarUrl; - private String bannerUrl; private String feedUrl; private long subscriberCount = -1; private String description; private String[] donationLinks; + @Nonnull + private List avatars = List.of(); + @Nonnull + private List banners = List.of(); + @Nonnull + private List parentChannelAvatars = List.of(); private boolean verified; private List tabs = List.of(); private List tags = List.of(); @@ -161,28 +165,31 @@ public void setParentChannelUrl(final String parentChannelUrl) { this.parentChannelUrl = parentChannelUrl; } - public String getParentChannelAvatarUrl() { - return parentChannelAvatarUrl; + @Nonnull + public List getParentChannelAvatars() { + return parentChannelAvatars; } - public void setParentChannelAvatarUrl(final String parentChannelAvatarUrl) { - this.parentChannelAvatarUrl = parentChannelAvatarUrl; + public void setParentChannelAvatars(@Nonnull final List parentChannelAvatars) { + this.parentChannelAvatars = parentChannelAvatars; } - public String getAvatarUrl() { - return avatarUrl; + @Nonnull + public List getAvatars() { + return avatars; } - public void setAvatarUrl(final String avatarUrl) { - this.avatarUrl = avatarUrl; + public void setAvatars(@Nonnull final List avatars) { + this.avatars = avatars; } - public String getBannerUrl() { - return bannerUrl; + @Nonnull + public List getBanners() { + return banners; } - public void setBannerUrl(final String bannerUrl) { - this.bannerUrl = bannerUrl; + public void setBanners(@Nonnull final List banners) { + this.banners = banners; } public String getFeedUrl() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java index 97a0d530d8..bb29ca7d0f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.playlist; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.NewPipe; @@ -12,6 +13,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.utils.ExtractorHelper; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -109,26 +111,23 @@ public static PlaylistInfo getInfo(final PlaylistExtractor extractor) info.addError(e); } try { - info.setThumbnailUrl(extractor.getThumbnailUrl()); + info.setThumbnails(extractor.getThumbnails()); } catch (final Exception e) { info.addError(e); } try { info.setUploaderUrl(extractor.getUploaderUrl()); } catch (final Exception e) { - info.setUploaderUrl(""); uploaderParsingErrors.add(e); } try { info.setUploaderName(extractor.getUploaderName()); } catch (final Exception e) { - info.setUploaderName(""); uploaderParsingErrors.add(e); } try { - info.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl()); + info.setUploaderAvatars(extractor.getUploaderAvatars()); } catch (final Exception e) { - info.setUploaderAvatarUrl(""); uploaderParsingErrors.add(e); } try { @@ -142,12 +141,12 @@ public static PlaylistInfo getInfo(final PlaylistExtractor extractor) uploaderParsingErrors.add(e); } try { - info.setSubChannelAvatarUrl(extractor.getSubChannelAvatarUrl()); + info.setSubChannelAvatars(extractor.getSubChannelAvatars()); } catch (final Exception e) { uploaderParsingErrors.add(e); } try { - info.setBannerUrl(extractor.getBannerUrl()); + info.setBanners(extractor.getBanners()); } catch (final Exception e) { info.addError(e); } @@ -171,32 +170,38 @@ public static PlaylistInfo getInfo(final PlaylistExtractor extractor) return info; } - private String thumbnailUrl; - private String bannerUrl; - private String uploaderUrl; - private String uploaderName; - private String uploaderAvatarUrl; + private String uploaderUrl = ""; + private String uploaderName = ""; private String subChannelUrl; private String subChannelName; - private String subChannelAvatarUrl; - private long streamCount = 0; private Description description; + @Nonnull + private List banners = List.of(); + @Nonnull + private List subChannelAvatars = List.of(); + @Nonnull + private List thumbnails = List.of(); + @Nonnull + private List uploaderAvatars = List.of(); + private long streamCount; private PlaylistType playlistType; - public String getThumbnailUrl() { - return thumbnailUrl; + @Nonnull + public List getThumbnails() { + return thumbnails; } - public void setThumbnailUrl(final String thumbnailUrl) { - this.thumbnailUrl = thumbnailUrl; + public void setThumbnails(@Nonnull final List thumbnails) { + this.thumbnails = thumbnails; } - public String getBannerUrl() { - return bannerUrl; + @Nonnull + public List getBanners() { + return banners; } - public void setBannerUrl(final String bannerUrl) { - this.bannerUrl = bannerUrl; + public void setBanners(@Nonnull final List banners) { + this.banners = banners; } public String getUploaderUrl() { @@ -215,12 +220,13 @@ public void setUploaderName(final String uploaderName) { this.uploaderName = uploaderName; } - public String getUploaderAvatarUrl() { - return uploaderAvatarUrl; + @Nonnull + public List getUploaderAvatars() { + return uploaderAvatars; } - public void setUploaderAvatarUrl(final String uploaderAvatarUrl) { - this.uploaderAvatarUrl = uploaderAvatarUrl; + public void setUploaderAvatars(@Nonnull final List uploaderAvatars) { + this.uploaderAvatars = uploaderAvatars; } public String getSubChannelUrl() { @@ -239,12 +245,13 @@ public void setSubChannelName(final String subChannelName) { this.subChannelName = subChannelName; } - public String getSubChannelAvatarUrl() { - return subChannelAvatarUrl; + @Nonnull + public List getSubChannelAvatars() { + return subChannelAvatars; } - public void setSubChannelAvatarUrl(final String subChannelAvatarUrl) { - this.subChannelAvatarUrl = subChannelAvatarUrl; + public void setSubChannelAvatars(@Nonnull final List subChannelAvatars) { + this.subChannelAvatars = subChannelAvatars; } public long getStreamCount() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index fa979c34b2..252bc7f149 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -1,5 +1,26 @@ +/* + * Created by Christian Schabesberger on 26.08.15. + * + * Copyright (C) Christian Schabesberger 2016 + * StreamInfo.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + package org.schabi.newpipe.extractor.stream; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Info; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.MetaInfo; @@ -12,8 +33,6 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Locale; @@ -21,26 +40,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; -/* - * Created by Christian Schabesberger on 26.08.15. - * - * Copyright (C) Christian Schabesberger 2016 - * StreamInfo.java is part of NewPipe Extractor. - * - * NewPipe Extractor is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe Extractor is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe Extractor. If not, see . - */ - /** * Info object for opened contents, i.e. the content ready to play. */ @@ -106,9 +105,7 @@ private static StreamInfo extractImportantData(@Nonnull final StreamExtractor ex // Important data, without it the content can't be displayed. // If one of these is not available, the frontend will receive an exception directly. - final int serviceId = extractor.getServiceId(); final String url = extractor.getUrl(); - final String originalUrl = extractor.getOriginalUrl(); final StreamType streamType = extractor.getStreamType(); final String id = extractor.getId(); final String name = extractor.getName(); @@ -148,7 +145,6 @@ private static void extractStreams(final StreamInfo streamInfo, streamInfo.addError(new ExtractionException("Couldn't get HLS manifest", e)); } - /* Load and extract audio */ try { streamInfo.setAudioStreams(extractor.getAudioStreams()); } catch (final ContentNotSupportedException e) { @@ -157,31 +153,18 @@ private static void extractStreams(final StreamInfo streamInfo, streamInfo.addError(new ExtractionException("Couldn't get audio streams", e)); } - /* Extract video stream url */ try { streamInfo.setVideoStreams(extractor.getVideoStreams()); } catch (final Exception e) { streamInfo.addError(new ExtractionException("Couldn't get video streams", e)); } - /* Extract video only stream url */ try { streamInfo.setVideoOnlyStreams(extractor.getVideoOnlyStreams()); } catch (final Exception e) { streamInfo.addError(new ExtractionException("Couldn't get video only streams", e)); } - // Lists can be null if an exception was thrown during extraction - if (streamInfo.getVideoStreams() == null) { - streamInfo.setVideoStreams(Collections.emptyList()); - } - if (streamInfo.getVideoOnlyStreams() == null) { - streamInfo.setVideoOnlyStreams(Collections.emptyList()); - } - if (streamInfo.getAudioStreams() == null) { - streamInfo.setAudioStreams(Collections.emptyList()); - } - // Either audio or video has to be available, otherwise we didn't get a stream (since // videoOnly are optional, they don't count). if ((streamInfo.videoStreams.isEmpty()) && (streamInfo.audioStreams.isEmpty())) { @@ -199,7 +182,7 @@ private static void extractOptionalData(final StreamInfo streamInfo, // so the frontend can afterwards check where errors happened. try { - streamInfo.setThumbnailUrl(extractor.getThumbnailUrl()); + streamInfo.setThumbnails(extractor.getThumbnails()); } catch (final Exception e) { streamInfo.addError(e); } @@ -219,7 +202,7 @@ private static void extractOptionalData(final StreamInfo streamInfo, streamInfo.addError(e); } try { - streamInfo.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl()); + streamInfo.setUploaderAvatars(extractor.getUploaderAvatars()); } catch (final Exception e) { streamInfo.addError(e); } @@ -245,7 +228,7 @@ private static void extractOptionalData(final StreamInfo streamInfo, streamInfo.addError(e); } try { - streamInfo.setSubChannelAvatarUrl(extractor.getSubChannelAvatarUrl()); + streamInfo.setSubChannelAvatars(extractor.getSubChannelAvatars()); } catch (final Exception e) { streamInfo.addError(e); } @@ -353,7 +336,8 @@ private static void extractOptionalData(final StreamInfo streamInfo, } private StreamType streamType; - private String thumbnailUrl = ""; + @Nonnull + private List thumbnails = List.of(); private String textualUploadDate; private DateWrapper uploadDate; private long duration = -1; @@ -366,24 +350,26 @@ private static void extractOptionalData(final StreamInfo streamInfo, private String uploaderName = ""; private String uploaderUrl = ""; - private String uploaderAvatarUrl = ""; + @Nonnull + private List uploaderAvatars = List.of(); private boolean uploaderVerified = false; private long uploaderSubscriberCount = -1; private String subChannelName = ""; private String subChannelUrl = ""; - private String subChannelAvatarUrl = ""; + @Nonnull + private List subChannelAvatars = List.of(); - private List videoStreams = new ArrayList<>(); - private List audioStreams = new ArrayList<>(); - private List videoOnlyStreams = new ArrayList<>(); + private List videoStreams = List.of(); + private List audioStreams = List.of(); + private List videoOnlyStreams = List.of(); private String dashMpdUrl = ""; private String hlsUrl = ""; - private List relatedItems = new ArrayList<>(); + private List relatedItems = List.of(); private long startPosition = 0; - private List subtitles = new ArrayList<>(); + private List subtitles = List.of(); private String host = ""; private StreamExtractor.Privacy privacy; @@ -391,15 +377,15 @@ private static void extractOptionalData(final StreamInfo streamInfo, private String licence = ""; private String supportInfo = ""; private Locale language = null; - private List tags = new ArrayList<>(); - private List streamSegments = new ArrayList<>(); - private List metaInfo = new ArrayList<>(); + private List tags = List.of(); + private List streamSegments = List.of(); + private List metaInfo = List.of(); private boolean shortFormContent = false; /** * Preview frames, e.g. for the storyboard / seekbar thumbnail preview */ - private List previewFrames = Collections.emptyList(); + private List previewFrames = List.of(); /** * Get the stream type @@ -419,12 +405,13 @@ public void setStreamType(final StreamType streamType) { * * @return the thumbnail url as a string */ - public String getThumbnailUrl() { - return thumbnailUrl; + @Nonnull + public List getThumbnails() { + return thumbnails; } - public void setThumbnailUrl(final String thumbnailUrl) { - this.thumbnailUrl = thumbnailUrl; + public void setThumbnails(@Nonnull final List thumbnails) { + this.thumbnails = thumbnails; } public String getTextualUploadDate() { @@ -522,12 +509,13 @@ public void setUploaderUrl(final String uploaderUrl) { this.uploaderUrl = uploaderUrl; } - public String getUploaderAvatarUrl() { - return uploaderAvatarUrl; + @Nonnull + public List getUploaderAvatars() { + return uploaderAvatars; } - public void setUploaderAvatarUrl(final String uploaderAvatarUrl) { - this.uploaderAvatarUrl = uploaderAvatarUrl; + public void setUploaderAvatars(@Nonnull final List uploaderAvatars) { + this.uploaderAvatars = uploaderAvatars; } public boolean isUploaderVerified() { @@ -562,12 +550,13 @@ public void setSubChannelUrl(final String subChannelUrl) { this.subChannelUrl = subChannelUrl; } - public String getSubChannelAvatarUrl() { - return subChannelAvatarUrl; + @Nonnull + public List getSubChannelAvatars() { + return subChannelAvatars; } - public void setSubChannelAvatarUrl(final String subChannelAvatarUrl) { - this.subChannelAvatarUrl = subChannelAvatarUrl; + public void setSubChannelAvatars(@Nonnull final List subChannelAvatars) { + this.subChannelAvatars = subChannelAvatars; } public List getVideoStreams() { From adfad086ac21a8aa7af375bbf16ab2761592dc64 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 22 Jul 2022 17:28:39 +0200 Subject: [PATCH 08/35] [YouTube] Add utility methods to get images from InfoItems and thumbnails arrays Unmodifiable lists of Images are returned, parsed from a given YouTube "thumbnails" JSON array. These methods will be used in all YouTube extractors and InfoItems, as the structures between content types (videos, channels, playlists, ...) are common. --- .../youtube/YoutubeParsingHelper.java | 58 +++++++++++++++++-- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index aec5508972..1bce2c2f82 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -33,6 +33,9 @@ import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; import org.jsoup.nodes.Entities; + +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException; @@ -69,6 +72,7 @@ import java.util.Random; import java.util.Set; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -1133,17 +1137,61 @@ public static String fixThumbnailUrl(@Nonnull final String thumbnailUrl) { return result; } - public static String getThumbnailUrlFromInfoItem(final JsonObject infoItem) + /** + * Get thumbnails from a {@link JsonObject} representing a YouTube + * {@link org.schabi.newpipe.extractor.InfoItem InfoItem}. + * + *

+ * Thumbnails are got from the {@code thumbnails} {@link JsonArray} inside the {@code thumbnail} + * {@link JsonObject} of the YouTube {@link org.schabi.newpipe.extractor.InfoItem InfoItem}, + * using {@link #getImagesFromThumbnailsArray(JsonArray)}. + *

+ * + * @param infoItem a YouTube {@link org.schabi.newpipe.extractor.InfoItem InfoItem} + * @return an unmodifiable list of {@link Image}s found in the {@code thumbnails} + * {@link JsonArray} + * @throws ParsingException if an exception occurs when + * {@link #getImagesFromThumbnailsArray(JsonArray)} is executed + */ + @Nonnull + public static List getThumbnailsFromInfoItem(@Nonnull final JsonObject infoItem) throws ParsingException { - // TODO: Don't simply get the first item, but look at all thumbnails and their resolution try { - return fixThumbnailUrl(infoItem.getObject("thumbnail").getArray("thumbnails") - .getObject(0).getString("url")); + return getImagesFromThumbnailsArray(infoItem.getObject("thumbnail") + .getArray("thumbnails")); } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url", e); + throw new ParsingException("Could not get thumbnails from InfoItem", e); } } + /** + * Get images from a YouTube {@code thumbnails} {@link JsonArray}. + * + *

+ * The properties of the {@link Image}s created will be set using the corresponding ones of + * thumbnail items. + *

+ * + * @param thumbnails a YouTube {@code thumbnails} {@link JsonArray} + * @return an unmodifiable list of {@link Image}s extracted from the given {@link JsonArray} + */ + @Nonnull + public static List getImagesFromThumbnailsArray( + @Nonnull final JsonArray thumbnails) { + return thumbnails.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(thumbnail -> !isNullOrEmpty(thumbnail.getString("url"))) + .map(thumbnail -> { + final int height = thumbnail.getInt("height", Image.HEIGHT_UNKNOWN); + return new Image(fixThumbnailUrl(thumbnail.getString("url")), + height, + thumbnail.getInt("width", Image.WIDTH_UNKNOWN), + ResolutionLevel.fromHeight(height)); + }) + .collect(Collectors.toUnmodifiableList()); + } + @Nonnull public static String getValidJsonResponseBody(@Nonnull final Response response) throws ParsingException, MalformedURLException { From 4cc99f9ce1ba059441222b95386af722f6b19658 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 22 Jul 2022 21:34:12 +0200 Subject: [PATCH 09/35] [YouTube] Apply changes in InfoItemExtractors except YouTube Music ones --- .../YoutubeChannelInfoItemExtractor.java | 52 +++++++------ .../YoutubeCommentsInfoItemExtractor.java | 41 +++++----- .../YoutubeFeedInfoItemExtractor.java | 55 ++++++++++--- ...YoutubeMixOrPlaylistInfoItemExtractor.java | 11 ++- .../YoutubePlaylistInfoItemExtractor.java | 15 ++-- .../YoutubeReelInfoItemExtractor.java | 19 +++-- .../YoutubeStreamInfoItemExtractor.java | 77 ++++++++++--------- 7 files changed, 164 insertions(+), 106 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index e01f0cfb9a..302fb3ef90 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -1,37 +1,41 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; - -import com.grack.nanojson.JsonObject; - -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.utils.Utils; - -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; - /* * Created by Christian Schabesberger on 12.02.17. * * Copyright (C) Christian Schabesberger 2017 - * YoutubeChannelInfoItemExtractor.java is part of NewPipe. + * YoutubeChannelInfoItemExtractor.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; + +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; + public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor { private final JsonObject channelInfoItem; /** @@ -53,15 +57,13 @@ public YoutubeChannelInfoItemExtractor(final JsonObject channelInfoItem) { this.withHandle = wHandle; } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public List getThumbnails() throws ParsingException { try { - final String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails") - .getObject(0).getString("url"); - - return fixThumbnailUrl(url); + return getThumbnailsFromInfoItem(channelInfoItem); } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url", e); + throw new ParsingException("Could not get thumbnails", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java index 95209a65fa..fb6463a54b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -11,9 +12,12 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; import static org.schabi.newpipe.extractor.comments.CommentsInfoItem.UNKNOWN_REPLY_COUNT; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { @@ -42,20 +46,25 @@ private JsonObject getCommentRenderer() throws ParsingException { return commentRenderer; } + @Nonnull + private List getAuthorThumbnails() throws ParsingException { + try { + return getImagesFromThumbnailsArray(JsonUtils.getArray(getCommentRenderer(), + "authorThumbnail.thumbnails")); + } catch (final Exception e) { + throw new ParsingException("Could not get author thumbnails", e); + } + } + @Override public String getUrl() throws ParsingException { return url; } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - try { - final JsonArray arr = JsonUtils.getArray(getCommentRenderer(), - "authorThumbnail.thumbnails"); - return JsonUtils.getString(arr.getObject(2), "url"); - } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } + public List getThumbnails() throws ParsingException { + return getAuthorThumbnails(); } @Override @@ -204,15 +213,10 @@ public String getCommentId() throws ParsingException { } } + @Nonnull @Override - public String getUploaderAvatarUrl() throws ParsingException { - try { - final JsonArray arr = JsonUtils.getArray(getCommentRenderer(), - "authorThumbnail.thumbnails"); - return JsonUtils.getString(arr.getObject(2), "url"); - } catch (final Exception e) { - throw new ParsingException("Could not get author thumbnail", e); - } + public List getUploaderAvatars() throws ParsingException { + return getAuthorThumbnails(); } @Override @@ -228,6 +232,7 @@ public boolean isPinned() throws ParsingException { return getCommentRenderer().has("pinnedCommentBadge"); } + @Override public boolean isUploaderVerified() throws ParsingException { return getCommentRenderer().has("authorCommentBadge"); } @@ -261,7 +266,7 @@ public int getReplyCount() throws ParsingException { } @Override - public Page getReplies() throws ParsingException { + public Page getReplies() { try { final String id = JsonUtils.getString( JsonUtils.getArray(json, "replies.commentRepliesRenderer.contents") diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java index d5a83d3b71..d917eb2d7b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java @@ -1,14 +1,18 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.time.OffsetDateTime; import java.time.format.DateTimeParseException; +import java.util.List; public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { private final Element entryElement; @@ -51,12 +55,6 @@ public String getUploaderUrl() { return entryElement.select("author > uri").first().text(); } - @Nullable - @Override - public String getUploaderAvatarUrl() throws ParsingException { - return null; - } - @Override public boolean isUploaderVerified() throws ParsingException { return false; @@ -89,12 +87,51 @@ public String getUrl() { return entryElement.getElementsByTag("link").first().attr("href"); } + @Nonnull @Override - public String getThumbnailUrl() { + public List getThumbnails() { + final Element thumbnailElement = entryElement.getElementsByTag("media:thumbnail").first(); + if (thumbnailElement == null) { + return List.of(); + } + + final String feedThumbnailUrl = thumbnailElement.attr("url"); + + // If the thumbnail URL is empty, it means that no thumbnail is available, return an empty + // list in this case + if (feedThumbnailUrl.isEmpty()) { + return List.of(); + } + // The hqdefault thumbnail has some black bars at the top and at the bottom, while the // mqdefault doesn't, so return the mqdefault one. It should always exist, according to // https://stackoverflow.com/a/20542029/9481500. - return entryElement.getElementsByTag("media:thumbnail").first().attr("url") - .replace("hqdefault", "mqdefault"); + final String newFeedThumbnailUrl = feedThumbnailUrl.replace("hqdefault", "mqdefault"); + + int height; + int width; + + // If the new thumbnail URL is equal to the feed one, it means that a different image + // resolution is used on feeds, so use the height and width provided instead of the + // mqdefault ones + if (newFeedThumbnailUrl.equals(feedThumbnailUrl)) { + try { + height = Integer.parseInt(thumbnailElement.attr("height")); + } catch (final NumberFormatException e) { + height = Image.HEIGHT_UNKNOWN; + } + + try { + width = Integer.parseInt(thumbnailElement.attr("width")); + } catch (final NumberFormatException e) { + width = Image.WIDTH_UNKNOWN; + } + } else { + height = 320; + width = 180; + } + + return List.of( + new Image(newFeedThumbnailUrl, height, width, ResolutionLevel.fromHeight(height))); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java index 4847302e6c..16c7a3e3ea 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java @@ -2,11 +2,12 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfo; @@ -14,6 +15,7 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import javax.annotation.Nonnull; +import java.util.List; public class YoutubeMixOrPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private final JsonObject mixInfoItem; @@ -40,9 +42,10 @@ public String getUrl() throws ParsingException { return url; } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return getThumbnailUrlFromInfoItem(mixInfoItem); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromInfoItem(mixInfoItem); } @Override @@ -75,7 +78,7 @@ public long getStreamCount() throws ParsingException { return Integer.parseInt(countString); } catch (final NumberFormatException ignored) { // un-parsable integer: this is a mix with infinite items and "50+" as count string - // (though youtube music mixes do not necessarily have an infinite count of songs) + // (though YouTube Music mixes do not necessarily have an infinite count of songs) return ListExtractor.ITEM_COUNT_INFINITE; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index c3a7707dc5..a0584a20fb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -2,17 +2,21 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromObject; - public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private final JsonObject playlistInfoItem; @@ -20,8 +24,9 @@ public YoutubePlaylistInfoItemExtractor(final JsonObject playlistInfoItem) { this.playlistInfoItem = playlistInfoItem; } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public List getThumbnails() throws ParsingException { try { JsonArray thumbnails = playlistInfoItem.getArray("thumbnails") .getObject(0) @@ -31,9 +36,9 @@ public String getThumbnailUrl() throws ParsingException { .getArray("thumbnails"); } - return fixThumbnailUrl(thumbnails.getObject(0).getString("url")); + return getImagesFromThumbnailsArray(thumbnails); } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url", e); + throw new ParsingException("Could not get thumbnails", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java index 1b68c2ddfe..911cb4bedf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; @@ -13,9 +15,11 @@ import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import java.util.List; + /** * A {@link StreamInfoItemExtractor} for YouTube's {@code reelItemRenderers}. * @@ -53,9 +57,10 @@ public String getUrl() throws ParsingException { } } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return getThumbnailUrlFromInfoItem(reelInfo); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromInfoItem(reelInfo); } @Override @@ -101,7 +106,7 @@ public long getViewCount() throws ParsingException { } @Override - public boolean isShortFormContent() throws ParsingException { + public boolean isShortFormContent() { return true; } @@ -122,12 +127,6 @@ public String getUploaderUrl() throws ParsingException { return null; } - @Nullable - @Override - public String getUploaderAvatarUrl() throws ParsingException { - return null; - } - @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 1607e0a0f8..178cc2bf67 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -1,7 +1,33 @@ +/* + * Copyright (C) Christian Schabesberger 2016 + * YoutubeStreamInfoItemExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + package org.schabi.newpipe.extractor.services.youtube.extractors; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; @@ -15,35 +41,14 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; + import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.util.List; import java.util.regex.Pattern; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - -/* - * Copyright (C) Christian Schabesberger 2016 - * YoutubeStreamInfoItemExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { private static final Pattern ACCESSIBILITY_DATA_VIEW_COUNT_REGEX = @@ -215,21 +220,22 @@ public String getUploaderUrl() throws ParsingException { return url; } - @Nullable + @Nonnull @Override - public String getUploaderAvatarUrl() throws ParsingException { + public List getUploaderAvatars() throws ParsingException { if (videoInfo.has("channelThumbnailSupportedRenderers")) { - return JsonUtils.getArray(videoInfo, "channelThumbnailSupportedRenderers" - + ".channelThumbnailWithLinkRenderer.thumbnail.thumbnails") - .getObject(0).getString("url"); + return getImagesFromThumbnailsArray(JsonUtils.getArray(videoInfo, + // CHECKSTYLE:OFF + "channelThumbnailSupportedRenderers.channelThumbnailWithLinkRenderer.thumbnail.thumbnails")); + // CHECKSTYLE:ON } if (videoInfo.has("channelThumbnail")) { - return JsonUtils.getArray(videoInfo, "channelThumbnail.thumbnails") - .getObject(0).getString("url"); + return getImagesFromThumbnailsArray( + JsonUtils.getArray(videoInfo, "channelThumbnail.thumbnails")); } - return null; + return List.of(); } @Override @@ -371,9 +377,10 @@ private long getViewCountFromAccessibilityData() videoInfoTitleAccessibilityData))); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return getThumbnailUrlFromInfoItem(videoInfo); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromInfoItem(videoInfo); } private boolean isPremium() { @@ -409,10 +416,10 @@ private OffsetDateTime getDateFromPremiere() throws ParsingException { @Nullable @Override public String getShortDescription() throws ParsingException { - if (videoInfo.has("detailedMetadataSnippets")) { return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets") - .getObject(0).getObject("snippetText")); + .getObject(0) + .getObject("snippetText")); } if (videoInfo.has("descriptionSnippet")) { From c1981ed54feba983172fc6f1e0a74cbefb0f2da4 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 22 Jul 2022 22:31:43 +0200 Subject: [PATCH 10/35] [YouTube] Apply changes in Extractors except YoutubeMusicSearchExtractor Also improve a bit some code related to the changes. --- .../extractors/YoutubeChannelExtractor.java | 104 ++++++++---------- .../YoutubeMixPlaylistExtractor.java | 38 +++++-- .../extractors/YoutubePlaylistExtractor.java | 54 ++++----- .../extractors/YoutubeStreamExtractor.java | 44 +++----- 4 files changed, 110 insertions(+), 130 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 66b927e02a..f35d20ebcc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -1,3 +1,23 @@ +/* + * Created by Christian Schabesberger on 25.07.16. + * + * Copyright (C) Christian Schabesberger 2018 + * YoutubeChannelExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse; @@ -8,6 +28,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; @@ -36,26 +57,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -/* - * Created by Christian Schabesberger on 25.07.16. - * - * Copyright (C) Christian Schabesberger 2018 - * YoutubeChannelExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - public class YoutubeChannelExtractor extends ChannelExtractor { private JsonObject jsonResponse; @@ -190,16 +191,15 @@ public String getName() throws ParsingException { .orElseThrow(() -> new ParsingException("Could not get channel name")); } + @Nonnull @Override - public String getAvatarUrl() throws ParsingException { + public List getAvatars() throws ParsingException { assertPageFetched(); if (channelAgeGateRenderer != null) { return Optional.ofNullable(channelAgeGateRenderer.getObject("avatar") - .getArray("thumbnails") - .getObject(0) - .getString("url")) - .map(YoutubeParsingHelper::fixThumbnailUrl) - .orElseThrow(() -> new ParsingException("Could not get avatar URL")); + .getArray("thumbnails")) + .map(YoutubeParsingHelper::getImagesFromThumbnailsArray) + .orElseThrow(() -> new ParsingException("Could not get avatars")); } return channelHeader.map(header -> { @@ -210,56 +210,37 @@ public String getAvatarUrl() throws ParsingException { .getObject("image") .getObject("contentPreviewImageViewModel") .getObject("image") - .getArray("sources") - .getObject(0) - .getString("url"); + .getArray("sources"); case INTERACTIVE_TABBED: return header.json.getObject("boxArt") - .getArray("thumbnails") - .getObject(0) - .getString("url"); + .getArray("thumbnails"); case C4_TABBED: case CAROUSEL: default: return header.json.getObject("avatar") - .getArray("thumbnails") - .getObject(0) - .getString("url"); + .getArray("thumbnails"); } }) - .map(YoutubeParsingHelper::fixThumbnailUrl) - .orElseThrow(() -> new ParsingException("Could not get avatar URL")); + .map(YoutubeParsingHelper::getImagesFromThumbnailsArray) + .orElseThrow(() -> new ParsingException("Could not get avatars")); } + @Nonnull @Override - public String getBannerUrl() throws ParsingException { + public List getBanners() { assertPageFetched(); if (channelAgeGateRenderer != null) { - return null; - } - - if (channelHeader.isPresent()) { - final ChannelHeader header = channelHeader.get(); - if (header.headerType == HeaderType.PAGE) { - // No banner is available on pageHeaderRenderer headers - return null; - } - - return Optional.ofNullable(header.json.getObject("banner") - .getArray("thumbnails") - .getObject(0) - .getString("url")) - .filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner")) - .map(YoutubeParsingHelper::fixThumbnailUrl) - // Channels may not have a banner, so no exception should be thrown if no - // banner is found - // Return null in this case - .orElse(null); + return List.of(); } - return null; + // No banner is available on pageHeaderRenderer headers + return channelHeader.filter(header -> header.headerType != HeaderType.PAGE) + .map(header -> header.json.getObject("banner") + .getArray("thumbnails")) + .map(YoutubeParsingHelper::getImagesFromThumbnailsArray) + .orElse(List.of()); } @Override @@ -359,9 +340,10 @@ public String getParentChannelUrl() { return ""; } + @Nonnull @Override - public String getParentChannelAvatarUrl() { - return ""; + public List getParentChannelAvatars() { + return List.of(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index eefc10a94d..c88a2918e5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -17,6 +17,8 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; @@ -34,6 +36,7 @@ import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.ImageSuffix; import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; @@ -43,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -53,6 +57,12 @@ * {@code youtube.com/watch?v=videoId&list=playlistId} */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { + private static final List IMAGE_URL_SUFFIXES_AND_RESOLUTIONS = List.of( + // sqdefault and maxresdefault image resolutions are not available on all + // videos, so don't add them in the list of available resolutions + new ImageSuffix("default.jpg", 90, 120, ResolutionLevel.LOW), + new ImageSuffix("mqdefault.jpg", 180, 320, ResolutionLevel.MEDIUM), + new ImageSuffix("hqdefault.jpg", 360, 480, ResolutionLevel.MEDIUM)); /** * YouTube identifies mixes based on this cookie. With this information it can generate @@ -126,18 +136,18 @@ public String getName() throws ParsingException { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public List getThumbnails() throws ParsingException { try { - return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId")); + return getThumbnailsFromPlaylistId(playlistData.getString("playlistId")); } catch (final Exception e) { try { - // Fallback to thumbnail of current video. Always the case for channel mix - return getThumbnailUrlFromVideoId(initialData.getObject("currentVideoEndpoint") + // Fallback to thumbnail of current video. Always the case for channel mixes + return getThumbnailsFromVideoId(initialData.getObject("currentVideoEndpoint") .getObject("watchEndpoint").getString("videoId")); } catch (final Exception ignored) { } - throw new ParsingException("Could not get playlist thumbnail", e); + throw new ParsingException("Could not get playlist thumbnails", e); } } @@ -153,10 +163,11 @@ public String getUploaderName() { return "YouTube"; } + @Nonnull @Override - public String getUploaderAvatarUrl() { + public List getUploaderAvatars() { // YouTube mixes are auto-generated by YouTube - return ""; + return List.of(); } @Override @@ -264,14 +275,19 @@ private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collecto } @Nonnull - private String getThumbnailUrlFromPlaylistId(@Nonnull final String playlistId) + private List getThumbnailsFromPlaylistId(@Nonnull final String playlistId) throws ParsingException { - return getThumbnailUrlFromVideoId(YoutubeParsingHelper.extractVideoIdFromMixId(playlistId)); + return getThumbnailsFromVideoId(YoutubeParsingHelper.extractVideoIdFromMixId(playlistId)); } @Nonnull - private String getThumbnailUrlFromVideoId(final String videoId) { - return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; + private List getThumbnailsFromVideoId(@Nonnull final String videoId) { + final String baseUrl = "https://i.ytimg.com/vi/" + videoId + "/"; + return IMAGE_URL_SUFFIXES_AND_RESOLUTIONS.stream() + .map(imageSuffix -> new Image(baseUrl + imageSuffix.getSuffix(), + imageSuffix.getHeight(), imageSuffix.getWidth(), + imageSuffix.getResolutionLevel())) + .collect(Collectors.toUnmodifiableList()); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index bbb5955e00..bb4df7cd29 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -3,10 +3,10 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -15,6 +15,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -33,6 +34,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -160,39 +162,35 @@ public String getName() throws ParsingException { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - String url; + public List getThumbnails() throws ParsingException { + final JsonArray playlistMetadataThumbnailsArray; if (isNewPlaylistInterface) { - url = getPlaylistHeader().getObject("playlistHeaderBanner") + playlistMetadataThumbnailsArray = getPlaylistHeader().getObject("playlistHeaderBanner") .getObject("heroPlaylistThumbnailRenderer") .getObject("thumbnail") - .getArray("thumbnails") - .getObject(0) - .getString("url"); + .getArray("thumbnails"); } else { - url = getPlaylistInfo().getObject("thumbnailRenderer") + playlistMetadataThumbnailsArray = playlistInfo.getObject("thumbnailRenderer") .getObject("playlistVideoThumbnailRenderer") .getObject("thumbnail") - .getArray("thumbnails") - .getObject(0) - .getString("url"); + .getArray("thumbnails"); + } + + if (!isNullOrEmpty(playlistMetadataThumbnailsArray)) { + return getImagesFromThumbnailsArray(playlistMetadataThumbnailsArray); } // This data structure is returned in both layouts - if (isNullOrEmpty(url)) { - url = browseResponse.getObject("microformat") + final JsonArray microFormatThumbnailsArray = browseResponse.getObject("microformat") .getObject("microformatDataRenderer") .getObject("thumbnail") - .getArray("thumbnails") - .getObject(0) - .getString("url"); + .getArray("thumbnails"); - if (isNullOrEmpty(url)) { - throw new ParsingException("Could not get playlist thumbnail"); - } + if (!isNullOrEmpty(microFormatThumbnailsArray)) { + return getImagesFromThumbnailsArray(microFormatThumbnailsArray); } - return fixThumbnailUrl(url); + throw new ParsingException("Could not get playlist thumbnails"); } @Override @@ -220,23 +218,19 @@ public String getUploaderName() throws ParsingException { } } + @Nonnull @Override - public String getUploaderAvatarUrl() throws ParsingException { + public List getUploaderAvatars() throws ParsingException { if (isNewPlaylistInterface) { // The new playlist interface doesn't provide an uploader avatar - return ""; + return List.of(); } try { - final String url = getUploaderInfo() - .getObject("thumbnail") - .getArray("thumbnails") - .getObject(0) - .getString("url"); - - return fixThumbnailUrl(url); + return getImagesFromThumbnailsArray(getUploaderInfo().getObject("thumbnail") + .getArray("thumbnails")); } catch (final Exception e) { - throw new ParsingException("Could not get playlist uploader avatar", e); + throw new ParsingException("Could not get playlist uploader avatars", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 70f6255a86..c78ee5b961 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -30,11 +30,12 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAttributedDescription; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAttributedDescription; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder; @@ -47,6 +48,7 @@ import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.MultiInfoItemsCollector; @@ -258,23 +260,15 @@ public DateWrapper getUploadDate() throws ParsingException { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public List getThumbnails() throws ParsingException { assertPageFetched(); try { - final JsonArray thumbnails = playerResponse - .getObject("videoDetails") + return getImagesFromThumbnailsArray(playerResponse.getObject("videoDetails") .getObject("thumbnail") - .getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - final String url = thumbnails - .getObject(thumbnails.size() - 1) - .getString("url"); - - return fixThumbnailUrl(url); + .getArray("thumbnails")); } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url"); + throw new ParsingException("Could not get thumbnails"); } - } @Nonnull @@ -552,26 +546,20 @@ public boolean isUploaderVerified() throws ParsingException { @Nonnull @Override - public String getUploaderAvatarUrl() throws ParsingException { + public List getUploaderAvatars() throws ParsingException { assertPageFetched(); - final String url = getVideoSecondaryInfoRenderer() - .getObject("owner") - .getObject("videoOwnerRenderer") - .getObject("thumbnail") - .getArray("thumbnails") - .getObject(0) - .getString("url"); - - if (isNullOrEmpty(url)) { - if (ageLimit == NO_AGE_LIMIT) { - throw new ParsingException("Could not get uploader avatar URL"); - } + final List imageList = getImagesFromThumbnailsArray( + getVideoSecondaryInfoRenderer().getObject("owner") + .getObject("videoOwnerRenderer") + .getObject("thumbnail") + .getArray("thumbnails")); - return ""; + if (imageList.isEmpty() && ageLimit == NO_AGE_LIMIT) { + throw new ParsingException("Could not get uploader avatars"); } - return fixThumbnailUrl(url); + return imageList; } @Override From 266cd1f76bf2bf8d6996cac8cecffd5474d7a471 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sun, 24 Jul 2022 19:52:07 +0200 Subject: [PATCH 11/35] [YouTube] Apply changes in YoutubeMusicSearchExtractor and split its InfoItemExtractors into separate classes Splitting YoutubeMusicSearchExtractor's InfoItemExtractors into separate classes (YoutubeMusicSongOrVideoInfoItemExtractor, YoutubeMusicAlbumOrPlaylistInfoItemExtractor and YoutubeMusicArtistInfoItemExtractor) allows to simplify YoutubeMusicSearchExtractor,improves reading and applying changes to InfoItems (no more losing at least quarter of a line due to indentations). These InfoItems, in which the image changes have been applied, don't extend the YouTube ones anymore, as most methods were overridden and the few ones that are not don't apply in YouTube Music items responses, so it was useless to extend them. The code of YoutubeMusicSearchExtractor have been also improved a bit. --- ...MusicAlbumOrPlaylistInfoItemExtractor.java | 157 ++++++++ .../YoutubeMusicArtistInfoItemExtractor.java | 95 +++++ .../YoutubeMusicSearchExtractor.java | 353 ++---------------- ...tubeMusicSongOrVideoInfoItemExtractor.java | 177 +++++++++ 4 files changed, 467 insertions(+), 315 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java new file mode 100644 index 0000000000..16619d0a19 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java @@ -0,0 +1,157 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import org.schabi.newpipe.extractor.utils.Utils; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; + +import static org.schabi.newpipe.extractor.ListExtractor.ITEM_COUNT_MORE_THAN_100; +import static org.schabi.newpipe.extractor.ListExtractor.ITEM_COUNT_UNKNOWN; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +public class YoutubeMusicAlbumOrPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { + private final JsonObject albumOrPlaylistInfoItem; + private final JsonArray descriptionElements; + private final String searchType; + + public YoutubeMusicAlbumOrPlaylistInfoItemExtractor(final JsonObject albumOrPlaylistInfoItem, + final JsonArray descriptionElements, + final String searchType) { + this.albumOrPlaylistInfoItem = albumOrPlaylistInfoItem; + this.descriptionElements = descriptionElements; + this.searchType = searchType; + } + + @Nonnull + @Override + public List getThumbnails() throws ParsingException { + try { + return getImagesFromThumbnailsArray( + albumOrPlaylistInfoItem.getObject("thumbnail") + .getObject("musicThumbnailRenderer") + .getObject("thumbnail") + .getArray("thumbnails")); + } catch (final Exception e) { + throw new ParsingException("Could not get thumbnails", e); + } + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(albumOrPlaylistInfoItem.getArray("flexColumns") + .getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text")); + + if (!isNullOrEmpty(name)) { + return name; + } + + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + String playlistId = albumOrPlaylistInfoItem.getObject("menu") + .getObject("menuRenderer") + .getArray("items") + .getObject(4) + .getObject("toggleMenuServiceItemRenderer") + .getObject("toggledServiceEndpoint") + .getObject("likeEndpoint") + .getObject("target") + .getString("playlistId"); + + if (isNullOrEmpty(playlistId)) { + playlistId = albumOrPlaylistInfoItem.getObject("overlay") + .getObject("musicItemThumbnailOverlayRenderer") + .getObject("content") + .getObject("musicPlayButtonRenderer") + .getObject("playNavigationEndpoint") + .getObject("watchPlaylistEndpoint") + .getString("playlistId"); + } + + if (!isNullOrEmpty(playlistId)) { + return "https://music.youtube.com/playlist?list=" + playlistId; + } + + throw new ParsingException("Could not get URL"); + } + + @Override + public String getUploaderName() throws ParsingException { + final String name; + if (searchType.equals(MUSIC_ALBUMS)) { + name = descriptionElements.getObject(2).getString("text"); + } else { + name = descriptionElements.getObject(0).getString("text"); + } + + if (!isNullOrEmpty(name)) { + return name; + } + + throw new ParsingException("Could not get uploader name"); + } + + @Nullable + @Override + public String getUploaderUrl() throws ParsingException { + if (searchType.equals(MUSIC_PLAYLISTS)) { + return null; + } + + final JsonArray items = albumOrPlaylistInfoItem.getObject("menu") + .getObject("menuRenderer") + .getArray("items"); + for (final Object item : items) { + final JsonObject menuNavigationItemRenderer = + ((JsonObject) item).getObject("menuNavigationItemRenderer"); + if (menuNavigationItemRenderer.getObject("icon") + .getString("iconType", "") + .equals("ARTIST")) { + return getUrlFromNavigationEndpoint( + menuNavigationItemRenderer.getObject("navigationEndpoint")); + } + } + + throw new ParsingException("Could not get uploader URL"); + } + + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + + @Override + public long getStreamCount() throws ParsingException { + if (searchType.equals(MUSIC_ALBUMS)) { + return ITEM_COUNT_UNKNOWN; + } + + final String count = descriptionElements.getObject(2) + .getString("text"); + + if (!isNullOrEmpty(count)) { + if (count.contains("100+")) { + return ITEM_COUNT_MORE_THAN_100; + } else { + return Long.parseLong(Utils.removeNonDigitCharacters(count)); + } + } + + throw new ParsingException("Could not get stream count"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java new file mode 100644 index 0000000000..477eaf7096 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java @@ -0,0 +1,95 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Utils; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +public class YoutubeMusicArtistInfoItemExtractor implements ChannelInfoItemExtractor { + private final JsonObject artistInfoItem; + + public YoutubeMusicArtistInfoItemExtractor(final JsonObject artistInfoItem) { + this.artistInfoItem = artistInfoItem; + } + + @Nonnull + @Override + public List getThumbnails() throws ParsingException { + try { + return getImagesFromThumbnailsArray( + artistInfoItem.getObject("thumbnail") + .getObject("musicThumbnailRenderer") + .getObject("thumbnail") + .getArray("thumbnails")); + } catch (final Exception e) { + throw new ParsingException("Could not get thumbnails", e); + } + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(artistInfoItem.getArray("flexColumns") + .getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text")); + if (!isNullOrEmpty(name)) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + final String url = getUrlFromNavigationEndpoint( + artistInfoItem.getObject("navigationEndpoint")); + if (!isNullOrEmpty(url)) { + return url; + } + throw new ParsingException("Could not get URL"); + } + + @Override + public long getSubscriberCount() throws ParsingException { + final String subscriberCount = getTextFromObject(artistInfoItem.getArray("flexColumns") + .getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text")); + if (!isNullOrEmpty(subscriberCount)) { + try { + return Utils.mixedNumberWordToLong(subscriberCount); + } catch (final Parser.RegexException ignored) { + // probably subscriberCount == "No subscribers" or similar + return 0; + } + } + throw new ParsingException("Could not get subscriber count"); + } + + @Override + public long getStreamCount() { + return -1; + } + + @Override + public boolean isVerified() { + // An artist on YouTube Music is always verified + return true; + } + + @Nullable + @Override + public String getDescription() { + return null; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index ae6b10a652..3efe6a74ea 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -1,9 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicHeaders; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; @@ -29,19 +27,16 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; -import org.schabi.newpipe.extractor.utils.Parser; -import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -222,7 +217,7 @@ public InfoItemsPage getPage(final Page page) .object("client") .value("clientName", "WEB_REMIX") .value("clientVersion", youtubeMusicKeys[2]) - .value("hl", "en") + .value("hl", "en-GB") .value("gl", getExtractorContentCountry().getCountryCode()) .array("experimentIds").end() .value("experimentsToken", "") @@ -263,316 +258,44 @@ public InfoItemsPage getPage(final Page page) return new InfoItemsPage<>(collector, getNextPageFrom(continuations)); } - @SuppressWarnings("MethodLength") private void collectMusicStreamsFrom(final MultiInfoItemsCollector collector, @Nonnull final JsonArray videos) { - final TimeAgoParser timeAgoParser = getTimeAgoParser(); - - for (final Object item : videos) { - final JsonObject info = ((JsonObject) item) - .getObject("musicResponsiveListItemRenderer", null); - if (info != null) { - final String displayPolicy = info.getString("musicItemRendererDisplayPolicy", - ""); - if (displayPolicy.equals("MUSIC_ITEM_RENDERER_DISPLAY_POLICY_GREY_OUT")) { - continue; // No info about video URL available - } - - final JsonObject flexColumnRenderer = info.getArray("flexColumns") - .getObject(1) - .getObject("musicResponsiveListItemFlexColumnRenderer"); - final JsonArray descriptionElements = flexColumnRenderer.getObject("text") - .getArray("runs"); - final String searchType = getLinkHandler().getContentFilters().get(0); - if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { - collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) { - @Override - public String getUrl() throws ParsingException { - final String id = info.getObject("playlistItemData") - .getString("videoId"); - if (!isNullOrEmpty(id)) { - return "https://music.youtube.com/watch?v=" + id; - } - throw new ParsingException("Could not get url"); - } - - @Override - public String getName() throws ParsingException { - final String name = getTextFromObject(info.getArray("flexColumns") - .getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get name"); - } - - @Override - public long getDuration() throws ParsingException { - final String duration = descriptionElements - .getObject(descriptionElements.size() - 1) - .getString("text"); - if (!isNullOrEmpty(duration)) { - return YoutubeParsingHelper.parseDurationString(duration); - } - throw new ParsingException("Could not get duration"); - } - - @Override - public String getUploaderName() throws ParsingException { - final String name = descriptionElements.getObject(0).getString("text"); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get uploader name"); - } - - @Override - public String getUploaderUrl() throws ParsingException { - if (searchType.equals(MUSIC_VIDEOS)) { - final JsonArray items = info.getObject("menu") - .getObject("menuRenderer") - .getArray("items"); - for (final Object item : items) { - final JsonObject menuNavigationItemRenderer = - ((JsonObject) item).getObject( - "menuNavigationItemRenderer"); - if (menuNavigationItemRenderer.getObject("icon") - .getString("iconType", "") - .equals("ARTIST")) { - return getUrlFromNavigationEndpoint( - menuNavigationItemRenderer - .getObject("navigationEndpoint")); - } - } - - return null; - } else { - final JsonObject navigationEndpointHolder = info - .getArray("flexColumns") - .getObject(1) - .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text").getArray("runs").getObject(0); - - if (!navigationEndpointHolder.has("navigationEndpoint")) { - return null; - } - - final String url = getUrlFromNavigationEndpoint( - navigationEndpointHolder.getObject("navigationEndpoint")); - - if (!isNullOrEmpty(url)) { - return url; - } - - throw new ParsingException("Could not get uploader URL"); - } - } - - @Override - public String getTextualUploadDate() { - return null; - } - - @Override - public DateWrapper getUploadDate() { - return null; - } - - @Override - public long getViewCount() throws ParsingException { - if (searchType.equals(MUSIC_SONGS)) { - return -1; - } - final String viewCount = descriptionElements - .getObject(descriptionElements.size() - 3) - .getString("text"); - if (!isNullOrEmpty(viewCount)) { - try { - return Utils.mixedNumberWordToLong(viewCount); - } catch (final Parser.RegexException e) { - // probably viewCount == "No views" or similar - return 0; - } - } - throw new ParsingException("Could not get view count"); - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - final JsonArray thumbnails = info.getObject("thumbnail") - .getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - final String url = thumbnails.getObject(thumbnails.size() - 1) - .getString("url"); - - return fixThumbnailUrl(url); - } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }); - } else if (searchType.equals(MUSIC_ARTISTS)) { - collector.commit(new YoutubeChannelInfoItemExtractor(info) { - @Override - public String getThumbnailUrl() throws ParsingException { - try { - final JsonArray thumbnails = info.getObject("thumbnail") - .getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - final String url = thumbnails.getObject(thumbnails.size() - 1) - .getString("url"); - - return fixThumbnailUrl(url); - } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - - @Override - public String getName() throws ParsingException { - final String name = getTextFromObject(info.getArray("flexColumns") - .getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get name"); - } - - @Override - public String getUrl() throws ParsingException { - final String url = getUrlFromNavigationEndpoint(info - .getObject("navigationEndpoint")); - if (!isNullOrEmpty(url)) { - return url; - } - throw new ParsingException("Could not get url"); - } - - @Override - public long getSubscriberCount() throws ParsingException { - final String subscriberCount = getTextFromObject(info - .getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - if (!isNullOrEmpty(subscriberCount)) { - try { - return Utils.mixedNumberWordToLong(subscriberCount); - } catch (final Parser.RegexException ignored) { - // probably subscriberCount == "No subscribers" or similar - return 0; - } - } - throw new ParsingException("Could not get subscriber count"); - } - - @Override - public long getStreamCount() { - return -1; - } - - @Override - public String getDescription() { - return null; - } - }); - } else if (searchType.equals(MUSIC_ALBUMS) || searchType.equals(MUSIC_PLAYLISTS)) { - collector.commit(new YoutubePlaylistInfoItemExtractor(info) { - @Override - public String getThumbnailUrl() throws ParsingException { - try { - final JsonArray thumbnails = info.getObject("thumbnail") - .getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - final String url = thumbnails.getObject(thumbnails.size() - 1) - .getString("url"); - - return fixThumbnailUrl(url); - } catch (final Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - - @Override - public String getName() throws ParsingException { - final String name = getTextFromObject(info.getArray("flexColumns") - .getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get name"); - } - - @Override - public String getUrl() throws ParsingException { - String playlistId = info.getObject("menu") - .getObject("menuRenderer") - .getArray("items") - .getObject(4) - .getObject("toggleMenuServiceItemRenderer") - .getObject("toggledServiceEndpoint") - .getObject("likeEndpoint") - .getObject("target") - .getString("playlistId"); - - if (isNullOrEmpty(playlistId)) { - playlistId = info.getObject("overlay") - .getObject("musicItemThumbnailOverlayRenderer") - .getObject("content") - .getObject("musicPlayButtonRenderer") - .getObject("playNavigationEndpoint") - .getObject("watchPlaylistEndpoint") - .getString("playlistId"); - } - if (!isNullOrEmpty(playlistId)) { - return "https://music.youtube.com/playlist?list=" + playlistId; - } - throw new ParsingException("Could not get url"); - } - - @Override - public String getUploaderName() throws ParsingException { - final String name; - if (searchType.equals(MUSIC_ALBUMS)) { - name = descriptionElements.getObject(2).getString("text"); - } else { - name = descriptionElements.getObject(0).getString("text"); - } - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get uploader name"); - } - - @Override - public long getStreamCount() throws ParsingException { - if (searchType.equals(MUSIC_ALBUMS)) { - return ITEM_COUNT_UNKNOWN; - } - final String count = descriptionElements.getObject(2) - .getString("text"); - if (!isNullOrEmpty(count)) { - if (count.contains("100+")) { - return ITEM_COUNT_MORE_THAN_100; - } else { - return Long.parseLong(Utils.removeNonDigitCharacters(count)); - } - } - throw new ParsingException("Could not get count"); - } - }); - } - } - } + final String searchType = getLinkHandler().getContentFilters().get(0); + videos.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .map(item -> item.getObject("musicResponsiveListItemRenderer", null)) + .filter(Objects::nonNull) + .forEachOrdered(infoItem -> { + final String displayPolicy = infoItem.getString( + "musicItemRendererDisplayPolicy", ""); + if (displayPolicy.equals("MUSIC_ITEM_RENDERER_DISPLAY_POLICY_GREY_OUT")) { + // No info about URL available + return; + } + + final JsonArray descriptionElements = infoItem.getArray("flexColumns") + .getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text") + .getArray("runs"); + + switch (searchType) { + case MUSIC_SONGS: + case MUSIC_VIDEOS: + collector.commit(new YoutubeMusicSongOrVideoInfoItemExtractor( + infoItem, descriptionElements, searchType)); + break; + case MUSIC_ARTISTS: + collector.commit(new YoutubeMusicArtistInfoItemExtractor(infoItem)); + break; + case MUSIC_ALBUMS: + case MUSIC_PLAYLISTS: + collector.commit(new YoutubeMusicAlbumOrPlaylistInfoItemExtractor( + infoItem, descriptionElements, searchType)); + break; + } + }); } @Nullable diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java new file mode 100644 index 0000000000..11b220288c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java @@ -0,0 +1,177 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Utils; + +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +public class YoutubeMusicSongOrVideoInfoItemExtractor implements StreamInfoItemExtractor { + private final JsonObject songOrVideoInfoItem; + private final JsonArray descriptionElements; + private final String searchType; + + public YoutubeMusicSongOrVideoInfoItemExtractor(final JsonObject songOrVideoInfoItem, + final JsonArray descriptionElements, + final String searchType) { + this.songOrVideoInfoItem = songOrVideoInfoItem; + this.descriptionElements = descriptionElements; + this.searchType = searchType; + } + + + @Override + public String getUrl() throws ParsingException { + final String id = songOrVideoInfoItem.getObject("playlistItemData").getString("videoId"); + if (!isNullOrEmpty(id)) { + return "https://music.youtube.com/watch?v=" + id; + } + throw new ParsingException("Could not get URL"); + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(songOrVideoInfoItem.getArray("flexColumns") + .getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text")); + if (!isNullOrEmpty(name)) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public StreamType getStreamType() { + return StreamType.VIDEO_STREAM; + } + + @Override + public boolean isAd() { + return false; + } + + @Override + public long getDuration() throws ParsingException { + final String duration = descriptionElements.getObject(descriptionElements.size() - 1) + .getString("text"); + if (!isNullOrEmpty(duration)) { + return YoutubeParsingHelper.parseDurationString(duration); + } + throw new ParsingException("Could not get duration"); + } + + @Override + public String getUploaderName() throws ParsingException { + final String name = descriptionElements.getObject(0).getString("text"); + if (!isNullOrEmpty(name)) { + return name; + } + throw new ParsingException("Could not get uploader name"); + } + + @Override + public String getUploaderUrl() throws ParsingException { + if (searchType.equals(MUSIC_VIDEOS)) { + final JsonArray items = songOrVideoInfoItem.getObject("menu") + .getObject("menuRenderer") + .getArray("items"); + for (final Object item : items) { + final JsonObject menuNavigationItemRenderer = + ((JsonObject) item).getObject("menuNavigationItemRenderer"); + if (menuNavigationItemRenderer.getObject("icon") + .getString("iconType", "") + .equals("ARTIST")) { + return getUrlFromNavigationEndpoint( + menuNavigationItemRenderer.getObject("navigationEndpoint")); + } + } + + return null; + } else { + final JsonObject navigationEndpointHolder = songOrVideoInfoItem.getArray("flexColumns") + .getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text") + .getArray("runs") + .getObject(0); + + if (!navigationEndpointHolder.has("navigationEndpoint")) { + return null; + } + + final String url = getUrlFromNavigationEndpoint( + navigationEndpointHolder.getObject("navigationEndpoint")); + + if (!isNullOrEmpty(url)) { + return url; + } + + throw new ParsingException("Could not get uploader URL"); + } + } + + @Override + public boolean isUploaderVerified() { + // We don't have the ability to know this information on YouTube Music + return false; + } + + @Override + public String getTextualUploadDate() { + return null; + } + + @Override + public DateWrapper getUploadDate() { + return null; + } + + @Override + public long getViewCount() throws ParsingException { + if (searchType.equals(MUSIC_SONGS)) { + return -1; + } + final String viewCount = descriptionElements + .getObject(descriptionElements.size() - 3) + .getString("text"); + if (!isNullOrEmpty(viewCount)) { + try { + return Utils.mixedNumberWordToLong(viewCount); + } catch (final Parser.RegexException e) { + // probably viewCount == "No views" or similar + return 0; + } + } + throw new ParsingException("Could not get view count"); + } + + @Nonnull + @Override + public List getThumbnails() throws ParsingException { + try { + return getImagesFromThumbnailsArray( + songOrVideoInfoItem.getObject("thumbnail") + .getObject("musicThumbnailRenderer") + .getObject("thumbnail") + .getArray("thumbnails")); + } catch (final Exception e) { + throw new ParsingException("Could not get thumbnails", e); + } + } +} From 7f818217d28f3066174bd9cf97088c821938df68 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Mon, 25 Jul 2022 18:40:02 +0200 Subject: [PATCH 12/35] [SoundCloud] Add utility methods to get images from track JSON objects and image URLs These new public and static methods, added in SoundcloudParsingHelper, getAllImagesFromArtworkOrAvatarUrl(String) and getAllImagesFromVisualUrl(String) (which call a common private method, getAllImagesFromImageUrlReturned(String, List, List)), return an unmodifiable list of JPEG images containing almost every image resolution provided by SoundCloud except the original size and the tiny resolution (for artworks and avatars, as the image size is 20x20 for artworks and 18x18 for avatars, so very close to or equal to the t20x20 resolution): - for artworks and avatars: - mini: 16x16; - t20x20: 20x20; - small: 32x32; - badge: 47x47; - t50x50: 50x50; - t60x60: 60x60; - t67x67: 67x67; - large: 100x100; - t120x120: 120x120; - t200x200: 200x200; - t240x240: 240x240; - t250x250: 250x250; - t300x300: 300x300; - t500x500: 500x500. - for visuals/user banners: - t1240x260: 1240x260; - t2480x520: 2480x520. Duplicated code in two methods of SoundcloudParsingHelper (getUsersFromApi(ChannelInfoItemsCollector, String) and getStreamsFromApi(StreamInfoItemsCollector, String, boolean)) has been merged into one common private method, getNextPageUrlFromResponseObject(JsonObject). --- .../soundcloud/SoundcloudParsingHelper.java | 105 +++++++++++++++++- 1 file changed, 101 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index a029d85da4..bae7c4fe4a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -1,5 +1,11 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import static org.schabi.newpipe.extractor.Image.ResolutionLevel.LOW; +import static org.schabi.newpipe.extractor.Image.ResolutionLevel.MEDIUM; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; @@ -9,6 +15,7 @@ import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.MultiInfoItemsCollector; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -20,12 +27,14 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudPlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.ImageSuffix; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -35,12 +44,47 @@ import java.util.Collections; import java.util.List; import java.util.Map; - -import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; -import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; +import java.util.stream.Collectors; public final class SoundcloudParsingHelper { + // CHECKSTYLE:OFF + // From https://web.archive.org/web/20210214185000/https://developers.soundcloud.com/docs/api/reference#tracks + // and researches on images used by the websites + // CHECKSTYLE:ON + /* + SoundCloud avatars and artworks are almost squares + + When we get non-square pictures, all these images variants are still squares, except the + original and the crop versions provides images which are respecting aspect ratios. + The websites only use the square variants. + + t2400x2400 and t3000x3000 variants also exists, but are not returned as several images are + uploaded with a lower size than these variants: in this case, these variants return an upscaled + version of the original image. + */ + private static final List ALBUMS_AND_ARTWORKS_URL_SUFFIXES_AND_RESOLUTIONS = + List.of(new ImageSuffix("mini.jpg", 16, 16, LOW), + new ImageSuffix("t20x20.jpg", 20, 20, LOW), + new ImageSuffix("small.jpg", 32, 32, LOW), + new ImageSuffix("badge.jpg", 47, 47, LOW), + new ImageSuffix("t50x50.jpg", 50, 50, LOW), + new ImageSuffix("t60x60.jpg", 60, 60, LOW), + // Seems to work also on avatars, even if it is written to be not the case in + // the old API docs + new ImageSuffix("t67x67.jpg", 67, 67, LOW), + new ImageSuffix("t80x80.jpg", 80, 80, LOW), + new ImageSuffix("large.jpg", 100, 100, LOW), + new ImageSuffix("t120x120.jpg", 120, 120, LOW), + new ImageSuffix("t200x200.jpg", 200, 200, MEDIUM), + new ImageSuffix("t240x240.jpg", 240, 240, MEDIUM), + new ImageSuffix("t250x250.jpg", 250, 250, MEDIUM), + new ImageSuffix("t300x300.jpg", 300, 300, MEDIUM), + new ImageSuffix("t500x500.jpg", 500, 500, MEDIUM)); + + private static final List VISUALS_URL_SUFFIXES_AND_RESOLUTIONS = + List.of(new ImageSuffix("t1240x260.jpg", 1240, 260, MEDIUM), + new ImageSuffix("t2480x520.jpg", 2480, 520, MEDIUM)); + private static String clientId; public static final String SOUNDCLOUD_API_V2_URL = "https://api-v2.soundcloud.com/"; @@ -366,4 +410,57 @@ public static String getAvatarUrl(final JsonObject object) { public static String getUploaderName(final JsonObject object) { return object.getObject("user").getString("username", ""); } + + @Nonnull + public static List getAllImagesFromTrackObject(@Nonnull final JsonObject trackObject) + throws ParsingException { + final String artworkUrl = trackObject.getString("artwork_url"); + if (artworkUrl != null) { + return getAllImagesFromArtworkOrAvatarUrl(artworkUrl); + } + final String avatarUrl = trackObject.getObject("user").getString("avatar_url"); + if (avatarUrl != null) { + return getAllImagesFromArtworkOrAvatarUrl(avatarUrl); + } + + throw new ParsingException("Could not get track or track user's thumbnails"); + } + + @Nonnull + public static List getAllImagesFromArtworkOrAvatarUrl( + @Nullable final String originalArtworkOrAvatarUrl) { + if (isNullOrEmpty(originalArtworkOrAvatarUrl)) { + return List.of(); + } + + return getAllImagesFromImageUrlReturned( + // Artwork and avatars are originally returned with the "large" resolution, which + // is 100x100 + originalArtworkOrAvatarUrl.replace("large.jpg", ""), + ALBUMS_AND_ARTWORKS_URL_SUFFIXES_AND_RESOLUTIONS); + } + + @Nonnull + public static List getAllImagesFromVisualUrl( + @Nullable final String originalVisualUrl) { + if (isNullOrEmpty(originalVisualUrl)) { + return List.of(); + } + + return getAllImagesFromImageUrlReturned( + // Images are originally returned with the "original" resolution, which may be + // huge so don't include it for size purposes + originalVisualUrl.replace("original.jpg", ""), + VISUALS_URL_SUFFIXES_AND_RESOLUTIONS); + } + + private static List getAllImagesFromImageUrlReturned( + @Nonnull final String baseImageUrl, + @Nonnull final List imageSuffixes) { + return imageSuffixes.stream() + .map(imageSuffix -> new Image(baseImageUrl + imageSuffix.getSuffix(), + imageSuffix.getHeight(), imageSuffix.getWidth(), + imageSuffix.getResolutionLevel())) + .collect(Collectors.toUnmodifiableList()); + } } From a3a74cd5663dddb3f034469d84d3736cbc153f26 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Mon, 25 Jul 2022 18:58:20 +0200 Subject: [PATCH 13/35] [SoundCloud] Apply changes in InfoItemExtractors and return track user avatars as uploader avatars in SoundcloudStreamInfoItemExtractor --- .../SoundcloudChannelInfoItemExtractor.java | 16 ++++--- .../SoundcloudCommentsInfoItemExtractor.java | 21 ++++++--- .../SoundcloudPlaylistInfoItemExtractor.java | 47 ++++++++++--------- .../SoundcloudStreamInfoItemExtractor.java | 32 +++++++------ 4 files changed, 67 insertions(+), 49 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java index 86b3d48ff7..45241f090a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java @@ -1,11 +1,15 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; -import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; - import com.grack.nanojson.JsonObject; - +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor { private final JsonObject itemObject; @@ -23,10 +27,10 @@ public String getUrl() { return replaceHttpWithHttps(itemObject.getString("permalink_url")); } + @Nonnull @Override - public String getThumbnailUrl() { - // An avatar URL with a better resolution - return itemObject.getString("avatar_url", "").replace("large.jpg", "crop.jpg"); + public List getThumbnails() { + return getAllImagesFromArtworkOrAvatarUrl(itemObject.getString("avatar_url")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java index ec3f353e62..46194a3b68 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -1,15 +1,20 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.Description; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; import java.util.Objects; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDateFrom; + public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor { private final JsonObject json; private final String url; @@ -34,9 +39,10 @@ public String getUploaderName() { return json.getObject("user").getString("username"); } + @Nonnull @Override - public String getUploaderAvatarUrl() { - return json.getObject("user").getString("avatar_url"); + public List getUploaderAvatars() { + return getAllImagesFromArtworkOrAvatarUrl(json.getObject("user").getString("avatar_url")); } @Override @@ -45,7 +51,7 @@ public boolean isUploaderVerified() throws ParsingException { } @Override - public int getStreamPosition() throws ParsingException { + public int getStreamPosition() { return json.getInt("timestamp") / 1000; // convert milliseconds to seconds } @@ -62,7 +68,7 @@ public String getTextualUploadDate() { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate())); + return new DateWrapper(parseDateFrom(getTextualUploadDate())); } @Override @@ -75,8 +81,9 @@ public String getUrl() { return url; } + @Nonnull @Override - public String getThumbnailUrl() { - return json.getObject("user").getString("avatar_url"); + public List getThumbnails() { + return getAllImagesFromArtworkOrAvatarUrl(json.getObject("user").getString("avatar_url")); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java index 3f3f3b6755..9201b436b6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java @@ -1,12 +1,17 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; -import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; - import com.grack.nanojson.JsonObject; - +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private static final String USER_KEY = "user"; private static final String AVATAR_URL_KEY = "avatar_url"; @@ -28,36 +33,35 @@ public String getUrl() { return replaceHttpWithHttps(itemObject.getString("permalink_url")); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public List getThumbnails() throws ParsingException { // Over-engineering at its finest if (itemObject.isString(ARTWORK_URL_KEY)) { - final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, ""); - if (!artworkUrl.isEmpty()) { - // An artwork URL with a better resolution - return artworkUrl.replace("large.jpg", "crop.jpg"); + final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY); + if (!isNullOrEmpty(artworkUrl)) { + return getAllImagesFromArtworkOrAvatarUrl(artworkUrl); } } try { - // Look for artwork url inside the track list + // Look for artwork URL inside the track list for (final Object track : itemObject.getArray("tracks")) { final JsonObject trackObject = (JsonObject) track; - // First look for track artwork url + // First look for track artwork URL if (trackObject.isString(ARTWORK_URL_KEY)) { - final String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, ""); - if (!artworkUrl.isEmpty()) { - // An artwork URL with a better resolution - return artworkUrl.replace("large.jpg", "crop.jpg"); + final String artworkUrl = trackObject.getString(ARTWORK_URL_KEY); + if (!isNullOrEmpty(artworkUrl)) { + return getAllImagesFromArtworkOrAvatarUrl(artworkUrl); } } - // Then look for track creator avatar url + // Then look for track creator avatar URL final JsonObject creator = trackObject.getObject(USER_KEY); - final String creatorAvatar = creator.getString(AVATAR_URL_KEY, ""); - if (!creatorAvatar.isEmpty()) { - return creatorAvatar; + final String creatorAvatar = creator.getString(AVATAR_URL_KEY); + if (!isNullOrEmpty(creatorAvatar)) { + return getAllImagesFromArtworkOrAvatarUrl(creatorAvatar); } } } catch (final Exception ignored) { @@ -65,10 +69,11 @@ public String getThumbnailUrl() throws ParsingException { } try { - // Last resort, use user avatar url. If still not found, then throw exception. - return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, ""); + // Last resort, use user avatar URL. If still not found, then throw an exception. + return getAllImagesFromArtworkOrAvatarUrl( + itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY)); } catch (final Exception e) { - throw new ParsingException("Failed to extract playlist thumbnail url", e); + throw new ParsingException("Failed to extract playlist thumbnails", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java index 3d265e4e41..6fd6232e97 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java @@ -1,20 +1,24 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; -import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; - import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; -import javax.annotation.Nullable; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromTrackObject; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDateFrom; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor { - protected final JsonObject itemObject; + private final JsonObject itemObject; public SoundcloudStreamInfoItemExtractor(final JsonObject itemObject) { this.itemObject = itemObject; @@ -45,10 +49,11 @@ public String getUploaderUrl() { return replaceHttpWithHttps(itemObject.getObject("user").getString("permalink_url")); } - @Nullable + @Nonnull @Override - public String getUploaderAvatarUrl() { - return null; + public List getUploaderAvatars() { + return getAllImagesFromArtworkOrAvatarUrl( + itemObject.getObject("user").getString("avatar_url")); } @Override @@ -63,7 +68,7 @@ public String getTextualUploadDate() { @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate())); + return new DateWrapper(parseDateFrom(getTextualUploadDate())); } @Override @@ -71,13 +76,10 @@ public long getViewCount() { return itemObject.getLong("playback_count"); } + @Nonnull @Override - public String getThumbnailUrl() { - String artworkUrl = itemObject.getString("artwork_url", ""); - if (artworkUrl.isEmpty()) { - artworkUrl = itemObject.getObject("user").getString("avatar_url"); - } - return artworkUrl.replace("large.jpg", "crop.jpg"); + public List getThumbnails() throws ParsingException { + return getAllImagesFromTrackObject(itemObject); } @Override From 31da5beb51120d6d24484d0207eed9779593e81b Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Mon, 25 Jul 2022 19:09:17 +0200 Subject: [PATCH 14/35] [SoundCloud] Apply changes in Extractors --- .../SoundcloudChannelExtractor.java | 22 +++++--- .../SoundcloudPlaylistExtractor.java | 53 ++++++++++--------- .../extractors/SoundcloudStreamExtractor.java | 20 +++---- 3 files changed, 53 insertions(+), 42 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java index 143c8bf903..d8cd186d53 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java @@ -1,11 +1,14 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromVisualUrl; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; @@ -59,15 +62,19 @@ public String getName() { return user.getString("username"); } + @Nonnull @Override - public String getAvatarUrl() { - return user.getString("avatar_url"); + public List getAvatars() { + return getAllImagesFromArtworkOrAvatarUrl(user.getString("avatar_url")); } + @Nonnull @Override - public String getBannerUrl() { - return user.getObject("visuals").getArray("visuals").getObject(0) - .getString("visual_url"); + public List getBanners() { + return getAllImagesFromVisualUrl(user.getObject("visuals") + .getArray("visuals") + .getObject(0) + .getString("visual_url")); } @Override @@ -95,9 +102,10 @@ public String getParentChannelUrl() { return ""; } + @Nonnull @Override - public String getParentChannelAvatarUrl() { - return ""; + public List getParentChannelAvatars() { + return List.of(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java index 88f38b1e1a..46bba0a0d9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java @@ -1,9 +1,16 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAvatarUrl; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; @@ -24,9 +31,6 @@ import java.util.List; import java.util.Objects; -import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - public class SoundcloudPlaylistExtractor extends PlaylistExtractor { private static final int STREAMS_PER_REQUESTED_PAGE = 15; @@ -68,30 +72,28 @@ public String getName() { @Nonnull @Override - public String getThumbnailUrl() { - String artworkUrl = playlist.getString("artwork_url"); - - if (artworkUrl == null) { - // If the thumbnail is null, traverse the items list and get a valid one, - // if it also fails, return null - try { - final InfoItemsPage infoItems = getInitialPage(); - - for (final StreamInfoItem item : infoItems.getItems()) { - artworkUrl = item.getThumbnailUrl(); - if (!isNullOrEmpty(artworkUrl)) { - break; - } - } - } catch (final Exception ignored) { - } + public List getThumbnails() { + final String artworkUrl = playlist.getString("artwork_url"); + + if (!isNullOrEmpty(artworkUrl)) { + return getAllImagesFromArtworkOrAvatarUrl(artworkUrl); + } - if (artworkUrl == null) { - return ""; + // If the thumbnail is null or empty, traverse the items list and get a valid one + // If it also fails, return an empty list + try { + final InfoItemsPage infoItems = getInitialPage(); + + for (final StreamInfoItem item : infoItems.getItems()) { + final List thumbnails = item.getThumbnails(); + if (!isNullOrEmpty(thumbnails)) { + return thumbnails; + } } + } catch (final Exception ignored) { } - return artworkUrl.replace("large.jpg", "crop.jpg"); + return List.of(); } @Override @@ -104,9 +106,10 @@ public String getUploaderName() { return SoundcloudParsingHelper.getUploaderName(playlist); } + @Nonnull @Override - public String getUploaderAvatarUrl() { - return SoundcloudParsingHelper.getAvatarUrl(playlist); + public List getUploaderAvatars() { + return getAllImagesFromArtworkOrAvatarUrl(getAvatarUrl(playlist)); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 22d4d4beaa..aeb070153f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -2,6 +2,10 @@ import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.clientId; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromTrackObject; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAvatarUrl; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDateFrom; import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -11,6 +15,7 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -96,18 +101,13 @@ public String getTextualUploadDate() { @Nonnull @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(track.getString( - "created_at"))); + return new DateWrapper(parseDateFrom(track.getString("created_at"))); } @Nonnull @Override - public String getThumbnailUrl() { - String artworkUrl = track.getString("artwork_url", ""); - if (artworkUrl.isEmpty()) { - artworkUrl = track.getObject("user").getString("avatar_url", ""); - } - return artworkUrl.replace("large.jpg", "crop.jpg"); + public List getThumbnails() throws ParsingException { + return getAllImagesFromTrackObject(track); } @Nonnull @@ -155,8 +155,8 @@ public boolean isUploaderVerified() throws ParsingException { @Nonnull @Override - public String getUploaderAvatarUrl() { - return SoundcloudParsingHelper.getAvatarUrl(track); + public List getUploaderAvatars() { + return getAllImagesFromArtworkOrAvatarUrl(getAvatarUrl(track)); } @Override From 81c0d80a54f3b280957ba77908225e2cb91a591c Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 27 Jul 2022 19:30:59 +0200 Subject: [PATCH 15/35] [PeerTube] Add utility methods to get avatars and banners of accounts and channels Four new static methods have been added in PeertubeParsingHelper to do so: - two public methods to get the corresponding image type: getAvatarsFromOwnerAccountOrVideoChannelObject(String, JsonObject) and getBannersFromAccountOrVideoChannelObject(String, JsonObject); - two private methods as helper methods: getImagesFromAvatarsOrBanners(String, JsonObject, String, String) and getImagesFromAvatarOrBannerArray(String, JsonArray). --- .../peertube/PeertubeParsingHelper.java | 150 +++++++++++++++++- 1 file changed, 147 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index b57aa6b88c..ad11e6b39f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -3,6 +3,8 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.InfoItemExtractor; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.Page; @@ -14,12 +16,20 @@ import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamInfoItemExtractor; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; -import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeParseException; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.schabi.newpipe.extractor.Image.HEIGHT_UNKNOWN; +import static org.schabi.newpipe.extractor.Image.WIDTH_UNKNOWN; +import static org.schabi.newpipe.extractor.utils.Utils.isBlank; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public final class PeertubeParsingHelper { public static final String START_KEY = "start"; @@ -32,7 +42,7 @@ private PeertubeParsingHelper() { public static void validate(final JsonObject json) throws ContentNotAvailableException { final String error = json.getString("error"); - if (!Utils.isBlank(error)) { + if (!isBlank(error)) { throw new ContentNotAvailableException(error); } } @@ -53,7 +63,7 @@ public static Page getNextPage(final String prevPageUrl, final long total) { } catch (final Parser.RegexException e) { return null; } - if (Utils.isBlank(prevStart)) { + if (isBlank(prevStart)) { return null; } @@ -128,4 +138,138 @@ public static void collectItemsFrom(final InfoItemsCollector collector, } } + /** + * Get avatars from a {@code ownerAccount} or a {@code videoChannel} {@link JsonObject}. + * + *

+ * If the {@code avatars} {@link JsonArray} is present and non null or empty, avatars will be + * extracted from this array using {@link #getImagesFromAvatarOrBannerArray(String, JsonArray)}. + *

+ * + *

+ * If that's not the case, an avatar will extracted using the {@code avatar} {@link JsonObject}. + *

+ * + *

+ * Note that only images for which paths are not null and not empty will be added to the + * unmodifiable {@link Image} list returned. + *

+ * + * @param baseUrl the base URL of the PeerTube instance + * @param ownerAccountOrVideoChannelObject the {@code ownerAccount} or {@code videoChannel} + * {@link JsonObject} + * @return an unmodifiable list of {@link Image}s, which may be empty but never null + */ + @Nonnull + public static List getAvatarsFromOwnerAccountOrVideoChannelObject( + @Nonnull final String baseUrl, + @Nonnull final JsonObject ownerAccountOrVideoChannelObject) { + return getImagesFromAvatarsOrBanners(baseUrl, ownerAccountOrVideoChannelObject, + "avatars", "avatar"); + } + + /** + * Get banners from a {@code ownerAccount} or a {@code videoChannel} {@link JsonObject}. + * + *

+ * If the {@code banners} {@link JsonArray} is present and non null or empty, banners will be + * extracted from this array using {@link #getImagesFromAvatarOrBannerArray(String, JsonArray)}. + *

+ * + *

+ * If that's not the case, a banner will extracted using the {@code banner} {@link JsonObject}. + *

+ * + *

+ * Note that only images for which paths are not null and not empty will be added to the + * unmodifiable {@link Image} list returned. + *

+ * + * @param baseUrl the base URL of the PeerTube instance + * @param ownerAccountOrVideoChannelObject the {@code ownerAccount} or {@code videoChannel} + * {@link JsonObject} + * @return an unmodifiable list of {@link Image}s, which may be empty but never null + */ + @Nonnull + public static List getBannersFromAccountOrVideoChannelObject( + @Nonnull final String baseUrl, + @Nonnull final JsonObject ownerAccountOrVideoChannelObject) { + return getImagesFromAvatarsOrBanners(baseUrl, ownerAccountOrVideoChannelObject, + "banners", "banner"); + } + + /** + * Utility method to get avatars and banners from video channels and accounts from given name + * keys. + * + *

+ * Only images for which paths are not null and not empty will be added to the unmodifiable + * {@link Image} list returned and only the width of avatars or banners is provided by the API, + * and so is the only image dimension known. + *

+ * + * @param baseUrl the base URL of the PeerTube instance + * @param ownerAccountOrVideoChannelObject the {@code ownerAccount} or {@code videoChannel} + * {@link JsonObject} + * @param jsonArrayName the key name of the {@link JsonArray} to which + * extract all images available ({@code avatars} or + * {@code banners}) + * @param jsonObjectName the key name of the {@link JsonObject} to which + * extract a single image ({@code avatar} or + * {@code banner}), used as a fallback if the images + * {@link JsonArray} is null or empty + * @return an unmodifiable list of {@link Image}s, which may be empty but never null + */ + @Nonnull + private static List getImagesFromAvatarsOrBanners( + @Nonnull final String baseUrl, + @Nonnull final JsonObject ownerAccountOrVideoChannelObject, + @Nonnull final String jsonArrayName, + @Nonnull final String jsonObjectName) { + final JsonArray images = ownerAccountOrVideoChannelObject.getArray(jsonArrayName); + + if (!isNullOrEmpty(images)) { + return getImagesFromAvatarOrBannerArray(baseUrl, images); + } + + final JsonObject image = ownerAccountOrVideoChannelObject.getObject(jsonObjectName); + final String path = image.getString("path"); + if (!isNullOrEmpty(path)) { + return List.of(new Image(baseUrl + path, HEIGHT_UNKNOWN, + image.getInt("width", WIDTH_UNKNOWN), ResolutionLevel.UNKNOWN)); + } + + return List.of(); + } + + /** + * Get {@link Image}s from an {@code avatars} or a {@code banners} {@link JsonArray}. + * + *

+ * Only images for which paths are not null and not empty will be added to the + * unmodifiable {@link Image} list returned. + *

+ * + *

+ * Note that only the width of avatars or banners is provided by the API, and so only is the + * only dimension known of images. + *

+ * + * @param baseUrl the base URL of the PeerTube instance from which the + * {@code avatarsOrBannersArray} {@link JsonArray} comes from + * @param avatarsOrBannersArray an {@code avatars} or {@code banners} {@link JsonArray} + * @return an unmodifiable list of {@link Image}s, which may be empty but never null + */ + @Nonnull + private static List getImagesFromAvatarOrBannerArray( + @Nonnull final String baseUrl, + @Nonnull final JsonArray avatarsOrBannersArray) { + return avatarsOrBannersArray.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(image -> !isNullOrEmpty(image.getString("path"))) + .map(image -> new Image(baseUrl + image.getString("path"), HEIGHT_UNKNOWN, + image.getInt("width", WIDTH_UNKNOWN), ResolutionLevel.UNKNOWN)) + .collect(Collectors.toUnmodifiableList()); + } } From 6f8331524b49a973f449a96cdc943aa0415a2b70 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Thu, 9 Feb 2023 19:43:27 +0100 Subject: [PATCH 16/35] [PeerTube] Add utility method to get thumbnails of playlists and videos This method, getThumbnailsFromPlaylistOrVideoItem, has been added in PeertubeParsingHelper and returns the two image variants for playlists and videos. --- .../peertube/PeertubeParsingHelper.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index ad11e6b39f..4e8bd2d350 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -22,6 +22,7 @@ import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeParseException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -198,6 +199,47 @@ public static List getBannersFromAccountOrVideoChannelObject( "banners", "banner"); } + /** + * Get thumbnails from a playlist or a video item {@link JsonObject}. + * + *

+ * PeerTube provides two thumbnails in its API: a low one, represented by the value of the + * {@code thumbnailPath} key, and a medium one, represented by the value of the + * {@code previewPath} key. + *

+ * + *

+ * If a value is not null or empty, an {@link Image} will be added to the list returned with + * the URL to the thumbnail ({@code baseUrl + value}), a height and a width unknown and the + * corresponding resolution level (see above). + *

+ * + * @param baseUrl the base URL of the PeerTube instance + * @param playlistOrVideoItemObject the playlist or the video item {@link JsonObject}, which + * must not be null + * @return an unmodifiable list of {@link Image}s, which is never null but can be empty + */ + @Nonnull + public static List getThumbnailsFromPlaylistOrVideoItem( + @Nonnull final String baseUrl, + @Nonnull final JsonObject playlistOrVideoItemObject) { + final List imageList = new ArrayList<>(2); + + final String thumbnailPath = playlistOrVideoItemObject.getString("thumbnailPath"); + if (!isNullOrEmpty(thumbnailPath)) { + imageList.add(new Image(baseUrl + thumbnailPath, HEIGHT_UNKNOWN, WIDTH_UNKNOWN, + ResolutionLevel.LOW)); + } + + final String previewPath = playlistOrVideoItemObject.getString("previewPath"); + if (!isNullOrEmpty(previewPath)) { + imageList.add(new Image(baseUrl + previewPath, HEIGHT_UNKNOWN, WIDTH_UNKNOWN, + ResolutionLevel.MEDIUM)); + } + + return Collections.unmodifiableList(imageList); + } + /** * Utility method to get avatars and banners from video channels and accounts from given name * keys. From 0a6011a50ed0dbe4d8acfaeaaf84bb968040e956 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 27 Jul 2022 19:36:44 +0200 Subject: [PATCH 17/35] [PeerTube] Apply changes in InfoItemExtractors Also lower the visibility of attributes of channels and playlists InfoItems to private. --- .../PeertubeChannelInfoItemExtractor.java | 22 +++++++------- .../PeertubeCommentsInfoItemExtractor.java | 29 +++++++------------ .../PeertubePlaylistInfoItemExtractor.java | 16 ++++++---- .../PeertubeStreamInfoItemExtractor.java | 26 +++++++++-------- 4 files changed, 45 insertions(+), 48 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelInfoItemExtractor.java index 0faa25ea81..ac960b07ef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelInfoItemExtractor.java @@ -1,22 +1,24 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import javax.annotation.Nonnull; -import java.util.Comparator; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; public class PeertubeChannelInfoItemExtractor implements ChannelInfoItemExtractor { - final JsonObject item; - final JsonObject uploader; - final String baseUrl; + private final JsonObject item; + private final String baseUrl; + public PeertubeChannelInfoItemExtractor(@Nonnull final JsonObject item, @Nonnull final String baseUrl) { this.item = item; - this.uploader = item.getObject("uploader"); this.baseUrl = baseUrl; } @@ -30,14 +32,10 @@ public String getUrl() throws ParsingException { return item.getString("url"); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return item.getArray("avatars").stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) - .max(Comparator.comparingInt(avatar -> avatar.getInt("width"))) - .map(avatar -> baseUrl + avatar.getString("path")) - .orElse(null); + public List getThumbnails() throws ParsingException { + return getAvatarsFromOwnerAccountOrVideoChannelObject(baseUrl, item); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index 804b23802d..f2179cbdec 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -6,21 +6,24 @@ import com.grack.nanojson.JsonWriter; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.nio.charset.StandardCharsets; +import java.util.List; import java.util.Objects; import static org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor.CHILDREN; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseDateFrom; public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { @Nonnull @@ -52,15 +55,10 @@ public String getUrl() throws ParsingException { return url + "/" + getCommentId(); } + @Nonnull @Override - public String getThumbnailUrl() { - String value; - try { - value = JsonUtils.getString(item, "account.avatar.path"); - } catch (final Exception e) { - value = "/client/assets/images/default-avatar.png"; - } - return baseUrl + value; + public List getThumbnails() { + return getUploaderAvatars(); } @Override @@ -76,7 +74,7 @@ public String getTextualUploadDate() throws ParsingException { @Override public DateWrapper getUploadDate() throws ParsingException { final String textualUploadDate = getTextualUploadDate(); - return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); + return new DateWrapper(parseDateFrom(textualUploadDate)); } @Override @@ -97,15 +95,10 @@ public String getCommentId() { return Objects.toString(item.getLong("id"), null); } + @Nonnull @Override - public String getUploaderAvatarUrl() { - String value; - try { - value = JsonUtils.getString(item, "account.avatar.path"); - } catch (final Exception e) { - value = "/client/assets/images/default-avatar.png"; - } - return baseUrl + value; + public List getUploaderAvatars() { + return getAvatarsFromOwnerAccountOrVideoChannelObject(baseUrl, item.getObject("account")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java index 168872e159..6dbe0272a2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java @@ -2,19 +2,22 @@ import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.stream.Description; import javax.annotation.Nonnull; +import java.util.List; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getThumbnailsFromPlaylistOrVideoItem; -public class PeertubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { +public class PeertubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { - final JsonObject item; - final JsonObject uploader; - final String baseUrl; + private final JsonObject item; + private final JsonObject uploader; + private final String baseUrl; public PeertubePlaylistInfoItemExtractor(@Nonnull final JsonObject item, @Nonnull final String baseUrl) { @@ -33,9 +36,10 @@ public String getUrl() throws ParsingException { return item.getString("url"); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return baseUrl + item.getString("thumbnailPath"); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromPlaylistOrVideoItem(baseUrl, item); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index b9c7f4b13e..46aae43ccf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -1,15 +1,20 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.JsonUtils; -import javax.annotation.Nullable; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getThumbnailsFromPlaylistOrVideoItem; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseDateFrom; public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @@ -27,9 +32,10 @@ public String getUrl() throws ParsingException { return ServiceList.PeerTube.getStreamLHFactory().fromId(uuid, baseUrl).getUrl(); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return baseUrl + JsonUtils.getString(item, "thumbnailPath"); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromPlaylistOrVideoItem(baseUrl, item); } @Override @@ -56,14 +62,10 @@ public String getUploaderUrl() throws ParsingException { .fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); } - @Nullable + @Nonnull @Override - public String getUploaderAvatarUrl() { - final JsonObject account = item.getObject("account"); - if (account.has("avatar") && !account.isNull("avatar")) { - return baseUrl + account.getObject("avatar").getString("path"); - } - return null; + public List getUploaderAvatars() { + return getAvatarsFromOwnerAccountOrVideoChannelObject(baseUrl, item.getObject("account")); } @Override @@ -89,7 +91,7 @@ public DateWrapper getUploadDate() throws ParsingException { return null; } - return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); + return new DateWrapper(parseDateFrom(textualUploadDate)); } @Override From 4e6fb368bc49d261a1e3196414449c994249df42 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 29 Jul 2022 12:52:13 +0200 Subject: [PATCH 18/35] [PeerTube] Apply changes in Extractors and remove usages of default avatar picture The default avatar picture was used when no profile picture was found, but it was removed and split in multiple images. Thumbnails' size is not known, as this data is not provided by the API. --- .../extractors/PeertubeAccountExtractor.java | 25 ++++++++------- .../extractors/PeertubeChannelExtractor.java | 32 ++++++++----------- .../extractors/PeertubePlaylistExtractor.java | 21 +++++++----- .../extractors/PeertubeStreamExtractor.java | 28 ++++++---------- 4 files changed, 49 insertions(+), 57 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index eb95dcbe98..8b86a3a441 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; @@ -22,6 +23,9 @@ import java.io.IOException; import java.util.List; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getBannersFromAccountOrVideoChannelObject; + public class PeertubeAccountExtractor extends ChannelExtractor { private JsonObject json; private final String baseUrl; @@ -33,20 +37,16 @@ public PeertubeAccountExtractor(final StreamingService service, this.baseUrl = getBaseUrl(); } + @Nonnull @Override - public String getAvatarUrl() { - String value; - try { - value = JsonUtils.getString(json, "avatar.path"); - } catch (final Exception e) { - value = "/client/assets/images/default-avatar.png"; - } - return baseUrl + value; + public List getAvatars() { + return getAvatarsFromOwnerAccountOrVideoChannelObject(baseUrl, json); } + @Nonnull @Override - public String getBannerUrl() { - return null; + public List getBanners() { + return getBannersFromAccountOrVideoChannelObject(baseUrl, json); } @Override @@ -99,9 +99,10 @@ public String getParentChannelUrl() { return ""; } + @Nonnull @Override - public String getParentChannelAvatarUrl() { - return ""; + public List getParentChannelAvatars() { + return List.of(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index e7de3f061f..f8594d6978 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -3,6 +3,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; @@ -20,6 +21,9 @@ import java.io.IOException; import java.util.List; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getBannersFromAccountOrVideoChannelObject; + public class PeertubeChannelExtractor extends ChannelExtractor { private JsonObject json; private final String baseUrl; @@ -30,20 +34,16 @@ public PeertubeChannelExtractor(final StreamingService service, this.baseUrl = getBaseUrl(); } + @Nonnull @Override - public String getAvatarUrl() { - String value; - try { - value = JsonUtils.getString(json, "avatar.path"); - } catch (final Exception e) { - value = "/client/assets/images/default-avatar.png"; - } - return baseUrl + value; + public List getAvatars() { + return getAvatarsFromOwnerAccountOrVideoChannelObject(baseUrl, json); } + @Nonnull @Override - public String getBannerUrl() { - return null; + public List getBanners() { + return getBannersFromAccountOrVideoChannelObject(baseUrl, json); } @Override @@ -72,15 +72,11 @@ public String getParentChannelUrl() throws ParsingException { return JsonUtils.getString(json, "ownerAccount.url"); } + @Nonnull @Override - public String getParentChannelAvatarUrl() { - String value; - try { - value = JsonUtils.getString(json, "ownerAccount.avatar.path"); - } catch (final Exception e) { - value = "/client/assets/images/default-avatar.png"; - } - return baseUrl + value; + public List getParentChannelAvatars() { + return getAvatarsFromOwnerAccountOrVideoChannelObject( + baseUrl, json.getObject("ownerAccount")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index a950375f02..e1d05417c2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -3,6 +3,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -19,11 +20,14 @@ import javax.annotation.Nonnull; import java.io.IOException; +import java.util.List; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getThumbnailsFromPlaylistOrVideoItem; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class PeertubePlaylistExtractor extends PlaylistExtractor { @@ -36,8 +40,8 @@ public PeertubePlaylistExtractor(final StreamingService service, @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return getBaseUrl() + playlistInfo.getString("thumbnailPath"); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromPlaylistOrVideoItem(getBaseUrl(), playlistInfo); } @Override @@ -50,10 +54,11 @@ public String getUploaderName() { return playlistInfo.getObject("ownerAccount").getString("displayName"); } + @Nonnull @Override - public String getUploaderAvatarUrl() throws ParsingException { - return getBaseUrl() - + playlistInfo.getObject("ownerAccount").getObject("avatar").getString("path"); + public List getUploaderAvatars() throws ParsingException { + return getAvatarsFromOwnerAccountOrVideoChannelObject(getBaseUrl(), + playlistInfo.getObject("ownerAccount")); } @Override @@ -90,9 +95,9 @@ public String getSubChannelUrl() { @Nonnull @Override - public String getSubChannelAvatarUrl() throws ParsingException { - return getBaseUrl() - + playlistInfo.getObject("videoChannel").getObject("avatar").getString("path"); + public List getSubChannelAvatars() throws ParsingException { + return getAvatarsFromOwnerAccountOrVideoChannelObject(getBaseUrl(), + playlistInfo.getObject("videoChannel")); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 32d3ed4a79..699083df75 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getThumbnailsFromPlaylistOrVideoItem; import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -7,7 +9,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; - +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -87,8 +89,8 @@ public DateWrapper getUploadDate() throws ParsingException { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return baseUrl + JsonUtils.getString(json, "previewPath"); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromPlaylistOrVideoItem(baseUrl, json); } @Nonnull @@ -176,14 +178,8 @@ public String getUploaderName() throws ParsingException { @Nonnull @Override - public String getUploaderAvatarUrl() { - String value; - try { - value = JsonUtils.getString(json, "account.avatar.path"); - } catch (final Exception e) { - value = "/client/assets/images/default-avatar.png"; - } - return baseUrl + value; + public List getUploaderAvatars() { + return getAvatarsFromOwnerAccountOrVideoChannelObject(baseUrl, json.getObject("account")); } @Nonnull @@ -200,14 +196,8 @@ public String getSubChannelName() throws ParsingException { @Nonnull @Override - public String getSubChannelAvatarUrl() { - String value; - try { - value = JsonUtils.getString(json, "channel.avatar.path"); - } catch (final Exception e) { - value = "/client/assets/images/default-avatar.png"; - } - return baseUrl + value; + public List getSubChannelAvatars() { + return getAvatarsFromOwnerAccountOrVideoChannelObject(baseUrl, json.getObject("channel")); } @Nonnull From 4b80d737a4cba22f38767514b1fab6f863be636e Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 30 Jul 2022 17:02:19 +0200 Subject: [PATCH 19/35] [Bandcamp] Add utility methods to get multiple images Bandcamp images work with image IDs, which provide different resolutions. Images on Bandcamp are not always squares, and some IDs respect aspect ratios where some others not. The extractor will only use the ones which preserve aspect ratio and will not provide original images, for performance and size purposes. Because of this aspect ratio preservation constraint, only one dimension will be known at a time. The image IDs with their respective dimension used are: - 10: 1200w; - 101: 90h; - 170: 422h; - 171: 646h; - 20: 1024w; - 200: 420h; - 201: 280h; - 202: 140h; - 204: 360h; - 205: 240h; - 206: 180h; - 207: 120h; - 43: 100h; - 44: 200h. (Where w represents the width of the image and h the height of the image) Note that these dimensions are theoretical because if the image size is less than the dimensions of the image ID, it will be not upscaled but kept to its original size. All these resolutions are stored in a private static list of ThumbnailSuffixes in BandcampExtractorHelper, in which the methods to get mutliple images have been added: - getImagesFromImageUrl(String): public method to get images from an image URL; - getImagesFromImageId(long, boolean): public method to get images from an image ID; - getImagesFromImageBaseUrl(String): private utility method to get images from the static list of ThumbnailSuffixes from a given image base URL, containing the path to the image, a "a" letter if it comes from an album, its ID and an underscore. Some existing methods have been also edited: - the documentation of getImageUrl(long, boolean) has been changed to reflect the Bandcamp images findings; - getThumbnailUrlFromSearchResult has been renamed to getImagesFromSearchResult, and a documentation has been added to this method. The method replaceHttpWithHttps of the Utils class has been also used in BandcampExtractorHelper instead of doing manually what the method does. --- .../extractors/BandcampExtractorHelper.java | 178 ++++++++++++++++-- 1 file changed, 162 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index aff8660534..e7bbac999c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -6,25 +6,81 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; + import org.jsoup.Jsoup; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.utils.Utils; +import org.schabi.newpipe.extractor.utils.ImageSuffix; -import javax.annotation.Nullable; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.DateTimeException; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Collections; +import java.util.List; import java.util.Locale; +import java.util.stream.Collectors; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.Image.HEIGHT_UNKNOWN; +import static org.schabi.newpipe.extractor.Image.WIDTH_UNKNOWN; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public final class BandcampExtractorHelper { + /** + * List of image IDs which preserve aspect ratio with their theoretical dimension known. + * + *

+ * Bandcamp images are not always squares, so images which preserve aspect ratio are only used. + *

+ * + *

+ * One of the direct consequences of this specificity is that only one dimension of images is + * known at time, depending of the image ID. + *

+ * + *

+ * Note also that dimensions are only theoretical because if the image size is less than the + * dimensions of the image ID, it will be not upscaled but kept to its original size. + *

+ * + *

+ * IDs come from the + * GitHub Gist "Bandcamp File Format Parameters" by f2k1de + *

+ */ + private static final List IMAGE_URL_SUFFIXES_AND_RESOLUTIONS = List.of( + // ID | HEIGHT | WIDTH + new ImageSuffix("10.jpg", HEIGHT_UNKNOWN, 1200, ResolutionLevel.HIGH), + new ImageSuffix("101.jpg", 90, WIDTH_UNKNOWN, ResolutionLevel.LOW), + new ImageSuffix("170.jpg", 422, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM), + // 180 returns the same image aspect ratio and size as 171 + new ImageSuffix("171.jpg", 646, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM), + new ImageSuffix("20.jpg", HEIGHT_UNKNOWN, 1024, ResolutionLevel.HIGH), + // 203 returns the same image aspect ratio and size as 200 + new ImageSuffix("200.jpg", 420, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM), + new ImageSuffix("201.jpg", 280, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM), + new ImageSuffix("202.jpg", 140, WIDTH_UNKNOWN, ResolutionLevel.LOW), + new ImageSuffix("204.jpg", 360, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM), + new ImageSuffix("205.jpg", 240, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM), + new ImageSuffix("206.jpg", 180, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM), + new ImageSuffix("207.jpg", 120, WIDTH_UNKNOWN, ResolutionLevel.LOW), + new ImageSuffix("43.jpg", 100, WIDTH_UNKNOWN, ResolutionLevel.LOW), + new ImageSuffix("44.jpg", 200, WIDTH_UNKNOWN, ResolutionLevel.MEDIUM)); + + private static final String IMAGE_URL_APPENDIX_AND_EXTENSION_REGEX = "_\\d+\\.\\w+"; + private static final String IMAGES_DOMAIN_AND_PATH = "https://f4.bcbits.com/img/"; + public static final String BASE_URL = "https://bandcamp.com"; public static final String BASE_API_URL = BASE_URL + "/api"; @@ -44,7 +100,7 @@ public static String getStreamUrlFromIds(final long bandId, + "&tralbum_id=" + itemId + "&tralbum_type=" + itemType.charAt(0)) .responseBody(); - return Utils.replaceHttpWithHttps(JsonParser.object().from(jsonString) + return replaceHttpWithHttps(JsonParser.object().from(jsonString) .getString("bandcamp_url")); } catch (final JsonParserException | ReCaptchaException | IOException e) { @@ -76,17 +132,26 @@ public static JsonObject getArtistDetails(final String id) throws ParsingExcepti } /** - * Generate image url from image ID. + * Generate an image url from an image ID. + * + *

+ * The image ID {@code 10} was chosen because it provides images wide up to 1200px (when + * the original image width is more than or equal this resolution). + *

+ * *

- * The appendix "_10" was chosen because it provides images sized 1200x1200. Other integer - * values are possible as well (e.g. 0 is a very large resolution, possibly the original). + * Other integer values are possible as well (e.g. 0 is a very large resolution, possibly the + * original); see {@link #IMAGE_URL_SUFFIXES_AND_RESOLUTIONS} for more details about image + * resolution IDs. + *

* - * @param id The image ID - * @param album True if this is the cover of an album or track - * @return URL of image with this ID sized 1200x1200 + * @param id the image ID + * @param isAlbum whether the image is the cover of an album or a track + * @return a URL of the image with this ID with a width up to 1200px */ - public static String getImageUrl(final long id, final boolean album) { - return "https://f4.bcbits.com/img/" + (album ? 'a' : "") + id + "_10.jpg"; + @Nonnull + public static String getImageUrl(final long id, final boolean isAlbum) { + return IMAGES_DOMAIN_AND_PATH + (isAlbum ? 'a' : "") + id + "_10.jpg"; } /** @@ -136,13 +201,94 @@ public static DateWrapper parseDate(final String textDate) throws ParsingExcepti } } - @Nullable - public static String getThumbnailUrlFromSearchResult(final Element searchResult) { - return searchResult.getElementsByClass("art").stream() + /** + * Get a list of images from a search result {@link Element}. + * + *

+ * This method will call {@link #getImagesFromImageUrl(String)} using the first non null and + * non empty image URL found from the {@code src} attribute of {@code img} HTML elements, or an + * empty string if no valid image URL was found. + *

+ * + * @param searchResult a search result {@link Element} + * @return an unmodifiable list of {@link Image}s, which is never null but can be empty, in the + * case where no valid image URL was found + */ + @Nonnull + public static List getImagesFromSearchResult(@Nonnull final Element searchResult) { + return getImagesFromImageUrl(searchResult.getElementsByClass("art") + .stream() .flatMap(element -> element.getElementsByTag("img").stream()) .map(element -> element.attr("src")) - .filter(string -> !string.isEmpty()) + .filter(imageUrl -> !isNullOrEmpty(imageUrl)) .findFirst() - .orElse(null); + .orElse("")); + } + + /** + * Get all images which have resolutions preserving aspect ratio from an image URL. + * + *

+ * This method will remove the image ID and its extension from the end of the URL and then call + * {@link #getImagesFromImageBaseUrl(String)}. + *

+ * + * @param imageUrl the full URL of an image provided by Bandcamp, such as in its HTML code + * @return an unmodifiable list of {@link Image}s, which is never null but can be empty, in the + * case where the image URL has been not extracted (and so is null or empty) + */ + @Nonnull + public static List getImagesFromImageUrl(@Nullable final String imageUrl) { + if (isNullOrEmpty(imageUrl)) { + return List.of(); + } + + return getImagesFromImageBaseUrl( + imageUrl.replaceFirst(IMAGE_URL_APPENDIX_AND_EXTENSION_REGEX, "_")); + } + + /** + * Get all images which have resolutions preserving aspect ratio from an image ID. + * + *

+ * This method will call {@link #getImagesFromImageBaseUrl(String)}. + *

+ * + * @param id the id of an image provided by Bandcamp + * @param isAlbum whether the image is the cover of an album + * @return an unmodifiable list of {@link Image}s, which is never null but can be empty, in the + * case where the image ID has been not extracted (and so equal to 0) + */ + @Nonnull + public static List getImagesFromImageId(final long id, final boolean isAlbum) { + if (id == 0) { + return List.of(); + } + + return getImagesFromImageBaseUrl(IMAGES_DOMAIN_AND_PATH + (isAlbum ? 'a' : "") + id + "_"); + } + + /** + * Get all images resolutions preserving aspect ratio from a base image URL. + * + *

+ * Base image URLs are images containing the image path, a {@code a} letter if it comes from an + * album, its ID and an underscore. + *

+ * + *

+ * Images resolutions returned are the ones of {@link #IMAGE_URL_SUFFIXES_AND_RESOLUTIONS}. + *

+ * + * @param baseUrl the base URL of the image + * @return an unmodifiable and non-empty list of {@link Image}s + */ + @Nonnull + private static List getImagesFromImageBaseUrl(@Nonnull final String baseUrl) { + return IMAGE_URL_SUFFIXES_AND_RESOLUTIONS.stream() + .map(imageSuffix -> new Image(baseUrl + imageSuffix.getSuffix(), + imageSuffix.getHeight(), imageSuffix.getWidth(), + imageSuffix.getResolutionLevel())) + .collect(Collectors.toUnmodifiableList()); } } From 7e01eaac3332567e7c3dc17eb42bb7e508231d40 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 30 Jul 2022 17:09:09 +0200 Subject: [PATCH 20/35] [Bandcamp] Apply changes in InfoItemExtractors --- .../BandcampAlbumInfoItemExtractor.java | 11 +++++-- .../BandcampChannelInfoItemExtractor.java | 11 +++++-- .../BandcampCommentsInfoItemExtractor.java | 16 +++++++--- .../BandcampPlaylistInfoItemExtractor.java | 9 ++++-- ...campPlaylistInfoItemFeaturedExtractor.java | 17 +++++++--- .../BandcampRadioInfoItemExtractor.java | 20 ++++++------ ...dcampRelatedPlaylistInfoItemExtractor.java | 9 ++++-- ...campDiscographStreamInfoItemExtractor.java | 19 +++++------ ...ndcampPlaylistStreamInfoItemExtractor.java | 32 +++++++++---------- ...BandcampSearchStreamInfoItemExtractor.java | 18 +++++------ 10 files changed, 96 insertions(+), 66 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampAlbumInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampAlbumInfoItemExtractor.java index 3f41ea047d..5b9b1018f8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampAlbumInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampAlbumInfoItemExtractor.java @@ -1,10 +1,16 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import java.util.List; + +import javax.annotation.Nonnull; + public class BandcampAlbumInfoItemExtractor implements PlaylistInfoItemExtractor { private final JsonObject albumInfoItem; private final String uploaderUrl; @@ -28,9 +34,10 @@ public String getUrl() throws ParsingException { albumInfoItem.getString("item_type")); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return BandcampExtractorHelper.getImageUrl(albumInfoItem.getLong("art_id"), true); + public List getThumbnails() throws ParsingException { + return BandcampExtractorHelper.getImagesFromImageId(albumInfoItem.getLong("art_id"), true); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java index f89a634051..268494e4aa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java @@ -3,9 +3,15 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromSearchResult; + public class BandcampChannelInfoItemExtractor implements ChannelInfoItemExtractor { private final Element resultInfo; @@ -26,9 +32,10 @@ public String getUrl() throws ParsingException { return resultInfo.getElementsByClass("itemurl").text(); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return BandcampExtractorHelper.getThumbnailUrlFromSearchResult(searchResult); + public List getThumbnails() throws ParsingException { + return getImagesFromSearchResult(searchResult); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsInfoItemExtractor.java index e65b2cc4de..1a9f691c3b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsInfoItemExtractor.java @@ -1,13 +1,17 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.Description; +import javax.annotation.Nonnull; +import java.util.List; + public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtractor { private final JsonObject review; @@ -28,9 +32,10 @@ public String getUrl() { return url; } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return getUploaderAvatarUrl(); + public List getThumbnails() throws ParsingException { + return getUploaderAvatars(); } @Override @@ -43,8 +48,9 @@ public String getUploaderName() throws ParsingException { return review.getString("name"); } + @Nonnull @Override - public String getUploaderAvatarUrl() { - return getImageUrl(review.getLong("image_id"), false); + public List getUploaderAvatars() { + return getImagesFromImageId(review.getLong("image_id"), false); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java index 4bc4a1c655..8c34675f0d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java @@ -1,9 +1,13 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromSearchResult; public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private final Element searchResult; @@ -46,8 +50,9 @@ public String getUrl() { return resultInfo.getElementsByClass("itemurl").text(); } + @Nonnull @Override - public String getThumbnailUrl() { - return BandcampExtractorHelper.getThumbnailUrlFromSearchResult(searchResult); + public List getThumbnails() { + return getImagesFromSearchResult(searchResult); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java index 0808e10cbf..e7de5691d9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java @@ -1,9 +1,14 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import org.schabi.newpipe.extractor.utils.Utils; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; public class BandcampPlaylistInfoItemFeaturedExtractor implements PlaylistInfoItemExtractor { @@ -40,12 +45,14 @@ public String getName() { @Override public String getUrl() { - return featuredStory.getString("item_url").replaceAll("http://", "https://"); + return Utils.replaceHttpWithHttps(featuredStory.getString("item_url")); } + @Nonnull @Override - public String getThumbnailUrl() { - return featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) - : getImageUrl(featuredStory.getLong("item_art_id"), true); + public List getThumbnails() { + return featuredStory.has("art_id") + ? getImagesFromImageId(featuredStory.getLong("art_id"), true) + : getImagesFromImageId(featuredStory.getLong("item_art_id"), true); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index be6ba0b7ad..023234aa31 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -3,15 +3,20 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; + import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.parseDate; public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { @@ -39,7 +44,7 @@ public String getTextualUploadDate() { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - return BandcampExtractorHelper.parseDate(getTextualUploadDate()); + return parseDate(getTextualUploadDate()); } @Override @@ -52,9 +57,10 @@ public String getUrl() { return BASE_URL + "/?show=" + show.getInt("id"); } + @Nonnull @Override - public String getThumbnailUrl() { - return getImageUrl(show.getLong("image_id"), false); + public List getThumbnails() { + return getImagesFromImageId(show.getLong("image_id"), false); } @Override @@ -78,12 +84,6 @@ public String getUploaderUrl() { return ""; } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRelatedPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRelatedPlaylistInfoItemExtractor.java index 55b3199360..64d3885281 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRelatedPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRelatedPlaylistInfoItemExtractor.java @@ -3,10 +3,14 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageUrl; /** * Extracts recommended albums from tracks' website @@ -28,9 +32,10 @@ public String getUrl() throws ParsingException { return relatedAlbum.getElementsByClass("album-link").attr("abs:href"); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return relatedAlbum.getElementsByClass("album-art").attr("src"); + public List getThumbnails() throws ParsingException { + return getImagesFromImageUrl(relatedAlbum.getElementsByClass("album-art").attr("src")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java index 42b3170b3e..cea3504608 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java @@ -1,10 +1,14 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; -import javax.annotation.Nullable; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; public class BandcampDiscographStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor { @@ -20,12 +24,6 @@ public String getUploaderName() { return discograph.getString("band_name"); } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public String getName() { return discograph.getString("title"); @@ -40,11 +38,10 @@ public String getUrl() throws ParsingException { ); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return BandcampExtractorHelper.getImageUrl( - discograph.getLong("art_id"), true - ); + public List getThumbnails() throws ParsingException { + return getImagesFromImageId(discograph.getLong("art_id"), true); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java index 1b2e7e886c..5c37ec4573 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java @@ -3,19 +3,21 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import javax.annotation.Nullable; +import javax.annotation.Nonnull; import java.io.IOException; - +import java.util.Collections; +import java.util.List; public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor { private final JsonObject track; - private String substituteCoverUrl; + private List substituteCovers; private final StreamingService service; public BandcampPlaylistStreamInfoItemExtractor(final JsonObject track, @@ -24,13 +26,14 @@ public BandcampPlaylistStreamInfoItemExtractor(final JsonObject track, super(uploaderUrl); this.track = track; this.service = service; + substituteCovers = Collections.emptyList(); } public BandcampPlaylistStreamInfoItemExtractor(final JsonObject track, final String uploaderUrl, - final String substituteCoverUrl) { + final List substituteCovers) { this(track, uploaderUrl, (StreamingService) null); - this.substituteCoverUrl = substituteCoverUrl; + this.substituteCovers = substituteCovers; } @Override @@ -56,28 +59,23 @@ public String getUploaderName() { return ""; } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - /** * Each track can have its own cover art. Therefore, unless a substitute is provided, * the thumbnail is extracted using a stream extractor. */ + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - if (substituteCoverUrl != null) { - return substituteCoverUrl; - } else { + public List getThumbnails() throws ParsingException { + if (substituteCovers.isEmpty()) { try { final StreamExtractor extractor = service.getStreamExtractor(getUrl()); extractor.fetchPage(); - return extractor.getThumbnailUrl(); + return extractor.getThumbnails(); } catch (final ExtractionException | IOException e) { - throw new ParsingException("could not download cover art location", e); + throw new ParsingException("Could not download cover art location", e); } } + + return substituteCovers; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java index 5e585e7e00..18c0c0dcce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java @@ -1,10 +1,13 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; -import javax.annotation.Nullable; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromSearchResult; public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor { @@ -29,12 +32,6 @@ public String getUploaderName() { } } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public String getName() throws ParsingException { return resultInfo.getElementsByClass("heading").text(); @@ -45,9 +42,10 @@ public String getUrl() throws ParsingException { return resultInfo.getElementsByClass("itemurl").text(); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return BandcampExtractorHelper.getThumbnailUrlFromSearchResult(searchResult); + public List getThumbnails() throws ParsingException { + return getImagesFromSearchResult(searchResult); } @Override From 71cda03c4c048209842177983a628d391bf35ac3 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 30 Jul 2022 17:14:15 +0200 Subject: [PATCH 21/35] [Bandcamp] Apply changes in Extractors --- .../extractors/BandcampChannelExtractor.java | 33 ++++++++++++------- .../extractors/BandcampPlaylistExtractor.java | 21 +++++++----- .../BandcampRadioStreamExtractor.java | 13 +++++--- .../extractors/BandcampStreamExtractor.java | 23 ++++++++----- 4 files changed, 57 insertions(+), 33 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index dc2f984075..ac44cbb8d8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -2,12 +2,18 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; +import static org.schabi.newpipe.extractor.Image.HEIGHT_UNKNOWN; +import static org.schabi.newpipe.extractor.Image.WIDTH_UNKNOWN; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getArtistDetails; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import org.jsoup.Jsoup; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor; @@ -25,6 +31,7 @@ import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -38,17 +45,15 @@ public BandcampChannelExtractor(final StreamingService service, super(service, linkHandler); } + @Nonnull @Override - public String getAvatarUrl() { - if (channelInfo.getLong("bio_image_id") == 0) { - return ""; - } - - return BandcampExtractorHelper.getImageUrl(channelInfo.getLong("bio_image_id"), false); + public List getAvatars() { + return getImagesFromImageId(channelInfo.getLong("bio_image_id"), false); } + @Nonnull @Override - public String getBannerUrl() throws ParsingException { + public List getBanners() throws ParsingException { /* * Mobile API does not return the header or not the correct header. * Therefore, we need to query the website @@ -62,8 +67,11 @@ public String getBannerUrl() throws ParsingException { .filter(Objects::nonNull) .flatMap(element -> element.getElementsByTag("img").stream()) .map(element -> element.attr("src")) - .findFirst() - .orElse(""); // no banner available + .filter(url -> !url.isEmpty()) + .map(url -> new Image( + replaceHttpWithHttps(url), HEIGHT_UNKNOWN, WIDTH_UNKNOWN, + ResolutionLevel.UNKNOWN)) + .collect(Collectors.toUnmodifiableList()); } catch (final IOException | ReCaptchaException e) { throw new ParsingException("Could not download artist web site", e); @@ -98,9 +106,10 @@ public String getParentChannelUrl() { return null; } + @Nonnull @Override - public String getParentChannelAvatarUrl() { - return null; + public List getParentChannelAvatars() { + return List.of(); } @Override @@ -156,7 +165,7 @@ public List getTabs() throws ParsingException { @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { - channelInfo = BandcampExtractorHelper.getArtistDetails(getId()); + channelInfo = getArtistDetails(getId()); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index aa6dd79683..ee5dc92134 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageUrl; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor.getAlbumInfoJson; import static org.schabi.newpipe.extractor.utils.JsonUtils.getJsonData; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; @@ -13,6 +14,7 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -28,6 +30,7 @@ import java.io.IOException; import java.util.Objects; +import java.util.List; import javax.annotation.Nonnull; @@ -74,11 +77,11 @@ public void onFetchPage(@Nonnull final Downloader downloader) @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public List getThumbnails() throws ParsingException { if (albumJson.isNull("art_id")) { - return ""; + return List.of(); } else { - return getImageUrl(albumJson.getLong("art_id"), true); + return getImagesFromImageId(albumJson.getLong("art_id"), true); } } @@ -94,12 +97,14 @@ public String getUploaderName() { return albumJson.getString("artist"); } + @Nonnull @Override - public String getUploaderAvatarUrl() { - return document.getElementsByClass("band-photo").stream() + public List getUploaderAvatars() { + return getImagesFromImageUrl(document.getElementsByClass("band-photo") + .stream() .map(element -> element.attr("src")) .findFirst() - .orElse(""); + .orElse("")); } @Override @@ -154,7 +159,7 @@ public InfoItemsPage getInitialPage() throws ExtractionException } else { // Pretend every track has the same cover art as the album collector.commit(new BandcampPlaylistStreamInfoItemExtractor( - track, getUploaderUrl(), getThumbnailUrl())); + track, getUploaderUrl(), getThumbnails())); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index 03a9c2222b..2b28f03a79 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -3,6 +3,7 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -11,6 +12,8 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -97,14 +100,16 @@ public String getTextualUploadDate() { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return getImageUrl(showInfo.getLong("show_image_id"), false); + public List getThumbnails() throws ParsingException { + return getImagesFromImageId(showInfo.getLong("show_image_id"), false); } @Nonnull @Override - public String getUploaderAvatarUrl() { - return BASE_URL + "/img/buttons/bandcamp-button-circle-whitecolor-512.png"; + public List getUploaderAvatars() { + return Collections.singletonList( + new Image(BASE_URL + "/img/buttons/bandcamp-button-circle-whitecolor-512.png", + 512, 512, ResolutionLevel.MEDIUM)); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index b4e6098ae6..dfdbb56965 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -2,8 +2,11 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageId; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImagesFromImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.parseDate; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParserException; @@ -11,6 +14,7 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -98,7 +102,7 @@ public String getUploaderUrl() throws ParsingException { @Nonnull @Override public String getUrl() throws ParsingException { - return albumJson.getString("url").replace("http://", "https://"); + return replaceHttpWithHttps(albumJson.getString("url")); } @Nonnull @@ -116,26 +120,27 @@ public String getTextualUploadDate() { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - return BandcampExtractorHelper.parseDate(getTextualUploadDate()); + return parseDate(getTextualUploadDate()); } @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public List getThumbnails() throws ParsingException { if (albumJson.isNull("art_id")) { - return ""; + return List.of(); } - return getImageUrl(albumJson.getLong("art_id"), true); + return getImagesFromImageId(albumJson.getLong("art_id"), true); } @Nonnull @Override - public String getUploaderAvatarUrl() { - return document.getElementsByClass("band-photo").stream() + public List getUploaderAvatars() { + return getImagesFromImageUrl(document.getElementsByClass("band-photo") + .stream() .map(element -> element.attr("src")) .findFirst() - .orElse(""); + .orElse("")); } @Nonnull From 2f40861428eb41e482982ee2e9cc90ecfabc050c Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 30 Jul 2022 18:31:08 +0200 Subject: [PATCH 22/35] [MediaCCC] Add utility methods to get image lists from conference logos and streams These three new methods, added in MediaCCCParsingHelper, getImageListFromImageUrl(String), getThumbnailsFromStreamItem(JsonObject) and getThumbnailsFromLiveStreamItem(JsonObject) (the last two are based on a common method, getThumbnailsFromObject(JsonObject, String, String)), return an empty list if the case no image URL could be extracted. Images returned have their height and width unknown and a resolution level depending on the image key of the JSON API response. --- .../extractors/MediaCCCParsingHelper.java | 108 +++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java index ad07c4720d..2a4ea6a9f8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java @@ -1,21 +1,33 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.Localization; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.time.OffsetDateTime; import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.regex.Pattern; +import static org.schabi.newpipe.extractor.Image.HEIGHT_UNKNOWN; +import static org.schabi.newpipe.extractor.Image.WIDTH_UNKNOWN; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + public final class MediaCCCParsingHelper { - // {conference_slug}/{room_slug} + // conference_slug/room_slug private static final Pattern LIVE_STREAM_ID_PATTERN = Pattern.compile("\\w+/\\w+"); private static JsonArray liveStreams = null; @@ -69,4 +81,98 @@ public static JsonArray getLiveStreams(final Downloader downloader, } return liveStreams; } + + /** + * Get an {@link Image} list from a given image logo URL. + * + *

+ * If the image URL is null or empty, an empty list is returned; otherwise, a singleton list is + * returned containing an {@link Image} with the image URL with its height, width and + * resolution unknown. + *

+ * + * @param logoImageUrl a logo image URL, which can be null or empty + * @return an unmodifiable list of {@link Image}s, which is always empty or a singleton + */ + @Nonnull + public static List getImageListFromLogoImageUrl(@Nullable final String logoImageUrl) { + if (isNullOrEmpty(logoImageUrl)) { + return List.of(); + } + + return List.of(new Image(logoImageUrl, HEIGHT_UNKNOWN, WIDTH_UNKNOWN, + ResolutionLevel.UNKNOWN)); + } + + /** + * Get the {@link Image} list of thumbnails from a given stream item. + * + *

+ * MediaCCC API provides two thumbnails for a stream item: a {@code thumb_url} one, which is + * medium quality and a {@code poster_url} one, which is high quality in most cases. + *

+ * + * @param streamItem a stream JSON item of MediaCCC's API, which must not be null + * @return an unmodifiable list, which is never null but can be empty. + */ + @Nonnull + public static List getThumbnailsFromStreamItem(@Nonnull final JsonObject streamItem) { + return getThumbnailsFromObject(streamItem, "thumb_url", "poster_url"); + } + + /** + * Get the {@link Image} list of thumbnails from a given live stream item. + * + *

+ * MediaCCC API provides two URL thumbnails for a livestream item: a {@code thumb} one, + * which should be medium quality and a {@code poster_url} one, which should be high quality. + *

+ * + * @param liveStreamItem a stream JSON item of MediaCCC's API, which must not be null + * @return an unmodifiable list, which is never null but can be empty. + */ + @Nonnull + public static List getThumbnailsFromLiveStreamItem( + @Nonnull final JsonObject liveStreamItem) { + return getThumbnailsFromObject(liveStreamItem, "thumb", "poster"); + } + + /** + * Utility method to get an {@link Image} list of thumbnails from a stream or a livestream. + * + *

+ * MediaCCC's API thumbnails come from two elements: a {@code thumb} element, which links to a + * medium thumbnail and a {@code poster} element, which links to a high thumbnail. + *

+ *

+ * Thumbnails are only added if their URLs are not null or empty. + *

+ * + * @param streamOrLivestreamItem a (live)stream JSON item of MediaCCC's API, which must not be + * null + * @param thumbUrlKey the name of the {@code thumb} URL key + * @param posterUrlKey the name of the {@code poster} URL key + * @return an unmodifiable list, which is never null but can be empty. + */ + @Nonnull + private static List getThumbnailsFromObject( + @Nonnull final JsonObject streamOrLivestreamItem, + @Nonnull final String thumbUrlKey, + @Nonnull final String posterUrlKey) { + final List imageList = new ArrayList<>(2); + + final String thumbUrl = streamOrLivestreamItem.getString(thumbUrlKey); + if (!isNullOrEmpty(thumbUrl)) { + imageList.add(new Image(thumbUrl, HEIGHT_UNKNOWN, WIDTH_UNKNOWN, + ResolutionLevel.MEDIUM)); + } + + final String posterUrl = streamOrLivestreamItem.getString(posterUrlKey); + if (!isNullOrEmpty(posterUrl)) { + imageList.add(new Image(posterUrl, HEIGHT_UNKNOWN, WIDTH_UNKNOWN, + ResolutionLevel.HIGH)); + } + + return Collections.unmodifiableList(imageList); + } } From 306068a63b3f4dfa557eaff2b1fd0ddeb7a5ff62 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 30 Jul 2022 18:39:37 +0200 Subject: [PATCH 23/35] [MediaCCC] Apply changes in InfoItemExtractors --- .../MediaCCCLiveStreamKioskExtractor.java | 16 ++++++++-------- .../extractors/MediaCCCRecentKioskExtractor.java | 16 ++++++++-------- .../MediaCCCConferenceInfoItemExtractor.java | 11 +++++++++-- .../MediaCCCStreamInfoItemExtractor.java | 16 ++++++++-------- 4 files changed, 33 insertions(+), 26 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java index 2b311fb352..f6c5ac8627 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java @@ -1,12 +1,17 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getThumbnailsFromLiveStreamItem; public class MediaCCCLiveStreamKioskExtractor implements StreamInfoItemExtractor { @@ -32,9 +37,10 @@ public String getUrl() throws ParsingException { return roomInfo.getString("link"); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return roomInfo.getString("thumb"); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromLiveStreamItem(roomInfo); } @Override @@ -75,12 +81,6 @@ public String getUploaderUrl() throws ParsingException { return "https://media.ccc.de/c/" + conferenceInfo.getString("slug"); } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java index cef496d9db..df25b28d89 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java @@ -2,6 +2,7 @@ import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; @@ -10,9 +11,13 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.List; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getImageListFromLogoImageUrl; + public class MediaCCCRecentKioskExtractor implements StreamInfoItemExtractor { private final JsonObject event; @@ -31,9 +36,10 @@ public String getUrl() throws ParsingException { return event.getString("frontend_link"); } + @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return event.getString("thumb_url"); + public List getThumbnails() throws ParsingException { + return getImageListFromLogoImageUrl(event.getString("poster_url")); } @Override @@ -70,12 +76,6 @@ public String getUploaderUrl() throws ParsingException { .getUrl(); // web URL } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java index b69d7a9083..3f56aec3af 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java @@ -2,10 +2,16 @@ import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import javax.annotation.Nonnull; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getImageListFromLogoImageUrl; + public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtractor { private final JsonObject conference; @@ -43,8 +49,9 @@ public String getUrl() throws ParsingException { return conference.getString("url"); } + @Nonnull @Override - public String getThumbnailUrl() { - return conference.getString("logo_url"); + public List getThumbnails() { + return getImageListFromLogoImageUrl(conference.getString("logo_url")); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java index 92f0894b9f..ec9d00f3a0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java @@ -1,13 +1,18 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getThumbnailsFromStreamItem; public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor { private final JsonObject event; @@ -46,12 +51,6 @@ public String getUploaderUrl() { return event.getString("conference_url"); } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public boolean isUploaderVerified() throws ParsingException { return false; @@ -84,8 +83,9 @@ public String getUrl() throws ParsingException { + event.getString("guid"); } + @Nonnull @Override - public String getThumbnailUrl() { - return event.getString("thumb_url"); + public List getThumbnails() { + return getThumbnailsFromStreamItem(event); } } From e16d521b7bbdd3947fe388164ef7f9eb5287466c Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 30 Jul 2022 18:46:54 +0200 Subject: [PATCH 24/35] [MediaCCC] Apply changes in Extractors Also remove usage of the conference logo as the banner of a conference, as it is a logo and not a banner. --- .../MediaCCCConferenceExtractor.java | 19 +++++++++++++------ .../MediaCCCLiveStreamExtractor.java | 6 ++++-- .../extractors/MediaCCCSearchExtractor.java | 8 +++++--- .../extractors/MediaCCCStreamExtractor.java | 14 +++++++++----- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java index 48d85f7cad..e4574cb887 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.MultiInfoItemsCollector; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -21,10 +22,13 @@ import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; import java.io.IOException; +import java.util.Collections; import java.util.List; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getImageListFromLogoImageUrl; + public class MediaCCCConferenceExtractor extends ChannelExtractor { private JsonObject conferenceData; @@ -33,14 +37,16 @@ public MediaCCCConferenceExtractor(final StreamingService service, super(service, linkHandler); } + @Nonnull @Override - public String getAvatarUrl() { - return conferenceData.getString("logo_url"); + public List getAvatars() { + return getImageListFromLogoImageUrl(conferenceData.getString("logo_url")); } + @Nonnull @Override - public String getBannerUrl() { - return conferenceData.getString("logo_url"); + public List getBanners() { + return Collections.emptyList(); } @Override @@ -68,9 +74,10 @@ public String getParentChannelUrl() { return ""; } + @Nonnull @Override - public String getParentChannelAvatarUrl() { - return ""; + public List getParentChannelAvatars() { + return Collections.emptyList(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java index 9271bbc322..4c3edbd06c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java @@ -1,11 +1,13 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getThumbnailsFromLiveStreamItem; import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -77,8 +79,8 @@ public String getName() throws ParsingException { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { - return room.getString("thumb"); + public List getThumbnails() throws ParsingException { + return getThumbnailsFromLiveStreamItem(room); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index c4501f5c2c..e72d26cb22 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -9,8 +9,10 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; @@ -18,7 +20,6 @@ import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory; @@ -157,9 +158,10 @@ public String getUrl() { return item.getUrl(); } + @Nonnull @Override - public String getThumbnailUrl() { - return item.getThumbnailUrl(); + public List getThumbnails() { + return item.getThumbnails(); } }); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 07dfc870f2..9f299a71af 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -1,5 +1,8 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getImageListFromLogoImageUrl; +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getThumbnailsFromStreamItem; +import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.parseDateFrom; import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; @@ -8,6 +11,7 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -51,13 +55,13 @@ public String getTextualUploadDate() { @Nonnull @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(getTextualUploadDate())); + return new DateWrapper(parseDateFrom(getTextualUploadDate())); } @Nonnull @Override - public String getThumbnailUrl() { - return data.getString("thumb_url"); + public List getThumbnails() { + return getThumbnailsFromStreamItem(data); } @Nonnull @@ -91,8 +95,8 @@ public String getUploaderName() { @Nonnull @Override - public String getUploaderAvatarUrl() { - return conferenceData.getString("logo_url"); + public List getUploaderAvatars() { + return getImageListFromLogoImageUrl(conferenceData.getString("logo_url")); } @Override From 70fb3aa38e2c15d9c379ce0b4eb84c53dd424cc2 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Mon, 1 Aug 2022 21:07:44 +0200 Subject: [PATCH 25/35] Update BaseExtractorTests image methods' name Also suppress unused warnings in BaseStreamExtractorTest, like it is done on other BaseExtractorTests interfaces. --- .../extractor/services/BaseChannelExtractorTest.java | 4 ++-- .../extractor/services/BasePlaylistExtractorTest.java | 6 +++--- .../extractor/services/BaseStreamExtractorTest.java | 7 ++++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseChannelExtractorTest.java index c776ee5145..693b20e18c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseChannelExtractorTest.java @@ -6,9 +6,9 @@ public interface BaseChannelExtractorTest extends BaseExtractorTest { @Test void testDescription() throws Exception; @Test - void testAvatarUrl() throws Exception; + void testAvatars() throws Exception; @Test - void testBannerUrl() throws Exception; + void testBanners() throws Exception; @Test void testFeedUrl() throws Exception; @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java index 241bbbee56..00478d7936 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java @@ -4,13 +4,13 @@ public interface BasePlaylistExtractorTest extends BaseListExtractorTest { @Test - void testThumbnailUrl() throws Exception; + void testThumbnails() throws Exception; @Test - void testBannerUrl() throws Exception; + void testBanners() throws Exception; @Test void testUploaderName() throws Exception; @Test - void testUploaderAvatarUrl() throws Exception; + void testUploaderAvatars() throws Exception; @Test void testStreamCount() throws Exception; @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java index a224a347db..c797220e88 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java @@ -2,6 +2,7 @@ import org.junit.jupiter.api.Test; +@SuppressWarnings("unused") public interface BaseStreamExtractorTest extends BaseExtractorTest { @Test void testStreamType() throws Exception; @@ -10,7 +11,7 @@ public interface BaseStreamExtractorTest extends BaseExtractorTest { @Test void testUploaderUrl() throws Exception; @Test - void testUploaderAvatarUrl() throws Exception; + void testUploaderAvatars() throws Exception; @Test void testSubscriberCount() throws Exception; @Test @@ -18,9 +19,9 @@ public interface BaseStreamExtractorTest extends BaseExtractorTest { @Test void testSubChannelUrl() throws Exception; @Test - void testSubChannelAvatarUrl() throws Exception; + void testSubChannelAvatars() throws Exception; @Test - void testThumbnailUrl() throws Exception; + void testThumbnails() throws Exception; @Test void testDescription() throws Exception; @Test From 515847285299c69d0d56be23a9c71e0197b284ec Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Mon, 1 Aug 2022 21:17:09 +0200 Subject: [PATCH 26/35] Apply changes in DefaultTests and add utility method to test image lists This new method, defaultTestImageList(List thumbnails = item.getThumbnails(); + if (!isNullOrEmpty(thumbnails)) { + defaultTestImageCollection(thumbnails); } assertNotNull(item.getInfoType(), "InfoItem type not set: " + item); assertEquals(expectedService.getServiceId(), item.getServiceId(), "Unexpected item service id"); @@ -44,9 +55,9 @@ public static void defaultTestListOfItems(StreamingService expectedService, List assertExpectedLinkType(expectedService, uploaderUrl, LinkType.CHANNEL); } - final String uploaderAvatarUrl = streamInfoItem.getUploaderAvatarUrl(); - if (!isNullOrEmpty(uploaderAvatarUrl)) { - assertIsSecureUrl(uploaderAvatarUrl); + final List uploaderAvatars = streamInfoItem.getUploaderAvatars(); + if (!isNullOrEmpty(uploaderAvatars)) { + defaultTestImageCollection(uploaderAvatars); } assertExpectedLinkType(expectedService, streamInfoItem.getUrl(), LinkType.STREAM); @@ -134,4 +145,16 @@ public static void defaultTestGetPageInNewExtractor(ListExtractor page = newExtractor.getPage(nextPage); defaultTestListOfItems(extractor.getService(), page.getItems(), page.getErrors()); } + + public static void defaultTestImageCollection( + @Nullable final Collection imageCollection) { + assertNotNull(imageCollection); + imageCollection.forEach(image -> { + assertIsSecureUrl(image.getUrl()); + assertGreaterOrEqual(Image.HEIGHT_UNKNOWN, image.getHeight(), + "Unexpected image height: " + image.getHeight()); + assertGreaterOrEqual(Image.WIDTH_UNKNOWN, image.getWidth(), + "Unexpected image width: " + image.getWidth()); + }); + } } From 434e88570866a36ab5e767f32ba74e9b43aa36b7 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 13:03:50 +0200 Subject: [PATCH 27/35] Add utility methods in ExtractorAsserts to check whether a collection is empty and to test image collections Two new methods have been added in ExtractorAsserts to check if a collection is empty: - assertNotEmpty(String, Collection), checking: - the non nullity of the collection; - its non emptiness (if that's not case, an exception will be thrown using the provided message). - assertNotEmpty(Collection), calling assertNotEmpty(String, Collection) with null as the value of the string argument. A new one has been added to this assertion class to check the contrary: assertEmpty(Collection), checking emptiness of the collection only if it is not null. Three new methods have been added in ExtractorAsserts as utility test methods for image collections: - assertContainsImageUrlInImageCollection(String, Collection), checking that: - the provided URL and image collection are not null; - the image collection contains at least one image which has the provided string value as its URL (which is a string) property. - assertContainsOnlyEquivalentImages(Collection, Collection), checking that: - both collections are not null; - they have the same size; - each image of the first collection has its equivalent in the second one. This means that the properties of an image in the first collection must be equal in an image of the second one. - assertNotOnlyContainsEquivalentImages(Collection, Collection), checking that: - both collections are not null; - one of the following conditions is met: - they have different sizes; - an image of the first collection has not its equivalent in the second one. This means that the properties of an image in the first collection must be not equal in an image of the second one. These methods will be used by services extractors tests (and default ones) to test image collections. --- .../newpipe/extractor/ExtractorAsserts.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java index b47f6d5ee0..d66e433ec9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java @@ -4,6 +4,7 @@ import java.net.URL; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Set; @@ -60,6 +61,16 @@ public static void assertNotEmpty(@Nullable String message, String stringToCheck assertFalse(stringToCheck.isEmpty(), message); } + public static void assertNotEmpty(@Nullable final Collection collectionToCheck) { + assertNotEmpty(null, collectionToCheck); + } + + public static void assertNotEmpty(@Nullable final String message, + @Nullable final Collection collectionToCheck) { + assertNotNull(collectionToCheck); + assertFalse(collectionToCheck.isEmpty(), message); + } + public static void assertEmpty(String stringToCheck) { assertEmpty(null, stringToCheck); } @@ -70,6 +81,12 @@ public static void assertEmpty(@Nullable String message, String stringToCheck) { } } + public static void assertEmpty(@Nullable final Collection collectionToCheck) { + if (collectionToCheck != null) { + assertTrue(collectionToCheck.isEmpty()); + } + } + public static void assertNotBlank(String stringToCheck) { assertNotBlank(stringToCheck, null); } @@ -160,4 +177,50 @@ public static void assertTabsContain(@Nonnull final List tabs, .forEach(expectedTab -> assertTrue(tabSet.contains(expectedTab), "Missing " + expectedTab + " tab (got " + tabSet + ")")); } + + public static void assertContainsImageUrlInImageCollection( + @Nullable final String exceptedImageUrlContained, + @Nullable final Collection imageCollection) { + assertNotNull(exceptedImageUrlContained, "exceptedImageUrlContained is null"); + assertNotNull(imageCollection, "imageCollection is null"); + assertTrue(imageCollection.stream().anyMatch(image -> + image.getUrl().equals(exceptedImageUrlContained))); + } + + public static void assertContainsOnlyEquivalentImages( + @Nullable final Collection firstImageCollection, + @Nullable final Collection secondImageCollection) { + assertNotNull(firstImageCollection); + assertNotNull(secondImageCollection); + assertEquals(firstImageCollection.size(), secondImageCollection.size()); + + firstImageCollection.forEach(exceptedImage -> + assertTrue(secondImageCollection.stream().anyMatch(image -> + exceptedImage.getUrl().equals(image.getUrl()) + && exceptedImage.getHeight() == image.getHeight() + && exceptedImage.getWidth() == image.getWidth()))); + } + + public static void assertNotOnlyContainsEquivalentImages( + @Nullable final Collection firstImageCollection, + @Nullable final Collection secondImageCollection) { + assertNotNull(firstImageCollection); + assertNotNull(secondImageCollection); + + if (secondImageCollection.size() != firstImageCollection.size()) { + return; + } + + for (final Image unexpectedImage : firstImageCollection) { + for (final Image image : secondImageCollection) { + if (!image.getUrl().equals(unexpectedImage.getUrl()) + || image.getHeight() != unexpectedImage.getHeight() + || image.getWidth() != unexpectedImage.getWidth()) { + return; + } + } + } + + throw new AssertionError("All excepted images have an equivalent in the image list"); + } } From d381f3b70b20b5a50ce28e744d37980ff0ed3136 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 16:01:38 +0200 Subject: [PATCH 28/35] Update avatar, banners and thumbnail methods' name and apply changes in DefaultStreamExtractorTest --- .../services/DefaultStreamExtractorTest.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index d9b4e6cde2..fab5c8f3a1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -29,10 +29,12 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreaterOrEqual; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEqualsOrderIndependent; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreaterOrEqual; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection; import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems; import static org.schabi.newpipe.extractor.stream.StreamExtractor.UNKNOWN_SUBSCRIBER_COUNT; @@ -98,8 +100,8 @@ public void testUploaderUrl() throws Exception { @Test @Override - public void testUploaderAvatarUrl() throws Exception { - assertIsSecureUrl(extractor().getUploaderAvatarUrl()); + public void testUploaderAvatars() throws Exception { + defaultTestImageCollection(extractor().getUploaderAvatars()); } @Test @@ -137,20 +139,20 @@ public void testSubChannelUrl() throws Exception { @Test @Override - public void testSubChannelAvatarUrl() throws Exception { + public void testSubChannelAvatars() throws Exception { if (expectedSubChannelName().isEmpty() && expectedSubChannelUrl().isEmpty()) { // this stream has no subchannel - assertEquals("", extractor().getSubChannelAvatarUrl()); + assertEmpty(extractor().getSubChannelAvatars()); } else { // this stream has a subchannel - assertIsSecureUrl(extractor().getSubChannelAvatarUrl()); + defaultTestImageCollection(extractor().getSubChannelAvatars()); } } @Test @Override - public void testThumbnailUrl() throws Exception { - assertIsSecureUrl(extractor().getThumbnailUrl()); + public void testThumbnails() throws Exception { + defaultTestImageCollection(extractor().getThumbnails()); } @Test From 2c436d428cf7e27d9df68f46002935f6408a8171 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 16:06:21 +0200 Subject: [PATCH 29/35] [YouTube] Add utility test method to test images in YoutubeTestsUtils This method, testImages(Collection), will use first the default image collection test in DefaultTests and then will check that each image URL contains the string yt. The JavaDoc of the class has been also updated to reflect the changes made in it (it is now more general). --- .../services/youtube/YoutubeTestsUtils.java | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java index a980398732..fec933c004 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java @@ -1,11 +1,16 @@ package org.schabi.newpipe.extractor.services.youtube; +import org.schabi.newpipe.extractor.ExtractorAsserts; +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.services.DefaultTests; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import javax.annotation.Nullable; +import java.util.Collection; import java.util.Random; /** - * Utility class for keeping YouTube tests stateless. + * Utility class for YouTube tests. */ public final class YoutubeTestsUtils { private YoutubeTestsUtils() { @@ -26,4 +31,20 @@ public static void ensureStateless() { YoutubeParsingHelper.setNumberGenerator(new Random(1)); YoutubeStreamExtractor.resetDeobfuscationCode(); } + + /** + * Test that YouTube images of a {@link Collection} respect + * {@link DefaultTests#defaultTestImageCollection(Collection) default requirements} and contain + * the string {@code yt} in their URL. + * + * @param images a YouTube {@link Image} {@link Collection} + */ + public static void testImages(@Nullable final Collection images) { + DefaultTests.defaultTestImageCollection(images); + // Disable NPE warning because if the collection is null, an AssertionError would be thrown + // by DefaultTests.defaultTestImageCollection + //noinspection DataFlowIssue + images.forEach(image -> + ExtractorAsserts.assertContains("yt", image.getUrl())); + } } From 93a210394d1588173b799f7f403f2ddd700c06ab Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 16:41:03 +0200 Subject: [PATCH 30/35] [YouTube] Apply changes in extractor tests Also remove some public test methods modifiers, add missing Test annotations on old Junit 4 tests (and update them if needed), and use final in some places where it was possible. --- .../youtube/YoutubeChannelExtractorTest.java | 102 +++++++----------- .../youtube/YoutubeCommentsExtractorTest.java | 44 ++++---- .../YoutubeMixPlaylistExtractorTest.java | 51 ++++----- .../youtube/YoutubePlaylistExtractorTest.java | 92 ++++++---------- .../search/YoutubeSearchExtractorTest.java | 23 ++-- .../YoutubeStreamExtractorDefaultTest.java | 4 +- .../YoutubeStreamExtractorRelatedMixTest.java | 5 +- 7 files changed, 134 insertions(+), 187 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 1c2289e4b1..856b331bad 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -2,11 +2,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertNotBlank; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContain; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -203,17 +202,13 @@ public void testDescription() throws Exception { } @Test - public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test - public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - assertContains("yt3", bannerUrl); + public void testBanners() throws Exception { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test @@ -226,6 +221,7 @@ public void testSubscriberCount() throws Exception { ExtractorAsserts.assertGreaterOrEqual(4_900_000, extractor.getSubscriberCount()); } + @Test @Override public void testVerified() throws Exception { assertTrue(extractor.isVerified()); @@ -248,7 +244,7 @@ public void testTags() throws Exception { } } - // Youtube RED/Premium ad blocking test + // YouTube RED/Premium ad blocking test public static class VSauce implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; @@ -300,17 +296,13 @@ public void testDescription() throws Exception { } @Test - public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test - public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - assertContains("yt3", bannerUrl); + public void testBanners() throws Exception { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test @@ -400,17 +392,13 @@ public void testDescription() throws Exception { } @Test - public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test - public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - assertContains("yt3", bannerUrl); + public void testBanners() throws Exception { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test @@ -524,17 +512,13 @@ public void testDescription() throws Exception { } @Test - public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test - public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - assertContains("yt3", bannerUrl); + public void testBanners() throws Exception { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test @@ -621,17 +605,13 @@ public void testDescription() throws Exception { } @Test - public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test - public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - assertContains("yt3", bannerUrl); + public void testBanners() throws Exception { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test @@ -689,7 +669,7 @@ public void testServiceId() { @Test public void testName() throws Exception { - assertEquals(extractor.getName(), "Coachella"); + assertEquals("Coachella", extractor.getName()); } @Test @@ -718,16 +698,14 @@ public void testDescription() throws ParsingException { } @Test - public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test - public void testBannerUrl() throws Exception { - // CarouselHeaderRenders do not contain a banner - assertNull(extractor.getBannerUrl()); + public void testBanners() { + // A CarouselHeaderRenderer doesn't contain a banner + assertEmpty(extractor.getBanners()); } @Test @@ -795,17 +773,15 @@ public void testDescription() throws Exception { @Test @Override - public void testAvatarUrl() throws Exception { - final String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test @Override - public void testBannerUrl() throws Exception { + public void testBanners() throws Exception { // Banners cannot be extracted from age-restricted channels - assertTrue(isNullOrEmpty(extractor.getBannerUrl())); + assertEmpty(extractor.getBanners()); } @Test @@ -913,18 +889,14 @@ public void testDescription() throws Exception { @Test @Override - public void testAvatarUrl() throws Exception { - final String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertContains("yt3", avatarUrl); + public void testAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getAvatars()); } @Test @Override - public void testBannerUrl() throws Exception { - final String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - assertContains("yt3", bannerUrl); + public void testBanners() throws Exception { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index b05502a8ae..611c6b457d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -49,7 +49,7 @@ public static void setUp() throws Exception { } @Test - public void testGetComments() throws IOException, ExtractionException { + void testGetComments() throws IOException, ExtractionException { assertTrue(getCommentsHelper(extractor)); } @@ -66,11 +66,11 @@ private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOE } @Test - public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { + void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { assertTrue(getCommentsFromCommentsInfoHelper(url)); } - private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException { + private boolean getCommentsFromCommentsInfoHelper(final String url) throws IOException, ExtractionException { final CommentsInfo commentsInfo = CommentsInfo.getInfo(url); assertEquals("Comments", commentsInfo.getName()); @@ -87,21 +87,21 @@ private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException } @Test - public void testGetCommentsAllData() throws IOException, ExtractionException { + void testGetCommentsAllData() throws IOException, ExtractionException { InfoItemsPage comments = extractor.getInitialPage(); assertTrue(extractor.getCommentsCount() > 5); // at least 5 comments DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); - for (CommentsInfoItem c : comments.getItems()) { + for (final CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderUrl())); assertFalse(Utils.isBlank(c.getUploaderName())); - assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + YoutubeTestsUtils.testImages(c.getUploaderAvatars()); assertFalse(Utils.isBlank(c.getCommentId())); assertFalse(Utils.isBlank(c.getCommentText().getContent())); assertFalse(Utils.isBlank(c.getName())); assertFalse(Utils.isBlank(c.getTextualUploadDate())); assertNotNull(c.getUploadDate()); - assertFalse(Utils.isBlank(c.getThumbnailUrl())); + YoutubeTestsUtils.testImages(c.getThumbnails()); assertFalse(Utils.isBlank(c.getUrl())); assertTrue(c.getLikeCount() >= 0); } @@ -138,19 +138,19 @@ public static void setUp() throws Exception { } @Test - public void testGetCommentsAllData() throws IOException, ExtractionException { + void testGetCommentsAllData() throws IOException, ExtractionException { final InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); - for (CommentsInfoItem c : comments.getItems()) { + for (final CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderUrl())); assertFalse(Utils.isBlank(c.getUploaderName())); - assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + YoutubeTestsUtils.testImages(c.getUploaderAvatars()); assertFalse(Utils.isBlank(c.getCommentId())); assertFalse(Utils.isBlank(c.getName())); assertFalse(Utils.isBlank(c.getTextualUploadDate())); assertNotNull(c.getUploadDate()); - assertFalse(Utils.isBlank(c.getThumbnailUrl())); + YoutubeTestsUtils.testImages(c.getThumbnails()); assertFalse(Utils.isBlank(c.getUrl())); assertTrue(c.getLikeCount() >= 0); if (c.getCommentId().equals("Ugga_h1-EXdHB3gCoAEC")) { // comment without text @@ -177,22 +177,22 @@ public static void setUp() throws Exception { } @Test - public void testGetCommentsAllData() throws IOException, ExtractionException { + void testGetCommentsAllData() throws IOException, ExtractionException { final InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); boolean heartedByUploader = false; - for (CommentsInfoItem c : comments.getItems()) { + for (final CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderUrl())); assertFalse(Utils.isBlank(c.getUploaderName())); - assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + YoutubeTestsUtils.testImages(c.getUploaderAvatars()); assertFalse(Utils.isBlank(c.getCommentId())); assertFalse(Utils.isBlank(c.getName())); assertFalse(Utils.isBlank(c.getTextualUploadDate())); assertNotNull(c.getUploadDate()); - assertFalse(Utils.isBlank(c.getThumbnailUrl())); + YoutubeTestsUtils.testImages(c.getThumbnails()); assertFalse(Utils.isBlank(c.getUrl())); assertTrue(c.getLikeCount() >= 0); assertFalse(Utils.isBlank(c.getCommentText().getContent())); @@ -219,20 +219,20 @@ public static void setUp() throws Exception { } @Test - public void testGetCommentsAllData() throws IOException, ExtractionException { + void testGetCommentsAllData() throws IOException, ExtractionException { final InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); - for (CommentsInfoItem c : comments.getItems()) { + for (final CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderUrl())); assertFalse(Utils.isBlank(c.getUploaderName())); - assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + YoutubeTestsUtils.testImages(c.getUploaderAvatars()); assertFalse(Utils.isBlank(c.getCommentId())); assertFalse(Utils.isBlank(c.getName())); assertFalse(Utils.isBlank(c.getTextualUploadDate())); assertNotNull(c.getUploadDate()); - assertFalse(Utils.isBlank(c.getThumbnailUrl())); + YoutubeTestsUtils.testImages(c.getThumbnails()); assertFalse(Utils.isBlank(c.getUrl())); assertTrue(c.getLikeCount() >= 0); assertFalse(Utils.isBlank(c.getCommentText().getContent())); @@ -260,7 +260,7 @@ public static void setUp() throws Exception { } @Test - public void testGetCommentsFirst() throws IOException, ExtractionException { + void testGetCommentsFirst() throws IOException, ExtractionException { final InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); @@ -293,7 +293,7 @@ public static void setUp() throws Exception { } @Test - public void testGetCommentsFirst() throws IOException, ExtractionException { + void testGetCommentsFirst() throws IOException, ExtractionException { final InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); @@ -319,7 +319,7 @@ public static void setUp() throws Exception { } @Test - public void testGetCommentsFirstReplies() throws IOException, ExtractionException { + void testGetCommentsFirstReplies() throws IOException, ExtractionException { final InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index fc765bcdef..3913ca2a25 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -4,7 +4,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; @@ -69,11 +68,10 @@ void getName() throws Exception { } @Test - void getThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); - ExtractorAsserts.assertContains(VIDEO_ID, thumbnailUrl); + void getThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); + extractor.getThumbnails().forEach(thumbnail -> + ExtractorAsserts.assertContains(VIDEO_ID, thumbnail.getUrl())); } @Test @@ -158,11 +156,10 @@ void getName() throws Exception { } @Test - void getThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); - ExtractorAsserts.assertContains(VIDEO_ID, thumbnailUrl); + void getThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); + extractor.getThumbnails().forEach(thumbnail -> + ExtractorAsserts.assertContains(VIDEO_ID, thumbnail.getUrl())); } @Test @@ -248,10 +245,10 @@ void getName() throws Exception { } @Test - void getThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/" + VIDEO_ID)); + void getThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); + extractor.getThumbnails().forEach(thumbnail -> + ExtractorAsserts.assertContains(VIDEO_ID, thumbnail.getUrl())); } @Test @@ -366,10 +363,10 @@ void getName() throws Exception { } @Test - void getThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); + void getThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); + extractor.getThumbnails().forEach(thumbnail -> + ExtractorAsserts.assertContains(VIDEO_ID_OF_CHANNEL, thumbnail.getUrl())); } @Test @@ -433,11 +430,10 @@ void getName() throws Exception { } @Test - void getThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); - ExtractorAsserts.assertContains(VIDEO_ID, thumbnailUrl); + void getThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); + extractor.getThumbnails().forEach(thumbnail -> + ExtractorAsserts.assertContains(VIDEO_ID, thumbnail.getUrl())); } @Test @@ -523,10 +519,9 @@ void getName() throws Exception { @Test void getThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); - ExtractorAsserts.assertContains(VIDEO_ID, thumbnailUrl); + YoutubeTestsUtils.testImages(extractor.getThumbnails()); + extractor.getThumbnails().forEach(thumbnail -> + ExtractorAsserts.assertContains(VIDEO_ID, thumbnail.getUrl())); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 1e2a3be1d5..5ccec5afb9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems; import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; @@ -121,22 +120,17 @@ public void testMoreRelatedItems() throws Exception { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); + public void testThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); } - @Disabled @Test - public void testBannerUrl() throws ParsingException { - final String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - ExtractorAsserts.assertContains("yt", bannerUrl); + public void testBanners() throws ParsingException { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test - public void testUploaderUrl() throws Exception { + void testUploaderUrl() throws Exception { assertEquals("https://www.youtube.com/channel/UCs72iRpTEuwV3y6pdWYLgiw", extractor.getUploaderUrl()); } @@ -147,9 +141,8 @@ public void testUploaderName() throws Exception { } @Test - public void testUploaderAvatarUrl() throws Exception { - final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl(); - ExtractorAsserts.assertContains("yt", uploaderAvatarUrl); + public void testUploaderAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getUploaderAvatars()); } @Test @@ -157,6 +150,7 @@ public void testStreamCount() throws Exception { ExtractorAsserts.assertGreater(100, extractor.getStreamCount()); } + @Test @Override public void testUploaderVerified() throws Exception { assertFalse(extractor.isUploaderVerified()); @@ -191,7 +185,7 @@ public static void setUp() throws Exception { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testGetPageInNewExtractor() throws Exception { + void testGetPageInNewExtractor() throws Exception { final PlaylistExtractor newExtractor = YouTube.getPlaylistExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor); } @@ -251,22 +245,17 @@ public void testMoreRelatedItems() throws Exception { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); + public void testThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); } - @Disabled @Test - public void testBannerUrl() throws ParsingException { - final String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - ExtractorAsserts.assertContains("yt", bannerUrl); + public void testBanners() throws ParsingException { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test - public void testUploaderUrl() throws Exception { + void testUploaderUrl() throws Exception { assertEquals("https://www.youtube.com/channel/UCHSPWoY1J5fbDVbcnyeqwdw", extractor.getUploaderUrl()); } @@ -276,9 +265,8 @@ public void testUploaderName() throws Exception { } @Test - public void testUploaderAvatarUrl() throws Exception { - final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl(); - ExtractorAsserts.assertContains("yt", uploaderAvatarUrl); + public void testUploaderAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getUploaderAvatars()); } @Test @@ -286,9 +274,10 @@ public void testStreamCount() throws Exception { ExtractorAsserts.assertGreater(100, extractor.getStreamCount()); } + @Test @Override public void testUploaderVerified() throws Exception { - assertTrue(extractor.isUploaderVerified()); + assertFalse(extractor.isUploaderVerified()); } @Test @@ -364,22 +353,17 @@ public void testMoreRelatedItems() throws Exception { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); + public void testThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); } - @Disabled @Test - public void testBannerUrl() throws ParsingException { - final String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - ExtractorAsserts.assertContains("yt", bannerUrl); + public void testBanners() throws ParsingException { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test - public void testUploaderUrl() throws Exception { + void testUploaderUrl() throws Exception { assertEquals("https://www.youtube.com/channel/UCX6b17PVsYBQ0ip5gyeme-Q", extractor.getUploaderUrl()); } @@ -390,9 +374,8 @@ public void testUploaderName() throws Exception { } @Test - public void testUploaderAvatarUrl() throws Exception { - final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl(); - ExtractorAsserts.assertContains("yt", uploaderAvatarUrl); + public void testUploaderAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getUploaderAvatars()); } @Test @@ -400,9 +383,10 @@ public void testStreamCount() throws Exception { ExtractorAsserts.assertGreater(40, extractor.getStreamCount()); } + @Test @Override public void testUploaderVerified() throws Exception { - assertTrue(extractor.isUploaderVerified()); + assertFalse(extractor.isUploaderVerified()); } @Test @@ -480,18 +464,14 @@ public void testMoreRelatedItems() throws Exception { @Test @Override - public void testThumbnailUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); + public void testThumbnails() throws Exception { + YoutubeTestsUtils.testImages(extractor.getThumbnails()); } @Test @Override - public void testBannerUrl() throws Exception { - final String thumbnailUrl = extractor.getThumbnailUrl(); - assertIsSecureUrl(thumbnailUrl); - ExtractorAsserts.assertContains("yt", thumbnailUrl); + public void testBanners() throws Exception { + YoutubeTestsUtils.testImages(extractor.getBanners()); } @Test @@ -501,10 +481,8 @@ public void testUploaderName() throws Exception { } @Test - @Override - public void testUploaderAvatarUrl() throws Exception { - final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl(); - ExtractorAsserts.assertContains("yt", uploaderAvatarUrl); + public void testUploaderAvatars() throws Exception { + YoutubeTestsUtils.testImages(extractor.getUploaderAvatars()); } @Test @@ -540,7 +518,7 @@ public static void setUp() throws IOException { } @Test - public void testNoContinuations() throws Exception { + void testNoContinuations() throws Exception { final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube .getPlaylistExtractor( "https://www.youtube.com/playlist?list=PLXJg25X-OulsVsnvZ7RVtSDW-id9_RzAO"); @@ -550,7 +528,7 @@ public void testNoContinuations() throws Exception { } @Test - public void testOnlySingleContinuation() throws Exception { + void testOnlySingleContinuation() throws Exception { final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube .getPlaylistExtractor( "https://www.youtube.com/playlist?list=PLoumn5BIsUDeGF1vy5Nylf_RJKn5aL_nr"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index 3c9f310fc8..33f2fe43d8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -234,9 +234,9 @@ public void testMoreRelatedItems() throws Exception { } } - public static class PagingTest { + static class PagingTest { @Test - public void duplicatedItemsCheck() throws Exception { + void duplicatedItemsCheck() throws Exception { YoutubeTestsUtils.ensureStateless(); NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "paging")); final SearchExtractor extractor = YouTube.getSearchExtractor("cirque du soleil", singletonList(VIDEOS), ""); @@ -275,7 +275,7 @@ public static void setUp() throws Exception { )); } // testMoreRelatedItems is broken because a video has no duration shown - @Override public void testMoreRelatedItems() { } + @Test @Override public void testMoreRelatedItems() { } @Override public SearchExtractor extractor() { return extractor; } @Override public StreamingService expectedService() { return YouTube; } @Override public String expectedName() { return QUERY; } @@ -307,7 +307,7 @@ public static void setUp() throws Exception { @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; } @Test - public void testAtLeastOneVerified() throws IOException, ExtractionException { + void testAtLeastOneVerified() throws IOException, ExtractionException { final List items = extractor.getInitialPage().getItems(); boolean verified = false; for (InfoItem item : items) { @@ -344,11 +344,14 @@ public static void setUp() throws Exception { @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } @Test - public void testUploaderAvatar() throws IOException, ExtractionException { - final List items = extractor.getInitialPage().getItems(); - for (final InfoItem item : items) { - assertNotNull(((StreamInfoItem) item).getUploaderAvatarUrl()); - } + void testUploaderAvatars() throws IOException, ExtractionException { + extractor.getInitialPage() + .getItems() + .stream() + .filter(StreamInfoItem.class::isInstance) + .map(StreamInfoItem.class::cast) + .forEach(streamInfoItem -> + YoutubeTestsUtils.testImages(streamInfoItem.getUploaderAvatars())); } } @@ -375,7 +378,7 @@ public static void setUp() throws Exception { @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } @Test - public void testVideoDescription() throws IOException, ExtractionException { + void testVideoDescription() throws IOException, ExtractionException { final List items = extractor.getInitialPage().getItems(); assertNotNull(((StreamInfoItem) items.get(0)).getShortDescription()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index fff75450c9..22b5f6f99f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -491,8 +491,8 @@ public void testLikeCount() { @Test @Override - public void testUploaderAvatarUrl() { - assertThrows(ParsingException.class, () -> extractor.getUploaderAvatarUrl()); + public void testUploaderAvatars() { + assertThrows(ParsingException.class, () -> extractor.getUploaderAvatars()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorRelatedMixTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorRelatedMixTest.java index cd7feb3527..6906471848 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorRelatedMixTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorRelatedMixTest.java @@ -4,7 +4,6 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertSame; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.youtube.stream.YoutubeStreamExtractorDefaultTest.YOUTUBE_LICENCE; @@ -109,7 +108,7 @@ public void testRelatedItems() throws Exception { assertContains(URL, streamMix.getUrl()); assertContains("list=RD" + ID, streamMix.getUrl()); assertEquals("Mix – " + TITLE, streamMix.getName()); - assertIsSecureUrl(streamMix.getThumbnailUrl()); + YoutubeTestsUtils.testImages(streamMix.getThumbnails()); final List musicMixes = playlists.stream() .filter(item -> item.getPlaylistType().equals(PlaylistType.MIX_MUSIC)) @@ -121,6 +120,6 @@ public void testRelatedItems() throws Exception { assertEquals(YouTube.getServiceId(), musicMix.getServiceId()); assertContains("list=RDCLAK", musicMix.getUrl()); assertEquals("Hip Hop Essentials", musicMix.getName()); - assertIsSecureUrl(musicMix.getThumbnailUrl()); + YoutubeTestsUtils.testImages(musicMix.getThumbnails()); } } From 1d72bac53d4b7a30275ccc479eb0c14f25a4efb9 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 16:42:49 +0200 Subject: [PATCH 31/35] [SoundCloud] Apply changes in extractor tests --- .../SoundcloudChannelExtractorTest.java | 22 +++---- .../SoundcloudPlaylistExtractorTest.java | 59 +++++++++++-------- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java index bd94dfd26c..7a6e2f1e36 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java @@ -9,11 +9,13 @@ import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelExtractor; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContain; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection; /** * Test for {@link SoundcloudChannelExtractor} @@ -69,13 +71,13 @@ public void testDescription() { } @Test - public void testAvatarUrl() { - assertIsSecureUrl(extractor.getAvatarUrl()); + public void testAvatars() { + defaultTestImageCollection(extractor.getAvatars()); } @Test - public void testBannerUrl() { - assertIsSecureUrl(extractor.getBannerUrl()); + public void testBanners() { + defaultTestImageCollection(extractor.getBanners()); } @Test @@ -157,13 +159,13 @@ public void testDescription() { } @Test - public void testAvatarUrl() { - assertIsSecureUrl(extractor.getAvatarUrl()); + public void testAvatars() { + defaultTestImageCollection(extractor.getAvatars()); } @Test - public void testBannerUrl() { - assertIsSecureUrl(extractor.getBannerUrl()); + public void testBanners() { + defaultTestImageCollection(extractor.getBanners()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 1b8930dc68..38fe024e98 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -13,11 +13,18 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; -import static org.schabi.newpipe.extractor.services.DefaultTests.*; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; /** * Test for {@link PlaylistExtractor} @@ -82,14 +89,14 @@ public void testMoreRelatedItems() throws Exception { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testThumbnailUrl() { - assertIsSecureUrl(extractor.getThumbnailUrl()); + public void testThumbnails() { + defaultTestImageCollection(extractor.getThumbnails()); } @Test - public void testBannerUrl() throws ParsingException { + public void testBanners() throws ParsingException { // SoundCloud playlists do not have a banner - assertEmpty(extractor.getBannerUrl()); + assertEmpty(extractor.getBanners()); } @Test @@ -105,8 +112,8 @@ public void testUploaderName() { } @Test - public void testUploaderAvatarUrl() { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + public void testUploaderAvatars() { + defaultTestImageCollection(extractor.getUploaderAvatars()); } @Test @@ -179,14 +186,14 @@ public void testMoreRelatedItems() throws Exception { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testThumbnailUrl() { - assertIsSecureUrl(extractor.getThumbnailUrl()); + public void testThumbnails() { + defaultTestImageCollection(extractor.getThumbnails()); } @Test - public void testBannerUrl() throws ParsingException { + public void testBanners() throws ParsingException { // SoundCloud playlists do not have a banner - assertEmpty(extractor.getBannerUrl()); + assertEmpty(extractor.getBanners()); } @Test @@ -202,8 +209,8 @@ public void testUploaderName() { } @Test - public void testUploaderAvatarUrl() { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + public void testUploaderAvatars() { + defaultTestImageCollection(extractor.getUploaderAvatars()); } @Test @@ -291,14 +298,14 @@ public void testMoreRelatedItems() throws Exception { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testThumbnailUrl() { - assertIsSecureUrl(extractor.getThumbnailUrl()); + public void testThumbnails() { + defaultTestImageCollection(extractor.getThumbnails()); } @Test - public void testBannerUrl() throws ParsingException { + public void testBanners() throws ParsingException { // SoundCloud playlists do not have a banner - assertEmpty(extractor.getBannerUrl()); + assertEmpty(extractor.getBanners()); } @Test @@ -314,8 +321,8 @@ public void testUploaderName() { } @Test - public void testUploaderAvatarUrl() { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + public void testUploaderAvatars() { + defaultTestImageCollection(extractor.getUploaderAvatars()); } @Test @@ -395,14 +402,14 @@ public void testMoreRelatedItems() throws Exception { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testThumbnailUrl() { - assertIsSecureUrl(extractor.getThumbnailUrl()); + public void testThumbnails() { + defaultTestImageCollection(extractor.getThumbnails()); } @Test - public void testBannerUrl() throws ParsingException { + public void testBanners() throws ParsingException { // SoundCloud playlists do not have a banner - assertEmpty(extractor.getBannerUrl()); + assertEmpty(extractor.getBanners()); } @Test @@ -418,8 +425,8 @@ public void testUploaderName() { } @Test - public void testUploaderAvatarUrl() { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + public void testUploaderAvatars() { + defaultTestImageCollection(extractor.getUploaderAvatars()); } @Test From ba5315c72dcdfc54839cbecd321310bb8d005117 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 16:51:46 +0200 Subject: [PATCH 32/35] [PeerTube] Apply changes in extractor tests Also remove some public test methods modifiers, add missing Test annotations on old Junit 4 tests (and update them if needed), and improve some code. --- .../PeertubeAccountExtractorTest.java | 23 ++++---- .../PeertubeChannelExtractorTest.java | 52 ++++++++++--------- .../PeertubeCommentsExtractorTest.java | 5 +- .../PeertubePlaylistExtractorTest.java | 21 +++----- 4 files changed, 50 insertions(+), 51 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java index 63397c6ba3..869ba31ff3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java @@ -15,9 +15,10 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContain; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection; /** * Test for {@link PeertubeAccountExtractor} @@ -76,13 +77,13 @@ public void testDescription() throws ParsingException { } @Test - public void testAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getAvatarUrl()); + public void testAvatars() { + defaultTestImageCollection(extractor.getAvatars()); } @Test - public void testBannerUrl() { - assertNull(extractor.getBannerUrl()); + public void testBanners() { + assertEmpty(extractor.getBanners()); } @Test @@ -95,6 +96,7 @@ public void testSubscriberCount() throws ParsingException { ExtractorAsserts.assertGreaterOrEqual(700, extractor.getSubscriberCount()); } + @Test @Override public void testVerified() throws Exception { assertFalse(extractor.isVerified()); @@ -160,18 +162,18 @@ public void testOriginalUrl() throws ParsingException { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testDescription() throws ParsingException { + public void testDescription() { assertNotNull(extractor.getDescription()); } @Test - public void testAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getAvatarUrl()); + public void testAvatars() { + defaultTestImageCollection(extractor.getAvatars()); } @Test - public void testBannerUrl() throws ParsingException { - assertNull(extractor.getBannerUrl()); + public void testBanners() { + assertEmpty(extractor.getBanners()); } @Test @@ -184,6 +186,7 @@ public void testSubscriberCount() throws ParsingException { ExtractorAsserts.assertGreaterOrEqual(100, extractor.getSubscriberCount()); } + @Test @Override public void testVerified() throws Exception { assertFalse(extractor.isVerified()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 72ce048397..5b48849f8b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -3,7 +3,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; -import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -13,11 +12,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreaterOrEqual; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContain; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection; /** * Test for {@link PeertubeChannelExtractor} @@ -71,33 +71,33 @@ public void testOriginalUrl() throws ParsingException { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testDescription() throws ParsingException { + public void testDescription() { assertNotNull(extractor.getDescription()); } @Test - public void testParentChannelName() throws ParsingException { + void testParentChannelName() throws ParsingException { assertEquals("lqdn", extractor.getParentChannelName()); } @Test - public void testParentChannelUrl() throws ParsingException { + void testParentChannelUrl() throws ParsingException { assertEquals("https://video.lqdn.fr/accounts/lqdn", extractor.getParentChannelUrl()); } @Test - public void testParentChannelAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getParentChannelAvatarUrl()); + void testParentChannelAvatarUrl() { + defaultTestImageCollection(extractor.getParentChannelAvatars()); } @Test - public void testAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getAvatarUrl()); + public void testAvatars() { + defaultTestImageCollection(extractor.getAvatars()); } @Test - public void testBannerUrl() throws ParsingException { - assertNull(extractor.getBannerUrl()); + public void testBanners() { + assertEmpty(extractor.getBanners()); } @Test @@ -106,10 +106,11 @@ public void testFeedUrl() throws ParsingException { } @Test - public void testSubscriberCount() throws ParsingException { - ExtractorAsserts.assertGreaterOrEqual(230, extractor.getSubscriberCount()); + public void testSubscriberCount() { + assertGreaterOrEqual(230, extractor.getSubscriberCount()); } + @Test @Override public void testVerified() throws Exception { assertFalse(extractor.isVerified()); @@ -176,33 +177,33 @@ public void testOriginalUrl() throws ParsingException { //////////////////////////////////////////////////////////////////////////*/ @Test - public void testDescription() throws ParsingException { + public void testDescription() { assertNotNull(extractor.getDescription()); } @Test - public void testParentChannelName() throws ParsingException { + void testParentChannelName() throws ParsingException { assertEquals("nathan", extractor.getParentChannelName()); } @Test - public void testParentChannelUrl() throws ParsingException { + void testParentChannelUrl() throws ParsingException { assertEquals("https://skeptikon.fr/accounts/nathan", extractor.getParentChannelUrl()); } @Test - public void testParentChannelAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getParentChannelAvatarUrl()); + void testParentChannelAvatars() { + defaultTestImageCollection(extractor.getParentChannelAvatars()); } @Test - public void testAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getAvatarUrl()); + public void testAvatars() { + defaultTestImageCollection(extractor.getAvatars()); } @Test - public void testBannerUrl() throws ParsingException { - assertNull(extractor.getBannerUrl()); + public void testBanners() throws ParsingException { + assertEmpty(extractor.getBanners()); } @Test @@ -211,10 +212,11 @@ public void testFeedUrl() throws ParsingException { } @Test - public void testSubscriberCount() throws ParsingException { - ExtractorAsserts.assertGreaterOrEqual(700, extractor.getSubscriberCount()); + public void testSubscriberCount() { + assertGreaterOrEqual(700, extractor.getSubscriberCount()); } + @Test @Override public void testVerified() throws Exception { assertFalse(extractor.isVerified()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java index 353e00482d..ee2303706e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.*; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection; public class PeertubeCommentsExtractorTest { public static class Default { @@ -73,12 +74,12 @@ void testGetCommentsAllData() throws IOException, ExtractionException { .forEach(commentsInfoItem -> { assertFalse(Utils.isBlank(commentsInfoItem.getUploaderUrl())); assertFalse(Utils.isBlank(commentsInfoItem.getUploaderName())); - assertFalse(Utils.isBlank(commentsInfoItem.getUploaderAvatarUrl())); + defaultTestImageCollection(commentsInfoItem.getUploaderAvatars()); assertFalse(Utils.isBlank(commentsInfoItem.getCommentId())); assertFalse(Utils.isBlank(commentsInfoItem.getCommentText().getContent())); assertFalse(Utils.isBlank(commentsInfoItem.getName())); assertFalse(Utils.isBlank(commentsInfoItem.getTextualUploadDate())); - assertFalse(Utils.isBlank(commentsInfoItem.getThumbnailUrl())); + defaultTestImageCollection(commentsInfoItem.getThumbnails()); assertFalse(Utils.isBlank(commentsInfoItem.getUrl())); assertEquals(-1, commentsInfoItem.getLikeCount()); assertTrue(Utils.isBlank(commentsInfoItem.getTextualLikeCount())); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java index 37853305ae..4c64314cdf 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java @@ -2,9 +2,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.ExtractorAsserts; @@ -31,11 +31,8 @@ void testGetName() throws ParsingException { } @Test - @Disabled("URL changes with every request") - void testGetThumbnailUrl() throws ParsingException { - assertEquals( - "https://framatube.org/static/thumbnails/playlist-96b0ee2b-a5a7-4794-8769-58d8ccb79ab7.jpg", - extractor.getThumbnailUrl()); + void testGetThumbnails() throws ParsingException { + defaultTestImageCollection(extractor.getThumbnails()); } @Test @@ -44,10 +41,8 @@ void testGetUploaderUrl() { } @Test - void testGetUploaderAvatarUrl() throws ParsingException { - assertEquals( - "https://framatube.org/lazy-static/avatars/c6801ff9-cb49-42e6-b2db-3db623248115.jpg", - extractor.getUploaderAvatarUrl()); + void testGetUploaderAvatars() throws ParsingException { + defaultTestImageCollection(extractor.getUploaderAvatars()); } @Test @@ -76,10 +71,8 @@ void testGetSubChannelName() { } @Test - void testGetSubChannelAvatarUrl() throws ParsingException { - assertEquals( - "https://framatube.org/lazy-static/avatars/e801ccce-8694-4309-b0ab-e6f0e552ef77.png", - extractor.getSubChannelAvatarUrl()); + void testGetSubChannelAvatars() throws ParsingException { + defaultTestImageCollection(extractor.getSubChannelAvatars()); } } } From 2578f220540b04ed68ef1f971f7bf75d5e48e8c4 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 17:47:58 +0200 Subject: [PATCH 33/35] [Bandcamp] Add utility test method to test images This method, testImages(Collection), will use first the default image collection test in DefaultTests and then will check that each image URL contains f4.bcbits.com/img and ends with .jpg or .png. To do so, a new non-instantiable final class has been added: BandcampTestUtils. --- .../services/bandcamp/BandcampTestUtils.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampTestUtils.java diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampTestUtils.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampTestUtils.java new file mode 100644 index 0000000000..bc17fc515d --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampTestUtils.java @@ -0,0 +1,34 @@ +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.services.DefaultTests; + +import javax.annotation.Nullable; +import java.util.Collection; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Utility class for Bandcamp tests. + */ +public final class BandcampTestUtils { + private BandcampTestUtils() { + } + + /** + * Test that Bandcamp images of a {@link Collection} respect + * {@link DefaultTests#defaultTestImageCollection(Collection) default requirements}, contain + * the string {@code f4.bcbits.com/img} in their URL and end with {@code .jpg} or {@code .png}. + * + * @param images a Bandcamp {@link Image} {@link Collection} + */ + public static void testImages(@Nullable final Collection images) { + DefaultTests.defaultTestImageCollection(images); + // Disable NPE warning because if the collection is null, an AssertionError would be thrown + // by DefaultTests.defaultTestImageCollection + //noinspection DataFlowIssue + assertTrue(images.stream() + .allMatch(image -> image.getUrl().contains("f4.bcbits.com/img") + && (image.getUrl().endsWith(".jpg") || image.getUrl().endsWith(".png")))); + } +} From 0292c4f3e848726de0bd697eed10cf2af702ba37 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 18:05:44 +0200 Subject: [PATCH 34/35] [Bandcamp] Apply changes in extractor tests Also remove some public test methods modifiers, add missing Test annotations on old Junit 4 tests (and update them if needed), and use final in some places where it was possible. BandcampChannelExtractorTest.testLength has been removed as the test is always true. --- .../BandcampChannelExtractorTest.java | 23 ++++++-- .../BandcampCommentsExtractorTest.java | 10 ++-- .../BandcampPlaylistExtractorTest.java | 53 +++++++++++-------- .../BandcampRadioStreamExtractorTest.java | 22 +++++--- .../bandcamp/BandcampSearchExtractorTest.java | 14 ++--- .../bandcamp/BandcampStreamExtractorTest.java | 8 ++- 6 files changed, 76 insertions(+), 54 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index 066c7a5ae9..0fae98b530 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -10,7 +10,10 @@ import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContain; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; @@ -31,51 +34,61 @@ public void testDescription() throws Exception { assertEquals("making music:)", extractor.getDescription()); } + @Test @Override - public void testAvatarUrl() throws Exception { - assertTrue(extractor.getAvatarUrl().contains("://f4.bcbits.com/"), "unexpected avatar URL"); + public void testAvatars() throws Exception { + BandcampTestUtils.testImages(extractor.getAvatars()); } + @Test @Override - public void testBannerUrl() throws Exception { - assertTrue(extractor.getBannerUrl().contains("://f4.bcbits.com/"), "unexpected banner URL"); + public void testBanners() throws Exception { + BandcampTestUtils.testImages(extractor.getBanners()); } + @Test @Override public void testFeedUrl() throws Exception { assertNull(extractor.getFeedUrl()); } + @Test @Override public void testSubscriberCount() throws Exception { assertEquals(-1, extractor.getSubscriberCount()); } + @Test @Override public void testVerified() throws Exception { assertFalse(extractor.isVerified()); } + @Test @Override public void testServiceId() { assertEquals(Bandcamp.getServiceId(), extractor.getServiceId()); } + @Test @Override public void testName() throws Exception { assertEquals("toupie", extractor.getName()); } + @Test @Override public void testId() throws Exception { assertEquals("2450875064", extractor.getId()); } + @Test @Override public void testUrl() throws Exception { assertEquals("https://toupie.bandcamp.com", extractor.getUrl()); } + @Test @Override public void testOriginalUrl() throws Exception { assertEquals("https://toupie.bandcamp.com", extractor.getUrl()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsExtractorTest.java index 2fd98e2559..ed49852961 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsExtractorTest.java @@ -30,22 +30,22 @@ public static void setUp() throws ExtractionException, IOException { } @Test - public void hasComments() throws IOException, ExtractionException { + void hasComments() throws IOException, ExtractionException { assertTrue(extractor.getInitialPage().getItems().size() >= 3); } @Test - public void testGetCommentsAllData() throws IOException, ExtractionException { + void testGetCommentsAllData() throws IOException, ExtractionException { ListExtractor.InfoItemsPage comments = extractor.getInitialPage(); assertTrue(comments.hasNextPage()); DefaultTests.defaultTestListOfItems(Bandcamp, comments.getItems(), comments.getErrors()); - for (CommentsInfoItem c : comments.getItems()) { + for (final CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderName())); - assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + BandcampTestUtils.testImages(c.getUploaderAvatars()); assertFalse(Utils.isBlank(c.getCommentText().getContent())); assertFalse(Utils.isBlank(c.getName())); - assertFalse(Utils.isBlank(c.getThumbnailUrl())); + BandcampTestUtils.testImages(c.getThumbnails()); assertFalse(Utils.isBlank(c.getUrl())); assertEquals(-1, c.getLikeCount()); assertTrue(Utils.isBlank(c.getTextualLikeCount())); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java index f6f0979786..448cca262c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -19,8 +19,17 @@ import java.io.IOException; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContainsOnlyEquivalentImages; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertNotOnlyContainsEquivalentImages; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; /** @@ -37,7 +46,7 @@ public static void setUp() { * Test whether playlists contain the correct amount of items */ @Test - public void testCount() throws ExtractionException, IOException { + void testCount() throws ExtractionException, IOException { final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://macbenson.bandcamp.com/album/coming-of-age"); extractor.fetchPage(); @@ -48,13 +57,13 @@ public void testCount() throws ExtractionException, IOException { * Tests whether different stream thumbnails (track covers) get loaded correctly */ @Test - public void testDifferentTrackCovers() throws ExtractionException, IOException { + void testDifferentTrackCovers() throws ExtractionException, IOException { final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://zachbensonarchive.bandcamp.com/album/results-of-boredom"); extractor.fetchPage(); final List l = extractor.getInitialPage().getItems(); - assertEquals(extractor.getThumbnailUrl(), l.get(0).getThumbnailUrl()); - assertNotEquals(extractor.getThumbnailUrl(), l.get(5).getThumbnailUrl()); + assertContainsOnlyEquivalentImages(extractor.getThumbnails(), l.get(0).getThumbnails()); + assertNotOnlyContainsEquivalentImages(extractor.getThumbnails(), l.get(5).getThumbnails()); } /** @@ -62,23 +71,23 @@ public void testDifferentTrackCovers() throws ExtractionException, IOException { */ @Test @Timeout(10) - public void testDifferentTrackCoversDuration() throws ExtractionException, IOException { + void testDifferentTrackCoversDuration() throws ExtractionException, IOException { final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://infiniteammo.bandcamp.com/album/night-in-the-woods-vol-1-at-the-end-of-everything"); extractor.fetchPage(); - /* All tracks in this album have the same cover art, but I don't know any albums with more than 10 tracks - * that has at least one track with a cover art different from the rest. + /* All tracks on this album have the same cover art, but I don't know any albums with more + * than 10 tracks that has at least one track with a cover art different from the rest. */ final List l = extractor.getInitialPage().getItems(); - assertEquals(extractor.getThumbnailUrl(), l.get(0).getThumbnailUrl()); - assertEquals(extractor.getThumbnailUrl(), l.get(5).getThumbnailUrl()); + assertContainsOnlyEquivalentImages(extractor.getThumbnails(), l.get(0).getThumbnails()); + assertContainsOnlyEquivalentImages(extractor.getThumbnails(), l.get(5).getThumbnails()); } /** * Test playlists with locked content */ @Test - public void testLockedContent() throws ExtractionException, IOException { + void testLockedContent() throws ExtractionException { final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://billwurtz.bandcamp.com/album/high-enough"); assertThrows(ContentNotAvailableException.class, extractor::fetchPage); @@ -88,12 +97,11 @@ public void testLockedContent() throws ExtractionException, IOException { * Test playlist with just one track */ @Test - public void testSingleStreamPlaylist() throws ExtractionException, IOException { + void testSingleStreamPlaylist() throws ExtractionException, IOException { final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://zachjohnson1.bandcamp.com/album/endless"); extractor.fetchPage(); assertEquals(1, extractor.getStreamCount()); - } public static class ComingOfAge implements BasePlaylistExtractorTest { @@ -108,17 +116,17 @@ public static void setUp() throws ExtractionException, IOException { } @Test - public void testThumbnailUrl() throws ParsingException { - assertTrue(extractor.getThumbnailUrl().contains("f4.bcbits.com/img")); + public void testThumbnails() throws ParsingException { + BandcampTestUtils.testImages(extractor.getThumbnails()); } @Test - public void testBannerUrl() throws ParsingException { - assertEquals("", extractor.getBannerUrl()); + public void testBanners() throws ParsingException { + assertEmpty(extractor.getBanners()); } @Test - public void testUploaderUrl() throws ParsingException { + void testUploaderUrl() throws ParsingException { assertTrue(extractor.getUploaderUrl().contains("macbenson.bandcamp.com")); } @@ -128,8 +136,8 @@ public void testUploaderName() throws ParsingException { } @Test - public void testUploaderAvatarUrl() throws ParsingException { - assertTrue(extractor.getUploaderAvatarUrl().contains("f4.bcbits.com/img")); + public void testUploaderAvatars() throws ParsingException { + BandcampTestUtils.testImages(extractor.getUploaderAvatars()); } @Test @@ -147,13 +155,14 @@ public void testDescription() throws ParsingException { assertContains("all rights reserved", description.getContent()); // license } + @Test @Override public void testUploaderVerified() throws Exception { assertFalse(extractor.isUploaderVerified()); } @Test - public void testInitialPage() throws IOException, ExtractionException { + void testInitialPage() throws IOException, ExtractionException { assertNotNull(extractor.getInitialPage().getItems().get(0)); } @@ -183,7 +192,7 @@ public void testOriginalUrl() throws Exception { } @Test - public void testNextPageUrl() throws IOException, ExtractionException { + void testNextPageUrl() throws IOException, ExtractionException { assertNull(extractor.getPage(extractor.getInitialPage().getNextPage())); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index e00a61f3b6..920c32a7b9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -3,12 +3,14 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; +import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; +import org.schabi.newpipe.extractor.services.DefaultTests; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioStreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamType; @@ -36,7 +38,7 @@ public static void setUp() throws IOException, ExtractionException { } @Test - public void testGettingCorrectStreamExtractor() throws ExtractionException { + void testGettingCorrectStreamExtractor() throws ExtractionException { assertTrue(Bandcamp.getStreamExtractor("https://bandcamp.com/?show=3") instanceof BandcampRadioStreamExtractor); assertFalse(Bandcamp.getStreamExtractor("https://zachbenson.bandcamp.com/track/deflated") instanceof BandcampRadioStreamExtractor); @@ -57,15 +59,16 @@ public void testGettingCorrectStreamExtractor() throws ExtractionException { @Override public int expectedStreamSegmentsCount() { return 30; } @Test - public void testGetUploaderUrl() { + void testGetUploaderUrl() { assertThrows(ContentNotSupportedException.class, extractor::getUploaderUrl); } @Test @Override - public void testUploaderUrl() throws Exception { + public void testUploaderUrl() { assertThrows(ContentNotSupportedException.class, super::testUploaderUrl); } + @Override public String expectedUploaderUrl() { return null; } @Override @@ -93,16 +96,19 @@ public void testUploadDate() throws ParsingException { } @Test - public void testGetThumbnailUrl() throws ParsingException { - assertTrue(extractor.getThumbnailUrl().contains("bcbits.com/img")); + void testGetThumbnails() throws ParsingException { + BandcampTestUtils.testImages(extractor.getThumbnails()); } @Test - public void testGetUploaderAvatarUrl() throws ParsingException { - assertTrue(extractor.getUploaderAvatarUrl().contains("bandcamp-button")); + void testGetUploaderAvatars() throws ParsingException { + DefaultTests.defaultTestImageCollection(extractor.getUploaderAvatars()); + extractor.getUploaderAvatars().forEach(image -> + ExtractorAsserts.assertContains("bandcamp-button", image.getUrl())); } - @Test public void testGetAudioStreams() throws ExtractionException, IOException { + @Test + void testGetAudioStreams() throws ExtractionException, IOException { assertEquals(1, extractor.getAudioStreams().size()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java index d60178bd79..3b4052424a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.bandcamp; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; import org.junit.jupiter.api.BeforeAll; @@ -50,8 +49,7 @@ void testStreamSearch() throws ExtractionException, IOException { // The track by Zach Benson should be the first result, no? assertEquals("Best Friend's Basement", bestFriendsBasement.getName()); assertEquals("Zach Benson", bestFriendsBasement.getUploaderName()); - assertTrue(bestFriendsBasement.getThumbnailUrl().endsWith(".jpg")); - assertTrue(bestFriendsBasement.getThumbnailUrl().contains("f4.bcbits.com/img/")); + BandcampTestUtils.testImages(bestFriendsBasement.getThumbnails()); assertEquals(InfoItem.InfoType.STREAM, bestFriendsBasement.getInfoType()); } @@ -66,10 +64,8 @@ void testChannelSearch() throws ExtractionException, IOException { // C418's artist profile should be the first result, no? assertEquals("C418", c418.getName()); - assertTrue(c418.getThumbnailUrl().endsWith(".jpg")); - assertTrue(c418.getThumbnailUrl().contains("f4.bcbits.com/img/")); + BandcampTestUtils.testImages(c418.getThumbnails()); assertEquals("https://c418.bandcamp.com", c418.getUrl()); - } /** @@ -82,9 +78,9 @@ void testAlbumSearch() throws ExtractionException, IOException { // Minecraft volume alpha should be the first result, no? assertEquals("Minecraft - Volume Alpha", minecraft.getName()); - assertTrue(minecraft.getThumbnailUrl().endsWith(".jpg")); - assertTrue(minecraft.getThumbnailUrl().contains("f4.bcbits.com/img/")); - assertEquals("https://c418.bandcamp.com/album/minecraft-volume-alpha", + BandcampTestUtils.testImages(minecraft.getThumbnails()); + assertEquals( + "https://c418.bandcamp.com/album/minecraft-volume-alpha", minecraft.getUrl()); // Verify that playlist tracks counts get extracted correctly diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index 3a582db287..38bb44f3c0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -20,7 +20,6 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; /** @@ -150,13 +149,12 @@ public String expectedCategory() { } @Test - public void testArtistProfilePicture() throws Exception { - final String url = extractor().getUploaderAvatarUrl(); - assertTrue(url.contains("://f4.bcbits.com/img/") && url.endsWith(".jpg")); + void testArtistProfilePictures() { + BandcampTestUtils.testImages(extractor.getUploaderAvatars()); } @Test - public void testTranslateIdsToUrl() throws ParsingException { + void testTranslateIdsToUrl() throws ParsingException { // To add tests: look at website's source, search for `band_id` and `item_id` assertEquals( "https://teaganbear.bandcamp.com/track/just-for-the-halibut-creative-commons-attribution", From e8bfd20170581d8ec4722792acca75f37bafcec3 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Wed, 3 Aug 2022 18:09:52 +0200 Subject: [PATCH 35/35] [MediaCCC] Apply changes in extractor tests Also remove some public test methods modifiers. --- .../MediaCCCConferenceExtractorTest.java | 29 +++++++++------- .../MediaCCCStreamExtractorTest.java | 33 ++++++++++++------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java index 976af4e422..86561c971b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java @@ -9,6 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContainsImageUrlInImageCollection; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** @@ -30,27 +31,29 @@ public static void setUpClass() throws Exception { } @Test - public void testName() throws Exception { + void testName() throws Exception { assertEquals("FrOSCon 2017", extractor.getName()); } @Test - public void testGetUrl() throws Exception { + void testGetUrl() throws Exception { assertEquals("https://media.ccc.de/c/froscon2017", extractor.getUrl()); } @Test - public void testGetOriginalUrl() throws Exception { + void testGetOriginalUrl() throws Exception { assertEquals("https://media.ccc.de/c/froscon2017", extractor.getOriginalUrl()); } @Test - public void testGetThumbnailUrl() throws Exception { - assertEquals("https://static.media.ccc.de/media/events/froscon/2017/logo.png", extractor.getAvatarUrl()); + void testGetThumbnails() { + assertContainsImageUrlInImageCollection( + "https://static.media.ccc.de/media/events/froscon/2017/logo.png", + extractor.getAvatars()); } @Test - public void testGetInitalPage() throws Exception { + void testGetInitalPage() throws Exception { assertEquals(97, tabExtractor.getInitialPage().getItems().size()); } } @@ -70,27 +73,29 @@ public static void setUpClass() throws Exception { } @Test - public void testName() throws Exception { + void testName() throws Exception { assertEquals("Open Source Conference Albania 2019", extractor.getName()); } @Test - public void testGetUrl() throws Exception { + void testGetUrl() throws Exception { assertEquals("https://media.ccc.de/c/oscal19", extractor.getUrl()); } @Test - public void testGetOriginalUrl() throws Exception { + void testGetOriginalUrl() throws Exception { assertEquals("https://media.ccc.de/c/oscal19", extractor.getOriginalUrl()); } @Test - public void testGetThumbnailUrl() throws Exception { - assertEquals("https://static.media.ccc.de/media/events/oscal/2019/oscal-19.png", extractor.getAvatarUrl()); + void testGetThumbnailUrl() { + assertContainsImageUrlInImageCollection( + "https://static.media.ccc.de/media/events/oscal/2019/oscal-19.png", + extractor.getAvatars()); } @Test - public void testGetInitalPage() throws Exception { + void testGetInitalPage() throws Exception { assertTrue(tabExtractor.getInitialPage().getItems().size() >= 21); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index feb0b7c723..b68ce9ff44 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContainsImageUrlInImageCollection; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** @@ -66,16 +67,20 @@ public static void setUp() throws Exception { @Override @Test - public void testThumbnailUrl() throws Exception { - super.testThumbnailUrl(); - assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl()); + public void testThumbnails() throws Exception { + super.testThumbnails(); + assertContainsImageUrlInImageCollection( + "https://static.media.ccc.de/media/events/gpn/gpn18/105-hd_preview.jpg", + extractor.getThumbnails()); } @Override @Test - public void testUploaderAvatarUrl() throws Exception { - super.testUploaderAvatarUrl(); - assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl()); + public void testUploaderAvatars() throws Exception { + super.testUploaderAvatars(); + assertContainsImageUrlInImageCollection( + "https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", + extractor.getUploaderAvatars()); } @Override @@ -140,16 +145,20 @@ public static void setUp() throws Exception { @Override @Test - public void testThumbnailUrl() throws Exception { - super.testThumbnailUrl(); - assertEquals("https://static.media.ccc.de/media/congress/2019/10565-hd.jpg", extractor.getThumbnailUrl()); + public void testThumbnails() throws Exception { + super.testThumbnails(); + assertContainsImageUrlInImageCollection( + "https://static.media.ccc.de/media/congress/2019/10565-hd_preview.jpg", + extractor.getThumbnails()); } @Override @Test - public void testUploaderAvatarUrl() throws Exception { - super.testUploaderAvatarUrl(); - assertEquals("https://static.media.ccc.de/media/congress/2019/logo.png", extractor.getUploaderAvatarUrl()); + public void testUploaderAvatars() throws Exception { + super.testUploaderAvatars(); + assertContainsImageUrlInImageCollection( + "https://static.media.ccc.de/media/congress/2019/logo.png", + extractor.getUploaderAvatars()); } @Override