Merge branch 'dev_newpipe_extractor' into dev

# Conflicts: # .github/workflows/docs.yml
MaintainTeam · Jul 13, 2024 · 4faa566 · 4faa566
2 parents 062f70d + 592f159
commit 4faa566
Show file tree

Hide file tree

Showing 9 changed files with 98 additions and 47 deletions.
diff --git a/build.gradle b/build.gradle
@@ -8,7 +8,7 @@ allprojects {
     sourceCompatibility = JavaVersion.VERSION_11
     targetCompatibility = JavaVersion.VERSION_11
 
-    version 'v0.23.1'
+    version 'v0.24.0'
     group 'com.github.TeamNewPipe'
 
     repositories {
@@ -28,8 +28,8 @@ allprojects {
 
     ext {
         nanojsonVersion = "1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751"
-        spotbugsVersion = "4.8.3"
-        junitVersion = "5.10.2"
+        spotbugsVersion = "4.8.6"
+        junitVersion = "5.10.3"
         checkstyleVersion = "10.4"
     }
 }

diff --git a/extractor/build.gradle b/extractor/build.gradle
@@ -31,7 +31,7 @@ dependencies {
 
     // do not upgrade to 1.7.14, since in 1.7.14 Rhino uses the `SourceVersion` class, which is not
     // available on Android (even when using desugaring), and `NoClassDefFoundError` is thrown
-    implementation 'org.mozilla:rhino:1.7.13'
+    implementation 'org.mozilla:rhino:1.7.15'
 
     checkstyle "com.puppycrawl.tools:checkstyle:$checkstyleVersion"
 
@@ -41,5 +41,5 @@ dependencies {
     testImplementation 'org.junit.jupiter:junit-jupiter-params'
 
     testImplementation "com.squareup.okhttp3:okhttp:3.12.13"
-    testImplementation 'com.google.code.gson:gson:2.10.1'
+    testImplementation 'com.google.code.gson:gson:2.11.0'
 }
diff --git a/...or/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSignatureUtils.java b/...or/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSignatureUtils.java
@@ -1,5 +1,7 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import static org.schabi.newpipe.extractor.utils.Parser.matchGroup1MultiplePatterns;
+
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.JavaScript;
 import org.schabi.newpipe.extractor.utils.Parser;
@@ -20,13 +22,13 @@ final class YoutubeSignatureUtils {
      */
     static final String DEOBFUSCATION_FUNCTION_NAME = "deobfuscate";
 
-    private static final String[] FUNCTION_REGEXES = {
-            "\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)",
-            "\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)",
+    private static final Pattern[] FUNCTION_REGEXES = {
             // CHECKSTYLE:OFF
-            "(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)",
+            Pattern.compile("\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)"),
+            Pattern.compile("\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)"),
+            Pattern.compile("(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)"),
+            Pattern.compile("([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;")
             // CHECKSTYLE:ON
-            "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"
     };
 
     private static final String STS_REGEX = "signatureTimestamp[=:](\\d+)";
@@ -104,19 +106,12 @@ static String getDeobfuscationCode(@Nonnull final String javaScriptPlayerCode)
     @Nonnull
     private static String getDeobfuscationFunctionName(@Nonnull final String javaScriptPlayerCode)
             throws ParsingException {
-        Parser.RegexException exception = null;
-        for (final String regex : FUNCTION_REGEXES) {
-            try {
-                return Parser.matchGroup1(regex, javaScriptPlayerCode);
-            } catch (final Parser.RegexException e) {
-                if (exception == null) {
-                    exception = e;
-                }
-            }
+        try {
+            return matchGroup1MultiplePatterns(FUNCTION_REGEXES, javaScriptPlayerCode);
+        } catch (final Parser.RegexException e) {
+            throw new ParsingException(
+                    "Could not find deobfuscation function with any of the known patterns", e);
         }
-
-        throw new ParsingException(
-                "Could not find deobfuscation function with any of the known patterns", exception);
     }
 
     @Nonnull

diff --git a/...n/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java b/...n/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java
@@ -1,5 +1,7 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import static org.schabi.newpipe.extractor.utils.Parser.matchMultiplePatterns;
+
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.JavaScript;
 import org.schabi.newpipe.extractor.utils.Parser;
@@ -18,10 +20,33 @@ final class YoutubeThrottlingParameterUtils {
 
     private static final Pattern THROTTLING_PARAM_PATTERN = Pattern.compile("[&?]n=([^&]+)");
 
-    private static final Pattern DEOBFUSCATION_FUNCTION_NAME_PATTERN = Pattern.compile(
-            // CHECKSTYLE:OFF
-            "\\.get\\(\"n\"\\)\\)&&\\([a-zA-Z0-9$_]=([a-zA-Z0-9$_]+)(?:\\[(\\d+)])?\\([a-zA-Z0-9$_]\\)");
-            // CHECKSTYLE:ON
+    private static final String SINGLE_CHAR_VARIABLE_REGEX = "[a-zA-Z0-9$_]";
+
+    private static final String FUNCTION_NAME_REGEX = SINGLE_CHAR_VARIABLE_REGEX + "+";
+
+    private static final String ARRAY_ACCESS_REGEX = "\\[(\\d+)]";
+
+    /**
+     * The first regex matches this, where we want BDa:
+     * <p>
+     * (b=String.fromCharCode(110),c=a.get(b))&&(c=<strong>BDa</strong><strong>[0]</strong>(c)
+     * <p>
+     * Array access is optional, but needs to be handled, since the actual function is inside the
+     * array.
+     */
+    // CHECKSTYLE:OFF
+    private static final Pattern[] DEOBFUSCATION_FUNCTION_NAME_REGEXES = {
+            Pattern.compile("\\(" + SINGLE_CHAR_VARIABLE_REGEX + "=String\\.fromCharCode\\(110\\),"
+                    + SINGLE_CHAR_VARIABLE_REGEX + "=" + SINGLE_CHAR_VARIABLE_REGEX + "\\.get\\("
+                    + SINGLE_CHAR_VARIABLE_REGEX + "\\)\\)" + "&&\\(" + SINGLE_CHAR_VARIABLE_REGEX
+                    + "=(" + FUNCTION_NAME_REGEX + ")" + "(?:" + ARRAY_ACCESS_REGEX + ")?\\("
+                    + SINGLE_CHAR_VARIABLE_REGEX + "\\)"),
+            Pattern.compile("\\.get\\(\"n\"\\)\\)&&\\(" + SINGLE_CHAR_VARIABLE_REGEX
+                    + "=(" + FUNCTION_NAME_REGEX + ")(?:" + ARRAY_ACCESS_REGEX + ")?\\("
+                    + SINGLE_CHAR_VARIABLE_REGEX + "\\)"),
+    };
+    // CHECKSTYLE:ON
+
 
     // Escape the curly end brace to allow compatibility with Android's regex engine
     // See https://stackoverflow.com/q/45074813
@@ -48,11 +73,13 @@ private YoutubeThrottlingParameterUtils() {
     @Nonnull
     static String getDeobfuscationFunctionName(@Nonnull final String javaScriptPlayerCode)
             throws ParsingException {
-        final Matcher matcher = DEOBFUSCATION_FUNCTION_NAME_PATTERN.matcher(javaScriptPlayerCode);
-        if (!matcher.find()) {
-            throw new ParsingException("Failed to find deobfuscation function name pattern \""
-                    + DEOBFUSCATION_FUNCTION_NAME_PATTERN
-                    + "\" in the base JavaScript player code");
+        final Matcher matcher;
+        try {
+            matcher = matchMultiplePatterns(DEOBFUSCATION_FUNCTION_NAME_REGEXES,
+                    javaScriptPlayerCode);
+        } catch (final Parser.RegexException e) {
+            throw new ParsingException("Could not find deobfuscation function with any of the "
+                    + "known patterns in the base JavaScript player code", e);
         }
 
         final String functionName = matcher.group(1);

diff --git a/...or/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java b/...or/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java
@@ -75,19 +75,17 @@ private YoutubePostLiveStreamDvrDashManifestCreator() {
      *     parsed from the first sequence of the stream.</li>
      * </ul>
      *
-     *
-     * <p>In order to generate the DASH manifest, this method will:
-     *      <ul>
-     *          <li>request the first sequence of the stream (the base URL on which the first
-     *          sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
-     *          with a {@code POST} or {@code GET} request (depending of the client on which the
-     *          streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));</li>
-     *          <li>follow its redirection(s), if any;</li>
-     *          <li>save the last URL, remove the first sequence parameters;</li>
-     *          <li>use the information provided in the {@link ItagItem} to generate all elements
-     *          of the DASH manifest.</li>
-     *      </ul>
-     * </p>
+     * In order to generate the DASH manifest, this method will:
+     * <ul>
+     *     <li>request the first sequence of the stream (the base URL on which the first
+     *     sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
+     *     with a {@code POST} or {@code GET} request (depending of the client on which the
+     *     streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));</li>
+     *     <li>follow its redirection(s), if any;</li>
+     *     <li>save the last URL, remove the first sequence parameters;</li>
+     *     <li>use the information provided in the {@link ItagItem} to generate all elements
+     *     of the DASH manifest.</li>
+     * </ul>
      *
      * <p>
      * If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used

diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java
@@ -78,6 +78,37 @@ public static String matchGroup(@Nonnull final Pattern pat,
         }
     }
 
+    public static String matchGroup1MultiplePatterns(final Pattern[] patterns, final String input)
+            throws RegexException {
+        return matchMultiplePatterns(patterns, input).group(1);
+    }
+
+    public static Matcher matchMultiplePatterns(final Pattern[] patterns, final String input)
+            throws RegexException {
+        Parser.RegexException exception = null;
+        for (final Pattern pattern : patterns) {
+            final Matcher matcher = pattern.matcher(input);
+            if (matcher.find()) {
+                return matcher;
+            } else if (exception == null) {
+                // only pass input to exception message when it is not too long
+                if (input.length() > 1024) {
+                    exception = new RegexException("Failed to find pattern \"" + pattern.pattern()
+                            + "\"");
+                } else {
+                    exception = new RegexException("Failed to find pattern \"" + pattern.pattern()
+                            + "\" inside of \"" + input + "\"");
+                }
+            }
+        }
+
+        if (exception == null) {
+            throw new RegexException("Empty patterns array passed to matchMultiplePatterns");
+        } else {
+            throw exception;
+        }
+    }
+
     public static boolean isMatch(final String pattern, final String input) {
         final Pattern pat = Pattern.compile(pattern);
         final Matcher mat = pat.matcher(input);

diff --git a/...st/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/...st/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java
@@ -48,7 +48,7 @@ void testGetComments() throws IOException, ExtractionException {
 
         @Test
         void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
-            final String comment = "Thanks for this nice video explanation of Peertube!";
+            final String comment = "I love this. ❤";
 
             final CommentsInfo commentsInfo =
                     CommentsInfo.getInfo("https://framatube.org/w/kkGMgK9ZtnKfYAgnEtQxbv");

diff --git a/...test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java b/...test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java
@@ -85,7 +85,7 @@ public void testGetLanguageInformation() throws ParsingException {
         @Override public long expectedViewCountAtLeast() { return 38600; }
         @Nullable @Override public String expectedUploadDate() { return "2018-10-01 10:52:46.396"; }
         @Nullable @Override public String expectedTextualUploadDate() { return "2018-10-01T10:52:46.396Z"; }
-        @Override public long expectedLikeCountAtLeast() { return 19; }
+        @Override public long expectedLikeCountAtLeast() { return 18; }
         @Override public long expectedDislikeCountAtLeast() { return 0; }
         @Override public String expectedHost() { return "framatube.org"; }
         @Override public String expectedCategory() { return "Science & Technology"; }

diff --git a/...ava/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/...ava/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
@@ -317,7 +317,7 @@ void testUploaderUrl() {
 
         @Test
         public void testUploaderName() {
-            assertEquals("user350509423", extractor.getUploaderName());
+            assertEquals("Chaazyy", extractor.getUploaderName());
         }
 
         @Test