Improve tests and randomness

- Use the existing RNG inside YoutubeParsingHelper
- Deduplicated test-setup for YouTube tests
- Minor improvements

Author: litetex

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -1,3 +1,23 @@
+/*
+ * Created by Christian Schabesberger on 02.03.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * YoutubeParsingHelper.java is part of NewPipe Extractor.
+ *
+ * NewPipe Extractor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe Extractor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package org.schabi.newpipe.extractor.services.youtube;
 
 import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
@@ -7,7 +27,6 @@
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-import static org.schabi.newpipe.extractor.utils.Utils.randomStringFromAlphabet;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonBuilder;
@@ -29,6 +48,7 @@
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
+import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
@@ -37,6 +57,7 @@
 import java.net.URL;
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
+import java.security.SecureRandom;
 import java.time.LocalDate;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
@@ -54,26 +75,6 @@
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
-/*
- * Created by Christian Schabesberger on 02.03.16.
- *
- * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
- * YoutubeParsingHelper.java is part of NewPipe Extractor.
- *
- * NewPipe Extractor is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * NewPipe Extractor is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
- */
-
 public final class YoutubeParsingHelper {
 
     private YoutubeParsingHelper() {
@@ -83,11 +84,6 @@ private YoutubeParsingHelper() {
     public static final String CPN = "cpn";
     public static final String VIDEO_ID = "videoId";
 
-    /**
-     * Seed that will be used for video tests, in order to mock video requests.
-     */
-    private static final long SEED_FOR_VIDEOS_TESTS = 3000;
-
     private static final String HARDCODED_CLIENT_VERSION = "2.20220114.01.00";
     private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
 
@@ -122,7 +118,7 @@ private YoutubeParsingHelper() {
     private static final String CONTENT_PLAYBACK_NONCE_ALPHABET =
             "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
 
-    private static Random numberGenerator = new Random();
+    private static Random numberGenerator = new SecureRandom();
 
     /**
      * {@code PENDING+} means that the user did not yet submit their choices.
@@ -606,12 +602,13 @@ public static String getClientVersion() throws IOException, ExtractionException
 
         if (keyAndVersionExtracted) {
             return clientVersion;
-        } else {
-            if (areHardcodedClientVersionAndKeyValid()) {
-                clientVersion = HARDCODED_CLIENT_VERSION;
-                return clientVersion;
-            }
         }
+
+        if (areHardcodedClientVersionAndKeyValid()) {
+            clientVersion = HARDCODED_CLIENT_VERSION;
+            return clientVersion;
+        }
+
         throw new ExtractionException("Could not get YouTube WEB client version");
     }
 
@@ -631,11 +628,11 @@ public static String getKey() throws IOException, ExtractionException {
 
         if (keyAndVersionExtracted) {
             return key;
-        } else {
-            if (areHardcodedClientVersionAndKeyValid()) {
-                key = HARDCODED_KEY;
-                return key;
-            }
+        }
+
+        if (areHardcodedClientVersionAndKeyValid()) {
+            key = HARDCODED_KEY;
+            return key;
         }
 
         // The ANDROID API key is also valid with the WEB client so return it if we couldn't
@@ -1508,7 +1505,8 @@ public static String unescapeDocument(@Nonnull final String doc) {
      */
     @Nonnull
     public static String generateContentPlaybackNonce() {
-        return randomStringFromAlphabet(CONTENT_PLAYBACK_NONCE_ALPHABET, 16);
+        return RandomStringFromAlphabetGenerator.generate(
+                CONTENT_PLAYBACK_NONCE_ALPHABET, 16, numberGenerator);
     }
 
     /**
@@ -1524,23 +1522,7 @@ public static String generateContentPlaybackNonce() {
      */
     @Nonnull
     public static String generateTParameter() {
-        return randomStringFromAlphabet(CONTENT_PLAYBACK_NONCE_ALPHABET, 12);
-    }
-
-    /**
-     * Set the seed for video tests.
-     *
-     * <p>
-     * This seed will be used to generate the same {@code t} and {@code cpn} values between
-     * different execution of tests so mocks can be used for stream tests.
-     * </p>
-     *
-     * <p>
-     * This method will call {@link Utils#setSecureRandomSeed(long)} with the
-     * {@link #SEED_FOR_VIDEOS_TESTS value}.
-     * </p>
-     */
-    public static void setSeedForVideoTests() {
-        Utils.setSecureRandomSeed(SEED_FOR_VIDEOS_TESTS);
+        return RandomStringFromAlphabetGenerator.generate(
+                CONTENT_PLAYBACK_NONCE_ALPHABET, 12, numberGenerator);
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/RandomStringFromAlphabetGenerator.java

@@ -0,0 +1,35 @@
+package org.schabi.newpipe.extractor.utils;
+
+import java.util.Random;
+
+import javax.annotation.Nonnull;
+
+/**
+ * Generates a random string from a predefined alphabet.
+ */
+public final class RandomStringFromAlphabetGenerator {
+    private RandomStringFromAlphabetGenerator() {
+        // No impl
+    }
+
+    /**
+     * Generate a random string from an alphabet.
+     *
+     * @param alphabet the characters' alphabet to use
+     * @param length   the length of the returned string (> 0)
+     * @param random   {@link Random} (or better {@link java.security.SecureRandom}) used for generating the random string
+     * @return a random string of the requested length made of only characters from the provided
+     * alphabet
+     */
+    @Nonnull
+    public static String generate(
+            final String alphabet,
+            final int length,
+            final Random random) {
+        final StringBuilder stringBuilder = new StringBuilder(length);
+        for (int i = 0; i < length; i++) {
+            stringBuilder.append(alphabet.charAt(random.nextInt(alphabet.length())));
+        }
+        return stringBuilder.toString();
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -8,14 +8,13 @@
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.LinkedList;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.security.SecureRandom;
 import java.util.regex.Pattern;
 
 public final class Utils {
@@ -26,7 +25,6 @@ public final class Utils {
     public static final String EMPTY_STRING = "";
     private static final Pattern M_PATTERN = Pattern.compile("(https?)?:\\/\\/m\\.");
     private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\.");
-    private static final SecureRandom random = new SecureRandom();
 
     private Utils() {
         // no instance
@@ -75,8 +73,9 @@ public static long mixedNumberWordToLong(final String numberWord)
             multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2);
         } catch (final ParsingException ignored) {
         }
-        final double count = Double.parseDouble(
-                Parser.matchGroup1("([\\d]+([\\.,][\\d]+)?)", numberWord).replace(",", "."));
+
+        final double count = Double.parseDouble(Parser.matchGroup1("([\\d]+([\\.,][\\d]+)?)",
+                numberWord).replace(",", "."));
         switch (multiplier.toUpperCase()) {
             case "K":
                 return (long) (count * 1e3);
@@ -443,43 +442,4 @@ public static String getStringResultFromRegexArray(@Nonnull final String input,
         }
         return result;
     }
-
-    /**
-     * Generate a random string using the secure random device {@link #random}.
-     *
-     * <p>
-     * {@link #setSecureRandomSeed(long)} might be useful when mocking tests.
-     * </p>
-     *
-     * @param alphabet the characters' alphabet to use
-     * @param length   the length of the returned string
-     * @return a random string of the requested length made of only characters from the provided
-     * alphabet
-     */
-    @Nonnull
-    public static String randomStringFromAlphabet(final String alphabet, final int length) {
-        final StringBuilder stringBuilder = new StringBuilder();
-        for (int i = 0; i < length; ++i) {
-            stringBuilder.append(alphabet.charAt(random.nextInt(alphabet.length())));
-        }
-        return stringBuilder.toString();
-    }
-
-    /**
-     * Seed the secure random device used for {@link #randomStringFromAlphabet(String, int)}.
-     *
-     * <p>
-     * Use this in tests so that they can be mocked as the same random numbers are always
-     * generated.
-     * </p>
-     *
-     * <p>
-     * This is not intended to be used outside of tests.
-     * </p>
-     *
-     * @param seed the seed to pass to {@link SecureRandom#setSeed(long)}
-     */
-    public static void setSecureRandomSeed(final long seed) {
-        random.setSeed(seed);
-    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -1,5 +1,17 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
@@ -15,13 +27,6 @@
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
 
 import java.io.IOException;
-import java.util.Random;
-
-import static org.junit.jupiter.api.Assertions.*;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.DefaultTests.*;
 
 /**
  * Test for {@link ChannelExtractor}
@@ -33,8 +38,7 @@ public class YoutubeChannelExtractorTest {
     public static class NotAvailable {
         @BeforeAll
         public static void setUp() throws IOException {
-            YoutubeParsingHelper.resetClientVersionAndKey();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            YoutubeTestsUtils.ensureStateless();
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "notAvailable"));
         }
 
@@ -130,8 +134,7 @@ public void accountTerminatedCopyrightFetch() throws Exception {
     public static class NotSupported {
         @BeforeAll
         public static void setUp() throws IOException {
-            YoutubeParsingHelper.resetClientVersionAndKey();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            YoutubeTestsUtils.ensureStateless();
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "notSupported"));
         }
 
@@ -149,8 +152,7 @@ public static class Gronkh implements BaseChannelExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            YoutubeTestsUtils.ensureStateless();
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "gronkh"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("http://www.youtube.com/user/Gronkh");
@@ -246,8 +248,7 @@ public static class VSauce implements BaseChannelExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            YoutubeTestsUtils.ensureStateless();
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "VSauce"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/user/Vsauce");
@@ -342,8 +343,7 @@ public static class Kurzgesagt implements BaseChannelExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            YoutubeTestsUtils.ensureStateless();
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "kurzgesagt"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q");
@@ -460,8 +460,7 @@ public static class CaptainDisillusion implements BaseChannelExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            YoutubeTestsUtils.ensureStateless();
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "captainDisillusion"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/user/CaptainDisillusion/videos");
@@ -555,8 +554,7 @@ public static class RandomChannel implements BaseChannelExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            YoutubeTestsUtils.ensureStateless();
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "random"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w");