[YouTube] Remove usage of API keys for InnerTube requests, bump versions

The API keys are not used anymore by official clients in almost all cases
(still used by the Android app until it gets a configuration) for all requests
we made.

Clients and device OS versions have been bumped to their latest stable version
known.

Methods and fields related to API keys have been renamed or deleted if they're
no longer relevant.

Author: AudricV

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -29,7 +29,6 @@
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@@ -422,8 +421,8 @@ private Page getNextPageFrom(final JsonObject continuations,
                         .done())
                 .getBytes(StandardCharsets.UTF_8);
 
-        return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
-                + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
+        return new Page(YOUTUBEI_V1_URL + "browse?" + DISABLE_PRETTY_PRINT_PARAMETER, null,
+                channelIds, null, body);
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java

@@ -4,7 +4,6 @@
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
@@ -103,8 +102,8 @@ public void onFetchPage(@Nonnull final Downloader downloader)
         final var headers = getYouTubeHeaders();
 
         final Response response = getDownloader().postWithContentTypeJson(
-                YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
-                headers, body, localization);
+                YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, headers, body,
+                localization);
 
         initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
         playlistData = initialData
@@ -225,8 +224,8 @@ private Page getNextPageFrom(@Nonnull final JsonObject playlistJson,
                 .done())
                 .getBytes(StandardCharsets.UTF_8);
 
-        return new Page(YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
-                null, null, cookies, body);
+        return new Page(YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null,
+                cookies, body);
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java

@@ -3,6 +3,7 @@
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicClientVersion;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicHeaders;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
@@ -25,10 +26,8 @@
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
-import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 
 import java.io.IOException;
@@ -52,10 +51,8 @@ public YoutubeMusicSearchExtractor(final StreamingService service,
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader)
             throws IOException, ExtractionException {
-        final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
-
-        final String url = "https://music.youtube.com/youtubei/v1/search?key="
-                + youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
+        final String url = "https://music.youtube.com/youtubei/v1/search?"
+                + DISABLE_PRETTY_PRINT_PARAMETER;
 
         final String params;
 
@@ -86,7 +83,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
                 .object("context")
                     .object("client")
                         .value("clientName", "WEB_REMIX")
-                        .value("clientVersion", youtubeMusicKeys[2])
+                        .value("clientVersion", getYoutubeMusicClientVersion())
                         .value("hl", "en-GB")
                         .value("gl", getExtractorContentCountry().getCountryCode())
                         .value("platform", "DESKTOP")
@@ -206,15 +203,13 @@ public InfoItemsPage<InfoItem> getPage(final Page page)
 
         final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
 
-        final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
-
         // @formatter:off
         final byte[] json = JsonWriter.string()
             .object()
                 .object("context")
                     .object("client")
                         .value("clientName", "WEB_REMIX")
-                        .value("clientVersion", youtubeMusicKeys[2])
+                        .value("clientVersion", getYoutubeMusicClientVersion())
                         .value("hl", "en-GB")
                         .value("gl", getExtractorContentCountry().getCountryCode())
                         .value("platform", "DESKTOP")
@@ -295,8 +290,7 @@ private void collectMusicStreamsFrom(final MultiInfoItemsCollector collector,
     }
 
     @Nullable
-    private Page getNextPageFrom(final JsonArray continuations)
-            throws IOException, ParsingException, ReCaptchaException {
+    private Page getNextPageFrom(final JsonArray continuations) {
         if (isNullOrEmpty(continuations)) {
             return null;
         }
@@ -306,7 +300,6 @@ private Page getNextPageFrom(final JsonArray continuations)
         final String continuation = nextContinuationData.getString("continuation");
 
         return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation
-                + "&continuation=" + continuation + "&key="
-                + YoutubeParsingHelper.getYoutubeMusicKey()[0] + DISABLE_PRETTY_PRINT_PARAMETER);
+                + "&continuation=" + continuation + "&" + DISABLE_PRETTY_PRINT_PARAMETER);
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java

@@ -4,7 +4,6 @@
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
@@ -387,8 +386,7 @@ private Page getNextPageFrom(final JsonArray contents)
                             .done())
                     .getBytes(StandardCharsets.UTF_8);
 
-            return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
-                    + DISABLE_PRETTY_PRINT_PARAMETER, body);
+            return new Page(YOUTUBEI_V1_URL + "browse?" + DISABLE_PRETTY_PRINT_PARAMETER, body);
         } else {
             return null;
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -3,7 +3,6 @@
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.ALL;
@@ -247,8 +246,7 @@ private void collectStreamsFrom(final MultiInfoItemsCollector collector,
     }
 
     @Nullable
-    private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws IOException,
-            ExtractionException {
+    private Page getNextPageFrom(final JsonObject continuationItemRenderer) {
         if (isNullOrEmpty(continuationItemRenderer)) {
             return null;
         }
@@ -257,8 +255,7 @@ private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws I
                 .getObject("continuationCommand")
                 .getString("token");
 
-        final String url = YOUTUBEI_V1_URL + "search?key=" + getKey()
-                + DISABLE_PRETTY_PRINT_PARAMETER;
+        final String url = YOUTUBEI_V1_URL + "search?" + DISABLE_PRETTY_PRINT_PARAMETER;
 
         return new Page(url, token);
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -8,7 +8,6 @@
 import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeOtfDashManifestCreator;
 import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeProgressiveDashManifestCreator;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
-import org.schabi.newpipe.extractor.stream.AudioTrackType;
 import org.schabi.newpipe.extractor.stream.DeliveryMethod;
 import org.schabi.newpipe.extractor.stream.Stream;
 import org.w3c.dom.Document;
@@ -89,7 +88,7 @@ class YoutubeDashManifestCreatorsTest {
 
     @BeforeAll
     public static void setUp() throws Exception {
-        YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeParsingHelper.resetClientVersion();
         YoutubeParsingHelper.setNumberGenerator(new Random(1));
         NewPipe.init(DownloaderTestImpl.getInstance());
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java

@@ -7,7 +7,6 @@
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
 
 import com.grack.nanojson.JsonWriter;
@@ -91,7 +90,7 @@ void getPage() throws Exception {
                     .getBytes(StandardCharsets.UTF_8);
 
             final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
-                    YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+                    YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER,
                     null, null, dummyCookie, body));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
@@ -180,7 +179,7 @@ void getPage() throws Exception {
                     .getBytes(StandardCharsets.UTF_8);
 
             final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
-                    YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+                    YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER,
                     null, null, dummyCookie, body));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
@@ -268,7 +267,7 @@ void getPage() throws Exception {
                     .getBytes(StandardCharsets.UTF_8);
 
             final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
-                    YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+                    YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER,
                     null, null, dummyCookie, body));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
@@ -386,7 +385,7 @@ void getPage() throws Exception {
                     .getBytes(StandardCharsets.UTF_8);
 
             final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
-                    YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+                    YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER,
                     null, null, dummyCookie, body));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
@@ -453,7 +452,7 @@ void getPage() throws Exception {
                     .getBytes(StandardCharsets.UTF_8);
 
             final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
-                    YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+                    YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER,
                     null, null, dummyCookie, body));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
@@ -542,7 +541,7 @@ void getPage() throws Exception {
                     .getBytes(StandardCharsets.UTF_8);
 
             final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
-                    YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+                    YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER,
                     null, null, dummyCookie, body));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java

@@ -25,15 +25,15 @@ public static void setUp() throws IOException {
     }
 
     @Test
-    void testAreHardcodedClientVersionAndKeyValid() throws IOException, ExtractionException {
-        assertTrue(YoutubeParsingHelper.areHardcodedClientVersionAndKeyValid(),
-                "Hardcoded client version and key are not valid anymore");
+    void testIsHardcodedClientVersionValid() throws IOException, ExtractionException {
+        assertTrue(YoutubeParsingHelper.isHardcodedClientVersionValid(),
+                "Hardcoded client version is not valid anymore");
     }
 
     @Test
-    void testAreHardcodedYoutubeMusicKeysValid() throws IOException, ExtractionException {
-        assertTrue(YoutubeParsingHelper.isHardcodedYoutubeMusicKeyValid(),
-                "Hardcoded YouTube Music keys are not valid anymore");
+    void testIsHardcodedYoutubeMusicClientVersionValid() throws IOException, ExtractionException {
+        assertTrue(YoutubeParsingHelper.isHardcodedYoutubeMusicClientVersionValid(),
+                "Hardcoded YouTube Music client version is not valid anymore");
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java

@@ -26,7 +26,7 @@ private YoutubeTestsUtils() {
      */
     public static void ensureStateless() {
         YoutubeParsingHelper.setConsentAccepted(false);
-        YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeParsingHelper.resetClientVersion();
         YoutubeParsingHelper.setNumberGenerator(new Random(1));
         YoutubeJavaScriptPlayerManager.clearAllCaches();
     }