[YouTube] Add the cpn param to playback requests and try to spoof better the Android client

The cpn param, aka the content playback nonce param, is a parameter sent by YouTube web client in videoplayback requests, and for some of them, in the player request body. This PR adds it everywhere.

For the desktop/WEB client, some params were missing from the playbackContext object, which seemed (or not) to make YouTube throttle streams extracted from the WEB client. This PR adds them.

Fingerprinting on the WEB client basing on the client version used is not possible anymore, because the latest client version is extracted at the first time of a YouTube request on a session which require the extractor to fetch again the website (and this may come back the reCaptcha issues again unfortunately, but it seems there is no other way to get it).

For the Android client, the video id is now also sent as a query parameter, like a 12 characters string, in the t query parameter, in order to spoof better this client. Researches need to be done on this parameter, unique to each request, and how it is generated by clients.

This commit also fixes a small bug with the Android User-Agent string.

Some code improvements have been also made.

Author: AudricV

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -15,7 +15,6 @@
 import com.grack.nanojson.JsonWriter;
 
 import org.schabi.newpipe.extractor.MetaInfo;
-import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@@ -35,6 +34,8 @@
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.security.SecureRandom;
 import java.time.LocalDate;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
@@ -78,11 +79,15 @@ private YoutubeParsingHelper() {
     }
 
     public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
+    public static final String CPN = "cpn";
+    public static final String VIDEO_ID = "videoId";
 
     private static final String HARDCODED_CLIENT_VERSION = "2.20220107.00.00";
     private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
-    private static final String MOBILE_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
+
+    private static final String ANDROID_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
     private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.49.37";
+
     private static String clientVersion;
     private static String key;
 
@@ -94,6 +99,9 @@ private YoutubeParsingHelper() {
     @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
     private static Optional<Boolean> hardcodedClientVersionAndKeyValid = Optional.empty();
 
+    private static final String CONTENT_PLAYBACK_NONCE_ALPHABET =
+            "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
+
     private static Random numberGenerator = new Random();
 
     /**
@@ -593,7 +601,7 @@ public static String getKey() throws IOException, ExtractionException {
 
         // The ANDROID API key is also valid with the WEB client so return it if we couldn't
         // extract the WEB API key.
-        return MOBILE_YOUTUBE_KEY;
+        return ANDROID_YOUTUBE_KEY;
     }
 
     /**
@@ -769,7 +777,7 @@ public static String getUrlFromNavigationEndpoint(@Nonnull final JsonObject navi
         } else if (navigationEndpoint.has("watchEndpoint")) {
             final StringBuilder url = new StringBuilder();
             url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint
-                    .getObject("watchEndpoint").getString("videoId"));
+                    .getObject("watchEndpoint").getString(VIDEO_ID));
             if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) {
                 url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint")
                         .getString("playlistId"));
@@ -906,17 +914,6 @@ public static String getValidJsonResponseBody(@Nonnull final Response response)
         return responseBody;
     }
 
-    public static Response getResponse(final String url, final Localization localization)
-            throws IOException, ExtractionException {
-        final Map<String, List<String>> headers = new HashMap<>();
-        addYouTubeHeaders(headers);
-
-        final Response response = getDownloader().get(url, headers, localization);
-        getValidJsonResponseBody(response);
-
-        return response;
-    }
-
     public static JsonObject getJsonPostResponse(final String endpoint,
                                                  final byte[] body,
                                                  final Localization localization)
@@ -931,46 +928,28 @@ public static JsonObject getJsonPostResponse(final String endpoint,
         return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
     }
 
-    public static JsonObject getJsonMobilePostResponse(final String endpoint,
-                                                       final byte[] body,
-                                                       @Nonnull final ContentCountry
-                                                               contentCountry,
-                                                       final Localization localization)
-            throws IOException, ExtractionException {
+    public static JsonObject getJsonAndroidPostResponse(
+            final String endpoint,
+            final byte[] body,
+            @Nonnull final ContentCountry contentCountry,
+            final Localization localization,
+            @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
         final Map<String, List<String>> headers = new HashMap<>();
         headers.put("Content-Type", Collections.singletonList("application/json"));
         // Spoofing an Android 11 device with the hardcoded version of the Android app
         headers.put("User-Agent", Collections.singletonList("com.google.android.youtube/"
-                + MOBILE_YOUTUBE_CLIENT_VERSION + "Linux; U; Android 11; "
+                + MOBILE_YOUTUBE_CLIENT_VERSION + " (Linux; U; Android 11; "
                 + contentCountry.getCountryCode() + ") gzip"));
         headers.put("x-goog-api-format-version", Collections.singletonList("2"));
 
-        final Response response = getDownloader().post(
-                "https://youtubei.googleapis.com/youtubei/v1/" + endpoint + "?key="
-                        + MOBILE_YOUTUBE_KEY, headers, body, localization);
-
-        return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
-    }
-
-    public static JsonArray getJsonResponse(final String url, final Localization localization)
-            throws IOException, ExtractionException {
-        final Map<String, List<String>> headers = new HashMap<>();
-        addYouTubeHeaders(headers);
-
-        final Response response = getDownloader().get(url, headers, localization);
-
-        return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
-    }
+        final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint
+                + "?key=" + ANDROID_YOUTUBE_KEY;
 
-    public static JsonArray getJsonResponse(@Nonnull final Page page,
-                                            final Localization localization)
-            throws IOException, ExtractionException {
-        final Map<String, List<String>> headers = new HashMap<>();
-        addYouTubeHeaders(headers);
+        final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
+                        ? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
+                headers, body, localization);
 
-        final Response response = getDownloader().get(page.getUrl(), headers, localization);
-
-        return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
+        return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
     }
 
     @Nonnull
@@ -986,6 +965,13 @@ public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
                         .value("gl", contentCountry.getCountryCode())
                         .value("clientName", "WEB")
                         .value("clientVersion", getClientVersion())
+                        .value("originalUrl", "https://www.youtube.com")
+                        .value("platform", "DESKTOP")
+                    .end()
+                    .object("request")
+                        .array("internalExperimentFlags")
+                        .end()
+                        .value("useSsl", true)
                     .end()
                     .object("user")
                         // TO DO: provide a way to enable restricted mode with:
@@ -1032,25 +1018,32 @@ public static JsonBuilder<JsonObject> prepareDesktopEmbedVideoJsonBuilder(
                         .value("clientName", "WEB")
                         .value("clientVersion", getClientVersion())
                         .value("clientScreen", "EMBED")
+                        .value("originalUrl", "https://www.youtube.com")
+                        .value("platform", "DESKTOP")
                     .end()
                     .object("thirdParty")
                         .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
                     .end()
+                    .object("request")
+                        .array("internalExperimentFlags")
+                        .end()
+                        .value("useSsl", true)
+                    .end()
                     .object("user")
                         // TO DO: provide a way to enable restricted mode with:
                         // .value("enableSafetyMode", boolean)
                         .value("lockedSafetyMode", false)
                     .end()
-                .end()
-                .value("videoId", videoId);
+                .end();
         // @formatter:on
     }
 
     @Nonnull
     public static JsonBuilder<JsonObject> prepareAndroidMobileEmbedVideoJsonBuilder(
             @Nonnull final Localization localization,
             @Nonnull final ContentCountry contentCountry,
-            @Nonnull final String videoId) {
+            @Nonnull final String videoId,
+            @Nonnull final String contentPlaybackNonce) {
         // @formatter:off
         return JsonObject.builder()
                 .object("context")
@@ -1064,48 +1057,53 @@ public static JsonBuilder<JsonObject> prepareAndroidMobileEmbedVideoJsonBuilder(
                     .object("thirdParty")
                         .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
                     .end()
+                    .object("request")
+                        .array("internalExperimentFlags")
+                        .end()
+                        .value("useSsl", true)
+                    .end()
                     .object("user")
                         // TO DO: provide a way to enable restricted mode with:
                         // .value("enableSafetyMode", boolean)
                         .value("lockedSafetyMode", false)
                     .end()
                 .end()
-                .value("videoId", videoId);
+                .value(CPN, contentPlaybackNonce)
+                .value(VIDEO_ID, videoId);
         // @formatter:on
     }
 
     @Nonnull
-    public static byte[] createPlayerBodyWithSts(final Localization localization,
-                                                 final ContentCountry contentCountry,
-                                                 final String videoId,
-                                                 final boolean withThirdParty,
-                                                 @Nullable final String sts)
-            throws IOException, ExtractionException {
-        if (withThirdParty) {
-            // @formatter:off
-            return JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(
-                    localization, contentCountry, videoId)
-                    .object("playbackContext")
-                        .object("contentPlaybackContext")
-                            .value("signatureTimestamp", sts)
-                        .end()
-                    .end()
-                    .done())
-                    .getBytes(UTF_8);
-            // @formatter:on
-        } else {
-            // @formatter:off
-            return JsonWriter.string(prepareDesktopJsonBuilder(localization, contentCountry)
-                    .value("videoId", videoId)
-                    .object("playbackContext")
-                        .object("contentPlaybackContext")
-                            .value("signatureTimestamp", sts)
-                        .end()
+    public static byte[] createDesktopPlayerBody(
+            @Nonnull final Localization localization,
+            @Nonnull final ContentCountry contentCountry,
+            @Nonnull final String videoId,
+            @Nonnull final String sts,
+            final boolean isEmbedClientScreen,
+            @Nonnull final String contentPlaybackNonce) throws IOException, ExtractionException {
+        // @formatter:off
+        return JsonWriter.string((isEmbedClientScreen
+                        ? prepareDesktopEmbedVideoJsonBuilder(localization, contentCountry,
+                            videoId)
+                        : prepareDesktopJsonBuilder(localization, contentCountry))
+                .object("playbackContext")
+                    .object("contentPlaybackContext")
+                        .value("currentUrl", "/watch?v=" + videoId)
+                        .value("vis", 0)
+                        .value("splay", false)
+                        .value("autoCaptionsDefaultOn", false)
+                        .value("autonavState", "STATE_NONE")
+                        .value("html5Preference", "HTML5_PREF_WANTS")
+                        .value("signatureTimestamp", sts)
+                        .value("referer", "https://www.youtube.com/watch?v=" + videoId)
+                        .value("lactMilliseconds", "-1")
                     .end()
-                    .done())
-                    .getBytes(UTF_8);
-            // @formatter:on
-        }
+                .end()
+                .value(CPN, contentPlaybackNonce)
+                .value(VIDEO_ID, videoId)
+                .done())
+                .getBytes(StandardCharsets.UTF_8);
+        // @formatter:on
     }
 
     /**
@@ -1381,4 +1379,47 @@ public static String unescapeDocument(@Nonnull final String doc) {
                 .replaceAll("\\\\x5b", "[")
                 .replaceAll("\\\\x5d", "]");
     }
+
+    /**
+     * Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
+     * playback requests (and also for some clients, in the player request body).
+     *
+     * @return a content playback nonce string
+     */
+    @Nonnull
+    public static String generateContentPlaybackNonce() {
+        final SecureRandom random = new SecureRandom();
+        final StringBuilder stringBuilder = new StringBuilder();
+
+        for (int i = 0; i < 16; i++) {
+            stringBuilder.append(CONTENT_PLAYBACK_NONCE_ALPHABET.charAt(
+                    (random.nextInt(128) + 1) & 63));
+        }
+
+        return stringBuilder.toString();
+    }
+
+    /**
+     * Try to generate a {@code t} parameter, sent by mobile clients as a query of the player
+     * request.
+     *
+     * <p>
+     * Some researches needs to be done to know how this parameter, unique at each request, is
+     * generated.
+     * </p>
+     *
+     * @return a 12 characters string to try to reproduce the {@code} parameter
+     */
+    @Nonnull
+    public static String generateTParameter() {
+        final SecureRandom random = new SecureRandom();
+        final StringBuilder stringBuilder = new StringBuilder();
+
+        for (int i = 0; i < 12; i++) {
+            stringBuilder.append(CONTENT_PLAYBACK_NONCE_ALPHABET.charAt(
+                    (random.nextInt(128) + 1) & 63));
+        }
+
+        return stringBuilder.toString();
+    }
 }