fix: update iOS client, add visitor data to YouTube requests

Author: Theta-Dev

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ProtoBuilder.java

@@ -0,0 +1,71 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+public class ProtoBuilder {
+    ByteArrayOutputStream byteBuffer;
+
+    public ProtoBuilder() {
+        this.byteBuffer = new ByteArrayOutputStream();
+    }
+
+    public byte[] toBytes() {
+        return byteBuffer.toByteArray();
+    }
+
+    public String toUrlencodedBase64() {
+        final String b64 = Base64.getUrlEncoder().encodeToString(toBytes());
+        return URLEncoder.encode(b64, StandardCharsets.UTF_8);
+    }
+
+    private void writeVarint(long val) {
+        try {
+            if (val == 0) {
+                byteBuffer.write(new byte[]{(byte) 0});
+            } else {
+                while (val != 0) {
+                    byte b = (byte) (val & 0x7f);
+                    val >>= 7;
+
+                    if (val != 0) {
+                        b |= (byte) 0x80;
+                    }
+                    byteBuffer.write(new byte[]{b});
+                }
+            }
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void field(final int field, final byte wire) {
+        final long fbits = ((long) field) << 3;
+        final long wbits = ((long) wire) & 0x07;
+        final long val = fbits | wbits;
+        writeVarint(val);
+    }
+
+    public void varint(final int field, final long val) {
+        field(field, (byte) 0);
+        writeVarint(val);
+    }
+
+    public void string(final int field, final String string) {
+        final byte[] strBts = string.getBytes(StandardCharsets.UTF_8);
+        bytes(field, strBts);
+    }
+
+    public void bytes(final int field, final byte[] bytes) {
+        field(field, (byte) 2);
+        writeVarint(bytes.length);
+        try {
+            byteBuffer.write(bytes);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -174,7 +174,7 @@ private YoutubeParsingHelper() {
      * Store page of the YouTube app</a>, in the {@code What’s New} section.
      * </p>
      */
-    private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.28.1";
+    private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.45.4";
 
     /**
      * The hardcoded client version used for InnerTube requests with the TV HTML5 embed client.
@@ -235,15 +235,15 @@ private YoutubeParsingHelper() {
      *
      * @see #IOS_USER_AGENT_VERSION
      */
-    private static final String IOS_OS_VERSION = "17.5.1.21F90";
+    private static final String IOS_OS_VERSION = "18.1.0.22B83";
 
     /**
      * Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app. To be
      * used in the user agent for requests.
      *
      * @see #IOS_OS_VERSION
      */
-    private static final String IOS_USER_AGENT_VERSION = "17_5_1";
+    private static final String IOS_USER_AGENT_VERSION = "18_1_0";
 
     private static Random numberGenerator = new Random();
 
@@ -303,6 +303,23 @@ public static boolean isY2ubeURL(@Nonnull final URL url) {
         return url.getHost().equalsIgnoreCase("y2u.be");
     }
 
+    public static String randomVisitorData(final ContentCountry country) {
+        final ProtoBuilder pbE2 = new ProtoBuilder();
+        pbE2.string(2, "");
+        pbE2.varint(4, numberGenerator.nextInt(1, 256));
+
+        final ProtoBuilder pbE = new ProtoBuilder();
+        pbE.string(1, country.getCountryCode());
+        pbE.bytes(2, pbE2.toBytes());
+
+        final ProtoBuilder pb = new ProtoBuilder();
+        pb.string(1, RandomStringFromAlphabetGenerator.generate(
+                CONTENT_PLAYBACK_NONCE_ALPHABET, 11, numberGenerator));
+        pb.varint(5, System.currentTimeMillis() / 1000 - numberGenerator.nextInt(600000));
+        pb.bytes(6, pbE.toBytes());
+        return pb.toUrlencodedBase64();
+    }
+
     /**
      * Parses the duration string of the video expecting ":" or "." as separators
      *
@@ -1164,10 +1181,14 @@ public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
     public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
             @Nonnull final Localization localization,
             @Nonnull final ContentCountry contentCountry,
-            @Nullable final String visitorData)
+            @Nullable String visitorData)
             throws IOException, ExtractionException {
+        if (visitorData == null) {
+            visitorData = randomVisitorData(contentCountry);
+        }
+
         // @formatter:off
-        final JsonBuilder<JsonObject> builder = JsonObject.builder()
+        return JsonObject.builder()
                 .object("context")
                     .object("client")
                         .value("hl", localization.getLocalizationCode())
@@ -1176,13 +1197,9 @@ public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
                         .value("clientVersion", getClientVersion())
                         .value("originalUrl", "https://www.youtube.com")
                         .value("platform", "DESKTOP")
-                        .value("utcOffsetMinutes", 0);
-
-        if (visitorData != null) {
-            builder.value("visitorData", visitorData);
-        }
-
-        return builder.end()
+                        .value("utcOffsetMinutes", 0)
+                        .value("visitorData", visitorData)
+                    .end()
                     .object("request")
                         .array("internalExperimentFlags")
                         .end()
@@ -1256,6 +1273,7 @@ public static JsonBuilder<JsonObject> prepareIosMobileJsonBuilder(
                         .value("platform", "MOBILE")
                         .value("osName", "iOS")
                         .value("osVersion", IOS_OS_VERSION)
+                        .value("visitorData", randomVisitorData(contentCountry))
                         .value("hl", localization.getLocalizationCode())
                         .value("gl", contentCountry.getCountryCode())
                         .value("utcOffsetMinutes", 0)