[YouTube] Send Content-Type header in all POST requests

This header was not sent partially before and was added and guessed by OkHttp. This can create issues when using other HTTP clients than OkHttp, such as Cronet.

Some code in the modified classes has been improved and / or deduplicated, and usages of the UTF_8 constant of the Utils class has been replaced by StandardCharsets.UTF_8 where possible.

Note that this header has been not added in except in YoutubeDashManifestCreatorsUtils, as an empty body is sent in the POST requests made by this class.

Author: AudricV

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -597,13 +597,14 @@ public static boolean areHardcodedClientVersionAndKeyValid()
                 .end()
                 .value("fetchLiveState", true)
                 .end()
-            .end().done().getBytes(UTF_8);
+            .end().done().getBytes(StandardCharsets.UTF_8);
         // @formatter:on
 
         final Map<String, List<String>> headers = new HashMap<>();
         headers.put("X-YouTube-Client-Name", singletonList("1"));
         headers.put("X-YouTube-Client-Version",
                 singletonList(HARDCODED_CLIENT_VERSION));
+        headers.put("Content-Type", Collections.singletonList("application/json"));
 
         // This endpoint is fetched by the YouTube website to get the items of its main menu and is
         // pretty lightweight (around 30kB)
@@ -842,7 +843,7 @@ public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException,
                     .end()
                 .end()
                 .value("input", "")
-            .end().done().getBytes(UTF_8);
+            .end().done().getBytes(StandardCharsets.UTF_8);
         // @formatter:on
 
         final Map<String, List<String>> headers = new HashMap<>();
@@ -851,7 +852,7 @@ public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException,
         headers.put("X-YouTube-Client-Version", singletonList(
                 HARDCODED_YOUTUBE_MUSIC_KEY[2]));
         headers.put("Origin", singletonList("https://music.youtube.com"));
-        headers.put("Referer", singletonList("music.youtube.com"));
+        headers.put("Referer", singletonList("https://music.youtube.com"));
         headers.put("Content-Type", singletonList("application/json"));
 
         final Response response = getDownloader().post(url, headers, json);
@@ -1088,13 +1089,12 @@ public static JsonObject getJsonPostResponse(final String endpoint,
                                                  final Localization localization)
             throws IOException, ExtractionException {
         final Map<String, List<String>> headers = new HashMap<>();
-        addClientInfoHeaders(headers);
+        addYouTubeHeaders(headers);
         headers.put("Content-Type", singletonList("application/json"));
 
-        final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
-                + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
-
-        return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
+        return JsonUtils.toJsonObject(getValidJsonResponseBody(
+                getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key=" + getKey()
+                        + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization)));
     }
 
     public static JsonObject getJsonAndroidPostResponse(
@@ -1123,17 +1123,18 @@ private static JsonObject getMobilePostResponse(
             @Nonnull final String innerTubeApiKey,
             @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
         final Map<String, List<String>> headers = new HashMap<>();
-        headers.put("Content-Type", singletonList("application/json"));
         headers.put("User-Agent", singletonList(userAgent));
         headers.put("X-Goog-Api-Format-Version", singletonList("2"));
+        headers.put("Content-Type", singletonList("application/json"));
 
         final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey
                 + DISABLE_PRETTY_PRINT_PARAMETER;
 
-        final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
-                        ? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
-                headers, body, localization);
-        return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
+        return JsonUtils.toJsonObject(getValidJsonResponseBody(
+                getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
+                                ? baseEndpointUrl
+                                : baseEndpointUrl + endPartOfUrlRequest,
+                        headers, body, localization)));
     }
 
     @Nonnull
@@ -1336,6 +1337,17 @@ public static String getIosUserAgent(@Nullable final Localization localization)
                 + ")";
     }
 
+    @Nonnull
+    public static Map<String, List<String>> getYoutubeMusicHeaders() {
+        final Map<String, List<String>> headers = new HashMap<>();
+        headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKey[1]));
+        headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKey[2]));
+        headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
+        headers.put("Referer", Collections.singletonList("https://music.youtube.com"));
+        headers.put("Content-Type", Collections.singletonList("application/json"));
+        return headers;
+    }
+
     /**
      * Add required headers and cookies to an existing headers Map.
      * @see #addClientInfoHeaders(Map)

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java

@@ -35,7 +35,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
@@ -707,7 +706,8 @@ private static Response getStreamingWebUrlWithoutRedirects(
             throws CreationException {
         try {
             final Map<String, List<String>> headers = new HashMap<>();
-            addClientInfoHeaders(headers);
+            headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
+            headers.put("Referer", Collections.singletonList("https://www.youtube.com"));
 
             String responseMimeType = "";