Skip to content

Commit 7c16d97

Browse files
AudricVAudricV
committed
[YouTube] Send Content-Type header in all POST requests
This header was not sent partially before and was added and guessed by OkHttp. This can create issues when using other HTTP clients than OkHttp, such as Cronet. Some code in the modified classes has been improved and / or deduplicated, and usages of the UTF_8 constant of the Utils class has been replaced by StandardCharsets.UTF_8 where possible. Note that this header has been not added in except in YoutubeDashManifestCreatorsUtils, as an empty body is sent in the POST requests made by this class.
1 parent f4020be commit 7c16d97

7 files changed

Lines changed: 144 additions & 165 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 25 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -596,13 +596,14 @@ public static boolean areHardcodedClientVersionAndKeyValid()
596596
.end()
597597
.value("fetchLiveState", true)
598598
.end()
599-
.end().done().getBytes(UTF_8);
599+
.end().done().getBytes(StandardCharsets.UTF_8);
600600
// @formatter:on
601601

602602
final Map<String, List<String>> headers = new HashMap<>();
603603
headers.put("X-YouTube-Client-Name", singletonList("1"));
604604
headers.put("X-YouTube-Client-Version",
605605
singletonList(HARDCODED_CLIENT_VERSION));
606+
headers.put("Content-Type", Collections.singletonList("application/json"));
606607

607608
// This endpoint is fetched by the YouTube website to get the items of its main menu and is
608609
// pretty lightweight (around 30kB)
@@ -841,7 +842,7 @@ public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException,
841842
.end()
842843
.end()
843844
.value("input", "")
844-
.end().done().getBytes(UTF_8);
845+
.end().done().getBytes(StandardCharsets.UTF_8);
845846
// @formatter:on
846847

847848
final Map<String, List<String>> headers = new HashMap<>();
@@ -850,7 +851,7 @@ public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException,
850851
headers.put("X-YouTube-Client-Version", singletonList(
851852
HARDCODED_YOUTUBE_MUSIC_KEY[2]));
852853
headers.put("Origin", singletonList("https://music.youtube.com"));
853-
headers.put("Referer", singletonList("music.youtube.com"));
854+
headers.put("Referer", singletonList("https://music.youtube.com"));
854855
headers.put("Content-Type", singletonList("application/json"));
855856

856857
final Response response = getDownloader().post(url, headers, json);
@@ -1087,13 +1088,12 @@ public static JsonObject getJsonPostResponse(final String endpoint,
10871088
final Localization localization)
10881089
throws IOException, ExtractionException {
10891090
final Map<String, List<String>> headers = new HashMap<>();
1090-
addClientInfoHeaders(headers);
1091+
addYouTubeHeaders(headers);
10911092
headers.put("Content-Type", singletonList("application/json"));
10921093

1093-
final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
1094-
+ getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
1095-
1096-
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
1094+
return JsonUtils.toJsonObject(getValidJsonResponseBody(
1095+
getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key=" + getKey()
1096+
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization)));
10971097
}
10981098

10991099
public static JsonObject getJsonAndroidPostResponse(
@@ -1122,17 +1122,18 @@ private static JsonObject getMobilePostResponse(
11221122
@Nonnull final String innerTubeApiKey,
11231123
@Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
11241124
final Map<String, List<String>> headers = new HashMap<>();
1125-
headers.put("Content-Type", singletonList("application/json"));
11261125
headers.put("User-Agent", singletonList(userAgent));
11271126
headers.put("X-Goog-Api-Format-Version", singletonList("2"));
1127+
headers.put("Content-Type", singletonList("application/json"));
11281128

11291129
final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey
11301130
+ DISABLE_PRETTY_PRINT_PARAMETER;
11311131

1132-
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
1133-
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
1134-
headers, body, localization);
1135-
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
1132+
return JsonUtils.toJsonObject(getValidJsonResponseBody(
1133+
getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
1134+
? baseEndpointUrl
1135+
: baseEndpointUrl + endPartOfUrlRequest,
1136+
headers, body, localization)));
11361137
}
11371138

11381139
@Nonnull
@@ -1335,6 +1336,17 @@ public static String getIosUserAgent(@Nullable final Localization localization)
13351336
+ ")";
13361337
}
13371338

1339+
@Nonnull
1340+
public static Map<String, List<String>> getYoutubeMusicHeaders() {
1341+
final Map<String, List<String>> headers = new HashMap<>();
1342+
headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKey[1]));
1343+
headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKey[2]));
1344+
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
1345+
headers.put("Referer", Collections.singletonList("https://music.youtube.com"));
1346+
headers.put("Content-Type", Collections.singletonList("application/json"));
1347+
return headers;
1348+
}
1349+
13381350
/**
13391351
* Add required headers and cookies to an existing headers Map.
13401352
* @see #addClientInfoHeaders(Map)

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
import java.util.Map;
3636
import java.util.Objects;
3737

38-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
3938
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
4039
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
4140
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
@@ -707,7 +706,8 @@ private static Response getStreamingWebUrlWithoutRedirects(
707706
throws CreationException {
708707
try {
709708
final Map<String, List<String>> headers = new HashMap<>();
710-
addClientInfoHeaders(headers);
709+
headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
710+
headers.put("Referer", Collections.singletonList("https://www.youtube.com"));
711711

712712
String responseMimeType = "";
713713

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 57 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,11 @@
22

33
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
44
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
5-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
65
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
76
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
87
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
98
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
10-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
119
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
12-
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
1310
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
1411

1512
import com.grack.nanojson.JsonArray;
@@ -20,7 +17,6 @@
2017
import org.schabi.newpipe.extractor.StreamingService;
2118
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
2219
import org.schabi.newpipe.extractor.downloader.Downloader;
23-
import org.schabi.newpipe.extractor.downloader.Response;
2420
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
2521
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
2622
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -31,14 +27,13 @@
3127
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
3228
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
3329
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
34-
import org.schabi.newpipe.extractor.utils.JsonUtils;
3530
import org.schabi.newpipe.extractor.utils.Utils;
3631

3732
import java.io.IOException;
33+
import java.nio.charset.StandardCharsets;
3834
import java.util.ArrayList;
39-
import java.util.HashMap;
4035
import java.util.List;
41-
import java.util.Map;
36+
import java.util.Objects;
4237

4338
import javax.annotation.Nonnull;
4439
import javax.annotation.Nullable;
@@ -85,8 +80,8 @@ public YoutubeChannelExtractor(final StreamingService service,
8580
}
8681

8782
@Override
88-
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
89-
ExtractionException {
83+
public void onFetchPage(@Nonnull final Downloader downloader)
84+
throws IOException, ExtractionException {
9085
final String channelPath = super.getId();
9186
final String[] channelId = channelPath.split("/");
9287
String id = "";
@@ -98,22 +93,12 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
9893
getExtractorLocalization(), getExtractorContentCountry())
9994
.value("url", "https://www.youtube.com/" + channelPath)
10095
.done())
101-
.getBytes(UTF_8);
96+
.getBytes(StandardCharsets.UTF_8);
10297

10398
final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
10499
body, getExtractorLocalization());
105100

106-
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
107-
final JsonObject errorJsonObject = jsonResponse.getObject("error");
108-
final int errorCode = errorJsonObject.getInt("code");
109-
if (errorCode == 404) {
110-
throw new ContentNotAvailableException("This channel doesn't exist.");
111-
} else {
112-
throw new ContentNotAvailableException("Got error:\""
113-
+ errorJsonObject.getString("status") + "\": "
114-
+ errorJsonObject.getString("message"));
115-
}
116-
}
101+
checkIfChannelResponseIsValid(jsonResponse);
117102

118103
final JsonObject endpoint = jsonResponse.getObject("endpoint");
119104

@@ -146,22 +131,12 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
146131
.value("browseId", id)
147132
.value("params", "EgZ2aWRlb3M%3D") // Equal to videos
148133
.done())
149-
.getBytes(UTF_8);
134+
.getBytes(StandardCharsets.UTF_8);
150135

151136
final JsonObject jsonResponse = getJsonPostResponse("browse", body,
152137
getExtractorLocalization());
153138

154-
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
155-
final JsonObject errorJsonObject = jsonResponse.getObject("error");
156-
final int errorCode = errorJsonObject.getInt("code");
157-
if (errorCode == 404) {
158-
throw new ContentNotAvailableException("This channel doesn't exist.");
159-
} else {
160-
throw new ContentNotAvailableException("Got error:\""
161-
+ errorJsonObject.getString("status") + "\": "
162-
+ errorJsonObject.getString("message"));
163-
}
164-
}
139+
checkIfChannelResponseIsValid(jsonResponse);
165140

166141
final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
167142
.getObject(0)
@@ -199,6 +174,21 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
199174
YoutubeParsingHelper.defaultAlertsCheck(initialData);
200175
}
201176

177+
private void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse)
178+
throws ContentNotAvailableException {
179+
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
180+
final JsonObject errorJsonObject = jsonResponse.getObject("error");
181+
final int errorCode = errorJsonObject.getInt("code");
182+
if (errorCode == 404) {
183+
throw new ContentNotAvailableException("This channel doesn't exist.");
184+
} else {
185+
throw new ContentNotAvailableException("Got error:\""
186+
+ errorJsonObject.getString("status") + "\": "
187+
+ errorJsonObject.getString("message"));
188+
}
189+
}
190+
}
191+
202192
@Nonnull
203193
@Override
204194
public String getUrl() throws ParsingException {
@@ -340,8 +330,8 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
340330
final List<String> channelIds = new ArrayList<>();
341331
channelIds.add(getName());
342332
channelIds.add(getUrl());
343-
final JsonObject continuation = collectStreamsFrom(collector, gridRenderer
344-
.getArray("items"), channelIds);
333+
final JsonObject continuation = collectStreamsFrom(collector,
334+
gridRenderer.getArray("items"), channelIds);
345335

346336
nextPage = getNextPageFrom(continuation, channelIds);
347337
}
@@ -350,23 +340,19 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
350340
}
351341

352342
@Override
353-
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
354-
ExtractionException {
343+
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
344+
throws IOException, ExtractionException {
355345
if (page == null || isNullOrEmpty(page.getUrl())) {
356346
throw new IllegalArgumentException("Page doesn't contain an URL");
357347
}
358348

359349
final List<String> channelIds = page.getIds();
360350

361351
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
362-
final Map<String, List<String>> headers = new HashMap<>();
363-
addClientInfoHeaders(headers);
364352

365-
final Response response = getDownloader().post(page.getUrl(), null, page.getBody(),
353+
final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
366354
getExtractorLocalization());
367355

368-
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
369-
370356
final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
371357
.getObject(0)
372358
.getObject("appendContinuationItemsAction");
@@ -379,8 +365,8 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
379365

380366
@Nullable
381367
private Page getNextPageFrom(final JsonObject continuations,
382-
final List<String> channelIds) throws IOException,
383-
ExtractionException {
368+
final List<String> channelIds)
369+
throws IOException, ExtractionException {
384370
if (isNullOrEmpty(continuations)) {
385371
return null;
386372
}
@@ -393,7 +379,7 @@ private Page getNextPageFrom(final JsonObject continuations,
393379
getExtractorContentCountry())
394380
.value("continuation", continuation)
395381
.done())
396-
.getBytes(UTF_8);
382+
.getBytes(StandardCharsets.UTF_8);
397383

398384
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
399385
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
@@ -443,39 +429,43 @@ public String getUploaderUrl() {
443429

444430
@Nullable
445431
private JsonObject getVideoTab() throws ParsingException {
446-
if (this.videoTab != null) {
447-
return this.videoTab;
432+
if (videoTab != null) {
433+
return videoTab;
448434
}
449435

450436
final JsonArray tabs = initialData.getObject("contents")
451437
.getObject("twoColumnBrowseResultsRenderer")
452438
.getArray("tabs");
453439

454-
JsonObject foundVideoTab = null;
455-
for (final Object tab : tabs) {
456-
if (((JsonObject) tab).has("tabRenderer")) {
457-
if (((JsonObject) tab).getObject("tabRenderer").getString("title",
458-
"").equals("Videos")) {
459-
foundVideoTab = ((JsonObject) tab).getObject("tabRenderer");
460-
break;
461-
}
462-
}
463-
}
464-
465-
if (foundVideoTab == null) {
466-
throw new ContentNotSupportedException("This channel has no Videos tab");
467-
}
468-
469-
final String messageRendererText = getTextFromObject(foundVideoTab.getObject("content")
470-
.getObject("sectionListRenderer").getArray("contents").getObject(0)
471-
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
472-
.getObject("messageRenderer").getObject("text"));
440+
final JsonObject foundVideoTab = tabs.stream()
441+
.filter(Objects::nonNull)
442+
.filter(JsonObject.class::isInstance)
443+
.map(JsonObject.class::cast)
444+
.filter(tab -> tab.has("tabRenderer")
445+
&& tab.getObject("tabRenderer")
446+
.getString("title", "")
447+
.equals("Videos"))
448+
.findFirst()
449+
.map(tab -> tab.getObject("tabRenderer"))
450+
.orElseThrow(
451+
() -> new ContentNotSupportedException("This channel has no Videos tab"));
452+
453+
final String messageRendererText = getTextFromObject(
454+
foundVideoTab.getObject("content")
455+
.getObject("sectionListRenderer")
456+
.getArray("contents")
457+
.getObject(0)
458+
.getObject("itemSectionRenderer")
459+
.getArray("contents")
460+
.getObject(0)
461+
.getObject("messageRenderer")
462+
.getObject("text"));
473463
if (messageRendererText != null
474464
&& messageRendererText.equals("This channel has no videos.")) {
475465
return null;
476466
}
477467

478-
this.videoTab = foundVideoTab;
468+
videoTab = foundVideoTab;
479469
return foundVideoTab;
480470
}
481471
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import java.io.IOException;
3939
import java.net.URL;
4040
import java.nio.charset.StandardCharsets;
41+
import java.util.Collections;
4142
import java.util.HashMap;
4243
import java.util.List;
4344
import java.util.Map;
@@ -91,6 +92,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
9192
final Map<String, List<String>> headers = new HashMap<>();
9293
// Cookie is required due to consent
9394
addYouTubeHeaders(headers);
95+
headers.put("Content-Type", Collections.singletonList("application/json"));
9496

9597
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
9698
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
@@ -224,6 +226,7 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
224226
final Map<String, List<String>> headers = new HashMap<>();
225227
// Cookie is required due to consent
226228
addYouTubeHeaders(headers);
229+
headers.put("Content-Type", Collections.singletonList("application/json"));
227230

228231
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
229232
getExtractorLocalization());

0 commit comments

Comments
 (0)