Skip to content

Commit 3d38459

Browse files
AudricVAudricV
committed
[YouTube] Reduce InnerTube response sizes by adding the prettyPrint parameter with the false value
InnerTube responses return pretty printed responses, which increase responses' size for nothing. By using the prettyPrint parameter on requests and setting its value to false, responses are not pretty printed anymore, which reduces responses size, and so data transfer and processing times. This usage has been recently deployed by YouTube on their websites.
1 parent 349ba8d commit 3d38459

7 files changed

Lines changed: 33 additions & 18 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,16 @@ private YoutubeParsingHelper() {
8181
}
8282

8383
public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
84+
85+
/**
86+
* A parameter to disable pretty-printed response of InnerTube requests, to reduce response
87+
* sizes.
88+
*
89+
* <p>
90+
* Sent in query parameters of the requests, <b>after</b> the API key.
91+
* </p>
92+
**/
93+
public static final String DISABLE_PRETTY_PRINT_PARAMETER = "&prettyPrint=false";
8494
public static final String CPN = "cpn";
8595
public static final String VIDEO_ID = "videoId";
8696

@@ -495,7 +505,7 @@ public static boolean areHardcodedClientVersionAndKeyValid()
495505
// This endpoint is fetched by the YouTube website to get the items of its main menu and is
496506
// pretty lightweight (around 30kB)
497507
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "guide?key="
498-
+ HARDCODED_KEY, headers, body);
508+
+ HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER, headers, body);
499509
final String responseBody = response.responseBody();
500510
final int responseCode = response.responseCode();
501511

@@ -674,7 +684,7 @@ public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException,
674684
ReCaptchaException {
675685
final String url =
676686
"https://music.youtube.com/youtubei/v1/music/get_search_suggestions?alt=json&key="
677-
+ HARDCODED_YOUTUBE_MUSIC_KEY[0];
687+
+ HARDCODED_YOUTUBE_MUSIC_KEY[0] + DISABLE_PRETTY_PRINT_PARAMETER;
678688

679689
// @formatter:off
680690
final byte[] json = JsonWriter.string()
@@ -953,7 +963,7 @@ public static JsonObject getJsonPostResponse(final String endpoint,
953963
headers.put("Content-Type", Collections.singletonList("application/json"));
954964

955965
final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
956-
+ getKey(), headers, body, localization);
966+
+ getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
957967

958968
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
959969
}
@@ -972,7 +982,7 @@ public static JsonObject getJsonAndroidPostResponse(
972982
headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2"));
973983

974984
final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint
975-
+ "?key=" + ANDROID_YOUTUBE_KEY;
985+
+ "?key=" + ANDROID_YOUTUBE_KEY + DISABLE_PRETTY_PRINT_PARAMETER;
976986

977987
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
978988
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
@@ -995,7 +1005,7 @@ public static JsonObject getJsonIosPostResponse(
9951005
headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2"));
9961006

9971007
final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint
998-
+ "?key=" + IOS_YOUTUBE_KEY;
1008+
+ "?key=" + IOS_YOUTUBE_KEY + DISABLE_PRETTY_PRINT_PARAMETER;
9991009

10001010
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
10011011
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -395,7 +395,8 @@ private Page getNextPageFrom(final JsonObject continuations,
395395
.done())
396396
.getBytes(UTF_8);
397397

398-
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelIds, null, body);
398+
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
399+
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
399400
}
400401

401402
/**

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,8 @@ public void onFetchPage(@Nonnull final Downloader downloader)
9090
final Map<String, List<String>> headers = new HashMap<>();
9191
addClientInfoHeaders(headers);
9292

93-
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey(),
94-
headers, body, localization);
93+
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
94+
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
9595

9696
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
9797
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
6060
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
6161

6262
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key="
63-
+ youtubeMusicKeys[0];
63+
+ youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
6464

6565
final String params;
6666

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,8 @@ private Page getNextPageFrom(final JsonArray contents)
317317
.done())
318318
.getBytes(StandardCharsets.UTF_8);
319319

320-
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body);
320+
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
321+
+ DISABLE_PRETTY_PRINT_PARAMETER, body);
321322
} else {
322323
return null;
323324
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,8 @@ private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws I
239239
final String token = continuationItemRenderer.getObject("continuationEndpoint")
240240
.getObject("continuationCommand").getString("token");
241241

242-
final String url = YOUTUBEI_V1_URL + "search?key=" + getKey();
242+
final String url = YOUTUBEI_V1_URL + "search?key=" + getKey()
243+
+ DISABLE_PRETTY_PRINT_PARAMETER;
243244

244245
return new Page(url, token);
245246
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,7 @@
66
import static org.junit.jupiter.api.Assertions.assertTrue;
77
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
88
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
9-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
10-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
11-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
9+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
1210

1311
import com.grack.nanojson.JsonWriter;
1412

@@ -93,7 +91,8 @@ void getPage() throws Exception {
9391
.getBytes(StandardCharsets.UTF_8);
9492

9593
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
96-
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
94+
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
95+
null, null, dummyCookie, body));
9796
assertFalse(streams.getItems().isEmpty());
9897
assertTrue(streams.hasNextPage());
9998
}
@@ -183,7 +182,8 @@ void getPage() throws Exception {
183182
.getBytes(StandardCharsets.UTF_8);
184183

185184
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
186-
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
185+
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
186+
null, null, dummyCookie, body));
187187
assertFalse(streams.getItems().isEmpty());
188188
assertTrue(streams.hasNextPage());
189189
}
@@ -270,7 +270,8 @@ void getPage() throws Exception {
270270
.getBytes(StandardCharsets.UTF_8);
271271

272272
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
273-
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
273+
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
274+
null, null, dummyCookie, body));
274275
assertFalse(streams.getItems().isEmpty());
275276
assertTrue(streams.hasNextPage());
276277
}
@@ -389,7 +390,8 @@ void getPage() throws Exception {
389390
.getBytes(StandardCharsets.UTF_8);
390391

391392
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
392-
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
393+
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
394+
null, null, dummyCookie, body));
393395
assertFalse(streams.getItems().isEmpty());
394396
assertTrue(streams.hasNextPage());
395397
}

0 commit comments

Comments
 (0)