Skip to content

Commit 4299d80

Browse files
AudricVAudricV
committed
Adress changes
1 parent 1a6b8da commit 4299d80

7 files changed

Lines changed: 95 additions & 110 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 36 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -66,15 +66,15 @@ private YoutubeParsingHelper() {
6666

6767
public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
6868

69-
private static final String HARDCODED_CLIENT_VERSION = "2.20210603.07.00";
69+
private static final String HARDCODED_CLIENT_VERSION = "2.20210622.10.00";
7070
private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
7171
private static final String MOBILE_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
72-
private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.20.36";
72+
private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.23.36";
7373
private static String clientVersion;
7474
private static String key;
7575

7676
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEY =
77-
{"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
77+
{"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20210621.00.00"};
7878
private static String[] youtubeMusicKey;
7979

8080
private static boolean keyAndVersionExtracted = false;
@@ -102,7 +102,8 @@ private static boolean isGoogleURL(String url) {
102102
try {
103103
final URL u = new URL(url);
104104
final String host = u.getHost();
105-
return host.startsWith("google.") || host.startsWith("m.google.")
105+
return host.startsWith("google.")
106+
|| host.startsWith("m.google.")
106107
|| host.startsWith("www.google.");
107108
} catch (final MalformedURLException e) {
108109
return false;
@@ -111,7 +112,8 @@ private static boolean isGoogleURL(String url) {
111112

112113
public static boolean isYoutubeURL(@Nonnull final URL url) {
113114
final String host = url.getHost();
114-
return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com")
115+
return host.equalsIgnoreCase("youtube.com")
116+
|| host.equalsIgnoreCase("www.youtube.com")
115117
|| host.equalsIgnoreCase("m.youtube.com")
116118
|| host.equalsIgnoreCase("music.youtube.com");
117119
}
@@ -234,7 +236,7 @@ public static OffsetDateTime parseDateFrom(final String textualUploadDate)
234236
* Checks if the given playlist id is a YouTube Mix (auto-generated playlist)
235237
* Ids from a YouTube Mix start with "RD"
236238
*
237-
* @param playlistId the id of the playlist
239+
* @param playlistId the playlist id
238240
* @return Whether given id belongs to a YouTube Mix
239241
*/
240242
public static boolean isYoutubeMixId(@Nonnull final String playlistId) {
@@ -306,8 +308,8 @@ public static JsonObject getInitialData(final String html) throws ParsingExcepti
306308
}
307309
}
308310

309-
public static boolean areHardcodedClientVersionAndKeyValid() throws IOException,
310-
ExtractionException {
311+
public static boolean areHardcodedClientVersionAndKeyValid()
312+
throws IOException, ExtractionException {
311313
if (areHardcodedClientVersionAndKeyValidValue != null) {
312314
return areHardcodedClientVersionAndKeyValidValue;
313315
}
@@ -316,11 +318,17 @@ public static boolean areHardcodedClientVersionAndKeyValid() throws IOException,
316318
.object()
317319
.object("context")
318320
.object("client")
319-
.value("hl", "en")
321+
.value("hl", "en-GB")
320322
.value("gl", "GB")
321-
.value("clientName", "1")
323+
.value("clientName", "WEB")
322324
.value("clientVersion", HARDCODED_CLIENT_VERSION)
323325
.end()
326+
.object("user")
327+
// TO DO: provide a way to enable restricted mode with:
328+
// .value("enableSafetyMode", boolean)
329+
.value("lockedSafetyMode", false)
330+
.end()
331+
.value("fetchLiveState", true)
324332
.end()
325333
.end().done().getBytes(UTF_8);
326334
// @formatter:on
@@ -337,9 +345,8 @@ public static boolean areHardcodedClientVersionAndKeyValid() throws IOException,
337345
final String responseBody = response.responseBody();
338346
final int responseCode = response.responseCode();
339347

340-
areHardcodedClientVersionAndKeyValidValue = responseBody.length() > 5000
348+
return areHardcodedClientVersionAndKeyValidValue = responseBody.length() > 5000
341349
&& responseCode == 200; // Ensure to have a valid response
342-
return areHardcodedClientVersionAndKeyValidValue;
343350
}
344351

345352
private static void extractClientVersionAndKey() throws IOException, ExtractionException {
@@ -485,8 +492,7 @@ public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException,
485492
.value("hl", "en-GB")
486493
.value("gl", "GB")
487494
.array("experimentIds").end()
488-
.value("experimentsToken", "")
489-
.value("utcOffsetMinutes", 0)
495+
.value("experimentsToken", EMPTY_STRING)
490496
.object("locationInfo").end()
491497
.object("musicAppInfo").end()
492498
.end()
@@ -802,10 +808,13 @@ public static JsonBuilder<JsonObject> prepareJsonBuilder(@Nonnull final Localiza
802808
return JsonObject.builder()
803809
.object("context")
804810
.object("client")
805-
.value("clientName", "WEB")
806-
.value("clientVersion", getClientVersion())
807811
.value("hl", localization.getLocalizationCode())
808812
.value("gl", contentCountry.getCountryCode())
813+
.value("clientName", "WEB")
814+
.value("clientVersion", getClientVersion())
815+
.end()
816+
.object("user")
817+
.value("lockedSafetyMode", false)
809818
.end()
810819
.end();
811820
// @formatter:on
@@ -826,6 +835,11 @@ public static JsonBuilder<JsonObject> prepareMobileJsonBuilder(@Nonnull final Lo
826835
.value("hl", localization.getLocalizationCode())
827836
.value("gl", contentCountry.getCountryCode())
828837
.end()
838+
.object("user")
839+
// TO DO: provide a way to enable restricted mode with:
840+
// .value("enableSafetyMode", boolean)
841+
.value("lockedSafetyMode", false)
842+
.end()
829843
.end();
830844
// @formatter:on
831845
}
@@ -848,15 +862,11 @@ public static void addYouTubeHeaders(final Map<String, List<String>> headers)
848862
*/
849863
public static void addClientInfoHeaders(@Nonnull final Map<String, List<String>> headers)
850864
throws IOException, ExtractionException {
851-
if (headers.get("Origin") == null) {
852-
headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
853-
}
854-
if (headers.get("Referer") == null) {
855-
headers.put("Referer", Collections.singletonList("https://www.youtube.com"));
856-
}
857-
if (headers.get("X-YouTube-Client-Name") == null) {
858-
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
859-
}
865+
headers.computeIfAbsent("Origin", k -> Collections.singletonList(
866+
"https://www.youtube.com"));
867+
headers.computeIfAbsent("Referer", k -> Collections.singletonList(
868+
"https://www.youtube.com"));
869+
headers.computeIfAbsent("X-YouTube-Client-Name", k -> Collections.singletonList("1"));
860870
if (headers.get("X-YouTube-Client-Version") == null) {
861871
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
862872
}
@@ -867,7 +877,7 @@ public static void addClientInfoHeaders(@Nonnull final Map<String, List<String>>
867877
* @see #CONSENT_COOKIE
868878
* @param headers the headers which should be completed
869879
*/
870-
public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
880+
public static void addCookieHeader(final Map<String, List<String>> headers) {
871881
if (headers.get("Cookie") == null) {
872882
headers.put("Cookie", Arrays.asList(generateConsentCookie()));
873883
} else {
@@ -1092,5 +1102,4 @@ public static String unescapeDocument(@Nonnull final String doc) {
10921102
.replaceAll("\\\\x5b", "[")
10931103
.replaceAll("\\\\x5d", "]");
10941104
}
1095-
10961105
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.util.Map;
3030

3131
import javax.annotation.Nonnull;
32+
import javax.annotation.Nullable;
3233

3334
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
3435
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
@@ -84,7 +85,7 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
8485
final String[] channelInfo = channel_path.split("/");
8586
String id = "";
8687
// If the url is an URL which is not a /channel URL, we need to use the
87-
// navigation/resolve_url endpoint of the youtubei API to get the channel id. Otherwise, we
88+
// navigation/resolve_url endpoint of the internal API to get the channel id. Otherwise, we
8889
// couldn't get information about the channel associated with this URL, if there is one.
8990
if (!channelInfo[0].equals("channel")) {
9091
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
@@ -293,17 +294,17 @@ public String getDescription() throws ParsingException {
293294
}
294295

295296
@Override
296-
public String getParentChannelName() throws ParsingException {
297+
public String getParentChannelName() {
297298
return "";
298299
}
299300

300301
@Override
301-
public String getParentChannelUrl() throws ParsingException {
302+
public String getParentChannelUrl() {
302303
return "";
303304
}
304305

305306
@Override
306-
public String getParentChannelAvatarUrl() throws ParsingException {
307+
public String getParentChannelAvatarUrl() {
307308
return "";
308309
}
309310

@@ -329,13 +330,13 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
329330
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
330331
.getArray("contents").getObject(0).getObject("gridRenderer");
331332

332-
final List<String> channelInformations = new ArrayList<>();
333-
channelInformations.add(getName());
334-
channelInformations.add(getUrl());
333+
final List<String> channelInfo = new ArrayList<>();
334+
channelInfo.add(getName());
335+
channelInfo.add(getUrl());
335336
final JsonObject continuation = collectStreamsFrom(collector, gridRenderer
336-
.getArray("items"), channelInformations);
337+
.getArray("items"), channelInfo);
337338

338-
nextPage = getNextPageFrom(continuation, channelInformations);
339+
nextPage = getNextPageFrom(continuation, channelInfo);
339340
}
340341

341342
return new InfoItemsPage<>(collector, nextPage);
@@ -348,7 +349,7 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
348349
throw new IllegalArgumentException("Page doesn't contain an URL");
349350
}
350351

351-
final List<String> channelInformations = page.getIds();
352+
final List<String> channelInfos = page.getIds();
352353

353354
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
354355
final Map<String, List<String>> headers = new HashMap<>();
@@ -364,13 +365,14 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
364365
.getObject("appendContinuationItemsAction");
365366

366367
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation
367-
.getArray("continuationItems"), channelInformations);
368+
.getArray("continuationItems"), channelInfos);
368369

369-
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelInformations));
370+
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelInfos));
370371
}
371372

373+
@Nullable
372374
private Page getNextPageFrom(final JsonObject continuations,
373-
final List<String> channelInformations) throws IOException,
375+
final List<String> channelInfo) throws IOException,
374376
ExtractionException {
375377
if (isNullOrEmpty(continuations)) {
376378
return null;
@@ -386,8 +388,7 @@ private Page getNextPageFrom(final JsonObject continuations,
386388
.done())
387389
.getBytes(UTF_8);
388390

389-
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelInformations,
390-
null, body);
391+
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelInfo, null, body);
391392
}
392393

393394
/**
@@ -397,13 +398,13 @@ private Page getNextPageFrom(final JsonObject continuations,
397398
* @param videos the array to get videos from
398399
* @return the continuation object
399400
*/
400-
private JsonObject collectStreamsFrom(final StreamInfoItemsCollector collector,
401-
final JsonArray videos,
402-
final List<String> channelInformations) {
401+
private JsonObject collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
402+
@Nonnull final JsonArray videos,
403+
@Nonnull final List<String> channelInfo) {
403404
collector.reset();
404405

405-
final String uploaderName = channelInformations.get(0);
406-
final String uploaderUrl = channelInformations.get(1);
406+
final String uploaderName = channelInfo.get(0);
407+
final String uploaderUrl = channelInfo.get(1);
407408
final TimeAgoParser timeAgoParser = getTimeAgoParser();
408409

409410
JsonObject continuation = null;
@@ -431,6 +432,7 @@ public String getUploaderUrl() {
431432
return continuation;
432433
}
433434

435+
@Nullable
434436
private JsonObject getVideoTab() throws ParsingException {
435437
if (this.videoTab != null) return this.videoTab;
436438

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 9 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.schabi.newpipe.extractor.services.youtube.extractors;
22

33
import com.grack.nanojson.JsonArray;
4+
import com.grack.nanojson.JsonBuilder;
45
import com.grack.nanojson.JsonObject;
56

67
import com.grack.nanojson.JsonWriter;
@@ -61,31 +62,16 @@ public void onFetchPage(@Nonnull final Downloader downloader)
6162
final String videoId = getQueryValue(url, "v");
6263
final String playlistIndexString = getQueryValue(url, "index");
6364

64-
final byte[] body;
65+
final JsonBuilder<JsonObject> jsonBody = prepareJsonBuilder(localization,
66+
getExtractorContentCountry()).value("playlistId", mixPlaylistId);
6567
if (videoId != null) {
66-
if (playlistIndexString != null) {
67-
body = JsonWriter.string(prepareJsonBuilder(localization,
68-
getExtractorContentCountry())
69-
.value("videoId", videoId)
70-
.value("playlistId", mixPlaylistId)
71-
.value("playlistIndex", Integer.parseInt(playlistIndexString))
72-
.done())
73-
.getBytes(UTF_8);
74-
} else {
75-
body = JsonWriter.string(prepareJsonBuilder(localization,
76-
getExtractorContentCountry())
77-
.value("videoId", videoId)
78-
.value("playlistId", mixPlaylistId)
79-
.done())
80-
.getBytes(UTF_8);
81-
}
82-
} else {
83-
body = JsonWriter.string(prepareJsonBuilder(localization,
84-
getExtractorContentCountry())
85-
.value("playlistId", mixPlaylistId)
86-
.done())
87-
.getBytes(UTF_8);
68+
jsonBody.value("videoId", videoId);
8869
}
70+
if (playlistIndexString != null) {
71+
jsonBody.value("playlistIndex", Integer.parseInt(playlistIndexString));
72+
}
73+
74+
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(UTF_8);
8975

9076
final Map<String, List<String>> headers = new HashMap<>();
9177
addClientInfoHeaders(headers);

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,11 +74,10 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
7474
.object("client")
7575
.value("clientName", "WEB_REMIX")
7676
.value("clientVersion", youtubeMusicKeys[2])
77-
.value("hl", "en")
77+
.value("hl", "en-GB")
7878
.value("gl", getExtractorContentCountry().getCountryCode())
7979
.array("experimentIds").end()
80-
.value("experimentsToken", "")
81-
.value("utcOffsetMinutes", 0)
80+
.value("experimentsToken", EMPTY_STRING)
8281
.object("locationInfo").end()
8382
.object("musicAppInfo").end()
8483
.end()
@@ -89,6 +88,7 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
8988
.end()
9089
.object("activePlayers").end()
9190
.object("user")
91+
// TO DO: provide a way to enable restricted mode with:
9292
.value("enableSafetyMode", false)
9393
.end()
9494
.end()

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
1010
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1111
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
12-
import org.schabi.newpipe.extractor.localization.ContentCountry;
1312
import org.schabi.newpipe.extractor.localization.Localization;
1413
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
1514
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
@@ -60,7 +59,6 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
6059
ExtractionException {
6160
final String query = super.getSearchString();
6261
final Localization localization = getExtractorLocalization();
63-
final ContentCountry contentCountry = getExtractorContentCountry();
6462

6563
// Get the search parameter of the request
6664
final List<String> contentFilters = super.getLinkHandler().getContentFilters();
@@ -72,20 +70,15 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
7270
params = "";
7371
}
7472

75-
final byte[] body;
73+
final JsonBuilder<JsonObject> jsonBody = prepareJsonBuilder(localization,
74+
getExtractorContentCountry())
75+
.value("query", query);
7676
if (!isNullOrEmpty(params)) {
77-
body = JsonWriter.string(prepareJsonBuilder(localization, contentCountry)
78-
.value("query", query)
79-
.value("params", params)
80-
.done())
81-
.getBytes(UTF_8);
82-
} else {
83-
body = JsonWriter.string(prepareJsonBuilder(localization, contentCountry)
84-
.value("query", query)
85-
.done())
86-
.getBytes(UTF_8);
77+
jsonBody.value("params", params);
8778
}
8879

80+
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(UTF_8);
81+
8982
initialData = getJsonPostResponse("search", body, localization);
9083
}
9184

0 commit comments

Comments
 (0)