Skip to content

Commit 883f16e

Browse files
TobiGrTobiGr
committed
[YouTube] Set CONSENT cookie
1 parent e61ceef commit 883f16e

4 files changed

Lines changed: 93 additions & 43 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 69 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,7 @@
2828
import java.time.OffsetDateTime;
2929
import java.time.ZoneOffset;
3030
import java.time.format.DateTimeParseException;
31-
import java.util.ArrayList;
32-
import java.util.Collections;
33-
import java.util.HashMap;
34-
import java.util.List;
35-
import java.util.Map;
36-
import java.util.Objects;
31+
import java.util.*;
3732

3833
import javax.annotation.Nonnull;
3934
import javax.annotation.Nullable;
@@ -79,6 +74,17 @@ private YoutubeParsingHelper() {
7974
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
8075
private static String[] youtubeMusicKeys;
8176

77+
/**
78+
* <code>PENDING+</code> means that the user did not yet submit their choices.
79+
* Therefore, YouTube & Google should not track the user, because they did not give consent.
80+
* The three digits at the end can be random, but are required.
81+
*/
82+
public static final String CONSENT_COOKIE_VALUE = "PENDING+" + (100 + new Random().nextInt(900));
83+
/**
84+
* Youtube <code>CONSENT</code> cookie. Should prevent redirect to consent.youtube.com
85+
*/
86+
public static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
87+
8288
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
8389
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
8490

@@ -427,6 +433,7 @@ public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, Re
427433
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
428434
headers.put("Referer", Collections.singletonList("music.youtube.com"));
429435
headers.put("Content-Type", Collections.singletonList("application/json"));
436+
addCookieHeader(headers);
430437

431438
final String response = getDownloader().post(url, headers, json).responseBody();
432439

@@ -629,34 +636,19 @@ public static String getValidJsonResponseBody(final Response response)
629636
public static Response getResponse(final String url, final Localization localization)
630637
throws IOException, ExtractionException {
631638
final Map<String, List<String>> headers = new HashMap<>();
632-
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
633-
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
639+
addYouTubeHeaders(headers);
634640

635641
final Response response = getDownloader().get(url, headers, localization);
636642
getValidJsonResponseBody(response);
637643

638644
return response;
639645
}
640646

641-
public static String extractCookieValue(final String cookieName, final Response response) {
642-
final List<String> cookies = response.responseHeaders().get("set-cookie");
643-
int startIndex;
644-
String result = "";
645-
for (final String cookie : cookies) {
646-
startIndex = cookie.indexOf(cookieName);
647-
if (startIndex != -1) {
648-
result = cookie.substring(startIndex + cookieName.length() + "=".length(),
649-
cookie.indexOf(";", startIndex));
650-
}
651-
}
652-
return result;
653-
}
654-
655647
public static JsonArray getJsonResponse(final String url, final Localization localization)
656648
throws IOException, ExtractionException {
657649
Map<String, List<String>> headers = new HashMap<>();
658-
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
659-
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
650+
addYouTubeHeaders(headers);
651+
660652
final Response response = getDownloader().get(url, headers, localization);
661653

662654
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
@@ -665,17 +657,65 @@ public static JsonArray getJsonResponse(final String url, final Localization loc
665657
public static JsonArray getJsonResponse(final Page page, final Localization localization)
666658
throws IOException, ExtractionException {
667659
final Map<String, List<String>> headers = new HashMap<>();
668-
if (!isNullOrEmpty(page.getCookies())) {
669-
headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies())));
670-
}
671-
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
672-
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
660+
addYouTubeHeaders(headers);
673661

674662
final Response response = getDownloader().get(page.getUrl(), headers, localization);
675663

676664
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
677665
}
678666

667+
/**
668+
* Add required headers and cookies to an existing headers Map.
669+
* @see #addClientInfoHeaders(Map)
670+
* @see #addCookieHeader(Map)
671+
*/
672+
public static void addYouTubeHeaders(final Map<String, List<String>> headers)
673+
throws IOException, ExtractionException {
674+
addClientInfoHeaders(headers);
675+
addCookieHeader(headers);
676+
}
677+
678+
/**
679+
* Add the <code>X-YouTube-Client-Name</code> and <code>X-YouTube-Client-Version</code> headers.
680+
* @param headers The headers which should be completed
681+
*/
682+
public static void addClientInfoHeaders(final Map<String, List<String>> headers)
683+
throws IOException, ExtractionException {
684+
if (headers.get("X-YouTube-Client-Name") == null) {
685+
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
686+
}
687+
if (headers.get("X-YouTube-Client-Version") == null) {
688+
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
689+
}
690+
}
691+
692+
/**
693+
* Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
694+
* @see #CONSENT_COOKIE
695+
* @param headers the headers which should be completed
696+
*/
697+
public static void addCookieHeader(final Map<String, List<String>> headers) {
698+
if (headers.get("Cookie") == null) {
699+
headers.put("Cookie", Arrays.asList(CONSENT_COOKIE));
700+
} else {
701+
headers.get("Cookie").add(CONSENT_COOKIE);
702+
}
703+
}
704+
705+
public static String extractCookieValue(final String cookieName, final Response response) {
706+
final List<String> cookies = response.responseHeaders().get("set-cookie");
707+
int startIndex;
708+
String result = "";
709+
for (final String cookie : cookies) {
710+
startIndex = cookie.indexOf(cookieName);
711+
if (startIndex != -1) {
712+
result = cookie.substring(startIndex + cookieName.length() + "=".length(),
713+
cookie.indexOf(";", startIndex));
714+
}
715+
}
716+
return result;
717+
}
718+
679719
/**
680720
* Shared alert detection function, multiple endpoints return the error similarly structured.
681721
* <p>

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@
2020

2121
import java.io.IOException;
2222
import java.util.Collections;
23+
import java.util.HashMap;
2324
import java.util.List;
25+
import java.util.Map;
2426

2527
import javax.annotation.Nonnull;
2628
import javax.annotation.Nullable;
@@ -130,8 +132,11 @@ public long getStreamCount() {
130132
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
131133
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
132134
collectStreamsFrom(collector, playlistData.getArray("contents"));
133-
return new InfoItemsPage<>(collector,
134-
new Page(getNextPageUrlFrom(playlistData), Collections.singletonMap(COOKIE_NAME, cookieValue)));
135+
136+
final Map<String, String> cookies = new HashMap<>();
137+
cookies.put(COOKIE_NAME, cookieValue);
138+
139+
return new InfoItemsPage<>(collector, new Page(getNextPageUrlFrom(playlistData), cookies));
135140
}
136141

137142
private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException {

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,10 @@
1212

1313
import java.io.IOException;
1414
import java.net.URLEncoder;
15-
import java.util.ArrayList;
16-
import java.util.List;
15+
import java.util.*;
1716

17+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONSENT_COOKIE_VALUE;
18+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addCookieHeader;
1819
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
1920

2021
/*
@@ -45,17 +46,20 @@ public YoutubeSuggestionExtractor(StreamingService service) {
4546

4647
@Override
4748
public List<String> suggestionList(String query) throws IOException, ExtractionException {
48-
Downloader dl = NewPipe.getDownloader();
49-
List<String> suggestions = new ArrayList<>();
49+
final Downloader dl = NewPipe.getDownloader();
50+
final List<String> suggestions = new ArrayList<>();
5051

51-
String url = "https://suggestqueries.google.com/complete/search"
52+
final String url = "https://suggestqueries.google.com/complete/search"
5253
+ "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
5354
+ "&jsonp=" + "JP"
5455
+ "&ds=" + "yt"
5556
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8)
5657
+ "&q=" + URLEncoder.encode(query, UTF_8);
5758

58-
String response = dl.get(url, getExtractorLocalization()).responseBody();
59+
final Map<String, List<String>> headers = new HashMap<>();
60+
addCookieHeader(headers);
61+
62+
String response = dl.get(url, headers, getExtractorLocalization()).responseBody();
5963
// trim JSONP part "JP(...)"
6064
response = response.substring(3, response.length() - 1);
6165
try {

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,7 @@
2121
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
2222

2323
import java.io.IOException;
24-
import java.util.Collections;
25-
import java.util.HashSet;
26-
import java.util.Map;
27-
import java.util.Set;
24+
import java.util.*;
2825

2926
import static org.hamcrest.CoreMatchers.containsString;
3027
import static org.hamcrest.CoreMatchers.startsWith;
@@ -44,8 +41,7 @@ public class YoutubeMixPlaylistExtractorTest {
4441
private static final String VIDEO_TITLE =
4542
"Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
4643
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/";
47-
private static final Map<String, String> dummyCookie
48-
= Collections.singletonMap(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
44+
private static final Map<String, String> dummyCookie = new HashMap<>();
4945

5046
private static YoutubeMixPlaylistExtractor extractor;
5147

@@ -55,6 +51,7 @@ public static class Mix {
5551
public static void setUp() throws Exception {
5652
YoutubeParsingHelper.resetClientVersionAndKey();
5753
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mix"));
54+
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
5855
extractor = (YoutubeMixPlaylistExtractor) YouTube
5956
.getPlaylistExtractor(
6057
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID);
@@ -133,6 +130,7 @@ public static class MixWithIndex {
133130
public static void setUp() throws Exception {
134131
YoutubeParsingHelper.resetClientVersionAndKey();
135132
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mixWithIndex"));
133+
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
136134
extractor = (YoutubeMixPlaylistExtractor) YouTube
137135
.getPlaylistExtractor(
138136
"https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
@@ -203,6 +201,7 @@ public static class MyMix {
203201
public static void setUp() throws Exception {
204202
YoutubeParsingHelper.resetClientVersionAndKey();
205203
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "myMix"));
204+
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
206205
extractor = (YoutubeMixPlaylistExtractor) YouTube
207206
.getPlaylistExtractor(
208207
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM"
@@ -277,6 +276,7 @@ public static class Invalid {
277276
public static void setUp() throws IOException {
278277
YoutubeParsingHelper.resetClientVersionAndKey();
279278
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "invalid"));
279+
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
280280
}
281281

282282
@Test(expected = IllegalArgumentException.class)
@@ -309,6 +309,7 @@ public static class ChannelMix {
309309
public static void setUp() throws Exception {
310310
YoutubeParsingHelper.resetClientVersionAndKey();
311311
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "channelMix"));
312+
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
312313
extractor = (YoutubeMixPlaylistExtractor) YouTube
313314
.getPlaylistExtractor(
314315
"https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL

0 commit comments

Comments
 (0)