Skip to content

Commit 55e0f8e

Browse files
TobiGrTobiGr
committed
Merge branch 'master' into dev
2 parents eafad3a + e747a89 commit 55e0f8e

190 files changed

Lines changed: 4946 additions & 5357 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Request.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -123,12 +123,10 @@ public Builder url(String url) {
123123
* Any default headers that the implementation may have, <b>should</b> be overridden by these.
124124
*/
125125
public Builder headers(@Nullable Map<String, List<String>> headers) {
126-
if (headers == null) {
127-
this.headers.clear();
128-
return this;
129-
}
130126
this.headers.clear();
131-
this.headers.putAll(headers);
127+
if (headers != null) {
128+
this.headers.putAll(headers);
129+
}
132130
return this;
133131
}
134132

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 84 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,7 @@
2929
import java.time.OffsetDateTime;
3030
import java.time.ZoneOffset;
3131
import java.time.format.DateTimeParseException;
32-
import java.util.ArrayList;
33-
import java.util.Collections;
34-
import java.util.HashMap;
35-
import java.util.List;
36-
import java.util.Map;
37-
import java.util.Objects;
32+
import java.util.*;
3833

3934
import javax.annotation.Nonnull;
4035
import javax.annotation.Nullable;
@@ -80,6 +75,19 @@ private YoutubeParsingHelper() {
8075
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
8176
private static String[] youtubeMusicKeys;
8277

78+
private static Random numberGenerator = new Random();
79+
80+
/**
81+
* <code>PENDING+</code> means that the user did not yet submit their choices.
82+
* Therefore, YouTube & Google should not track the user, because they did not give consent.
83+
* The three digits at the end can be random, but are required.
84+
*/
85+
private static final String CONSENT_COOKIE_VALUE = "PENDING+";
86+
/**
87+
* Youtube <code>CONSENT</code> cookie. Should prevent redirect to consent.youtube.com
88+
*/
89+
private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
90+
8391
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
8492
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
8593

@@ -389,6 +397,15 @@ public static void resetClientVersionAndKey() {
389397
key = null;
390398
}
391399

400+
/**
401+
* <p>
402+
* <b>Only use in tests.</b>
403+
* </p>
404+
*/
405+
public static void setNumberGenerator(Random random) {
406+
numberGenerator = random;
407+
}
408+
392409
public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException {
393410
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + HARDCODED_YOUTUBE_MUSIC_KEYS[0];
394411

@@ -428,6 +445,7 @@ public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, Re
428445
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
429446
headers.put("Referer", Collections.singletonList("music.youtube.com"));
430447
headers.put("Content-Type", Collections.singletonList("application/json"));
448+
addCookieHeader(headers);
431449

432450
final String response = getDownloader().post(url, headers, json).responseBody();
433451

@@ -630,34 +648,19 @@ public static String getValidJsonResponseBody(final Response response)
630648
public static Response getResponse(final String url, final Localization localization)
631649
throws IOException, ExtractionException {
632650
final Map<String, List<String>> headers = new HashMap<>();
633-
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
634-
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
651+
addYouTubeHeaders(headers);
635652

636653
final Response response = getDownloader().get(url, headers, localization);
637654
getValidJsonResponseBody(response);
638655

639656
return response;
640657
}
641658

642-
public static String extractCookieValue(final String cookieName, final Response response) {
643-
final List<String> cookies = response.responseHeaders().get("set-cookie");
644-
int startIndex;
645-
String result = "";
646-
for (final String cookie : cookies) {
647-
startIndex = cookie.indexOf(cookieName);
648-
if (startIndex != -1) {
649-
result = cookie.substring(startIndex + cookieName.length() + "=".length(),
650-
cookie.indexOf(";", startIndex));
651-
}
652-
}
653-
return result;
654-
}
655-
656659
public static JsonArray getJsonResponse(final String url, final Localization localization)
657660
throws IOException, ExtractionException {
658661
Map<String, List<String>> headers = new HashMap<>();
659-
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
660-
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
662+
addYouTubeHeaders(headers);
663+
661664
final Response response = getDownloader().get(url, headers, localization);
662665

663666
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
@@ -666,11 +669,7 @@ public static JsonArray getJsonResponse(final String url, final Localization loc
666669
public static JsonArray getJsonResponse(final Page page, final Localization localization)
667670
throws IOException, ExtractionException {
668671
final Map<String, List<String>> headers = new HashMap<>();
669-
if (!isNullOrEmpty(page.getCookies())) {
670-
headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies())));
671-
}
672-
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
673-
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
672+
addYouTubeHeaders(headers);
674673

675674
final Response response = getDownloader().get(page.getUrl(), headers, localization);
676675

@@ -690,6 +689,62 @@ public static JsonBuilder<JsonObject> prepareJsonBuilder()
690689
// @formatter:on
691690
}
692691

692+
/**
693+
* Add required headers and cookies to an existing headers Map.
694+
* @see #addClientInfoHeaders(Map)
695+
* @see #addCookieHeader(Map)
696+
*/
697+
public static void addYouTubeHeaders(final Map<String, List<String>> headers)
698+
throws IOException, ExtractionException {
699+
addClientInfoHeaders(headers);
700+
addCookieHeader(headers);
701+
}
702+
703+
/**
704+
* Add the <code>X-YouTube-Client-Name</code> and <code>X-YouTube-Client-Version</code> headers.
705+
* @param headers The headers which should be completed
706+
*/
707+
public static void addClientInfoHeaders(final Map<String, List<String>> headers)
708+
throws IOException, ExtractionException {
709+
if (headers.get("X-YouTube-Client-Name") == null) {
710+
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
711+
}
712+
if (headers.get("X-YouTube-Client-Version") == null) {
713+
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
714+
}
715+
}
716+
717+
/**
718+
* Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
719+
* @see #CONSENT_COOKIE
720+
* @param headers the headers which should be completed
721+
*/
722+
public static void addCookieHeader(final Map<String, List<String>> headers) {
723+
if (headers.get("Cookie") == null) {
724+
headers.put("Cookie", Arrays.asList(generateConsentCookie()));
725+
} else {
726+
headers.get("Cookie").add(generateConsentCookie());
727+
}
728+
}
729+
730+
public static String generateConsentCookie() {
731+
return CONSENT_COOKIE + 100 + numberGenerator.nextInt(900);
732+
}
733+
734+
public static String extractCookieValue(final String cookieName, final Response response) {
735+
final List<String> cookies = response.responseHeaders().get("set-cookie");
736+
int startIndex;
737+
String result = "";
738+
for (final String cookie : cookies) {
739+
startIndex = cookie.indexOf(cookieName);
740+
if (startIndex != -1) {
741+
result = cookie.substring(startIndex + cookieName.length() + "=".length(),
742+
cookie.indexOf(";", startIndex));
743+
}
744+
}
745+
return result;
746+
}
747+
693748
/**
694749
* Shared alert detection function, multiple endpoints return the error similarly structured.
695750
* <p>

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@
2020

2121
import java.io.IOException;
2222
import java.util.Collections;
23+
import java.util.HashMap;
2324
import java.util.List;
25+
import java.util.Map;
2426

2527
import javax.annotation.Nonnull;
2628
import javax.annotation.Nullable;
@@ -130,8 +132,11 @@ public long getStreamCount() {
130132
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
131133
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
132134
collectStreamsFrom(collector, playlistData.getArray("contents"));
133-
return new InfoItemsPage<>(collector,
134-
new Page(getNextPageUrlFrom(playlistData), Collections.singletonMap(COOKIE_NAME, cookieValue)));
135+
136+
final Map<String, String> cookies = new HashMap<>();
137+
cookies.put(COOKIE_NAME, cookieValue);
138+
139+
return new InfoItemsPage<>(collector, new Page(getNextPageUrlFrom(playlistData), cookies));
135140
}
136141

137142
private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException {

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@
1212

1313
import java.io.IOException;
1414
import java.net.URLEncoder;
15-
import java.util.ArrayList;
16-
import java.util.List;
15+
import java.util.*;
1716

17+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addCookieHeader;
1818
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
1919

2020
/*
@@ -45,17 +45,20 @@ public YoutubeSuggestionExtractor(StreamingService service) {
4545

4646
@Override
4747
public List<String> suggestionList(String query) throws IOException, ExtractionException {
48-
Downloader dl = NewPipe.getDownloader();
49-
List<String> suggestions = new ArrayList<>();
48+
final Downloader dl = NewPipe.getDownloader();
49+
final List<String> suggestions = new ArrayList<>();
5050

51-
String url = "https://suggestqueries.google.com/complete/search"
51+
final String url = "https://suggestqueries.google.com/complete/search"
5252
+ "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
5353
+ "&jsonp=" + "JP"
5454
+ "&ds=" + "yt"
5555
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8)
5656
+ "&q=" + URLEncoder.encode(query, UTF_8);
5757

58-
String response = dl.get(url, getExtractorLocalization()).responseBody();
58+
final Map<String, List<String>> headers = new HashMap<>();
59+
addCookieHeader(headers);
60+
61+
String response = dl.get(url, headers, getExtractorLocalization()).responseBody();
5962
// trim JSONP part "JP(...)"
6063
response = response.substring(3, response.length() - 1);
6164
try {

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
1414

1515
import java.io.IOException;
16+
import java.util.Random;
1617

1718
import static org.hamcrest.CoreMatchers.containsString;
1819
import static org.hamcrest.MatcherAssert.assertThat;
@@ -32,6 +33,7 @@ public static class NotAvailable {
3233
@BeforeClass
3334
public static void setUp() throws IOException {
3435
YoutubeParsingHelper.resetClientVersionAndKey();
36+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
3537
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
3638
}
3739

@@ -54,6 +56,7 @@ public static class NotSupported {
5456
@BeforeClass
5557
public static void setUp() throws IOException {
5658
YoutubeParsingHelper.resetClientVersionAndKey();
59+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
5760
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notSupported"));
5861
}
5962

@@ -71,6 +74,7 @@ public static class Gronkh implements BaseChannelExtractorTest {
7174
@BeforeClass
7275
public static void setUp() throws Exception {
7376
YoutubeParsingHelper.resetClientVersionAndKey();
77+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
7478
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "gronkh"));
7579
extractor = (YoutubeChannelExtractor) YouTube
7680
.getChannelExtractor("http://www.youtube.com/user/Gronkh");
@@ -168,6 +172,7 @@ public static class VSauce implements BaseChannelExtractorTest {
168172
@BeforeClass
169173
public static void setUp() throws Exception {
170174
YoutubeParsingHelper.resetClientVersionAndKey();
175+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
171176
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "VSauce"));
172177
extractor = (YoutubeChannelExtractor) YouTube
173178
.getChannelExtractor("https://www.youtube.com/user/Vsauce");
@@ -265,6 +270,7 @@ public static class Kurzgesagt implements BaseChannelExtractorTest {
265270
@BeforeClass
266271
public static void setUp() throws Exception {
267272
YoutubeParsingHelper.resetClientVersionAndKey();
273+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
268274
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "kurzgesagt"));
269275
extractor = (YoutubeChannelExtractor) YouTube
270276
.getChannelExtractor("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q");
@@ -383,6 +389,7 @@ public static class CaptainDisillusion implements BaseChannelExtractorTest {
383389
@BeforeClass
384390
public static void setUp() throws Exception {
385391
YoutubeParsingHelper.resetClientVersionAndKey();
392+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
386393
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "captainDisillusion"));
387394
extractor = (YoutubeChannelExtractor) YouTube
388395
.getChannelExtractor("https://www.youtube.com/user/CaptainDisillusion/videos");
@@ -478,6 +485,7 @@ public static class RandomChannel implements BaseChannelExtractorTest {
478485
@BeforeClass
479486
public static void setUp() throws Exception {
480487
YoutubeParsingHelper.resetClientVersionAndKey();
488+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
481489
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "random"));
482490
extractor = (YoutubeChannelExtractor) YouTube
483491
.getChannelExtractor("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w");

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import java.util.LinkedHashMap;
1515
import java.util.List;
1616
import java.util.Map;
17+
import java.util.Random;
1718

1819
import static org.junit.Assert.fail;
1920
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@@ -30,6 +31,7 @@ public class YoutubeChannelLocalizationTest {
3031
@Test
3132
public void testAllSupportedLocalizations() throws Exception {
3233
YoutubeParsingHelper.resetClientVersionAndKey();
34+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
3335
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "localization"));
3436

3537
testLocalizationsFor("https://www.youtube.com/user/NBCNews");

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import java.io.IOException;
1717
import java.util.List;
18+
import java.util.Random;
1819

1920
import static org.junit.Assert.assertEquals;
2021
import static org.junit.Assert.assertFalse;
@@ -36,6 +37,7 @@ public static class Thomas {
3637
@BeforeClass
3738
public static void setUp() throws Exception {
3839
YoutubeParsingHelper.resetClientVersionAndKey();
40+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
3941
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "thomas"));
4042
extractor = (YoutubeCommentsExtractor) YouTube
4143
.getCommentsExtractor(url);
@@ -124,6 +126,7 @@ public static class EmptyComment {
124126
@BeforeClass
125127
public static void setUp() throws Exception {
126128
YoutubeParsingHelper.resetClientVersionAndKey();
129+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
127130
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "empty"));
128131
extractor = (YoutubeCommentsExtractor) YouTube
129132
.getCommentsExtractor(url);
@@ -163,6 +166,7 @@ public static class HeartedByCreator {
163166
@BeforeClass
164167
public static void setUp() throws Exception {
165168
YoutubeParsingHelper.resetClientVersionAndKey();
169+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
166170
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "hearted"));
167171
extractor = (YoutubeCommentsExtractor) YouTube
168172
.getCommentsExtractor(url);
@@ -205,6 +209,7 @@ public static class Pinned {
205209
@BeforeClass
206210
public static void setUp() throws Exception {
207211
YoutubeParsingHelper.resetClientVersionAndKey();
212+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
208213
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "pinned"));
209214
extractor = (YoutubeCommentsExtractor) YouTube
210215
.getCommentsExtractor(url);

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
99
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
1010

11+
import java.util.Random;
12+
1113
import static org.junit.Assert.assertEquals;
1214
import static org.junit.Assert.assertTrue;
1315
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@@ -24,6 +26,7 @@ public static class Kurzgesagt implements BaseListExtractorTest {
2426
@BeforeClass
2527
public static void setUp() throws Exception {
2628
YoutubeParsingHelper.resetClientVersionAndKey();
29+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
2730
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH));
2831
extractor = (YoutubeFeedExtractor) YouTube
2932
.getFeedExtractor("https://www.youtube.com/user/Kurzgesagt");

0 commit comments

Comments
 (0)