Skip to content

Commit 39a911d

Browse files
TobiGrTobiGr
authored
Merge pull request #1084 from AudricV/yt_android-403s-workaround-and-streams-tests-fixes
[YouTube] Workaround again 403 HTTP issues on the ANDROID InnerTube client and fix stream tests
2 parents 8ebbe1b + 522c781 commit 39a911d

110 files changed

Lines changed: 1285 additions & 1148 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -981,11 +981,9 @@ private void fetchAndroidMobileJsonPlayer(@Nonnull final ContentCountry contentC
981981
.value(CPN, androidCpn)
982982
.value(CONTENT_CHECK_OK, true)
983983
.value(RACY_CHECK_OK, true)
984-
// Workaround getting streaming URLs which can return 403 HTTP response
985-
// codes by using stories parameter for Android client requests
986-
// This behavior only happen in certain countries such as UK as of
987-
// 10.29.2022
988-
.value("params", "8AEB")
984+
// Workaround getting streaming URLs which return 403 HTTP response code by
985+
// using some parameters for Android client requests
986+
.value("params", "CgIQBg")
989987
.done())
990988
.getBytes(StandardCharsets.UTF_8);
991989

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java‎

Lines changed: 21 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -380,7 +380,7 @@ public void testTags() {}
380380
}
381381

382382
public static class PublicBroadcasterTest extends DefaultStreamExtractorTest {
383-
private static final String ID = "q6fgbYWsMgw";
383+
private static final String ID = "cJ9to6EmElQ";
384384
private static final int TIMESTAMP = 0;
385385
private static final String URL = BASE_URL + ID;
386386
private static StreamExtractor extractor;
@@ -396,41 +396,39 @@ public static void setUp() throws Exception {
396396
// @formatter:off
397397
@Override public StreamExtractor extractor() { return extractor; }
398398
@Override public StreamingService expectedService() { return YouTube; }
399-
@Override public String expectedName() { return "Was verbirgt sich am tiefsten Punkt des Ozeans?"; }
399+
@Override public String expectedName() { return "Merci pour les 3 millions d'abonnés \uD83C\uDF89| ARTE"; }
400400
@Override public String expectedId() { return ID; }
401401
@Override public String expectedUrlContains() { return BASE_URL + ID; }
402402
@Override public String expectedOriginalUrlContains() { return URL; }
403403

404404
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
405-
@Override public String expectedUploaderName() { return "Dinge Erklärt – Kurzgesagt"; }
406-
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCwRH985XgMYXQ6NxXDo8npw"; }
407-
@Override public long expectedUploaderSubscriberCountAtLeast() { return 1_500_000; }
408-
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("Lasst uns abtauchen!", "Angebot von funk", "Dinge"); }
409-
@Override public long expectedLength() { return 631; }
405+
@Override public String expectedUploaderName() { return "ARTE"; }
406+
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCwI-JbGNsojunnHbFAc0M4Q"; }
407+
@Override public long expectedUploaderSubscriberCountAtLeast() { return 3_000_000; }
408+
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("sommets", "fans", "cadeau"); }
409+
@Override public long expectedLength() { return 45; }
410410
@Override public long expectedTimestamp() { return TIMESTAMP; }
411-
@Override public long expectedViewCountAtLeast() { return 1_600_000; }
412-
@Nullable @Override public String expectedUploadDate() { return "2019-06-12 00:00:00.000"; }
413-
@Nullable @Override public String expectedTextualUploadDate() { return "2019-06-12"; }
414-
@Override public long expectedLikeCountAtLeast() { return 70000; }
411+
@Override public long expectedViewCountAtLeast() { return 20_000; }
412+
@Nullable @Override public String expectedUploadDate() { return "2023-07-07 00:00:00.000"; }
413+
@Nullable @Override public String expectedTextualUploadDate() { return "2023-07-07"; }
414+
@Override public long expectedLikeCountAtLeast() { return 1000; }
415415
@Override public long expectedDislikeCountAtLeast() { return -1; }
416416
@Override public List<MetaInfo> expectedMetaInfo() throws MalformedURLException {
417417
return Collections.singletonList(new MetaInfo(
418418
"",
419-
new Description("Funk is a German public broadcast service.", Description.PLAIN_TEXT),
420-
Collections.singletonList(new URL("https://de.wikipedia.org/wiki/Funk_(Medienangebot)?wprov=yicw1")),
421-
Collections.singletonList("Wikipedia (German)")
419+
new Description("Arte is a French/German public broadcast service.",
420+
Description.PLAIN_TEXT),
421+
List.of(new URL(
422+
"https://en.wikipedia.org/wiki/Arte?wprov=yicw1")),
423+
List.of("Wikipedia")
422424
));
423425
}
424426
@Override public boolean expectedUploaderVerified() { return true; }
425427
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
426-
@Override public String expectedCategory() { return "Education"; }
428+
@Override public String expectedCategory() { return "News & Politics"; }
427429
@Override public List<String> expectedTags() {
428-
return Arrays.asList("Abgrund", "Algen", "Bakterien", "Challengertief", "Dumbooktopus",
429-
"Dunkel", "Dunkelheit", "Fische", "Flohkrebs", "Hadal-Zone", "Kontinentalschelf",
430-
"Licht", "Mariannengraben", "Meer", "Meeresbewohner", "Meeresschnee", "Mesopelagial",
431-
"Ozean", "Photosynthese", "Plankton", "Plastik", "Polypen", "Pottwale",
432-
"Staatsquelle", "Tauchen", "Tauchgang", "Tentakel", "Tiefe", "Tiefsee", "Tintenfische",
433-
"Titanic", "Vampirtintenfisch", "Verschmutzung", "Viperfisch", "Wale");
430+
return Arrays.asList("arte", "arte 3 millions", "arte remerciement",
431+
"documentaire arte", "arte documentaire", "fan d'arte", "arte youtube");
434432
}
435433
// @formatter:on
436434
}
@@ -559,7 +557,8 @@ void testCheckAudioStreams() throws Exception {
559557
}
560558

561559
assertTrue(audioStreams.stream()
562-
.anyMatch(audioStream -> "English".equals(audioStream.getAudioTrackName())));
560+
.anyMatch(audioStream ->
561+
"English original".equals(audioStream.getAudioTrackName())));
563562

564563
final Locale hindiLocale = LocaleCompat.forLanguageTag("hi");
565564
assertTrue(audioStreams.stream()

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public void testUploaderName() throws Exception {
4040

4141
@Override public StreamExtractor extractor() { return extractor; }
4242
@Override public StreamingService expectedService() { return YouTube; }
43-
@Override public String expectedName() { return "lofi hip hop radio - beats to relax/study to"; }
43+
@Override public String expectedName() { return "lofi hip hop radio \uD83D\uDCDA - beats to relax/study to"; }
4444
@Override public String expectedId() { return ID; }
4545
@Override public String expectedUrlContains() { return YoutubeStreamExtractorDefaultTest.BASE_URL + ID; }
4646
@Override public String expectedOriginalUrlContains() { return URL; }

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/stream/ageRestricted/generated_mock_0.json‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,15 @@
1717
"responseMessage": "",
1818
"responseHeaders": {
1919
"alt-svc": [
20-
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
20+
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
2121
],
2222
"cache-control": [
2323
"private, max-age\u003d0"
2424
],
25+
"content-security-policy-report-only": [
26+
"base-uri \u0027self\u0027;default-src \u0027self\u0027 https: blob:;font-src https: data:;img-src https: data: android-webview-video-poster:;media-src blob: https:;object-src \u0027none\u0027;script-src \u0027nonce-YdAQShUg0To_xhoSabl-vQ\u0027 \u0027unsafe-inline\u0027 \u0027strict-dynamic\u0027 https: http: \u0027unsafe-eval\u0027;style-src https: \u0027unsafe-inline\u0027;report-uri /cspreport",
27+
"require-trusted-types-for \u0027script\u0027;report-uri /cspreport"
28+
],
2529
"content-type": [
2630
"text/javascript; charset\u003dutf-8"
2731
],
@@ -32,16 +36,19 @@
3236
"cross-origin"
3337
],
3438
"date": [
35-
"Tue, 22 Nov 2022 10:41:17 GMT"
39+
"Sat, 22 Jul 2023 18:24:10 GMT"
3640
],
3741
"expires": [
38-
"Tue, 22 Nov 2022 10:41:17 GMT"
42+
"Sat, 22 Jul 2023 18:24:10 GMT"
43+
],
44+
"origin-trial": [
45+
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
3946
],
4047
"p3p": [
4148
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
4249
],
4350
"permissions-policy": [
44-
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
51+
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-form-factor\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
4552
],
4653
"report-to": [
4754
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
@@ -50,9 +57,9 @@
5057
"ESF"
5158
],
5259
"set-cookie": [
53-
"YSC\u003dARiIxvc7aiA; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
54-
"VISITOR_INFO1_LIVE\u003d2gvEtKcFruk; Domain\u003d.youtube.com; Expires\u003dSun, 21-May-2023 10:41:17 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
55-
"CONSENT\u003dPENDING+076; expires\u003dThu, 21-Nov-2024 10:41:17 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
60+
"YSC\u003dRBncugXb0cY; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
61+
"VISITOR_INFO1_LIVE\u003dF1G_Uux5N7o; Domain\u003d.youtube.com; Expires\u003dThu, 18-Jan-2024 18:24:10 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
62+
"CONSENT\u003dPENDING+446; expires\u003dMon, 21-Jul-2025 18:24:10 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
5663
],
5764
"strict-transport-security": [
5865
"max-age\u003d31536000"
@@ -67,7 +74,7 @@
6774
"0"
6875
]
6976
},
70-
"responseBody": "var scriptUrl \u003d \u0027https:\\/\\/www.youtube.com\\/s\\/player\\/041a7965\\/www-widgetapi.vflset\\/www-widgetapi.js\u0027;window[\u0027yt_embedsEnableIframeSrcWithIntent\u0027] \u003d true ;try{var ttPolicy\u003dwindow.trustedTypes.createPolicy(\"youtube-widget-api\",{createScriptURL:function(x){return x}});scriptUrl\u003dttPolicy.createScriptURL(scriptUrl)}catch(e){}var YT;if(!window[\"YT\"])YT\u003d{loading:0,loaded:0};var YTConfig;if(!window[\"YTConfig\"])YTConfig\u003d{\"host\":\"https://www.youtube.com\"};\nif(!YT.loading){YT.loading\u003d1;(function(){var l\u003d[];YT.ready\u003dfunction(f){if(YT.loaded)f();else l.push(f)};window.onYTReady\u003dfunction(){YT.loaded\u003d1;for(var i\u003d0;i\u003cl.length;i++)try{l[i]()}catch(e$0){}};YT.setConfig\u003dfunction(c){for(var k in c)if(c.hasOwnProperty(k))YTConfig[k]\u003dc[k]};var a\u003ddocument.createElement(\"script\");a.type\u003d\"text/javascript\";a.id\u003d\"www-widgetapi-script\";a.src\u003dscriptUrl;a.async\u003dtrue;var c\u003ddocument.currentScript;if(c){var n\u003dc.nonce||c.getAttribute(\"nonce\");if(n)a.setAttribute(\"nonce\",n)}var b\u003d\ndocument.getElementsByTagName(\"script\")[0];b.parentNode.insertBefore(a,b)})()};\n",
77+
"responseBody": "var scriptUrl \u003d \u0027https:\\/\\/www.youtube.com\\/s\\/player\\/8e83803a\\/www-widgetapi.vflset\\/www-widgetapi.js\u0027;try{var ttPolicy\u003dwindow.trustedTypes.createPolicy(\"youtube-widget-api\",{createScriptURL:function(x){return x}});scriptUrl\u003dttPolicy.createScriptURL(scriptUrl)}catch(e){}var YT;if(!window[\"YT\"])YT\u003d{loading:0,loaded:0};var YTConfig;if(!window[\"YTConfig\"])YTConfig\u003d{\"host\":\"https://www.youtube.com\"};\nif(!YT.loading){YT.loading\u003d1;(function(){var l\u003d[];YT.ready\u003dfunction(f){if(YT.loaded)f();else l.push(f)};window.onYTReady\u003dfunction(){YT.loaded\u003d1;var i\u003d0;for(;i\u003cl.length;i++)try{l[i]()}catch(e$1271022551$0){}};YT.setConfig\u003dfunction(c){var k;for(k in c)if(c.hasOwnProperty(k))YTConfig[k]\u003dc[k]};var a\u003ddocument.createElement(\"script\");a.type\u003d\"text/javascript\";a.id\u003d\"www-widgetapi-script\";a.src\u003dscriptUrl;a.async\u003dtrue;var c\u003ddocument.currentScript;if(c){var n\u003dc.nonce||c.getAttribute(\"nonce\");if(n)a.setAttribute(\"nonce\",\nn)}var b\u003ddocument.getElementsByTagName(\"script\")[0];b.parentNode.insertBefore(a,b)})()};\n",
7178
"latestUrl": "https://www.youtube.com/iframe_api"
7279
}
7380
}

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/stream/ageRestricted/generated_mock_1.json‎

Lines changed: 8 additions & 11 deletions
Large diffs are not rendered by default.

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/stream/ageRestricted/generated_mock_2.json‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,28 +29,31 @@
2929
"https://www.youtube.com"
3030
],
3131
"alt-svc": [
32-
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
32+
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
3333
],
3434
"cache-control": [
3535
"private, max-age\u003d0"
3636
],
3737
"content-type": [
3838
"text/javascript; charset\u003dutf-8"
3939
],
40-
"cross-origin-opener-policy-report-only": [
40+
"cross-origin-opener-policy": [
4141
"same-origin; report-to\u003d\"youtube_main\""
4242
],
4343
"date": [
44-
"Tue, 22 Nov 2022 10:41:18 GMT"
44+
"Sat, 22 Jul 2023 18:24:11 GMT"
4545
],
4646
"expires": [
47-
"Tue, 22 Nov 2022 10:41:18 GMT"
47+
"Sat, 22 Jul 2023 18:24:11 GMT"
48+
],
49+
"origin-trial": [
50+
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
4851
],
4952
"p3p": [
5053
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
5154
],
5255
"permissions-policy": [
53-
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
56+
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-form-factor\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
5457
],
5558
"report-to": [
5659
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
@@ -59,9 +62,9 @@
5962
"ESF"
6063
],
6164
"set-cookie": [
62-
"YSC\u003dqOvGL5EH0kk; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
63-
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 26-Feb-2020 10:41:18 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
64-
"CONSENT\u003dPENDING+190; expires\u003dThu, 21-Nov-2024 10:41:18 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
65+
"YSC\u003dKzlu5knnLDg; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
66+
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSun, 25-Oct-2020 18:24:11 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
67+
"CONSENT\u003dPENDING+519; expires\u003dMon, 21-Jul-2025 18:24:11 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
6568
],
6669
"strict-transport-security": [
6770
"max-age\u003d31536000"

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/stream/ageRestricted/generated_mock_3.json‎

Lines changed: 10 additions & 7 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)