Skip to content

Commit 22fa131

Browse files
TobiGrTobiGr
committed
Merge branch 'dev' into bandcamp
2 parents a1688fe + cb07ffa commit 22fa131

65 files changed

Lines changed: 5029 additions & 534 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo.
1111
If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps:
1212

1313
1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`.
14-
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.10'`the `dependencies` in your `build.gradle`. Replace `v0.20.10` with the latest release.
14+
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.11'`the `dependencies` in your `build.gradle`. Replace `v0.20.11` with the latest release.
1515

1616
**Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required.
1717

‎build.gradle‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,13 @@ allprojects {
22
apply plugin: 'java-library'
33
apply plugin: 'maven'
44

5+
compileJava.options.encoding = 'UTF-8'
6+
compileTestJava.options.encoding = 'UTF-8'
7+
58
sourceCompatibility = 1.8
69
targetCompatibility = 1.8
710

8-
version 'v0.20.10'
11+
version 'v0.20.11'
912
group 'com.github.TeamNewPipe'
1013

1114
repositories {

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
import static org.schabi.newpipe.extractor.utils.Utils.*;
4141

4242
public class SoundcloudParsingHelper {
43-
private static final String HARDCODED_CLIENT_ID = "H2c34Q0E7hftqnuDHGsk88DbNqhYpgMm"; // Updated on 24/06/20
43+
private static final String HARDCODED_CLIENT_ID = "Kl9G8jQT22DxqatQk09IjWRujGlut5Vd"; // Updated on 04/03/21
4444
private static String clientId;
4545

4646
private SoundcloudParsingHelper() {
@@ -67,7 +67,7 @@ public synchronized static String clientId() throws ExtractionException, IOExcep
6767
Collections.reverse(possibleScripts);
6868

6969
final HashMap<String, List<String>> headers = new HashMap<>();
70-
headers.put("Range", singletonList("bytes=0-16384"));
70+
headers.put("Range", singletonList("bytes=0-50000"));
7171

7272
for (Element element : possibleScripts) {
7373
final String srcUrl = element.attr("src");

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

3-
import com.grack.nanojson.*;
3+
import com.grack.nanojson.JsonArray;
4+
import com.grack.nanojson.JsonObject;
5+
import com.grack.nanojson.JsonParser;
6+
import com.grack.nanojson.JsonParserException;
7+
import com.grack.nanojson.JsonWriter;
8+
49
import org.schabi.newpipe.extractor.MetaInfo;
510
import org.schabi.newpipe.extractor.Page;
611
import org.schabi.newpipe.extractor.downloader.Response;
@@ -10,6 +15,7 @@
1015
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
1116
import org.schabi.newpipe.extractor.localization.Localization;
1217
import org.schabi.newpipe.extractor.stream.Description;
18+
import org.schabi.newpipe.extractor.utils.JsonUtils;
1319
import org.schabi.newpipe.extractor.utils.Parser;
1420
import org.schabi.newpipe.extractor.utils.Utils;
1521

@@ -33,7 +39,12 @@
3339
import javax.annotation.Nullable;
3440

3541
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
36-
import static org.schabi.newpipe.extractor.utils.Utils.*;
42+
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
43+
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
44+
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
45+
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
46+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
47+
import static org.schabi.newpipe.extractor.utils.Utils.join;
3748

3849
/*
3950
* Created by Christian Schabesberger on 02.03.16.
@@ -638,7 +649,7 @@ public static JsonArray getJsonResponse(final String url, final Localization loc
638649
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
639650
final Response response = getDownloader().get(url, headers, localization);
640651

641-
return toJsonArray(getValidJsonResponseBody(response));
652+
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
642653
}
643654

644655
public static JsonArray getJsonResponse(final Page page, final Localization localization)
@@ -652,15 +663,7 @@ public static JsonArray getJsonResponse(final Page page, final Localization loca
652663

653664
final Response response = getDownloader().get(page.getUrl(), headers, localization);
654665

655-
return toJsonArray(getValidJsonResponseBody(response));
656-
}
657-
658-
public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
659-
try {
660-
return JsonParser.array().from(responseBody);
661-
} catch (JsonParserException e) {
662-
throw new ParsingException("Could not parse JSON", e);
663-
}
666+
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
664667
}
665668

666669
/**
@@ -821,4 +824,14 @@ public static boolean isVerified(final JsonArray badges) {
821824

822825
return false;
823826
}
827+
828+
public static String unescapeDocument(final String doc) {
829+
return doc
830+
.replaceAll("\\\\x22", "\"")
831+
.replaceAll("\\\\x7b", "{")
832+
.replaceAll("\\\\x7d", "}")
833+
.replaceAll("\\\\x5b", "[")
834+
.replaceAll("\\\\x5d", "]");
835+
}
836+
824837
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 36 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,13 @@
22

33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonObject;
5+
import com.grack.nanojson.JsonWriter;
6+
57
import org.schabi.newpipe.extractor.Page;
68
import org.schabi.newpipe.extractor.StreamingService;
79
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
810
import org.schabi.newpipe.extractor.downloader.Downloader;
11+
import org.schabi.newpipe.extractor.downloader.Response;
912
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
1013
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
1114
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -15,13 +18,20 @@
1518
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
1619
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
1720
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
21+
import org.schabi.newpipe.extractor.utils.JsonUtils;
1822
import org.schabi.newpipe.extractor.utils.Utils;
1923

2024
import javax.annotation.Nonnull;
2125
import java.io.IOException;
2226

23-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
27+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
28+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
29+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
30+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
31+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
32+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
2433
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
34+
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
2535
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
2636

2737
/*
@@ -226,7 +236,7 @@ public boolean isVerified() throws ParsingException {
226236

227237
@Nonnull
228238
@Override
229-
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
239+
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
230240
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
231241

232242
Page nextPage = null;
@@ -254,27 +264,44 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
254264
// as they don't deliver enough information on their own (the channel name, for example).
255265
fetchPage();
256266

267+
// @formatter:off
268+
byte[] json = JsonWriter.string()
269+
.object()
270+
.object("context")
271+
.object("client")
272+
.value("clientName", "1")
273+
.value("clientVersion", getClientVersion())
274+
.end()
275+
.end()
276+
.value("continuation", page.getId())
277+
.end()
278+
.done()
279+
.getBytes(UTF_8);
280+
// @formatter:on
281+
257282
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
258-
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
283+
final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
284+
285+
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
259286

260-
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
261-
.getArray("onResponseReceivedActions").getObject(0).getObject("appendContinuationItemsAction");
287+
JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
288+
.getObject(0)
289+
.getObject("appendContinuationItemsAction");
262290

263291
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation.getArray("continuationItems"));
264292

265293
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
266294
}
267295

268-
private Page getNextPageFrom(final JsonObject continuations) {
296+
private Page getNextPageFrom(final JsonObject continuations) throws IOException, ExtractionException {
269297
if (isNullOrEmpty(continuations)) {
270298
return null;
271299
}
272300

273301
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
274302
final String continuation = continuationEndpoint.getObject("continuationCommand").getString("token");
275-
final String clickTrackingParams = continuationEndpoint.getString("clickTrackingParams");
276-
return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation
277-
+ "&continuation=" + continuation + "&itct=" + clickTrackingParams);
303+
return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
304+
continuation);
278305
}
279306

280307
/**

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1616
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
1717
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
18+
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
1819
import org.schabi.newpipe.extractor.utils.JsonUtils;
1920
import org.schabi.newpipe.extractor.utils.Parser;
2021

@@ -46,7 +47,10 @@ public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHand
4647

4748
@Override
4849
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
49-
final String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
50+
String commentsTokenInside = findValue(responseBody, "sectionListRenderer", "}");
51+
if (!commentsTokenInside.contains("continuation\":\"")) {
52+
commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
53+
}
5054
final String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\"");
5155
return getPage(getNextPage(commentsToken));
5256
}
@@ -128,7 +132,7 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
128132
final Map<String, List<String>> requestHeaders = new HashMap<>();
129133
requestHeaders.put("User-Agent", singletonList(USER_AGENT));
130134
final Response response = downloader.get(getUrl(), requestHeaders, getExtractorLocalization());
131-
responseBody = response.responseBody();
135+
responseBody = YoutubeParsingHelper.unescapeDocument(response.responseBody());
132136
ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\"");
133137
ytClientName = Parser.matchGroup1(YT_CLIENT_NAME_PATTERN, responseBody);
134138
}
@@ -158,16 +162,9 @@ private String getDataString(Map<String, String> params) throws UnsupportedEncod
158162
return result.toString();
159163
}
160164

161-
private String findValue(String doc, String start, String end) {
162-
final String unescaped = doc
163-
.replaceAll("\\\\x22", "\"")
164-
.replaceAll("\\\\x7b", "{")
165-
.replaceAll("\\\\x7d", "}")
166-
.replaceAll("\\\\x5b", "[")
167-
.replaceAll("\\\\x5d", "]");
168-
169-
final int beginIndex = unescaped.indexOf(start) + start.length();
170-
final int endIndex = unescaped.indexOf(end, beginIndex);
171-
return unescaped.substring(beginIndex, endIndex);
165+
private String findValue(final String doc, final String start, final String end) {
166+
final int beginIndex = doc.indexOf(start) + start.length();
167+
final int endIndex = doc.indexOf(end, beginIndex);
168+
return doc.substring(beginIndex, endIndex);
172169
}
173170
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonObject;
5+
56
import org.schabi.newpipe.extractor.ListExtractor;
67
import org.schabi.newpipe.extractor.Page;
78
import org.schabi.newpipe.extractor.StreamingService;
@@ -14,14 +15,19 @@
1415
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
1516
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
1617
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
18+
import org.schabi.newpipe.extractor.utils.JsonUtils;
1719

18-
import javax.annotation.Nonnull;
19-
import javax.annotation.Nullable;
2020
import java.io.IOException;
2121
import java.util.Collections;
2222
import java.util.List;
2323

24-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
24+
import javax.annotation.Nonnull;
25+
import javax.annotation.Nullable;
26+
27+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
28+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
29+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse;
30+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
2531
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
2632

2733
/**
@@ -51,7 +57,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
5157
throws IOException, ExtractionException {
5258
final String url = getUrl() + "&pbj=1";
5359
final Response response = getResponse(url, getExtractorLocalization());
54-
final JsonArray ajaxJson = toJsonArray(response.responseBody());
60+
final JsonArray ajaxJson = JsonUtils.toJsonArray(response.responseBody());
5561
initialData = ajaxJson.getObject(3).getObject("response");
5662
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
5763
.getObject("playlist").getObject("playlist");

0 commit comments

Comments
 (0)