Skip to content

Commit 9256b3b

Browse files
XiangRongLinXiangRongLin
authored
Merge pull request #567 from XiangRongLin/playlist_continuations
Playlist continuations
2 parents 5d594cf + 506cc5f commit 9256b3b

25 files changed

Lines changed: 1847 additions & 271 deletions

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

3-
import com.grack.nanojson.*;
3+
import com.grack.nanojson.JsonArray;
4+
import com.grack.nanojson.JsonObject;
5+
import com.grack.nanojson.JsonParser;
6+
import com.grack.nanojson.JsonParserException;
7+
import com.grack.nanojson.JsonWriter;
8+
49
import org.schabi.newpipe.extractor.MetaInfo;
510
import org.schabi.newpipe.extractor.Page;
611
import org.schabi.newpipe.extractor.downloader.Response;
@@ -10,6 +15,7 @@
1015
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
1116
import org.schabi.newpipe.extractor.localization.Localization;
1217
import org.schabi.newpipe.extractor.stream.Description;
18+
import org.schabi.newpipe.extractor.utils.JsonUtils;
1319
import org.schabi.newpipe.extractor.utils.Parser;
1420
import org.schabi.newpipe.extractor.utils.Utils;
1521

@@ -33,7 +39,12 @@
3339
import javax.annotation.Nullable;
3440

3541
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
36-
import static org.schabi.newpipe.extractor.utils.Utils.*;
42+
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
43+
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
44+
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
45+
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
46+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
47+
import static org.schabi.newpipe.extractor.utils.Utils.join;
3748

3849
/*
3950
* Created by Christian Schabesberger on 02.03.16.
@@ -638,7 +649,7 @@ public static JsonArray getJsonResponse(final String url, final Localization loc
638649
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
639650
final Response response = getDownloader().get(url, headers, localization);
640651

641-
return toJsonArray(getValidJsonResponseBody(response));
652+
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
642653
}
643654

644655
public static JsonArray getJsonResponse(final Page page, final Localization localization)
@@ -652,15 +663,7 @@ public static JsonArray getJsonResponse(final Page page, final Localization loca
652663

653664
final Response response = getDownloader().get(page.getUrl(), headers, localization);
654665

655-
return toJsonArray(getValidJsonResponseBody(response));
656-
}
657-
658-
public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
659-
try {
660-
return JsonParser.array().from(responseBody);
661-
} catch (JsonParserException e) {
662-
throw new ParsingException("Could not parse JSON", e);
663-
}
666+
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
664667
}
665668

666669
/**

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonObject;
5+
56
import org.schabi.newpipe.extractor.ListExtractor;
67
import org.schabi.newpipe.extractor.Page;
78
import org.schabi.newpipe.extractor.StreamingService;
@@ -14,14 +15,19 @@
1415
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
1516
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
1617
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
18+
import org.schabi.newpipe.extractor.utils.JsonUtils;
1719

18-
import javax.annotation.Nonnull;
19-
import javax.annotation.Nullable;
2020
import java.io.IOException;
2121
import java.util.Collections;
2222
import java.util.List;
2323

24-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
24+
import javax.annotation.Nonnull;
25+
import javax.annotation.Nullable;
26+
27+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
28+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
29+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse;
30+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
2531
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
2632

2733
/**
@@ -51,7 +57,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
5157
throws IOException, ExtractionException {
5258
final String url = getUrl() + "&pbj=1";
5359
final Response response = getResponse(url, getExtractorLocalization());
54-
final JsonArray ajaxJson = toJsonArray(response.responseBody());
60+
final JsonArray ajaxJson = JsonUtils.toJsonArray(response.responseBody());
5561
initialData = ajaxJson.getObject(3).getObject("response");
5662
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
5763
.getObject("playlist").getObject("playlist");

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java‎

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,12 @@
22

33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonObject;
5+
import com.grack.nanojson.JsonWriter;
6+
57
import org.schabi.newpipe.extractor.Page;
68
import org.schabi.newpipe.extractor.StreamingService;
79
import org.schabi.newpipe.extractor.downloader.Downloader;
10+
import org.schabi.newpipe.extractor.downloader.Response;
811
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
912
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1013
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -19,11 +22,20 @@
1922
import org.schabi.newpipe.extractor.stream.StreamType;
2023
import org.schabi.newpipe.extractor.utils.Utils;
2124

25+
import java.io.IOException;
26+
2227
import javax.annotation.Nonnull;
2328
import javax.annotation.Nullable;
24-
import java.io.IOException;
2529

26-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
30+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
31+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
32+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
33+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
34+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
35+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
36+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
37+
import static org.schabi.newpipe.extractor.utils.JsonUtils.toJsonObject;
38+
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
2739
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
2840

2941
@SuppressWarnings("WeakerAccess")
@@ -168,7 +180,7 @@ public String getSubChannelAvatarUrl() {
168180

169181
@Nonnull
170182
@Override
171-
public InfoItemsPage<StreamInfoItem> getInitialPage() {
183+
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
172184
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
173185
Page nextPage = null;
174186

@@ -205,12 +217,27 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
205217
throw new IllegalArgumentException("Page doesn't contain an URL");
206218
}
207219

220+
// @formatter:off
221+
byte[] json = JsonWriter.string()
222+
.object()
223+
.object("context")
224+
.object("client")
225+
.value("clientName", "1")
226+
.value("clientVersion", getClientVersion())
227+
.end()
228+
.end()
229+
.value("continuation", page.getId())
230+
.end()
231+
.done()
232+
.getBytes(UTF_8);
233+
// @formatter:on
234+
208235
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
209-
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
236+
final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
237+
238+
final JsonObject ajaxJson = toJsonObject(getValidJsonResponseBody(response));
210239

211-
final JsonArray continuation = ajaxJson.getObject(1)
212-
.getObject("response")
213-
.getArray("onResponseReceivedActions")
240+
final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions")
214241
.getObject(0)
215242
.getObject("appendContinuationItemsAction")
216243
.getArray("continuationItems");
@@ -220,7 +247,7 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
220247
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
221248
}
222249

223-
private Page getNextPageFrom(final JsonArray contents) {
250+
private Page getNextPageFrom(final JsonArray contents) throws IOException, ExtractionException {
224251
if (isNullOrEmpty(contents)) {
225252
return null;
226253
}
@@ -232,7 +259,9 @@ private Page getNextPageFrom(final JsonArray contents) {
232259
.getObject("continuationEndpoint")
233260
.getObject("continuationCommand")
234261
.getString("token");
235-
return new Page("https://www.youtube.com/browse_ajax?continuation=" + continuation);
262+
return new Page(
263+
"https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
264+
continuation);
236265
} else {
237266
return null;
238267
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,18 @@
22

33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonObject;
5+
import com.grack.nanojson.JsonParser;
6+
import com.grack.nanojson.JsonParserException;
7+
58
import org.schabi.newpipe.extractor.exceptions.ParsingException;
69

7-
import javax.annotation.Nonnull;
8-
import javax.annotation.Nullable;
910
import java.util.ArrayList;
1011
import java.util.Arrays;
1112
import java.util.List;
1213

14+
import javax.annotation.Nonnull;
15+
import javax.annotation.Nullable;
16+
1317
public class JsonUtils {
1418
public static final JsonObject EMPTY_OBJECT = new JsonObject();
1519
public static final JsonArray EMPTY_ARRAY = new JsonArray();
@@ -99,4 +103,19 @@ private static JsonObject getObject(@Nonnull JsonObject object, @Nonnull List<St
99103
return result;
100104
}
101105

106+
public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
107+
try {
108+
return JsonParser.array().from(responseBody);
109+
} catch (JsonParserException e) {
110+
throw new ParsingException("Could not parse JSON", e);
111+
}
112+
}
113+
114+
public static JsonObject toJsonObject(final String responseBody) throws ParsingException {
115+
try {
116+
return JsonParser.object().from(responseBody);
117+
} catch (JsonParserException e) {
118+
throw new ParsingException("Could not parse JSON", e);
119+
}
120+
}
102121
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,11 @@
1313
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1414
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
1515
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
16-
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.*;
16+
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.ContinuationsTests;
17+
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.HugePlaylist;
18+
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.LearningPlaylist;
19+
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.NotAvailable;
20+
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.TimelessPopHits;
1721
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
1822
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
1923

@@ -410,7 +414,7 @@ public void testNoContinuations() throws Exception {
410414
public void testOnlySingleContinuation() throws Exception {
411415
final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube
412416
.getPlaylistExtractor(
413-
"https://www.youtube.com/playlist?list=PLjgwFL8urN2DFRuRkFTkmtHjyoNWHHdZX");
417+
"https://www.youtube.com/playlist?list=PLoumn5BIsUDeGF1vy5Nylf_RJKn5aL_nr");
414418
extractor.fetchPage();
415419

416420
final ListExtractor.InfoItemsPage<StreamInfoItem> page = defaultTestMoreItems(

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/TimelessPopHits/generated_mock_0.json‎

Lines changed: 6 additions & 6 deletions
Large diffs are not rendered by default.

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/TimelessPopHits/generated_mock_1.json‎

Lines changed: 6 additions & 6 deletions
Large diffs are not rendered by default.

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/TimelessPopHits/generated_mock_2.json‎

Lines changed: 16 additions & 24 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)