Skip to content

Commit bdb0f2a

Browse files
authored
Merge pull request #332 from wb9688/learning-playlist
Support YouTube's learning playlists
2 parents cf18cdb + ab77961 commit bdb0f2a

2 files changed

Lines changed: 203 additions & 16 deletions

File tree

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

Lines changed: 105 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,21 @@
88
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
99
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1010
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
11+
import org.schabi.newpipe.extractor.localization.DateWrapper;
1112
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
1213
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
1314
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
15+
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
1416
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
17+
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
1518
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
19+
import org.schabi.newpipe.extractor.stream.StreamType;
1620
import org.schabi.newpipe.extractor.utils.Utils;
1721

1822
import java.io.IOException;
1923

2024
import javax.annotation.Nonnull;
25+
import javax.annotation.Nullable;
2126

2227
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
2328
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
@@ -27,6 +32,7 @@
2732

2833
@SuppressWarnings("WeakerAccess")
2934
public class YoutubePlaylistExtractor extends PlaylistExtractor {
35+
private JsonArray initialAjaxJson;
3036
private JsonObject initialData;
3137
private JsonObject playlistInfo;
3238

@@ -38,9 +44,9 @@ public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHa
3844
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
3945
final String url = getUrl() + "&pbj=1";
4046

41-
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
47+
initialAjaxJson = getJsonResponse(url, getExtractorLocalization());
4248

43-
initialData = ajaxJson.getObject(1).getObject("response");
49+
initialData = initialAjaxJson.getObject(1).getObject("response");
4450
YoutubeParsingHelper.defaultAlertsCheck(initialData);
4551

4652
playlistInfo = getPlaylistInfo();
@@ -152,34 +158,47 @@ public long getStreamCount() throws ParsingException {
152158

153159
@Nonnull
154160
@Override
155-
public String getSubChannelName() throws ParsingException {
161+
public String getSubChannelName() {
156162
return "";
157163
}
158164

159165
@Nonnull
160166
@Override
161-
public String getSubChannelUrl() throws ParsingException {
167+
public String getSubChannelUrl() {
162168
return "";
163169
}
164170

165171
@Nonnull
166172
@Override
167-
public String getSubChannelAvatarUrl() throws ParsingException {
173+
public String getSubChannelAvatarUrl() {
168174
return "";
169175
}
170176

171177
@Nonnull
172178
@Override
173179
public InfoItemsPage<StreamInfoItem> getInitialPage() {
174-
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
180+
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
175181

176-
JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
182+
final JsonArray contents = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
177183
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
178184
.getObject("sectionListRenderer").getArray("contents").getObject(0)
179-
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
180-
.getObject("playlistVideoListRenderer").getArray("contents");
185+
.getObject("itemSectionRenderer").getArray("contents");
186+
187+
if (contents.getObject(0).has("playlistSegmentRenderer")) {
188+
for (final Object segment : contents) {
189+
if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("trailer")) {
190+
collectTrailerFrom(collector, ((JsonObject) segment));
191+
} else if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("videoList")) {
192+
collectStreamsFrom(collector, ((JsonObject) segment).getObject("playlistSegmentRenderer")
193+
.getObject("videoList").getObject("playlistVideoListRenderer").getArray("contents"));
194+
}
195+
}
196+
} else if (contents.getObject(0).has("playlistVideoListRenderer")) {
197+
final JsonArray videos = contents.getObject(0)
198+
.getObject("playlistVideoListRenderer").getArray("contents");
199+
collectStreamsFrom(collector, videos);
200+
}
181201

182-
collectStreamsFrom(collector, videos);
183202
return new InfoItemsPage<>(collector, getNextPageUrl());
184203
}
185204

@@ -189,18 +208,18 @@ public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl) throws IOExce
189208
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
190209
}
191210

192-
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
211+
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
193212
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
194213

195-
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
214+
final JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
196215
.getObject("continuationContents").getObject("playlistVideoListContinuation");
197216

198217
collectStreamsFrom(collector, sectionListContinuation.getArray("contents"));
199218

200219
return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations")));
201220
}
202221

203-
private String getNextPageUrlFrom(JsonArray continuations) {
222+
private String getNextPageUrlFrom(final JsonArray continuations) {
204223
if (isNullOrEmpty(continuations)) {
205224
return "";
206225
}
@@ -212,9 +231,7 @@ private String getNextPageUrlFrom(JsonArray continuations) {
212231
+ "&itct=" + clickTrackingParams;
213232
}
214233

215-
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) {
216-
collector.reset();
217-
234+
private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) {
218235
final TimeAgoParser timeAgoParser = getTimeAgoParser();
219236

220237
for (Object video : videos) {
@@ -228,4 +245,76 @@ public long getViewCount() {
228245
}
229246
}
230247
}
248+
249+
private void collectTrailerFrom(final StreamInfoItemsCollector collector,
250+
final JsonObject segment) {
251+
collector.commit(new StreamInfoItemExtractor() {
252+
@Override
253+
public String getName() throws ParsingException {
254+
return getTextFromObject(segment.getObject("playlistSegmentRenderer")
255+
.getObject("title"));
256+
}
257+
258+
@Override
259+
public String getUrl() throws ParsingException {
260+
return YoutubeStreamLinkHandlerFactory.getInstance()
261+
.fromId(segment.getObject("playlistSegmentRenderer").getObject("trailer")
262+
.getObject("playlistVideoPlayerRenderer").getString("videoId"))
263+
.getUrl();
264+
}
265+
266+
@Override
267+
public String getThumbnailUrl() {
268+
final JsonArray thumbnails = initialAjaxJson.getObject(1).getObject("playerResponse")
269+
.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
270+
// the last thumbnail is the one with the highest resolution
271+
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
272+
return fixThumbnailUrl(url);
273+
}
274+
275+
@Override
276+
public StreamType getStreamType() {
277+
return StreamType.VIDEO_STREAM;
278+
}
279+
280+
@Override
281+
public boolean isAd() {
282+
return false;
283+
}
284+
285+
@Override
286+
public long getDuration() throws ParsingException {
287+
return YoutubeParsingHelper.parseDurationString(
288+
getTextFromObject(segment.getObject("playlistSegmentRenderer")
289+
.getObject("segmentAnnotation")).split("•")[0]);
290+
}
291+
292+
@Override
293+
public long getViewCount() {
294+
return -1;
295+
}
296+
297+
@Override
298+
public String getUploaderName() throws ParsingException {
299+
return YoutubePlaylistExtractor.this.getUploaderName();
300+
}
301+
302+
@Override
303+
public String getUploaderUrl() throws ParsingException {
304+
return YoutubePlaylistExtractor.this.getUploaderUrl();
305+
}
306+
307+
@Nullable
308+
@Override
309+
public String getTextualUploadDate() {
310+
return null;
311+
}
312+
313+
@Nullable
314+
@Override
315+
public DateWrapper getUploadDate() {
316+
return null;
317+
}
318+
});
319+
}
231320
}

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,4 +254,102 @@ public void testStreamCount() throws Exception {
254254
assertTrue("Error in the streams count", extractor.getStreamCount() > 100);
255255
}
256256
}
257+
258+
public static class LearningPlaylist implements BasePlaylistExtractorTest {
259+
private static YoutubePlaylistExtractor extractor;
260+
261+
@BeforeClass
262+
public static void setUp() throws Exception {
263+
NewPipe.init(DownloaderTestImpl.getInstance());
264+
extractor = (YoutubePlaylistExtractor) YouTube
265+
.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8");
266+
extractor.fetchPage();
267+
}
268+
269+
/*//////////////////////////////////////////////////////////////////////////
270+
// Extractor
271+
//////////////////////////////////////////////////////////////////////////*/
272+
273+
@Test
274+
public void testServiceId() {
275+
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
276+
}
277+
278+
@Test
279+
public void testName() throws Exception {
280+
String name = extractor.getName();
281+
assertTrue(name, name.startsWith("Anatomy & Physiology"));
282+
}
283+
284+
@Test
285+
public void testId() throws Exception {
286+
assertEquals("PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getId());
287+
}
288+
289+
@Test
290+
public void testUrl() throws ParsingException {
291+
assertEquals("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getUrl());
292+
}
293+
294+
@Test
295+
public void testOriginalUrl() throws ParsingException {
296+
assertEquals("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getOriginalUrl());
297+
}
298+
299+
/*//////////////////////////////////////////////////////////////////////////
300+
// ListExtractor
301+
//////////////////////////////////////////////////////////////////////////*/
302+
303+
@Test
304+
public void testRelatedItems() throws Exception {
305+
defaultTestRelatedItems(extractor);
306+
}
307+
308+
@Ignore
309+
@Test
310+
public void testMoreRelatedItems() throws Exception {
311+
defaultTestMoreItems(extractor);
312+
}
313+
314+
/*//////////////////////////////////////////////////////////////////////////
315+
// PlaylistExtractor
316+
//////////////////////////////////////////////////////////////////////////*/
317+
318+
@Test
319+
public void testThumbnailUrl() throws Exception {
320+
final String thumbnailUrl = extractor.getThumbnailUrl();
321+
assertIsSecureUrl(thumbnailUrl);
322+
assertTrue(thumbnailUrl, thumbnailUrl.contains("yt"));
323+
}
324+
325+
@Ignore
326+
@Test
327+
public void testBannerUrl() throws Exception {
328+
final String bannerUrl = extractor.getBannerUrl();
329+
assertIsSecureUrl(bannerUrl);
330+
assertTrue(bannerUrl, bannerUrl.contains("yt"));
331+
}
332+
333+
@Test
334+
public void testUploaderUrl() throws Exception {
335+
assertEquals("https://www.youtube.com/channel/UCX6b17PVsYBQ0ip5gyeme-Q", extractor.getUploaderUrl());
336+
}
337+
338+
@Test
339+
public void testUploaderName() throws Exception {
340+
final String uploaderName = extractor.getUploaderName();
341+
assertTrue(uploaderName, uploaderName.contains("CrashCourse"));
342+
}
343+
344+
@Test
345+
public void testUploaderAvatarUrl() throws Exception {
346+
final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl();
347+
assertTrue(uploaderAvatarUrl, uploaderAvatarUrl.contains("yt"));
348+
}
349+
350+
@Test
351+
public void testStreamCount() throws Exception {
352+
assertTrue("Error in the streams count", extractor.getStreamCount() > 40);
353+
}
354+
}
257355
}

0 commit comments

Comments
 (0)